diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index f79505d7c5..635c4373cd 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -1,9 +1,12 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; +using osu.Framework.Development; +using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Statistics; @@ -144,6 +147,18 @@ namespace osu.Game.Database Database = { AutoTransactionsEnabled = false } }; + public void CreateBackup(string backupFilename) + { + Logger.Log($"Creating full EF database backup at {backupFilename}", LoggingTarget.Database); + + if (DebugUtils.IsDebugBuild) + Logger.Log("Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.", level: LogLevel.Important); + + using (var source = storage.GetStream(DATABASE_NAME)) + using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew)) + source.CopyTo(destination); + } + public void ResetDatabase() { lock (writeLock) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 7683accc5c..76e5219f87 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -1,8 +1,11 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System; +using System.Collections.Generic; using System.Linq; using Microsoft.EntityFrameworkCore; +using osu.Framework.Logging; using osu.Game.Beatmaps; using osu.Game.Configuration; using osu.Game.Models; @@ -21,6 +24,8 @@ namespace osu.Game.Database private readonly RealmContextFactory realmContextFactory; private readonly OsuConfigManager config; + private bool hasTakenBackup; + public EFToRealmMigrator(DatabaseContextFactory efContextFactory, RealmContextFactory realmContextFactory, OsuConfigManager config) { this.efContextFactory = efContextFactory; @@ -30,97 +35,125 @@ namespace osu.Game.Database public void Run() { - using (var db = efContextFactory.GetForWrite()) + using (var ef = efContextFactory.GetForWrite()) { - migrateSettings(db); - migrateSkins(db); - - migrateBeatmaps(db); - migrateScores(db); + migrateSettings(ef); + migrateSkins(ef); + migrateBeatmaps(ef); + migrateScores(ef); } + + // Delete the database permanently. + // Will cause future startups to not attempt migration. + Logger.Log("Migration successful, deleting EF database", LoggingTarget.Database); + efContextFactory.ResetDatabase(); } - private void migrateBeatmaps(DatabaseWriteUsage db) + private void migrateBeatmaps(DatabaseWriteUsage ef) { // can be removed 20220730. - var existingBeatmapSets = db.Context.EFBeatmapSetInfo - .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) - .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) - .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) - .Include(s => s.Files).ThenInclude(f => f.FileInfo) - .Include(s => s.Metadata) - .ToList(); + List existingBeatmapSets = ef.Context.EFBeatmapSetInfo + .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) + .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) + .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) + .Include(s => s.Files).ThenInclude(f => f.FileInfo) + .Include(s => s.Metadata) + .ToList(); + + Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database); // previous entries in EF are removed post migration. if (!existingBeatmapSets.Any()) + { + Logger.Log("No beatmaps found to migrate", LoggingTarget.Database); return; + } using (var realm = realmContextFactory.CreateContext()) - using (var transaction = realm.BeginWrite()) { - // only migrate data if the realm database is empty. - // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. - if (!realm.All().Any(s => !s.Protected)) + Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database); + + if (!hasTakenBackup) { - foreach (var beatmapSet in existingBeatmapSets) + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + + efContextFactory.CreateBackup($"client.{migration}.db"); + realmContextFactory.CreateBackup($"client.{migration}.realm"); + + hasTakenBackup = true; + } + + // only migrate data if the realm database is empty. + // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. + if (realm.All().Any(s => !s.Protected)) + { + Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database); + } + else + { + using (var transaction = realm.BeginWrite()) { - var realmBeatmapSet = new BeatmapSetInfo + foreach (var beatmapSet in existingBeatmapSets) { - OnlineID = beatmapSet.OnlineID ?? -1, - DateAdded = beatmapSet.DateAdded, - Status = beatmapSet.Status, - DeletePending = beatmapSet.DeletePending, - Hash = beatmapSet.Hash, - Protected = beatmapSet.Protected, - }; - - migrateFiles(beatmapSet, realm, realmBeatmapSet); - - foreach (var beatmap in beatmapSet.Beatmaps) - { - var realmBeatmap = new BeatmapInfo + var realmBeatmapSet = new BeatmapSetInfo { - DifficultyName = beatmap.DifficultyName, - Status = beatmap.Status, - OnlineID = beatmap.OnlineID ?? -1, - Length = beatmap.Length, - BPM = beatmap.BPM, - Hash = beatmap.Hash, - StarRating = beatmap.StarRating, - MD5Hash = beatmap.MD5Hash, - Hidden = beatmap.Hidden, - AudioLeadIn = beatmap.AudioLeadIn, - StackLeniency = beatmap.StackLeniency, - SpecialStyle = beatmap.SpecialStyle, - LetterboxInBreaks = beatmap.LetterboxInBreaks, - WidescreenStoryboard = beatmap.WidescreenStoryboard, - EpilepsyWarning = beatmap.EpilepsyWarning, - SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate, - DistanceSpacing = beatmap.DistanceSpacing, - BeatDivisor = beatmap.BeatDivisor, - GridSize = beatmap.GridSize, - TimelineZoom = beatmap.TimelineZoom, - Countdown = beatmap.Countdown, - CountdownOffset = beatmap.CountdownOffset, - MaxCombo = beatmap.MaxCombo, - Bookmarks = beatmap.Bookmarks, - Ruleset = realm.Find(beatmap.RulesetInfo.ShortName), - Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty), - Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata), - BeatmapSet = realmBeatmapSet, + OnlineID = beatmapSet.OnlineID ?? -1, + DateAdded = beatmapSet.DateAdded, + Status = beatmapSet.Status, + DeletePending = beatmapSet.DeletePending, + Hash = beatmapSet.Hash, + Protected = beatmapSet.Protected, }; - realmBeatmapSet.Beatmaps.Add(realmBeatmap); + migrateFiles(beatmapSet, realm, realmBeatmapSet); + + foreach (var beatmap in beatmapSet.Beatmaps) + { + var realmBeatmap = new BeatmapInfo + { + DifficultyName = beatmap.DifficultyName, + Status = beatmap.Status, + OnlineID = beatmap.OnlineID ?? -1, + Length = beatmap.Length, + BPM = beatmap.BPM, + Hash = beatmap.Hash, + StarRating = beatmap.StarRating, + MD5Hash = beatmap.MD5Hash, + Hidden = beatmap.Hidden, + AudioLeadIn = beatmap.AudioLeadIn, + StackLeniency = beatmap.StackLeniency, + SpecialStyle = beatmap.SpecialStyle, + LetterboxInBreaks = beatmap.LetterboxInBreaks, + WidescreenStoryboard = beatmap.WidescreenStoryboard, + EpilepsyWarning = beatmap.EpilepsyWarning, + SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate, + DistanceSpacing = beatmap.DistanceSpacing, + BeatDivisor = beatmap.BeatDivisor, + GridSize = beatmap.GridSize, + TimelineZoom = beatmap.TimelineZoom, + Countdown = beatmap.Countdown, + CountdownOffset = beatmap.CountdownOffset, + MaxCombo = beatmap.MaxCombo, + Bookmarks = beatmap.Bookmarks, + Ruleset = realm.Find(beatmap.RulesetInfo.ShortName), + Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty), + Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata), + BeatmapSet = realmBeatmapSet, + }; + + realmBeatmapSet.Beatmaps.Add(realmBeatmap); + } + + realm.Add(realmBeatmapSet); } - realm.Add(realmBeatmapSet); + transaction.Commit(); + Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); } } - db.Context.RemoveRange(existingBeatmapSets); + ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. - - transaction.Commit(); } } @@ -151,62 +184,85 @@ namespace osu.Game.Database private void migrateScores(DatabaseWriteUsage db) { // can be removed 20220730. - var existingScores = db.Context.ScoreInfo - .Include(s => s.Ruleset) - .Include(s => s.BeatmapInfo) - .Include(s => s.Files) - .ThenInclude(f => f.FileInfo) - .ToList(); + List existingScores = db.Context.ScoreInfo + .Include(s => s.Ruleset) + .Include(s => s.BeatmapInfo) + .Include(s => s.Files) + .ThenInclude(f => f.FileInfo) + .ToList(); + + Logger.Log("Beginning scores migration to realm", LoggingTarget.Database); // previous entries in EF are removed post migration. if (!existingScores.Any()) + { + Logger.Log("No scores found to migrate", LoggingTarget.Database); return; + } using (var realm = realmContextFactory.CreateContext()) - using (var transaction = realm.BeginWrite()) { - // only migrate data if the realm database is empty. - // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. - if (!realm.All().Any()) + Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database); + + if (!hasTakenBackup) { - foreach (var score in existingScores) + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + + efContextFactory.CreateBackup($"client.{migration}.db"); + realmContextFactory.CreateBackup($"client.{migration}.realm"); + + hasTakenBackup = true; + } + + // only migrate data if the realm database is empty. + if (realm.All().Any()) + { + Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database); + } + else + { + using (var transaction = realm.BeginWrite()) { - var realmScore = new ScoreInfo + foreach (var score in existingScores) { - Hash = score.Hash, - DeletePending = score.DeletePending, - OnlineID = score.OnlineID ?? -1, - ModsJson = score.ModsJson, - StatisticsJson = score.StatisticsJson, - User = score.User, - TotalScore = score.TotalScore, - MaxCombo = score.MaxCombo, - Accuracy = score.Accuracy, - HasReplay = ((IScoreInfo)score).HasReplay, - Date = score.Date, - PP = score.PP, - BeatmapInfo = realm.All().First(b => b.Hash == score.BeatmapInfo.Hash), - Ruleset = realm.Find(score.Ruleset.ShortName), - Rank = score.Rank, - HitEvents = score.HitEvents, - Passed = score.Passed, - Combo = score.Combo, - Position = score.Position, - Statistics = score.Statistics, - Mods = score.Mods, - APIMods = score.APIMods, - }; + var realmScore = new ScoreInfo + { + Hash = score.Hash, + DeletePending = score.DeletePending, + OnlineID = score.OnlineID ?? -1, + ModsJson = score.ModsJson, + StatisticsJson = score.StatisticsJson, + User = score.User, + TotalScore = score.TotalScore, + MaxCombo = score.MaxCombo, + Accuracy = score.Accuracy, + HasReplay = ((IScoreInfo)score).HasReplay, + Date = score.Date, + PP = score.PP, + BeatmapInfo = realm.All().First(b => b.Hash == score.BeatmapInfo.Hash), + Ruleset = realm.Find(score.Ruleset.ShortName), + Rank = score.Rank, + HitEvents = score.HitEvents, + Passed = score.Passed, + Combo = score.Combo, + Position = score.Position, + Statistics = score.Statistics, + Mods = score.Mods, + APIMods = score.APIMods, + }; - migrateFiles(score, realm, realmScore); + migrateFiles(score, realm, realmScore); - realm.Add(realmScore); + realm.Add(realmScore); + } + + transaction.Commit(); + Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); } } db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. - - transaction.Commit(); } } @@ -243,6 +299,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any(s => !s.Protected)) { + Logger.Log($"Migrating {existingSkins.Count} skins", LoggingTarget.Database); + foreach (var skin in existingSkins) { var realmSkin = new SkinInfo @@ -286,18 +344,22 @@ namespace osu.Game.Database private void migrateSettings(DatabaseWriteUsage db) { // migrate ruleset settings. can be removed 20220315. - var existingSettings = db.Context.DatabasedSetting; + var existingSettings = db.Context.DatabasedSetting.ToList(); // previous entries in EF are removed post migration. if (!existingSettings.Any()) return; + Logger.Log("Beginning settings migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { // only migrate data if the realm database is empty. if (!realm.All().Any()) { + Logger.Log($"Migrating {existingSettings.Count} settings", LoggingTarget.Database); + foreach (var dkb in existingSettings) { if (dkb.RulesetID == null) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index b9c0d1b681..8be8eab567 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -361,6 +361,17 @@ namespace osu.Game.Database private string? getRulesetShortNameFromLegacyID(long rulesetId) => efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName; + public void CreateBackup(string backupFilename) + { + using (BlockAllOperations()) + { + Logger.Log($"Creating full realm database backup at {backupFilename}", LoggingTarget.Database); + using (var source = storage.GetStream(Filename)) + using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew)) + source.CopyTo(destination); + } + } + /// /// Flush any active contexts and block any further writes. /// @@ -374,17 +385,17 @@ namespace osu.Game.Database if (isDisposed) throw new ObjectDisposedException(nameof(RealmContextFactory)); - if (!ThreadSafety.IsUpdateThread) - throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread."); - - Logger.Log(@"Blocking realm operations.", LoggingTarget.Database); - try { contextCreationLock.Wait(); lock (contextLock) { + if (!ThreadSafety.IsUpdateThread && context != null) + throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread."); + + Logger.Log(@"Blocking realm operations.", LoggingTarget.Database); + context?.Dispose(); context = null; } diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 2f9e9d5734..b24fdf2bfe 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -193,6 +193,7 @@ namespace osu.Game dependencies.Cache(RulesetStore = new RulesetStore(realmFactory, Storage)); dependencies.CacheAs(RulesetStore); + // A non-null context factory means there's still content to migrate. if (efContextFactory != null) new EFToRealmMigrator(efContextFactory, realmFactory, LocalConfig).Run();