From a714941f932cd07c0f1315a3a78a534a9e1101ae Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:06:22 +0900 Subject: [PATCH 01/13] Rename EF variable to make reading code easier --- osu.Game/Database/EFToRealmMigrator.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 7683accc5c..98037bb8a3 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -30,20 +30,20 @@ namespace osu.Game.Database public void Run() { - using (var db = efContextFactory.GetForWrite()) + using (var ef = efContextFactory.GetForWrite()) { - migrateSettings(db); - migrateSkins(db); + migrateSettings(ef); + migrateSkins(ef); - migrateBeatmaps(db); - migrateScores(db); + migrateBeatmaps(ef); + migrateScores(ef); } } - private void migrateBeatmaps(DatabaseWriteUsage db) + private void migrateBeatmaps(DatabaseWriteUsage ef) { // can be removed 20220730. - var existingBeatmapSets = db.Context.EFBeatmapSetInfo + var existingBeatmapSets = ef.Context.EFBeatmapSetInfo .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) @@ -117,7 +117,7 @@ namespace osu.Game.Database } } - db.Context.RemoveRange(existingBeatmapSets); + ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. transaction.Commit(); From b1a75ce48098116ee14a6eb9db39cd1339cdf4dc Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:17:43 +0900 Subject: [PATCH 02/13] Permanently delete `client.db` after migration completes --- osu.Game/Database/EFToRealmMigrator.cs | 4 ++++ osu.Game/OsuGameBase.cs | 1 + 2 files changed, 5 insertions(+) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 98037bb8a3..7ad8e507bb 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -38,6 +38,10 @@ namespace osu.Game.Database migrateBeatmaps(ef); migrateScores(ef); } + + // Delete the database permanently. + // Will cause future startups to not attempt migration. + efContextFactory.ResetDatabase(); } private void migrateBeatmaps(DatabaseWriteUsage ef) diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index a9538c1e8c..18b22588a6 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -193,6 +193,7 @@ namespace osu.Game dependencies.Cache(RulesetStore = new RulesetStore(realmFactory, Storage)); dependencies.CacheAs(RulesetStore); + // A non-null context factory means there's still content to migrate. if (efContextFactory != null) new EFToRealmMigrator(efContextFactory, realmFactory, LocalConfig).Run(); From 798482c94122eb9d23221e90e33ccc390a52380a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:19:25 +0900 Subject: [PATCH 03/13] Create backups before deleting scores and beatmaps from EF database --- osu.Game/Database/DatabaseContextFactory.cs | 8 +++++++ osu.Game/Database/EFToRealmMigrator.cs | 25 +++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index f79505d7c5..c2a60122eb 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -1,6 +1,7 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; @@ -144,6 +145,13 @@ namespace osu.Game.Database Database = { AutoTransactionsEnabled = false } }; + public void CreateBackup(string filename) + { + using (var source = storage.GetStream(DATABASE_NAME)) + using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + source.CopyTo(destination); + } + public void ResetDatabase() { lock (writeLock) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 7ad8e507bb..3ecbd50b3e 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -1,6 +1,8 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System; +using System.Collections.Generic; using System.Linq; using Microsoft.EntityFrameworkCore; using osu.Game.Beatmaps; @@ -31,13 +33,16 @@ namespace osu.Game.Database public void Run() { using (var ef = efContextFactory.GetForWrite()) - { migrateSettings(ef); + + using (var ef = efContextFactory.GetForWrite()) migrateSkins(ef); + using (var ef = efContextFactory.GetForWrite()) migrateBeatmaps(ef); + + using (var ef = efContextFactory.GetForWrite()) migrateScores(ef); - } // Delete the database permanently. // Will cause future startups to not attempt migration. @@ -47,13 +52,13 @@ namespace osu.Game.Database private void migrateBeatmaps(DatabaseWriteUsage ef) { // can be removed 20220730. - var existingBeatmapSets = ef.Context.EFBeatmapSetInfo - .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) - .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) - .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) - .Include(s => s.Files).ThenInclude(f => f.FileInfo) - .Include(s => s.Metadata) - .ToList(); + List existingBeatmapSets = ef.Context.EFBeatmapSetInfo + .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) + .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) + .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) + .Include(s => s.Files).ThenInclude(f => f.FileInfo) + .Include(s => s.Metadata) + .ToList(); // previous entries in EF are removed post migration. if (!existingBeatmapSets.Any()) @@ -121,6 +126,7 @@ namespace osu.Game.Database } } + efContextFactory.CreateBackup($"client.before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. @@ -207,6 +213,7 @@ namespace osu.Game.Database } } + efContextFactory.CreateBackup($"client.before_scores_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. From cf30d48721242567eb44d7d0e31fb6cb9e2d1b69 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:19:31 +0900 Subject: [PATCH 04/13] Add more logging during migration process --- osu.Game/Database/EFToRealmMigrator.cs | 29 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 3ecbd50b3e..d8a3b66c7c 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Linq; using Microsoft.EntityFrameworkCore; +using osu.Framework.Logging; using osu.Game.Beatmaps; using osu.Game.Configuration; using osu.Game.Models; @@ -64,6 +65,8 @@ namespace osu.Game.Database if (!existingBeatmapSets.Any()) return; + Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { @@ -71,6 +74,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any(s => !s.Protected)) { + Logger.Log($"Migrating {existingBeatmapSets.Count} beatmaps", LoggingTarget.Database); + foreach (var beatmapSet in existingBeatmapSets) { var realmBeatmapSet = new BeatmapSetInfo @@ -161,17 +166,19 @@ namespace osu.Game.Database private void migrateScores(DatabaseWriteUsage db) { // can be removed 20220730. - var existingScores = db.Context.ScoreInfo - .Include(s => s.Ruleset) - .Include(s => s.BeatmapInfo) - .Include(s => s.Files) - .ThenInclude(f => f.FileInfo) - .ToList(); + List existingScores = db.Context.ScoreInfo + .Include(s => s.Ruleset) + .Include(s => s.BeatmapInfo) + .Include(s => s.Files) + .ThenInclude(f => f.FileInfo) + .ToList(); // previous entries in EF are removed post migration. if (!existingScores.Any()) return; + Logger.Log("Beginning scores migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { @@ -179,6 +186,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any()) { + Logger.Log($"Migrating {existingScores.Count} scores", LoggingTarget.Database); + foreach (var score in existingScores) { var realmScore = new ScoreInfo @@ -254,6 +263,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any(s => !s.Protected)) { + Logger.Log($"Migrating {existingSkins.Count} skins", LoggingTarget.Database); + foreach (var skin in existingSkins) { var realmSkin = new SkinInfo @@ -297,18 +308,22 @@ namespace osu.Game.Database private void migrateSettings(DatabaseWriteUsage db) { // migrate ruleset settings. can be removed 20220315. - var existingSettings = db.Context.DatabasedSetting; + var existingSettings = db.Context.DatabasedSetting.ToList(); // previous entries in EF are removed post migration. if (!existingSettings.Any()) return; + Logger.Log("Beginning settings migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { // only migrate data if the realm database is empty. if (!realm.All().Any()) { + Logger.Log($"Migrating {existingSettings.Count} settings", LoggingTarget.Database); + foreach (var dkb in existingSettings) { if (dkb.RulesetID == null) From 2b1c15b6cc1b7a6f51157cb598e63b835322ea3c Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:30:32 +0900 Subject: [PATCH 05/13] Allow `BlockAllOperations` to be called from a non-update thread if update has never run --- osu.Game/Database/RealmContextFactory.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 9307e06be0..01c54d6ee2 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -366,17 +366,17 @@ namespace osu.Game.Database if (isDisposed) throw new ObjectDisposedException(nameof(RealmContextFactory)); - if (!ThreadSafety.IsUpdateThread) - throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread."); - - Logger.Log(@"Blocking realm operations.", LoggingTarget.Database); - try { contextCreationLock.Wait(); lock (contextLock) { + if (!ThreadSafety.IsUpdateThread && context != null) + throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread."); + + Logger.Log(@"Blocking realm operations.", LoggingTarget.Database); + context?.Dispose(); context = null; } From bf50a9b8f8d92d0a23a498822513d65d9fade133 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:30:41 +0900 Subject: [PATCH 06/13] Also backup the realm database before migration --- osu.Game/Database/EFToRealmMigrator.cs | 8 ++++++-- osu.Game/Database/RealmContextFactory.cs | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index d8a3b66c7c..9fad29b3f4 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -75,6 +75,9 @@ namespace osu.Game.Database if (!realm.All().Any(s => !s.Protected)) { Logger.Log($"Migrating {existingBeatmapSets.Count} beatmaps", LoggingTarget.Database); + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + realmContextFactory.CreateBackup($"client.{migration}.realm"); + efContextFactory.CreateBackup($"client.{migration}.db"); foreach (var beatmapSet in existingBeatmapSets) { @@ -131,7 +134,6 @@ namespace osu.Game.Database } } - efContextFactory.CreateBackup($"client.before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. @@ -187,6 +189,9 @@ namespace osu.Game.Database if (!realm.All().Any()) { Logger.Log($"Migrating {existingScores.Count} scores", LoggingTarget.Database); + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + realmContextFactory.CreateBackup($"client.{migration}.realm"); + efContextFactory.CreateBackup($"client.{migration}.db"); foreach (var score in existingScores) { @@ -222,7 +227,6 @@ namespace osu.Game.Database } } - efContextFactory.CreateBackup($"client.before_scores_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 01c54d6ee2..c86a9e54e1 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -2,6 +2,7 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.IO; using System.Linq; using System.Reflection; using System.Threading; @@ -353,6 +354,16 @@ namespace osu.Game.Database private string? getRulesetShortNameFromLegacyID(long rulesetId) => efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName; + public void CreateBackup(string filename) + { + using (BlockAllOperations()) + { + using (var source = storage.GetStream(Filename)) + using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + source.CopyTo(destination); + } + } + /// /// Flush any active contexts and block any further writes. /// From 3429fd87681b928a52954b4a3b32ae38fd2bd794 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:41:02 +0900 Subject: [PATCH 07/13] Fix transaction scope and add even more logging --- osu.Game/Database/DatabaseContextFactory.cs | 2 + osu.Game/Database/EFToRealmMigrator.cs | 215 +++++++++++--------- osu.Game/Database/RealmContextFactory.cs | 1 + 3 files changed, 122 insertions(+), 96 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index c2a60122eb..cc690a9fda 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; +using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Statistics; @@ -147,6 +148,7 @@ namespace osu.Game.Database public void CreateBackup(string filename) { + Logger.Log($"Creating full EF database backup at {filename}", LoggingTarget.Database); using (var source = storage.GetStream(DATABASE_NAME)) using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 9fad29b3f4..8f5de80e3b 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -47,6 +47,7 @@ namespace osu.Game.Database // Delete the database permanently. // Will cause future startups to not attempt migration. + Logger.Log("Migration successful, deleting EF database", LoggingTarget.Database); efContextFactory.ResetDatabase(); } @@ -61,83 +62,94 @@ namespace osu.Game.Database .Include(s => s.Metadata) .ToList(); - // previous entries in EF are removed post migration. - if (!existingBeatmapSets.Any()) - return; - Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database); - using (var realm = realmContextFactory.CreateContext()) - using (var transaction = realm.BeginWrite()) + // previous entries in EF are removed post migration. + if (!existingBeatmapSets.Any()) { + Logger.Log("No beatmaps found to migrate", LoggingTarget.Database); + return; + } + + using (var realm = realmContextFactory.CreateContext()) + { + Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database); + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + efContextFactory.CreateBackup($"client.{migration}.db"); + // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. - if (!realm.All().Any(s => !s.Protected)) + if (realm.All().Any(s => !s.Protected)) + { + Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database); + } + else { - Logger.Log($"Migrating {existingBeatmapSets.Count} beatmaps", LoggingTarget.Database); - string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; realmContextFactory.CreateBackup($"client.{migration}.realm"); - efContextFactory.CreateBackup($"client.{migration}.db"); - foreach (var beatmapSet in existingBeatmapSets) + using (var transaction = realm.BeginWrite()) { - var realmBeatmapSet = new BeatmapSetInfo + foreach (var beatmapSet in existingBeatmapSets) { - OnlineID = beatmapSet.OnlineID ?? -1, - DateAdded = beatmapSet.DateAdded, - Status = beatmapSet.Status, - DeletePending = beatmapSet.DeletePending, - Hash = beatmapSet.Hash, - Protected = beatmapSet.Protected, - }; - - migrateFiles(beatmapSet, realm, realmBeatmapSet); - - foreach (var beatmap in beatmapSet.Beatmaps) - { - var realmBeatmap = new BeatmapInfo + var realmBeatmapSet = new BeatmapSetInfo { - DifficultyName = beatmap.DifficultyName, - Status = beatmap.Status, - OnlineID = beatmap.OnlineID ?? -1, - Length = beatmap.Length, - BPM = beatmap.BPM, - Hash = beatmap.Hash, - StarRating = beatmap.StarRating, - MD5Hash = beatmap.MD5Hash, - Hidden = beatmap.Hidden, - AudioLeadIn = beatmap.AudioLeadIn, - StackLeniency = beatmap.StackLeniency, - SpecialStyle = beatmap.SpecialStyle, - LetterboxInBreaks = beatmap.LetterboxInBreaks, - WidescreenStoryboard = beatmap.WidescreenStoryboard, - EpilepsyWarning = beatmap.EpilepsyWarning, - SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate, - DistanceSpacing = beatmap.DistanceSpacing, - BeatDivisor = beatmap.BeatDivisor, - GridSize = beatmap.GridSize, - TimelineZoom = beatmap.TimelineZoom, - Countdown = beatmap.Countdown, - CountdownOffset = beatmap.CountdownOffset, - MaxCombo = beatmap.MaxCombo, - Bookmarks = beatmap.Bookmarks, - Ruleset = realm.Find(beatmap.RulesetInfo.ShortName), - Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty), - Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata), - BeatmapSet = realmBeatmapSet, + OnlineID = beatmapSet.OnlineID ?? -1, + DateAdded = beatmapSet.DateAdded, + Status = beatmapSet.Status, + DeletePending = beatmapSet.DeletePending, + Hash = beatmapSet.Hash, + Protected = beatmapSet.Protected, }; - realmBeatmapSet.Beatmaps.Add(realmBeatmap); + migrateFiles(beatmapSet, realm, realmBeatmapSet); + + foreach (var beatmap in beatmapSet.Beatmaps) + { + var realmBeatmap = new BeatmapInfo + { + DifficultyName = beatmap.DifficultyName, + Status = beatmap.Status, + OnlineID = beatmap.OnlineID ?? -1, + Length = beatmap.Length, + BPM = beatmap.BPM, + Hash = beatmap.Hash, + StarRating = beatmap.StarRating, + MD5Hash = beatmap.MD5Hash, + Hidden = beatmap.Hidden, + AudioLeadIn = beatmap.AudioLeadIn, + StackLeniency = beatmap.StackLeniency, + SpecialStyle = beatmap.SpecialStyle, + LetterboxInBreaks = beatmap.LetterboxInBreaks, + WidescreenStoryboard = beatmap.WidescreenStoryboard, + EpilepsyWarning = beatmap.EpilepsyWarning, + SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate, + DistanceSpacing = beatmap.DistanceSpacing, + BeatDivisor = beatmap.BeatDivisor, + GridSize = beatmap.GridSize, + TimelineZoom = beatmap.TimelineZoom, + Countdown = beatmap.Countdown, + CountdownOffset = beatmap.CountdownOffset, + MaxCombo = beatmap.MaxCombo, + Bookmarks = beatmap.Bookmarks, + Ruleset = realm.Find(beatmap.RulesetInfo.ShortName), + Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty), + Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata), + BeatmapSet = realmBeatmapSet, + }; + + realmBeatmapSet.Beatmaps.Add(realmBeatmap); + } + + realm.Add(realmBeatmapSet); } - realm.Add(realmBeatmapSet); + transaction.Commit(); } } + Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. - - transaction.Commit(); } } @@ -175,62 +187,73 @@ namespace osu.Game.Database .ThenInclude(f => f.FileInfo) .ToList(); - // previous entries in EF are removed post migration. - if (!existingScores.Any()) - return; - Logger.Log("Beginning scores migration to realm", LoggingTarget.Database); - using (var realm = realmContextFactory.CreateContext()) - using (var transaction = realm.BeginWrite()) + // previous entries in EF are removed post migration. + if (!existingScores.Any()) { + Logger.Log("No scores found to migrate", LoggingTarget.Database); + return; + } + + using (var realm = realmContextFactory.CreateContext()) + { + Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database); + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + efContextFactory.CreateBackup($"client.{migration}.db"); + // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. - if (!realm.All().Any()) + if (realm.All().Any()) + { + Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database); + } + else { - Logger.Log($"Migrating {existingScores.Count} scores", LoggingTarget.Database); - string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; realmContextFactory.CreateBackup($"client.{migration}.realm"); - efContextFactory.CreateBackup($"client.{migration}.db"); - foreach (var score in existingScores) + using (var transaction = realm.BeginWrite()) { - var realmScore = new ScoreInfo + foreach (var score in existingScores) { - Hash = score.Hash, - DeletePending = score.DeletePending, - OnlineID = score.OnlineID ?? -1, - ModsJson = score.ModsJson, - StatisticsJson = score.StatisticsJson, - User = score.User, - TotalScore = score.TotalScore, - MaxCombo = score.MaxCombo, - Accuracy = score.Accuracy, - HasReplay = ((IScoreInfo)score).HasReplay, - Date = score.Date, - PP = score.PP, - BeatmapInfo = realm.All().First(b => b.Hash == score.BeatmapInfo.Hash), - Ruleset = realm.Find(score.Ruleset.ShortName), - Rank = score.Rank, - HitEvents = score.HitEvents, - Passed = score.Passed, - Combo = score.Combo, - Position = score.Position, - Statistics = score.Statistics, - Mods = score.Mods, - APIMods = score.APIMods, - }; + var realmScore = new ScoreInfo + { + Hash = score.Hash, + DeletePending = score.DeletePending, + OnlineID = score.OnlineID ?? -1, + ModsJson = score.ModsJson, + StatisticsJson = score.StatisticsJson, + User = score.User, + TotalScore = score.TotalScore, + MaxCombo = score.MaxCombo, + Accuracy = score.Accuracy, + HasReplay = ((IScoreInfo)score).HasReplay, + Date = score.Date, + PP = score.PP, + BeatmapInfo = realm.All().First(b => b.Hash == score.BeatmapInfo.Hash), + Ruleset = realm.Find(score.Ruleset.ShortName), + Rank = score.Rank, + HitEvents = score.HitEvents, + Passed = score.Passed, + Combo = score.Combo, + Position = score.Position, + Statistics = score.Statistics, + Mods = score.Mods, + APIMods = score.APIMods, + }; - migrateFiles(score, realm, realmScore); + migrateFiles(score, realm, realmScore); - realm.Add(realmScore); + realm.Add(realmScore); + } + + transaction.Commit(); } } + Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. - - transaction.Commit(); } } diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index c86a9e54e1..8548e63e94 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -358,6 +358,7 @@ namespace osu.Game.Database { using (BlockAllOperations()) { + Logger.Log($"Creating full realm database backup at {filename}", LoggingTarget.Database); using (var source = storage.GetStream(Filename)) using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); From 6b0bf38c93cb994f2c8d144e26d4001e5ad01060 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 20:47:53 +0900 Subject: [PATCH 08/13] Use a single EF context to avoid scores getting cascade deleted along the way --- osu.Game/Database/EFToRealmMigrator.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 8f5de80e3b..d479f81ead 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -34,16 +34,12 @@ namespace osu.Game.Database public void Run() { using (var ef = efContextFactory.GetForWrite()) + { migrateSettings(ef); - - using (var ef = efContextFactory.GetForWrite()) migrateSkins(ef); - - using (var ef = efContextFactory.GetForWrite()) migrateBeatmaps(ef); - - using (var ef = efContextFactory.GetForWrite()) migrateScores(ef); + } // Delete the database permanently. // Will cause future startups to not attempt migration. From 64a023665e9944f93615de503de190a64144bcf2 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:16:54 +0900 Subject: [PATCH 09/13] Avoid taking more than one backup per migration run --- osu.Game/Database/EFToRealmMigrator.cs | 30 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index d479f81ead..b7f3eebe7a 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -24,6 +24,8 @@ namespace osu.Game.Database private readonly RealmContextFactory realmContextFactory; private readonly OsuConfigManager config; + private bool hasTakenBackup; + public EFToRealmMigrator(DatabaseContextFactory efContextFactory, RealmContextFactory realmContextFactory, OsuConfigManager config) { this.efContextFactory = efContextFactory; @@ -70,8 +72,16 @@ namespace osu.Game.Database using (var realm = realmContextFactory.CreateContext()) { Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database); - string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; - efContextFactory.CreateBackup($"client.{migration}.db"); + + if (!hasTakenBackup) + { + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + + efContextFactory.CreateBackup($"client.{migration}.db"); + realmContextFactory.CreateBackup($"client.{migration}.realm"); + + hasTakenBackup = true; + } // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. @@ -81,8 +91,6 @@ namespace osu.Game.Database } else { - realmContextFactory.CreateBackup($"client.{migration}.realm"); - using (var transaction = realm.BeginWrite()) { foreach (var beatmapSet in existingBeatmapSets) @@ -195,8 +203,16 @@ namespace osu.Game.Database using (var realm = realmContextFactory.CreateContext()) { Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database); - string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; - efContextFactory.CreateBackup($"client.{migration}.db"); + + if (!hasTakenBackup) + { + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + + efContextFactory.CreateBackup($"client.{migration}.db"); + realmContextFactory.CreateBackup($"client.{migration}.realm"); + + hasTakenBackup = true; + } // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. @@ -206,8 +222,6 @@ namespace osu.Game.Database } else { - realmContextFactory.CreateBackup($"client.{migration}.realm"); - using (var transaction = realm.BeginWrite()) { foreach (var score in existingScores) From 04e9ffa966afd6b95698591de468006302328441 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:20:43 +0900 Subject: [PATCH 10/13] Freshen some comments --- osu.Game/Database/EFToRealmMigrator.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index b7f3eebe7a..6b911ebad0 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -84,7 +84,7 @@ namespace osu.Game.Database } // only migrate data if the realm database is empty. - // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. + // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (realm.All().Any(s => !s.Protected)) { Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database); @@ -215,7 +215,6 @@ namespace osu.Game.Database } // only migrate data if the realm database is empty. - // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (realm.All().Any()) { Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database); From e1a35714be3720178f2a56ef88bcdb038946f04a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:30:17 +0900 Subject: [PATCH 11/13] Add notification for debug builds when database migration occurs --- osu.Game/Database/DatabaseContextFactory.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index cc690a9fda..635c4373cd 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; +using osu.Framework.Development; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Statistics; @@ -146,11 +147,15 @@ namespace osu.Game.Database Database = { AutoTransactionsEnabled = false } }; - public void CreateBackup(string filename) + public void CreateBackup(string backupFilename) { - Logger.Log($"Creating full EF database backup at {filename}", LoggingTarget.Database); + Logger.Log($"Creating full EF database backup at {backupFilename}", LoggingTarget.Database); + + if (DebugUtils.IsDebugBuild) + Logger.Log("Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.", level: LogLevel.Important); + using (var source = storage.GetStream(DATABASE_NAME)) - using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); } From 195534a1d2e0a890be7b16feb3e936ed2fe426c4 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:31:09 +0900 Subject: [PATCH 12/13] Only output "successful" messages when copy actually occurred --- osu.Game/Database/EFToRealmMigrator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 6b911ebad0..76e5219f87 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -148,10 +148,10 @@ namespace osu.Game.Database } transaction.Commit(); + Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); } } - Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. } @@ -257,10 +257,10 @@ namespace osu.Game.Database } transaction.Commit(); + Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); } } - Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. } From c52899b1fb0f6a6e52d73ee93d74247b931bc6f4 Mon Sep 17 00:00:00 2001 From: Dan Balasescu Date: Wed, 19 Jan 2022 11:56:44 +0900 Subject: [PATCH 13/13] Rename property --- osu.Game/Database/RealmContextFactory.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 10c2438df9..8be8eab567 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -361,13 +361,13 @@ namespace osu.Game.Database private string? getRulesetShortNameFromLegacyID(long rulesetId) => efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName; - public void CreateBackup(string filename) + public void CreateBackup(string backupFilename) { using (BlockAllOperations()) { - Logger.Log($"Creating full realm database backup at {filename}", LoggingTarget.Database); + Logger.Log($"Creating full realm database backup at {backupFilename}", LoggingTarget.Database); using (var source = storage.GetStream(Filename)) - using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); } }