From 5be41a189bb00e22926511b35424db85c992ead6 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 21 Jan 2022 14:56:28 +0900 Subject: [PATCH 1/6] Expose EF context factory for use in external migration logic --- osu.Game/OsuGameBase.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 5af992f800..3d40126b4f 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -161,6 +161,11 @@ namespace osu.Game private readonly BindableNumber globalTrackVolumeAdjust = new BindableNumber(global_track_volume_adjust); + /// + /// A legacy EF context factory if migration has not been performed to realm yet. + /// + protected DatabaseContextFactory EFContextFactory { get; private set; } + public OsuGameBase() { UseDevelopmentServer = DebugUtils.IsDebugBuild; @@ -184,19 +189,14 @@ namespace osu.Game Resources.AddStore(new DllResourceStore(OsuResources.ResourceAssembly)); - DatabaseContextFactory efContextFactory = Storage.Exists(DatabaseContextFactory.DATABASE_NAME) - ? new DatabaseContextFactory(Storage) - : null; + if (Storage.Exists(DatabaseContextFactory.DATABASE_NAME)) + dependencies.Cache(EFContextFactory = new DatabaseContextFactory(Storage)); - dependencies.Cache(realmFactory = new RealmContextFactory(Storage, "client", efContextFactory)); + dependencies.Cache(realmFactory = new RealmContextFactory(Storage, "client", EFContextFactory)); dependencies.Cache(RulesetStore = new RulesetStore(realmFactory, Storage)); dependencies.CacheAs(RulesetStore); - // A non-null context factory means there's still content to migrate. - if (efContextFactory != null) - new EFToRealmMigrator(efContextFactory, realmFactory, LocalConfig, Storage).Run(); - dependencies.CacheAs(Storage); var largeStore = new LargeTextureStore(Host.CreateTextureLoaderStore(new NamespacedResourceStore(Resources, @"Textures"))); From 5622d2ba4f704fe707ea5a9e647ee9859277ed34 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 21 Jan 2022 14:56:49 +0900 Subject: [PATCH 2/6] Show realm migration progress at `Loader` --- osu.Game/Database/EFToRealmMigrator.cs | 150 +++++++++++++++++++------ osu.Game/Screens/Loader.cs | 12 +- 2 files changed, 125 insertions(+), 37 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 727815cc4d..85d65fea82 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -4,52 +4,130 @@ using System; using System.IO; using System.Linq; +using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; +using osu.Framework.Allocation; +using osu.Framework.Graphics; +using osu.Framework.Graphics.Containers; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Game.Beatmaps; using osu.Game.Configuration; +using osu.Game.Graphics; +using osu.Game.Graphics.Sprites; +using osu.Game.Graphics.UserInterface; using osu.Game.Models; using osu.Game.Rulesets; using osu.Game.Scoring; using osu.Game.Skinning; +using osuTK; using Realms; #nullable enable namespace osu.Game.Database { - internal class EFToRealmMigrator + internal class EFToRealmMigrator : CompositeDrawable { - private readonly DatabaseContextFactory efContextFactory; - private readonly RealmContextFactory realmContextFactory; - private readonly OsuConfigManager config; - private readonly Storage storage; + public bool FinishedMigrating { get; private set; } - public EFToRealmMigrator(DatabaseContextFactory efContextFactory, RealmContextFactory realmContextFactory, OsuConfigManager config, Storage storage) + [Resolved] + private DatabaseContextFactory efContextFactory { get; set; } = null!; + + [Resolved] + private RealmContextFactory realmContextFactory { get; set; } = null!; + + [Resolved] + private OsuConfigManager config { get; set; } = null!; + + [Resolved] + private Storage storage { get; set; } = null!; + + private readonly OsuSpriteText currentOperationText; + + public EFToRealmMigrator() { - this.efContextFactory = efContextFactory; - this.realmContextFactory = realmContextFactory; - this.config = config; - this.storage = storage; + RelativeSizeAxes = Axes.Both; + + InternalChildren = new Drawable[] + { + new FillFlowContainer + { + AutoSizeAxes = Axes.Both, + Direction = FillDirection.Vertical, + Anchor = Anchor.Centre, + Origin = Anchor.Centre, + Spacing = new Vector2(10), + Children = new Drawable[] + { + new OsuSpriteText + { + Anchor = Anchor.Centre, + Origin = Anchor.Centre, + Text = "Database migration in progress", + Font = OsuFont.Default.With(size: 40) + }, + new OsuSpriteText + { + Anchor = Anchor.Centre, + Origin = Anchor.Centre, + Text = "This could take a few minutes depending on the speed of your disk(s).", + Font = OsuFont.Default.With(size: 30) + }, + new OsuSpriteText + { + Anchor = Anchor.Centre, + Origin = Anchor.Centre, + Text = "Please keep the window open until this completes!", + Font = OsuFont.Default.With(size: 30) + }, + new LoadingSpinner(true) + { + State = { Value = Visibility.Visible } + }, + currentOperationText = new OsuSpriteText + { + Anchor = Anchor.Centre, + Origin = Anchor.Centre, + Font = OsuFont.Default.With(size: 30) + }, + } + }, + }; } - public void Run() + protected override void LoadComplete() { + base.LoadComplete(); + + // needs to be run on the update thread because of realm BlockAllOperations. + // maybe we can work around this? not sure.. createBackup(); - using (var ef = efContextFactory.Get()) + Task.Factory.StartNew(() => { - migrateSettings(ef); - migrateSkins(ef); - migrateBeatmaps(ef); - migrateScores(ef); - } + using (var ef = efContextFactory.Get()) + { + migrateSettings(ef); + migrateSkins(ef); + migrateBeatmaps(ef); + migrateScores(ef); + } - // Delete the database permanently. - // Will cause future startups to not attempt migration. - Logger.Log("Migration successful, deleting EF database", LoggingTarget.Database); - efContextFactory.ResetDatabase(); + // Delete the database permanently. + // Will cause future startups to not attempt migration. + log("Migration successful, deleting EF database"); + efContextFactory.ResetDatabase(); + }, TaskCreationOptions.LongRunning).ContinueWith(t => + { + FinishedMigrating = true; + }); + } + + private void log(string message) + { + Logger.Log(message, LoggingTarget.Database); + Scheduler.AddOnce(m => currentOperationText.Text = m, message); } private void migrateBeatmaps(OsuDbContext ef) @@ -62,12 +140,12 @@ namespace osu.Game.Database .Include(s => s.Files).ThenInclude(f => f.FileInfo) .Include(s => s.Metadata); - Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database); + log("Beginning beatmaps migration to realm"); // previous entries in EF are removed post migration. if (!existingBeatmapSets.Any()) { - Logger.Log("No beatmaps found to migrate", LoggingTarget.Database); + log("No beatmaps found to migrate"); return; } @@ -75,13 +153,13 @@ namespace osu.Game.Database using (var realm = realmContextFactory.CreateContext()) { - Logger.Log($"Found {count} beatmaps in EF", LoggingTarget.Database); + log($"Found {count} beatmaps in EF"); // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (realm.All().Any(s => !s.Protected)) { - Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database); + log("Skipping migration as realm already has beatmaps loaded"); } else { @@ -96,7 +174,7 @@ namespace osu.Game.Database { transaction.Commit(); transaction = realm.BeginWrite(); - Logger.Log($"Migrated {written}/{count} beatmaps...", LoggingTarget.Database); + log($"Migrated {written}/{count} beatmaps..."); } var realmBeatmapSet = new BeatmapSetInfo @@ -156,7 +234,7 @@ namespace osu.Game.Database transaction.Commit(); } - Logger.Log($"Successfully migrated {count} beatmaps to realm", LoggingTarget.Database); + log($"Successfully migrated {count} beatmaps to realm"); } } } @@ -193,12 +271,12 @@ namespace osu.Game.Database .Include(s => s.Files) .ThenInclude(f => f.FileInfo); - Logger.Log("Beginning scores migration to realm", LoggingTarget.Database); + log("Beginning scores migration to realm"); // previous entries in EF are removed post migration. if (!existingScores.Any()) { - Logger.Log("No scores found to migrate", LoggingTarget.Database); + log("No scores found to migrate"); return; } @@ -206,12 +284,12 @@ namespace osu.Game.Database using (var realm = realmContextFactory.CreateContext()) { - Logger.Log($"Found {count} scores in EF", LoggingTarget.Database); + log($"Found {count} scores in EF"); // only migrate data if the realm database is empty. if (realm.All().Any()) { - Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database); + log("Skipping migration as realm already has scores loaded"); } else { @@ -226,7 +304,7 @@ namespace osu.Game.Database { transaction.Commit(); transaction = realm.BeginWrite(); - Logger.Log($"Migrated {written}/{count} scores...", LoggingTarget.Database); + log($"Migrated {written}/{count} scores..."); } var beatmap = realm.All().First(b => b.Hash == score.BeatmapInfo.Hash); @@ -270,7 +348,7 @@ namespace osu.Game.Database transaction.Commit(); } - Logger.Log($"Successfully migrated {count} scores to realm", LoggingTarget.Database); + log($"Successfully migrated {count} scores to realm"); } } } @@ -308,7 +386,7 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any(s => !s.Protected)) { - Logger.Log($"Migrating {existingSkins.Count} skins", LoggingTarget.Database); + log($"Migrating {existingSkins.Count} skins"); foreach (var skin in existingSkins) { @@ -356,7 +434,7 @@ namespace osu.Game.Database if (!existingSettings.Any()) return; - Logger.Log("Beginning settings migration to realm", LoggingTarget.Database); + log("Beginning settings migration to realm"); using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) @@ -364,7 +442,7 @@ namespace osu.Game.Database // only migrate data if the realm database is empty. if (!realm.All().Any()) { - Logger.Log($"Migrating {existingSettings.Count} settings", LoggingTarget.Database); + log($"Migrating {existingSettings.Count} settings"); foreach (var dkb in existingSettings) { diff --git a/osu.Game/Screens/Loader.cs b/osu.Game/Screens/Loader.cs index 41097a4c74..8c4a13f2bd 100644 --- a/osu.Game/Screens/Loader.cs +++ b/osu.Game/Screens/Loader.cs @@ -12,6 +12,7 @@ using osu.Game.Screens.Menu; using osu.Framework.Screens; using osu.Framework.Threading; using osu.Game.Configuration; +using osu.Game.Database; using osu.Game.Graphics.UserInterface; using IntroSequence = osu.Game.Configuration.IntroSequence; @@ -63,6 +64,11 @@ namespace osu.Game.Screens protected virtual ShaderPrecompiler CreateShaderPrecompiler() => new ShaderPrecompiler(); + [Resolved(canBeNull: true)] + private DatabaseContextFactory efContextFactory { get; set; } + + private EFToRealmMigrator realmMigrator; + public override void OnEntering(IScreen last) { base.OnEntering(last); @@ -70,6 +76,10 @@ namespace osu.Game.Screens LoadComponentAsync(precompiler = CreateShaderPrecompiler(), AddInternal); LoadComponentAsync(loadableScreen = CreateLoadableScreen()); + // A non-null context factory means there's still content to migrate. + if (efContextFactory != null) + LoadComponentAsync(realmMigrator = new EFToRealmMigrator(), AddInternal); + LoadComponentAsync(spinner = new LoadingSpinner(true, true) { Anchor = Anchor.BottomRight, @@ -86,7 +96,7 @@ namespace osu.Game.Screens private void checkIfLoaded() { - if (loadableScreen.LoadState != LoadState.Ready || !precompiler.FinishedCompiling) + if (loadableScreen.LoadState != LoadState.Ready || !precompiler.FinishedCompiling || realmMigrator?.FinishedMigrating == false) { Schedule(checkIfLoaded); return; From 855ef3fa92ef76e53d2db54eeddf7ec70482e248 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Sat, 22 Jan 2022 11:50:47 +0900 Subject: [PATCH 3/6] Create backup before any realm contexts are used --- osu.Game/Database/EFToRealmMigrator.cs | 16 ---------------- osu.Game/OsuGameBase.cs | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 85d65fea82..1e21107628 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -100,10 +100,6 @@ namespace osu.Game.Database { base.LoadComplete(); - // needs to be run on the update thread because of realm BlockAllOperations. - // maybe we can work around this? not sure.. - createBackup(); - Task.Factory.StartNew(() => { using (var ef = efContextFactory.Get()) @@ -470,17 +466,5 @@ namespace osu.Game.Database private string? getRulesetShortNameFromLegacyID(long rulesetId) => efContextFactory.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName; - - private void createBackup() - { - string migration = $"before_final_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; - - efContextFactory.CreateBackup($"client.{migration}.db"); - realmContextFactory.CreateBackup($"client.{migration}.realm"); - - using (var source = storage.GetStream("collection.db")) - using (var destination = storage.GetStream($"collection.{migration}.db", FileAccess.Write, FileMode.CreateNew)) - source.CopyTo(destination); - } } } diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 3d40126b4f..710a7be8d4 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -197,6 +197,21 @@ namespace osu.Game dependencies.Cache(RulesetStore = new RulesetStore(realmFactory, Storage)); dependencies.CacheAs(RulesetStore); + // Backup is taken here rather than in EFToRealmMigrator to avoid recycling realm contexts + // after initial usages below. It can be moved once a direction is established for handling re-subscription. + // See https://github.com/ppy/osu/pull/16547 for more discussion. + if (EFContextFactory != null) + { + string migration = $"before_final_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + + EFContextFactory.CreateBackup($"client.{migration}.db"); + realmFactory.CreateBackup($"client.{migration}.realm"); + + using (var source = Storage.GetStream("collection.db")) + using (var destination = Storage.GetStream($"collection.{migration}.db", FileAccess.Write, FileMode.CreateNew)) + source.CopyTo(destination); + } + dependencies.CacheAs(Storage); var largeStore = new LargeTextureStore(Host.CreateTextureLoaderStore(new NamespacedResourceStore(Resources, @"Textures"))); From daed0b04dc97e8a50375f6955451bd3175af6946 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Sat, 22 Jan 2022 11:54:40 +0900 Subject: [PATCH 4/6] Remove using statements --- osu.Game/Database/EFToRealmMigrator.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 1e21107628..f5b338062a 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -1,8 +1,6 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. -using System; -using System.IO; using System.Linq; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; From c99f2278797abd9c2e259b422ccd602ae14eea30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Sat, 22 Jan 2022 13:48:49 +0100 Subject: [PATCH 5/6] Remove no longer used resolved storage --- osu.Game/Database/EFToRealmMigrator.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index f5b338062a..161c12e35e 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -38,9 +38,6 @@ namespace osu.Game.Database [Resolved] private OsuConfigManager config { get; set; } = null!; - [Resolved] - private Storage storage { get; set; } = null!; - private readonly OsuSpriteText currentOperationText; public EFToRealmMigrator() From 7e68371d28b9ae109a8e89039bf1db1ed801436b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Sat, 22 Jan 2022 14:20:28 +0100 Subject: [PATCH 6/6] Move log statement about migration completed closer to rest of migration code --- osu.Game/Database/DatabaseContextFactory.cs | 4 ---- osu.Game/Database/EFToRealmMigrator.cs | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index 635c4373cd..c84edbfb81 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -5,7 +5,6 @@ using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; -using osu.Framework.Development; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Statistics; @@ -151,9 +150,6 @@ namespace osu.Game.Database { Logger.Log($"Creating full EF database backup at {backupFilename}", LoggingTarget.Database); - if (DebugUtils.IsDebugBuild) - Logger.Log("Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.", level: LogLevel.Important); - using (var source = storage.GetStream(DATABASE_NAME)) using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index c224399dbc..bbbdac352e 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using osu.Framework.Allocation; +using osu.Framework.Development; using osu.Framework.Graphics; using osu.Framework.Graphics.Containers; using osu.Framework.Logging; @@ -108,6 +109,9 @@ namespace osu.Game.Database // Will cause future startups to not attempt migration. log("Migration successful, deleting EF database"); efContextFactory.ResetDatabase(); + + if (DebugUtils.IsDebugBuild) + Logger.Log("Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.", level: LogLevel.Important); }, TaskCreationOptions.LongRunning).ContinueWith(t => { FinishedMigrating = true;