osu-lazer/osu.Game/Database/EFToRealmMigrator.cs

// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.

using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using osu.Framework;
using osu.Framework.Allocation;
using osu.Framework.Development;
using osu.Framework.Graphics;
using osu.Framework.Graphics.Containers;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Game.Beatmaps;
using osu.Game.Configuration;
using osu.Game.Graphics;
using osu.Game.Graphics.Containers;
using osu.Game.Graphics.Sprites;
using osu.Game.Graphics.UserInterface;
using osu.Game.Models;
using osu.Game.Overlays;
using osu.Game.Overlays.Notifications;
using osu.Game.Rulesets;
using osu.Game.Scoring;
using osu.Game.Skinning;
using osuTK;
using Realms;
using SharpCompress.Archives;
using SharpCompress.Archives.Zip;
using SharpCompress.Common;
using SharpCompress.Writers.Zip;

namespace osu.Game.Database
{
    internal class EFToRealmMigrator : CompositeDrawable
    {
        public Task<bool> MigrationCompleted => migrationCompleted.Task;

        private readonly TaskCompletionSource<bool> migrationCompleted = new TaskCompletionSource<bool>();

        [Resolved]
        private DatabaseContextFactory efContextFactory { get; set; } = null!;

        [Resolved]
        private RealmAccess realm { get; set; } = null!;

        [Resolved]
        private OsuConfigManager config { get; set; } = null!;

        [Resolved]
        private INotificationOverlay notificationOverlay { get; set; } = null!;

        [Resolved]
        private OsuGame game { get; set; } = null!;

        [Resolved]
        private Storage storage { get; set; } = null!;

        private readonly OsuTextFlowContainer currentOperationText;

        public EFToRealmMigrator()
        {
            RelativeSizeAxes = Axes.Both;

            InternalChildren = new Drawable[]
            {
                new FillFlowContainer
                {
                    AutoSizeAxes = Axes.Both,
                    Direction = FillDirection.Vertical,
                    Anchor = Anchor.Centre,
                    Origin = Anchor.Centre,
                    Spacing = new Vector2(10),
                    Children = new Drawable[]
                    {
                        new OsuSpriteText
                        {
                            Anchor = Anchor.Centre,
                            Origin = Anchor.Centre,
                            Text = "Database migration in progress",
                            Font = OsuFont.Default.With(size: 40)
                        },
                        new OsuSpriteText
                        {
                            Anchor = Anchor.Centre,
                            Origin = Anchor.Centre,
                            Text = "This could take a few minutes depending on the speed of your disk(s).",
                            Font = OsuFont.Default.With(size: 30)
                        },
                        new OsuSpriteText
                        {
                            Anchor = Anchor.Centre,
                            Origin = Anchor.Centre,
                            Text = "Please keep the window open until this completes!",
                            Font = OsuFont.Default.With(size: 30)
                        },
                        new LoadingSpinner(true)
                        {
                            State = { Value = Visibility.Visible }
                        },
                        currentOperationText = new OsuTextFlowContainer(cp => cp.Font = OsuFont.Default.With(size: 30))
                        {
                            Anchor = Anchor.Centre,
                            Origin = Anchor.Centre,
                            AutoSizeAxes = Axes.Y,
                            RelativeSizeAxes = Axes.X,
                            TextAnchor = Anchor.TopCentre,
                        },
                    }
                },
            };
        }

        protected override void LoadComplete()
        {
            base.LoadComplete();
            beginMigration();
        }

        private void beginMigration()
        {
            const string backup_folder = "backups";

            string backupSuffix = $"before_final_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";

            // required for initial backup.
            var realmBlockOperations = realm.BlockAllOperations("EF migration");

            Task.Factory.StartNew(() =>
            {
                try
                {
                    realm.CreateBackup(Path.Combine(backup_folder, $"client.{backupSuffix}.realm"), realmBlockOperations);
                }
                finally
                {
                    // Above call will dispose of the blocking token when done.
                    // Clean up here so we don't accidentally dispose twice.
                    realmBlockOperations = null;
                }

                efContextFactory.CreateBackup(Path.Combine(backup_folder, $"client.{backupSuffix}.db"));

                using (var ef = efContextFactory.Get())
                {
                    realm.Write(r =>
                    {
                        // Before beginning, ensure realm is in an empty state.
                        // Migrations which are half-completed could lead to issues if the user tries a second time.
                        // Note that we only do this for beatmaps and scores since the other migrations are yonks old.
                        r.RemoveAll<BeatmapSetInfo>();
                        r.RemoveAll<BeatmapInfo>();
                        r.RemoveAll<BeatmapMetadata>();
                        r.RemoveAll<ScoreInfo>();
                    });

                    ef.Migrate();

                    migrateSettings(ef);
                    migrateSkins(ef);
                    migrateBeatmaps(ef);
                    migrateScores(ef);
                }
            }, TaskCreationOptions.LongRunning).ContinueWith(t =>
            {
                if (t.Exception == null)
                {
                    log("Migration successful!");

                    if (DebugUtils.IsDebugBuild)
                    {
                        Logger.Log(
                            "Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.",
                            level: LogLevel.Important);
                    }
                }
                else
                {
                    log("Migration failed!");
                    Logger.Log(t.Exception.ToString(), LoggingTarget.Database);

                    if (RuntimeInfo.OS == RuntimeInfo.Platform.macOS && t.Exception.Flatten().InnerException is TypeInitializationException)
                    {
                        // Not guaranteed to be the only cause of exception, but let's roll with it for now.
                        log("Please download and run the intel version of osu! once\nto allow data migration to complete!");
                        efContextFactory.SetMigrationCompletion();
                        return;
                    }

                    notificationOverlay.Post(new SimpleErrorNotification
                    {
                        Text =
                            "IMPORTANT: During data migration, some of your data could not be successfully migrated. The previous version has been backed up.\n\nFor further assistance, please open a discussion on github and attach your backup files (click to get started).",
                        Activated = () =>
                        {
                            game.OpenUrlExternally(
                                $@"https://github.com/ppy/osu/discussions/new?title=Realm%20migration%20issue ({t.Exception.Message})&body=Please%20drag%20the%20""attach_me.zip""%20file%20here!&category=q-a",
                                true);

                            const string attachment_filename = "attach_me.zip";

                            var backupStorage = storage.GetStorageForDirectory(backup_folder);

                            backupStorage.Delete(attachment_filename);

                            try
                            {
                                using (var zip = ZipArchive.Create())
                                {
                                    zip.AddAllFromDirectory(backupStorage.GetFullPath(string.Empty));
                                    zip.SaveTo(Path.Combine(backupStorage.GetFullPath(string.Empty), attachment_filename), new ZipWriterOptions(CompressionType.Deflate));
                                }
                            }
                            catch { }

                            backupStorage.PresentFileExternally(attachment_filename);

                            return true;
                        }
                    });
                }

                // Regardless of success, since the game is going to continue with startup let's move the ef database out of the way.
                // If we were to not do this, the migration would run another time the next time the user starts the game.
                deletePreRealmData();

                // If something went wrong and the disposal token wasn't invoked above, ensure it is here.
                realmBlockOperations?.Dispose();

                migrationCompleted.SetResult(true);
                efContextFactory.SetMigrationCompletion();
            });
        }

        private void deletePreRealmData()
        {
            // Delete the database permanently.
            // Will cause future startups to not attempt migration.
            efContextFactory.ResetDatabase();
        }

        private void log(string message)
        {
            Logger.Log(message, LoggingTarget.Database);
            Scheduler.AddOnce(m => currentOperationText.Text = m, message);
        }

        private void migrateBeatmaps(OsuDbContext ef)
        {
            // can be removed 20220730.
            var existingBeatmapSets = ef.EFBeatmapSetInfo
                                        .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo)
                                        .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata)
                                        .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty)
                                        .Include(s => s.Files).ThenInclude(f => f.FileInfo)
                                        .Include(s => s.Metadata)
                                        .AsSplitQuery();

            log("Beginning beatmaps migration to realm");

            // previous entries in EF are removed post migration.
            if (!existingBeatmapSets.Any())
            {
                log("No beatmaps found to migrate");
                return;
            }

            int count = existingBeatmapSets.Count();

            realm.Run(r =>
            {
                log($"Found {count} beatmaps in EF");

                var transaction = r.BeginWrite();
                int written = 0;
                int missing = 0;

                try
                {
                    foreach (var beatmapSet in existingBeatmapSets)
                    {
                        if (++written % 1000 == 0)
                        {
                            transaction.Commit();
                            transaction = r.BeginWrite();
                            log($"Migrated {written}/{count} beatmaps...");
                        }

                        var realmBeatmapSet = new BeatmapSetInfo
                        {
                            OnlineID = beatmapSet.OnlineID ?? -1,
                            DateAdded = beatmapSet.DateAdded,
                            Status = beatmapSet.Status,
                            DeletePending = beatmapSet.DeletePending,
                            Hash = beatmapSet.Hash,
                            Protected = beatmapSet.Protected,
                        };

                        migrateFiles(beatmapSet, r, realmBeatmapSet);

                        foreach (var beatmap in beatmapSet.Beatmaps)
                        {
                            var ruleset = r.Find<RulesetInfo>(beatmap.RulesetInfo.ShortName);
                            var metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata);

                            if (ruleset == null)
                            {
                                log($"Skipping {++missing} beatmaps with missing ruleset");
                                continue;
                            }

                            var realmBeatmap = new BeatmapInfo(ruleset, new BeatmapDifficulty(beatmap.BaseDifficulty), metadata)
                            {
                                DifficultyName = beatmap.DifficultyName,
                                Status = beatmap.Status,
                                OnlineID = beatmap.OnlineID ?? -1,
                                Length = beatmap.Length,
                                BPM = beatmap.BPM,
                                Hash = beatmap.Hash,
                                StarRating = beatmap.StarRating,
                                MD5Hash = beatmap.MD5Hash,
                                Hidden = beatmap.Hidden,
                                AudioLeadIn = beatmap.AudioLeadIn,
                                StackLeniency = beatmap.StackLeniency,
                                SpecialStyle = beatmap.SpecialStyle,
                                LetterboxInBreaks = beatmap.LetterboxInBreaks,
                                WidescreenStoryboard = beatmap.WidescreenStoryboard,
                                EpilepsyWarning = beatmap.EpilepsyWarning,
                                SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate,
                                DistanceSpacing = beatmap.DistanceSpacing,
                                BeatDivisor = beatmap.BeatDivisor,
                                GridSize = beatmap.GridSize,
                                TimelineZoom = beatmap.TimelineZoom,
                                Countdown = beatmap.Countdown,
                                CountdownOffset = beatmap.CountdownOffset,
                                Bookmarks = beatmap.Bookmarks,
                                BeatmapSet = realmBeatmapSet,
                            };

                            realmBeatmapSet.Beatmaps.Add(realmBeatmap);
                        }

                        r.Add(realmBeatmapSet);
                    }
                }
                finally
                {
                    transaction.Commit();
                }

                log($"Successfully migrated {count} beatmaps to realm");
            });
        }

        private BeatmapMetadata getBestMetadata(EFBeatmapMetadata? beatmapMetadata, EFBeatmapMetadata? beatmapSetMetadata)
        {
            var metadata = beatmapMetadata ?? beatmapSetMetadata ?? new EFBeatmapMetadata();

            return new BeatmapMetadata
            {
                Title = metadata.Title,
                TitleUnicode = metadata.TitleUnicode,
                Artist = metadata.Artist,
                ArtistUnicode = metadata.ArtistUnicode,
                Author =
                {
                    OnlineID = metadata.Author.Id,
                    Username = metadata.Author.Username,
                },
                Source = metadata.Source,
                Tags = metadata.Tags,
                PreviewTime = metadata.PreviewTime,
                AudioFile = metadata.AudioFile,
                BackgroundFile = metadata.BackgroundFile,
            };
        }

        private void migrateScores(OsuDbContext db)
        {
            // can be removed 20220730.
            var existingScores = db.ScoreInfo
                                   .Include(s => s.Ruleset)
                                   .Include(s => s.BeatmapInfo)
                                   .Include(s => s.Files)
                                   .ThenInclude(f => f.FileInfo)
                                   .AsSplitQuery();

            log("Beginning scores migration to realm");

            // previous entries in EF are removed post migration.
            if (!existingScores.Any())
            {
                log("No scores found to migrate");
                return;
            }

            int count = existingScores.Count();

            realm.Run(r =>
            {
                log($"Found {count} scores in EF");

                var transaction = r.BeginWrite();
                int written = 0;
                int missing = 0;

                try
                {
                    foreach (var score in existingScores)
                    {
                        if (++written % 1000 == 0)
                        {
                            transaction.Commit();
                            transaction = r.BeginWrite();
                            log($"Migrated {written}/{count} scores...");
                        }

                        var beatmap = r.All<BeatmapInfo>().FirstOrDefault(b => b.Hash == score.BeatmapInfo.Hash);
                        var ruleset = r.Find<RulesetInfo>(score.Ruleset.ShortName);

                        if (beatmap == null || ruleset == null)
                        {
                            log($"Skipping {++missing} scores with missing ruleset or beatmap");
                            continue;
                        }

                        var user = new RealmUser
                        {
                            OnlineID = score.User.OnlineID,
                            Username = score.User.Username
                        };

                        var realmScore = new ScoreInfo(beatmap, ruleset, user)
                        {
                            Hash = score.Hash,
                            DeletePending = score.DeletePending,
                            OnlineID = score.OnlineID ?? -1,
                            ModsJson = score.ModsJson,
                            StatisticsJson = score.StatisticsJson,
                            TotalScore = score.TotalScore,
                            MaxCombo = score.MaxCombo,
                            Accuracy = score.Accuracy,
                            HasReplay = ((IScoreInfo)score).HasReplay,
                            Date = score.Date,
                            PP = score.PP,
                            Rank = score.Rank,
                            HitEvents = score.HitEvents,
                            Passed = score.Passed,
                            Combo = score.Combo,
                            Position = score.Position,
                            Statistics = score.Statistics,
                            Mods = score.Mods,
                            APIMods = score.APIMods,
                        };

                        migrateFiles(score, r, realmScore);

                        r.Add(realmScore);
                    }
                }
                finally
                {
                    transaction.Commit();
                }

                log($"Successfully migrated {count} scores to realm");
            });
        }

        private void migrateSkins(OsuDbContext db)
        {
            // can be removed 20220530.
            var existingSkins = db.SkinInfo
                                  .Include(s => s.Files)
                                  .ThenInclude(f => f.FileInfo)
                                  .AsSplitQuery()
                                  .ToList();

            // previous entries in EF are removed post migration.
            if (!existingSkins.Any())
                return;

            var userSkinChoice = config.GetBindable<string>(OsuSetting.Skin);
            int.TryParse(userSkinChoice.Value, out int userSkinInt);

            switch (userSkinInt)
            {
                case EFSkinInfo.DEFAULT_SKIN:
                    userSkinChoice.Value = SkinInfo.DEFAULT_SKIN.ToString();
                    break;

                case EFSkinInfo.CLASSIC_SKIN:
                    userSkinChoice.Value = SkinInfo.CLASSIC_SKIN.ToString();
                    break;
            }

            realm.Run(r =>
            {
                using (var transaction = r.BeginWrite())
                {
                    // only migrate data if the realm database is empty.
                    // note that this cannot be written as: `r.All<SkinInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
                    if (!r.All<SkinInfo>().Any(s => !s.Protected))
                    {
                        log($"Migrating {existingSkins.Count} skins");

                        foreach (var skin in existingSkins)
                        {
                            var realmSkin = new SkinInfo
                            {
                                Name = skin.Name,
                                Creator = skin.Creator,
                                Hash = skin.Hash,
                                Protected = false,
                                InstantiationInfo = skin.InstantiationInfo,
                            };

                            migrateFiles(skin, r, realmSkin);

                            r.Add(realmSkin);

                            if (skin.ID == userSkinInt)
                                userSkinChoice.Value = realmSkin.ID.ToString();
                        }
                    }

                    transaction.Commit();
                }
            });
        }

        private static void migrateFiles<T>(IHasFiles<T> fileSource, Realm realm, IHasRealmFiles realmObject) where T : INamedFileInfo
        {
            foreach (var file in fileSource.Files)
            {
                var realmFile = realm.Find<RealmFile>(file.FileInfo.Hash);

                if (realmFile == null)
                    realm.Add(realmFile = new RealmFile { Hash = file.FileInfo.Hash });

                realmObject.Files.Add(new RealmNamedFileUsage(realmFile, file.Filename));
            }
        }

        private void migrateSettings(OsuDbContext db)
        {
            // migrate ruleset settings. can be removed 20220315.
            var existingSettings = db.DatabasedSetting.ToList();

            // previous entries in EF are removed post migration.
            if (!existingSettings.Any())
                return;

            log("Beginning settings migration to realm");

            realm.Run(r =>
            {
                using (var transaction = r.BeginWrite())
                {
                    // only migrate data if the realm database is empty.
                    if (!r.All<RealmRulesetSetting>().Any())
                    {
                        log($"Migrating {existingSettings.Count} settings");

                        foreach (var dkb in existingSettings)
                        {
                            if (dkb.RulesetID == null)
                                continue;

                            string? shortName = getRulesetShortNameFromLegacyID(dkb.RulesetID.Value);

                            if (string.IsNullOrEmpty(shortName))
                                continue;

                            r.Add(new RealmRulesetSetting
                            {
                                Key = dkb.Key,
                                Value = dkb.StringValue,
                                RulesetName = shortName,
                                Variant = dkb.Variant ?? 0,
                            });
                        }
                    }

                    transaction.Commit();
                }
            });
        }

        private string? getRulesetShortNameFromLegacyID(long rulesetId) =>
            efContextFactory.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName;
    }
}