h1or4dme8/osu-lazer
1
0
Fork 0
mirror of https://github.com/ppy/osu.git synced 2024-11-11 09:07:52 +08:00
Code Issues Releases Wiki Activity

Merge pull request #16491 from peppy/realm-integration/safer-migration

Browse Source 
Improve safety and logging of realm migration process
This commit is contained in:
Dan Balasescu 2022-01-19 12:23:48 +09:00 committed by GitHub
commit 8c120c6bfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 200 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
osu.Game/Database/DatabaseContextFactory.cs
View File

@ -1,9 +1,12 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System.IO;
using System.Linq;
using System.Threading;
using Microsoft.EntityFrameworkCore.Storage;
using osu.Framework.Development;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Framework.Statistics;
@ -144,6 +147,18 @@ namespace osu.Game.Database
Database = { AutoTransactionsEnabled = false }
};
public void CreateBackup(string backupFilename)
{
Logger.Log($"Creating full EF database backup at {backupFilename}", LoggingTarget.Database);
if (DebugUtils.IsDebugBuild)
Logger.Log("Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.", level: LogLevel.Important);
using (var source = storage.GetStream(DATABASE_NAME))
using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew))
source.CopyTo(destination);
}
public void ResetDatabase()
{
lock (writeLock)

108
osu.Game/Database/EFToRealmMigrator.cs
View File

@ -1,8 +1,11 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.EntityFrameworkCore;
using osu.Framework.Logging;
using osu.Game.Beatmaps;
using osu.Game.Configuration;
using osu.Game.Models;
@ -21,6 +24,8 @@ namespace osu.Game.Database
private readonly RealmContextFactory realmContextFactory;
private readonly OsuConfigManager config;
private bool hasTakenBackup;
public EFToRealmMigrator(DatabaseContextFactory efContextFactory, RealmContextFactory realmContextFactory, OsuConfigManager config)
{
this.efContextFactory = efContextFactory;
@ -30,20 +35,24 @@ namespace osu.Game.Database
public void Run()
{
using (var db = efContextFactory.GetForWrite())
using (var ef = efContextFactory.GetForWrite())
{
migrateSettings(db);
migrateSkins(db);
migrateBeatmaps(db);
migrateScores(db);
}
migrateSettings(ef);
migrateSkins(ef);
migrateBeatmaps(ef);
migrateScores(ef);
}
private void migrateBeatmaps(DatabaseWriteUsage db)
// Delete the database permanently.
// Will cause future startups to not attempt migration.
Logger.Log("Migration successful, deleting EF database", LoggingTarget.Database);
efContextFactory.ResetDatabase();
}
private void migrateBeatmaps(DatabaseWriteUsage ef)
{
// can be removed 20220730.
var existingBeatmapSets = db.Context.EFBeatmapSetInfo
List<EFBeatmapSetInfo> existingBeatmapSets = ef.Context.EFBeatmapSetInfo
.Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo)
.Include(s => s.Beatmaps).ThenInclude(b => b.Metadata)
.Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty)
@ -51,16 +60,38 @@ namespace osu.Game.Database
.Include(s => s.Metadata)
.ToList();
Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database);
// previous entries in EF are removed post migration.
if (!existingBeatmapSets.Any())
{
Logger.Log("No beatmaps found to migrate", LoggingTarget.Database);
return;
}
using (var realm = realmContextFactory.CreateContext())
using (var transaction = realm.BeginWrite())
{
Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database);
if (!hasTakenBackup)
{
string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
efContextFactory.CreateBackup($"client.{migration}.db");
realmContextFactory.CreateBackup($"client.{migration}.realm");
hasTakenBackup = true;
}
// only migrate data if the realm database is empty.
// note that this cannot be written as: `realm.All<BeatmapInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (!realm.All<BeatmapSetInfo>().Any(s => !s.Protected))
// note that this cannot be written as: `realm.All<BeatmapSetInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (realm.All<BeatmapSetInfo>().Any(s => !s.Protected))
{
Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database);
}
else
{
using (var transaction = realm.BeginWrite())
{
foreach (var beatmapSet in existingBeatmapSets)
{
@ -115,12 +146,14 @@ namespace osu.Game.Database
realm.Add(realmBeatmapSet);
}
}
db.Context.RemoveRange(existingBeatmapSets);
// Intentionally don't clean up the files, so they don't get purged by EF.
transaction.Commit();
Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database);
}
}
ef.Context.RemoveRange(existingBeatmapSets);
// Intentionally don't clean up the files, so they don't get purged by EF.
}
}
@ -151,23 +184,44 @@ namespace osu.Game.Database
private void migrateScores(DatabaseWriteUsage db)
{
// can be removed 20220730.
var existingScores = db.Context.ScoreInfo
List<EFScoreInfo> existingScores = db.Context.ScoreInfo
.Include(s => s.Ruleset)
.Include(s => s.BeatmapInfo)
.Include(s => s.Files)
.ThenInclude(f => f.FileInfo)
.ToList();
Logger.Log("Beginning scores migration to realm", LoggingTarget.Database);
// previous entries in EF are removed post migration.
if (!existingScores.Any())
{
Logger.Log("No scores found to migrate", LoggingTarget.Database);
return;
}
using (var realm = realmContextFactory.CreateContext())
using (var transaction = realm.BeginWrite())
{
Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database);
if (!hasTakenBackup)
{
string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
efContextFactory.CreateBackup($"client.{migration}.db");
realmContextFactory.CreateBackup($"client.{migration}.realm");
hasTakenBackup = true;
}
// only migrate data if the realm database is empty.
// note that this cannot be written as: `realm.All<ScoreInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (!realm.All<ScoreInfo>().Any())
if (realm.All<ScoreInfo>().Any())
{
Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database);
}
else
{
using (var transaction = realm.BeginWrite())
{
foreach (var score in existingScores)
{
@ -201,12 +255,14 @@ namespace osu.Game.Database
realm.Add(realmScore);
}
transaction.Commit();
Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database);
}
}
db.Context.RemoveRange(existingScores);
// Intentionally don't clean up the files, so they don't get purged by EF.
transaction.Commit();
}
}
@ -243,6 +299,8 @@ namespace osu.Game.Database
// note that this cannot be written as: `realm.All<SkinInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (!realm.All<SkinInfo>().Any(s => !s.Protected))
{
Logger.Log($"Migrating {existingSkins.Count} skins", LoggingTarget.Database);
foreach (var skin in existingSkins)
{
var realmSkin = new SkinInfo
@ -286,18 +344,22 @@ namespace osu.Game.Database
private void migrateSettings(DatabaseWriteUsage db)
{
// migrate ruleset settings. can be removed 20220315.
var existingSettings = db.Context.DatabasedSetting;
var existingSettings = db.Context.DatabasedSetting.ToList();
// previous entries in EF are removed post migration.
if (!existingSettings.Any())
return;
Logger.Log("Beginning settings migration to realm", LoggingTarget.Database);
using (var realm = realmContextFactory.CreateContext())
using (var transaction = realm.BeginWrite())
{
// only migrate data if the realm database is empty.
if (!realm.All<RealmRulesetSetting>().Any())
{
Logger.Log($"Migrating {existingSettings.Count} settings", LoggingTarget.Database);
foreach (var dkb in existingSettings)
{
if (dkb.RulesetID == null)

21
osu.Game/Database/RealmContextFactory.cs
View File

@ -361,6 +361,17 @@ namespace osu.Game.Database
private string? getRulesetShortNameFromLegacyID(long rulesetId) =>
efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName;
public void CreateBackup(string backupFilename)
{
using (BlockAllOperations())
{
Logger.Log($"Creating full realm database backup at {backupFilename}", LoggingTarget.Database);
using (var source = storage.GetStream(Filename))
using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew))
source.CopyTo(destination);
}
}
/// <summary>
/// Flush any active contexts and block any further writes.
/// </summary>
@ -374,17 +385,17 @@ namespace osu.Game.Database
if (isDisposed)
throw new ObjectDisposedException(nameof(RealmContextFactory));
if (!ThreadSafety.IsUpdateThread)
throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread.");
Logger.Log(@"Blocking realm operations.", LoggingTarget.Database);
try
{
contextCreationLock.Wait();
lock (contextLock)
{
if (!ThreadSafety.IsUpdateThread && context != null)
throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread.");
Logger.Log(@"Blocking realm operations.", LoggingTarget.Database);
context?.Dispose();
context = null;
}

1
osu.Game/OsuGameBase.cs
View File

@ -193,6 +193,7 @@ namespace osu.Game
dependencies.Cache(RulesetStore = new RulesetStore(realmFactory, Storage));
dependencies.CacheAs<IRulesetStore>(RulesetStore);
// A non-null context factory means there's still content to migrate.
if (efContextFactory != null)
new EFToRealmMigrator(efContextFactory, realmFactory, LocalConfig).Run();