h1or4dme8/osu-lazer
1
0
Fork 0
mirror of https://github.com/ppy/osu.git synced 2024-09-21 17:27:24 +08:00
Code Issues Releases Wiki Activity

Merge pull request #16491 from peppy/realm-integration/safer-migration

Browse Source 
Improve safety and logging of realm migration process
This commit is contained in:
Dan Balasescu 2022-01-19 12:23:48 +09:00 committed by GitHub
commit 8c120c6bfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 200 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
osu.Game/Database/DatabaseContextFactory.cs
View File

@ -1,9 +1,12 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System.IO;
using System.Linq;
using System.Threading;
using Microsoft.EntityFrameworkCore.Storage;
using osu.Framework.Development;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Framework.Statistics;
@ -144,6 +147,18 @@ namespace osu.Game.Database
Database = { AutoTransactionsEnabled = false }
};
public void CreateBackup(string backupFilename)
{
Logger.Log($"Creating full EF database backup at {backupFilename}", LoggingTarget.Database);
if (DebugUtils.IsDebugBuild)
Logger.Log("Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.", level: LogLevel.Important);
using (var source = storage.GetStream(DATABASE_NAME))
using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew))
source.CopyTo(destination);
}
public void ResetDatabase()
{
lock (writeLock)

274
osu.Game/Database/EFToRealmMigrator.cs
View File

@ -1,8 +1,11 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.EntityFrameworkCore;
using osu.Framework.Logging;
using osu.Game.Beatmaps;
using osu.Game.Configuration;
using osu.Game.Models;
@ -21,6 +24,8 @@ namespace osu.Game.Database
private readonly RealmContextFactory realmContextFactory;
private readonly OsuConfigManager config;
private bool hasTakenBackup;
public EFToRealmMigrator(DatabaseContextFactory efContextFactory, RealmContextFactory realmContextFactory, OsuConfigManager config)
{
this.efContextFactory = efContextFactory;
@ -30,97 +35,125 @@ namespace osu.Game.Database
public void Run()
{
using (var db = efContextFactory.GetForWrite())
using (var ef = efContextFactory.GetForWrite())
{
migrateSettings(db);
migrateSkins(db);
migrateBeatmaps(db);
migrateScores(db);
migrateSettings(ef);
migrateSkins(ef);
migrateBeatmaps(ef);
migrateScores(ef);
}
// Delete the database permanently.
// Will cause future startups to not attempt migration.
Logger.Log("Migration successful, deleting EF database", LoggingTarget.Database);
efContextFactory.ResetDatabase();
}
private void migrateBeatmaps(DatabaseWriteUsage db)
private void migrateBeatmaps(DatabaseWriteUsage ef)
{
// can be removed 20220730.
var existingBeatmapSets = db.Context.EFBeatmapSetInfo
.Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo)
.Include(s => s.Beatmaps).ThenInclude(b => b.Metadata)
.Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty)
.Include(s => s.Files).ThenInclude(f => f.FileInfo)
.Include(s => s.Metadata)
.ToList();
List<EFBeatmapSetInfo> existingBeatmapSets = ef.Context.EFBeatmapSetInfo
.Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo)
.Include(s => s.Beatmaps).ThenInclude(b => b.Metadata)
.Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty)
.Include(s => s.Files).ThenInclude(f => f.FileInfo)
.Include(s => s.Metadata)
.ToList();
Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database);
// previous entries in EF are removed post migration.
if (!existingBeatmapSets.Any())
{
Logger.Log("No beatmaps found to migrate", LoggingTarget.Database);
return;
}
using (var realm = realmContextFactory.CreateContext())
using (var transaction = realm.BeginWrite())
{
// only migrate data if the realm database is empty.
// note that this cannot be written as: `realm.All<BeatmapInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (!realm.All<BeatmapSetInfo>().Any(s => !s.Protected))
Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database);
if (!hasTakenBackup)
{
foreach (var beatmapSet in existingBeatmapSets)
string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
efContextFactory.CreateBackup($"client.{migration}.db");
realmContextFactory.CreateBackup($"client.{migration}.realm");
hasTakenBackup = true;
}
// only migrate data if the realm database is empty.
// note that this cannot be written as: `realm.All<BeatmapSetInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (realm.All<BeatmapSetInfo>().Any(s => !s.Protected))
{
Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database);
}
else
{
using (var transaction = realm.BeginWrite())
{
var realmBeatmapSet = new BeatmapSetInfo
foreach (var beatmapSet in existingBeatmapSets)
{
OnlineID = beatmapSet.OnlineID ?? -1,
DateAdded = beatmapSet.DateAdded,
Status = beatmapSet.Status,
DeletePending = beatmapSet.DeletePending,
Hash = beatmapSet.Hash,
Protected = beatmapSet.Protected,
};
migrateFiles(beatmapSet, realm, realmBeatmapSet);
foreach (var beatmap in beatmapSet.Beatmaps)
{
var realmBeatmap = new BeatmapInfo
var realmBeatmapSet = new BeatmapSetInfo
{
DifficultyName = beatmap.DifficultyName,
Status = beatmap.Status,
OnlineID = beatmap.OnlineID ?? -1,
Length = beatmap.Length,
BPM = beatmap.BPM,
Hash = beatmap.Hash,
StarRating = beatmap.StarRating,
MD5Hash = beatmap.MD5Hash,
Hidden = beatmap.Hidden,
AudioLeadIn = beatmap.AudioLeadIn,
StackLeniency = beatmap.StackLeniency,
SpecialStyle = beatmap.SpecialStyle,
LetterboxInBreaks = beatmap.LetterboxInBreaks,
WidescreenStoryboard = beatmap.WidescreenStoryboard,
EpilepsyWarning = beatmap.EpilepsyWarning,
SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate,
DistanceSpacing = beatmap.DistanceSpacing,
BeatDivisor = beatmap.BeatDivisor,
GridSize = beatmap.GridSize,
TimelineZoom = beatmap.TimelineZoom,
Countdown = beatmap.Countdown,
CountdownOffset = beatmap.CountdownOffset,
MaxCombo = beatmap.MaxCombo,
Bookmarks = beatmap.Bookmarks,
Ruleset = realm.Find<RulesetInfo>(beatmap.RulesetInfo.ShortName),
Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty),
Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata),
BeatmapSet = realmBeatmapSet,
OnlineID = beatmapSet.OnlineID ?? -1,
DateAdded = beatmapSet.DateAdded,
Status = beatmapSet.Status,
DeletePending = beatmapSet.DeletePending,
Hash = beatmapSet.Hash,
Protected = beatmapSet.Protected,
};
realmBeatmapSet.Beatmaps.Add(realmBeatmap);
migrateFiles(beatmapSet, realm, realmBeatmapSet);
foreach (var beatmap in beatmapSet.Beatmaps)
{
var realmBeatmap = new BeatmapInfo
{
DifficultyName = beatmap.DifficultyName,
Status = beatmap.Status,
OnlineID = beatmap.OnlineID ?? -1,
Length = beatmap.Length,
BPM = beatmap.BPM,
Hash = beatmap.Hash,
StarRating = beatmap.StarRating,
MD5Hash = beatmap.MD5Hash,
Hidden = beatmap.Hidden,
AudioLeadIn = beatmap.AudioLeadIn,
StackLeniency = beatmap.StackLeniency,
SpecialStyle = beatmap.SpecialStyle,
LetterboxInBreaks = beatmap.LetterboxInBreaks,
WidescreenStoryboard = beatmap.WidescreenStoryboard,
EpilepsyWarning = beatmap.EpilepsyWarning,
SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate,
DistanceSpacing = beatmap.DistanceSpacing,
BeatDivisor = beatmap.BeatDivisor,
GridSize = beatmap.GridSize,
TimelineZoom = beatmap.TimelineZoom,
Countdown = beatmap.Countdown,
CountdownOffset = beatmap.CountdownOffset,
MaxCombo = beatmap.MaxCombo,
Bookmarks = beatmap.Bookmarks,
Ruleset = realm.Find<RulesetInfo>(beatmap.RulesetInfo.ShortName),
Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty),
Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata),
BeatmapSet = realmBeatmapSet,
};
realmBeatmapSet.Beatmaps.Add(realmBeatmap);
}
realm.Add(realmBeatmapSet);
}
realm.Add(realmBeatmapSet);
transaction.Commit();
Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database);
}
}
db.Context.RemoveRange(existingBeatmapSets);
ef.Context.RemoveRange(existingBeatmapSets);
// Intentionally don't clean up the files, so they don't get purged by EF.
transaction.Commit();
}
}
@ -151,62 +184,85 @@ namespace osu.Game.Database
private void migrateScores(DatabaseWriteUsage db)
{
// can be removed 20220730.
var existingScores = db.Context.ScoreInfo
.Include(s => s.Ruleset)
.Include(s => s.BeatmapInfo)
.Include(s => s.Files)
.ThenInclude(f => f.FileInfo)
.ToList();
List<EFScoreInfo> existingScores = db.Context.ScoreInfo
.Include(s => s.Ruleset)
.Include(s => s.BeatmapInfo)
.Include(s => s.Files)
.ThenInclude(f => f.FileInfo)
.ToList();
Logger.Log("Beginning scores migration to realm", LoggingTarget.Database);
// previous entries in EF are removed post migration.
if (!existingScores.Any())
{
Logger.Log("No scores found to migrate", LoggingTarget.Database);
return;
}
using (var realm = realmContextFactory.CreateContext())
using (var transaction = realm.BeginWrite())
{
// only migrate data if the realm database is empty.
// note that this cannot be written as: `realm.All<ScoreInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (!realm.All<ScoreInfo>().Any())
Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database);
if (!hasTakenBackup)
{
foreach (var score in existingScores)
string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
efContextFactory.CreateBackup($"client.{migration}.db");
realmContextFactory.CreateBackup($"client.{migration}.realm");
hasTakenBackup = true;
}
// only migrate data if the realm database is empty.
if (realm.All<ScoreInfo>().Any())
{
Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database);
}
else
{
using (var transaction = realm.BeginWrite())
{
var realmScore = new ScoreInfo
foreach (var score in existingScores)
{
Hash = score.Hash,
DeletePending = score.DeletePending,
OnlineID = score.OnlineID ?? -1,
ModsJson = score.ModsJson,
StatisticsJson = score.StatisticsJson,
User = score.User,
TotalScore = score.TotalScore,
MaxCombo = score.MaxCombo,
Accuracy = score.Accuracy,
HasReplay = ((IScoreInfo)score).HasReplay,
Date = score.Date,
PP = score.PP,
BeatmapInfo = realm.All<BeatmapInfo>().First(b => b.Hash == score.BeatmapInfo.Hash),
Ruleset = realm.Find<RulesetInfo>(score.Ruleset.ShortName),
Rank = score.Rank,
HitEvents = score.HitEvents,
Passed = score.Passed,
Combo = score.Combo,
Position = score.Position,
Statistics = score.Statistics,
Mods = score.Mods,
APIMods = score.APIMods,
};
var realmScore = new ScoreInfo
{
Hash = score.Hash,
DeletePending = score.DeletePending,
OnlineID = score.OnlineID ?? -1,
ModsJson = score.ModsJson,
StatisticsJson = score.StatisticsJson,
User = score.User,
TotalScore = score.TotalScore,
MaxCombo = score.MaxCombo,
Accuracy = score.Accuracy,
HasReplay = ((IScoreInfo)score).HasReplay,
Date = score.Date,
PP = score.PP,
BeatmapInfo = realm.All<BeatmapInfo>().First(b => b.Hash == score.BeatmapInfo.Hash),
Ruleset = realm.Find<RulesetInfo>(score.Ruleset.ShortName),
Rank = score.Rank,
HitEvents = score.HitEvents,
Passed = score.Passed,
Combo = score.Combo,
Position = score.Position,
Statistics = score.Statistics,
Mods = score.Mods,
APIMods = score.APIMods,
};
migrateFiles(score, realm, realmScore);
migrateFiles(score, realm, realmScore);
realm.Add(realmScore);
realm.Add(realmScore);
}
transaction.Commit();
Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database);
}
}
db.Context.RemoveRange(existingScores);
// Intentionally don't clean up the files, so they don't get purged by EF.
transaction.Commit();
}
}
@ -243,6 +299,8 @@ namespace osu.Game.Database
// note that this cannot be written as: `realm.All<SkinInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (!realm.All<SkinInfo>().Any(s => !s.Protected))
{
Logger.Log($"Migrating {existingSkins.Count} skins", LoggingTarget.Database);
foreach (var skin in existingSkins)
{
var realmSkin = new SkinInfo
@ -286,18 +344,22 @@ namespace osu.Game.Database
private void migrateSettings(DatabaseWriteUsage db)
{
// migrate ruleset settings. can be removed 20220315.
var existingSettings = db.Context.DatabasedSetting;
var existingSettings = db.Context.DatabasedSetting.ToList();
// previous entries in EF are removed post migration.
if (!existingSettings.Any())
return;
Logger.Log("Beginning settings migration to realm", LoggingTarget.Database);
using (var realm = realmContextFactory.CreateContext())
using (var transaction = realm.BeginWrite())
{
// only migrate data if the realm database is empty.
if (!realm.All<RealmRulesetSetting>().Any())
{
Logger.Log($"Migrating {existingSettings.Count} settings", LoggingTarget.Database);
foreach (var dkb in existingSettings)
{
if (dkb.RulesetID == null)

21
osu.Game/Database/RealmContextFactory.cs
View File

@ -361,6 +361,17 @@ namespace osu.Game.Database
private string? getRulesetShortNameFromLegacyID(long rulesetId) =>
efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName;
public void CreateBackup(string backupFilename)
{
using (BlockAllOperations())
{
Logger.Log($"Creating full realm database backup at {backupFilename}", LoggingTarget.Database);
using (var source = storage.GetStream(Filename))
using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew))
source.CopyTo(destination);
}
}
/// <summary>
/// Flush any active contexts and block any further writes.
/// </summary>
@ -374,17 +385,17 @@ namespace osu.Game.Database
if (isDisposed)
throw new ObjectDisposedException(nameof(RealmContextFactory));
if (!ThreadSafety.IsUpdateThread)
throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread.");
Logger.Log(@"Blocking realm operations.", LoggingTarget.Database);
try
{
contextCreationLock.Wait();
lock (contextLock)
{
if (!ThreadSafety.IsUpdateThread && context != null)
throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread.");
Logger.Log(@"Blocking realm operations.", LoggingTarget.Database);
context?.Dispose();
context = null;
}

1
osu.Game/OsuGameBase.cs
View File

@ -193,6 +193,7 @@ namespace osu.Game
dependencies.Cache(RulesetStore = new RulesetStore(realmFactory, Storage));
dependencies.CacheAs<IRulesetStore>(RulesetStore);
// A non-null context factory means there's still content to migrate.
if (efContextFactory != null)
new EFToRealmMigrator(efContextFactory, realmFactory, LocalConfig).Run();