h1or4dme8/osu-lazer
1
0
Fork 0
mirror of https://github.com/ppy/osu.git synced 2025-01-12 17:43:05 +08:00
Code Issues Releases Wiki Activity

Split out legacy import path from realm manager

Browse Source
This commit is contained in:
Dean Herbert 2022-07-27 15:04:09 +09:00
parent c30e8047ab
commit 6b73f7c7ec
10 changed files with 237 additions and 333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
osu.Game.Tests/Collections/IO/ImportCollectionsTest.cs
View File

@ -11,6 +11,7 @@ using NUnit.Framework;
using osu.Framework.Extensions;
using osu.Framework.Platform;
using osu.Framework.Testing;
using osu.Game.Database;
using osu.Game.Tests.Resources;
namespace osu.Game.Tests.Collections.IO
@ -187,7 +188,7 @@ namespace osu.Game.Tests.Collections.IO
{
// intentionally spin this up on a separate task to avoid disposal deadlocks.
// see https://github.com/EventStore/EventStore/issues/1179
await Task.Factory.StartNew(() => osu.CollectionManager.Import(stream).WaitSafely(), TaskCreationOptions.LongRunning);
await Task.Factory.StartNew(() => new LegacyCollectionImporter(osu.CollectionManager).Import(stream).WaitSafely(), TaskCreationOptions.LongRunning);
}
}
}

2
osu.Game.Tests/ImportTest.cs
View File

@ -63,7 +63,7 @@ namespace osu.Game.Tests
if (withBeatmap)
BeatmapManager.Import(TestResources.GetTestBeatmapForImport()).WaitSafely();
AddInternal(CollectionManager = new CollectionManager(Storage));
AddInternal(CollectionManager = new CollectionManager());
}
}
}

2
osu.Game.Tests/Visual/Collections/TestSceneManageCollectionsDialog.cs
View File

@ -46,7 +46,7 @@ namespace osu.Game.Tests.Visual.Collections
base.Content.AddRange(new Drawable[]
{
manager = new CollectionManager(LocalStorage),
manager = new CollectionManager(),
Content,
dialogOverlay = new DialogOverlay(),
});

2
osu.Game.Tests/Visual/SongSelect/TestSceneFilterControl.cs
View File

@ -46,7 +46,7 @@ namespace osu.Game.Tests.Visual.SongSelect
base.Content.AddRange(new Drawable[]
{
collectionManager = new CollectionManager(LocalStorage),
collectionManager = new CollectionManager(),
Content
});

40
osu.Game/Beatmaps/RealmBeatmapCollection.cs Normal file
View File

@ -0,0 +1,40 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Collections.Generic;
using JetBrains.Annotations;
using osu.Game.Database;
using Realms;
namespace osu.Game.Beatmaps
{
public class RealmBeatmapCollection : RealmObject, IHasGuidPrimaryKey
{
[PrimaryKey]
public Guid ID { get; }
public string Name { get; set; } = string.Empty;
public List<string> BeatmapMD5Hashes { get; set; } = null!;
/// <summary>
/// The date when this collection was last modified.
/// </summary>
public DateTimeOffset LastModified { get; set; }
public RealmBeatmapCollection(string? name, List<string>? beatmapMD5Hashes)
{
ID = Guid.NewGuid();
Name = name ?? string.Empty;
BeatmapMD5Hashes = beatmapMD5Hashes ?? new List<string>();
LastModified = DateTimeOffset.UtcNow;
}
[UsedImplicitly]
private RealmBeatmapCollection()
{
}
}
}

17
osu.Game/Collections/BeatmapCollection.cs
View File

@ -14,11 +14,6 @@ namespace osu.Game.Collections
/// </summary>
public class BeatmapCollection
{
/// <summary>
/// Invoked whenever any change occurs on this <see cref="BeatmapCollection"/>.
/// </summary>
public event Action Changed;
/// <summary>
/// The collection's name.
/// </summary>
@ -33,17 +28,5 @@ namespace osu.Game.Collections
/// The date when this collection was last modified.
/// </summary>
public DateTimeOffset LastModifyDate { get; private set; } = DateTimeOffset.UtcNow;
public BeatmapCollection()
{
BeatmapHashes.CollectionChanged += (_, _) => onChange();
Name.ValueChanged += _ => onChange();
}
private void onChange()
{
LastModifyDate = DateTimeOffset.Now;
Changed?.Invoke();
}
}
}

331
osu.Game/Collections/CollectionManager.cs
View File

@ -4,346 +4,59 @@
#nullable disable
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using osu.Framework.Allocation;
using osu.Framework.Bindables;
using osu.Framework.Graphics;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Game.Beatmaps;
using osu.Game.Database;
using osu.Game.IO;
using osu.Game.IO.Legacy;
using osu.Game.Overlays.Notifications;
using Realms;
namespace osu.Game.Collections
{
/// <summary>
/// Handles user-defined collections of beatmaps.
/// </summary>
/// <remarks>
/// This is currently reading and writing from the osu-stable file format. This is a temporary arrangement until we refactor the
/// database backing the game. Going forward writing should be done in a similar way to other model stores.
/// </remarks>
public class CollectionManager : Component, IPostNotifications
{
/// <summary>
/// Database version in stable-compatible YYYYMMDD format.
/// </summary>
private const int database_version = 30000000;
private const string database_name = "collection.db";
private const string database_backup_name = "collection.db.bak";
public readonly BindableList<BeatmapCollection> Collections = new BindableList<BeatmapCollection>();
private readonly Storage storage;
public CollectionManager(Storage storage)
{
this.storage = storage;
}
[Resolved(canBeNull: true)]
private DatabaseContextFactory efContextFactory { get; set; } = null!;
[Resolved]
private RealmAccess realm { get; set; }
[BackgroundDependencyLoader]
private void load()
{
efContextFactory?.WaitForMigrationCompletion();
Collections.CollectionChanged += collectionsChanged;
if (storage.Exists(database_backup_name))
{
// If a backup file exists, it means the previous write operation didn't run to completion.
// Always prefer the backup file in such a case as it's the most recent copy that is guaranteed to not be malformed.
//
// The database is saved 100ms after any change, and again when the game is closed, so there shouldn't be a large diff between the two files in the worst case.
if (storage.Exists(database_name))
storage.Delete(database_name);
File.Copy(storage.GetFullPath(database_backup_name), storage.GetFullPath(database_name));
}
if (storage.Exists(database_name))
{
List<BeatmapCollection> beatmapCollections;
using (var stream = storage.GetStream(database_name))
beatmapCollections = readCollections(stream);
// intentionally fire-and-forget async.
importCollections(beatmapCollections);
}
}
private void collectionsChanged(object sender, NotifyCollectionChangedEventArgs e) => Schedule(() =>
protected override void LoadComplete()
{
switch (e.Action)
base.LoadComplete();
realm.RegisterForNotifications(r => r.All<RealmBeatmapCollection>(), collectionsChanged);
}
private void collectionsChanged(IRealmCollection<RealmBeatmapCollection> sender, ChangeSet changes, Exception error)
{
// TODO: hook up with realm changes.
if (changes == null)
{
case NotifyCollectionChangedAction.Add:
foreach (var c in e.NewItems.Cast<BeatmapCollection>())
c.Changed += backgroundSave;
break;
case NotifyCollectionChangedAction.Remove:
foreach (var c in e.OldItems.Cast<BeatmapCollection>())
c.Changed -= backgroundSave;
break;
case NotifyCollectionChangedAction.Replace:
foreach (var c in e.OldItems.Cast<BeatmapCollection>())
c.Changed -= backgroundSave;
foreach (var c in e.NewItems.Cast<BeatmapCollection>())
c.Changed += backgroundSave;
break;
foreach (var collection in sender)
Collections.Add(new BeatmapCollection
{
Name = { Value = collection.Name },
BeatmapHashes = { Value = collection.BeatmapMD5Hashes },
});
}
backgroundSave();
});
}
public Action<Notification> PostNotification { protected get; set; }
public Task<int> GetAvailableCount(StableStorage stableStorage)
{
if (!stableStorage.Exists(database_name))
return Task.FromResult(0);
return Task.Run(() =>
{
using (var stream = stableStorage.GetStream(database_name))
return readCollections(stream).Count;
});
}
/// <summary>
/// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future.
/// </summary>
public Task ImportFromStableAsync(StableStorage stableStorage)
{
if (!stableStorage.Exists(database_name))
{
// This handles situations like when the user does not have a collections.db file
Logger.Log($"No {database_name} available in osu!stable installation", LoggingTarget.Information, LogLevel.Error);
return Task.CompletedTask;
}
return Task.Run(async () =>
{
using (var stream = stableStorage.GetStream(database_name))
await Import(stream).ConfigureAwait(false);
});
}
public async Task Import(Stream stream)
{
var notification = new ProgressNotification
{
State = ProgressNotificationState.Active,
Text = "Collections import is initialising..."
};
PostNotification?.Invoke(notification);
var collections = readCollections(stream, notification);
await importCollections(collections).ConfigureAwait(false);
notification.CompletionText = $"Imported {collections.Count} collections";
notification.State = ProgressNotificationState.Completed;
}
private Task importCollections(List<BeatmapCollection> newCollections)
{
var tcs = new TaskCompletionSource<bool>();
Schedule(() =>
{
try
{
foreach (var newCol in newCollections)
{
var existing = Collections.FirstOrDefault(c => c.Name.Value == newCol.Name.Value);
if (existing == null)
Collections.Add(existing = new BeatmapCollection { Name = { Value = newCol.Name.Value } });
foreach (string newBeatmap in newCol.BeatmapHashes)
{
if (!existing.BeatmapHashes.Contains(newBeatmap))
existing.BeatmapHashes.Add(newBeatmap);
}
}
tcs.SetResult(true);
}
catch (Exception e)
{
Logger.Error(e, "Failed to import collection.");
tcs.SetException(e);
}
});
return tcs.Task;
}
private List<BeatmapCollection> readCollections(Stream stream, ProgressNotification notification = null)
{
if (notification != null)
{
notification.Text = "Reading collections...";
notification.Progress = 0;
}
var result = new List<BeatmapCollection>();
try
{
using (var sr = new SerializationReader(stream))
{
sr.ReadInt32(); // Version
int collectionCount = sr.ReadInt32();
result.Capacity = collectionCount;
for (int i = 0; i < collectionCount; i++)
{
if (notification?.CancellationToken.IsCancellationRequested == true)
return result;
var collection = new BeatmapCollection { Name = { Value = sr.ReadString() } };
int mapCount = sr.ReadInt32();
for (int j = 0; j < mapCount; j++)
{
if (notification?.CancellationToken.IsCancellationRequested == true)
return result;
string checksum = sr.ReadString();
collection.BeatmapHashes.Add(checksum);
}
if (notification != null)
{
notification.Text = $"Imported {i + 1} of {collectionCount} collections";
notification.Progress = (float)(i + 1) / collectionCount;
}
result.Add(collection);
}
}
}
catch (Exception e)
{
Logger.Error(e, "Failed to read collection database.");
}
return result;
}
public void DeleteAll()
{
Collections.Clear();
PostNotification?.Invoke(new ProgressCompletionNotification { Text = "Deleted all collections!" });
}
private readonly object saveLock = new object();
private int lastSave;
private int saveFailures;
/// <summary>
/// Perform a save with debounce.
/// </summary>
private void backgroundSave()
{
int current = Interlocked.Increment(ref lastSave);
Task.Delay(100).ContinueWith(_ =>
{
if (current != lastSave)
return;
if (!save())
backgroundSave();
});
}
private bool save()
{
lock (saveLock)
{
Interlocked.Increment(ref lastSave);
// This is NOT thread-safe!!
try
{
string tempPath = Path.GetTempFileName();
using (var ms = new MemoryStream())
{
using (var sw = new SerializationWriter(ms, true))
{
sw.Write(database_version);
var collectionsCopy = Collections.ToArray();
sw.Write(collectionsCopy.Length);
foreach (var c in collectionsCopy)
{
sw.Write(c.Name.Value);
string[] beatmapsCopy = c.BeatmapHashes.ToArray();
sw.Write(beatmapsCopy.Length);
foreach (string b in beatmapsCopy)
sw.Write(b);
}
}
using (var fs = File.OpenWrite(tempPath))
ms.WriteTo(fs);
string databasePath = storage.GetFullPath(database_name);
string databaseBackupPath = storage.GetFullPath(database_backup_name);
// Back up the existing database, clearing any existing backup.
if (File.Exists(databaseBackupPath))
File.Delete(databaseBackupPath);
if (File.Exists(databasePath))
File.Move(databasePath, databaseBackupPath);
// Move the new database in-place of the existing one.
File.Move(tempPath, databasePath);
// If everything succeeded up to this point, remove the backup file.
if (File.Exists(databaseBackupPath))
File.Delete(databaseBackupPath);
}
if (saveFailures < 10)
saveFailures = 0;
return true;
}
catch (Exception e)
{
// Since this code is not thread-safe, we may run into random exceptions (such as collection enumeration or out of range indexing).
// Failures are thus only alerted if they exceed a threshold (once) to indicate "actual" errors having occurred.
if (++saveFailures == 10)
Logger.Error(e, "Failed to save collection database!");
}
return false;
}
}
protected override void Dispose(bool isDisposing)
{
base.Dispose(isDisposing);
save();
}
}
}

167
osu.Game/Database/LegacyCollectionImporter.cs Normal file
View File

@ -0,0 +1,167 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
#nullable disable
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using osu.Framework.Logging;
using osu.Game.Collections;
using osu.Game.IO;
using osu.Game.IO.Legacy;
using osu.Game.Overlays.Notifications;
namespace osu.Game.Database
{
public class LegacyCollectionImporter
{
private readonly CollectionManager collections;
public LegacyCollectionImporter(CollectionManager collections)
{
this.collections = collections;
}
public Action<Notification> PostNotification { protected get; set; }
private const string database_name = "collection.db";
public Task<int> GetAvailableCount(StableStorage stableStorage)
{
if (!stableStorage.Exists(database_name))
return Task.FromResult(0);
return Task.Run(() =>
{
using (var stream = stableStorage.GetStream(database_name))
return readCollections(stream).Count;
});
}
/// <summary>
/// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future.
/// </summary>
public Task ImportFromStableAsync(StableStorage stableStorage)
{
if (!stableStorage.Exists(database_name))
{
// This handles situations like when the user does not have a collections.db file
Logger.Log($"No {database_name} available in osu!stable installation", LoggingTarget.Information, LogLevel.Error);
return Task.CompletedTask;
}
return Task.Run(async () =>
{
using (var stream = stableStorage.GetStream(database_name))
await Import(stream).ConfigureAwait(false);
});
}
public async Task Import(Stream stream)
{
var notification = new ProgressNotification
{
State = ProgressNotificationState.Active,
Text = "Collections import is initialising..."
};
PostNotification?.Invoke(notification);
var importedCollections = readCollections(stream, notification);
await importCollections(importedCollections).ConfigureAwait(false);
notification.CompletionText = $"Imported {importedCollections.Count} collections";
notification.State = ProgressNotificationState.Completed;
}
private Task importCollections(List<BeatmapCollection> newCollections)
{
var tcs = new TaskCompletionSource<bool>();
// Schedule(() =>
// {
try
{
foreach (var newCol in newCollections)
{
var existing = collections.Collections.FirstOrDefault(c => c.Name.Value == newCol.Name.Value);
if (existing == null)
collections.Collections.Add(existing = new BeatmapCollection { Name = { Value = newCol.Name.Value } });
foreach (string newBeatmap in newCol.BeatmapHashes)
{
if (!existing.BeatmapHashes.Contains(newBeatmap))
existing.BeatmapHashes.Add(newBeatmap);
}
}
tcs.SetResult(true);
}
catch (Exception e)
{
Logger.Error(e, "Failed to import collection.");
tcs.SetException(e);
}
// });
return tcs.Task;
}
private List<BeatmapCollection> readCollections(Stream stream, ProgressNotification notification = null)
{
if (notification != null)
{
notification.Text = "Reading collections...";
notification.Progress = 0;
}
var result = new List<BeatmapCollection>();
try
{
using (var sr = new SerializationReader(stream))
{
sr.ReadInt32(); // Version
int collectionCount = sr.ReadInt32();
result.Capacity = collectionCount;
for (int i = 0; i < collectionCount; i++)
{
if (notification?.CancellationToken.IsCancellationRequested == true)
return result;
var collection = new BeatmapCollection { Name = { Value = sr.ReadString() } };
int mapCount = sr.ReadInt32();
for (int j = 0; j < mapCount; j++)
{
if (notification?.CancellationToken.IsCancellationRequested == true)
return result;
string checksum = sr.ReadString();
collection.BeatmapHashes.Add(checksum);
}
if (notification != null)
{
notification.Text = $"Imported {i + 1} of {collectionCount} collections";
notification.Progress = (float)(i + 1) / collectionCount;
}
result.Add(collection);
}
}
}
catch (Exception e)
{
Logger.Error(e, "Failed to read collection database.");
}
return result;
}
}
}

4
osu.Game/Database/LegacyImportManager.cs
View File

@ -72,7 +72,7 @@ namespace osu.Game.Database
return await new LegacySkinImporter(skins).GetAvailableCount(stableStorage);
case StableContent.Collections:
return await collections.GetAvailableCount(stableStorage);
return await new LegacyCollectionImporter(collections).GetAvailableCount(stableStorage);
case StableContent.Scores:
return await new LegacyScoreImporter(scores).GetAvailableCount(stableStorage);
@ -109,7 +109,7 @@ namespace osu.Game.Database
importTasks.Add(new LegacySkinImporter(skins).ImportFromStableAsync(stableStorage));
if (content.HasFlagFast(StableContent.Collections))
importTasks.Add(beatmapImportTask.ContinueWith(_ => collections.ImportFromStableAsync(stableStorage), TaskContinuationOptions.OnlyOnRanToCompletion));
importTasks.Add(beatmapImportTask.ContinueWith(_ => new LegacyCollectionImporter(collections).ImportFromStableAsync(stableStorage), TaskContinuationOptions.OnlyOnRanToCompletion));
if (content.HasFlagFast(StableContent.Scores))
importTasks.Add(beatmapImportTask.ContinueWith(_ => new LegacyScoreImporter(scores).ImportFromStableAsync(stableStorage), TaskContinuationOptions.OnlyOnRanToCompletion));

2
osu.Game/OsuGame.cs
View File

@ -858,7 +858,7 @@ namespace osu.Game
d.Origin = Anchor.TopRight;
}), rightFloatingOverlayContent.Add, true);
loadComponentSingleFile(new CollectionManager(Storage)
loadComponentSingleFile(new CollectionManager
{
PostNotification = n => Notifications.Post(n),
}, Add, true);