diff --git a/osu.Game.Tests/Collections/IO/ImportCollectionsTest.cs b/osu.Game.Tests/Collections/IO/ImportCollectionsTest.cs index 9a8f29647d..685586ff02 100644 --- a/osu.Game.Tests/Collections/IO/ImportCollectionsTest.cs +++ b/osu.Game.Tests/Collections/IO/ImportCollectionsTest.cs @@ -11,6 +11,7 @@ using NUnit.Framework; using osu.Framework.Extensions; using osu.Framework.Platform; using osu.Framework.Testing; +using osu.Game.Database; using osu.Game.Tests.Resources; namespace osu.Game.Tests.Collections.IO @@ -187,7 +188,7 @@ namespace osu.Game.Tests.Collections.IO { // intentionally spin this up on a separate task to avoid disposal deadlocks. // see https://github.com/EventStore/EventStore/issues/1179 - await Task.Factory.StartNew(() => osu.CollectionManager.Import(stream).WaitSafely(), TaskCreationOptions.LongRunning); + await Task.Factory.StartNew(() => new LegacyCollectionImporter(osu.CollectionManager).Import(stream).WaitSafely(), TaskCreationOptions.LongRunning); } } } diff --git a/osu.Game.Tests/ImportTest.cs b/osu.Game.Tests/ImportTest.cs index 32b6dc649c..1f18f92158 100644 --- a/osu.Game.Tests/ImportTest.cs +++ b/osu.Game.Tests/ImportTest.cs @@ -63,7 +63,7 @@ namespace osu.Game.Tests if (withBeatmap) BeatmapManager.Import(TestResources.GetTestBeatmapForImport()).WaitSafely(); - AddInternal(CollectionManager = new CollectionManager(Storage)); + AddInternal(CollectionManager = new CollectionManager()); } } } diff --git a/osu.Game.Tests/Visual/Collections/TestSceneManageCollectionsDialog.cs b/osu.Game.Tests/Visual/Collections/TestSceneManageCollectionsDialog.cs index 3f30fa367c..789139f483 100644 --- a/osu.Game.Tests/Visual/Collections/TestSceneManageCollectionsDialog.cs +++ b/osu.Game.Tests/Visual/Collections/TestSceneManageCollectionsDialog.cs @@ -46,7 +46,7 @@ namespace osu.Game.Tests.Visual.Collections base.Content.AddRange(new Drawable[] { - manager = new CollectionManager(LocalStorage), + manager = new CollectionManager(), Content, dialogOverlay = new DialogOverlay(), }); diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneFilterControl.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneFilterControl.cs index 6807180640..01bf312ddd 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestSceneFilterControl.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestSceneFilterControl.cs @@ -46,7 +46,7 @@ namespace osu.Game.Tests.Visual.SongSelect base.Content.AddRange(new Drawable[] { - collectionManager = new CollectionManager(LocalStorage), + collectionManager = new CollectionManager(), Content }); diff --git a/osu.Game/Beatmaps/RealmBeatmapCollection.cs b/osu.Game/Beatmaps/RealmBeatmapCollection.cs new file mode 100644 index 0000000000..d3261fc39e --- /dev/null +++ b/osu.Game/Beatmaps/RealmBeatmapCollection.cs @@ -0,0 +1,40 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using JetBrains.Annotations; +using osu.Game.Database; +using Realms; + +namespace osu.Game.Beatmaps +{ + public class RealmBeatmapCollection : RealmObject, IHasGuidPrimaryKey + { + [PrimaryKey] + public Guid ID { get; } + + public string Name { get; set; } = string.Empty; + + public List BeatmapMD5Hashes { get; set; } = null!; + + /// + /// The date when this collection was last modified. + /// + public DateTimeOffset LastModified { get; set; } + + public RealmBeatmapCollection(string? name, List? beatmapMD5Hashes) + { + ID = Guid.NewGuid(); + Name = name ?? string.Empty; + BeatmapMD5Hashes = beatmapMD5Hashes ?? new List(); + + LastModified = DateTimeOffset.UtcNow; + } + + [UsedImplicitly] + private RealmBeatmapCollection() + { + } + } +} diff --git a/osu.Game/Collections/BeatmapCollection.cs b/osu.Game/Collections/BeatmapCollection.cs index 742d757bec..abfd0e6dd0 100644 --- a/osu.Game/Collections/BeatmapCollection.cs +++ b/osu.Game/Collections/BeatmapCollection.cs @@ -14,11 +14,6 @@ namespace osu.Game.Collections /// public class BeatmapCollection { - /// - /// Invoked whenever any change occurs on this . - /// - public event Action Changed; - /// /// The collection's name. /// @@ -33,17 +28,5 @@ namespace osu.Game.Collections /// The date when this collection was last modified. /// public DateTimeOffset LastModifyDate { get; private set; } = DateTimeOffset.UtcNow; - - public BeatmapCollection() - { - BeatmapHashes.CollectionChanged += (_, _) => onChange(); - Name.ValueChanged += _ => onChange(); - } - - private void onChange() - { - LastModifyDate = DateTimeOffset.Now; - Changed?.Invoke(); - } } } diff --git a/osu.Game/Collections/CollectionManager.cs b/osu.Game/Collections/CollectionManager.cs index 796b3c426c..0d4ee5c722 100644 --- a/osu.Game/Collections/CollectionManager.cs +++ b/osu.Game/Collections/CollectionManager.cs @@ -4,346 +4,59 @@ #nullable disable using System; -using System.Collections.Generic; -using System.Collections.Specialized; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using osu.Framework.Allocation; using osu.Framework.Bindables; using osu.Framework.Graphics; -using osu.Framework.Logging; -using osu.Framework.Platform; +using osu.Game.Beatmaps; using osu.Game.Database; -using osu.Game.IO; -using osu.Game.IO.Legacy; using osu.Game.Overlays.Notifications; +using Realms; namespace osu.Game.Collections { /// /// Handles user-defined collections of beatmaps. /// - /// - /// This is currently reading and writing from the osu-stable file format. This is a temporary arrangement until we refactor the - /// database backing the game. Going forward writing should be done in a similar way to other model stores. - /// public class CollectionManager : Component, IPostNotifications { - /// - /// Database version in stable-compatible YYYYMMDD format. - /// - private const int database_version = 30000000; - - private const string database_name = "collection.db"; - private const string database_backup_name = "collection.db.bak"; - public readonly BindableList Collections = new BindableList(); - private readonly Storage storage; - - public CollectionManager(Storage storage) - { - this.storage = storage; - } - - [Resolved(canBeNull: true)] - private DatabaseContextFactory efContextFactory { get; set; } = null!; + [Resolved] + private RealmAccess realm { get; set; } [BackgroundDependencyLoader] private void load() { - efContextFactory?.WaitForMigrationCompletion(); - - Collections.CollectionChanged += collectionsChanged; - - if (storage.Exists(database_backup_name)) - { - // If a backup file exists, it means the previous write operation didn't run to completion. - // Always prefer the backup file in such a case as it's the most recent copy that is guaranteed to not be malformed. - // - // The database is saved 100ms after any change, and again when the game is closed, so there shouldn't be a large diff between the two files in the worst case. - if (storage.Exists(database_name)) - storage.Delete(database_name); - File.Copy(storage.GetFullPath(database_backup_name), storage.GetFullPath(database_name)); - } - - if (storage.Exists(database_name)) - { - List beatmapCollections; - - using (var stream = storage.GetStream(database_name)) - beatmapCollections = readCollections(stream); - - // intentionally fire-and-forget async. - importCollections(beatmapCollections); - } } - private void collectionsChanged(object sender, NotifyCollectionChangedEventArgs e) => Schedule(() => + protected override void LoadComplete() { - switch (e.Action) + base.LoadComplete(); + + realm.RegisterForNotifications(r => r.All(), collectionsChanged); + } + + private void collectionsChanged(IRealmCollection sender, ChangeSet changes, Exception error) + { + // TODO: hook up with realm changes. + + if (changes == null) { - case NotifyCollectionChangedAction.Add: - foreach (var c in e.NewItems.Cast()) - c.Changed += backgroundSave; - break; - - case NotifyCollectionChangedAction.Remove: - foreach (var c in e.OldItems.Cast()) - c.Changed -= backgroundSave; - break; - - case NotifyCollectionChangedAction.Replace: - foreach (var c in e.OldItems.Cast()) - c.Changed -= backgroundSave; - - foreach (var c in e.NewItems.Cast()) - c.Changed += backgroundSave; - break; + foreach (var collection in sender) + Collections.Add(new BeatmapCollection + { + Name = { Value = collection.Name }, + BeatmapHashes = { Value = collection.BeatmapMD5Hashes }, + }); } - - backgroundSave(); - }); + } public Action PostNotification { protected get; set; } - public Task GetAvailableCount(StableStorage stableStorage) - { - if (!stableStorage.Exists(database_name)) - return Task.FromResult(0); - - return Task.Run(() => - { - using (var stream = stableStorage.GetStream(database_name)) - return readCollections(stream).Count; - }); - } - - /// - /// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future. - /// - public Task ImportFromStableAsync(StableStorage stableStorage) - { - if (!stableStorage.Exists(database_name)) - { - // This handles situations like when the user does not have a collections.db file - Logger.Log($"No {database_name} available in osu!stable installation", LoggingTarget.Information, LogLevel.Error); - return Task.CompletedTask; - } - - return Task.Run(async () => - { - using (var stream = stableStorage.GetStream(database_name)) - await Import(stream).ConfigureAwait(false); - }); - } - - public async Task Import(Stream stream) - { - var notification = new ProgressNotification - { - State = ProgressNotificationState.Active, - Text = "Collections import is initialising..." - }; - - PostNotification?.Invoke(notification); - - var collections = readCollections(stream, notification); - await importCollections(collections).ConfigureAwait(false); - - notification.CompletionText = $"Imported {collections.Count} collections"; - notification.State = ProgressNotificationState.Completed; - } - - private Task importCollections(List newCollections) - { - var tcs = new TaskCompletionSource(); - - Schedule(() => - { - try - { - foreach (var newCol in newCollections) - { - var existing = Collections.FirstOrDefault(c => c.Name.Value == newCol.Name.Value); - if (existing == null) - Collections.Add(existing = new BeatmapCollection { Name = { Value = newCol.Name.Value } }); - - foreach (string newBeatmap in newCol.BeatmapHashes) - { - if (!existing.BeatmapHashes.Contains(newBeatmap)) - existing.BeatmapHashes.Add(newBeatmap); - } - } - - tcs.SetResult(true); - } - catch (Exception e) - { - Logger.Error(e, "Failed to import collection."); - tcs.SetException(e); - } - }); - - return tcs.Task; - } - - private List readCollections(Stream stream, ProgressNotification notification = null) - { - if (notification != null) - { - notification.Text = "Reading collections..."; - notification.Progress = 0; - } - - var result = new List(); - - try - { - using (var sr = new SerializationReader(stream)) - { - sr.ReadInt32(); // Version - - int collectionCount = sr.ReadInt32(); - result.Capacity = collectionCount; - - for (int i = 0; i < collectionCount; i++) - { - if (notification?.CancellationToken.IsCancellationRequested == true) - return result; - - var collection = new BeatmapCollection { Name = { Value = sr.ReadString() } }; - int mapCount = sr.ReadInt32(); - - for (int j = 0; j < mapCount; j++) - { - if (notification?.CancellationToken.IsCancellationRequested == true) - return result; - - string checksum = sr.ReadString(); - - collection.BeatmapHashes.Add(checksum); - } - - if (notification != null) - { - notification.Text = $"Imported {i + 1} of {collectionCount} collections"; - notification.Progress = (float)(i + 1) / collectionCount; - } - - result.Add(collection); - } - } - } - catch (Exception e) - { - Logger.Error(e, "Failed to read collection database."); - } - - return result; - } - public void DeleteAll() { Collections.Clear(); PostNotification?.Invoke(new ProgressCompletionNotification { Text = "Deleted all collections!" }); } - - private readonly object saveLock = new object(); - private int lastSave; - private int saveFailures; - - /// - /// Perform a save with debounce. - /// - private void backgroundSave() - { - int current = Interlocked.Increment(ref lastSave); - Task.Delay(100).ContinueWith(_ => - { - if (current != lastSave) - return; - - if (!save()) - backgroundSave(); - }); - } - - private bool save() - { - lock (saveLock) - { - Interlocked.Increment(ref lastSave); - - // This is NOT thread-safe!! - try - { - string tempPath = Path.GetTempFileName(); - - using (var ms = new MemoryStream()) - { - using (var sw = new SerializationWriter(ms, true)) - { - sw.Write(database_version); - - var collectionsCopy = Collections.ToArray(); - sw.Write(collectionsCopy.Length); - - foreach (var c in collectionsCopy) - { - sw.Write(c.Name.Value); - - string[] beatmapsCopy = c.BeatmapHashes.ToArray(); - - sw.Write(beatmapsCopy.Length); - - foreach (string b in beatmapsCopy) - sw.Write(b); - } - } - - using (var fs = File.OpenWrite(tempPath)) - ms.WriteTo(fs); - - string databasePath = storage.GetFullPath(database_name); - string databaseBackupPath = storage.GetFullPath(database_backup_name); - - // Back up the existing database, clearing any existing backup. - if (File.Exists(databaseBackupPath)) - File.Delete(databaseBackupPath); - if (File.Exists(databasePath)) - File.Move(databasePath, databaseBackupPath); - - // Move the new database in-place of the existing one. - File.Move(tempPath, databasePath); - - // If everything succeeded up to this point, remove the backup file. - if (File.Exists(databaseBackupPath)) - File.Delete(databaseBackupPath); - } - - if (saveFailures < 10) - saveFailures = 0; - return true; - } - catch (Exception e) - { - // Since this code is not thread-safe, we may run into random exceptions (such as collection enumeration or out of range indexing). - // Failures are thus only alerted if they exceed a threshold (once) to indicate "actual" errors having occurred. - if (++saveFailures == 10) - Logger.Error(e, "Failed to save collection database!"); - } - - return false; - } - } - - protected override void Dispose(bool isDisposing) - { - base.Dispose(isDisposing); - save(); - } } } diff --git a/osu.Game/Database/LegacyCollectionImporter.cs b/osu.Game/Database/LegacyCollectionImporter.cs new file mode 100644 index 0000000000..8168419e80 --- /dev/null +++ b/osu.Game/Database/LegacyCollectionImporter.cs @@ -0,0 +1,167 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +#nullable disable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using osu.Framework.Logging; +using osu.Game.Collections; +using osu.Game.IO; +using osu.Game.IO.Legacy; +using osu.Game.Overlays.Notifications; + +namespace osu.Game.Database +{ + public class LegacyCollectionImporter + { + private readonly CollectionManager collections; + + public LegacyCollectionImporter(CollectionManager collections) + { + this.collections = collections; + } + + public Action PostNotification { protected get; set; } + + private const string database_name = "collection.db"; + + public Task GetAvailableCount(StableStorage stableStorage) + { + if (!stableStorage.Exists(database_name)) + return Task.FromResult(0); + + return Task.Run(() => + { + using (var stream = stableStorage.GetStream(database_name)) + return readCollections(stream).Count; + }); + } + + /// + /// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future. + /// + public Task ImportFromStableAsync(StableStorage stableStorage) + { + if (!stableStorage.Exists(database_name)) + { + // This handles situations like when the user does not have a collections.db file + Logger.Log($"No {database_name} available in osu!stable installation", LoggingTarget.Information, LogLevel.Error); + return Task.CompletedTask; + } + + return Task.Run(async () => + { + using (var stream = stableStorage.GetStream(database_name)) + await Import(stream).ConfigureAwait(false); + }); + } + + public async Task Import(Stream stream) + { + var notification = new ProgressNotification + { + State = ProgressNotificationState.Active, + Text = "Collections import is initialising..." + }; + + PostNotification?.Invoke(notification); + + var importedCollections = readCollections(stream, notification); + await importCollections(importedCollections).ConfigureAwait(false); + + notification.CompletionText = $"Imported {importedCollections.Count} collections"; + notification.State = ProgressNotificationState.Completed; + } + + private Task importCollections(List newCollections) + { + var tcs = new TaskCompletionSource(); + + // Schedule(() => + // { + try + { + foreach (var newCol in newCollections) + { + var existing = collections.Collections.FirstOrDefault(c => c.Name.Value == newCol.Name.Value); + if (existing == null) + collections.Collections.Add(existing = new BeatmapCollection { Name = { Value = newCol.Name.Value } }); + + foreach (string newBeatmap in newCol.BeatmapHashes) + { + if (!existing.BeatmapHashes.Contains(newBeatmap)) + existing.BeatmapHashes.Add(newBeatmap); + } + } + + tcs.SetResult(true); + } + catch (Exception e) + { + Logger.Error(e, "Failed to import collection."); + tcs.SetException(e); + } + // }); + + return tcs.Task; + } + + private List readCollections(Stream stream, ProgressNotification notification = null) + { + if (notification != null) + { + notification.Text = "Reading collections..."; + notification.Progress = 0; + } + + var result = new List(); + + try + { + using (var sr = new SerializationReader(stream)) + { + sr.ReadInt32(); // Version + + int collectionCount = sr.ReadInt32(); + result.Capacity = collectionCount; + + for (int i = 0; i < collectionCount; i++) + { + if (notification?.CancellationToken.IsCancellationRequested == true) + return result; + + var collection = new BeatmapCollection { Name = { Value = sr.ReadString() } }; + int mapCount = sr.ReadInt32(); + + for (int j = 0; j < mapCount; j++) + { + if (notification?.CancellationToken.IsCancellationRequested == true) + return result; + + string checksum = sr.ReadString(); + + collection.BeatmapHashes.Add(checksum); + } + + if (notification != null) + { + notification.Text = $"Imported {i + 1} of {collectionCount} collections"; + notification.Progress = (float)(i + 1) / collectionCount; + } + + result.Add(collection); + } + } + } + catch (Exception e) + { + Logger.Error(e, "Failed to read collection database."); + } + + return result; + } + } +} diff --git a/osu.Game/Database/LegacyImportManager.cs b/osu.Game/Database/LegacyImportManager.cs index f40e0d33c2..05bd5ceb54 100644 --- a/osu.Game/Database/LegacyImportManager.cs +++ b/osu.Game/Database/LegacyImportManager.cs @@ -72,7 +72,7 @@ namespace osu.Game.Database return await new LegacySkinImporter(skins).GetAvailableCount(stableStorage); case StableContent.Collections: - return await collections.GetAvailableCount(stableStorage); + return await new LegacyCollectionImporter(collections).GetAvailableCount(stableStorage); case StableContent.Scores: return await new LegacyScoreImporter(scores).GetAvailableCount(stableStorage); @@ -109,7 +109,7 @@ namespace osu.Game.Database importTasks.Add(new LegacySkinImporter(skins).ImportFromStableAsync(stableStorage)); if (content.HasFlagFast(StableContent.Collections)) - importTasks.Add(beatmapImportTask.ContinueWith(_ => collections.ImportFromStableAsync(stableStorage), TaskContinuationOptions.OnlyOnRanToCompletion)); + importTasks.Add(beatmapImportTask.ContinueWith(_ => new LegacyCollectionImporter(collections).ImportFromStableAsync(stableStorage), TaskContinuationOptions.OnlyOnRanToCompletion)); if (content.HasFlagFast(StableContent.Scores)) importTasks.Add(beatmapImportTask.ContinueWith(_ => new LegacyScoreImporter(scores).ImportFromStableAsync(stableStorage), TaskContinuationOptions.OnlyOnRanToCompletion)); diff --git a/osu.Game/OsuGame.cs b/osu.Game/OsuGame.cs index 1ee53e2848..8d8864a46a 100644 --- a/osu.Game/OsuGame.cs +++ b/osu.Game/OsuGame.cs @@ -858,7 +858,7 @@ namespace osu.Game d.Origin = Anchor.TopRight; }), rightFloatingOverlayContent.Add, true); - loadComponentSingleFile(new CollectionManager(Storage) + loadComponentSingleFile(new CollectionManager { PostNotification = n => Notifications.Post(n), }, Add, true);