From 7affef75b05ac886b74f7d5580ba8d2ab4714faa Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 14 Jun 2022 20:25:38 +0900 Subject: [PATCH 1/7] Remove unused `Import` method in `SkinManager` --- osu.Game/Skinning/SkinManager.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/osu.Game/Skinning/SkinManager.cs b/osu.Game/Skinning/SkinManager.cs index 7ec922987b..6002bafa8d 100644 --- a/osu.Game/Skinning/SkinManager.cs +++ b/osu.Game/Skinning/SkinManager.cs @@ -281,9 +281,6 @@ namespace osu.Game.Skinning public Task> Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default) => skinModelManager.Import(archive, lowPriority, cancellationToken); - public Live Import(SkinInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) => - skinModelManager.Import(item, archive, cancellationToken); - #endregion #region Implementation of IModelManager From 2f8290831a3a370ae2d677b4227a60e9e5589f1c Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 14 Jun 2022 20:34:23 +0900 Subject: [PATCH 2/7] Skip quick import clause when importing a single item Closes https://github.com/ppy/osu/issues/18600. --- ...eneOnlinePlayBeatmapAvailabilityTracker.cs | 4 +- osu.Game/Beatmaps/BeatmapManager.cs | 2 +- osu.Game/Scoring/ScoreManager.cs | 2 +- osu.Game/Stores/RealmArchiveModelImporter.cs | 156 +++++++++--------- 4 files changed, 81 insertions(+), 83 deletions(-) diff --git a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs index 278acce3e5..5cae576db1 100644 --- a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs +++ b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs @@ -225,10 +225,10 @@ namespace osu.Game.Tests.Online this.testBeatmapManager = testBeatmapManager; } - public override Live Import(BeatmapSetInfo item, ArchiveReader archive = null, CancellationToken cancellationToken = default) + public override Live Import(BeatmapSetInfo item, ArchiveReader archive = null, bool quickSkipIfExisting = false, CancellationToken cancellationToken = default) { testBeatmapManager.AllowImport.Task.WaitSafely(); - return (testBeatmapManager.CurrentImport = base.Import(item, archive, cancellationToken)); + return (testBeatmapManager.CurrentImport = base.Import(item, archive, quickSkipIfExisting, cancellationToken)); } } } diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 51f4d23bb4..ab614106cc 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -357,7 +357,7 @@ namespace osu.Game.Beatmaps public Task?> Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default) => beatmapModelManager.Import(archive, lowPriority, cancellationToken); - public Live? Import(BeatmapSetInfo item, ArchiveReader? archive = null, CancellationToken cancellationToken = default) => beatmapModelManager.Import(item, archive, cancellationToken); + public Live? Import(BeatmapSetInfo item, ArchiveReader? archive = null, CancellationToken cancellationToken = default) => beatmapModelManager.Import(item, archive, false, cancellationToken); public IEnumerable HandledExtensions => beatmapModelManager.HandledExtensions; diff --git a/osu.Game/Scoring/ScoreManager.cs b/osu.Game/Scoring/ScoreManager.cs index d42bf1b454..e086135296 100644 --- a/osu.Game/Scoring/ScoreManager.cs +++ b/osu.Game/Scoring/ScoreManager.cs @@ -280,7 +280,7 @@ namespace osu.Game.Scoring public Task>> Import(ProgressNotification notification, params ImportTask[] tasks) => scoreModelManager.Import(notification, tasks); - public Live Import(ScoreInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) => scoreModelManager.Import(item, archive, cancellationToken); + public Live Import(ScoreInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) => scoreModelManager.Import(item, archive, lowPriority, cancellationToken); public bool IsAvailableLocally(ScoreInfo model) => scoreModelManager.IsAvailableLocally(model); diff --git a/osu.Game/Stores/RealmArchiveModelImporter.cs b/osu.Game/Stores/RealmArchiveModelImporter.cs index 82b77bfb05..ac8a38fe37 100644 --- a/osu.Game/Stores/RealmArchiveModelImporter.cs +++ b/osu.Game/Stores/RealmArchiveModelImporter.cs @@ -235,7 +235,7 @@ namespace osu.Game.Stores return null; } - var scheduledImport = Task.Factory.StartNew(() => Import(model, archive, cancellationToken), + var scheduledImport = Task.Factory.StartNew(() => Import(model, archive, lowPriority, cancellationToken), cancellationToken, TaskCreationOptions.HideScheduler, lowPriority ? import_scheduler_low_priority : import_scheduler); @@ -248,106 +248,104 @@ namespace osu.Game.Stores /// /// The model to be imported. /// An optional archive to use for model population. + /// If true, imports will be skipped before they begin, given an existing model matches on hash and filenames. Should generally only be used for large batch imports, as it may defy user expectations when updating an existing model. /// An optional cancellation token. - public virtual Live? Import(TModel item, ArchiveReader? archive = null, CancellationToken cancellationToken = default) + public virtual Live? Import(TModel item, ArchiveReader? archive = null, bool quickSkipIfExisting = false, CancellationToken cancellationToken = default) => Realm.Run(realm => { - return Realm.Run(realm => + cancellationToken.ThrowIfCancellationRequested(); + + bool checkedExisting = false; + TModel? existing = null; + + if (quickSkipIfExisting && archive != null && !HasCustomHashFunction) { - cancellationToken.ThrowIfCancellationRequested(); + // this is a fast bail condition to improve large import performance. + item.Hash = computeHashFast(archive); - bool checkedExisting = false; - TModel? existing = null; + checkedExisting = true; + existing = CheckForExisting(item, realm); - if (archive != null && !HasCustomHashFunction) + if (existing != null) { - // this is a fast bail condition to improve large import performance. - item.Hash = computeHashFast(archive); + // bare minimum comparisons + // + // note that this should really be checking filesizes on disk (of existing files) for some degree of sanity. + // or alternatively doing a faster hash check. either of these require database changes and reprocessing of existing files. + if (CanSkipImport(existing, item) && + getFilenames(existing.Files).SequenceEqual(getShortenedFilenames(archive).Select(p => p.shortened).OrderBy(f => f)) && + checkAllFilesExist(existing)) + { + LogForModel(item, @$"Found existing (optimised) {HumanisedModelName} for {item} (ID {existing.ID}) – skipping import."); - checkedExisting = true; - existing = CheckForExisting(item, realm); + using (var transaction = realm.BeginWrite()) + { + UndeleteForReuse(existing); + transaction.Commit(); + } + + return existing.ToLive(Realm); + } + + LogForModel(item, @"Found existing (optimised) but failed pre-check."); + } + } + + try + { + LogForModel(item, @"Beginning import..."); + + // TODO: do we want to make the transaction this local? not 100% sure, will need further investigation. + using (var transaction = realm.BeginWrite()) + { + if (archive != null) + // TODO: look into rollback of file additions (or delayed commit). + item.Files.AddRange(createFileInfos(archive, Files, realm)); + + item.Hash = ComputeHash(item); + + // TODO: we may want to run this outside of the transaction. + Populate(item, archive, realm, cancellationToken); + + if (!checkedExisting) + existing = CheckForExisting(item, realm); if (existing != null) { - // bare minimum comparisons - // - // note that this should really be checking filesizes on disk (of existing files) for some degree of sanity. - // or alternatively doing a faster hash check. either of these require database changes and reprocessing of existing files. - if (CanSkipImport(existing, item) && - getFilenames(existing.Files).SequenceEqual(getShortenedFilenames(archive).Select(p => p.shortened).OrderBy(f => f)) && - checkAllFilesExist(existing)) + if (CanReuseExisting(existing, item)) { - LogForModel(item, @$"Found existing (optimised) {HumanisedModelName} for {item} (ID {existing.ID}) – skipping import."); + LogForModel(item, @$"Found existing {HumanisedModelName} for {item} (ID {existing.ID}) – skipping import."); - using (var transaction = realm.BeginWrite()) - { - UndeleteForReuse(existing); - transaction.Commit(); - } + UndeleteForReuse(existing); + transaction.Commit(); return existing.ToLive(Realm); } - LogForModel(item, @"Found existing (optimised) but failed pre-check."); - } - } + LogForModel(item, @"Found existing but failed re-use check."); - try - { - LogForModel(item, @"Beginning import..."); - - // TODO: do we want to make the transaction this local? not 100% sure, will need further investigation. - using (var transaction = realm.BeginWrite()) - { - if (archive != null) - // TODO: look into rollback of file additions (or delayed commit). - item.Files.AddRange(createFileInfos(archive, Files, realm)); - - item.Hash = ComputeHash(item); - - // TODO: we may want to run this outside of the transaction. - Populate(item, archive, realm, cancellationToken); - - if (!checkedExisting) - existing = CheckForExisting(item, realm); - - if (existing != null) - { - if (CanReuseExisting(existing, item)) - { - LogForModel(item, @$"Found existing {HumanisedModelName} for {item} (ID {existing.ID}) – skipping import."); - - UndeleteForReuse(existing); - transaction.Commit(); - - return existing.ToLive(Realm); - } - - LogForModel(item, @"Found existing but failed re-use check."); - - existing.DeletePending = true; - } - - PreImport(item, realm); - - // import to store - realm.Add(item); - - transaction.Commit(); + existing.DeletePending = true; } - LogForModel(item, @"Import successfully completed!"); - } - catch (Exception e) - { - if (!(e is TaskCanceledException)) - LogForModel(item, @"Database import or population failed and has been rolled back.", e); + PreImport(item, realm); - throw; + // import to store + realm.Add(item); + + transaction.Commit(); } - return (Live?)item.ToLive(Realm); - }); - } + LogForModel(item, @"Import successfully completed!"); + } + catch (Exception e) + { + if (!(e is TaskCanceledException)) + LogForModel(item, @"Database import or population failed and has been rolled back.", e); + + throw; + } + + return (Live?)item.ToLive(Realm); + }); /// /// Any file extensions which should be included in hash creation. From f11fe543859fcefed2a94145e47d399c5d5b14f9 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 15 Jun 2022 00:27:09 +0900 Subject: [PATCH 3/7] Remove custom hash function flow --- .../Database/BeatmapImporterTests.cs | 23 +++++-------------- osu.Game/Skinning/SkinModelManager.cs | 2 -- osu.Game/Stores/RealmArchiveModelImporter.cs | 10 ++------ osu.Game/Tests/Visual/EditorTestScene.cs | 13 +---------- 4 files changed, 9 insertions(+), 39 deletions(-) diff --git a/osu.Game.Tests/Database/BeatmapImporterTests.cs b/osu.Game.Tests/Database/BeatmapImporterTests.cs index d4956e97e0..ab40ed2453 100644 --- a/osu.Game.Tests/Database/BeatmapImporterTests.cs +++ b/osu.Game.Tests/Database/BeatmapImporterTests.cs @@ -1,6 +1,8 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +#nullable enable + using System; using System.Collections.Generic; using System.Diagnostics; @@ -12,7 +14,6 @@ using NUnit.Framework; using osu.Framework.Extensions; using osu.Framework.Extensions.ObjectExtensions; using osu.Framework.Logging; -using osu.Framework.Platform; using osu.Game.Beatmaps; using osu.Game.Database; using osu.Game.Extensions; @@ -28,8 +29,6 @@ using SharpCompress.Archives.Zip; using SharpCompress.Common; using SharpCompress.Writers.Zip; -#nullable enable - namespace osu.Game.Tests.Database { [TestFixture] @@ -622,7 +621,7 @@ namespace osu.Game.Tests.Database using var importer = new BeatmapModelManager(realm, storage); using var store = new RealmRulesetStore(realm, storage); - var imported = await LoadOszIntoStore(importer, realm.Realm); + var imported = await LoadOszIntoStore(importer, realm.Realm, batchImport: true); deleteBeatmapSet(imported, realm.Realm); @@ -678,7 +677,7 @@ namespace osu.Game.Tests.Database { RunTestWithRealmAsync(async (realm, storage) => { - using var importer = new NonOptimisedBeatmapImporter(realm, storage); + using var importer = new BeatmapModelManager(realm, storage); using var store = new RealmRulesetStore(realm, storage); var imported = await LoadOszIntoStore(importer, realm.Realm); @@ -960,11 +959,11 @@ namespace osu.Game.Tests.Database return realm.All().FirstOrDefault(beatmapSet => beatmapSet.ID == importedSet!.ID); } - public static async Task LoadOszIntoStore(BeatmapImporter importer, Realm realm, string? path = null, bool virtualTrack = false) + public static async Task LoadOszIntoStore(BeatmapImporter importer, Realm realm, string? path = null, bool virtualTrack = false, bool batchImport = false) { string? temp = path ?? TestResources.GetTestBeatmapForImport(virtualTrack); - var importedSet = await importer.Import(new ImportTask(temp)); + var importedSet = await importer.Import(new ImportTask(temp), batchImport); Assert.NotNull(importedSet); Debug.Assert(importedSet != null); @@ -1081,15 +1080,5 @@ namespace osu.Game.Tests.Database Assert.Fail(failureMessage); } - - public class NonOptimisedBeatmapImporter : BeatmapImporter - { - public NonOptimisedBeatmapImporter(RealmAccess realm, Storage storage) - : base(realm, storage) - { - } - - protected override bool HasCustomHashFunction => true; - } } } diff --git a/osu.Game/Skinning/SkinModelManager.cs b/osu.Game/Skinning/SkinModelManager.cs index 23813e8eb2..0a18449800 100644 --- a/osu.Game/Skinning/SkinModelManager.cs +++ b/osu.Game/Skinning/SkinModelManager.cs @@ -46,8 +46,6 @@ namespace osu.Game.Skinning private const string unknown_creator_string = @"Unknown"; - protected override bool HasCustomHashFunction => true; - protected override void Populate(SkinInfo model, ArchiveReader? archive, Realm realm, CancellationToken cancellationToken = default) { var skinInfoFile = model.Files.SingleOrDefault(f => f.Filename == skin_info_file); diff --git a/osu.Game/Stores/RealmArchiveModelImporter.cs b/osu.Game/Stores/RealmArchiveModelImporter.cs index ac8a38fe37..976b26aac2 100644 --- a/osu.Game/Stores/RealmArchiveModelImporter.cs +++ b/osu.Game/Stores/RealmArchiveModelImporter.cs @@ -257,7 +257,7 @@ namespace osu.Game.Stores bool checkedExisting = false; TModel? existing = null; - if (quickSkipIfExisting && archive != null && !HasCustomHashFunction) + if (quickSkipIfExisting && archive != null) { // this is a fast bail condition to improve large import performance. item.Hash = computeHashFast(archive); @@ -373,19 +373,13 @@ namespace osu.Game.Stores Logger.Log($"{prefix} {message}", LoggingTarget.Database); } - /// - /// Whether the implementation overrides with a custom implementation. - /// Custom hash implementations must bypass the early exit in the import flow (see usage). - /// - protected virtual bool HasCustomHashFunction => false; - /// /// Create a SHA-2 hash from the provided archive based on file content of all files matching . /// /// /// In the case of no matching files, a hash will be generated from the passed archive's . /// - protected virtual string ComputeHash(TModel item) + protected string ComputeHash(TModel item) { // for now, concatenate all hashable files in the set to create a unique hash. MemoryStream hashable = new MemoryStream(); diff --git a/osu.Game/Tests/Visual/EditorTestScene.cs b/osu.Game/Tests/Visual/EditorTestScene.cs index 46f31ae53b..8e9bb6ff78 100644 --- a/osu.Game/Tests/Visual/EditorTestScene.cs +++ b/osu.Game/Tests/Visual/EditorTestScene.cs @@ -147,7 +147,7 @@ namespace osu.Game.Tests.Visual protected override BeatmapModelManager CreateBeatmapModelManager(Storage storage, RealmAccess realm, RulesetStore rulesets, BeatmapOnlineLookupQueue onlineLookupQueue) { - return new TestBeatmapModelManager(storage, realm, onlineLookupQueue); + return new BeatmapModelManager(realm, storage, onlineLookupQueue); } protected override WorkingBeatmapCache CreateWorkingBeatmapCache(AudioManager audioManager, IResourceStore resources, IResourceStore storage, WorkingBeatmap defaultBeatmap, GameHost host) @@ -181,17 +181,6 @@ namespace osu.Game.Tests.Visual => testBeatmapManager.TestBeatmap; } - internal class TestBeatmapModelManager : BeatmapModelManager - { - public TestBeatmapModelManager(Storage storage, RealmAccess databaseAccess, BeatmapOnlineLookupQueue beatmapOnlineLookupQueue) - : base(databaseAccess, storage, beatmapOnlineLookupQueue) - { - } - - protected override string ComputeHash(BeatmapSetInfo item) - => string.Empty; - } - public override void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null) { // don't actually care about saving for this context. From 88d5e074a8c6ca4060101ca0747bfc5bf01b4f1a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 15 Jun 2022 00:46:00 +0900 Subject: [PATCH 4/7] Rename `lowPriority` to `batchImport` --- ...eneOnlinePlayBeatmapAvailabilityTracker.cs | 4 +-- osu.Game/Beatmaps/BeatmapManager.cs | 4 +-- osu.Game/Scoring/ScoreManager.cs | 2 +- osu.Game/Skinning/SkinManager.cs | 6 ++-- osu.Game/Stores/RealmArchiveModelImporter.cs | 33 ++++++++----------- 5 files changed, 22 insertions(+), 27 deletions(-) diff --git a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs index 5cae576db1..4f255ddd00 100644 --- a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs +++ b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs @@ -225,10 +225,10 @@ namespace osu.Game.Tests.Online this.testBeatmapManager = testBeatmapManager; } - public override Live Import(BeatmapSetInfo item, ArchiveReader archive = null, bool quickSkipIfExisting = false, CancellationToken cancellationToken = default) + public override Live Import(BeatmapSetInfo item, ArchiveReader archive = null, bool batchImport = false, CancellationToken cancellationToken = default) { testBeatmapManager.AllowImport.Task.WaitSafely(); - return (testBeatmapManager.CurrentImport = base.Import(item, archive, quickSkipIfExisting, cancellationToken)); + return (testBeatmapManager.CurrentImport = base.Import(item, archive, batchImport, cancellationToken)); } } } diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index ab614106cc..e156c2b430 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -353,9 +353,9 @@ namespace osu.Game.Beatmaps public Task>> Import(ProgressNotification notification, params ImportTask[] tasks) => beatmapModelManager.Import(notification, tasks); - public Task?> Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default) => beatmapModelManager.Import(task, lowPriority, cancellationToken); + public Task?> Import(ImportTask task, bool batchImport = false, CancellationToken cancellationToken = default) => beatmapModelManager.Import(task, batchImport, cancellationToken); - public Task?> Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default) => beatmapModelManager.Import(archive, lowPriority, cancellationToken); + public Task?> Import(ArchiveReader archive, bool batchImport = false, CancellationToken cancellationToken = default) => beatmapModelManager.Import(archive, batchImport, cancellationToken); public Live? Import(BeatmapSetInfo item, ArchiveReader? archive = null, CancellationToken cancellationToken = default) => beatmapModelManager.Import(item, archive, false, cancellationToken); diff --git a/osu.Game/Scoring/ScoreManager.cs b/osu.Game/Scoring/ScoreManager.cs index e086135296..e857d2109f 100644 --- a/osu.Game/Scoring/ScoreManager.cs +++ b/osu.Game/Scoring/ScoreManager.cs @@ -280,7 +280,7 @@ namespace osu.Game.Scoring public Task>> Import(ProgressNotification notification, params ImportTask[] tasks) => scoreModelManager.Import(notification, tasks); - public Live Import(ScoreInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) => scoreModelManager.Import(item, archive, lowPriority, cancellationToken); + public Live Import(ScoreInfo item, ArchiveReader archive = null, bool batchImport = false, CancellationToken cancellationToken = default) => scoreModelManager.Import(item, archive, batchImport, cancellationToken); public bool IsAvailableLocally(ScoreInfo model) => scoreModelManager.IsAvailableLocally(model); diff --git a/osu.Game/Skinning/SkinManager.cs b/osu.Game/Skinning/SkinManager.cs index 6002bafa8d..f296386d34 100644 --- a/osu.Game/Skinning/SkinManager.cs +++ b/osu.Game/Skinning/SkinManager.cs @@ -276,10 +276,10 @@ namespace osu.Game.Skinning public Task>> Import(ProgressNotification notification, params ImportTask[] tasks) => skinModelManager.Import(notification, tasks); - public Task> Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default) => skinModelManager.Import(task, lowPriority, cancellationToken); + public Task> Import(ImportTask task, bool batchImport = false, CancellationToken cancellationToken = default) => skinModelManager.Import(task, batchImport, cancellationToken); - public Task> Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default) => - skinModelManager.Import(archive, lowPriority, cancellationToken); + public Task> Import(ArchiveReader archive, bool batchImport = false, CancellationToken cancellationToken = default) => + skinModelManager.Import(archive, batchImport, cancellationToken); #endregion diff --git a/osu.Game/Stores/RealmArchiveModelImporter.cs b/osu.Game/Stores/RealmArchiveModelImporter.cs index 976b26aac2..c6231e5583 100644 --- a/osu.Game/Stores/RealmArchiveModelImporter.cs +++ b/osu.Game/Stores/RealmArchiveModelImporter.cs @@ -34,11 +34,6 @@ namespace osu.Game.Stores { private const int import_queue_request_concurrency = 1; - /// - /// The size of a batch import operation before considering it a lower priority operation. - /// - private const int low_priority_import_batch_size = 1; - /// /// A singleton scheduler shared by all . /// @@ -49,11 +44,11 @@ namespace osu.Game.Stores private static readonly ThreadedTaskScheduler import_scheduler = new ThreadedTaskScheduler(import_queue_request_concurrency, nameof(RealmArchiveModelImporter)); /// - /// A second scheduler for lower priority imports. + /// A second scheduler for batch imports. /// For simplicity, these will just run in parallel with normal priority imports, but a future refactor would see this implemented via a custom scheduler/queue. /// See https://gist.github.com/peppy/f0e118a14751fc832ca30dd48ba3876b for an incomplete version of this. /// - private static readonly ThreadedTaskScheduler import_scheduler_low_priority = new ThreadedTaskScheduler(import_queue_request_concurrency, nameof(RealmArchiveModelImporter)); + private static readonly ThreadedTaskScheduler import_scheduler_batch = new ThreadedTaskScheduler(import_queue_request_concurrency, nameof(RealmArchiveModelImporter)); public virtual IEnumerable HandledExtensions => new[] { @".zip" }; @@ -105,7 +100,7 @@ namespace osu.Game.Stores var imported = new List>(); - bool isLowPriorityImport = tasks.Length > low_priority_import_batch_size; + bool isBatchImport = tasks.Length > 1; try { @@ -115,7 +110,7 @@ namespace osu.Game.Stores try { - var model = await Import(task, isLowPriorityImport, notification.CancellationToken).ConfigureAwait(false); + var model = await Import(task, isBatchImport, notification.CancellationToken).ConfigureAwait(false); lock (imported) { @@ -178,16 +173,16 @@ namespace osu.Game.Stores /// Note that this bypasses the UI flow and should only be used for special cases or testing. /// /// The containing data about the to import. - /// Whether this is a low priority import. + /// Whether this is a low priority import. /// An optional cancellation token. /// The imported model, if successful. - public async Task?> Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default) + public async Task?> Import(ImportTask task, bool batchImport = false, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); Live? import; using (ArchiveReader reader = task.GetReader()) - import = await Import(reader, lowPriority, cancellationToken).ConfigureAwait(false); + import = await Import(reader, batchImport, cancellationToken).ConfigureAwait(false); // We may or may not want to delete the file depending on where it is stored. // e.g. reconstructing/repairing database with items from default storage. @@ -210,9 +205,9 @@ namespace osu.Game.Stores /// Silently import an item from an . /// /// The archive to be imported. - /// Whether this is a low priority import. + /// Whether this is a low priority import. /// An optional cancellation token. - public async Task?> Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default) + public async Task?> Import(ArchiveReader archive, bool batchImport = false, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -235,10 +230,10 @@ namespace osu.Game.Stores return null; } - var scheduledImport = Task.Factory.StartNew(() => Import(model, archive, lowPriority, cancellationToken), + var scheduledImport = Task.Factory.StartNew(() => Import(model, archive, batchImport, cancellationToken), cancellationToken, TaskCreationOptions.HideScheduler, - lowPriority ? import_scheduler_low_priority : import_scheduler); + batchImport ? import_scheduler_batch : import_scheduler); return await scheduledImport.ConfigureAwait(false); } @@ -248,16 +243,16 @@ namespace osu.Game.Stores /// /// The model to be imported. /// An optional archive to use for model population. - /// If true, imports will be skipped before they begin, given an existing model matches on hash and filenames. Should generally only be used for large batch imports, as it may defy user expectations when updating an existing model. + /// If true, imports will be skipped before they begin, given an existing model matches on hash and filenames. Should generally only be used for large batch imports, as it may defy user expectations when updating an existing model. /// An optional cancellation token. - public virtual Live? Import(TModel item, ArchiveReader? archive = null, bool quickSkipIfExisting = false, CancellationToken cancellationToken = default) => Realm.Run(realm => + public virtual Live? Import(TModel item, ArchiveReader? archive = null, bool batchImport = false, CancellationToken cancellationToken = default) => Realm.Run(realm => { cancellationToken.ThrowIfCancellationRequested(); bool checkedExisting = false; TModel? existing = null; - if (quickSkipIfExisting && archive != null) + if (batchImport && archive != null) { // this is a fast bail condition to improve large import performance. item.Hash = computeHashFast(archive); From cac724f43679950579ba217d3a337d6d96d0d9da Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 15 Jun 2022 00:48:00 +0900 Subject: [PATCH 5/7] Re-enable ignored tests that will work again now --- osu.Game.Tests/Database/BeatmapImporterTests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/osu.Game.Tests/Database/BeatmapImporterTests.cs b/osu.Game.Tests/Database/BeatmapImporterTests.cs index ab40ed2453..de2bbeb791 100644 --- a/osu.Game.Tests/Database/BeatmapImporterTests.cs +++ b/osu.Game.Tests/Database/BeatmapImporterTests.cs @@ -393,7 +393,6 @@ namespace osu.Game.Tests.Database } [Test] - [Ignore("intentionally broken by import optimisations")] public void TestImportThenImportWithChangedFile() { RunTestWithRealmAsync(async (realm, storage) => @@ -490,7 +489,6 @@ namespace osu.Game.Tests.Database } [Test] - [Ignore("intentionally broken by import optimisations")] public void TestImportCorruptThenImport() { RunTestWithRealmAsync(async (realm, storage) => @@ -502,16 +500,18 @@ namespace osu.Game.Tests.Database var firstFile = imported.Files.First(); + var fileStorage = storage.GetStorageForDirectory("files"); + long originalLength; - using (var stream = storage.GetStream(firstFile.File.GetStoragePath())) + using (var stream = fileStorage.GetStream(firstFile.File.GetStoragePath())) originalLength = stream.Length; - using (var stream = storage.CreateFileSafely(firstFile.File.GetStoragePath())) + using (var stream = fileStorage.CreateFileSafely(firstFile.File.GetStoragePath())) stream.WriteByte(0); var importedSecondTime = await LoadOszIntoStore(importer, realm.Realm); - using (var stream = storage.GetStream(firstFile.File.GetStoragePath())) + using (var stream = fileStorage.GetStream(firstFile.File.GetStoragePath())) Assert.AreEqual(stream.Length, originalLength, "Corruption was not fixed on second import"); // check the newly "imported" beatmap is actually just the restored previous import. since it matches hash. From abe947bcb07ff67e974b67f580dac0dd07f28d2f Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 15 Jun 2022 15:31:45 +0900 Subject: [PATCH 6/7] Update outdated xmldoc comments --- osu.Game/Stores/RealmArchiveModelImporter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game/Stores/RealmArchiveModelImporter.cs b/osu.Game/Stores/RealmArchiveModelImporter.cs index c6231e5583..4bbd6b2264 100644 --- a/osu.Game/Stores/RealmArchiveModelImporter.cs +++ b/osu.Game/Stores/RealmArchiveModelImporter.cs @@ -173,7 +173,7 @@ namespace osu.Game.Stores /// Note that this bypasses the UI flow and should only be used for special cases or testing. /// /// The containing data about the to import. - /// Whether this is a low priority import. + /// Whether this import is part of a larger batch. /// An optional cancellation token. /// The imported model, if successful. public async Task?> Import(ImportTask task, bool batchImport = false, CancellationToken cancellationToken = default) @@ -205,7 +205,7 @@ namespace osu.Game.Stores /// Silently import an item from an . /// /// The archive to be imported. - /// Whether this is a low priority import. + /// Whether this import is part of a larger batch. /// An optional cancellation token. public async Task?> Import(ArchiveReader archive, bool batchImport = false, CancellationToken cancellationToken = default) { From 5201057a626f5f5fa6eeff0105df50621e1d03df Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 15 Jun 2022 15:31:58 +0900 Subject: [PATCH 7/7] Change batch import to require at least 10 items --- osu.Game/Stores/RealmArchiveModelImporter.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/osu.Game/Stores/RealmArchiveModelImporter.cs b/osu.Game/Stores/RealmArchiveModelImporter.cs index 4bbd6b2264..7f31bc66be 100644 --- a/osu.Game/Stores/RealmArchiveModelImporter.cs +++ b/osu.Game/Stores/RealmArchiveModelImporter.cs @@ -32,8 +32,17 @@ namespace osu.Game.Stores public abstract class RealmArchiveModelImporter : IModelImporter where TModel : RealmObject, IHasRealmFiles, IHasGuidPrimaryKey, ISoftDelete { + /// + /// The maximum number of concurrent imports to run per import scheduler. + /// private const int import_queue_request_concurrency = 1; + /// + /// The minimum number of items in a single import call in order for the import to be processed as a batch. + /// Batch imports will apply optimisations preferring speed over consistency when detecting changes in already-imported items. + /// + private const int minimum_items_considered_batch_import = 10; + /// /// A singleton scheduler shared by all . /// @@ -100,7 +109,7 @@ namespace osu.Game.Stores var imported = new List>(); - bool isBatchImport = tasks.Length > 1; + bool isBatchImport = tasks.Length >= minimum_items_considered_batch_import; try {