From 6ffd9fdcfa20476f80a45fbbd60602782323148d Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 14:46:01 +0900 Subject: [PATCH 01/16] Split out `BeatmapOnlineLookupQueue` from `BeatmapManager` --- ...eneOnlinePlayBeatmapAvailabilityTracker.cs | 4 +- osu.Game/Beatmaps/BeatmapManager.cs | 25 +- ...BeatmapManager_BeatmapOnlineLookupQueue.cs | 215 ------------------ osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs | 215 ++++++++++++++++++ osu.Game/OsuGameBase.cs | 9 +- 5 files changed, 234 insertions(+), 234 deletions(-) delete mode 100644 osu.Game/Beatmaps/BeatmapManager_BeatmapOnlineLookupQueue.cs create mode 100644 osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs diff --git a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs index 7e7e5ebc45..a7d34fadbe 100644 --- a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs +++ b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs @@ -161,8 +161,8 @@ namespace osu.Game.Tests.Online protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) => new TestDownloadRequest(set); - public TestBeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, WorkingBeatmap defaultBeatmap = null, bool performOnlineLookups = false) - : base(storage, contextFactory, rulesets, api, audioManager, resources, host, defaultBeatmap, performOnlineLookups) + public TestBeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, WorkingBeatmap defaultBeatmap = null) + : base(storage, contextFactory, rulesets, api, audioManager, resources, host, defaultBeatmap) { } diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index bd85017d58..a2f9740779 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -40,7 +40,7 @@ namespace osu.Game.Beatmaps /// Handles the storage and retrieval of Beatmaps/WorkingBeatmaps. /// [ExcludeFromDynamicCompile] - public partial class BeatmapManager : DownloadableArchiveModelManager, IDisposable, IBeatmapResourceProvider + public partial class BeatmapManager : DownloadableArchiveModelManager, IBeatmapResourceProvider { /// /// Fired when a single difficulty has been hidden. @@ -54,6 +54,12 @@ namespace osu.Game.Beatmaps /// public IBindable> BeatmapRestored => beatmapRestored; + /// + /// A function which populates online information during the import process. + /// It is run as the final step of import. + /// + public Func PopulateOnlineInformation; + private readonly Bindable> beatmapRestored = new Bindable>(); /// @@ -79,11 +85,8 @@ namespace osu.Game.Beatmaps [CanBeNull] private readonly GameHost host; - [CanBeNull] - private readonly BeatmapOnlineLookupQueue onlineLookupQueue; - public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, - WorkingBeatmap defaultBeatmap = null, bool performOnlineLookups = false) + WorkingBeatmap defaultBeatmap = null) : base(storage, contextFactory, api, new BeatmapStore(contextFactory), host) { this.rulesets = rulesets; @@ -99,9 +102,6 @@ namespace osu.Game.Beatmaps beatmaps.ItemRemoved += removeWorkingCache; beatmaps.ItemUpdated += removeWorkingCache; - if (performOnlineLookups) - onlineLookupQueue = new BeatmapOnlineLookupQueue(api, storage); - largeTextureStore = new LargeTextureStore(host?.CreateTextureLoaderStore(Files.Store)); trackStore = audioManager.GetTrackStore(Files.Store); } @@ -156,8 +156,8 @@ namespace osu.Game.Beatmaps bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0); - if (onlineLookupQueue != null) - await onlineLookupQueue.UpdateAsync(beatmapSet, cancellationToken).ConfigureAwait(false); + if (PopulateOnlineInformation != null) + await PopulateOnlineInformation(beatmapSet, cancellationToken).ConfigureAwait(false); // ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID. if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0)) @@ -533,11 +533,6 @@ namespace osu.Game.Beatmaps } } - public void Dispose() - { - onlineLookupQueue?.Dispose(); - } - #region IResourceStorageProvider TextureStore IBeatmapResourceProvider.LargeTextureStore => largeTextureStore; diff --git a/osu.Game/Beatmaps/BeatmapManager_BeatmapOnlineLookupQueue.cs b/osu.Game/Beatmaps/BeatmapManager_BeatmapOnlineLookupQueue.cs deleted file mode 100644 index 3dd34f6c2f..0000000000 --- a/osu.Game/Beatmaps/BeatmapManager_BeatmapOnlineLookupQueue.cs +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Data.Sqlite; -using osu.Framework.Development; -using osu.Framework.IO.Network; -using osu.Framework.Logging; -using osu.Framework.Platform; -using osu.Framework.Testing; -using osu.Framework.Threading; -using osu.Game.Database; -using osu.Game.Online.API; -using osu.Game.Online.API.Requests; -using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; - -namespace osu.Game.Beatmaps -{ - public partial class BeatmapManager - { - [ExcludeFromDynamicCompile] - private class BeatmapOnlineLookupQueue : IDisposable - { - private readonly IAPIProvider api; - private readonly Storage storage; - - private const int update_queue_request_concurrency = 4; - - private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency, nameof(BeatmapOnlineLookupQueue)); - - private FileWebRequest cacheDownloadRequest; - - private const string cache_database_name = "online.db"; - - public BeatmapOnlineLookupQueue(IAPIProvider api, Storage storage) - { - this.api = api; - this.storage = storage; - - // avoid downloading / using cache for unit tests. - if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name)) - prepareLocalCache(); - } - - public Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) - { - return Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray()); - } - - // todo: expose this when we need to do individual difficulty lookups. - protected Task UpdateAsync(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken) - => Task.Factory.StartNew(() => lookup(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler | TaskCreationOptions.RunContinuationsAsynchronously, updateScheduler); - - private void lookup(BeatmapSetInfo set, BeatmapInfo beatmap) - { - if (checkLocalCache(set, beatmap)) - return; - - if (api?.State.Value != APIState.Online) - return; - - var req = new GetBeatmapRequest(beatmap); - - req.Failure += fail; - - try - { - // intentionally blocking to limit web request concurrency - api.Perform(req); - - var res = req.Result; - - if (res != null) - { - beatmap.Status = res.Status; - beatmap.BeatmapSet.Status = res.BeatmapSet.Status; - beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID; - beatmap.OnlineBeatmapID = res.OnlineBeatmapID; - - if (beatmap.Metadata != null) - beatmap.Metadata.AuthorID = res.AuthorID; - - if (beatmap.BeatmapSet.Metadata != null) - beatmap.BeatmapSet.Metadata.AuthorID = res.AuthorID; - - LogForModel(set, $"Online retrieval mapped {beatmap} to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}."); - } - } - catch (Exception e) - { - fail(e); - } - - void fail(Exception e) - { - beatmap.OnlineBeatmapID = null; - LogForModel(set, $"Online retrieval failed for {beatmap} ({e.Message})"); - } - } - - private void prepareLocalCache() - { - string cacheFilePath = storage.GetFullPath(cache_database_name); - string compressedCacheFilePath = $"{cacheFilePath}.bz2"; - - cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}"); - - cacheDownloadRequest.Failed += ex => - { - File.Delete(compressedCacheFilePath); - File.Delete(cacheFilePath); - - Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache download failed: {ex}", LoggingTarget.Database); - }; - - cacheDownloadRequest.Finished += () => - { - try - { - using (var stream = File.OpenRead(cacheDownloadRequest.Filename)) - using (var outStream = File.OpenWrite(cacheFilePath)) - using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false)) - bz2.CopyTo(outStream); - - // set to null on completion to allow lookups to begin using the new source - cacheDownloadRequest = null; - } - catch (Exception ex) - { - Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache extraction failed: {ex}", LoggingTarget.Database); - File.Delete(cacheFilePath); - } - finally - { - File.Delete(compressedCacheFilePath); - } - }; - - cacheDownloadRequest.PerformAsync(); - } - - private bool checkLocalCache(BeatmapSetInfo set, BeatmapInfo beatmap) - { - // download is in progress (or was, and failed). - if (cacheDownloadRequest != null) - return false; - - // database is unavailable. - if (!storage.Exists(cache_database_name)) - return false; - - if (string.IsNullOrEmpty(beatmap.MD5Hash) - && string.IsNullOrEmpty(beatmap.Path) - && beatmap.OnlineBeatmapID == null) - return false; - - try - { - using (var db = new SqliteConnection(DatabaseContextFactory.CreateDatabaseConnectionString("online.db", storage))) - { - db.Open(); - - using (var cmd = db.CreateCommand()) - { - cmd.CommandText = "SELECT beatmapset_id, beatmap_id, approved, user_id FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineBeatmapID OR filename = @Path"; - - cmd.Parameters.Add(new SqliteParameter("@MD5Hash", beatmap.MD5Hash)); - cmd.Parameters.Add(new SqliteParameter("@OnlineBeatmapID", beatmap.OnlineBeatmapID ?? (object)DBNull.Value)); - cmd.Parameters.Add(new SqliteParameter("@Path", beatmap.Path)); - - using (var reader = cmd.ExecuteReader()) - { - if (reader.Read()) - { - var status = (BeatmapSetOnlineStatus)reader.GetByte(2); - - beatmap.Status = status; - beatmap.BeatmapSet.Status = status; - beatmap.BeatmapSet.OnlineBeatmapSetID = reader.GetInt32(0); - beatmap.OnlineBeatmapID = reader.GetInt32(1); - - if (beatmap.Metadata != null) - beatmap.Metadata.AuthorID = reader.GetInt32(3); - - if (beatmap.BeatmapSet.Metadata != null) - beatmap.BeatmapSet.Metadata.AuthorID = reader.GetInt32(3); - - LogForModel(set, $"Cached local retrieval for {beatmap}."); - return true; - } - } - } - } - } - catch (Exception ex) - { - LogForModel(set, $"Cached local retrieval for {beatmap} failed with {ex}."); - } - - return false; - } - - public void Dispose() - { - cacheDownloadRequest?.Dispose(); - updateScheduler?.Dispose(); - } - } - } -} diff --git a/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs b/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs new file mode 100644 index 0000000000..bbac30f2bb --- /dev/null +++ b/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs @@ -0,0 +1,215 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using osu.Framework.Development; +using osu.Framework.IO.Network; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Framework.Testing; +using osu.Framework.Threading; +using osu.Game.Database; +using osu.Game.Online.API; +using osu.Game.Online.API.Requests; +using SharpCompress.Compressors; +using SharpCompress.Compressors.BZip2; + +namespace osu.Game.Beatmaps +{ + [ExcludeFromDynamicCompile] + public class BeatmapOnlineLookupQueue : IDisposable + { + private readonly IAPIProvider api; + private readonly Storage storage; + + private const int update_queue_request_concurrency = 4; + + private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency, nameof(BeatmapOnlineLookupQueue)); + + private FileWebRequest cacheDownloadRequest; + + private const string cache_database_name = "online.db"; + + public BeatmapOnlineLookupQueue(IAPIProvider api, Storage storage) + { + this.api = api; + this.storage = storage; + + // avoid downloading / using cache for unit tests. + if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name)) + prepareLocalCache(); + } + + public Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) + { + return Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray()); + } + + // todo: expose this when we need to do individual difficulty lookups. + protected Task UpdateAsync(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken) + => Task.Factory.StartNew(() => lookup(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler | TaskCreationOptions.RunContinuationsAsynchronously, updateScheduler); + + private void lookup(BeatmapSetInfo set, BeatmapInfo beatmap) + { + if (checkLocalCache(set, beatmap)) + return; + + if (api?.State.Value != APIState.Online) + return; + + var req = new GetBeatmapRequest(beatmap); + + req.Failure += fail; + + try + { + // intentionally blocking to limit web request concurrency + api.Perform(req); + + var res = req.Result; + + if (res != null) + { + beatmap.Status = res.Status; + beatmap.BeatmapSet.Status = res.BeatmapSet.Status; + beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID; + beatmap.OnlineBeatmapID = res.OnlineBeatmapID; + + if (beatmap.Metadata != null) + beatmap.Metadata.AuthorID = res.AuthorID; + + if (beatmap.BeatmapSet.Metadata != null) + beatmap.BeatmapSet.Metadata.AuthorID = res.AuthorID; + + logForModel(set, $"Online retrieval mapped {beatmap} to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}."); + } + } + catch (Exception e) + { + fail(e); + } + + void fail(Exception e) + { + beatmap.OnlineBeatmapID = null; + logForModel(set, $"Online retrieval failed for {beatmap} ({e.Message})"); + } + } + + private void prepareLocalCache() + { + string cacheFilePath = storage.GetFullPath(cache_database_name); + string compressedCacheFilePath = $"{cacheFilePath}.bz2"; + + cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}"); + + cacheDownloadRequest.Failed += ex => + { + File.Delete(compressedCacheFilePath); + File.Delete(cacheFilePath); + + Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache download failed: {ex}", LoggingTarget.Database); + }; + + cacheDownloadRequest.Finished += () => + { + try + { + using (var stream = File.OpenRead(cacheDownloadRequest.Filename)) + using (var outStream = File.OpenWrite(cacheFilePath)) + using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false)) + bz2.CopyTo(outStream); + + // set to null on completion to allow lookups to begin using the new source + cacheDownloadRequest = null; + } + catch (Exception ex) + { + Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache extraction failed: {ex}", LoggingTarget.Database); + File.Delete(cacheFilePath); + } + finally + { + File.Delete(compressedCacheFilePath); + } + }; + + cacheDownloadRequest.PerformAsync(); + } + + private bool checkLocalCache(BeatmapSetInfo set, BeatmapInfo beatmap) + { + // download is in progress (or was, and failed). + if (cacheDownloadRequest != null) + return false; + + // database is unavailable. + if (!storage.Exists(cache_database_name)) + return false; + + if (string.IsNullOrEmpty(beatmap.MD5Hash) + && string.IsNullOrEmpty(beatmap.Path) + && beatmap.OnlineBeatmapID == null) + return false; + + try + { + using (var db = new SqliteConnection(DatabaseContextFactory.CreateDatabaseConnectionString("online.db", storage))) + { + db.Open(); + + using (var cmd = db.CreateCommand()) + { + cmd.CommandText = "SELECT beatmapset_id, beatmap_id, approved, user_id FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineBeatmapID OR filename = @Path"; + + cmd.Parameters.Add(new SqliteParameter("@MD5Hash", beatmap.MD5Hash)); + cmd.Parameters.Add(new SqliteParameter("@OnlineBeatmapID", beatmap.OnlineBeatmapID ?? (object)DBNull.Value)); + cmd.Parameters.Add(new SqliteParameter("@Path", beatmap.Path)); + + using (var reader = cmd.ExecuteReader()) + { + if (reader.Read()) + { + var status = (BeatmapSetOnlineStatus)reader.GetByte(2); + + beatmap.Status = status; + beatmap.BeatmapSet.Status = status; + beatmap.BeatmapSet.OnlineBeatmapSetID = reader.GetInt32(0); + beatmap.OnlineBeatmapID = reader.GetInt32(1); + + if (beatmap.Metadata != null) + beatmap.Metadata.AuthorID = reader.GetInt32(3); + + if (beatmap.BeatmapSet.Metadata != null) + beatmap.BeatmapSet.Metadata.AuthorID = reader.GetInt32(3); + + logForModel(set, $"Cached local retrieval for {beatmap}."); + return true; + } + } + } + } + } + catch (Exception ex) + { + logForModel(set, $"Cached local retrieval for {beatmap} failed with {ex}."); + } + + return false; + } + + private void logForModel(BeatmapSetInfo set, string message) => + ArchiveModelManager.LogForModel(set, $"{nameof(BeatmapOnlineLookupQueue)}] {message}"); + + public void Dispose() + { + cacheDownloadRequest?.Dispose(); + updateScheduler?.Dispose(); + } + } +} diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 7aa460981a..8263e26dec 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -138,6 +138,8 @@ namespace osu.Game private UserLookupCache userCache; + private BeatmapOnlineLookupQueue onlineBeatmapLookupCache; + private FileStore fileStore; private RulesetConfigCache rulesetConfigCache; @@ -242,7 +244,11 @@ namespace osu.Game // ordering is important here to ensure foreign keys rules are not broken in ModelStore.Cleanup() dependencies.Cache(ScoreManager = new ScoreManager(RulesetStore, () => BeatmapManager, Storage, API, contextFactory, Scheduler, Host, () => difficultyCache, LocalConfig)); - dependencies.Cache(BeatmapManager = new BeatmapManager(Storage, contextFactory, RulesetStore, API, Audio, Resources, Host, defaultBeatmap, true)); + dependencies.Cache(BeatmapManager = new BeatmapManager(Storage, contextFactory, RulesetStore, API, Audio, Resources, Host, defaultBeatmap)); + + onlineBeatmapLookupCache = new BeatmapOnlineLookupQueue(API, Storage); + + BeatmapManager.PopulateOnlineInformation = onlineBeatmapLookupCache.UpdateAsync; // this should likely be moved to ArchiveModelManager when another case appears where it is necessary // to have inter-dependent model managers. this could be obtained with an IHasForeign interface to @@ -524,7 +530,6 @@ namespace osu.Game base.Dispose(isDisposing); RulesetStore?.Dispose(); - BeatmapManager?.Dispose(); LocalConfig?.Dispose(); contextFactory?.FlushConnections(); From 8a6501fa58f67f49aa71df20dbb76a5caba92516 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 14:46:07 +0900 Subject: [PATCH 02/16] Add basic component level xmldoc --- osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs b/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs index bbac30f2bb..19f02c82ec 100644 --- a/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs +++ b/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs @@ -21,6 +21,13 @@ using SharpCompress.Compressors.BZip2; namespace osu.Game.Beatmaps { + /// + /// A component which handles population of online IDs for beatmaps using a two part lookup procedure. + /// + /// + /// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to ). + /// This will always be checked before doing a second online query to get required metadata. + /// [ExcludeFromDynamicCompile] public class BeatmapOnlineLookupQueue : IDisposable { From e7e04733234cf2de40abf9241548c786f64d8ed6 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 15:40:41 +0900 Subject: [PATCH 03/16] Split out `WorkingBeatmapCache` from `BeatmapManager` --- osu.Game/Beatmaps/BeatmapManager.cs | 134 +-------- .../Beatmaps/BeatmapManager_WorkingBeatmap.cs | 147 --------- osu.Game/Beatmaps/WorkingBeatmapCache.cs | 279 ++++++++++++++++++ 3 files changed, 289 insertions(+), 271 deletions(-) delete mode 100644 osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs create mode 100644 osu.Game/Beatmaps/WorkingBeatmapCache.cs diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index a2f9740779..1c0e7dc319 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -9,18 +9,13 @@ using System.Linq.Expressions; using System.Text; using System.Threading; using System.Threading.Tasks; -using JetBrains.Annotations; using Microsoft.EntityFrameworkCore; -using osu.Framework.Audio; using osu.Framework.Audio.Track; using osu.Framework.Bindables; using osu.Framework.Extensions; using osu.Framework.Graphics.Textures; -using osu.Framework.IO.Stores; -using osu.Framework.Lists; using osu.Framework.Logging; using osu.Framework.Platform; -using osu.Framework.Statistics; using osu.Framework.Testing; using osu.Game.Beatmaps.Formats; using osu.Game.Database; @@ -31,7 +26,6 @@ using osu.Game.Online.API.Requests; using osu.Game.Rulesets; using osu.Game.Rulesets.Objects; using osu.Game.Skinning; -using osu.Game.Users; using Decoder = osu.Game.Beatmaps.Formats.Decoder; namespace osu.Game.Beatmaps @@ -40,7 +34,7 @@ namespace osu.Game.Beatmaps /// Handles the storage and retrieval of Beatmaps/WorkingBeatmaps. /// [ExcludeFromDynamicCompile] - public partial class BeatmapManager : DownloadableArchiveModelManager, IBeatmapResourceProvider + public class BeatmapManager : DownloadableArchiveModelManager { /// /// Fired when a single difficulty has been hidden. @@ -60,12 +54,12 @@ namespace osu.Game.Beatmaps /// public Func PopulateOnlineInformation; - private readonly Bindable> beatmapRestored = new Bindable>(); - /// - /// A default representation of a WorkingBeatmap to use when no beatmap is available. + /// The game working beatmap cache, used to invalidate entries on changes. /// - public readonly WorkingBeatmap DefaultBeatmap; + public WorkingBeatmapCache WorkingBeatmapCache { private get; set; } + + private readonly Bindable> beatmapRestored = new Bindable>(); public override IEnumerable HandledExtensions => new[] { ".osz" }; @@ -75,35 +69,19 @@ namespace osu.Game.Beatmaps protected override Storage PrepareStableStorage(StableStorage stableStorage) => stableStorage.GetSongStorage(); - private readonly RulesetStore rulesets; private readonly BeatmapStore beatmaps; - private readonly AudioManager audioManager; - private readonly IResourceStore resources; - private readonly LargeTextureStore largeTextureStore; - private readonly ITrackStore trackStore; + private readonly RulesetStore rulesets; - [CanBeNull] - private readonly GameHost host; - - public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, - WorkingBeatmap defaultBeatmap = null) + public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host = null) : base(storage, contextFactory, api, new BeatmapStore(contextFactory), host) { this.rulesets = rulesets; - this.audioManager = audioManager; - this.resources = resources; - this.host = host; - - DefaultBeatmap = defaultBeatmap; beatmaps = (BeatmapStore)ModelStore; beatmaps.BeatmapHidden += b => beatmapHidden.Value = new WeakReference(b); beatmaps.BeatmapRestored += b => beatmapRestored.Value = new WeakReference(b); - beatmaps.ItemRemoved += removeWorkingCache; - beatmaps.ItemUpdated += removeWorkingCache; - - largeTextureStore = new LargeTextureStore(host?.CreateTextureLoaderStore(Files.Store)); - trackStore = audioManager.GetTrackStore(Files.Store); + beatmaps.ItemRemoved += b => WorkingBeatmapCache?.Invalidate(b); + beatmaps.ItemUpdated += obj => WorkingBeatmapCache?.Invalidate(obj); } protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) => @@ -111,33 +89,6 @@ namespace osu.Game.Beatmaps protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path)?.ToLowerInvariant() == ".osz"; - public WorkingBeatmap CreateNew(RulesetInfo ruleset, User user) - { - var metadata = new BeatmapMetadata - { - Author = user, - }; - - var set = new BeatmapSetInfo - { - Metadata = metadata, - Beatmaps = new List - { - new BeatmapInfo - { - BaseDifficulty = new BeatmapDifficulty(), - Ruleset = ruleset, - Metadata = metadata, - WidescreenStoryboard = true, - SamplesMatchPlaybackRate = true, - } - } - }; - - var working = Import(set).Result; - return GetWorkingBeatmap(working.Beatmaps.First()); - } - protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) { if (archive != null) @@ -278,43 +229,7 @@ namespace osu.Game.Beatmaps } } - removeWorkingCache(info); - } - - private readonly WeakList workingCache = new WeakList(); - - /// - /// Retrieve a instance for the provided - /// - /// The beatmap to lookup. - /// A instance correlating to the provided . - public virtual WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo) - { - // if there are no files, presume the full beatmap info has not yet been fetched from the database. - if (beatmapInfo?.BeatmapSet?.Files.Count == 0) - { - int lookupId = beatmapInfo.ID; - beatmapInfo = QueryBeatmap(b => b.ID == lookupId); - } - - if (beatmapInfo?.BeatmapSet == null) - return DefaultBeatmap; - - lock (workingCache) - { - var working = workingCache.FirstOrDefault(w => w.BeatmapInfo?.ID == beatmapInfo.ID); - if (working != null) - return working; - - beatmapInfo.Metadata ??= beatmapInfo.BeatmapSet.Metadata; - - workingCache.Add(working = new BeatmapManagerWorkingBeatmap(beatmapInfo, this)); - - // best effort; may be higher than expected. - GlobalStatistics.Get(nameof(Beatmaps), $"Cached {nameof(WorkingBeatmap)}s").Value = workingCache.Count(); - - return working; - } + WorkingBeatmapCache?.Invalidate(info); } /// @@ -515,35 +430,6 @@ namespace osu.Game.Beatmaps return endTime - startTime; } - private void removeWorkingCache(BeatmapSetInfo info) - { - if (info.Beatmaps == null) return; - - foreach (var b in info.Beatmaps) - removeWorkingCache(b); - } - - private void removeWorkingCache(BeatmapInfo info) - { - lock (workingCache) - { - var working = workingCache.FirstOrDefault(w => w.BeatmapInfo?.ID == info.ID); - if (working != null) - workingCache.Remove(working); - } - } - - #region IResourceStorageProvider - - TextureStore IBeatmapResourceProvider.LargeTextureStore => largeTextureStore; - ITrackStore IBeatmapResourceProvider.Tracks => trackStore; - AudioManager IStorageResourceProvider.AudioManager => audioManager; - IResourceStore IStorageResourceProvider.Files => Files.Store; - IResourceStore IStorageResourceProvider.Resources => resources; - IResourceStore IStorageResourceProvider.CreateTextureLoaderStore(IResourceStore underlyingStore) => host?.CreateTextureLoaderStore(underlyingStore); - - #endregion - /// /// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation. /// diff --git a/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs b/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs deleted file mode 100644 index 45112ae74c..0000000000 --- a/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using osu.Framework.Audio.Track; -using osu.Framework.Graphics.Textures; -using osu.Framework.Logging; -using osu.Framework.Testing; -using osu.Game.Beatmaps.Formats; -using osu.Game.IO; -using osu.Game.Skinning; -using osu.Game.Storyboards; - -namespace osu.Game.Beatmaps -{ - public partial class BeatmapManager - { - [ExcludeFromDynamicCompile] - private class BeatmapManagerWorkingBeatmap : WorkingBeatmap - { - [NotNull] - private readonly IBeatmapResourceProvider resources; - - public BeatmapManagerWorkingBeatmap(BeatmapInfo beatmapInfo, [NotNull] IBeatmapResourceProvider resources) - : base(beatmapInfo, resources.AudioManager) - { - this.resources = resources; - } - - protected override IBeatmap GetBeatmap() - { - if (BeatmapInfo.Path == null) - return new Beatmap { BeatmapInfo = BeatmapInfo }; - - try - { - using (var stream = new LineBufferedReader(GetStream(BeatmapSetInfo.GetPathForFile(BeatmapInfo.Path)))) - return Decoder.GetDecoder(stream).Decode(stream); - } - catch (Exception e) - { - Logger.Error(e, "Beatmap failed to load"); - return null; - } - } - - protected override bool BackgroundStillValid(Texture b) => false; // bypass lazy logic. we want to return a new background each time for refcounting purposes. - - protected override Texture GetBackground() - { - if (Metadata?.BackgroundFile == null) - return null; - - try - { - return resources.LargeTextureStore.Get(BeatmapSetInfo.GetPathForFile(Metadata.BackgroundFile)); - } - catch (Exception e) - { - Logger.Error(e, "Background failed to load"); - return null; - } - } - - protected override Track GetBeatmapTrack() - { - if (Metadata?.AudioFile == null) - return null; - - try - { - return resources.Tracks.Get(BeatmapSetInfo.GetPathForFile(Metadata.AudioFile)); - } - catch (Exception e) - { - Logger.Error(e, "Track failed to load"); - return null; - } - } - - protected override Waveform GetWaveform() - { - if (Metadata?.AudioFile == null) - return null; - - try - { - var trackData = GetStream(BeatmapSetInfo.GetPathForFile(Metadata.AudioFile)); - return trackData == null ? null : new Waveform(trackData); - } - catch (Exception e) - { - Logger.Error(e, "Waveform failed to load"); - return null; - } - } - - protected override Storyboard GetStoryboard() - { - Storyboard storyboard; - - try - { - using (var stream = new LineBufferedReader(GetStream(BeatmapSetInfo.GetPathForFile(BeatmapInfo.Path)))) - { - var decoder = Decoder.GetDecoder(stream); - - // todo: support loading from both set-wide storyboard *and* beatmap specific. - if (BeatmapSetInfo?.StoryboardFile == null) - storyboard = decoder.Decode(stream); - else - { - using (var secondaryStream = new LineBufferedReader(GetStream(BeatmapSetInfo.GetPathForFile(BeatmapSetInfo.StoryboardFile)))) - storyboard = decoder.Decode(stream, secondaryStream); - } - } - } - catch (Exception e) - { - Logger.Error(e, "Storyboard failed to load"); - storyboard = new Storyboard(); - } - - storyboard.BeatmapInfo = BeatmapInfo; - - return storyboard; - } - - protected internal override ISkin GetSkin() - { - try - { - return new LegacyBeatmapSkin(BeatmapInfo, resources.Files, resources); - } - catch (Exception e) - { - Logger.Error(e, "Skin failed to load"); - return null; - } - } - - public override Stream GetStream(string storagePath) => resources.Files.GetStream(storagePath); - } - } -} diff --git a/osu.Game/Beatmaps/WorkingBeatmapCache.cs b/osu.Game/Beatmaps/WorkingBeatmapCache.cs new file mode 100644 index 0000000000..9f40eb4898 --- /dev/null +++ b/osu.Game/Beatmaps/WorkingBeatmapCache.cs @@ -0,0 +1,279 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using JetBrains.Annotations; +using osu.Framework.Audio; +using osu.Framework.Audio.Track; +using osu.Framework.Graphics.Textures; +using osu.Framework.IO.Stores; +using osu.Framework.Lists; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Framework.Statistics; +using osu.Framework.Testing; +using osu.Game.Beatmaps.Formats; +using osu.Game.IO; +using osu.Game.Rulesets; +using osu.Game.Skinning; +using osu.Game.Storyboards; +using osu.Game.Users; + +namespace osu.Game.Beatmaps +{ + public class WorkingBeatmapCache : IBeatmapResourceProvider + { + private readonly WeakList workingCache = new WeakList(); + + /// + /// A default representation of a WorkingBeatmap to use when no beatmap is available. + /// + public readonly WorkingBeatmap DefaultBeatmap; + + public BeatmapManager BeatmapManager { private get; set; } + + private readonly AudioManager audioManager; + private readonly IResourceStore resources; + private readonly LargeTextureStore largeTextureStore; + private readonly ITrackStore trackStore; + private readonly IResourceStore files; + + [CanBeNull] + private readonly GameHost host; + + public WorkingBeatmapCache([NotNull] AudioManager audioManager, IResourceStore resources, IResourceStore files, WorkingBeatmap defaultBeatmap = null, GameHost host = null) + { + DefaultBeatmap = defaultBeatmap; + + this.audioManager = audioManager; + this.resources = resources; + this.host = host; + this.files = files; + largeTextureStore = new LargeTextureStore(host?.CreateTextureLoaderStore(files)); + trackStore = audioManager.GetTrackStore(files); + } + + public void Invalidate(BeatmapSetInfo info) + { + if (info.Beatmaps == null) return; + + foreach (var b in info.Beatmaps) + Invalidate(b); + } + + public void Invalidate(BeatmapInfo info) + { + lock (workingCache) + { + var working = workingCache.FirstOrDefault(w => w.BeatmapInfo?.ID == info.ID); + if (working != null) + workingCache.Remove(working); + } + } + + /// + /// Create a new . + /// + public WorkingBeatmap CreateNew(RulesetInfo ruleset, User user) + { + var metadata = new BeatmapMetadata + { + Author = user, + }; + + var set = new BeatmapSetInfo + { + Metadata = metadata, + Beatmaps = new List + { + new BeatmapInfo + { + BaseDifficulty = new BeatmapDifficulty(), + Ruleset = ruleset, + Metadata = metadata, + WidescreenStoryboard = true, + SamplesMatchPlaybackRate = true, + } + } + }; + + var working = BeatmapManager.Import(set).Result; + return GetWorkingBeatmap(working.Beatmaps.First()); + } + + /// + /// Retrieve a instance for the provided + /// + /// The beatmap to lookup. + /// A instance correlating to the provided . + public virtual WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo) + { + // if there are no files, presume the full beatmap info has not yet been fetched from the database. + if (beatmapInfo?.BeatmapSet?.Files.Count == 0) + { + int lookupId = beatmapInfo.ID; + beatmapInfo = BeatmapManager.QueryBeatmap(b => b.ID == lookupId); + } + + if (beatmapInfo?.BeatmapSet == null) + return DefaultBeatmap; + + lock (workingCache) + { + var working = workingCache.FirstOrDefault(w => w.BeatmapInfo?.ID == beatmapInfo.ID); + if (working != null) + return working; + + beatmapInfo.Metadata ??= beatmapInfo.BeatmapSet.Metadata; + + workingCache.Add(working = new BeatmapManagerWorkingBeatmap(beatmapInfo, this)); + + // best effort; may be higher than expected. + GlobalStatistics.Get(nameof(Beatmaps), $"Cached {nameof(WorkingBeatmap)}s").Value = workingCache.Count(); + + return working; + } + } + + #region IResourceStorageProvider + + TextureStore IBeatmapResourceProvider.LargeTextureStore => largeTextureStore; + ITrackStore IBeatmapResourceProvider.Tracks => trackStore; + AudioManager IStorageResourceProvider.AudioManager => audioManager; + IResourceStore IStorageResourceProvider.Files => files; + IResourceStore IStorageResourceProvider.Resources => resources; + IResourceStore IStorageResourceProvider.CreateTextureLoaderStore(IResourceStore underlyingStore) => host?.CreateTextureLoaderStore(underlyingStore); + + #endregion + + [ExcludeFromDynamicCompile] + private class BeatmapManagerWorkingBeatmap : WorkingBeatmap + { + [NotNull] + private readonly IBeatmapResourceProvider resources; + + public BeatmapManagerWorkingBeatmap(BeatmapInfo beatmapInfo, [NotNull] IBeatmapResourceProvider resources) + : base(beatmapInfo, resources.AudioManager) + { + this.resources = resources; + } + + protected override IBeatmap GetBeatmap() + { + if (BeatmapInfo.Path == null) + return new Beatmap { BeatmapInfo = BeatmapInfo }; + + try + { + using (var stream = new LineBufferedReader(GetStream(BeatmapSetInfo.GetPathForFile(BeatmapInfo.Path)))) + return Decoder.GetDecoder(stream).Decode(stream); + } + catch (Exception e) + { + Logger.Error(e, "Beatmap failed to load"); + return null; + } + } + + protected override bool BackgroundStillValid(Texture b) => false; // bypass lazy logic. we want to return a new background each time for refcounting purposes. + + protected override Texture GetBackground() + { + if (Metadata?.BackgroundFile == null) + return null; + + try + { + return resources.LargeTextureStore.Get(BeatmapSetInfo.GetPathForFile(Metadata.BackgroundFile)); + } + catch (Exception e) + { + Logger.Error(e, "Background failed to load"); + return null; + } + } + + protected override Track GetBeatmapTrack() + { + if (Metadata?.AudioFile == null) + return null; + + try + { + return resources.Tracks.Get(BeatmapSetInfo.GetPathForFile(Metadata.AudioFile)); + } + catch (Exception e) + { + Logger.Error(e, "Track failed to load"); + return null; + } + } + + protected override Waveform GetWaveform() + { + if (Metadata?.AudioFile == null) + return null; + + try + { + var trackData = GetStream(BeatmapSetInfo.GetPathForFile(Metadata.AudioFile)); + return trackData == null ? null : new Waveform(trackData); + } + catch (Exception e) + { + Logger.Error(e, "Waveform failed to load"); + return null; + } + } + + protected override Storyboard GetStoryboard() + { + Storyboard storyboard; + + try + { + using (var stream = new LineBufferedReader(GetStream(BeatmapSetInfo.GetPathForFile(BeatmapInfo.Path)))) + { + var decoder = Decoder.GetDecoder(stream); + + // todo: support loading from both set-wide storyboard *and* beatmap specific. + if (BeatmapSetInfo?.StoryboardFile == null) + storyboard = decoder.Decode(stream); + else + { + using (var secondaryStream = new LineBufferedReader(GetStream(BeatmapSetInfo.GetPathForFile(BeatmapSetInfo.StoryboardFile)))) + storyboard = decoder.Decode(stream, secondaryStream); + } + } + } + catch (Exception e) + { + Logger.Error(e, "Storyboard failed to load"); + storyboard = new Storyboard(); + } + + storyboard.BeatmapInfo = BeatmapInfo; + + return storyboard; + } + + protected internal override ISkin GetSkin() + { + try + { + return new LegacyBeatmapSkin(BeatmapInfo, resources.Files, resources); + } + catch (Exception e) + { + Logger.Error(e, "Skin failed to load"); + return null; + } + } + + public override Stream GetStream(string storagePath) => resources.Files.GetStream(storagePath); + } + } +} From d21139b03efb77e7f5aeeea2d8236320d0e0d693 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 15:43:49 +0900 Subject: [PATCH 04/16] Split out database portion from `BeatmapManager` --- osu.Game/Beatmaps/BeatmapManager.cs | 466 +--------------------- osu.Game/Beatmaps/BeatmapModelManager.cs | 479 +++++++++++++++++++++++ 2 files changed, 483 insertions(+), 462 deletions(-) create mode 100644 osu.Game/Beatmaps/BeatmapModelManager.cs diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 1c0e7dc319..c445925a90 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -1,479 +1,21 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Linq.Expressions; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.EntityFrameworkCore; -using osu.Framework.Audio.Track; -using osu.Framework.Bindables; -using osu.Framework.Extensions; -using osu.Framework.Graphics.Textures; -using osu.Framework.Logging; -using osu.Framework.Platform; using osu.Framework.Testing; -using osu.Game.Beatmaps.Formats; -using osu.Game.Database; -using osu.Game.IO; -using osu.Game.IO.Archives; -using osu.Game.Online.API; -using osu.Game.Online.API.Requests; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Objects; -using osu.Game.Skinning; -using Decoder = osu.Game.Beatmaps.Formats.Decoder; namespace osu.Game.Beatmaps { /// - /// Handles the storage and retrieval of Beatmaps/WorkingBeatmaps. + /// Handles general operations related to global beatmap management. /// [ExcludeFromDynamicCompile] - public class BeatmapManager : DownloadableArchiveModelManager + public class BeatmapManager { - /// - /// Fired when a single difficulty has been hidden. - /// - public IBindable> BeatmapHidden => beatmapHidden; - - private readonly Bindable> beatmapHidden = new Bindable>(); - - /// - /// Fired when a single difficulty has been restored. - /// - public IBindable> BeatmapRestored => beatmapRestored; - - /// - /// A function which populates online information during the import process. - /// It is run as the final step of import. - /// - public Func PopulateOnlineInformation; - - /// - /// The game working beatmap cache, used to invalidate entries on changes. - /// - public WorkingBeatmapCache WorkingBeatmapCache { private get; set; } - - private readonly Bindable> beatmapRestored = new Bindable>(); - - public override IEnumerable HandledExtensions => new[] { ".osz" }; - - protected override string[] HashableFileTypes => new[] { ".osu" }; - - protected override string ImportFromStablePath => "."; - - protected override Storage PrepareStableStorage(StableStorage stableStorage) => stableStorage.GetSongStorage(); - - private readonly BeatmapStore beatmaps; - private readonly RulesetStore rulesets; - - public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host = null) - : base(storage, contextFactory, api, new BeatmapStore(contextFactory), host) + public BeatmapManager() { - this.rulesets = rulesets; - - beatmaps = (BeatmapStore)ModelStore; - beatmaps.BeatmapHidden += b => beatmapHidden.Value = new WeakReference(b); - beatmaps.BeatmapRestored += b => beatmapRestored.Value = new WeakReference(b); - beatmaps.ItemRemoved += b => WorkingBeatmapCache?.Invalidate(b); - beatmaps.ItemUpdated += obj => WorkingBeatmapCache?.Invalidate(obj); + beatmapModelManager = new BeatmapModelManager() } - protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) => - new DownloadBeatmapSetRequest(set, minimiseDownloadSize); - - protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path)?.ToLowerInvariant() == ".osz"; - - protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) - { - if (archive != null) - beatmapSet.Beatmaps = createBeatmapDifficulties(beatmapSet.Files); - - foreach (BeatmapInfo b in beatmapSet.Beatmaps) - { - // remove metadata from difficulties where it matches the set - if (beatmapSet.Metadata.Equals(b.Metadata)) - b.Metadata = null; - - b.BeatmapSet = beatmapSet; - } - - validateOnlineIds(beatmapSet); - - bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0); - - if (PopulateOnlineInformation != null) - await PopulateOnlineInformation(beatmapSet, cancellationToken).ConfigureAwait(false); - - // ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID. - if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0)) - { - if (beatmapSet.OnlineBeatmapSetID != null) - { - beatmapSet.OnlineBeatmapSetID = null; - LogForModel(beatmapSet, "Disassociating beatmap set ID due to loss of all beatmap IDs"); - } - } - } - - protected override void PreImport(BeatmapSetInfo beatmapSet) - { - if (beatmapSet.Beatmaps.Any(b => b.BaseDifficulty == null)) - throw new InvalidOperationException($"Cannot import {nameof(BeatmapInfo)} with null {nameof(BeatmapInfo.BaseDifficulty)}."); - - // check if a set already exists with the same online id, delete if it does. - if (beatmapSet.OnlineBeatmapSetID != null) - { - var existingOnlineId = beatmaps.ConsumableItems.FirstOrDefault(b => b.OnlineBeatmapSetID == beatmapSet.OnlineBeatmapSetID); - - if (existingOnlineId != null) - { - Delete(existingOnlineId); - - // in order to avoid a unique key constraint, immediately remove the online ID from the previous set. - existingOnlineId.OnlineBeatmapSetID = null; - foreach (var b in existingOnlineId.Beatmaps) - b.OnlineBeatmapID = null; - - LogForModel(beatmapSet, $"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been deleted."); - } - } - } - - private void validateOnlineIds(BeatmapSetInfo beatmapSet) - { - var beatmapIds = beatmapSet.Beatmaps.Where(b => b.OnlineBeatmapID.HasValue).Select(b => b.OnlineBeatmapID).ToList(); - - // ensure all IDs are unique - if (beatmapIds.GroupBy(b => b).Any(g => g.Count() > 1)) - { - LogForModel(beatmapSet, "Found non-unique IDs, resetting..."); - resetIds(); - return; - } - - // find any existing beatmaps in the database that have matching online ids - var existingBeatmaps = QueryBeatmaps(b => beatmapIds.Contains(b.OnlineBeatmapID)).ToList(); - - if (existingBeatmaps.Count > 0) - { - // reset the import ids (to force a re-fetch) *unless* they match the candidate CheckForExisting set. - // we can ignore the case where the new ids are contained by the CheckForExisting set as it will either be used (import skipped) or deleted. - var existing = CheckForExisting(beatmapSet); - - if (existing == null || existingBeatmaps.Any(b => !existing.Beatmaps.Contains(b))) - { - LogForModel(beatmapSet, "Found existing import with IDs already, resetting..."); - resetIds(); - } - } - - void resetIds() => beatmapSet.Beatmaps.ForEach(b => b.OnlineBeatmapID = null); - } - - protected override bool CheckLocalAvailability(BeatmapSetInfo model, IQueryable items) - => base.CheckLocalAvailability(model, items) - || (model.OnlineBeatmapSetID != null && items.Any(b => b.OnlineBeatmapSetID == model.OnlineBeatmapSetID)); - - /// - /// Delete a beatmap difficulty. - /// - /// The beatmap difficulty to hide. - public void Hide(BeatmapInfo beatmap) => beatmaps.Hide(beatmap); - - /// - /// Restore a beatmap difficulty. - /// - /// The beatmap difficulty to restore. - public void Restore(BeatmapInfo beatmap) => beatmaps.Restore(beatmap); - - /// - /// Saves an file against a given . - /// - /// The to save the content against. The file referenced by will be replaced. - /// The content to write. - /// The beatmap content to write, null if to be omitted. - public virtual void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null) - { - var setInfo = info.BeatmapSet; - - using (var stream = new MemoryStream()) - { - using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true)) - new LegacyBeatmapEncoder(beatmapContent, beatmapSkin).Encode(sw); - - stream.Seek(0, SeekOrigin.Begin); - - using (ContextFactory.GetForWrite()) - { - var beatmapInfo = setInfo.Beatmaps.Single(b => b.ID == info.ID); - var metadata = beatmapInfo.Metadata ?? setInfo.Metadata; - - // grab the original file (or create a new one if not found). - var fileInfo = setInfo.Files.SingleOrDefault(f => string.Equals(f.Filename, beatmapInfo.Path, StringComparison.OrdinalIgnoreCase)) ?? new BeatmapSetFileInfo(); - - // metadata may have changed; update the path with the standard format. - beatmapInfo.Path = $"{metadata.Artist} - {metadata.Title} ({metadata.Author}) [{beatmapInfo.Version}].osu"; - beatmapInfo.MD5Hash = stream.ComputeMD5Hash(); - - // update existing or populate new file's filename. - fileInfo.Filename = beatmapInfo.Path; - - stream.Seek(0, SeekOrigin.Begin); - ReplaceFile(setInfo, fileInfo, stream); - } - } - - WorkingBeatmapCache?.Invalidate(info); - } - - /// - /// Perform a lookup query on available s. - /// - /// The query. - /// The first result for the provided query, or null if no results were found. - public BeatmapSetInfo QueryBeatmapSet(Expression> query) => beatmaps.ConsumableItems.AsNoTracking().FirstOrDefault(query); - - protected override bool CanSkipImport(BeatmapSetInfo existing, BeatmapSetInfo import) - { - if (!base.CanSkipImport(existing, import)) - return false; - - return existing.Beatmaps.Any(b => b.OnlineBeatmapID != null); - } - - protected override bool CanReuseExisting(BeatmapSetInfo existing, BeatmapSetInfo import) - { - if (!base.CanReuseExisting(existing, import)) - return false; - - var existingIds = existing.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); - var importIds = import.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); - - // force re-import if we are not in a sane state. - return existing.OnlineBeatmapSetID == import.OnlineBeatmapSetID && existingIds.SequenceEqual(importIds); - } - - /// - /// Returns a list of all usable s. - /// - /// A list of available . - public List GetAllUsableBeatmapSets(IncludedDetails includes = IncludedDetails.All, bool includeProtected = false) => - GetAllUsableBeatmapSetsEnumerable(includes, includeProtected).ToList(); - - /// - /// Returns a list of all usable s. Note that files are not populated. - /// - /// The level of detail to include in the returned objects. - /// Whether to include protected (system) beatmaps. These should not be included for gameplay playable use cases. - /// A list of available . - public IEnumerable GetAllUsableBeatmapSetsEnumerable(IncludedDetails includes, bool includeProtected = false) - { - IQueryable queryable; - - switch (includes) - { - case IncludedDetails.Minimal: - queryable = beatmaps.BeatmapSetsOverview; - break; - - case IncludedDetails.AllButRuleset: - queryable = beatmaps.BeatmapSetsWithoutRuleset; - break; - - case IncludedDetails.AllButFiles: - queryable = beatmaps.BeatmapSetsWithoutFiles; - break; - - default: - queryable = beatmaps.ConsumableItems; - break; - } - - // AsEnumerable used here to avoid applying the WHERE in sql. When done so, ef core 2.x uses an incorrect ORDER BY - // clause which causes queries to take 5-10x longer. - // TODO: remove if upgrading to EF core 3.x. - return queryable.AsEnumerable().Where(s => !s.DeletePending && (includeProtected || !s.Protected)); - } - - /// - /// Perform a lookup query on available s. - /// - /// The query. - /// The level of detail to include in the returned objects. - /// Results from the provided query. - public IEnumerable QueryBeatmapSets(Expression> query, IncludedDetails includes = IncludedDetails.All) - { - IQueryable queryable; - - switch (includes) - { - case IncludedDetails.Minimal: - queryable = beatmaps.BeatmapSetsOverview; - break; - - case IncludedDetails.AllButRuleset: - queryable = beatmaps.BeatmapSetsWithoutRuleset; - break; - - case IncludedDetails.AllButFiles: - queryable = beatmaps.BeatmapSetsWithoutFiles; - break; - - default: - queryable = beatmaps.ConsumableItems; - break; - } - - return queryable.AsNoTracking().Where(query); - } - - /// - /// Perform a lookup query on available s. - /// - /// The query. - /// The first result for the provided query, or null if no results were found. - public BeatmapInfo QueryBeatmap(Expression> query) => beatmaps.Beatmaps.AsNoTracking().FirstOrDefault(query); - - /// - /// Perform a lookup query on available s. - /// - /// The query. - /// Results from the provided query. - public IQueryable QueryBeatmaps(Expression> query) => beatmaps.Beatmaps.AsNoTracking().Where(query); - - protected override string HumanisedModelName => "beatmap"; - - protected override BeatmapSetInfo CreateModel(ArchiveReader reader) - { - // let's make sure there are actually .osu files to import. - string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu", StringComparison.OrdinalIgnoreCase)); - - if (string.IsNullOrEmpty(mapName)) - { - Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database); - return null; - } - - Beatmap beatmap; - using (var stream = new LineBufferedReader(reader.GetStream(mapName))) - beatmap = Decoder.GetDecoder(stream).Decode(stream); - - return new BeatmapSetInfo - { - OnlineBeatmapSetID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID, - Beatmaps = new List(), - Metadata = beatmap.Metadata, - DateAdded = DateTimeOffset.UtcNow - }; - } - - /// - /// Create all required s for the provided archive. - /// - private List createBeatmapDifficulties(List files) - { - var beatmapInfos = new List(); - - foreach (var file in files.Where(f => f.Filename.EndsWith(".osu", StringComparison.OrdinalIgnoreCase))) - { - using (var raw = Files.Store.GetStream(file.FileInfo.StoragePath)) - using (var ms = new MemoryStream()) // we need a memory stream so we can seek - using (var sr = new LineBufferedReader(ms)) - { - raw.CopyTo(ms); - ms.Position = 0; - - var decoder = Decoder.GetDecoder(sr); - IBeatmap beatmap = decoder.Decode(sr); - - string hash = ms.ComputeSHA2Hash(); - - if (beatmapInfos.Any(b => b.Hash == hash)) - continue; - - beatmap.BeatmapInfo.Path = file.Filename; - beatmap.BeatmapInfo.Hash = hash; - beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash(); - - var ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID); - beatmap.BeatmapInfo.Ruleset = ruleset; - - // TODO: this should be done in a better place once we actually need to dynamically update it. - beatmap.BeatmapInfo.StarDifficulty = ruleset?.CreateInstance().CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating ?? 0; - beatmap.BeatmapInfo.Length = calculateLength(beatmap); - beatmap.BeatmapInfo.BPM = 60000 / beatmap.GetMostCommonBeatLength(); - - beatmapInfos.Add(beatmap.BeatmapInfo); - } - } - - return beatmapInfos; - } - - private double calculateLength(IBeatmap b) - { - if (!b.HitObjects.Any()) - return 0; - - var lastObject = b.HitObjects.Last(); - - //TODO: this isn't always correct (consider mania where a non-last object may last for longer than the last in the list). - double endTime = lastObject.GetEndTime(); - double startTime = b.HitObjects.First().StartTime; - - return endTime - startTime; - } - - /// - /// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation. - /// - private class DummyConversionBeatmap : WorkingBeatmap - { - private readonly IBeatmap beatmap; - - public DummyConversionBeatmap(IBeatmap beatmap) - : base(beatmap.BeatmapInfo, null) - { - this.beatmap = beatmap; - } - - protected override IBeatmap GetBeatmap() => beatmap; - protected override Texture GetBackground() => null; - protected override Track GetBeatmapTrack() => null; - protected internal override ISkin GetSkin() => null; - public override Stream GetStream(string storagePath) => null; - } } - /// - /// The level of detail to include in database results. - /// - public enum IncludedDetails - { - /// - /// Only include beatmap difficulties and set level metadata. - /// - Minimal, - - /// - /// Include all difficulties, rulesets, difficulty metadata but no files. - /// - AllButFiles, - - /// - /// Include everything except ruleset. Used for cases where we aren't sure the ruleset is present but still want to consume the beatmap. - /// - AllButRuleset, - - /// - /// Include everything. - /// - All - } } diff --git a/osu.Game/Beatmaps/BeatmapModelManager.cs b/osu.Game/Beatmaps/BeatmapModelManager.cs new file mode 100644 index 0000000000..be3adc412c --- /dev/null +++ b/osu.Game/Beatmaps/BeatmapModelManager.cs @@ -0,0 +1,479 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using osu.Framework.Audio.Track; +using osu.Framework.Bindables; +using osu.Framework.Extensions; +using osu.Framework.Graphics.Textures; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Framework.Testing; +using osu.Game.Beatmaps.Formats; +using osu.Game.Database; +using osu.Game.IO; +using osu.Game.IO.Archives; +using osu.Game.Online.API; +using osu.Game.Online.API.Requests; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Objects; +using osu.Game.Skinning; +using Decoder = osu.Game.Beatmaps.Formats.Decoder; + +namespace osu.Game.Beatmaps +{ + /// + /// Handles ef-core storage of beatmaps. + /// + [ExcludeFromDynamicCompile] + public class BeatmapModelManager : DownloadableArchiveModelManager + { + /// + /// Fired when a single difficulty has been hidden. + /// + public IBindable> BeatmapHidden => beatmapHidden; + + private readonly Bindable> beatmapHidden = new Bindable>(); + + /// + /// Fired when a single difficulty has been restored. + /// + public IBindable> BeatmapRestored => beatmapRestored; + + /// + /// A function which populates online information during the import process. + /// It is run as the final step of import. + /// + public Func PopulateOnlineInformation; + + /// + /// The game working beatmap cache, used to invalidate entries on changes. + /// + public WorkingBeatmapCache WorkingBeatmapCache { private get; set; } + + private readonly Bindable> beatmapRestored = new Bindable>(); + + public override IEnumerable HandledExtensions => new[] { ".osz" }; + + protected override string[] HashableFileTypes => new[] { ".osu" }; + + protected override string ImportFromStablePath => "."; + + protected override Storage PrepareStableStorage(StableStorage stableStorage) => stableStorage.GetSongStorage(); + + private readonly BeatmapStore beatmaps; + private readonly RulesetStore rulesets; + + public BeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host = null) + : base(storage, contextFactory, api, new BeatmapStore(contextFactory), host) + { + this.rulesets = rulesets; + + beatmaps = (BeatmapStore)ModelStore; + beatmaps.BeatmapHidden += b => beatmapHidden.Value = new WeakReference(b); + beatmaps.BeatmapRestored += b => beatmapRestored.Value = new WeakReference(b); + beatmaps.ItemRemoved += b => WorkingBeatmapCache?.Invalidate(b); + beatmaps.ItemUpdated += obj => WorkingBeatmapCache?.Invalidate(obj); + } + + protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) => + new DownloadBeatmapSetRequest(set, minimiseDownloadSize); + + protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path)?.ToLowerInvariant() == ".osz"; + + protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) + { + if (archive != null) + beatmapSet.Beatmaps = createBeatmapDifficulties(beatmapSet.Files); + + foreach (BeatmapInfo b in beatmapSet.Beatmaps) + { + // remove metadata from difficulties where it matches the set + if (beatmapSet.Metadata.Equals(b.Metadata)) + b.Metadata = null; + + b.BeatmapSet = beatmapSet; + } + + validateOnlineIds(beatmapSet); + + bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0); + + if (PopulateOnlineInformation != null) + await PopulateOnlineInformation(beatmapSet, cancellationToken).ConfigureAwait(false); + + // ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID. + if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0)) + { + if (beatmapSet.OnlineBeatmapSetID != null) + { + beatmapSet.OnlineBeatmapSetID = null; + LogForModel(beatmapSet, "Disassociating beatmap set ID due to loss of all beatmap IDs"); + } + } + } + + protected override void PreImport(BeatmapSetInfo beatmapSet) + { + if (beatmapSet.Beatmaps.Any(b => b.BaseDifficulty == null)) + throw new InvalidOperationException($"Cannot import {nameof(BeatmapInfo)} with null {nameof(BeatmapInfo.BaseDifficulty)}."); + + // check if a set already exists with the same online id, delete if it does. + if (beatmapSet.OnlineBeatmapSetID != null) + { + var existingOnlineId = beatmaps.ConsumableItems.FirstOrDefault(b => b.OnlineBeatmapSetID == beatmapSet.OnlineBeatmapSetID); + + if (existingOnlineId != null) + { + Delete(existingOnlineId); + + // in order to avoid a unique key constraint, immediately remove the online ID from the previous set. + existingOnlineId.OnlineBeatmapSetID = null; + foreach (var b in existingOnlineId.Beatmaps) + b.OnlineBeatmapID = null; + + LogForModel(beatmapSet, $"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been deleted."); + } + } + } + + private void validateOnlineIds(BeatmapSetInfo beatmapSet) + { + var beatmapIds = beatmapSet.Beatmaps.Where(b => b.OnlineBeatmapID.HasValue).Select(b => b.OnlineBeatmapID).ToList(); + + // ensure all IDs are unique + if (beatmapIds.GroupBy(b => b).Any(g => g.Count() > 1)) + { + LogForModel(beatmapSet, "Found non-unique IDs, resetting..."); + resetIds(); + return; + } + + // find any existing beatmaps in the database that have matching online ids + var existingBeatmaps = QueryBeatmaps(b => beatmapIds.Contains(b.OnlineBeatmapID)).ToList(); + + if (existingBeatmaps.Count > 0) + { + // reset the import ids (to force a re-fetch) *unless* they match the candidate CheckForExisting set. + // we can ignore the case where the new ids are contained by the CheckForExisting set as it will either be used (import skipped) or deleted. + var existing = CheckForExisting(beatmapSet); + + if (existing == null || existingBeatmaps.Any(b => !existing.Beatmaps.Contains(b))) + { + LogForModel(beatmapSet, "Found existing import with IDs already, resetting..."); + resetIds(); + } + } + + void resetIds() => beatmapSet.Beatmaps.ForEach(b => b.OnlineBeatmapID = null); + } + + protected override bool CheckLocalAvailability(BeatmapSetInfo model, IQueryable items) + => base.CheckLocalAvailability(model, items) + || (model.OnlineBeatmapSetID != null && items.Any(b => b.OnlineBeatmapSetID == model.OnlineBeatmapSetID)); + + /// + /// Delete a beatmap difficulty. + /// + /// The beatmap difficulty to hide. + public void Hide(BeatmapInfo beatmap) => beatmaps.Hide(beatmap); + + /// + /// Restore a beatmap difficulty. + /// + /// The beatmap difficulty to restore. + public void Restore(BeatmapInfo beatmap) => beatmaps.Restore(beatmap); + + /// + /// Saves an file against a given . + /// + /// The to save the content against. The file referenced by will be replaced. + /// The content to write. + /// The beatmap content to write, null if to be omitted. + public virtual void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null) + { + var setInfo = info.BeatmapSet; + + using (var stream = new MemoryStream()) + { + using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true)) + new LegacyBeatmapEncoder(beatmapContent, beatmapSkin).Encode(sw); + + stream.Seek(0, SeekOrigin.Begin); + + using (ContextFactory.GetForWrite()) + { + var beatmapInfo = setInfo.Beatmaps.Single(b => b.ID == info.ID); + var metadata = beatmapInfo.Metadata ?? setInfo.Metadata; + + // grab the original file (or create a new one if not found). + var fileInfo = setInfo.Files.SingleOrDefault(f => string.Equals(f.Filename, beatmapInfo.Path, StringComparison.OrdinalIgnoreCase)) ?? new BeatmapSetFileInfo(); + + // metadata may have changed; update the path with the standard format. + beatmapInfo.Path = $"{metadata.Artist} - {metadata.Title} ({metadata.Author}) [{beatmapInfo.Version}].osu"; + beatmapInfo.MD5Hash = stream.ComputeMD5Hash(); + + // update existing or populate new file's filename. + fileInfo.Filename = beatmapInfo.Path; + + stream.Seek(0, SeekOrigin.Begin); + ReplaceFile(setInfo, fileInfo, stream); + } + } + + WorkingBeatmapCache?.Invalidate(info); + } + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// The first result for the provided query, or null if no results were found. + public BeatmapSetInfo QueryBeatmapSet(Expression> query) => beatmaps.ConsumableItems.AsNoTracking().FirstOrDefault(query); + + protected override bool CanSkipImport(BeatmapSetInfo existing, BeatmapSetInfo import) + { + if (!base.CanSkipImport(existing, import)) + return false; + + return existing.Beatmaps.Any(b => b.OnlineBeatmapID != null); + } + + protected override bool CanReuseExisting(BeatmapSetInfo existing, BeatmapSetInfo import) + { + if (!base.CanReuseExisting(existing, import)) + return false; + + var existingIds = existing.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); + var importIds = import.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); + + // force re-import if we are not in a sane state. + return existing.OnlineBeatmapSetID == import.OnlineBeatmapSetID && existingIds.SequenceEqual(importIds); + } + + /// + /// Returns a list of all usable s. + /// + /// A list of available . + public List GetAllUsableBeatmapSets(IncludedDetails includes = IncludedDetails.All, bool includeProtected = false) => + GetAllUsableBeatmapSetsEnumerable(includes, includeProtected).ToList(); + + /// + /// Returns a list of all usable s. Note that files are not populated. + /// + /// The level of detail to include in the returned objects. + /// Whether to include protected (system) beatmaps. These should not be included for gameplay playable use cases. + /// A list of available . + public IEnumerable GetAllUsableBeatmapSetsEnumerable(IncludedDetails includes, bool includeProtected = false) + { + IQueryable queryable; + + switch (includes) + { + case IncludedDetails.Minimal: + queryable = beatmaps.BeatmapSetsOverview; + break; + + case IncludedDetails.AllButRuleset: + queryable = beatmaps.BeatmapSetsWithoutRuleset; + break; + + case IncludedDetails.AllButFiles: + queryable = beatmaps.BeatmapSetsWithoutFiles; + break; + + default: + queryable = beatmaps.ConsumableItems; + break; + } + + // AsEnumerable used here to avoid applying the WHERE in sql. When done so, ef core 2.x uses an incorrect ORDER BY + // clause which causes queries to take 5-10x longer. + // TODO: remove if upgrading to EF core 3.x. + return queryable.AsEnumerable().Where(s => !s.DeletePending && (includeProtected || !s.Protected)); + } + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// The level of detail to include in the returned objects. + /// Results from the provided query. + public IEnumerable QueryBeatmapSets(Expression> query, IncludedDetails includes = IncludedDetails.All) + { + IQueryable queryable; + + switch (includes) + { + case IncludedDetails.Minimal: + queryable = beatmaps.BeatmapSetsOverview; + break; + + case IncludedDetails.AllButRuleset: + queryable = beatmaps.BeatmapSetsWithoutRuleset; + break; + + case IncludedDetails.AllButFiles: + queryable = beatmaps.BeatmapSetsWithoutFiles; + break; + + default: + queryable = beatmaps.ConsumableItems; + break; + } + + return queryable.AsNoTracking().Where(query); + } + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// The first result for the provided query, or null if no results were found. + public BeatmapInfo QueryBeatmap(Expression> query) => beatmaps.Beatmaps.AsNoTracking().FirstOrDefault(query); + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// Results from the provided query. + public IQueryable QueryBeatmaps(Expression> query) => beatmaps.Beatmaps.AsNoTracking().Where(query); + + protected override string HumanisedModelName => "beatmap"; + + protected override BeatmapSetInfo CreateModel(ArchiveReader reader) + { + // let's make sure there are actually .osu files to import. + string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu", StringComparison.OrdinalIgnoreCase)); + + if (string.IsNullOrEmpty(mapName)) + { + Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database); + return null; + } + + Beatmap beatmap; + using (var stream = new LineBufferedReader(reader.GetStream(mapName))) + beatmap = Decoder.GetDecoder(stream).Decode(stream); + + return new BeatmapSetInfo + { + OnlineBeatmapSetID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID, + Beatmaps = new List(), + Metadata = beatmap.Metadata, + DateAdded = DateTimeOffset.UtcNow + }; + } + + /// + /// Create all required s for the provided archive. + /// + private List createBeatmapDifficulties(List files) + { + var beatmapInfos = new List(); + + foreach (var file in files.Where(f => f.Filename.EndsWith(".osu", StringComparison.OrdinalIgnoreCase))) + { + using (var raw = Files.Store.GetStream(file.FileInfo.StoragePath)) + using (var ms = new MemoryStream()) // we need a memory stream so we can seek + using (var sr = new LineBufferedReader(ms)) + { + raw.CopyTo(ms); + ms.Position = 0; + + var decoder = Decoder.GetDecoder(sr); + IBeatmap beatmap = decoder.Decode(sr); + + string hash = ms.ComputeSHA2Hash(); + + if (beatmapInfos.Any(b => b.Hash == hash)) + continue; + + beatmap.BeatmapInfo.Path = file.Filename; + beatmap.BeatmapInfo.Hash = hash; + beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash(); + + var ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID); + beatmap.BeatmapInfo.Ruleset = ruleset; + + // TODO: this should be done in a better place once we actually need to dynamically update it. + beatmap.BeatmapInfo.StarDifficulty = ruleset?.CreateInstance().CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating ?? 0; + beatmap.BeatmapInfo.Length = calculateLength(beatmap); + beatmap.BeatmapInfo.BPM = 60000 / beatmap.GetMostCommonBeatLength(); + + beatmapInfos.Add(beatmap.BeatmapInfo); + } + } + + return beatmapInfos; + } + + private double calculateLength(IBeatmap b) + { + if (!b.HitObjects.Any()) + return 0; + + var lastObject = b.HitObjects.Last(); + + //TODO: this isn't always correct (consider mania where a non-last object may last for longer than the last in the list). + double endTime = lastObject.GetEndTime(); + double startTime = b.HitObjects.First().StartTime; + + return endTime - startTime; + } + + /// + /// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation. + /// + private class DummyConversionBeatmap : WorkingBeatmap + { + private readonly IBeatmap beatmap; + + public DummyConversionBeatmap(IBeatmap beatmap) + : base(beatmap.BeatmapInfo, null) + { + this.beatmap = beatmap; + } + + protected override IBeatmap GetBeatmap() => beatmap; + protected override Texture GetBackground() => null; + protected override Track GetBeatmapTrack() => null; + protected internal override ISkin GetSkin() => null; + public override Stream GetStream(string storagePath) => null; + } + } + + /// + /// The level of detail to include in database results. + /// + public enum IncludedDetails + { + /// + /// Only include beatmap difficulties and set level metadata. + /// + Minimal, + + /// + /// Include all difficulties, rulesets, difficulty metadata but no files. + /// + AllButFiles, + + /// + /// Include everything except ruleset. Used for cases where we aren't sure the ruleset is present but still want to consume the beatmap. + /// + AllButRuleset, + + /// + /// Include everything. + /// + All + } +} From 5618c9933bfd61b5587160a1821248bf7b1fb214 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 16:44:39 +0900 Subject: [PATCH 05/16] Expose more pieces of `ArchiveModelManager` via interfaces --- osu.Game/Database/ArchiveModelManager.cs | 11 +- .../DownloadableArchiveModelManager.cs | 6 -- osu.Game/Database/IModelFileManager.cs | 36 +++++++ osu.Game/Database/IModelManager.cs | 101 +++++++++++++++++- osu.Game/Scoring/ScoreManager.cs | 2 +- 5 files changed, 141 insertions(+), 15 deletions(-) create mode 100644 osu.Game/Database/IModelFileManager.cs diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index ddd2bc5d1e..fc217d3058 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -30,7 +30,7 @@ namespace osu.Game.Database /// /// The model type. /// The associated file join type. - public abstract class ArchiveModelManager : ICanAcceptFiles, IModelManager + public abstract class ArchiveModelManager : ICanAcceptFiles, IModelManager, IModelFileManager where TModel : class, IHasFiles, IHasPrimaryKey, ISoftDelete where TFileModel : class, INamedFileInfo, new() { @@ -135,7 +135,7 @@ namespace osu.Game.Database return Import(notification, tasks); } - protected async Task> Import(ProgressNotification notification, params ImportTask[] tasks) + public async Task> Import(ProgressNotification notification, params ImportTask[] tasks) { if (tasks.Length == 0) { @@ -227,7 +227,7 @@ namespace osu.Game.Database /// Whether this is a low priority import. /// An optional cancellation token. /// The imported model, if successful. - internal async Task Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default) + public async Task Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -479,7 +479,7 @@ namespace osu.Game.Database /// /// The item to export. /// The output stream to export to. - protected virtual void ExportModelTo(TModel model, Stream outputStream) + public virtual void ExportModelTo(TModel model, Stream outputStream) { using (var archive = ZipArchive.Create()) { @@ -745,9 +745,6 @@ namespace osu.Game.Database /// Whether to perform deletion. protected virtual bool ShouldDeleteArchive(string path) => false; - /// - /// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future. - /// public Task ImportFromStableAsync(StableStorage stableStorage) { var storage = PrepareStableStorage(stableStorage); diff --git a/osu.Game/Database/DownloadableArchiveModelManager.cs b/osu.Game/Database/DownloadableArchiveModelManager.cs index da3144e8d0..e6d5b44b65 100644 --- a/osu.Game/Database/DownloadableArchiveModelManager.cs +++ b/osu.Game/Database/DownloadableArchiveModelManager.cs @@ -54,12 +54,6 @@ namespace osu.Game.Database /// The request object. protected abstract ArchiveDownloadRequest CreateDownloadRequest(TModel model, bool minimiseDownloadSize); - /// - /// Begin a download for the requested . - /// - /// The to be downloaded. - /// Whether this download should be optimised for slow connections. Generally means extras are not included in the download bundle. - /// Whether the download was started. public bool Download(TModel model, bool minimiseDownloadSize = false) { if (!canDownload(model)) return false; diff --git a/osu.Game/Database/IModelFileManager.cs b/osu.Game/Database/IModelFileManager.cs new file mode 100644 index 0000000000..c74b945eb7 --- /dev/null +++ b/osu.Game/Database/IModelFileManager.cs @@ -0,0 +1,36 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.IO; + +namespace osu.Game.Database +{ + public interface IModelFileManager + where TModel : class + where TFileModel : class + { + /// + /// Replace an existing file with a new version. + /// + /// The item to operate on. + /// The existing file to be replaced. + /// The new file contents. + /// An optional filename for the new file. Will use the previous filename if not specified. + void ReplaceFile(TModel model, TFileModel file, Stream contents, string filename = null); + + /// + /// Delete an existing file. + /// + /// The item to operate on. + /// The existing file to be deleted. + void DeleteFile(TModel model, TFileModel file); + + /// + /// Add a new file. + /// + /// The item to operate on. + /// The new file contents. + /// The filename for the new file. + void AddFile(TModel model, Stream contents, string filename); + } +} diff --git a/osu.Game/Database/IModelManager.cs b/osu.Game/Database/IModelManager.cs index 8c314f1617..8f0c6e1561 100644 --- a/osu.Game/Database/IModelManager.cs +++ b/osu.Game/Database/IModelManager.cs @@ -1,8 +1,15 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; using osu.Framework.Bindables; +using osu.Game.IO; +using osu.Game.IO.Archives; +using osu.Game.Overlays.Notifications; namespace osu.Game.Database { @@ -24,5 +31,97 @@ namespace osu.Game.Database /// This is not thread-safe and should be scheduled locally if consumed from a drawable component. /// IBindable> ItemRemoved { get; } + + /// + /// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future. + /// + Task ImportFromStableAsync(StableStorage stableStorage); + + /// + /// Exports an item to a legacy (.zip based) package. + /// + /// The item to export. + void Export(TModel item); + + /// + /// Exports an item to the given output stream. + /// + /// The item to export. + /// The output stream to export to. + void ExportModelTo(TModel model, Stream outputStream); + + /// + /// Perform an update of the specified item. + /// TODO: Support file additions/removals. + /// + /// The item to update. + void Update(TModel item); + + /// + /// Delete an item from the manager. + /// Is a no-op for already deleted items. + /// + /// The item to delete. + /// false if no operation was performed + bool Delete(TModel item); + + /// + /// Delete multiple items. + /// This will post notifications tracking progress. + /// + void Delete(List items, bool silent = false); + + /// + /// Restore multiple items that were previously deleted. + /// This will post notifications tracking progress. + /// + void Undelete(List items, bool silent = false); + + /// + /// Restore an item that was previously deleted. Is a no-op if the item is not in a deleted state, or has its protected flag set. + /// + /// The item to restore + void Undelete(TModel item); + + /// + /// Import one or more items from filesystem . + /// + /// + /// This will be treated as a low priority import if more than one path is specified; use to always import at standard priority. + /// This will post notifications tracking progress. + /// + /// One or more archive locations on disk. + Task Import(params string[] paths); + + Task Import(params ImportTask[] tasks); + + Task> Import(ProgressNotification notification, params ImportTask[] tasks); + + /// + /// Import one from the filesystem and delete the file on success. + /// Note that this bypasses the UI flow and should only be used for special cases or testing. + /// + /// The containing data about the to import. + /// Whether this is a low priority import. + /// An optional cancellation token. + /// The imported model, if successful. + Task Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default); + + /// + /// Silently import an item from an . + /// + /// The archive to be imported. + /// Whether this is a low priority import. + /// An optional cancellation token. + Task Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default); + + /// + /// Silently import an item from a . + /// + /// The model to be imported. + /// An optional archive to use for model population. + /// Whether this is a low priority import. + /// An optional cancellation token. + Task Import(TModel item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default); } } diff --git a/osu.Game/Scoring/ScoreManager.cs b/osu.Game/Scoring/ScoreManager.cs index 81e701f001..56c346d177 100644 --- a/osu.Game/Scoring/ScoreManager.cs +++ b/osu.Game/Scoring/ScoreManager.cs @@ -78,7 +78,7 @@ namespace osu.Game.Scoring protected override Task Populate(ScoreInfo model, ArchiveReader archive, CancellationToken cancellationToken = default) => Task.CompletedTask; - protected override void ExportModelTo(ScoreInfo model, Stream outputStream) + public override void ExportModelTo(ScoreInfo model, Stream outputStream) { var file = model.Files.SingleOrDefault(); if (file == null) From 90225f20820ed74fe6269a7c1c4105d2c3e4866a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 16:45:32 +0900 Subject: [PATCH 06/16] Hook up all required interfaces to new `BeatmapManager` --- ...eneOnlinePlayBeatmapAvailabilityTracker.cs | 26 +- osu.Game/Beatmaps/BeatmapManager.cs | 297 +++++++++++++++++- osu.Game/Beatmaps/IWorkingBeatmapCache.cs | 15 + osu.Game/Beatmaps/WorkingBeatmapCache.cs | 42 +-- osu.Game/OsuGameBase.cs | 6 - osu.Game/Tests/Visual/EditorTestScene.cs | 37 ++- 6 files changed, 364 insertions(+), 59 deletions(-) create mode 100644 osu.Game/Beatmaps/IWorkingBeatmapCache.cs diff --git a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs index a7d34fadbe..1a3f9e414d 100644 --- a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs +++ b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs @@ -158,18 +158,34 @@ namespace osu.Game.Tests.Online public Task CurrentImportTask { get; private set; } - protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) - => new TestDownloadRequest(set); + protected override BeatmapModelManager CreateBeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host) + { + return new TestBeatmapModelManager(this, storage, contextFactory, rulesets, api, host); + } public TestBeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, WorkingBeatmap defaultBeatmap = null) : base(storage, contextFactory, rulesets, api, audioManager, resources, host, defaultBeatmap) { } - public override async Task Import(BeatmapSetInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) + internal class TestBeatmapModelManager : BeatmapModelManager { - await AllowImport.Task.ConfigureAwait(false); - return await (CurrentImportTask = base.Import(item, archive, lowPriority, cancellationToken)).ConfigureAwait(false); + private readonly TestBeatmapManager testBeatmapManager; + + public TestBeatmapModelManager(TestBeatmapManager testBeatmapManager, Storage storage, IDatabaseContextFactory databaseContextFactory, RulesetStore rulesetStore, IAPIProvider apiProvider, GameHost gameHost) + : base(storage, databaseContextFactory, rulesetStore, apiProvider, gameHost) + { + this.testBeatmapManager = testBeatmapManager; + } + + protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) + => new TestDownloadRequest(set); + + public override async Task Import(BeatmapSetInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) + { + await testBeatmapManager.AllowImport.Task.ConfigureAwait(false); + return await (testBeatmapManager.CurrentImportTask = base.Import(item, archive, lowPriority, cancellationToken)).ConfigureAwait(false); + } } } diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index c445925a90..18513945e5 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -1,7 +1,27 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; +using JetBrains.Annotations; +using osu.Framework.Audio; +using osu.Framework.Bindables; +using osu.Framework.IO.Stores; +using osu.Framework.Platform; using osu.Framework.Testing; +using osu.Game.Database; +using osu.Game.IO; +using osu.Game.IO.Archives; +using osu.Game.Online.API; +using osu.Game.Overlays.Notifications; +using osu.Game.Rulesets; +using osu.Game.Skinning; +using osu.Game.Users; namespace osu.Game.Beatmaps { @@ -9,13 +29,282 @@ namespace osu.Game.Beatmaps /// Handles general operations related to global beatmap management. /// [ExcludeFromDynamicCompile] - public class BeatmapManager + public class BeatmapManager : IModelDownloader, IModelFileManager, ICanAcceptFiles, IWorkingBeatmapCache { - public BeatmapManager() + private readonly BeatmapModelManager beatmapModelManager; + private readonly WorkingBeatmapCache workingBeatmapCache; + + public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, + WorkingBeatmap defaultBeatmap = null, bool performOnlineLookups = false) { - beatmapModelManager = new BeatmapModelManager() + beatmapModelManager = CreateBeatmapModelManager(storage, contextFactory, rulesets, api, host); + workingBeatmapCache = CreateWorkingBeatmapCache(audioManager, resources, new FileStore(contextFactory, storage).Store, defaultBeatmap, host); + + workingBeatmapCache.BeatmapManager = beatmapModelManager; + + var onlineBeatmapLookupCache = new BeatmapOnlineLookupQueue(api, storage); + + beatmapModelManager.PopulateOnlineInformation = onlineBeatmapLookupCache.UpdateAsync; } - } + protected virtual WorkingBeatmapCache CreateWorkingBeatmapCache(AudioManager audioManager, IResourceStore resources, IResourceStore storage, WorkingBeatmap defaultBeatmap, GameHost host) => + new WorkingBeatmapCache(audioManager, resources, storage, defaultBeatmap, host); + protected virtual BeatmapModelManager CreateBeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host) => + new BeatmapModelManager(storage, contextFactory, rulesets, api, host); + + /// + /// Create a new . + /// + public WorkingBeatmap CreateNew(RulesetInfo ruleset, User user) + { + var metadata = new BeatmapMetadata + { + Author = user, + }; + + var set = new BeatmapSetInfo + { + Metadata = metadata, + Beatmaps = new List + { + new BeatmapInfo + { + BaseDifficulty = new BeatmapDifficulty(), + Ruleset = ruleset, + Metadata = metadata, + WidescreenStoryboard = true, + SamplesMatchPlaybackRate = true, + } + } + }; + + var working = beatmapModelManager.Import(set).Result; + return GetWorkingBeatmap(working.Beatmaps.First()); + } + + #region Delegation to BeatmapModelManager (methods which previously existed locally). + + /// + /// Fired when a single difficulty has been hidden. + /// + public IBindable> BeatmapHidden => beatmapModelManager.BeatmapHidden; + + /// + /// Fired when a single difficulty has been restored. + /// + public IBindable> BeatmapRestored => beatmapModelManager.BeatmapRestored; + + /// + /// Saves an file against a given . + /// + /// The to save the content against. The file referenced by will be replaced. + /// The content to write. + /// The beatmap content to write, null if to be omitted. + public virtual void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null) => beatmapModelManager.Save(info, beatmapContent, beatmapSkin); + + /// + /// Returns a list of all usable s. + /// + /// A list of available . + public List GetAllUsableBeatmapSets(IncludedDetails includes = IncludedDetails.All, bool includeProtected = false) => beatmapModelManager.GetAllUsableBeatmapSets(includes, includeProtected); + + /// + /// Returns a list of all usable s. Note that files are not populated. + /// + /// The level of detail to include in the returned objects. + /// Whether to include protected (system) beatmaps. These should not be included for gameplay playable use cases. + /// A list of available . + public IEnumerable GetAllUsableBeatmapSetsEnumerable(IncludedDetails includes, bool includeProtected = false) => beatmapModelManager.GetAllUsableBeatmapSetsEnumerable(includes, includeProtected); + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// The level of detail to include in the returned objects. + /// Results from the provided query. + public IEnumerable QueryBeatmapSets(Expression> query, IncludedDetails includes = IncludedDetails.All) => beatmapModelManager.QueryBeatmapSets(query, includes); + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// The first result for the provided query, or null if no results were found. + public BeatmapSetInfo QueryBeatmapSet(Expression> query) => beatmapModelManager.QueryBeatmapSet(query); + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// Results from the provided query. + public IQueryable QueryBeatmaps(Expression> query) => beatmapModelManager.QueryBeatmaps(query); + + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// The first result for the provided query, or null if no results were found. + public BeatmapInfo QueryBeatmap(Expression> query) => beatmapModelManager.QueryBeatmap(query); + + /// + /// A default representation of a WorkingBeatmap to use when no beatmap is available. + /// + public WorkingBeatmap DefaultBeatmap => workingBeatmapCache.DefaultBeatmap; + + /// + /// Fired when a notification should be presented to the user. + /// + public Action PostNotification { set => beatmapModelManager.PostNotification = value; } + + /// + /// Fired when the user requests to view the resulting import. + /// + public Action> PresentImport { set => beatmapModelManager.PresentImport = value; } + + /// + /// Delete a beatmap difficulty. + /// + /// The beatmap difficulty to hide. + public void Hide(BeatmapInfo beatmap) => beatmapModelManager.Hide(beatmap); + + /// + /// Restore a beatmap difficulty. + /// + /// The beatmap difficulty to restore. + public void Restore(BeatmapInfo beatmap) => beatmapModelManager.Restore(beatmap); + + #endregion + + #region Implementation of IModelManager + + public IBindable> ItemUpdated => beatmapModelManager.ItemUpdated; + + public IBindable> ItemRemoved => beatmapModelManager.ItemRemoved; + + public Task ImportFromStableAsync(StableStorage stableStorage) + { + return beatmapModelManager.ImportFromStableAsync(stableStorage); + } + + public void Export(BeatmapSetInfo item) + { + beatmapModelManager.Export(item); + } + + public void ExportModelTo(BeatmapSetInfo model, Stream outputStream) + { + beatmapModelManager.ExportModelTo(model, outputStream); + } + + public void Update(BeatmapSetInfo item) + { + beatmapModelManager.Update(item); + } + + public bool Delete(BeatmapSetInfo item) + { + return beatmapModelManager.Delete(item); + } + + public void Delete(List items, bool silent = false) + { + beatmapModelManager.Delete(items, silent); + } + + public void Undelete(List items, bool silent = false) + { + beatmapModelManager.Undelete(items, silent); + } + + public void Undelete(BeatmapSetInfo item) + { + beatmapModelManager.Undelete(item); + } + + #endregion + + #region Implementation of IModelDownloader + + public IBindable>> DownloadBegan => beatmapModelManager.DownloadBegan; + + public IBindable>> DownloadFailed => beatmapModelManager.DownloadFailed; + + public bool IsAvailableLocally(BeatmapSetInfo model) + { + return beatmapModelManager.IsAvailableLocally(model); + } + + public bool Download(BeatmapSetInfo model, bool minimiseDownloadSize = false) + { + return beatmapModelManager.Download(model, minimiseDownloadSize); + } + + public ArchiveDownloadRequest GetExistingDownload(BeatmapSetInfo model) + { + return beatmapModelManager.GetExistingDownload(model); + } + + #endregion + + #region Implementation of ICanAcceptFiles + + public Task Import(params string[] paths) + { + return beatmapModelManager.Import(paths); + } + + public Task Import(params ImportTask[] tasks) + { + return beatmapModelManager.Import(tasks); + } + + public Task> Import(ProgressNotification notification, params ImportTask[] tasks) + { + return beatmapModelManager.Import(notification, tasks); + } + + public Task Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default) + { + return beatmapModelManager.Import(task, lowPriority, cancellationToken); + } + + public Task Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default) + { + return beatmapModelManager.Import(archive, lowPriority, cancellationToken); + } + + public Task Import(BeatmapSetInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) + { + return beatmapModelManager.Import(item, archive, lowPriority, cancellationToken); + } + + public IEnumerable HandledExtensions => beatmapModelManager.HandledExtensions; + + #endregion + + #region Implementation of IWorkingBeatmapCache + + public WorkingBeatmap GetWorkingBeatmap(BeatmapInfo importedBeatmap) => workingBeatmapCache.GetWorkingBeatmap(importedBeatmap); + + #endregion + + #region Implementation of IModelFileManager + + public void ReplaceFile(BeatmapSetInfo model, BeatmapSetFileInfo file, Stream contents, string filename = null) + { + beatmapModelManager.ReplaceFile(model, file, contents, filename); + } + + public void DeleteFile(BeatmapSetInfo model, BeatmapSetFileInfo file) + { + beatmapModelManager.DeleteFile(model, file); + } + + public void AddFile(BeatmapSetInfo model, Stream contents, string filename) + { + beatmapModelManager.AddFile(model, contents, filename); + } + + #endregion + } } diff --git a/osu.Game/Beatmaps/IWorkingBeatmapCache.cs b/osu.Game/Beatmaps/IWorkingBeatmapCache.cs new file mode 100644 index 0000000000..881e734292 --- /dev/null +++ b/osu.Game/Beatmaps/IWorkingBeatmapCache.cs @@ -0,0 +1,15 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +namespace osu.Game.Beatmaps +{ + public interface IWorkingBeatmapCache + { + /// + /// Retrieve a instance for the provided + /// + /// The beatmap to lookup. + /// A instance correlating to the provided . + WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo); + } +} diff --git a/osu.Game/Beatmaps/WorkingBeatmapCache.cs b/osu.Game/Beatmaps/WorkingBeatmapCache.cs index 9f40eb4898..e117f1b82f 100644 --- a/osu.Game/Beatmaps/WorkingBeatmapCache.cs +++ b/osu.Game/Beatmaps/WorkingBeatmapCache.cs @@ -2,7 +2,6 @@ // See the LICENCE file in the repository root for full licence text. using System; -using System.Collections.Generic; using System.IO; using System.Linq; using JetBrains.Annotations; @@ -17,14 +16,12 @@ using osu.Framework.Statistics; using osu.Framework.Testing; using osu.Game.Beatmaps.Formats; using osu.Game.IO; -using osu.Game.Rulesets; using osu.Game.Skinning; using osu.Game.Storyboards; -using osu.Game.Users; namespace osu.Game.Beatmaps { - public class WorkingBeatmapCache : IBeatmapResourceProvider + public class WorkingBeatmapCache : IBeatmapResourceProvider, IWorkingBeatmapCache { private readonly WeakList workingCache = new WeakList(); @@ -33,7 +30,7 @@ namespace osu.Game.Beatmaps /// public readonly WorkingBeatmap DefaultBeatmap; - public BeatmapManager BeatmapManager { private get; set; } + public BeatmapModelManager BeatmapManager { private get; set; } private readonly AudioManager audioManager; private readonly IResourceStore resources; @@ -74,41 +71,6 @@ namespace osu.Game.Beatmaps } } - /// - /// Create a new . - /// - public WorkingBeatmap CreateNew(RulesetInfo ruleset, User user) - { - var metadata = new BeatmapMetadata - { - Author = user, - }; - - var set = new BeatmapSetInfo - { - Metadata = metadata, - Beatmaps = new List - { - new BeatmapInfo - { - BaseDifficulty = new BeatmapDifficulty(), - Ruleset = ruleset, - Metadata = metadata, - WidescreenStoryboard = true, - SamplesMatchPlaybackRate = true, - } - } - }; - - var working = BeatmapManager.Import(set).Result; - return GetWorkingBeatmap(working.Beatmaps.First()); - } - - /// - /// Retrieve a instance for the provided - /// - /// The beatmap to lookup. - /// A instance correlating to the provided . public virtual WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo) { // if there are no files, presume the full beatmap info has not yet been fetched from the database. diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 8263e26dec..dc1cb7a850 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -138,8 +138,6 @@ namespace osu.Game private UserLookupCache userCache; - private BeatmapOnlineLookupQueue onlineBeatmapLookupCache; - private FileStore fileStore; private RulesetConfigCache rulesetConfigCache; @@ -246,10 +244,6 @@ namespace osu.Game dependencies.Cache(ScoreManager = new ScoreManager(RulesetStore, () => BeatmapManager, Storage, API, contextFactory, Scheduler, Host, () => difficultyCache, LocalConfig)); dependencies.Cache(BeatmapManager = new BeatmapManager(Storage, contextFactory, RulesetStore, API, Audio, Resources, Host, defaultBeatmap)); - onlineBeatmapLookupCache = new BeatmapOnlineLookupQueue(API, Storage); - - BeatmapManager.PopulateOnlineInformation = onlineBeatmapLookupCache.UpdateAsync; - // this should likely be moved to ArchiveModelManager when another case appears where it is necessary // to have inter-dependent model managers. this could be obtained with an IHasForeign interface to // allow lookups to be done on the child (ScoreManager in this case) to perform the cascading delete. diff --git a/osu.Game/Tests/Visual/EditorTestScene.cs b/osu.Game/Tests/Visual/EditorTestScene.cs index 1e26036116..ac8773a840 100644 --- a/osu.Game/Tests/Visual/EditorTestScene.cs +++ b/osu.Game/Tests/Visual/EditorTestScene.cs @@ -123,11 +123,40 @@ namespace osu.Game.Tests.Visual this.testBeatmap = testBeatmap; } - protected override string ComputeHash(BeatmapSetInfo item, ArchiveReader reader = null) - => string.Empty; + protected override BeatmapModelManager CreateBeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host) + { + return new TestBeatmapModelManager(storage, contextFactory, rulesets, api, host); + } - public override WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo) - => testBeatmap; + protected override WorkingBeatmapCache CreateWorkingBeatmapCache(AudioManager audioManager, IResourceStore resources, IResourceStore storage, WorkingBeatmap defaultBeatmap, GameHost host) + { + return new TestWorkingBeatmapCache(this, audioManager, resources, storage, defaultBeatmap, host); + } + + private class TestWorkingBeatmapCache : WorkingBeatmapCache + { + private readonly TestBeatmapManager testBeatmapManager; + + public TestWorkingBeatmapCache(TestBeatmapManager testBeatmapManager, AudioManager audioManager, IResourceStore resourceStore, IResourceStore storage, WorkingBeatmap defaultBeatmap, GameHost gameHost) + : base(audioManager, resourceStore, storage, defaultBeatmap, gameHost) + { + this.testBeatmapManager = testBeatmapManager; + } + + public override WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo) + => testBeatmapManager.testBeatmap; + } + + internal class TestBeatmapModelManager : BeatmapModelManager + { + public TestBeatmapModelManager(Storage storage, IDatabaseContextFactory databaseContextFactory, RulesetStore rulesetStore, IAPIProvider apiProvider, GameHost gameHost) + : base(storage, databaseContextFactory, rulesetStore, apiProvider, gameHost) + { + } + + protected override string ComputeHash(BeatmapSetInfo item, ArchiveReader reader = null) + => string.Empty; + } public override void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null) { From 7a72747d886cc95e70c4abc94867527f2a6002e8 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 17:14:35 +0900 Subject: [PATCH 07/16] Add back optional online lookups --- osu.Game/Beatmaps/BeatmapManager.cs | 8 +++++--- osu.Game/OsuGameBase.cs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 18513945e5..6ffdfa24b5 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -42,9 +42,11 @@ namespace osu.Game.Beatmaps workingBeatmapCache.BeatmapManager = beatmapModelManager; - var onlineBeatmapLookupCache = new BeatmapOnlineLookupQueue(api, storage); - - beatmapModelManager.PopulateOnlineInformation = onlineBeatmapLookupCache.UpdateAsync; + if (performOnlineLookups) + { + var onlineBeatmapLookupCache = new BeatmapOnlineLookupQueue(api, storage); + beatmapModelManager.PopulateOnlineInformation = onlineBeatmapLookupCache.UpdateAsync; + } } protected virtual WorkingBeatmapCache CreateWorkingBeatmapCache(AudioManager audioManager, IResourceStore resources, IResourceStore storage, WorkingBeatmap defaultBeatmap, GameHost host) => diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index dc1cb7a850..e76436a75b 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -242,7 +242,7 @@ namespace osu.Game // ordering is important here to ensure foreign keys rules are not broken in ModelStore.Cleanup() dependencies.Cache(ScoreManager = new ScoreManager(RulesetStore, () => BeatmapManager, Storage, API, contextFactory, Scheduler, Host, () => difficultyCache, LocalConfig)); - dependencies.Cache(BeatmapManager = new BeatmapManager(Storage, contextFactory, RulesetStore, API, Audio, Resources, Host, defaultBeatmap)); + dependencies.Cache(BeatmapManager = new BeatmapManager(Storage, contextFactory, RulesetStore, API, Audio, Resources, Host, defaultBeatmap, performOnlineLookups: true)); // this should likely be moved to ArchiveModelManager when another case appears where it is necessary // to have inter-dependent model managers. this could be obtained with an IHasForeign interface to From fd13142a158b50f89caab9a516617137970e3388 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 18:20:20 +0900 Subject: [PATCH 08/16] Add missing interface to `BeatmapManager` --- osu.Game/Beatmaps/BeatmapManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 6ffdfa24b5..c72d1e8dec 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -29,7 +29,7 @@ namespace osu.Game.Beatmaps /// Handles general operations related to global beatmap management. /// [ExcludeFromDynamicCompile] - public class BeatmapManager : IModelDownloader, IModelFileManager, ICanAcceptFiles, IWorkingBeatmapCache + public class BeatmapManager : IModelDownloader, IModelManager, IModelFileManager, ICanAcceptFiles, IWorkingBeatmapCache { private readonly BeatmapModelManager beatmapModelManager; private readonly WorkingBeatmapCache workingBeatmapCache; From 0a00bc779542a3ce86c77a9970e14c8cfab06f34 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 17:42:12 +0900 Subject: [PATCH 09/16] Split out `IPostNotifications` into an interface --- osu.Game/Collections/CollectionManager.cs | 6 ++---- osu.Game/Database/ArchiveModelManager.cs | 3 --- osu.Game/Database/IModelManager.cs | 2 +- osu.Game/Database/IPostNotifications.cs | 16 ++++++++++++++++ 4 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 osu.Game/Database/IPostNotifications.cs diff --git a/osu.Game/Collections/CollectionManager.cs b/osu.Game/Collections/CollectionManager.cs index fe04c70d62..6f9d9cd8a8 100644 --- a/osu.Game/Collections/CollectionManager.cs +++ b/osu.Game/Collections/CollectionManager.cs @@ -14,6 +14,7 @@ using osu.Framework.Graphics; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Game.Beatmaps; +using osu.Game.Database; using osu.Game.IO; using osu.Game.IO.Legacy; using osu.Game.Overlays.Notifications; @@ -27,7 +28,7 @@ namespace osu.Game.Collections /// This is currently reading and writing from the osu-stable file format. This is a temporary arrangement until we refactor the /// database backing the game. Going forward writing should be done in a similar way to other model stores. /// - public class CollectionManager : Component + public class CollectionManager : Component, IPostNotifications { /// /// Database version in stable-compatible YYYYMMDD format. @@ -106,9 +107,6 @@ namespace osu.Game.Collections backgroundSave(); }); - /// - /// Set an endpoint for notifications to be posted to. - /// public Action PostNotification { protected get; set; } /// diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index fc217d3058..018b41ebc1 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -57,9 +57,6 @@ namespace osu.Game.Database /// private static readonly ThreadedTaskScheduler import_scheduler_low_priority = new ThreadedTaskScheduler(import_queue_request_concurrency, nameof(ArchiveModelManager)); - /// - /// Set an endpoint for notifications to be posted to. - /// public Action PostNotification { protected get; set; } /// diff --git a/osu.Game/Database/IModelManager.cs b/osu.Game/Database/IModelManager.cs index 8f0c6e1561..721de8f3a0 100644 --- a/osu.Game/Database/IModelManager.cs +++ b/osu.Game/Database/IModelManager.cs @@ -17,7 +17,7 @@ namespace osu.Game.Database /// Represents a model manager that publishes events when s are added or removed. /// /// The model type. - public interface IModelManager + public interface IModelManager : IPostNotifications where TModel : class { /// diff --git a/osu.Game/Database/IPostNotifications.cs b/osu.Game/Database/IPostNotifications.cs new file mode 100644 index 0000000000..d4fd64e79e --- /dev/null +++ b/osu.Game/Database/IPostNotifications.cs @@ -0,0 +1,16 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using osu.Game.Overlays.Notifications; + +namespace osu.Game.Database +{ + public interface IPostNotifications + { + /// + /// And action which will be fired when a notification should be presented to the user. + /// + public Action PostNotification { set; } + } +} From 3e3b9bc963583fa5f6ad3b822ad3e6e3818aa06c Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 18:21:16 +0900 Subject: [PATCH 10/16] Split out `IModelDownloader` and also split apart `ScoreManager` --- ...eneOnlinePlayBeatmapAvailabilityTracker.cs | 25 +- osu.Game/Beatmaps/BeatmapManager.cs | 37 ++- osu.Game/Beatmaps/BeatmapModelDownloader.cs | 21 ++ osu.Game/Beatmaps/BeatmapModelManager.cs | 21 +- osu.Game/Database/ArchiveModelManager.cs | 20 +- osu.Game/Database/IModelDownloader.cs | 9 +- osu.Game/Database/IModelManager.cs | 12 + osu.Game/Database/IPresentImports.cs | 17 ++ ...hiveModelManager.cs => ModelDownloader.cs} | 47 ++-- osu.Game/Online/DownloadTrackingComposite.cs | 10 +- osu.Game/Scoring/ScoreManager.cs | 222 ++++++++++++------ osu.Game/Scoring/ScoreModelDownloader.cs | 20 ++ osu.Game/Scoring/ScoreModelManager.cs | 88 +++++++ .../Screens/Ranking/ReplayDownloadButton.cs | 2 +- osu.Game/Tests/Visual/EditorTestScene.cs | 2 +- 15 files changed, 403 insertions(+), 150 deletions(-) create mode 100644 osu.Game/Beatmaps/BeatmapModelDownloader.cs create mode 100644 osu.Game/Database/IPresentImports.cs rename osu.Game/Database/{DownloadableArchiveModelManager.cs => ModelDownloader.cs} (68%) create mode 100644 osu.Game/Scoring/ScoreModelDownloader.cs create mode 100644 osu.Game/Scoring/ScoreModelManager.cs diff --git a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs index 1a3f9e414d..d38294aba9 100644 --- a/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs +++ b/osu.Game.Tests/Online/TestSceneOnlinePlayBeatmapAvailabilityTracker.cs @@ -158,14 +158,30 @@ namespace osu.Game.Tests.Online public Task CurrentImportTask { get; private set; } + public TestBeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, WorkingBeatmap defaultBeatmap = null) + : base(storage, contextFactory, rulesets, api, audioManager, resources, host, defaultBeatmap) + { + } + protected override BeatmapModelManager CreateBeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host) { return new TestBeatmapModelManager(this, storage, contextFactory, rulesets, api, host); } - public TestBeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, WorkingBeatmap defaultBeatmap = null) - : base(storage, contextFactory, rulesets, api, audioManager, resources, host, defaultBeatmap) + protected override BeatmapModelDownloader CreateBeatmapModelDownloader(BeatmapModelManager modelManager, IAPIProvider api, GameHost host) { + return new TestBeatmapModelDownloader(modelManager, api, host); + } + + internal class TestBeatmapModelDownloader : BeatmapModelDownloader + { + public TestBeatmapModelDownloader(BeatmapModelManager modelManager, IAPIProvider apiProvider, GameHost gameHost) + : base(modelManager, apiProvider, gameHost) + { + } + + protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) + => new TestDownloadRequest(set); } internal class TestBeatmapModelManager : BeatmapModelManager @@ -173,14 +189,11 @@ namespace osu.Game.Tests.Online private readonly TestBeatmapManager testBeatmapManager; public TestBeatmapModelManager(TestBeatmapManager testBeatmapManager, Storage storage, IDatabaseContextFactory databaseContextFactory, RulesetStore rulesetStore, IAPIProvider apiProvider, GameHost gameHost) - : base(storage, databaseContextFactory, rulesetStore, apiProvider, gameHost) + : base(storage, databaseContextFactory, rulesetStore, gameHost) { this.testBeatmapManager = testBeatmapManager; } - protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) - => new TestDownloadRequest(set); - public override async Task Import(BeatmapSetInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) { await testBeatmapManager.AllowImport.Task.ConfigureAwait(false); diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index c72d1e8dec..8dfd895987 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -32,12 +32,15 @@ namespace osu.Game.Beatmaps public class BeatmapManager : IModelDownloader, IModelManager, IModelFileManager, ICanAcceptFiles, IWorkingBeatmapCache { private readonly BeatmapModelManager beatmapModelManager; + private readonly BeatmapModelDownloader beatmapModelDownloader; + private readonly WorkingBeatmapCache workingBeatmapCache; public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, WorkingBeatmap defaultBeatmap = null, bool performOnlineLookups = false) { beatmapModelManager = CreateBeatmapModelManager(storage, contextFactory, rulesets, api, host); + beatmapModelDownloader = CreateBeatmapModelDownloader(beatmapModelManager, api, host); workingBeatmapCache = CreateWorkingBeatmapCache(audioManager, resources, new FileStore(contextFactory, storage).Store, defaultBeatmap, host); workingBeatmapCache.BeatmapManager = beatmapModelManager; @@ -49,11 +52,16 @@ namespace osu.Game.Beatmaps } } + protected virtual BeatmapModelDownloader CreateBeatmapModelDownloader(BeatmapModelManager modelManager, IAPIProvider api, GameHost host) + { + return new BeatmapModelDownloader(modelManager, api, host); + } + protected virtual WorkingBeatmapCache CreateWorkingBeatmapCache(AudioManager audioManager, IResourceStore resources, IResourceStore storage, WorkingBeatmap defaultBeatmap, GameHost host) => new WorkingBeatmapCache(audioManager, resources, storage, defaultBeatmap, host); protected virtual BeatmapModelManager CreateBeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host) => - new BeatmapModelManager(storage, contextFactory, rulesets, api, host); + new BeatmapModelManager(storage, contextFactory, rulesets, host); /// /// Create a new . @@ -156,7 +164,14 @@ namespace osu.Game.Beatmaps /// /// Fired when a notification should be presented to the user. /// - public Action PostNotification { set => beatmapModelManager.PostNotification = value; } + public Action PostNotification + { + set + { + beatmapModelManager.PostNotification = value; + beatmapModelDownloader.PostNotification = value; + } + } /// /// Fired when the user requests to view the resulting import. @@ -179,6 +194,11 @@ namespace osu.Game.Beatmaps #region Implementation of IModelManager + public bool IsAvailableLocally(BeatmapSetInfo model) + { + return beatmapModelManager.IsAvailableLocally(model); + } + public IBindable> ItemUpdated => beatmapModelManager.ItemUpdated; public IBindable> ItemRemoved => beatmapModelManager.ItemRemoved; @@ -227,23 +247,18 @@ namespace osu.Game.Beatmaps #region Implementation of IModelDownloader - public IBindable>> DownloadBegan => beatmapModelManager.DownloadBegan; + public IBindable>> DownloadBegan => beatmapModelDownloader.DownloadBegan; - public IBindable>> DownloadFailed => beatmapModelManager.DownloadFailed; - - public bool IsAvailableLocally(BeatmapSetInfo model) - { - return beatmapModelManager.IsAvailableLocally(model); - } + public IBindable>> DownloadFailed => beatmapModelDownloader.DownloadFailed; public bool Download(BeatmapSetInfo model, bool minimiseDownloadSize = false) { - return beatmapModelManager.Download(model, minimiseDownloadSize); + return beatmapModelDownloader.Download(model, minimiseDownloadSize); } public ArchiveDownloadRequest GetExistingDownload(BeatmapSetInfo model) { - return beatmapModelManager.GetExistingDownload(model); + return beatmapModelDownloader.GetExistingDownload(model); } #endregion diff --git a/osu.Game/Beatmaps/BeatmapModelDownloader.cs b/osu.Game/Beatmaps/BeatmapModelDownloader.cs new file mode 100644 index 0000000000..ae482eeafd --- /dev/null +++ b/osu.Game/Beatmaps/BeatmapModelDownloader.cs @@ -0,0 +1,21 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using osu.Framework.Platform; +using osu.Game.Database; +using osu.Game.Online.API; +using osu.Game.Online.API.Requests; + +namespace osu.Game.Beatmaps +{ + public class BeatmapModelDownloader : ModelDownloader + { + protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) => + new DownloadBeatmapSetRequest(set, minimiseDownloadSize); + + public BeatmapModelDownloader(BeatmapModelManager beatmapModelManager, IAPIProvider api, GameHost host = null) + : base(beatmapModelManager, api, host) + { + } + } +} diff --git a/osu.Game/Beatmaps/BeatmapModelManager.cs b/osu.Game/Beatmaps/BeatmapModelManager.cs index be3adc412c..1b6694b1b4 100644 --- a/osu.Game/Beatmaps/BeatmapModelManager.cs +++ b/osu.Game/Beatmaps/BeatmapModelManager.cs @@ -21,8 +21,6 @@ using osu.Game.Beatmaps.Formats; using osu.Game.Database; using osu.Game.IO; using osu.Game.IO.Archives; -using osu.Game.Online.API; -using osu.Game.Online.API.Requests; using osu.Game.Rulesets; using osu.Game.Rulesets.Objects; using osu.Game.Skinning; @@ -34,7 +32,7 @@ namespace osu.Game.Beatmaps /// Handles ef-core storage of beatmaps. /// [ExcludeFromDynamicCompile] - public class BeatmapModelManager : DownloadableArchiveModelManager + public class BeatmapModelManager : ArchiveModelManager { /// /// Fired when a single difficulty has been hidden. @@ -72,8 +70,8 @@ namespace osu.Game.Beatmaps private readonly BeatmapStore beatmaps; private readonly RulesetStore rulesets; - public BeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host = null) - : base(storage, contextFactory, api, new BeatmapStore(contextFactory), host) + public BeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, GameHost host = null) + : base(storage, contextFactory, new BeatmapStore(contextFactory), host) { this.rulesets = rulesets; @@ -84,9 +82,6 @@ namespace osu.Game.Beatmaps beatmaps.ItemUpdated += obj => WorkingBeatmapCache?.Invalidate(obj); } - protected override ArchiveDownloadRequest CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) => - new DownloadBeatmapSetRequest(set, minimiseDownloadSize); - protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path)?.ToLowerInvariant() == ".osz"; protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) @@ -176,10 +171,6 @@ namespace osu.Game.Beatmaps void resetIds() => beatmapSet.Beatmaps.ForEach(b => b.OnlineBeatmapID = null); } - protected override bool CheckLocalAvailability(BeatmapSetInfo model, IQueryable items) - => base.CheckLocalAvailability(model, items) - || (model.OnlineBeatmapSetID != null && items.Any(b => b.OnlineBeatmapSetID == model.OnlineBeatmapSetID)); - /// /// Delete a beatmap difficulty. /// @@ -347,7 +338,11 @@ namespace osu.Game.Beatmaps /// Results from the provided query. public IQueryable QueryBeatmaps(Expression> query) => beatmaps.Beatmaps.AsNoTracking().Where(query); - protected override string HumanisedModelName => "beatmap"; + public override string HumanisedModelName => "beatmap"; + + protected override bool CheckLocalAvailability(BeatmapSetInfo model, IQueryable items) + => base.CheckLocalAvailability(model, items) + || (model.OnlineBeatmapSetID != null && items.Any(b => b.OnlineBeatmapSetID == model.OnlineBeatmapSetID)); protected override BeatmapSetInfo CreateModel(ArchiveReader reader) { diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 018b41ebc1..0c309bbddb 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -30,7 +30,7 @@ namespace osu.Game.Database /// /// The model type. /// The associated file join type. - public abstract class ArchiveModelManager : ICanAcceptFiles, IModelManager, IModelFileManager + public abstract class ArchiveModelManager : ICanAcceptFiles, IModelManager, IModelFileManager, IPresentImports where TModel : class, IHasFiles, IHasPrimaryKey, ISoftDelete where TFileModel : class, INamedFileInfo, new() { @@ -249,10 +249,7 @@ namespace osu.Game.Database return import; } - /// - /// Fired when the user requests to view the resulting import. - /// - public Action> PresentImport; + public Action> PresentImport { protected get; set; } /// /// Silently import an item from an . @@ -799,6 +796,17 @@ namespace osu.Game.Database /// An existing model which matches the criteria to skip importing, else null. protected TModel CheckForExisting(TModel model) => model.Hash == null ? null : ModelStore.ConsumableItems.FirstOrDefault(b => b.Hash == model.Hash); + public bool IsAvailableLocally(TModel model) => CheckLocalAvailability(model, ModelStore.ConsumableItems.Where(m => !m.DeletePending)); + + /// + /// Performs implementation specific comparisons to determine whether a given model is present in the local store. + /// + /// The whose existence needs to be checked. + /// The usable items present in the store. + /// Whether the exists. + protected virtual bool CheckLocalAvailability(TModel model, IQueryable items) + => model.ID > 0 && items.Any(i => i.ID == model.ID && i.Files.Any()); + /// /// Whether import can be skipped after finding an existing import early in the process. /// Only valid when is not overridden. @@ -835,7 +843,7 @@ namespace osu.Game.Database private DbSet queryModel() => ContextFactory.Get().Set(); - protected virtual string HumanisedModelName => $"{typeof(TModel).Name.Replace(@"Info", "").ToLower()}"; + public virtual string HumanisedModelName => $"{typeof(TModel).Name.Replace(@"Info", "").ToLower()}"; #region Event handling / delaying diff --git a/osu.Game/Database/IModelDownloader.cs b/osu.Game/Database/IModelDownloader.cs index 0cb633280e..a5573b2190 100644 --- a/osu.Game/Database/IModelDownloader.cs +++ b/osu.Game/Database/IModelDownloader.cs @@ -11,7 +11,7 @@ namespace osu.Game.Database /// Represents a that can download new models from an external source. /// /// The model type. - public interface IModelDownloader : IModelManager + public interface IModelDownloader : IPostNotifications where TModel : class { /// @@ -26,13 +26,6 @@ namespace osu.Game.Database /// IBindable>> DownloadFailed { get; } - /// - /// Checks whether a given is already available in the local store. - /// - /// The whose existence needs to be checked. - /// Whether the exists. - bool IsAvailableLocally(TModel model); - /// /// Begin a download for the requested . /// diff --git a/osu.Game/Database/IModelManager.cs b/osu.Game/Database/IModelManager.cs index 721de8f3a0..7bfc8dbee3 100644 --- a/osu.Game/Database/IModelManager.cs +++ b/osu.Game/Database/IModelManager.cs @@ -123,5 +123,17 @@ namespace osu.Game.Database /// Whether this is a low priority import. /// An optional cancellation token. Task Import(TModel item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default); + + /// + /// Checks whether a given is already available in the local store. + /// + /// The whose existence needs to be checked. + /// Whether the exists. + bool IsAvailableLocally(TModel model); + + /// + /// A user displayable name for the model type associated with this manager. + /// + string HumanisedModelName => $"{typeof(TModel).Name.Replace(@"Info", "").ToLower()}"; } } diff --git a/osu.Game/Database/IPresentImports.cs b/osu.Game/Database/IPresentImports.cs new file mode 100644 index 0000000000..39b495ebd5 --- /dev/null +++ b/osu.Game/Database/IPresentImports.cs @@ -0,0 +1,17 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; + +namespace osu.Game.Database +{ + public interface IPresentImports + where TModel : class + { + /// + /// Fired when the user requests to view the resulting import. + /// + public Action> PresentImport { set; } + } +} diff --git a/osu.Game/Database/DownloadableArchiveModelManager.cs b/osu.Game/Database/ModelDownloader.cs similarity index 68% rename from osu.Game/Database/DownloadableArchiveModelManager.cs rename to osu.Game/Database/ModelDownloader.cs index e6d5b44b65..e613b39b6b 100644 --- a/osu.Game/Database/DownloadableArchiveModelManager.cs +++ b/osu.Game/Database/ModelDownloader.cs @@ -1,29 +1,24 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. -using Humanizer; -using osu.Framework.Logging; -using osu.Framework.Platform; -using osu.Game.Online.API; -using osu.Game.Overlays.Notifications; using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Humanizer; using osu.Framework.Bindables; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Game.Online.API; +using osu.Game.Overlays.Notifications; namespace osu.Game.Database { - /// - /// An that has the ability to download models using an and - /// import them into the store. - /// - /// The model type. - /// The associated file join type. - public abstract class DownloadableArchiveModelManager : ArchiveModelManager, IModelDownloader - where TModel : class, IHasFiles, IHasPrimaryKey, ISoftDelete, IEquatable - where TFileModel : class, INamedFileInfo, new() + public abstract class ModelDownloader : IModelDownloader + where TModel : class, IHasPrimaryKey, ISoftDelete, IEquatable { + public Action PostNotification { protected get; set; } + public IBindable>> DownloadBegan => downloadBegan; private readonly Bindable>> downloadBegan = new Bindable>>(); @@ -32,18 +27,15 @@ namespace osu.Game.Database private readonly Bindable>> downloadFailed = new Bindable>>(); + private readonly IModelManager modelManager; private readonly IAPIProvider api; private readonly List> currentDownloads = new List>(); - private readonly MutableDatabaseBackedStoreWithFileIncludes modelStore; - - protected DownloadableArchiveModelManager(Storage storage, IDatabaseContextFactory contextFactory, IAPIProvider api, MutableDatabaseBackedStoreWithFileIncludes modelStore, - IIpcHost importHost = null) - : base(storage, contextFactory, modelStore, importHost) + protected ModelDownloader(IModelManager modelManager, IAPIProvider api, IIpcHost importHost = null) { + this.modelManager = modelManager; this.api = api; - this.modelStore = modelStore; } /// @@ -76,7 +68,7 @@ namespace osu.Game.Database Task.Factory.StartNew(async () => { // This gets scheduled back to the update thread, but we want the import to run in the background. - var imported = await Import(notification, new ImportTask(filename)).ConfigureAwait(false); + var imported = await modelManager.Import(notification, new ImportTask(filename)).ConfigureAwait(false); // for now a failed import will be marked as a failed download for simplicity. if (!imported.Any()) @@ -111,21 +103,10 @@ namespace osu.Game.Database notification.State = ProgressNotificationState.Cancelled; if (!(error is OperationCanceledException)) - Logger.Error(error, $"{HumanisedModelName.Titleize()} download failed!"); + Logger.Error(error, $"{modelManager.HumanisedModelName.Titleize()} download failed!"); } } - public bool IsAvailableLocally(TModel model) => CheckLocalAvailability(model, modelStore.ConsumableItems.Where(m => !m.DeletePending)); - - /// - /// Performs implementation specific comparisons to determine whether a given model is present in the local store. - /// - /// The whose existence needs to be checked. - /// The usable items present in the store. - /// Whether the exists. - protected virtual bool CheckLocalAvailability(TModel model, IQueryable items) - => model.ID > 0 && items.Any(i => i.ID == model.ID && i.Files.Any()); - public ArchiveDownloadRequest GetExistingDownload(TModel model) => currentDownloads.Find(r => r.Model.Equals(model)); private bool canDownload(TModel model) => GetExistingDownload(model) == null && api != null; diff --git a/osu.Game/Online/DownloadTrackingComposite.cs b/osu.Game/Online/DownloadTrackingComposite.cs index d9599481e7..2a96051427 100644 --- a/osu.Game/Online/DownloadTrackingComposite.cs +++ b/osu.Game/Online/DownloadTrackingComposite.cs @@ -16,7 +16,7 @@ namespace osu.Game.Online /// public abstract class DownloadTrackingComposite : CompositeDrawable where TModel : class, IEquatable - where TModelManager : class, IModelDownloader + where TModelManager : class, IModelDownloader, IModelManager { protected readonly Bindable Model = new Bindable(); @@ -35,7 +35,7 @@ namespace osu.Game.Online Model.Value = model; } - private IBindable> managedUpdated; + private IBindable> managerUpdated; private IBindable> managerRemoved; private IBindable>> managerDownloadBegan; private IBindable>> managerDownloadFailed; @@ -60,8 +60,8 @@ namespace osu.Game.Online managerDownloadBegan.BindValueChanged(downloadBegan); managerDownloadFailed = Manager.DownloadFailed.GetBoundCopy(); managerDownloadFailed.BindValueChanged(downloadFailed); - managedUpdated = Manager.ItemUpdated.GetBoundCopy(); - managedUpdated.BindValueChanged(itemUpdated); + managerUpdated = Manager.ItemUpdated.GetBoundCopy(); + managerUpdated.BindValueChanged(itemUpdated); managerRemoved = Manager.ItemRemoved.GetBoundCopy(); managerRemoved.BindValueChanged(itemRemoved); } @@ -77,7 +77,7 @@ namespace osu.Game.Online /// /// Whether the given model is available in the database. - /// By default, this calls , + /// By default, this calls , /// but can be overriden to add additional checks for verifying the model in database. /// protected virtual bool IsModelAvailableLocally() => Manager?.IsAvailableLocally(Model.Value) == true; diff --git a/osu.Game/Scoring/ScoreManager.cs b/osu.Game/Scoring/ScoreManager.cs index 56c346d177..d83b4e3f1d 100644 --- a/osu.Game/Scoring/ScoreManager.cs +++ b/osu.Game/Scoring/ScoreManager.cs @@ -9,102 +9,48 @@ using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; -using Microsoft.EntityFrameworkCore; using osu.Framework.Bindables; -using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Threading; using osu.Game.Beatmaps; using osu.Game.Configuration; using osu.Game.Database; +using osu.Game.IO; using osu.Game.IO.Archives; using osu.Game.Online.API; -using osu.Game.Online.API.Requests; +using osu.Game.Overlays.Notifications; using osu.Game.Rulesets; using osu.Game.Rulesets.Judgements; using osu.Game.Rulesets.Scoring; -using osu.Game.Scoring.Legacy; namespace osu.Game.Scoring { - public class ScoreManager : DownloadableArchiveModelManager + public class ScoreManager : IModelManager, IModelFileManager, IModelDownloader, ICanAcceptFiles, IPresentImports { - public override IEnumerable HandledExtensions => new[] { ".osr" }; - - protected override string[] HashableFileTypes => new[] { ".osr" }; - - protected override string ImportFromStablePath => Path.Combine("Data", "r"); - - private readonly RulesetStore rulesets; - private readonly Func beatmaps; private readonly Scheduler scheduler; - - [CanBeNull] private readonly Func difficulties; - - [CanBeNull] private readonly OsuConfigManager configManager; + private readonly ScoreModelManager scoreModelManager; + private readonly ScoreModelDownloader scoreModelDownloader; public ScoreManager(RulesetStore rulesets, Func beatmaps, Storage storage, IAPIProvider api, IDatabaseContextFactory contextFactory, Scheduler scheduler, IIpcHost importHost = null, Func difficulties = null, OsuConfigManager configManager = null) - : base(storage, contextFactory, api, new ScoreStore(contextFactory, storage), importHost) { - this.rulesets = rulesets; - this.beatmaps = beatmaps; this.scheduler = scheduler; this.difficulties = difficulties; this.configManager = configManager; + + scoreModelManager = new ScoreModelManager(rulesets, beatmaps, storage, contextFactory, importHost); + scoreModelDownloader = new ScoreModelDownloader(scoreModelManager, api, importHost); } - protected override ScoreInfo CreateModel(ArchiveReader archive) - { - if (archive == null) - return null; + public Score GetScore(ScoreInfo score) => scoreModelManager.GetScore(score); - using (var stream = archive.GetStream(archive.Filenames.First(f => f.EndsWith(".osr", StringComparison.OrdinalIgnoreCase)))) - { - try - { - return new DatabasedLegacyScoreDecoder(rulesets, beatmaps()).Parse(stream).ScoreInfo; - } - catch (LegacyScoreDecoder.BeatmapNotFoundException e) - { - Logger.Log(e.Message, LoggingTarget.Information, LogLevel.Error); - return null; - } - } - } + public List GetAllUsableScores() => scoreModelManager.GetAllUsableScores(); - protected override Task Populate(ScoreInfo model, ArchiveReader archive, CancellationToken cancellationToken = default) - => Task.CompletedTask; + public IEnumerable QueryScores(Expression> query) => scoreModelManager.QueryScores(query); - public override void ExportModelTo(ScoreInfo model, Stream outputStream) - { - var file = model.Files.SingleOrDefault(); - if (file == null) - return; - - using (var inputStream = Files.Storage.GetStream(file.FileInfo.StoragePath)) - inputStream.CopyTo(outputStream); - } - - protected override IEnumerable GetStableImportPaths(Storage storage) - => storage.GetFiles(ImportFromStablePath).Where(p => HandledExtensions.Any(ext => Path.GetExtension(p)?.Equals(ext, StringComparison.OrdinalIgnoreCase) ?? false)) - .Select(path => storage.GetFullPath(path)); - - public Score GetScore(ScoreInfo score) => new LegacyDatabasedScore(score, rulesets, beatmaps(), Files.Store); - - public List GetAllUsableScores() => ModelStore.ConsumableItems.Where(s => !s.DeletePending).ToList(); - - public IEnumerable QueryScores(Expression> query) => ModelStore.ConsumableItems.AsNoTracking().Where(query); - - public ScoreInfo Query(Expression> query) => ModelStore.ConsumableItems.AsNoTracking().FirstOrDefault(query); - - protected override ArchiveDownloadRequest CreateDownloadRequest(ScoreInfo score, bool minimiseDownload) => new DownloadReplayRequest(score); - - protected override bool CheckLocalAvailability(ScoreInfo model, IQueryable items) - => base.CheckLocalAvailability(model, items) - || (model.OnlineScoreID != null && items.Any(i => i.OnlineScoreID == model.OnlineScoreID)); + public ScoreInfo Query(Expression> query) => scoreModelManager.Query(query); /// /// Orders an array of s by total score. @@ -281,5 +227,149 @@ namespace osu.Game.Scoring this.totalScore.BindValueChanged(v => Value = v.NewValue.ToString("N0"), true); } } + + #region Implementation of IPostNotifications + + public Action PostNotification + { + set + { + scoreModelManager.PostNotification = value; + scoreModelDownloader.PostNotification = value; + } + } + + #endregion + + #region Implementation of IModelManager + + public IBindable> ItemUpdated => scoreModelManager.ItemUpdated; + + public IBindable> ItemRemoved => scoreModelManager.ItemRemoved; + + public Task ImportFromStableAsync(StableStorage stableStorage) + { + return scoreModelManager.ImportFromStableAsync(stableStorage); + } + + public void Export(ScoreInfo item) + { + scoreModelManager.Export(item); + } + + public void ExportModelTo(ScoreInfo model, Stream outputStream) + { + scoreModelManager.ExportModelTo(model, outputStream); + } + + public void Update(ScoreInfo item) + { + scoreModelManager.Update(item); + } + + public bool Delete(ScoreInfo item) + { + return scoreModelManager.Delete(item); + } + + public void Delete(List items, bool silent = false) + { + scoreModelManager.Delete(items, silent); + } + + public void Undelete(List items, bool silent = false) + { + scoreModelManager.Undelete(items, silent); + } + + public void Undelete(ScoreInfo item) + { + scoreModelManager.Undelete(item); + } + + public Task Import(params string[] paths) + { + return scoreModelManager.Import(paths); + } + + public Task Import(params ImportTask[] tasks) + { + return scoreModelManager.Import(tasks); + } + + public IEnumerable HandledExtensions => scoreModelManager.HandledExtensions; + + public Task> Import(ProgressNotification notification, params ImportTask[] tasks) + { + return scoreModelManager.Import(notification, tasks); + } + + public Task Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default) + { + return scoreModelManager.Import(task, lowPriority, cancellationToken); + } + + public Task Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default) + { + return scoreModelManager.Import(archive, lowPriority, cancellationToken); + } + + public Task Import(ScoreInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) + { + return scoreModelManager.Import(item, archive, lowPriority, cancellationToken); + } + + public bool IsAvailableLocally(ScoreInfo model) + { + return scoreModelManager.IsAvailableLocally(model); + } + + #endregion + + #region Implementation of IModelFileManager + + public void ReplaceFile(ScoreInfo model, ScoreFileInfo file, Stream contents, string filename = null) + { + scoreModelManager.ReplaceFile(model, file, contents, filename); + } + + public void DeleteFile(ScoreInfo model, ScoreFileInfo file) + { + scoreModelManager.DeleteFile(model, file); + } + + public void AddFile(ScoreInfo model, Stream contents, string filename) + { + scoreModelManager.AddFile(model, contents, filename); + } + + #endregion + + #region Implementation of IModelDownloader + + public IBindable>> DownloadBegan => scoreModelDownloader.DownloadBegan; + + public IBindable>> DownloadFailed => scoreModelDownloader.DownloadFailed; + + public bool Download(ScoreInfo model, bool minimiseDownloadSize) + { + return scoreModelDownloader.Download(model, minimiseDownloadSize); + } + + public ArchiveDownloadRequest GetExistingDownload(ScoreInfo model) + { + return scoreModelDownloader.GetExistingDownload(model); + } + + #endregion + + #region Implementation of IPresentImports + + public Action> PresentImport + { + set => scoreModelManager.PresentImport = value; + } + + #endregion } } diff --git a/osu.Game/Scoring/ScoreModelDownloader.cs b/osu.Game/Scoring/ScoreModelDownloader.cs new file mode 100644 index 0000000000..b3c1e2928a --- /dev/null +++ b/osu.Game/Scoring/ScoreModelDownloader.cs @@ -0,0 +1,20 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using osu.Framework.Platform; +using osu.Game.Database; +using osu.Game.Online.API; +using osu.Game.Online.API.Requests; + +namespace osu.Game.Scoring +{ + public class ScoreModelDownloader : ModelDownloader + { + public ScoreModelDownloader(ScoreModelManager scoreManager, IAPIProvider api, IIpcHost importHost = null) + : base(scoreManager, api, importHost) + { + } + + protected override ArchiveDownloadRequest CreateDownloadRequest(ScoreInfo score, bool minimiseDownload) => new DownloadReplayRequest(score); + } +} diff --git a/osu.Game/Scoring/ScoreModelManager.cs b/osu.Game/Scoring/ScoreModelManager.cs new file mode 100644 index 0000000000..c65a6acdfb --- /dev/null +++ b/osu.Game/Scoring/ScoreModelManager.cs @@ -0,0 +1,88 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Game.Beatmaps; +using osu.Game.Database; +using osu.Game.IO.Archives; +using osu.Game.Rulesets; +using osu.Game.Scoring.Legacy; + +namespace osu.Game.Scoring +{ + public class ScoreModelManager : ArchiveModelManager + { + public override IEnumerable HandledExtensions => new[] { ".osr" }; + + protected override string[] HashableFileTypes => new[] { ".osr" }; + + protected override string ImportFromStablePath => Path.Combine("Data", "r"); + + private readonly RulesetStore rulesets; + private readonly Func beatmaps; + + public ScoreModelManager(RulesetStore rulesets, Func beatmaps, Storage storage, IDatabaseContextFactory contextFactory, IIpcHost importHost = null) + : base(storage, contextFactory, new ScoreStore(contextFactory, storage), importHost) + { + this.rulesets = rulesets; + this.beatmaps = beatmaps; + } + + protected override ScoreInfo CreateModel(ArchiveReader archive) + { + if (archive == null) + return null; + + using (var stream = archive.GetStream(archive.Filenames.First(f => f.EndsWith(".osr", StringComparison.OrdinalIgnoreCase)))) + { + try + { + return new DatabasedLegacyScoreDecoder(rulesets, beatmaps()).Parse(stream).ScoreInfo; + } + catch (LegacyScoreDecoder.BeatmapNotFoundException e) + { + Logger.Log(e.Message, LoggingTarget.Information, LogLevel.Error); + return null; + } + } + } + + public Score GetScore(ScoreInfo score) => new LegacyDatabasedScore(score, rulesets, beatmaps(), Files.Store); + + public List GetAllUsableScores() => ModelStore.ConsumableItems.Where(s => !s.DeletePending).ToList(); + + public IEnumerable QueryScores(Expression> query) => ModelStore.ConsumableItems.AsNoTracking().Where(query); + + public ScoreInfo Query(Expression> query) => ModelStore.ConsumableItems.AsNoTracking().FirstOrDefault(query); + + protected override Task Populate(ScoreInfo model, ArchiveReader archive, CancellationToken cancellationToken = default) + => Task.CompletedTask; + + protected override bool CheckLocalAvailability(ScoreInfo model, IQueryable items) + => base.CheckLocalAvailability(model, items) + || (model.OnlineScoreID != null && items.Any(i => i.OnlineScoreID == model.OnlineScoreID)); + + public override void ExportModelTo(ScoreInfo model, Stream outputStream) + { + var file = model.Files.SingleOrDefault(); + if (file == null) + return; + + using (var inputStream = Files.Storage.GetStream(file.FileInfo.StoragePath)) + inputStream.CopyTo(outputStream); + } + + protected override IEnumerable GetStableImportPaths(Storage storage) + => storage.GetFiles(ImportFromStablePath).Where(p => HandledExtensions.Any(ext => Path.GetExtension(p)?.Equals(ext, StringComparison.OrdinalIgnoreCase) ?? false)) + .Select(path => storage.GetFullPath(path)); + } +} diff --git a/osu.Game/Screens/Ranking/ReplayDownloadButton.cs b/osu.Game/Screens/Ranking/ReplayDownloadButton.cs index 18b8649a59..d96b6989b4 100644 --- a/osu.Game/Screens/Ranking/ReplayDownloadButton.cs +++ b/osu.Game/Screens/Ranking/ReplayDownloadButton.cs @@ -40,7 +40,7 @@ namespace osu.Game.Screens.Ranking } [BackgroundDependencyLoader(true)] - private void load(OsuGame game, ScoreManager scores) + private void load(OsuGame game, ScoreModelDownloader scores) { InternalChild = shakeContainer = new ShakeContainer { diff --git a/osu.Game/Tests/Visual/EditorTestScene.cs b/osu.Game/Tests/Visual/EditorTestScene.cs index ac8773a840..798b0d01ee 100644 --- a/osu.Game/Tests/Visual/EditorTestScene.cs +++ b/osu.Game/Tests/Visual/EditorTestScene.cs @@ -150,7 +150,7 @@ namespace osu.Game.Tests.Visual internal class TestBeatmapModelManager : BeatmapModelManager { public TestBeatmapModelManager(Storage storage, IDatabaseContextFactory databaseContextFactory, RulesetStore rulesetStore, IAPIProvider apiProvider, GameHost gameHost) - : base(storage, databaseContextFactory, rulesetStore, apiProvider, gameHost) + : base(storage, databaseContextFactory, rulesetStore, gameHost) { } From dde19f2e81ca08df3328799d6d244ae4e62ed9cd Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Oct 2021 01:37:51 +0900 Subject: [PATCH 11/16] Fix unbalanced brackets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bartłomiej Dach --- osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs b/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs index 19f02c82ec..bc86c6be5d 100644 --- a/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs +++ b/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs @@ -211,7 +211,7 @@ namespace osu.Game.Beatmaps } private void logForModel(BeatmapSetInfo set, string message) => - ArchiveModelManager.LogForModel(set, $"{nameof(BeatmapOnlineLookupQueue)}] {message}"); + ArchiveModelManager.LogForModel(set, $"[{nameof(BeatmapOnlineLookupQueue)}] {message}"); public void Dispose() { From 27c4f2b06ee70adab63aeabfa03d42cdcbfbeefc Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Oct 2021 01:38:50 +0900 Subject: [PATCH 12/16] Add missing disposal --- osu.Game/OsuGameBase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 8263e26dec..f239119e40 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -531,6 +531,7 @@ namespace osu.Game RulesetStore?.Dispose(); LocalConfig?.Dispose(); + onlineBeatmapLookupCache?.Dispose(); contextFactory?.FlushConnections(); } From 428c7830d958d731398bcb18193160461418a670 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Oct 2021 01:43:57 +0900 Subject: [PATCH 13/16] Pass online lookup queue in as a whole, rather than function --- osu.Game/Beatmaps/BeatmapManager.cs | 16 +++++++++++++--- osu.Game/Beatmaps/BeatmapModelManager.cs | 9 ++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index c72d1e8dec..1946e3f93f 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -29,10 +29,11 @@ namespace osu.Game.Beatmaps /// Handles general operations related to global beatmap management. /// [ExcludeFromDynamicCompile] - public class BeatmapManager : IModelDownloader, IModelManager, IModelFileManager, ICanAcceptFiles, IWorkingBeatmapCache + public class BeatmapManager : IModelDownloader, IModelManager, IModelFileManager, ICanAcceptFiles, IWorkingBeatmapCache, IDisposable { private readonly BeatmapModelManager beatmapModelManager; private readonly WorkingBeatmapCache workingBeatmapCache; + private readonly BeatmapOnlineLookupQueue onlineBetamapLookupQueue; public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore resources, GameHost host = null, WorkingBeatmap defaultBeatmap = null, bool performOnlineLookups = false) @@ -44,8 +45,8 @@ namespace osu.Game.Beatmaps if (performOnlineLookups) { - var onlineBeatmapLookupCache = new BeatmapOnlineLookupQueue(api, storage); - beatmapModelManager.PopulateOnlineInformation = onlineBeatmapLookupCache.UpdateAsync; + onlineBetamapLookupQueue = new BeatmapOnlineLookupQueue(api, storage); + beatmapModelManager.OnlineLookupQueue = onlineBetamapLookupQueue; } } @@ -308,5 +309,14 @@ namespace osu.Game.Beatmaps } #endregion + + #region Implementation of IDisposable + + public void Dispose() + { + onlineBetamapLookupQueue?.Dispose(); + } + + #endregion } } diff --git a/osu.Game/Beatmaps/BeatmapModelManager.cs b/osu.Game/Beatmaps/BeatmapModelManager.cs index be3adc412c..72df1f37ee 100644 --- a/osu.Game/Beatmaps/BeatmapModelManager.cs +++ b/osu.Game/Beatmaps/BeatmapModelManager.cs @@ -49,10 +49,9 @@ namespace osu.Game.Beatmaps public IBindable> BeatmapRestored => beatmapRestored; /// - /// A function which populates online information during the import process. - /// It is run as the final step of import. + /// An online lookup queue component which handles populating online beatmap metadata. /// - public Func PopulateOnlineInformation; + public BeatmapOnlineLookupQueue OnlineLookupQueue { private get; set; } /// /// The game working beatmap cache, used to invalidate entries on changes. @@ -107,8 +106,8 @@ namespace osu.Game.Beatmaps bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0); - if (PopulateOnlineInformation != null) - await PopulateOnlineInformation(beatmapSet, cancellationToken).ConfigureAwait(false); + if (OnlineLookupQueue != null) + await OnlineLookupQueue.UpdateAsync(beatmapSet, cancellationToken).ConfigureAwait(false); // ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID. if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0)) From 2ed28f625a6ce106ad76c86b2772b808daf975dc Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Oct 2021 01:46:37 +0900 Subject: [PATCH 14/16] Pass whole queue in rather than function --- osu.Game/Beatmaps/BeatmapManager.cs | 9 ++++----- osu.Game/OsuGameBase.cs | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index a2f9740779..1fc7aa3146 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -55,10 +55,9 @@ namespace osu.Game.Beatmaps public IBindable> BeatmapRestored => beatmapRestored; /// - /// A function which populates online information during the import process. - /// It is run as the final step of import. + /// An online lookup queue component which handles populating online beatmap metadata. /// - public Func PopulateOnlineInformation; + public BeatmapOnlineLookupQueue OnlineLookupQueue { private get; set; } private readonly Bindable> beatmapRestored = new Bindable>(); @@ -156,8 +155,8 @@ namespace osu.Game.Beatmaps bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0); - if (PopulateOnlineInformation != null) - await PopulateOnlineInformation(beatmapSet, cancellationToken).ConfigureAwait(false); + if (OnlineLookupQueue != null) + await OnlineLookupQueue.UpdateAsync(beatmapSet, cancellationToken).ConfigureAwait(false); // ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID. if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0)) diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index f239119e40..7772d5dfd8 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -138,7 +138,7 @@ namespace osu.Game private UserLookupCache userCache; - private BeatmapOnlineLookupQueue onlineBeatmapLookupCache; + private BeatmapOnlineLookupQueue onlineBeatmapLookupQueue; private FileStore fileStore; @@ -246,9 +246,9 @@ namespace osu.Game dependencies.Cache(ScoreManager = new ScoreManager(RulesetStore, () => BeatmapManager, Storage, API, contextFactory, Scheduler, Host, () => difficultyCache, LocalConfig)); dependencies.Cache(BeatmapManager = new BeatmapManager(Storage, contextFactory, RulesetStore, API, Audio, Resources, Host, defaultBeatmap)); - onlineBeatmapLookupCache = new BeatmapOnlineLookupQueue(API, Storage); + onlineBeatmapLookupQueue = new BeatmapOnlineLookupQueue(API, Storage); - BeatmapManager.PopulateOnlineInformation = onlineBeatmapLookupCache.UpdateAsync; + BeatmapManager.OnlineLookupQueue = onlineBeatmapLookupQueue; // this should likely be moved to ArchiveModelManager when another case appears where it is necessary // to have inter-dependent model managers. this could be obtained with an IHasForeign interface to @@ -531,7 +531,7 @@ namespace osu.Game RulesetStore?.Dispose(); LocalConfig?.Dispose(); - onlineBeatmapLookupCache?.Dispose(); + onlineBeatmapLookupQueue?.Dispose(); contextFactory?.FlushConnections(); } From c71cf1e2200bcbefb2d71400782f90ba50918bd1 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Oct 2021 01:51:29 +0900 Subject: [PATCH 15/16] Fix incomplete xmldoc --- osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs b/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs index bc86c6be5d..55164e2442 100644 --- a/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs +++ b/osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs @@ -25,7 +25,7 @@ namespace osu.Game.Beatmaps /// A component which handles population of online IDs for beatmaps using a two part lookup procedure. /// /// - /// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to ). + /// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to ) will be downloaded if not already present locally. /// This will always be checked before doing a second online query to get required metadata. /// [ExcludeFromDynamicCompile] From a32f5d44e279f617ad2933b28f26c0d0250882d6 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Oct 2021 22:23:51 +0900 Subject: [PATCH 16/16] Improve clarity of xmldoc Co-authored-by: Dan Balasescu --- osu.Game/Database/IRealmFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/IRealmFactory.cs b/osu.Game/Database/IRealmFactory.cs index 3b206d80eb..a957424584 100644 --- a/osu.Game/Database/IRealmFactory.cs +++ b/osu.Game/Database/IRealmFactory.cs @@ -13,7 +13,7 @@ namespace osu.Game.Database Realm Context { get; } /// - /// Create a new realm context for use on an arbitrary thread. + /// Create a new realm context for use on the current thread. /// Realm CreateContext(); }