diff --git a/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs b/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs new file mode 100644 index 0000000000..e1b01aaac5 --- /dev/null +++ b/osu.Game/Beatmaps/APIBeatmapMetadataSource.cs @@ -0,0 +1,78 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Diagnostics; +using osu.Game.Database; +using osu.Game.Online.API; +using osu.Game.Online.API.Requests; + +namespace osu.Game.Beatmaps +{ + public class APIBeatmapMetadataSource : IOnlineBeatmapMetadataSource + { + private readonly IAPIProvider api; + + public APIBeatmapMetadataSource(IAPIProvider api) + { + this.api = api; + } + + public bool Available => api.State.Value == APIState.Online; + + public OnlineBeatmapMetadata? Lookup(BeatmapInfo beatmapInfo) + { + if (!Available) + return null; + + Debug.Assert(beatmapInfo.BeatmapSet != null); + + var req = new GetBeatmapRequest(beatmapInfo); + + try + { + // intentionally blocking to limit web request concurrency + api.Perform(req); + + if (req.CompletionState == APIRequestCompletionState.Failed) + { + logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval failed for {beatmapInfo}"); + return null; + } + + var res = req.Response; + + if (res != null) + { + logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}."); + + return new OnlineBeatmapMetadata + { + BeatmapID = res.OnlineID, + BeatmapSetID = res.OnlineBeatmapSetID, + AuthorID = res.AuthorID, + BeatmapStatus = res.Status, + BeatmapSetStatus = res.BeatmapSet?.Status, + DateRanked = res.BeatmapSet?.Ranked, + DateSubmitted = res.BeatmapSet?.Submitted, + MD5Hash = res.MD5Hash, + LastUpdated = res.LastUpdated + }; + } + } + catch (Exception e) + { + logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval failed for {beatmapInfo} ({e.Message})"); + } + + return null; + } + + private void logForModel(BeatmapSetInfo set, string message) => + RealmArchiveModelImporter.LogForModel(set, $@"[{nameof(APIBeatmapMetadataSource)}] {message}"); + + public void Dispose() + { + } + } +} diff --git a/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs b/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs index fac91c23f5..b326e7ad1c 100644 --- a/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs +++ b/osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs @@ -1,62 +1,31 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. -#nullable disable - using System; using System.Diagnostics; -using System.IO; using System.Linq; -using System.Threading.Tasks; -using Microsoft.Data.Sqlite; -using osu.Framework.Development; -using osu.Framework.IO.Network; -using osu.Framework.Logging; using osu.Framework.Platform; -using osu.Game.Database; using osu.Game.Online.API; -using osu.Game.Online.API.Requests; -using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; -using SQLitePCL; namespace osu.Game.Beatmaps { /// /// A component which handles population of online IDs for beatmaps using a two part lookup procedure. /// - /// - /// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to ) will be downloaded if not already present locally. - /// This will always be checked before doing a second online query to get required metadata. - /// public class BeatmapUpdaterMetadataLookup : IDisposable { - private readonly IAPIProvider api; - private readonly Storage storage; - - private FileWebRequest cacheDownloadRequest; - - private const string cache_database_name = "online.db"; + private readonly IOnlineBeatmapMetadataSource apiMetadataSource; + private readonly IOnlineBeatmapMetadataSource localCachedMetadataSource; public BeatmapUpdaterMetadataLookup(IAPIProvider api, Storage storage) + : this(new APIBeatmapMetadataSource(api), new LocalCachedBeatmapMetadataSource(storage)) { - try - { - // required to initialise native SQLite libraries on some platforms. - Batteries_V2.Init(); - raw.sqlite3_config(2 /*SQLITE_CONFIG_MULTITHREAD*/); - } - catch - { - // may fail if platform not supported. - } + } - this.api = api; - this.storage = storage; - - // avoid downloading / using cache for unit tests. - if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name)) - prepareLocalCache(); + internal BeatmapUpdaterMetadataLookup(IOnlineBeatmapMetadataSource apiMetadataSource, IOnlineBeatmapMetadataSource localCachedMetadataSource) + { + this.apiMetadataSource = apiMetadataSource; + this.localCachedMetadataSource = localCachedMetadataSource; } /// @@ -69,196 +38,57 @@ namespace osu.Game.Beatmaps /// Whether metadata from an online source should be preferred. If true, the local cache will be skipped to ensure the freshest data state possible. public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch) { - foreach (var b in beatmapSet.Beatmaps) - lookup(beatmapSet, b, preferOnlineFetch); - } - - private void lookup(BeatmapSetInfo set, BeatmapInfo beatmapInfo, bool preferOnlineFetch) - { - bool apiAvailable = api?.State.Value == APIState.Online; - - bool useLocalCache = !apiAvailable || !preferOnlineFetch; - - if (useLocalCache && checkLocalCache(set, beatmapInfo)) - return; - - if (!apiAvailable) - return; - - var req = new GetBeatmapRequest(beatmapInfo); - - try + foreach (var beatmapInfo in beatmapSet.Beatmaps) { - // intentionally blocking to limit web request concurrency - api.Perform(req); + var res = lookup(beatmapSet, beatmapInfo, preferOnlineFetch); - if (req.CompletionState == APIRequestCompletionState.Failed) + if (res == null) { - logForModel(set, $"Online retrieval failed for {beatmapInfo}"); beatmapInfo.ResetOnlineInfo(); - return; + continue; } - var res = req.Response; + beatmapInfo.OnlineID = res.BeatmapID; + beatmapInfo.OnlineMD5Hash = res.MD5Hash; + beatmapInfo.LastOnlineUpdate = res.LastUpdated; - if (res != null) + Debug.Assert(beatmapInfo.BeatmapSet != null); + beatmapInfo.BeatmapSet.OnlineID = res.BeatmapSetID; + + // Some metadata should only be applied if there's no local changes. + if (shouldSaveOnlineMetadata(beatmapInfo)) { - beatmapInfo.OnlineID = res.OnlineID; - beatmapInfo.OnlineMD5Hash = res.MD5Hash; - beatmapInfo.LastOnlineUpdate = res.LastUpdated; - - Debug.Assert(beatmapInfo.BeatmapSet != null); - beatmapInfo.BeatmapSet.OnlineID = res.OnlineBeatmapSetID; - - // Some metadata should only be applied if there's no local changes. - if (shouldSaveOnlineMetadata(beatmapInfo)) - { - beatmapInfo.Status = res.Status; - beatmapInfo.Metadata.Author.OnlineID = res.AuthorID; - } - - if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata)) - { - beatmapInfo.BeatmapSet.Status = res.BeatmapSet?.Status ?? BeatmapOnlineStatus.None; - beatmapInfo.BeatmapSet.DateRanked = res.BeatmapSet?.Ranked; - beatmapInfo.BeatmapSet.DateSubmitted = res.BeatmapSet?.Submitted; - } - - logForModel(set, $"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}."); + beatmapInfo.Status = res.BeatmapStatus; + beatmapInfo.Metadata.Author.OnlineID = res.AuthorID; + } + + if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata)) + { + beatmapInfo.BeatmapSet.Status = res.BeatmapSetStatus ?? BeatmapOnlineStatus.None; + beatmapInfo.BeatmapSet.DateRanked = res.DateRanked; + beatmapInfo.BeatmapSet.DateSubmitted = res.DateSubmitted; } - } - catch (Exception e) - { - logForModel(set, $"Online retrieval failed for {beatmapInfo} ({e.Message})"); - beatmapInfo.ResetOnlineInfo(); } } - private void prepareLocalCache() + private OnlineBeatmapMetadata? lookup(BeatmapSetInfo set, BeatmapInfo beatmapInfo, bool preferOnlineFetch) { - string cacheFilePath = storage.GetFullPath(cache_database_name); - string compressedCacheFilePath = $"{cacheFilePath}.bz2"; + OnlineBeatmapMetadata? result = null; - cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}"); + bool useLocalCache = !apiMetadataSource.Available || !preferOnlineFetch; - cacheDownloadRequest.Failed += ex => - { - File.Delete(compressedCacheFilePath); - File.Delete(cacheFilePath); + if (useLocalCache) + result = localCachedMetadataSource.Lookup(beatmapInfo); - Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}", LoggingTarget.Database); - }; + if (result != null) + return result; - cacheDownloadRequest.Finished += () => - { - try - { - using (var stream = File.OpenRead(cacheDownloadRequest.Filename)) - using (var outStream = File.OpenWrite(cacheFilePath)) - using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false)) - bz2.CopyTo(outStream); + if (apiMetadataSource.Available) + result = apiMetadataSource.Lookup(beatmapInfo); - // set to null on completion to allow lookups to begin using the new source - cacheDownloadRequest = null; - } - catch (Exception ex) - { - Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache extraction failed: {ex}", LoggingTarget.Database); - File.Delete(cacheFilePath); - } - finally - { - File.Delete(compressedCacheFilePath); - } - }; - - Task.Run(async () => - { - try - { - await cacheDownloadRequest.PerformAsync().ConfigureAwait(false); - } - catch - { - // Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway. - } - }); + return result; } - private bool checkLocalCache(BeatmapSetInfo set, BeatmapInfo beatmapInfo) - { - // download is in progress (or was, and failed). - if (cacheDownloadRequest != null) - return false; - - // database is unavailable. - if (!storage.Exists(cache_database_name)) - return false; - - if (string.IsNullOrEmpty(beatmapInfo.MD5Hash) - && string.IsNullOrEmpty(beatmapInfo.Path) - && beatmapInfo.OnlineID <= 0) - return false; - - try - { - using (var db = new SqliteConnection(string.Concat("Data Source=", storage.GetFullPath($@"{"online.db"}", true)))) - { - db.Open(); - - using (var cmd = db.CreateCommand()) - { - cmd.CommandText = - "SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path"; - - cmd.Parameters.Add(new SqliteParameter("@MD5Hash", beatmapInfo.MD5Hash)); - cmd.Parameters.Add(new SqliteParameter("@OnlineID", beatmapInfo.OnlineID)); - cmd.Parameters.Add(new SqliteParameter("@Path", beatmapInfo.Path)); - - using (var reader = cmd.ExecuteReader()) - { - if (reader.Read()) - { - var status = (BeatmapOnlineStatus)reader.GetByte(2); - - // Some metadata should only be applied if there's no local changes. - if (shouldSaveOnlineMetadata(beatmapInfo)) - { - beatmapInfo.Status = status; - beatmapInfo.Metadata.Author.OnlineID = reader.GetInt32(3); - } - - // TODO: DateSubmitted and DateRanked are not provided by local cache. - beatmapInfo.OnlineID = reader.GetInt32(1); - beatmapInfo.OnlineMD5Hash = reader.GetString(4); - beatmapInfo.LastOnlineUpdate = reader.GetDateTimeOffset(5); - - Debug.Assert(beatmapInfo.BeatmapSet != null); - beatmapInfo.BeatmapSet.OnlineID = reader.GetInt32(0); - - if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata)) - { - beatmapInfo.BeatmapSet.Status = status; - } - - logForModel(set, $"Cached local retrieval for {beatmapInfo}."); - return true; - } - } - } - } - } - catch (Exception ex) - { - logForModel(set, $"Cached local retrieval for {beatmapInfo} failed with {ex}."); - } - - return false; - } - - private void logForModel(BeatmapSetInfo set, string message) => - RealmArchiveModelImporter.LogForModel(set, $"[{nameof(BeatmapUpdaterMetadataLookup)}] {message}"); - /// /// Check whether the provided beatmap is in a state where online "ranked" status metadata should be saved against it. /// Handles the case where a user may have locally modified a beatmap in the editor and expects the local status to stick. @@ -267,7 +97,8 @@ namespace osu.Game.Beatmaps public void Dispose() { - cacheDownloadRequest?.Dispose(); + apiMetadataSource.Dispose(); + localCachedMetadataSource.Dispose(); } } } diff --git a/osu.Game/Beatmaps/IOnlineBeatmapMetadataSource.cs b/osu.Game/Beatmaps/IOnlineBeatmapMetadataSource.cs new file mode 100644 index 0000000000..753462d493 --- /dev/null +++ b/osu.Game/Beatmaps/IOnlineBeatmapMetadataSource.cs @@ -0,0 +1,26 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; + +namespace osu.Game.Beatmaps +{ + /// + /// Unifying interface for sources of online beatmap metadata. + /// + public interface IOnlineBeatmapMetadataSource : IDisposable + { + /// + /// Whether this source can currently service lookups. + /// + bool Available { get; } + + /// + /// Looks up the online metadata for the supplied . + /// + /// + /// An instance if the lookup is successful, or if the lookup failed. + /// + OnlineBeatmapMetadata? Lookup(BeatmapInfo beatmapInfo); + } +} diff --git a/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs b/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs new file mode 100644 index 0000000000..435242aeab --- /dev/null +++ b/osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs @@ -0,0 +1,176 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using osu.Framework.Development; +using osu.Framework.IO.Network; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Game.Database; +using SharpCompress.Compressors; +using SharpCompress.Compressors.BZip2; +using SQLitePCL; + +namespace osu.Game.Beatmaps +{ + /// + /// + /// + /// + /// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to ) will be downloaded if not already present locally. + /// + public class LocalCachedBeatmapMetadataSource : IOnlineBeatmapMetadataSource + { + private readonly Storage storage; + + private FileWebRequest? cacheDownloadRequest; + + private const string cache_database_name = @"online.db"; + + public LocalCachedBeatmapMetadataSource(Storage storage) + { + try + { + // required to initialise native SQLite libraries on some platforms. + Batteries_V2.Init(); + raw.sqlite3_config(2 /*SQLITE_CONFIG_MULTITHREAD*/); + } + catch + { + // may fail if platform not supported. + } + + this.storage = storage; + + // avoid downloading / using cache for unit tests. + if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name)) + prepareLocalCache(); + } + + private void prepareLocalCache() + { + string cacheFilePath = storage.GetFullPath(cache_database_name); + string compressedCacheFilePath = $@"{cacheFilePath}.bz2"; + + cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $@"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}"); + + cacheDownloadRequest.Failed += ex => + { + File.Delete(compressedCacheFilePath); + File.Delete(cacheFilePath); + + Logger.Log($@"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}", LoggingTarget.Database); + }; + + cacheDownloadRequest.Finished += () => + { + try + { + using (var stream = File.OpenRead(cacheDownloadRequest.Filename)) + using (var outStream = File.OpenWrite(cacheFilePath)) + using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false)) + bz2.CopyTo(outStream); + + // set to null on completion to allow lookups to begin using the new source + cacheDownloadRequest = null; + } + catch (Exception ex) + { + Logger.Log($@"{nameof(LocalCachedBeatmapMetadataSource)}'s online cache extraction failed: {ex}", LoggingTarget.Database); + File.Delete(cacheFilePath); + } + finally + { + File.Delete(compressedCacheFilePath); + } + }; + + Task.Run(async () => + { + try + { + await cacheDownloadRequest.PerformAsync().ConfigureAwait(false); + } + catch + { + // Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway. + } + }); + } + + public bool Available => + // no download in progress. + cacheDownloadRequest == null + // cached database exists on disk. + && storage.Exists(cache_database_name); + + public OnlineBeatmapMetadata? Lookup(BeatmapInfo beatmapInfo) + { + if (!Available) + return null; + + if (string.IsNullOrEmpty(beatmapInfo.MD5Hash) + && string.IsNullOrEmpty(beatmapInfo.Path) + && beatmapInfo.OnlineID <= 0) + return null; + + Debug.Assert(beatmapInfo.BeatmapSet != null); + + try + { + using (var db = new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true)))) + { + db.Open(); + + using (var cmd = db.CreateCommand()) + { + cmd.CommandText = + @"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path"; + + cmd.Parameters.Add(new SqliteParameter(@"@MD5Hash", beatmapInfo.MD5Hash)); + cmd.Parameters.Add(new SqliteParameter(@"@OnlineID", beatmapInfo.OnlineID)); + cmd.Parameters.Add(new SqliteParameter(@"@Path", beatmapInfo.Path)); + + using (var reader = cmd.ExecuteReader()) + { + if (reader.Read()) + { + logForModel(beatmapInfo.BeatmapSet, $@"Cached local retrieval for {beatmapInfo}."); + + return new OnlineBeatmapMetadata + { + BeatmapSetID = reader.GetInt32(0), + BeatmapID = reader.GetInt32(1), + BeatmapStatus = (BeatmapOnlineStatus)reader.GetByte(2), + BeatmapSetStatus = (BeatmapOnlineStatus)reader.GetByte(2), + AuthorID = reader.GetInt32(3), + MD5Hash = reader.GetString(4), + LastUpdated = reader.GetDateTimeOffset(5), + // TODO: DateSubmitted and DateRanked are not provided by local cache. + }; + } + } + } + } + } + catch (Exception ex) + { + logForModel(beatmapInfo.BeatmapSet, $@"Cached local retrieval for {beatmapInfo} failed with {ex}."); + } + + return null; + } + + private void logForModel(BeatmapSetInfo set, string message) => + RealmArchiveModelImporter.LogForModel(set, $@"[{nameof(LocalCachedBeatmapMetadataSource)}] {message}"); + + public void Dispose() + { + cacheDownloadRequest?.Dispose(); + } + } +} diff --git a/osu.Game/Beatmaps/OnlineBeatmapMetadata.cs b/osu.Game/Beatmaps/OnlineBeatmapMetadata.cs new file mode 100644 index 0000000000..8640883ca1 --- /dev/null +++ b/osu.Game/Beatmaps/OnlineBeatmapMetadata.cs @@ -0,0 +1,61 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; + +namespace osu.Game.Beatmaps +{ + /// + /// This structure contains parts of beatmap metadata which are involved with the online parts + /// of the game, and therefore must be treated with particular care. + /// This data is retrieved from trusted sources (such as osu-web API, or a locally downloaded sqlite snapshot + /// of osu-web metadata). + /// + public class OnlineBeatmapMetadata + { + /// + /// The online ID of the beatmap. + /// + public int BeatmapID { get; init; } + + /// + /// The online ID of the beatmap set. + /// + public int BeatmapSetID { get; init; } + + /// + /// The online ID of the author. + /// + public int AuthorID { get; init; } + + /// + /// The online status of the beatmap. + /// + public BeatmapOnlineStatus BeatmapStatus { get; init; } + + /// + /// The online status of the associated beatmap set. + /// + public BeatmapOnlineStatus? BeatmapSetStatus { get; init; } + + /// + /// The rank date of the beatmap, if applicable and available. + /// + public DateTimeOffset? DateRanked { get; init; } + + /// + /// The submission date of the beatmap, if available. + /// + public DateTimeOffset? DateSubmitted { get; init; } + + /// + /// The MD5 hash of the beatmap. Used to verify integrity. + /// + public string MD5Hash { get; init; } = string.Empty; + + /// + /// The date when this metadata was last updated. + /// + public DateTimeOffset LastUpdated { get; init; } + } +}