// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. #nullable disable using System; using System.Diagnostics; using System.IO; using System.Threading.Tasks; using Microsoft.Data.Sqlite; using osu.Framework.Development; using osu.Framework.IO.Network; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Testing; using osu.Game.Database; using osu.Game.Online.API; using osu.Game.Online.API.Requests; using SharpCompress.Compressors; using SharpCompress.Compressors.BZip2; namespace osu.Game.Beatmaps { /// /// A component which handles population of online IDs for beatmaps using a two part lookup procedure. /// /// /// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to ) will be downloaded if not already present locally. /// This will always be checked before doing a second online query to get required metadata. /// [ExcludeFromDynamicCompile] public class BeatmapUpdaterMetadataLookup : IDisposable { private readonly IAPIProvider api; private readonly Storage storage; private FileWebRequest cacheDownloadRequest; private const string cache_database_name = "online.db"; public BeatmapUpdaterMetadataLookup(IAPIProvider api, Storage storage) { this.api = api; this.storage = storage; // avoid downloading / using cache for unit tests. if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name)) prepareLocalCache(); } /// /// Queue an update for a beatmap set. /// /// The beatmap set to update. Updates will be applied directly (so a transaction should be started if this instance is managed). /// Whether metadata from an online source should be preferred. If true, the local cache will be skipped to ensure the freshest data state possible. public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch) { foreach (var b in beatmapSet.Beatmaps) lookup(beatmapSet, b, preferOnlineFetch); } private void lookup(BeatmapSetInfo set, BeatmapInfo beatmapInfo, bool preferOnlineFetch) { bool apiAvailable = api?.State.Value == APIState.Online; bool useLocalCache = !apiAvailable || !preferOnlineFetch; if (useLocalCache && checkLocalCache(set, beatmapInfo)) return; if (!apiAvailable) return; var req = new GetBeatmapRequest(beatmapInfo); try { // intentionally blocking to limit web request concurrency api.Perform(req); if (req.CompletionState == APIRequestCompletionState.Failed) { logForModel(set, $"Online retrieval failed for {beatmapInfo}"); beatmapInfo.ResetOnlineInfo(); return; } var res = req.Response; if (res != null) { beatmapInfo.Status = res.Status; Debug.Assert(beatmapInfo.BeatmapSet != null); beatmapInfo.BeatmapSet.Status = res.BeatmapSet?.Status ?? BeatmapOnlineStatus.None; beatmapInfo.BeatmapSet.OnlineID = res.OnlineBeatmapSetID; beatmapInfo.BeatmapSet.DateRanked = res.BeatmapSet?.Ranked; beatmapInfo.BeatmapSet.DateSubmitted = res.BeatmapSet?.Submitted; beatmapInfo.OnlineMD5Hash = res.MD5Hash; beatmapInfo.LastOnlineUpdate = res.LastUpdated; beatmapInfo.OnlineID = res.OnlineID; beatmapInfo.Metadata.Author.OnlineID = res.AuthorID; logForModel(set, $"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}."); } } catch (Exception e) { logForModel(set, $"Online retrieval failed for {beatmapInfo} ({e.Message})"); beatmapInfo.ResetOnlineInfo(); } } private void prepareLocalCache() { string cacheFilePath = storage.GetFullPath(cache_database_name); string compressedCacheFilePath = $"{cacheFilePath}.bz2"; cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}"); cacheDownloadRequest.Failed += ex => { File.Delete(compressedCacheFilePath); File.Delete(cacheFilePath); Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}", LoggingTarget.Database); }; cacheDownloadRequest.Finished += () => { try { using (var stream = File.OpenRead(cacheDownloadRequest.Filename)) using (var outStream = File.OpenWrite(cacheFilePath)) using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false)) bz2.CopyTo(outStream); // set to null on completion to allow lookups to begin using the new source cacheDownloadRequest = null; } catch (Exception ex) { Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache extraction failed: {ex}", LoggingTarget.Database); File.Delete(cacheFilePath); } finally { File.Delete(compressedCacheFilePath); } }; Task.Run(async () => { try { await cacheDownloadRequest.PerformAsync(); } catch { // Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway. } }); } private bool checkLocalCache(BeatmapSetInfo set, BeatmapInfo beatmapInfo) { // download is in progress (or was, and failed). if (cacheDownloadRequest != null) return false; // database is unavailable. if (!storage.Exists(cache_database_name)) return false; if (string.IsNullOrEmpty(beatmapInfo.MD5Hash) && string.IsNullOrEmpty(beatmapInfo.Path) && beatmapInfo.OnlineID <= 0) return false; try { using (var db = new SqliteConnection(DatabaseContextFactory.CreateDatabaseConnectionString("online.db", storage))) { db.Open(); using (var cmd = db.CreateCommand()) { cmd.CommandText = "SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path"; cmd.Parameters.Add(new SqliteParameter("@MD5Hash", beatmapInfo.MD5Hash)); cmd.Parameters.Add(new SqliteParameter("@OnlineID", beatmapInfo.OnlineID)); cmd.Parameters.Add(new SqliteParameter("@Path", beatmapInfo.Path)); using (var reader = cmd.ExecuteReader()) { if (reader.Read()) { var status = (BeatmapOnlineStatus)reader.GetByte(2); beatmapInfo.Status = status; Debug.Assert(beatmapInfo.BeatmapSet != null); beatmapInfo.BeatmapSet.Status = status; beatmapInfo.BeatmapSet.OnlineID = reader.GetInt32(0); // TODO: DateSubmitted and DateRanked are not provided by local cache. beatmapInfo.OnlineID = reader.GetInt32(1); beatmapInfo.Metadata.Author.OnlineID = reader.GetInt32(3); beatmapInfo.OnlineMD5Hash = reader.GetString(4); beatmapInfo.LastOnlineUpdate = reader.GetDateTimeOffset(5); logForModel(set, $"Cached local retrieval for {beatmapInfo}."); return true; } } } } } catch (Exception ex) { logForModel(set, $"Cached local retrieval for {beatmapInfo} failed with {ex}."); } return false; } private void logForModel(BeatmapSetInfo set, string message) => RealmArchiveModelImporter.LogForModel(set, $"[{nameof(BeatmapUpdaterMetadataLookup)}] {message}"); public void Dispose() { cacheDownloadRequest?.Dispose(); } } }