h1or4dme8/osu-lazer
1
0
Fork 0
mirror of https://github.com/ppy/osu.git synced 2024-12-13 05:22:54 +08:00
Code Issues Releases Wiki Activity

Refactor metadata lookup to streamline online metadata application logic

Browse Source
This commit is contained in:
Bartłomiej Dach 2023-05-07 13:10:59 +02:00
parent a595b7804b
commit 7c8c6790d0
No known key found for this signature in database
5 changed files with 382 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
osu.Game/Beatmaps/APIBeatmapMetadataSource.cs Normal file
View File

@ -0,0 +1,78 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Diagnostics;
using osu.Game.Database;
using osu.Game.Online.API;
using osu.Game.Online.API.Requests;
namespace osu.Game.Beatmaps
{
public class APIBeatmapMetadataSource : IOnlineBeatmapMetadataSource
{
private readonly IAPIProvider api;
public APIBeatmapMetadataSource(IAPIProvider api)
{
this.api = api;
}
public bool Available => api.State.Value == APIState.Online;
public OnlineBeatmapMetadata? Lookup(BeatmapInfo beatmapInfo)
{
if (!Available)
return null;
Debug.Assert(beatmapInfo.BeatmapSet != null);
var req = new GetBeatmapRequest(beatmapInfo);
try
{
// intentionally blocking to limit web request concurrency
api.Perform(req);
if (req.CompletionState == APIRequestCompletionState.Failed)
{
logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval failed for {beatmapInfo}");
return null;
}
var res = req.Response;
if (res != null)
{
logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}.");
return new OnlineBeatmapMetadata
{
BeatmapID = res.OnlineID,
BeatmapSetID = res.OnlineBeatmapSetID,
AuthorID = res.AuthorID,
BeatmapStatus = res.Status,
BeatmapSetStatus = res.BeatmapSet?.Status,
DateRanked = res.BeatmapSet?.Ranked,
DateSubmitted = res.BeatmapSet?.Submitted,
MD5Hash = res.MD5Hash,
LastUpdated = res.LastUpdated
};
}
}
catch (Exception e)
{
logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval failed for {beatmapInfo} ({e.Message})");
}
return null;
}
private void logForModel(BeatmapSetInfo set, string message) =>
RealmArchiveModelImporter<BeatmapSetInfo>.LogForModel(set, $@"[{nameof(APIBeatmapMetadataSource)}] {message}");
public void Dispose()
{
}
}
}

251
osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs
View File

@ -1,62 +1,31 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
#nullable disable
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using osu.Framework.Development;
using osu.Framework.IO.Network;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Game.Database;
using osu.Game.Online.API;
using osu.Game.Online.API.Requests;
using SharpCompress.Compressors;
using SharpCompress.Compressors.BZip2;
using SQLitePCL;
namespace osu.Game.Beatmaps
{
/// <summary>
/// A component which handles population of online IDs for beatmaps using a two part lookup procedure.
/// </summary>
/// <remarks>
/// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to <see cref="cache_database_name"/>) will be downloaded if not already present locally.
/// This will always be checked before doing a second online query to get required metadata.
/// </remarks>
public class BeatmapUpdaterMetadataLookup : IDisposable
{
private readonly IAPIProvider api;
private readonly Storage storage;
private FileWebRequest cacheDownloadRequest;
private const string cache_database_name = "online.db";
private readonly IOnlineBeatmapMetadataSource apiMetadataSource;
private readonly IOnlineBeatmapMetadataSource localCachedMetadataSource;
public BeatmapUpdaterMetadataLookup(IAPIProvider api, Storage storage)
: this(new APIBeatmapMetadataSource(api), new LocalCachedBeatmapMetadataSource(storage))
{
try
{
// required to initialise native SQLite libraries on some platforms.
Batteries_V2.Init();
raw.sqlite3_config(2 /*SQLITE_CONFIG_MULTITHREAD*/);
}
catch
{
// may fail if platform not supported.
}
}
this.api = api;
this.storage = storage;
// avoid downloading / using cache for unit tests.
if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name))
prepareLocalCache();
internal BeatmapUpdaterMetadataLookup(IOnlineBeatmapMetadataSource apiMetadataSource, IOnlineBeatmapMetadataSource localCachedMetadataSource)
{
this.apiMetadataSource = apiMetadataSource;
this.localCachedMetadataSource = localCachedMetadataSource;
}
/// <summary>
@ -69,196 +38,57 @@ namespace osu.Game.Beatmaps
/// <param name="preferOnlineFetch">Whether metadata from an online source should be preferred. If <c>true</c>, the local cache will be skipped to ensure the freshest data state possible.</param>
public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch)
{
foreach (var b in beatmapSet.Beatmaps)
lookup(beatmapSet, b, preferOnlineFetch);
}
private void lookup(BeatmapSetInfo set, BeatmapInfo beatmapInfo, bool preferOnlineFetch)
{
bool apiAvailable = api?.State.Value == APIState.Online;
bool useLocalCache = !apiAvailable || !preferOnlineFetch;
if (useLocalCache && checkLocalCache(set, beatmapInfo))
return;
if (!apiAvailable)
return;
var req = new GetBeatmapRequest(beatmapInfo);
try
foreach (var beatmapInfo in beatmapSet.Beatmaps)
{
// intentionally blocking to limit web request concurrency
api.Perform(req);
var res = lookup(beatmapSet, beatmapInfo, preferOnlineFetch);
if (req.CompletionState == APIRequestCompletionState.Failed)
if (res == null)
{
logForModel(set, $"Online retrieval failed for {beatmapInfo}");
beatmapInfo.ResetOnlineInfo();
return;
continue;
}
var res = req.Response;
beatmapInfo.OnlineID = res.BeatmapID;
beatmapInfo.OnlineMD5Hash = res.MD5Hash;
beatmapInfo.LastOnlineUpdate = res.LastUpdated;
if (res != null)
Debug.Assert(beatmapInfo.BeatmapSet != null);
beatmapInfo.BeatmapSet.OnlineID = res.BeatmapSetID;
// Some metadata should only be applied if there's no local changes.
if (shouldSaveOnlineMetadata(beatmapInfo))
{
beatmapInfo.OnlineID = res.OnlineID;
beatmapInfo.OnlineMD5Hash = res.MD5Hash;
beatmapInfo.LastOnlineUpdate = res.LastUpdated;
Debug.Assert(beatmapInfo.BeatmapSet != null);
beatmapInfo.BeatmapSet.OnlineID = res.OnlineBeatmapSetID;
// Some metadata should only be applied if there's no local changes.
if (shouldSaveOnlineMetadata(beatmapInfo))
{
beatmapInfo.Status = res.Status;
beatmapInfo.Metadata.Author.OnlineID = res.AuthorID;
}
if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata))
{
beatmapInfo.BeatmapSet.Status = res.BeatmapSet?.Status ?? BeatmapOnlineStatus.None;
beatmapInfo.BeatmapSet.DateRanked = res.BeatmapSet?.Ranked;
beatmapInfo.BeatmapSet.DateSubmitted = res.BeatmapSet?.Submitted;
}
logForModel(set, $"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}.");
beatmapInfo.Status = res.BeatmapStatus;
beatmapInfo.Metadata.Author.OnlineID = res.AuthorID;
}
if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata))
{
beatmapInfo.BeatmapSet.Status = res.BeatmapSetStatus ?? BeatmapOnlineStatus.None;
beatmapInfo.BeatmapSet.DateRanked = res.DateRanked;
beatmapInfo.BeatmapSet.DateSubmitted = res.DateSubmitted;
}
}
catch (Exception e)
{
logForModel(set, $"Online retrieval failed for {beatmapInfo} ({e.Message})");
beatmapInfo.ResetOnlineInfo();
}
}
private void prepareLocalCache()
private OnlineBeatmapMetadata? lookup(BeatmapSetInfo set, BeatmapInfo beatmapInfo, bool preferOnlineFetch)
{
string cacheFilePath = storage.GetFullPath(cache_database_name);
string compressedCacheFilePath = $"{cacheFilePath}.bz2";
OnlineBeatmapMetadata? result = null;
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}");
bool useLocalCache = !apiMetadataSource.Available || !preferOnlineFetch;
cacheDownloadRequest.Failed += ex =>
{
File.Delete(compressedCacheFilePath);
File.Delete(cacheFilePath);
if (useLocalCache)
result = localCachedMetadataSource.Lookup(beatmapInfo);
Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}", LoggingTarget.Database);
};
if (result != null)
return result;
cacheDownloadRequest.Finished += () =>
{
try
{
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
using (var outStream = File.OpenWrite(cacheFilePath))
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
bz2.CopyTo(outStream);
if (apiMetadataSource.Available)
result = apiMetadataSource.Lookup(beatmapInfo);
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null;
}
catch (Exception ex)
{
Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
File.Delete(cacheFilePath);
}
finally
{
File.Delete(compressedCacheFilePath);
}
};
Task.Run(async () =>
{
try
{
await cacheDownloadRequest.PerformAsync().ConfigureAwait(false);
}
catch
{
// Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway.
}
});
return result;
}
private bool checkLocalCache(BeatmapSetInfo set, BeatmapInfo beatmapInfo)
{
// download is in progress (or was, and failed).
if (cacheDownloadRequest != null)
return false;
// database is unavailable.
if (!storage.Exists(cache_database_name))
return false;
if (string.IsNullOrEmpty(beatmapInfo.MD5Hash)
&& string.IsNullOrEmpty(beatmapInfo.Path)
&& beatmapInfo.OnlineID <= 0)
return false;
try
{
using (var db = new SqliteConnection(string.Concat("Data Source=", storage.GetFullPath($@"{"online.db"}", true))))
{
db.Open();
using (var cmd = db.CreateCommand())
{
cmd.CommandText =
"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path";
cmd.Parameters.Add(new SqliteParameter("@MD5Hash", beatmapInfo.MD5Hash));
cmd.Parameters.Add(new SqliteParameter("@OnlineID", beatmapInfo.OnlineID));
cmd.Parameters.Add(new SqliteParameter("@Path", beatmapInfo.Path));
using (var reader = cmd.ExecuteReader())
{
if (reader.Read())
{
var status = (BeatmapOnlineStatus)reader.GetByte(2);
// Some metadata should only be applied if there's no local changes.
if (shouldSaveOnlineMetadata(beatmapInfo))
{
beatmapInfo.Status = status;
beatmapInfo.Metadata.Author.OnlineID = reader.GetInt32(3);
}
// TODO: DateSubmitted and DateRanked are not provided by local cache.
beatmapInfo.OnlineID = reader.GetInt32(1);
beatmapInfo.OnlineMD5Hash = reader.GetString(4);
beatmapInfo.LastOnlineUpdate = reader.GetDateTimeOffset(5);
Debug.Assert(beatmapInfo.BeatmapSet != null);
beatmapInfo.BeatmapSet.OnlineID = reader.GetInt32(0);
if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata))
{
beatmapInfo.BeatmapSet.Status = status;
}
logForModel(set, $"Cached local retrieval for {beatmapInfo}.");
return true;
}
}
}
}
}
catch (Exception ex)
{
logForModel(set, $"Cached local retrieval for {beatmapInfo} failed with {ex}.");
}
return false;
}
private void logForModel(BeatmapSetInfo set, string message) =>
RealmArchiveModelImporter<BeatmapSetInfo>.LogForModel(set, $"[{nameof(BeatmapUpdaterMetadataLookup)}] {message}");
/// <summary>
/// Check whether the provided beatmap is in a state where online "ranked" status metadata should be saved against it.
/// Handles the case where a user may have locally modified a beatmap in the editor and expects the local status to stick.
@ -267,7 +97,8 @@ namespace osu.Game.Beatmaps
public void Dispose()
{
cacheDownloadRequest?.Dispose();
apiMetadataSource.Dispose();
localCachedMetadataSource.Dispose();
}
}
}

26
osu.Game/Beatmaps/IOnlineBeatmapMetadataSource.cs Normal file
View File

@ -0,0 +1,26 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
namespace osu.Game.Beatmaps
{
/// <summary>
/// Unifying interface for sources of online beatmap metadata.
/// </summary>
public interface IOnlineBeatmapMetadataSource : IDisposable
{
/// <summary>
/// Whether this source can currently service lookups.
/// </summary>
bool Available { get; }
/// <summary>
/// Looks up the online metadata for the supplied <paramref name="beatmapInfo"/>.
/// </summary>
/// <returns>
/// An <see cref="OnlineBeatmapMetadata"/> instance if the lookup is successful, or <see langword="null"/> if the lookup failed.
/// </returns>
OnlineBeatmapMetadata? Lookup(BeatmapInfo beatmapInfo);
}
}

176
osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs Normal file
View File

@ -0,0 +1,176 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Diagnostics;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using osu.Framework.Development;
using osu.Framework.IO.Network;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Game.Database;
using SharpCompress.Compressors;
using SharpCompress.Compressors.BZip2;
using SQLitePCL;
namespace osu.Game.Beatmaps
{
/// <summary>
///
/// </summary>
/// <remarks>
/// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to <see cref="cache_database_name"/>) will be downloaded if not already present locally.
/// </remarks>
public class LocalCachedBeatmapMetadataSource : IOnlineBeatmapMetadataSource
{
private readonly Storage storage;
private FileWebRequest? cacheDownloadRequest;
private const string cache_database_name = @"online.db";
public LocalCachedBeatmapMetadataSource(Storage storage)
{
try
{
// required to initialise native SQLite libraries on some platforms.
Batteries_V2.Init();
raw.sqlite3_config(2 /*SQLITE_CONFIG_MULTITHREAD*/);
}
catch
{
// may fail if platform not supported.
}
this.storage = storage;
// avoid downloading / using cache for unit tests.
if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name))
prepareLocalCache();
}
private void prepareLocalCache()
{
string cacheFilePath = storage.GetFullPath(cache_database_name);
string compressedCacheFilePath = $@"{cacheFilePath}.bz2";
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $@"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}");
cacheDownloadRequest.Failed += ex =>
{
File.Delete(compressedCacheFilePath);
File.Delete(cacheFilePath);
Logger.Log($@"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}", LoggingTarget.Database);
};
cacheDownloadRequest.Finished += () =>
{
try
{
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
using (var outStream = File.OpenWrite(cacheFilePath))
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
bz2.CopyTo(outStream);
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null;
}
catch (Exception ex)
{
Logger.Log($@"{nameof(LocalCachedBeatmapMetadataSource)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
File.Delete(cacheFilePath);
}
finally
{
File.Delete(compressedCacheFilePath);
}
};
Task.Run(async () =>
{
try
{
await cacheDownloadRequest.PerformAsync().ConfigureAwait(false);
}
catch
{
// Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway.
}
});
}
public bool Available =>
// no download in progress.
cacheDownloadRequest == null
// cached database exists on disk.
&& storage.Exists(cache_database_name);
public OnlineBeatmapMetadata? Lookup(BeatmapInfo beatmapInfo)
{
if (!Available)
return null;
if (string.IsNullOrEmpty(beatmapInfo.MD5Hash)
&& string.IsNullOrEmpty(beatmapInfo.Path)
&& beatmapInfo.OnlineID <= 0)
return null;
Debug.Assert(beatmapInfo.BeatmapSet != null);
try
{
using (var db = new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true))))
{
db.Open();
using (var cmd = db.CreateCommand())
{
cmd.CommandText =
@"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path";
cmd.Parameters.Add(new SqliteParameter(@"@MD5Hash", beatmapInfo.MD5Hash));
cmd.Parameters.Add(new SqliteParameter(@"@OnlineID", beatmapInfo.OnlineID));
cmd.Parameters.Add(new SqliteParameter(@"@Path", beatmapInfo.Path));
using (var reader = cmd.ExecuteReader())
{
if (reader.Read())
{
logForModel(beatmapInfo.BeatmapSet, $@"Cached local retrieval for {beatmapInfo}.");
return new OnlineBeatmapMetadata
{
BeatmapSetID = reader.GetInt32(0),
BeatmapID = reader.GetInt32(1),
BeatmapStatus = (BeatmapOnlineStatus)reader.GetByte(2),
BeatmapSetStatus = (BeatmapOnlineStatus)reader.GetByte(2),
AuthorID = reader.GetInt32(3),
MD5Hash = reader.GetString(4),
LastUpdated = reader.GetDateTimeOffset(5),
// TODO: DateSubmitted and DateRanked are not provided by local cache.
};
}
}
}
}
}
catch (Exception ex)
{
logForModel(beatmapInfo.BeatmapSet, $@"Cached local retrieval for {beatmapInfo} failed with {ex}.");
}
return null;
}
private void logForModel(BeatmapSetInfo set, string message) =>
RealmArchiveModelImporter<BeatmapSetInfo>.LogForModel(set, $@"[{nameof(LocalCachedBeatmapMetadataSource)}] {message}");
public void Dispose()
{
cacheDownloadRequest?.Dispose();
}
}
}

61
osu.Game/Beatmaps/OnlineBeatmapMetadata.cs Normal file
View File

@ -0,0 +1,61 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
namespace osu.Game.Beatmaps
{
/// <summary>
/// This structure contains parts of beatmap metadata which are involved with the online parts
/// of the game, and therefore must be treated with particular care.
/// This data is retrieved from trusted sources (such as osu-web API, or a locally downloaded sqlite snapshot
/// of osu-web metadata).
/// </summary>
public class OnlineBeatmapMetadata
{
/// <summary>
/// The online ID of the beatmap.
/// </summary>
public int BeatmapID { get; init; }
/// <summary>
/// The online ID of the beatmap set.
/// </summary>
public int BeatmapSetID { get; init; }
/// <summary>
/// The online ID of the author.
/// </summary>
public int AuthorID { get; init; }
/// <summary>
/// The online status of the beatmap.
/// </summary>
public BeatmapOnlineStatus BeatmapStatus { get; init; }
/// <summary>
/// The online status of the associated beatmap set.
/// </summary>
public BeatmapOnlineStatus? BeatmapSetStatus { get; init; }
/// <summary>
/// The rank date of the beatmap, if applicable and available.
/// </summary>
public DateTimeOffset? DateRanked { get; init; }
/// <summary>
/// The submission date of the beatmap, if available.
/// </summary>
public DateTimeOffset? DateSubmitted { get; init; }
/// <summary>
/// The MD5 hash of the beatmap. Used to verify integrity.
/// </summary>
public string MD5Hash { get; init; } = string.Empty;
/// <summary>
/// The date when this metadata was last updated.
/// </summary>
public DateTimeOffset LastUpdated { get; init; }
}
}