h1or4dme8/osu-lazer
1
0
Fork 0
mirror of https://github.com/ppy/osu.git synced 2025-01-12 21:02:54 +08:00
Code Issues Releases Wiki Activity

Merge pull request #23822 from bdach/refactor-metadata-lookup-sources

Browse Source 
Refactor metadata lookup flows
This commit is contained in:
Dean Herbert 2024-02-11 20:37:45 +08:00 committed by GitHub
commit 33d4e8f821
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 621 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

197
osu.Game.Tests/Beatmaps/BeatmapUpdaterMetadataLookupTest.cs Normal file
View File

@ -0,0 +1,197 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using Moq;
using NUnit.Framework;
using osu.Framework.Extensions.IEnumerableExtensions;
using osu.Game.Beatmaps;
namespace osu.Game.Tests.Beatmaps
{
[TestFixture]
public class BeatmapUpdaterMetadataLookupTest
{
private Mock<IOnlineBeatmapMetadataSource> apiMetadataSourceMock = null!;
private Mock<IOnlineBeatmapMetadataSource> localCachedMetadataSourceMock = null!;
private BeatmapUpdaterMetadataLookup metadataLookup = null!;
[SetUp]
public void SetUp()
{
apiMetadataSourceMock = new Mock<IOnlineBeatmapMetadataSource>();
localCachedMetadataSourceMock = new Mock<IOnlineBeatmapMetadataSource>();
metadataLookup = new BeatmapUpdaterMetadataLookup(apiMetadataSourceMock.Object, localCachedMetadataSourceMock.Object);
}
[Test]
public void TestLocalCacheQueriedFirst()
{
var localLookupResult = new OnlineBeatmapMetadata { BeatmapID = 123456, BeatmapStatus = BeatmapOnlineStatus.Ranked };
localCachedMetadataSourceMock.Setup(src => src.Available).Returns(true);
localCachedMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out localLookupResult))
.Returns(true);
apiMetadataSourceMock.Setup(src => src.Available).Returns(true);
var beatmap = new BeatmapInfo { OnlineID = 123456 };
var beatmapSet = new BeatmapSetInfo(beatmap.Yield());
beatmap.BeatmapSet = beatmapSet;
metadataLookup.Update(beatmapSet, preferOnlineFetch: false);
Assert.That(beatmap.Status, Is.EqualTo(BeatmapOnlineStatus.Ranked));
localCachedMetadataSourceMock.Verify(src => src.TryLookup(beatmap, out It.Ref<OnlineBeatmapMetadata>.IsAny!), Times.Once);
apiMetadataSourceMock.Verify(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out It.Ref<OnlineBeatmapMetadata>.IsAny!), Times.Never);
}
[Test]
public void TestAPIQueriedSecond()
{
OnlineBeatmapMetadata? localLookupResult = null;
localCachedMetadataSourceMock.Setup(src => src.Available).Returns(true);
localCachedMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out localLookupResult))
.Returns(false);
var onlineLookupResult = new OnlineBeatmapMetadata { BeatmapID = 123456, BeatmapStatus = BeatmapOnlineStatus.Ranked };
apiMetadataSourceMock.Setup(src => src.Available).Returns(true);
apiMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out onlineLookupResult))
.Returns(true);
var beatmap = new BeatmapInfo { OnlineID = 123456 };
var beatmapSet = new BeatmapSetInfo(beatmap.Yield());
beatmap.BeatmapSet = beatmapSet;
metadataLookup.Update(beatmapSet, preferOnlineFetch: false);
Assert.That(beatmap.Status, Is.EqualTo(BeatmapOnlineStatus.Ranked));
localCachedMetadataSourceMock.Verify(src => src.TryLookup(beatmap, out It.Ref<OnlineBeatmapMetadata?>.IsAny!), Times.Once);
apiMetadataSourceMock.Verify(src => src.TryLookup(beatmap, out It.Ref<OnlineBeatmapMetadata?>.IsAny!), Times.Once);
}
[Test]
public void TestPreferOnlineFetch()
{
var localLookupResult = new OnlineBeatmapMetadata { BeatmapID = 123456, BeatmapStatus = BeatmapOnlineStatus.Ranked };
localCachedMetadataSourceMock.Setup(src => src.Available).Returns(true);
localCachedMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out localLookupResult))
.Returns(true);
var onlineLookupResult = new OnlineBeatmapMetadata { BeatmapID = 123456, BeatmapStatus = BeatmapOnlineStatus.Graveyard };
apiMetadataSourceMock.Setup(src => src.Available).Returns(true);
apiMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out onlineLookupResult))
.Returns(true);
var beatmap = new BeatmapInfo { OnlineID = 123456 };
var beatmapSet = new BeatmapSetInfo(beatmap.Yield());
beatmap.BeatmapSet = beatmapSet;
metadataLookup.Update(beatmapSet, preferOnlineFetch: true);
Assert.That(beatmap.Status, Is.EqualTo(BeatmapOnlineStatus.Graveyard));
localCachedMetadataSourceMock.Verify(src => src.TryLookup(beatmap, out It.Ref<OnlineBeatmapMetadata?>.IsAny!), Times.Never);
apiMetadataSourceMock.Verify(src => src.TryLookup(beatmap, out It.Ref<OnlineBeatmapMetadata?>.IsAny!), Times.Once);
}
[Test]
public void TestPreferOnlineFetchFallsBackToLocalCacheIfOnlineSourceUnavailable()
{
var localLookupResult = new OnlineBeatmapMetadata { BeatmapID = 123456, BeatmapStatus = BeatmapOnlineStatus.Ranked };
localCachedMetadataSourceMock.Setup(src => src.Available).Returns(true);
localCachedMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out localLookupResult))
.Returns(true);
apiMetadataSourceMock.Setup(src => src.Available).Returns(false);
var beatmap = new BeatmapInfo { OnlineID = 123456 };
var beatmapSet = new BeatmapSetInfo(beatmap.Yield());
beatmap.BeatmapSet = beatmapSet;
metadataLookup.Update(beatmapSet, preferOnlineFetch: true);
Assert.That(beatmap.Status, Is.EqualTo(BeatmapOnlineStatus.Ranked));
localCachedMetadataSourceMock.Verify(src => src.TryLookup(beatmap, out It.Ref<OnlineBeatmapMetadata?>.IsAny!), Times.Once);
apiMetadataSourceMock.Verify(src => src.TryLookup(beatmap, out It.Ref<OnlineBeatmapMetadata?>.IsAny!), Times.Never);
}
[Test]
public void TestMetadataLookupFailed()
{
OnlineBeatmapMetadata? lookupResult = null;
localCachedMetadataSourceMock.Setup(src => src.Available).Returns(true);
localCachedMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out lookupResult))
.Returns(false);
apiMetadataSourceMock.Setup(src => src.Available).Returns(true);
apiMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out lookupResult))
.Returns(true);
var beatmap = new BeatmapInfo { OnlineID = 123456 };
var beatmapSet = new BeatmapSetInfo(beatmap.Yield());
beatmap.BeatmapSet = beatmapSet;
metadataLookup.Update(beatmapSet, preferOnlineFetch: false);
Assert.That(beatmap.Status, Is.EqualTo(BeatmapOnlineStatus.None));
Assert.That(beatmap.OnlineID, Is.EqualTo(-1));
localCachedMetadataSourceMock.Verify(src => src.TryLookup(beatmap, out It.Ref<OnlineBeatmapMetadata?>.IsAny!), Times.Once);
apiMetadataSourceMock.Verify(src => src.TryLookup(beatmap, out It.Ref<OnlineBeatmapMetadata?>.IsAny!), Times.Once);
}
/// <remarks>
/// For the time being, if we fail to find a match in the local cache but online retrieval is not available, we trust the incoming beatmap verbatim wrt online ID.
/// While this is suboptimal as it implicitly trusts the contents of the beatmap,
/// throwing away the online data would be anti-user as it would make all beatmaps imported offline stop working in online.
/// TODO: revisit if/when we have a better flow of queueing metadata retrieval.
/// </remarks>
[Test]
public void TestLocalMetadataLookupReturnedNoMatchAndOnlineLookupIsUnavailable([Values] bool preferOnlineFetch)
{
OnlineBeatmapMetadata? localLookupResult = null;
localCachedMetadataSourceMock.Setup(src => src.Available).Returns(true);
localCachedMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out localLookupResult))
.Returns(false);
apiMetadataSourceMock.Setup(src => src.Available).Returns(false);
var beatmap = new BeatmapInfo { OnlineID = 123456 };
var beatmapSet = new BeatmapSetInfo(beatmap.Yield());
beatmap.BeatmapSet = beatmapSet;
metadataLookup.Update(beatmapSet, preferOnlineFetch);
Assert.That(beatmap.Status, Is.EqualTo(BeatmapOnlineStatus.None));
Assert.That(beatmap.OnlineID, Is.EqualTo(123456));
}
/// <remarks>
/// For the time being, if there are no available metadata lookup sources, we trust the incoming beatmap verbatim wrt online ID.
/// While this is suboptimal as it implicitly trusts the contents of the beatmap,
/// throwing away the online data would be anti-user as it would make all beatmaps imported offline stop working in online.
/// TODO: revisit if/when we have a better flow of queueing metadata retrieval.
/// </remarks>
[Test]
public void TestNoAvailableSources([Values] bool preferOnlineFetch)
{
OnlineBeatmapMetadata? lookupResult = null;
localCachedMetadataSourceMock.Setup(src => src.Available).Returns(false);
localCachedMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out lookupResult))
.Returns(false);
apiMetadataSourceMock.Setup(src => src.Available).Returns(false);
apiMetadataSourceMock.Setup(src => src.TryLookup(It.IsAny<BeatmapInfo>(), out lookupResult))
.Returns(false);
var beatmap = new BeatmapInfo { OnlineID = 123456 };
var beatmapSet = new BeatmapSetInfo(beatmap.Yield());
beatmap.BeatmapSet = beatmapSet;
metadataLookup.Update(beatmapSet, preferOnlineFetch);
Assert.That(beatmap.OnlineID, Is.EqualTo(123456));
}
}
}

89
osu.Game/Beatmaps/APIBeatmapMetadataSource.cs Normal file
View File

@ -0,0 +1,89 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Diagnostics;
using osu.Game.Database;
using osu.Game.Online.API;
using osu.Game.Online.API.Requests;
namespace osu.Game.Beatmaps
{
/// <summary>
/// Performs online metadata lookups using the osu-web API.
/// </summary>
public class APIBeatmapMetadataSource : IOnlineBeatmapMetadataSource
{
private readonly IAPIProvider api;
public APIBeatmapMetadataSource(IAPIProvider api)
{
this.api = api;
}
public bool Available => api.State.Value == APIState.Online;
public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? onlineMetadata)
{
if (!Available)
{
onlineMetadata = null;
return false;
}
Debug.Assert(beatmapInfo.BeatmapSet != null);
var req = new GetBeatmapRequest(beatmapInfo);
try
{
// intentionally blocking to limit web request concurrency
api.Perform(req);
if (req.CompletionState == APIRequestCompletionState.Failed)
{
logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval failed for {beatmapInfo}");
onlineMetadata = null;
return true;
}
var res = req.Response;
if (res != null)
{
logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}.");
onlineMetadata = new OnlineBeatmapMetadata
{
BeatmapID = res.OnlineID,
BeatmapSetID = res.OnlineBeatmapSetID,
AuthorID = res.AuthorID,
BeatmapStatus = res.Status,
BeatmapSetStatus = res.BeatmapSet?.Status,
DateRanked = res.BeatmapSet?.Ranked,
DateSubmitted = res.BeatmapSet?.Submitted,
MD5Hash = res.MD5Hash,
LastUpdated = res.LastUpdated
};
return true;
}
}
catch (Exception e)
{
logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval failed for {beatmapInfo} ({e.Message})");
onlineMetadata = null;
return false;
}
onlineMetadata = null;
return false;
}
private void logForModel(BeatmapSetInfo set, string message) =>
RealmArchiveModelImporter<BeatmapSetInfo>.LogForModel(set, $@"[{nameof(APIBeatmapMetadataSource)}] {message}");
public void Dispose()
{
}
}
}

270
osu.Game/Beatmaps/BeatmapUpdaterMetadataLookup.cs
View File

@ -1,62 +1,31 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
#nullable disable
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using osu.Framework.Development;
using osu.Framework.IO.Network;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Game.Database;
using osu.Game.Online.API;
using osu.Game.Online.API.Requests;
using SharpCompress.Compressors;
using SharpCompress.Compressors.BZip2;
using SQLitePCL;
namespace osu.Game.Beatmaps
{
/// <summary>
/// A component which handles population of online IDs for beatmaps using a two part lookup procedure.
/// </summary>
/// <remarks>
/// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to <see cref="cache_database_name"/>) will be downloaded if not already present locally.
/// This will always be checked before doing a second online query to get required metadata.
/// </remarks>
public class BeatmapUpdaterMetadataLookup : IDisposable
{
private readonly IAPIProvider api;
private readonly Storage storage;
private FileWebRequest cacheDownloadRequest;
private const string cache_database_name = "online.db";
private readonly IOnlineBeatmapMetadataSource apiMetadataSource;
private readonly IOnlineBeatmapMetadataSource localCachedMetadataSource;
public BeatmapUpdaterMetadataLookup(IAPIProvider api, Storage storage)
: this(new APIBeatmapMetadataSource(api), new LocalCachedBeatmapMetadataSource(storage))
{
try
{
// required to initialise native SQLite libraries on some platforms.
Batteries_V2.Init();
raw.sqlite3_config(2 /*SQLITE_CONFIG_MULTITHREAD*/);
}
catch
{
// may fail if platform not supported.
}
}
this.api = api;
this.storage = storage;
// avoid downloading / using cache for unit tests.
if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name))
prepareLocalCache();
internal BeatmapUpdaterMetadataLookup(IOnlineBeatmapMetadataSource apiMetadataSource, IOnlineBeatmapMetadataSource localCachedMetadataSource)
{
this.apiMetadataSource = apiMetadataSource;
this.localCachedMetadataSource = localCachedMetadataSource;
}
/// <summary>
@ -69,196 +38,72 @@ namespace osu.Game.Beatmaps
/// <param name="preferOnlineFetch">Whether metadata from an online source should be preferred. If <c>true</c>, the local cache will be skipped to ensure the freshest data state possible.</param>
public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch)
{
foreach (var b in beatmapSet.Beatmaps)
lookup(beatmapSet, b, preferOnlineFetch);
}
private void lookup(BeatmapSetInfo set, BeatmapInfo beatmapInfo, bool preferOnlineFetch)
{
bool apiAvailable = api?.State.Value == APIState.Online;
bool useLocalCache = !apiAvailable || !preferOnlineFetch;
if (useLocalCache && checkLocalCache(set, beatmapInfo))
return;
if (!apiAvailable)
return;
var req = new GetBeatmapRequest(beatmapInfo);
try
foreach (var beatmapInfo in beatmapSet.Beatmaps)
{
// intentionally blocking to limit web request concurrency
api.Perform(req);
if (!tryLookup(beatmapInfo, preferOnlineFetch, out var res))
continue;
if (req.CompletionState == APIRequestCompletionState.Failed)
if (res == null)
{
logForModel(set, $"Online retrieval failed for {beatmapInfo}");
beatmapInfo.ResetOnlineInfo();
return;
continue;
}
var res = req.Response;
beatmapInfo.OnlineID = res.BeatmapID;
beatmapInfo.OnlineMD5Hash = res.MD5Hash;
beatmapInfo.LastOnlineUpdate = res.LastUpdated;
if (res != null)
Debug.Assert(beatmapInfo.BeatmapSet != null);
beatmapInfo.BeatmapSet.OnlineID = res.BeatmapSetID;
// Some metadata should only be applied if there's no local changes.
if (shouldSaveOnlineMetadata(beatmapInfo))
{
beatmapInfo.OnlineID = res.OnlineID;
beatmapInfo.OnlineMD5Hash = res.MD5Hash;
beatmapInfo.LastOnlineUpdate = res.LastUpdated;
Debug.Assert(beatmapInfo.BeatmapSet != null);
beatmapInfo.BeatmapSet.OnlineID = res.OnlineBeatmapSetID;
// Some metadata should only be applied if there's no local changes.
if (shouldSaveOnlineMetadata(beatmapInfo))
{
beatmapInfo.Status = res.Status;
beatmapInfo.Metadata.Author.OnlineID = res.AuthorID;
}
if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata))
{
beatmapInfo.BeatmapSet.Status = res.BeatmapSet?.Status ?? BeatmapOnlineStatus.None;
beatmapInfo.BeatmapSet.DateRanked = res.BeatmapSet?.Ranked;
beatmapInfo.BeatmapSet.DateSubmitted = res.BeatmapSet?.Submitted;
}
logForModel(set, $"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}.");
beatmapInfo.Status = res.BeatmapStatus;
beatmapInfo.Metadata.Author.OnlineID = res.AuthorID;
}
if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata))
{
beatmapInfo.BeatmapSet.Status = res.BeatmapSetStatus ?? BeatmapOnlineStatus.None;
beatmapInfo.BeatmapSet.DateRanked = res.DateRanked;
beatmapInfo.BeatmapSet.DateSubmitted = res.DateSubmitted;
}
}
catch (Exception e)
{
logForModel(set, $"Online retrieval failed for {beatmapInfo} ({e.Message})");
beatmapInfo.ResetOnlineInfo();
}
}
private void prepareLocalCache()
/// <summary>
/// Attempts to retrieve the <see cref="OnlineBeatmapMetadata"/> for the given <paramref name="beatmapInfo"/>.
/// </summary>
/// <param name="beatmapInfo">The beatmap to perform the online lookup for.</param>
/// <param name="preferOnlineFetch">Whether online sources should be preferred for the lookup.</param>
/// <param name="result">The result of the lookup. Can be <see langword="null"/> if no matching beatmap was found (or the lookup failed).</param>
/// <returns>
/// <see langword="true"/> if any of the metadata sources were available and returned a valid <paramref name="result"/>.
/// <see langword="false"/> if none of the metadata sources were available, or if there was insufficient data to return a valid <paramref name="result"/>.
/// </returns>
/// <remarks>
/// There are two cases wherein this method will return <see langword="false"/>:
/// <list type="bullet">
/// <item>If neither the local cache or the API are available to query.</item>
/// <item>If the API is not available to query, and a positive match was not made in the local cache.</item>
/// </list>
/// In either case, the online ID read from the .osu file will be preserved, which may not necessarily be what we want.
/// TODO: reconsider this if/when a better flow for queueing online retrieval is implemented.
/// </remarks>
private bool tryLookup(BeatmapInfo beatmapInfo, bool preferOnlineFetch, out OnlineBeatmapMetadata? result)
{
string cacheFilePath = storage.GetFullPath(cache_database_name);
string compressedCacheFilePath = $"{cacheFilePath}.bz2";
bool useLocalCache = !apiMetadataSource.Available || !preferOnlineFetch;
if (useLocalCache && localCachedMetadataSource.TryLookup(beatmapInfo, out result))
return true;
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}");
cacheDownloadRequest.Failed += ex =>
{
File.Delete(compressedCacheFilePath);
File.Delete(cacheFilePath);
Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}", LoggingTarget.Database);
};
cacheDownloadRequest.Finished += () =>
{
try
{
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
using (var outStream = File.OpenWrite(cacheFilePath))
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
bz2.CopyTo(outStream);
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null;
}
catch (Exception ex)
{
Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
File.Delete(cacheFilePath);
}
finally
{
File.Delete(compressedCacheFilePath);
}
};
Task.Run(async () =>
{
try
{
await cacheDownloadRequest.PerformAsync().ConfigureAwait(false);
}
catch
{
// Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway.
}
});
}
private bool checkLocalCache(BeatmapSetInfo set, BeatmapInfo beatmapInfo)
{
// download is in progress (or was, and failed).
if (cacheDownloadRequest != null)
return false;
// database is unavailable.
if (!storage.Exists(cache_database_name))
return false;
if (string.IsNullOrEmpty(beatmapInfo.MD5Hash)
&& string.IsNullOrEmpty(beatmapInfo.Path)
&& beatmapInfo.OnlineID <= 0)
return false;
try
{
using (var db = new SqliteConnection(string.Concat("Data Source=", storage.GetFullPath($@"{"online.db"}", true))))
{
db.Open();
using (var cmd = db.CreateCommand())
{
cmd.CommandText =
"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path";
cmd.Parameters.Add(new SqliteParameter("@MD5Hash", beatmapInfo.MD5Hash));
cmd.Parameters.Add(new SqliteParameter("@OnlineID", beatmapInfo.OnlineID));
cmd.Parameters.Add(new SqliteParameter("@Path", beatmapInfo.Path));
using (var reader = cmd.ExecuteReader())
{
if (reader.Read())
{
var status = (BeatmapOnlineStatus)reader.GetByte(2);
// Some metadata should only be applied if there's no local changes.
if (shouldSaveOnlineMetadata(beatmapInfo))
{
beatmapInfo.Status = status;
beatmapInfo.Metadata.Author.OnlineID = reader.GetInt32(3);
}
// TODO: DateSubmitted and DateRanked are not provided by local cache.
beatmapInfo.OnlineID = reader.GetInt32(1);
beatmapInfo.OnlineMD5Hash = reader.GetString(4);
beatmapInfo.LastOnlineUpdate = reader.GetDateTimeOffset(5);
Debug.Assert(beatmapInfo.BeatmapSet != null);
beatmapInfo.BeatmapSet.OnlineID = reader.GetInt32(0);
if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata))
{
beatmapInfo.BeatmapSet.Status = status;
}
logForModel(set, $"Cached local retrieval for {beatmapInfo}.");
return true;
}
}
}
}
}
catch (Exception ex)
{
logForModel(set, $"Cached local retrieval for {beatmapInfo} failed with {ex}.");
}
if (apiMetadataSource.TryLookup(beatmapInfo, out result))
return true;
result = null;
return false;
}
private void logForModel(BeatmapSetInfo set, string message) =>
RealmArchiveModelImporter<BeatmapSetInfo>.LogForModel(set, $"[{nameof(BeatmapUpdaterMetadataLookup)}] {message}");
/// <summary>
/// Check whether the provided beatmap is in a state where online "ranked" status metadata should be saved against it.
/// Handles the case where a user may have locally modified a beatmap in the editor and expects the local status to stick.
@ -267,7 +112,8 @@ namespace osu.Game.Beatmaps
public void Dispose()
{
cacheDownloadRequest?.Dispose();
apiMetadataSource.Dispose();
localCachedMetadataSource.Dispose();
}
}
}

32
osu.Game/Beatmaps/IOnlineBeatmapMetadataSource.cs Normal file
View File

@ -0,0 +1,32 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
namespace osu.Game.Beatmaps
{
/// <summary>
/// Unifying interface for sources of <see cref="OnlineBeatmapMetadata"/>.
/// </summary>
public interface IOnlineBeatmapMetadataSource : IDisposable
{
/// <summary>
/// Whether this source can currently service lookups.
/// </summary>
bool Available { get; }
/// <summary>
/// Looks up the online metadata for the supplied <paramref name="beatmapInfo"/>.
/// </summary>
/// <param name="beatmapInfo">The <see cref="BeatmapInfo"/> to look up.</param>
/// <param name="onlineMetadata">
/// An <see cref="OnlineBeatmapMetadata"/> instance if the lookup is successful.
/// <see langword="null"/> if a mismatch between the local instance and the looked-up data was detected.
/// The returned value is only valid if the return value of the method is <see langword="true"/>.
/// </param>
/// <returns>
/// Whether the lookup was performed.
/// </returns>
bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? onlineMetadata);
}
}

184
osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs Normal file
View File

@ -0,0 +1,184 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Diagnostics;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using osu.Framework.Development;
using osu.Framework.IO.Network;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Game.Database;
using SharpCompress.Compressors;
using SharpCompress.Compressors.BZip2;
using SQLitePCL;
namespace osu.Game.Beatmaps
{
/// <summary>
/// Performs online metadata lookups using a copy of a database containing metadata for a large subset of beatmaps (stored to <see cref="cache_database_name"/>).
/// The database will be asynchronously downloaded - if not already present locally - when this component is constructed.
/// </summary>
public class LocalCachedBeatmapMetadataSource : IOnlineBeatmapMetadataSource
{
private readonly Storage storage;
private FileWebRequest? cacheDownloadRequest;
private const string cache_database_name = @"online.db";
public LocalCachedBeatmapMetadataSource(Storage storage)
{
try
{
// required to initialise native SQLite libraries on some platforms.
Batteries_V2.Init();
raw.sqlite3_config(2 /*SQLITE_CONFIG_MULTITHREAD*/);
}
catch
{
// may fail if platform not supported.
}
this.storage = storage;
// avoid downloading / using cache for unit tests.
if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name))
prepareLocalCache();
}
public bool Available =>
// no download in progress.
cacheDownloadRequest == null
// cached database exists on disk.
&& storage.Exists(cache_database_name);
public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? onlineMetadata)
{
if (!Available)
{
onlineMetadata = null;
return false;
}
if (string.IsNullOrEmpty(beatmapInfo.MD5Hash)
&& string.IsNullOrEmpty(beatmapInfo.Path)
&& beatmapInfo.OnlineID <= 0)
{
onlineMetadata = null;
return false;
}
Debug.Assert(beatmapInfo.BeatmapSet != null);
try
{
using (var db = new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true))))
{
db.Open();
using (var cmd = db.CreateCommand())
{
cmd.CommandText =
@"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path";
cmd.Parameters.Add(new SqliteParameter(@"@MD5Hash", beatmapInfo.MD5Hash));
cmd.Parameters.Add(new SqliteParameter(@"@OnlineID", beatmapInfo.OnlineID));
cmd.Parameters.Add(new SqliteParameter(@"@Path", beatmapInfo.Path));
using (var reader = cmd.ExecuteReader())
{
if (reader.Read())
{
logForModel(beatmapInfo.BeatmapSet, $@"Cached local retrieval for {beatmapInfo}.");
onlineMetadata = new OnlineBeatmapMetadata
{
BeatmapSetID = reader.GetInt32(0),
BeatmapID = reader.GetInt32(1),
BeatmapStatus = (BeatmapOnlineStatus)reader.GetByte(2),
BeatmapSetStatus = (BeatmapOnlineStatus)reader.GetByte(2),
AuthorID = reader.GetInt32(3),
MD5Hash = reader.GetString(4),
LastUpdated = reader.GetDateTimeOffset(5),
// TODO: DateSubmitted and DateRanked are not provided by local cache.
};
return true;
}
}
}
}
}
catch (Exception ex)
{
logForModel(beatmapInfo.BeatmapSet, $@"Cached local retrieval for {beatmapInfo} failed with {ex}.");
onlineMetadata = null;
return false;
}
onlineMetadata = null;
return false;
}
private void prepareLocalCache()
{
string cacheFilePath = storage.GetFullPath(cache_database_name);
string compressedCacheFilePath = $@"{cacheFilePath}.bz2";
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $@"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}");
cacheDownloadRequest.Failed += ex =>
{
File.Delete(compressedCacheFilePath);
File.Delete(cacheFilePath);
Logger.Log($@"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}", LoggingTarget.Database);
};
cacheDownloadRequest.Finished += () =>
{
try
{
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
using (var outStream = File.OpenWrite(cacheFilePath))
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
bz2.CopyTo(outStream);
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null;
}
catch (Exception ex)
{
Logger.Log($@"{nameof(LocalCachedBeatmapMetadataSource)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
File.Delete(cacheFilePath);
}
finally
{
File.Delete(compressedCacheFilePath);
}
};
Task.Run(async () =>
{
try
{
await cacheDownloadRequest.PerformAsync().ConfigureAwait(false);
}
catch
{
// Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway.
}
});
}
private void logForModel(BeatmapSetInfo set, string message) =>
RealmArchiveModelImporter<BeatmapSetInfo>.LogForModel(set, $@"[{nameof(LocalCachedBeatmapMetadataSource)}] {message}");
public void Dispose()
{
cacheDownloadRequest?.Dispose();
}
}
}

61
osu.Game/Beatmaps/OnlineBeatmapMetadata.cs Normal file
View File

@ -0,0 +1,61 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
namespace osu.Game.Beatmaps
{
/// <summary>
/// This structure contains parts of beatmap metadata which are involved with the online parts
/// of the game, and therefore must be treated with particular care.
/// This data is retrieved from trusted sources (such as osu-web API, or a locally downloaded sqlite snapshot
/// of osu-web metadata).
/// </summary>
public class OnlineBeatmapMetadata
{
/// <summary>
/// The online ID of the beatmap.
/// </summary>
public int BeatmapID { get; init; }
/// <summary>
/// The online ID of the beatmap set.
/// </summary>
public int BeatmapSetID { get; init; }
/// <summary>
/// The online ID of the author.
/// </summary>
public int AuthorID { get; init; }
/// <summary>
/// The online status of the beatmap.
/// </summary>
public BeatmapOnlineStatus BeatmapStatus { get; init; }
/// <summary>
/// The online status of the associated beatmap set.
/// </summary>
public BeatmapOnlineStatus? BeatmapSetStatus { get; init; }
/// <summary>
/// The rank date of the beatmap, if applicable and available.
/// </summary>
public DateTimeOffset? DateRanked { get; init; }
/// <summary>
/// The submission date of the beatmap, if available.
/// </summary>
public DateTimeOffset? DateSubmitted { get; init; }
/// <summary>
/// The MD5 hash of the beatmap. Used to verify integrity.
/// </summary>
public string MD5Hash { get; init; } = string.Empty;
/// <summary>
/// The date when this metadata was last updated.
/// </summary>
public DateTimeOffset LastUpdated { get; init; }
}
}