mirror of
https://github.com/ppy/osu.git
synced 2024-12-14 02:22:59 +08:00
Refactor metadata lookup to streamline online metadata application logic
This commit is contained in:
parent
a595b7804b
commit
7c8c6790d0
78
osu.Game/Beatmaps/APIBeatmapMetadataSource.cs
Normal file
78
osu.Game/Beatmaps/APIBeatmapMetadataSource.cs
Normal file
@ -0,0 +1,78 @@
|
|||||||
|
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
|
||||||
|
// See the LICENCE file in the repository root for full licence text.
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using osu.Game.Database;
|
||||||
|
using osu.Game.Online.API;
|
||||||
|
using osu.Game.Online.API.Requests;
|
||||||
|
|
||||||
|
namespace osu.Game.Beatmaps
|
||||||
|
{
|
||||||
|
public class APIBeatmapMetadataSource : IOnlineBeatmapMetadataSource
|
||||||
|
{
|
||||||
|
private readonly IAPIProvider api;
|
||||||
|
|
||||||
|
public APIBeatmapMetadataSource(IAPIProvider api)
|
||||||
|
{
|
||||||
|
this.api = api;
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool Available => api.State.Value == APIState.Online;
|
||||||
|
|
||||||
|
public OnlineBeatmapMetadata? Lookup(BeatmapInfo beatmapInfo)
|
||||||
|
{
|
||||||
|
if (!Available)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
Debug.Assert(beatmapInfo.BeatmapSet != null);
|
||||||
|
|
||||||
|
var req = new GetBeatmapRequest(beatmapInfo);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// intentionally blocking to limit web request concurrency
|
||||||
|
api.Perform(req);
|
||||||
|
|
||||||
|
if (req.CompletionState == APIRequestCompletionState.Failed)
|
||||||
|
{
|
||||||
|
logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval failed for {beatmapInfo}");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var res = req.Response;
|
||||||
|
|
||||||
|
if (res != null)
|
||||||
|
{
|
||||||
|
logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}.");
|
||||||
|
|
||||||
|
return new OnlineBeatmapMetadata
|
||||||
|
{
|
||||||
|
BeatmapID = res.OnlineID,
|
||||||
|
BeatmapSetID = res.OnlineBeatmapSetID,
|
||||||
|
AuthorID = res.AuthorID,
|
||||||
|
BeatmapStatus = res.Status,
|
||||||
|
BeatmapSetStatus = res.BeatmapSet?.Status,
|
||||||
|
DateRanked = res.BeatmapSet?.Ranked,
|
||||||
|
DateSubmitted = res.BeatmapSet?.Submitted,
|
||||||
|
MD5Hash = res.MD5Hash,
|
||||||
|
LastUpdated = res.LastUpdated
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
logForModel(beatmapInfo.BeatmapSet, $@"Online retrieval failed for {beatmapInfo} ({e.Message})");
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void logForModel(BeatmapSetInfo set, string message) =>
|
||||||
|
RealmArchiveModelImporter<BeatmapSetInfo>.LogForModel(set, $@"[{nameof(APIBeatmapMetadataSource)}] {message}");
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,62 +1,31 @@
|
|||||||
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
|
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
|
||||||
// See the LICENCE file in the repository root for full licence text.
|
// See the LICENCE file in the repository root for full licence text.
|
||||||
|
|
||||||
#nullable disable
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Threading.Tasks;
|
|
||||||
using Microsoft.Data.Sqlite;
|
|
||||||
using osu.Framework.Development;
|
|
||||||
using osu.Framework.IO.Network;
|
|
||||||
using osu.Framework.Logging;
|
|
||||||
using osu.Framework.Platform;
|
using osu.Framework.Platform;
|
||||||
using osu.Game.Database;
|
|
||||||
using osu.Game.Online.API;
|
using osu.Game.Online.API;
|
||||||
using osu.Game.Online.API.Requests;
|
|
||||||
using SharpCompress.Compressors;
|
|
||||||
using SharpCompress.Compressors.BZip2;
|
|
||||||
using SQLitePCL;
|
|
||||||
|
|
||||||
namespace osu.Game.Beatmaps
|
namespace osu.Game.Beatmaps
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A component which handles population of online IDs for beatmaps using a two part lookup procedure.
|
/// A component which handles population of online IDs for beatmaps using a two part lookup procedure.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
|
||||||
/// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to <see cref="cache_database_name"/>) will be downloaded if not already present locally.
|
|
||||||
/// This will always be checked before doing a second online query to get required metadata.
|
|
||||||
/// </remarks>
|
|
||||||
public class BeatmapUpdaterMetadataLookup : IDisposable
|
public class BeatmapUpdaterMetadataLookup : IDisposable
|
||||||
{
|
{
|
||||||
private readonly IAPIProvider api;
|
private readonly IOnlineBeatmapMetadataSource apiMetadataSource;
|
||||||
private readonly Storage storage;
|
private readonly IOnlineBeatmapMetadataSource localCachedMetadataSource;
|
||||||
|
|
||||||
private FileWebRequest cacheDownloadRequest;
|
|
||||||
|
|
||||||
private const string cache_database_name = "online.db";
|
|
||||||
|
|
||||||
public BeatmapUpdaterMetadataLookup(IAPIProvider api, Storage storage)
|
public BeatmapUpdaterMetadataLookup(IAPIProvider api, Storage storage)
|
||||||
|
: this(new APIBeatmapMetadataSource(api), new LocalCachedBeatmapMetadataSource(storage))
|
||||||
{
|
{
|
||||||
try
|
}
|
||||||
{
|
|
||||||
// required to initialise native SQLite libraries on some platforms.
|
|
||||||
Batteries_V2.Init();
|
|
||||||
raw.sqlite3_config(2 /*SQLITE_CONFIG_MULTITHREAD*/);
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
// may fail if platform not supported.
|
|
||||||
}
|
|
||||||
|
|
||||||
this.api = api;
|
internal BeatmapUpdaterMetadataLookup(IOnlineBeatmapMetadataSource apiMetadataSource, IOnlineBeatmapMetadataSource localCachedMetadataSource)
|
||||||
this.storage = storage;
|
{
|
||||||
|
this.apiMetadataSource = apiMetadataSource;
|
||||||
// avoid downloading / using cache for unit tests.
|
this.localCachedMetadataSource = localCachedMetadataSource;
|
||||||
if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name))
|
|
||||||
prepareLocalCache();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@ -69,196 +38,57 @@ namespace osu.Game.Beatmaps
|
|||||||
/// <param name="preferOnlineFetch">Whether metadata from an online source should be preferred. If <c>true</c>, the local cache will be skipped to ensure the freshest data state possible.</param>
|
/// <param name="preferOnlineFetch">Whether metadata from an online source should be preferred. If <c>true</c>, the local cache will be skipped to ensure the freshest data state possible.</param>
|
||||||
public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch)
|
public void Update(BeatmapSetInfo beatmapSet, bool preferOnlineFetch)
|
||||||
{
|
{
|
||||||
foreach (var b in beatmapSet.Beatmaps)
|
foreach (var beatmapInfo in beatmapSet.Beatmaps)
|
||||||
lookup(beatmapSet, b, preferOnlineFetch);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void lookup(BeatmapSetInfo set, BeatmapInfo beatmapInfo, bool preferOnlineFetch)
|
|
||||||
{
|
|
||||||
bool apiAvailable = api?.State.Value == APIState.Online;
|
|
||||||
|
|
||||||
bool useLocalCache = !apiAvailable || !preferOnlineFetch;
|
|
||||||
|
|
||||||
if (useLocalCache && checkLocalCache(set, beatmapInfo))
|
|
||||||
return;
|
|
||||||
|
|
||||||
if (!apiAvailable)
|
|
||||||
return;
|
|
||||||
|
|
||||||
var req = new GetBeatmapRequest(beatmapInfo);
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
{
|
||||||
// intentionally blocking to limit web request concurrency
|
var res = lookup(beatmapSet, beatmapInfo, preferOnlineFetch);
|
||||||
api.Perform(req);
|
|
||||||
|
|
||||||
if (req.CompletionState == APIRequestCompletionState.Failed)
|
if (res == null)
|
||||||
{
|
{
|
||||||
logForModel(set, $"Online retrieval failed for {beatmapInfo}");
|
|
||||||
beatmapInfo.ResetOnlineInfo();
|
beatmapInfo.ResetOnlineInfo();
|
||||||
return;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
var res = req.Response;
|
beatmapInfo.OnlineID = res.BeatmapID;
|
||||||
|
beatmapInfo.OnlineMD5Hash = res.MD5Hash;
|
||||||
|
beatmapInfo.LastOnlineUpdate = res.LastUpdated;
|
||||||
|
|
||||||
if (res != null)
|
Debug.Assert(beatmapInfo.BeatmapSet != null);
|
||||||
|
beatmapInfo.BeatmapSet.OnlineID = res.BeatmapSetID;
|
||||||
|
|
||||||
|
// Some metadata should only be applied if there's no local changes.
|
||||||
|
if (shouldSaveOnlineMetadata(beatmapInfo))
|
||||||
{
|
{
|
||||||
beatmapInfo.OnlineID = res.OnlineID;
|
beatmapInfo.Status = res.BeatmapStatus;
|
||||||
beatmapInfo.OnlineMD5Hash = res.MD5Hash;
|
beatmapInfo.Metadata.Author.OnlineID = res.AuthorID;
|
||||||
beatmapInfo.LastOnlineUpdate = res.LastUpdated;
|
}
|
||||||
|
|
||||||
Debug.Assert(beatmapInfo.BeatmapSet != null);
|
if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata))
|
||||||
beatmapInfo.BeatmapSet.OnlineID = res.OnlineBeatmapSetID;
|
{
|
||||||
|
beatmapInfo.BeatmapSet.Status = res.BeatmapSetStatus ?? BeatmapOnlineStatus.None;
|
||||||
// Some metadata should only be applied if there's no local changes.
|
beatmapInfo.BeatmapSet.DateRanked = res.DateRanked;
|
||||||
if (shouldSaveOnlineMetadata(beatmapInfo))
|
beatmapInfo.BeatmapSet.DateSubmitted = res.DateSubmitted;
|
||||||
{
|
|
||||||
beatmapInfo.Status = res.Status;
|
|
||||||
beatmapInfo.Metadata.Author.OnlineID = res.AuthorID;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata))
|
|
||||||
{
|
|
||||||
beatmapInfo.BeatmapSet.Status = res.BeatmapSet?.Status ?? BeatmapOnlineStatus.None;
|
|
||||||
beatmapInfo.BeatmapSet.DateRanked = res.BeatmapSet?.Ranked;
|
|
||||||
beatmapInfo.BeatmapSet.DateSubmitted = res.BeatmapSet?.Submitted;
|
|
||||||
}
|
|
||||||
|
|
||||||
logForModel(set, $"Online retrieval mapped {beatmapInfo} to {res.OnlineBeatmapSetID} / {res.OnlineID}.");
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
logForModel(set, $"Online retrieval failed for {beatmapInfo} ({e.Message})");
|
|
||||||
beatmapInfo.ResetOnlineInfo();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void prepareLocalCache()
|
private OnlineBeatmapMetadata? lookup(BeatmapSetInfo set, BeatmapInfo beatmapInfo, bool preferOnlineFetch)
|
||||||
{
|
{
|
||||||
string cacheFilePath = storage.GetFullPath(cache_database_name);
|
OnlineBeatmapMetadata? result = null;
|
||||||
string compressedCacheFilePath = $"{cacheFilePath}.bz2";
|
|
||||||
|
|
||||||
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}");
|
bool useLocalCache = !apiMetadataSource.Available || !preferOnlineFetch;
|
||||||
|
|
||||||
cacheDownloadRequest.Failed += ex =>
|
if (useLocalCache)
|
||||||
{
|
result = localCachedMetadataSource.Lookup(beatmapInfo);
|
||||||
File.Delete(compressedCacheFilePath);
|
|
||||||
File.Delete(cacheFilePath);
|
|
||||||
|
|
||||||
Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}", LoggingTarget.Database);
|
if (result != null)
|
||||||
};
|
return result;
|
||||||
|
|
||||||
cacheDownloadRequest.Finished += () =>
|
if (apiMetadataSource.Available)
|
||||||
{
|
result = apiMetadataSource.Lookup(beatmapInfo);
|
||||||
try
|
|
||||||
{
|
|
||||||
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
|
|
||||||
using (var outStream = File.OpenWrite(cacheFilePath))
|
|
||||||
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
|
|
||||||
bz2.CopyTo(outStream);
|
|
||||||
|
|
||||||
// set to null on completion to allow lookups to begin using the new source
|
return result;
|
||||||
cacheDownloadRequest = null;
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
Logger.Log($"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
|
|
||||||
File.Delete(cacheFilePath);
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
File.Delete(compressedCacheFilePath);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Task.Run(async () =>
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await cacheDownloadRequest.PerformAsync().ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
// Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway.
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool checkLocalCache(BeatmapSetInfo set, BeatmapInfo beatmapInfo)
|
|
||||||
{
|
|
||||||
// download is in progress (or was, and failed).
|
|
||||||
if (cacheDownloadRequest != null)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
// database is unavailable.
|
|
||||||
if (!storage.Exists(cache_database_name))
|
|
||||||
return false;
|
|
||||||
|
|
||||||
if (string.IsNullOrEmpty(beatmapInfo.MD5Hash)
|
|
||||||
&& string.IsNullOrEmpty(beatmapInfo.Path)
|
|
||||||
&& beatmapInfo.OnlineID <= 0)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
using (var db = new SqliteConnection(string.Concat("Data Source=", storage.GetFullPath($@"{"online.db"}", true))))
|
|
||||||
{
|
|
||||||
db.Open();
|
|
||||||
|
|
||||||
using (var cmd = db.CreateCommand())
|
|
||||||
{
|
|
||||||
cmd.CommandText =
|
|
||||||
"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path";
|
|
||||||
|
|
||||||
cmd.Parameters.Add(new SqliteParameter("@MD5Hash", beatmapInfo.MD5Hash));
|
|
||||||
cmd.Parameters.Add(new SqliteParameter("@OnlineID", beatmapInfo.OnlineID));
|
|
||||||
cmd.Parameters.Add(new SqliteParameter("@Path", beatmapInfo.Path));
|
|
||||||
|
|
||||||
using (var reader = cmd.ExecuteReader())
|
|
||||||
{
|
|
||||||
if (reader.Read())
|
|
||||||
{
|
|
||||||
var status = (BeatmapOnlineStatus)reader.GetByte(2);
|
|
||||||
|
|
||||||
// Some metadata should only be applied if there's no local changes.
|
|
||||||
if (shouldSaveOnlineMetadata(beatmapInfo))
|
|
||||||
{
|
|
||||||
beatmapInfo.Status = status;
|
|
||||||
beatmapInfo.Metadata.Author.OnlineID = reader.GetInt32(3);
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: DateSubmitted and DateRanked are not provided by local cache.
|
|
||||||
beatmapInfo.OnlineID = reader.GetInt32(1);
|
|
||||||
beatmapInfo.OnlineMD5Hash = reader.GetString(4);
|
|
||||||
beatmapInfo.LastOnlineUpdate = reader.GetDateTimeOffset(5);
|
|
||||||
|
|
||||||
Debug.Assert(beatmapInfo.BeatmapSet != null);
|
|
||||||
beatmapInfo.BeatmapSet.OnlineID = reader.GetInt32(0);
|
|
||||||
|
|
||||||
if (beatmapInfo.BeatmapSet.Beatmaps.All(shouldSaveOnlineMetadata))
|
|
||||||
{
|
|
||||||
beatmapInfo.BeatmapSet.Status = status;
|
|
||||||
}
|
|
||||||
|
|
||||||
logForModel(set, $"Cached local retrieval for {beatmapInfo}.");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
logForModel(set, $"Cached local retrieval for {beatmapInfo} failed with {ex}.");
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void logForModel(BeatmapSetInfo set, string message) =>
|
|
||||||
RealmArchiveModelImporter<BeatmapSetInfo>.LogForModel(set, $"[{nameof(BeatmapUpdaterMetadataLookup)}] {message}");
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Check whether the provided beatmap is in a state where online "ranked" status metadata should be saved against it.
|
/// Check whether the provided beatmap is in a state where online "ranked" status metadata should be saved against it.
|
||||||
/// Handles the case where a user may have locally modified a beatmap in the editor and expects the local status to stick.
|
/// Handles the case where a user may have locally modified a beatmap in the editor and expects the local status to stick.
|
||||||
@ -267,7 +97,8 @@ namespace osu.Game.Beatmaps
|
|||||||
|
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
cacheDownloadRequest?.Dispose();
|
apiMetadataSource.Dispose();
|
||||||
|
localCachedMetadataSource.Dispose();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
26
osu.Game/Beatmaps/IOnlineBeatmapMetadataSource.cs
Normal file
26
osu.Game/Beatmaps/IOnlineBeatmapMetadataSource.cs
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
|
||||||
|
// See the LICENCE file in the repository root for full licence text.
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace osu.Game.Beatmaps
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Unifying interface for sources of online beatmap metadata.
|
||||||
|
/// </summary>
|
||||||
|
public interface IOnlineBeatmapMetadataSource : IDisposable
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Whether this source can currently service lookups.
|
||||||
|
/// </summary>
|
||||||
|
bool Available { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Looks up the online metadata for the supplied <paramref name="beatmapInfo"/>.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>
|
||||||
|
/// An <see cref="OnlineBeatmapMetadata"/> instance if the lookup is successful, or <see langword="null"/> if the lookup failed.
|
||||||
|
/// </returns>
|
||||||
|
OnlineBeatmapMetadata? Lookup(BeatmapInfo beatmapInfo);
|
||||||
|
}
|
||||||
|
}
|
176
osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs
Normal file
176
osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs
Normal file
@ -0,0 +1,176 @@
|
|||||||
|
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
|
||||||
|
// See the LICENCE file in the repository root for full licence text.
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.IO;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Microsoft.Data.Sqlite;
|
||||||
|
using osu.Framework.Development;
|
||||||
|
using osu.Framework.IO.Network;
|
||||||
|
using osu.Framework.Logging;
|
||||||
|
using osu.Framework.Platform;
|
||||||
|
using osu.Game.Database;
|
||||||
|
using SharpCompress.Compressors;
|
||||||
|
using SharpCompress.Compressors.BZip2;
|
||||||
|
using SQLitePCL;
|
||||||
|
|
||||||
|
namespace osu.Game.Beatmaps
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
///
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to <see cref="cache_database_name"/>) will be downloaded if not already present locally.
|
||||||
|
/// </remarks>
|
||||||
|
public class LocalCachedBeatmapMetadataSource : IOnlineBeatmapMetadataSource
|
||||||
|
{
|
||||||
|
private readonly Storage storage;
|
||||||
|
|
||||||
|
private FileWebRequest? cacheDownloadRequest;
|
||||||
|
|
||||||
|
private const string cache_database_name = @"online.db";
|
||||||
|
|
||||||
|
public LocalCachedBeatmapMetadataSource(Storage storage)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// required to initialise native SQLite libraries on some platforms.
|
||||||
|
Batteries_V2.Init();
|
||||||
|
raw.sqlite3_config(2 /*SQLITE_CONFIG_MULTITHREAD*/);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// may fail if platform not supported.
|
||||||
|
}
|
||||||
|
|
||||||
|
this.storage = storage;
|
||||||
|
|
||||||
|
// avoid downloading / using cache for unit tests.
|
||||||
|
if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name))
|
||||||
|
prepareLocalCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void prepareLocalCache()
|
||||||
|
{
|
||||||
|
string cacheFilePath = storage.GetFullPath(cache_database_name);
|
||||||
|
string compressedCacheFilePath = $@"{cacheFilePath}.bz2";
|
||||||
|
|
||||||
|
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $@"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}");
|
||||||
|
|
||||||
|
cacheDownloadRequest.Failed += ex =>
|
||||||
|
{
|
||||||
|
File.Delete(compressedCacheFilePath);
|
||||||
|
File.Delete(cacheFilePath);
|
||||||
|
|
||||||
|
Logger.Log($@"{nameof(BeatmapUpdaterMetadataLookup)}'s online cache download failed: {ex}", LoggingTarget.Database);
|
||||||
|
};
|
||||||
|
|
||||||
|
cacheDownloadRequest.Finished += () =>
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
|
||||||
|
using (var outStream = File.OpenWrite(cacheFilePath))
|
||||||
|
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
|
||||||
|
bz2.CopyTo(outStream);
|
||||||
|
|
||||||
|
// set to null on completion to allow lookups to begin using the new source
|
||||||
|
cacheDownloadRequest = null;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Logger.Log($@"{nameof(LocalCachedBeatmapMetadataSource)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
|
||||||
|
File.Delete(cacheFilePath);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
File.Delete(compressedCacheFilePath);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
Task.Run(async () =>
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await cacheDownloadRequest.PerformAsync().ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Prevent throwing unobserved exceptions, as they will be logged from the network request to the log file anyway.
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool Available =>
|
||||||
|
// no download in progress.
|
||||||
|
cacheDownloadRequest == null
|
||||||
|
// cached database exists on disk.
|
||||||
|
&& storage.Exists(cache_database_name);
|
||||||
|
|
||||||
|
public OnlineBeatmapMetadata? Lookup(BeatmapInfo beatmapInfo)
|
||||||
|
{
|
||||||
|
if (!Available)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(beatmapInfo.MD5Hash)
|
||||||
|
&& string.IsNullOrEmpty(beatmapInfo.Path)
|
||||||
|
&& beatmapInfo.OnlineID <= 0)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
Debug.Assert(beatmapInfo.BeatmapSet != null);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using (var db = new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true))))
|
||||||
|
{
|
||||||
|
db.Open();
|
||||||
|
|
||||||
|
using (var cmd = db.CreateCommand())
|
||||||
|
{
|
||||||
|
cmd.CommandText =
|
||||||
|
@"SELECT beatmapset_id, beatmap_id, approved, user_id, checksum, last_update FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineID OR filename = @Path";
|
||||||
|
|
||||||
|
cmd.Parameters.Add(new SqliteParameter(@"@MD5Hash", beatmapInfo.MD5Hash));
|
||||||
|
cmd.Parameters.Add(new SqliteParameter(@"@OnlineID", beatmapInfo.OnlineID));
|
||||||
|
cmd.Parameters.Add(new SqliteParameter(@"@Path", beatmapInfo.Path));
|
||||||
|
|
||||||
|
using (var reader = cmd.ExecuteReader())
|
||||||
|
{
|
||||||
|
if (reader.Read())
|
||||||
|
{
|
||||||
|
logForModel(beatmapInfo.BeatmapSet, $@"Cached local retrieval for {beatmapInfo}.");
|
||||||
|
|
||||||
|
return new OnlineBeatmapMetadata
|
||||||
|
{
|
||||||
|
BeatmapSetID = reader.GetInt32(0),
|
||||||
|
BeatmapID = reader.GetInt32(1),
|
||||||
|
BeatmapStatus = (BeatmapOnlineStatus)reader.GetByte(2),
|
||||||
|
BeatmapSetStatus = (BeatmapOnlineStatus)reader.GetByte(2),
|
||||||
|
AuthorID = reader.GetInt32(3),
|
||||||
|
MD5Hash = reader.GetString(4),
|
||||||
|
LastUpdated = reader.GetDateTimeOffset(5),
|
||||||
|
// TODO: DateSubmitted and DateRanked are not provided by local cache.
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
logForModel(beatmapInfo.BeatmapSet, $@"Cached local retrieval for {beatmapInfo} failed with {ex}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void logForModel(BeatmapSetInfo set, string message) =>
|
||||||
|
RealmArchiveModelImporter<BeatmapSetInfo>.LogForModel(set, $@"[{nameof(LocalCachedBeatmapMetadataSource)}] {message}");
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
cacheDownloadRequest?.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
61
osu.Game/Beatmaps/OnlineBeatmapMetadata.cs
Normal file
61
osu.Game/Beatmaps/OnlineBeatmapMetadata.cs
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
|
||||||
|
// See the LICENCE file in the repository root for full licence text.
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace osu.Game.Beatmaps
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// This structure contains parts of beatmap metadata which are involved with the online parts
|
||||||
|
/// of the game, and therefore must be treated with particular care.
|
||||||
|
/// This data is retrieved from trusted sources (such as osu-web API, or a locally downloaded sqlite snapshot
|
||||||
|
/// of osu-web metadata).
|
||||||
|
/// </summary>
|
||||||
|
public class OnlineBeatmapMetadata
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// The online ID of the beatmap.
|
||||||
|
/// </summary>
|
||||||
|
public int BeatmapID { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The online ID of the beatmap set.
|
||||||
|
/// </summary>
|
||||||
|
public int BeatmapSetID { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The online ID of the author.
|
||||||
|
/// </summary>
|
||||||
|
public int AuthorID { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The online status of the beatmap.
|
||||||
|
/// </summary>
|
||||||
|
public BeatmapOnlineStatus BeatmapStatus { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The online status of the associated beatmap set.
|
||||||
|
/// </summary>
|
||||||
|
public BeatmapOnlineStatus? BeatmapSetStatus { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The rank date of the beatmap, if applicable and available.
|
||||||
|
/// </summary>
|
||||||
|
public DateTimeOffset? DateRanked { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The submission date of the beatmap, if available.
|
||||||
|
/// </summary>
|
||||||
|
public DateTimeOffset? DateSubmitted { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The MD5 hash of the beatmap. Used to verify integrity.
|
||||||
|
/// </summary>
|
||||||
|
public string MD5Hash { get; init; } = string.Empty;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The date when this metadata was last updated.
|
||||||
|
/// </summary>
|
||||||
|
public DateTimeOffset LastUpdated { get; init; }
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user