2020-05-02 13:35:12 +08:00
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System ;
using System.Diagnostics.CodeAnalysis ;
using System.IO ;
using System.Linq ;
using System.Threading ;
using System.Threading.Tasks ;
using Microsoft.Data.Sqlite ;
2020-05-03 12:25:57 +08:00
using osu.Framework.Development ;
2020-05-02 13:35:12 +08:00
using osu.Framework.IO.Network ;
using osu.Framework.Logging ;
using osu.Framework.Platform ;
2020-10-16 13:39:02 +08:00
using osu.Framework.Testing ;
2020-05-02 13:35:12 +08:00
using osu.Framework.Threading ;
using osu.Game.Online.API ;
using osu.Game.Online.API.Requests ;
using SharpCompress.Compressors ;
using SharpCompress.Compressors.BZip2 ;
namespace osu.Game.Beatmaps
{
public partial class BeatmapManager
{
2020-10-16 13:39:02 +08:00
[ExcludeFromDynamicCompile]
2020-05-22 22:26:37 +08:00
private class BeatmapOnlineLookupQueue : IDisposable
2020-05-02 13:35:12 +08:00
{
private readonly IAPIProvider api ;
private readonly Storage storage ;
private const int update_queue_request_concurrency = 4 ;
2020-05-03 08:35:48 +08:00
private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler ( update_queue_request_concurrency , nameof ( BeatmapOnlineLookupQueue ) ) ;
2020-05-02 13:35:12 +08:00
private FileWebRequest cacheDownloadRequest ;
private const string cache_database_name = "online.db" ;
2020-05-03 08:35:48 +08:00
public BeatmapOnlineLookupQueue ( IAPIProvider api , Storage storage )
2020-05-02 13:35:12 +08:00
{
this . api = api ;
this . storage = storage ;
2020-05-03 12:25:57 +08:00
// avoid downloading / using cache for unit tests.
if ( ! DebugUtils . IsNUnitRunning & & ! storage . Exists ( cache_database_name ) )
2020-05-02 13:35:12 +08:00
prepareLocalCache ( ) ;
}
public Task UpdateAsync ( BeatmapSetInfo beatmapSet , CancellationToken cancellationToken )
{
LogForModel ( beatmapSet , "Performing online lookups..." ) ;
return Task . WhenAll ( beatmapSet . Beatmaps . Select ( b = > UpdateAsync ( beatmapSet , b , cancellationToken ) ) . ToArray ( ) ) ;
}
// todo: expose this when we need to do individual difficulty lookups.
protected Task UpdateAsync ( BeatmapSetInfo beatmapSet , BeatmapInfo beatmap , CancellationToken cancellationToken )
2020-07-14 15:03:40 +08:00
= > Task . Factory . StartNew ( ( ) = > lookup ( beatmapSet , beatmap ) , cancellationToken , TaskCreationOptions . HideScheduler | TaskCreationOptions . RunContinuationsAsynchronously , updateScheduler ) ;
2020-05-02 13:35:12 +08:00
2020-05-03 08:35:48 +08:00
private void lookup ( BeatmapSetInfo set , BeatmapInfo beatmap )
2020-05-02 13:35:12 +08:00
{
2020-05-03 08:35:48 +08:00
if ( checkLocalCache ( set , beatmap ) )
return ;
2020-05-02 13:35:12 +08:00
2020-10-22 13:19:12 +08:00
if ( api ? . State . Value ! = APIState . Online )
2020-05-02 13:35:12 +08:00
return ;
var req = new GetBeatmapRequest ( beatmap ) ;
req . Failure + = fail ;
try
{
// intentionally blocking to limit web request concurrency
api . Perform ( req ) ;
var res = req . Result ;
if ( res ! = null )
{
beatmap . Status = res . Status ;
beatmap . BeatmapSet . Status = res . BeatmapSet . Status ;
beatmap . BeatmapSet . OnlineBeatmapSetID = res . OnlineBeatmapSetID ;
beatmap . OnlineBeatmapID = res . OnlineBeatmapID ;
LogForModel ( set , $"Online retrieval mapped {beatmap} to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}." ) ;
}
}
catch ( Exception e )
{
fail ( e ) ;
}
void fail ( Exception e )
{
beatmap . OnlineBeatmapID = null ;
LogForModel ( set , $"Online retrieval failed for {beatmap} ({e.Message})" ) ;
}
}
private void prepareLocalCache ( )
{
string cacheFilePath = storage . GetFullPath ( cache_database_name ) ;
string compressedCacheFilePath = $"{cacheFilePath}.bz2" ;
2020-12-11 16:56:00 +08:00
cacheDownloadRequest = new FileWebRequest ( compressedCacheFilePath , $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}" ) ;
2020-05-02 13:35:12 +08:00
cacheDownloadRequest . Failed + = ex = >
{
File . Delete ( compressedCacheFilePath ) ;
File . Delete ( cacheFilePath ) ;
2020-05-03 08:35:48 +08:00
Logger . Log ( $"{nameof(BeatmapOnlineLookupQueue)}'s online cache download failed: {ex}" , LoggingTarget . Database ) ;
2020-05-02 13:35:12 +08:00
} ;
cacheDownloadRequest . Finished + = ( ) = >
{
try
{
using ( var stream = File . OpenRead ( cacheDownloadRequest . Filename ) )
using ( var outStream = File . OpenWrite ( cacheFilePath ) )
using ( var bz2 = new BZip2Stream ( stream , CompressionMode . Decompress , false ) )
bz2 . CopyTo ( outStream ) ;
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null ;
}
catch ( Exception ex )
{
2020-05-03 08:35:48 +08:00
Logger . Log ( $"{nameof(BeatmapOnlineLookupQueue)}'s online cache extraction failed: {ex}" , LoggingTarget . Database ) ;
2020-05-03 08:31:56 +08:00
File . Delete ( cacheFilePath ) ;
2020-05-02 13:35:12 +08:00
}
finally
{
File . Delete ( compressedCacheFilePath ) ;
}
} ;
cacheDownloadRequest . PerformAsync ( ) ;
}
2020-05-03 08:35:48 +08:00
private bool checkLocalCache ( BeatmapSetInfo set , BeatmapInfo beatmap )
{
// download is in progress (or was, and failed).
if ( cacheDownloadRequest ! = null )
return false ;
// database is unavailable.
if ( ! storage . Exists ( cache_database_name ) )
return false ;
try
{
using ( var db = new SqliteConnection ( storage . GetDatabaseConnectionString ( "online" ) ) )
{
2021-01-28 15:53:56 +08:00
db . Open ( ) ;
2020-05-03 08:35:48 +08:00
2021-01-28 15:53:56 +08:00
using ( var cmd = db . CreateCommand ( ) )
2020-05-03 08:35:48 +08:00
{
2021-01-28 15:53:56 +08:00
cmd . CommandText = "SELECT beatmapset_id, beatmap_id, approved FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineBeatmapID OR filename = @Path" ;
cmd . Parameters . Add ( new SqliteParameter ( "@MD5Hash" , beatmap . MD5Hash ) ) ;
2021-01-29 18:53:56 +08:00
cmd . Parameters . Add ( new SqliteParameter ( "@OnlineBeatmapID" , beatmap . OnlineBeatmapID ? ? ( object ) DBNull . Value ) ) ;
2021-01-28 15:53:56 +08:00
cmd . Parameters . Add ( new SqliteParameter ( "@Path" , beatmap . Path ) ) ;
using ( var reader = cmd . ExecuteReader ( ) )
{
if ( reader . Read ( ) )
{
var status = ( BeatmapSetOnlineStatus ) reader . GetByte ( 2 ) ;
beatmap . Status = status ;
beatmap . BeatmapSet . Status = status ;
beatmap . BeatmapSet . OnlineBeatmapSetID = reader . GetInt32 ( 0 ) ;
beatmap . OnlineBeatmapID = reader . GetInt32 ( 1 ) ;
LogForModel ( set , $"Cached local retrieval for {beatmap}." ) ;
return true ;
}
}
2020-05-03 08:35:48 +08:00
}
}
}
catch ( Exception ex )
{
LogForModel ( set , $"Cached local retrieval for {beatmap} failed with {ex}." ) ;
}
return false ;
}
2020-05-22 22:26:37 +08:00
public void Dispose ( )
{
cacheDownloadRequest ? . Dispose ( ) ;
2020-07-09 13:46:58 +08:00
updateScheduler ? . Dispose ( ) ;
2020-05-22 22:26:37 +08:00
}
2020-05-02 13:35:12 +08:00
[Serializable]
[SuppressMessage("ReSharper", "InconsistentNaming")]
private class CachedOnlineBeatmapLookup
{
public int approved { get ; set ; }
public int? beatmapset_id { get ; set ; }
public int? beatmap_id { get ; set ; }
}
}
}
}