Merge pull request #29862 from bdach/backpopulate-rank-submission-dates

Backpopulate missing ranked/submitted dates using new local metadata cache

osu.Game/Beatmaps/LocalCachedBeatmapMetadataSource.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.Data.Sqlite;
@@ -78,7 +79,7 @@ namespace osu.Game.Beatmaps
             // cached database exists on disk.
             && storage.Exists(cache_database_name);
 
-        public bool TryLookup(BeatmapInfo beatmapInfo, out OnlineBeatmapMetadata? onlineMetadata)
+        public bool TryLookup(BeatmapInfo beatmapInfo, [NotNullWhen(true)] out OnlineBeatmapMetadata? onlineMetadata)
         {
             Debug.Assert(beatmapInfo.BeatmapSet != null);
 
@@ -98,7 +99,7 @@ namespace osu.Game.Beatmaps
 
             try
             {
-                using (var db = new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true))))
+                using (var db = getConnection())
                 {
                     db.Open();
 
@@ -125,6 +126,9 @@ namespace osu.Game.Beatmaps
             return false;
         }
 
+        private SqliteConnection getConnection() =>
+            new SqliteConnection(string.Concat(@"Data Source=", storage.GetFullPath(@"online.db", true)));
+
         private void prepareLocalCache()
         {
             bool isRefetch = storage.Exists(cache_database_name);
@@ -191,6 +195,15 @@ namespace osu.Game.Beatmaps
             });
         }
 
+        public int GetCacheVersion()
+        {
+            using (var connection = getConnection())
+            {
+                connection.Open();
+                return getCacheVersion(connection);
+            }
+        }
+
         private int getCacheVersion(SqliteConnection connection)
         {
             using (var cmd = connection.CreateCommand())

osu.Game/Database/BackgroundDataStoreProcessor.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -11,6 +12,7 @@ using osu.Framework.Allocation;
 using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Logging;
+using osu.Framework.Platform;
 using osu.Game.Beatmaps;
 using osu.Game.Extensions;
 using osu.Game.Online.API;
@@ -61,6 +63,9 @@ namespace osu.Game.Database
         [Resolved]
         private IAPIProvider api { get; set; } = null!;
 
+        [Resolved]
+        private Storage storage { get; set; } = null!;
+
         protected virtual int TimeToSleepDuringGameplay => 30000;
 
         protected override void LoadComplete()
@@ -78,6 +83,7 @@ namespace osu.Game.Database
                 processScoresWithMissingStatistics();
                 convertLegacyTotalScoreToStandardised();
                 upgradeScoreRanks();
+                backpopulateMissingSubmissionAndRankDates();
             }, TaskCreationOptions.LongRunning).ContinueWith(t =>
             {
                 if (t.Exception?.InnerException is ObjectDisposedException)
@@ -443,6 +449,104 @@ namespace osu.Game.Database
             completeNotification(notification, processedCount, scoreIds.Count, failedCount);
         }
 
+        private void backpopulateMissingSubmissionAndRankDates()
+        {
+            var localMetadataSource = new LocalCachedBeatmapMetadataSource(storage);
+
+            if (!localMetadataSource.Available)
+            {
+                Logger.Log("Cannot backpopulate missing submission/rank dates because the local metadata cache is missing.");
+                return;
+            }
+
+            try
+            {
+                if (localMetadataSource.GetCacheVersion() < 2)
+                {
+                    Logger.Log("Cannot backpopulate missing submission/rank dates because the local metadata cache is too old.");
+                    return;
+                }
+            }
+            catch (Exception ex)
+            {
+                Logger.Log($"Error when trying to query version of local metadata cache: {ex}");
+                return;
+            }
+
+            Logger.Log("Querying for beatmap sets that contain missing submission/rank date...");
+
+            HashSet<Guid> beatmapSetIds = realmAccess.Run(r => new HashSet<Guid>(
+                r.All<BeatmapSetInfo>()
+                 .Where(b => b.StatusInt > 0 && (b.DateRanked == null || b.DateSubmitted == null))
+                 .AsEnumerable()
+                 .Select(b => b.ID)));
+
+            if (beatmapSetIds.Count == 0)
+                return;
+
+            Logger.Log($"Found {beatmapSetIds.Count} beatmap sets with missing submission/rank date.");
+
+            var notification = showProgressNotification(beatmapSetIds.Count, "Populating missing submission and rank dates", "beatmap sets now have correct submission and rank dates.");
+
+            int processedCount = 0;
+            int failedCount = 0;
+
+            foreach (var id in beatmapSetIds)
+            {
+                if (notification?.State == ProgressNotificationState.Cancelled)
+                    break;
+
+                updateNotificationProgress(notification, processedCount, beatmapSetIds.Count);
+
+                sleepIfRequired();
+
+                try
+                {
+                    // Can't use async overload because we're not on the update thread.
+                    // ReSharper disable once MethodHasAsyncOverload
+                    bool succeeded = realmAccess.Write(r =>
+                    {
+                        BeatmapSetInfo beatmapSet = r.Find<BeatmapSetInfo>(id)!;
+
+                        // we want any ranked representative of the set.
+                        // the reason for checking ranked status of the difficulty is that it can be locally modified,
+                        // at which point the lookup will fail - but there might still be another unmodified difficulty on which it will work.
+                        if (beatmapSet.Beatmaps.FirstOrDefault(b => b.Status >= BeatmapOnlineStatus.Ranked) is not BeatmapInfo beatmap)
+                            return false;
+
+                        bool lookupSucceeded = localMetadataSource.TryLookup(beatmap, out var result);
+
+                        if (lookupSucceeded)
+                        {
+                            Debug.Assert(result != null);
+                            beatmapSet.DateRanked = result.DateRanked;
+                            beatmapSet.DateSubmitted = result.DateSubmitted;
+                            return true;
+                        }
+
+                        Logger.Log($"Could not find {beatmapSet.GetDisplayString()} in local cache while backpopulating missing submission/rank date");
+                        return false;
+                    });
+
+                    if (succeeded)
+                        ++processedCount;
+                    else
+                        ++failedCount;
+                }
+                catch (ObjectDisposedException)
+                {
+                    throw;
+                }
+                catch (Exception e)
+                {
+                    Logger.Log($"Failed to update ranked/submitted dates for beatmap set {id}: {e}");
+                    ++failedCount;
+                }
+            }
+
+            completeNotification(notification, processedCount, beatmapSetIds.Count, failedCount);
+        }
+
         private void updateNotificationProgress(ProgressNotification? notification, int processedCount, int totalCount)
         {
             if (notification == null)