Merge pull request #19272 from peppy/beatmap-background-reprocessing

Add background beatmap processing

osu.Game.Rulesets.Catch/Difficulty/CatchDifficultyCalculator.cs

@@ -25,6 +25,8 @@ namespace osu.Game.Rulesets.Catch.Difficulty
 
         private float halfCatcherWidth;
 
+        public override int Version => 20220701;
+
         public CatchDifficultyCalculator(IRulesetInfo ruleset, IWorkingBeatmap beatmap)
             : base(ruleset, beatmap)
         {

osu.Game.Rulesets.Mania/Difficulty/ManiaDifficultyCalculator.cs

@@ -31,6 +31,8 @@ namespace osu.Game.Rulesets.Mania.Difficulty
         private readonly bool isForCurrentRuleset;
         private readonly double originalOverallDifficulty;
 
+        public override int Version => 20220701;
+
         public ManiaDifficultyCalculator(IRulesetInfo ruleset, IWorkingBeatmap beatmap)
             : base(ruleset, beatmap)
         {

osu.Game.Rulesets.Osu/Difficulty/OsuDifficultyCalculator.cs

@@ -25,6 +25,8 @@ namespace osu.Game.Rulesets.Osu.Difficulty
         private const double difficulty_multiplier = 0.0675;
         private double hitWindowGreat;
 
+        public override int Version => 20220701;
+
         public OsuDifficultyCalculator(IRulesetInfo ruleset, IWorkingBeatmap beatmap)
             : base(ruleset, beatmap)
         {

osu.Game.Rulesets.Taiko/Difficulty/TaikoDifficultyCalculator.cs

@@ -26,6 +26,8 @@ namespace osu.Game.Rulesets.Taiko.Difficulty
         private const double colour_skill_multiplier = 0.01;
         private const double stamina_skill_multiplier = 0.021;
 
+        public override int Version => 20220701;
+
         public TaikoDifficultyCalculator(IRulesetInfo ruleset, IWorkingBeatmap beatmap)
             : base(ruleset, beatmap)
         {

osu.Game.Tests/Database/BackgroundBeatmapProcessorTests.cs

@@ -0,0 +1,132 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Linq;
+using NUnit.Framework;
+using osu.Framework.Allocation;
+using osu.Framework.Bindables;
+using osu.Framework.Extensions;
+using osu.Framework.Testing;
+using osu.Game.Beatmaps;
+using osu.Game.Screens.Play;
+using osu.Game.Tests.Beatmaps.IO;
+using osu.Game.Tests.Visual;
+
+namespace osu.Game.Tests.Database
+{
+    [HeadlessTest]
+    public class BackgroundBeatmapProcessorTests : OsuTestScene, ILocalUserPlayInfo
+    {
+        public IBindable<bool> IsPlaying => isPlaying;
+
+        private readonly Bindable<bool> isPlaying = new Bindable<bool>();
+
+        private BeatmapSetInfo importedSet = null!;
+
+        [BackgroundDependencyLoader]
+        private void load(OsuGameBase osu)
+        {
+            importedSet = BeatmapImportHelper.LoadQuickOszIntoOsu(osu).GetResultSafely();
+        }
+
+        [SetUpSteps]
+        public void SetUpSteps()
+        {
+            AddStep("Set not playing", () => isPlaying.Value = false);
+        }
+
+        [Test]
+        public void TestDifficultyProcessing()
+        {
+            AddAssert("Difficulty is initially set", () =>
+            {
+                return Realm.Run(r =>
+                {
+                    var beatmapSetInfo = r.Find<BeatmapSetInfo>(importedSet.ID);
+                    return beatmapSetInfo.Beatmaps.All(b => b.StarRating > 0);
+                });
+            });
+
+            AddStep("Reset difficulty", () =>
+            {
+                Realm.Write(r =>
+                {
+                    var beatmapSetInfo = r.Find<BeatmapSetInfo>(importedSet.ID);
+                    foreach (var b in beatmapSetInfo.Beatmaps)
+                        b.StarRating = -1;
+                });
+            });
+
+            AddStep("Run background processor", () =>
+            {
+                Add(new TestBackgroundBeatmapProcessor());
+            });
+
+            AddUntilStep("wait for difficulties repopulated", () =>
+            {
+                return Realm.Run(r =>
+                {
+                    var beatmapSetInfo = r.Find<BeatmapSetInfo>(importedSet.ID);
+                    return beatmapSetInfo.Beatmaps.All(b => b.StarRating > 0);
+                });
+            });
+        }
+
+        [Test]
+        public void TestDifficultyProcessingWhilePlaying()
+        {
+            AddAssert("Difficulty is initially set", () =>
+            {
+                return Realm.Run(r =>
+                {
+                    var beatmapSetInfo = r.Find<BeatmapSetInfo>(importedSet.ID);
+                    return beatmapSetInfo.Beatmaps.All(b => b.StarRating > 0);
+                });
+            });
+
+            AddStep("Set playing", () => isPlaying.Value = true);
+
+            AddStep("Reset difficulty", () =>
+            {
+                Realm.Write(r =>
+                {
+                    var beatmapSetInfo = r.Find<BeatmapSetInfo>(importedSet.ID);
+                    foreach (var b in beatmapSetInfo.Beatmaps)
+                        b.StarRating = -1;
+                });
+            });
+
+            AddStep("Run background processor", () =>
+            {
+                Add(new TestBackgroundBeatmapProcessor());
+            });
+
+            AddWaitStep("wait some", 500);
+
+            AddAssert("Difficulty still not populated", () =>
+            {
+                return Realm.Run(r =>
+                {
+                    var beatmapSetInfo = r.Find<BeatmapSetInfo>(importedSet.ID);
+                    return beatmapSetInfo.Beatmaps.All(b => b.StarRating == -1);
+                });
+            });
+
+            AddStep("Set not playing", () => isPlaying.Value = false);
+
+            AddUntilStep("wait for difficulties repopulated", () =>
+            {
+                return Realm.Run(r =>
+                {
+                    var beatmapSetInfo = r.Find<BeatmapSetInfo>(importedSet.ID);
+                    return beatmapSetInfo.Beatmaps.All(b => b.StarRating > 0);
+                });
+            });
+        }
+
+        public class TestBackgroundBeatmapProcessor : BackgroundBeatmapProcessor
+        {
+            protected override int TimeToSleepDuringGameplay => 10;
+        }
+    }
+}

osu.Game/BackgroundBeatmapProcessor.cs

@@ -0,0 +1,137 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using osu.Framework.Allocation;
+using osu.Framework.Bindables;
+using osu.Framework.Graphics;
+using osu.Framework.Logging;
+using osu.Game.Beatmaps;
+using osu.Game.Database;
+using osu.Game.Rulesets;
+using osu.Game.Screens.Play;
+
+namespace osu.Game
+{
+    public class BackgroundBeatmapProcessor : Component
+    {
+        [Resolved]
+        private RulesetStore rulesetStore { get; set; } = null!;
+
+        [Resolved]
+        private RealmAccess realmAccess { get; set; } = null!;
+
+        [Resolved]
+        private BeatmapUpdater beatmapUpdater { get; set; } = null!;
+
+        [Resolved]
+        private IBindable<WorkingBeatmap> gameBeatmap { get; set; } = null!;
+
+        [Resolved]
+        private ILocalUserPlayInfo? localUserPlayInfo { get; set; }
+
+        protected virtual int TimeToSleepDuringGameplay => 30000;
+
+        protected override void LoadComplete()
+        {
+            base.LoadComplete();
+
+            Task.Run(() =>
+            {
+                Logger.Log("Beginning background beatmap processing..");
+                checkForOutdatedStarRatings();
+                processBeatmapSetsWithMissingMetrics();
+            }).ContinueWith(t =>
+            {
+                if (t.Exception?.InnerException is ObjectDisposedException)
+                {
+                    Logger.Log("Finished background aborted during shutdown");
+                    return;
+                }
+
+                Logger.Log("Finished background beatmap processing!");
+            });
+        }
+
+        /// <summary>
+        /// Check whether the databased difficulty calculation version matches the latest ruleset provided version.
+        /// If it doesn't, clear out any existing difficulties so they can be incrementally recalculated.
+        /// </summary>
+        private void checkForOutdatedStarRatings()
+        {
+            foreach (var ruleset in rulesetStore.AvailableRulesets)
+            {
+                // beatmap being passed in is arbitrary here. just needs to be non-null.
+                int currentVersion = ruleset.CreateInstance().CreateDifficultyCalculator(gameBeatmap.Value).Version;
+
+                if (ruleset.LastAppliedDifficultyVersion < currentVersion)
+                {
+                    Logger.Log($"Resetting star ratings for {ruleset.Name} (difficulty calculation version updated from {ruleset.LastAppliedDifficultyVersion} to {currentVersion})");
+
+                    int countReset = 0;
+
+                    realmAccess.Write(r =>
+                    {
+                        foreach (var b in r.All<BeatmapInfo>())
+                        {
+                            if (b.Ruleset.ShortName == ruleset.ShortName)
+                            {
+                                b.StarRating = -1;
+                                countReset++;
+                            }
+                        }
+
+                        r.Find<RulesetInfo>(ruleset.ShortName).LastAppliedDifficultyVersion = currentVersion;
+                    });
+
+                    Logger.Log($"Finished resetting {countReset} beatmap sets for {ruleset.Name}");
+                }
+            }
+        }
+
+        private void processBeatmapSetsWithMissingMetrics()
+        {
+            HashSet<Guid> beatmapSetIds = new HashSet<Guid>();
+
+            Logger.Log("Querying for beatmap sets to reprocess...");
+
+            realmAccess.Run(r =>
+            {
+                foreach (var b in r.All<BeatmapInfo>().Where(b => b.StarRating < 0 || (b.OnlineID > 0 && b.LastOnlineUpdate == null)))
+                {
+                    Debug.Assert(b.BeatmapSet != null);
+                    beatmapSetIds.Add(b.BeatmapSet.ID);
+                }
+            });
+
+            Logger.Log($"Found {beatmapSetIds.Count} beatmap sets which require reprocessing.");
+
+            int i = 0;
+
+            foreach (var id in beatmapSetIds)
+            {
+                while (localUserPlayInfo?.IsPlaying.Value == true)
+                {
+                    Logger.Log("Background processing sleeping due to active gameplay...");
+                    Thread.Sleep(TimeToSleepDuringGameplay);
+                }
+
+                realmAccess.Run(r =>
+                {
+                    var set = r.Find<BeatmapSetInfo>(id);
+
+                    if (set != null)
+                    {
+                        Logger.Log($"Background processing {set} ({++i} / {beatmapSetIds.Count})");
+                        beatmapUpdater.Process(set);
+                    }
+                });
+            }
+        }
+    }
+}

osu.Game/Beatmaps/BeatmapInfo.cs

@@ -87,7 +87,11 @@ namespace osu.Game.Beatmaps
 
         public string Hash { get; set; } = string.Empty;
 
-        public double StarRating { get; set; }
+        /// <summary>
+        /// Defaults to -1 (meaning not-yet-calculated).
+        /// Will likely be superseded with a better storage considering ruleset/mods.
+        /// </summary>
+        public double StarRating { get; set; } = -1;
 
         [Indexed]
         public string MD5Hash { get; set; } = string.Empty;

osu.Game/Database/RealmAccess.cs

@@ -63,8 +63,9 @@ namespace osu.Game.Database
         /// 17   2022-07-16    Added CountryCode to RealmUser.
         /// 18   2022-07-19    Added OnlineMD5Hash and LastOnlineUpdate to BeatmapInfo.
         /// 19   2022-07-19    Added DateSubmitted and DateRanked to BeatmapSetInfo.
+        /// 20   2022-07-21    Added LastAppliedDifficultyVersion to RulesetInfo, changed default value of BeatmapInfo.StarRating to -1.
         /// </summary>
-        private const int schema_version = 19;
+        private const int schema_version = 20;
 
         /// <summary>
         /// Lock object which is held during <see cref="BlockAllOperations"/> sections, blocking realm retrieval during blocking periods.
@@ -780,6 +781,15 @@ namespace osu.Game.Database
                 case 14:
                     foreach (var beatmap in migration.NewRealm.All<BeatmapInfo>())
                         beatmap.UserSettings = new BeatmapUserSettings();
+
+                    break;
+
+                case 20:
+                    // As we now have versioned difficulty calculations, let's reset
+                    // all star ratings and have `BackgroundBeatmapProcessor` recalculate them.
+                    foreach (var beatmap in migration.NewRealm.All<BeatmapInfo>())
+                        beatmap.StarRating = -1;
+
                     break;
             }
         }

osu.Game/OsuGame.cs

@@ -904,6 +904,8 @@ namespace osu.Game
 
             loadComponentSingleFile(CreateHighPerformanceSession(), Add);
 
+            loadComponentSingleFile(new BackgroundBeatmapProcessor(), Add);
+
             chatOverlay.State.BindValueChanged(_ => updateChatPollRate());
             // Multiplayer modes need to increase poll rate temporarily.
             API.Activity.BindValueChanged(_ => updateChatPollRate(), true);

osu.Game/OsuGameBase.cs

@@ -280,8 +280,7 @@ namespace osu.Game
             AddInternal(difficultyCache);
 
             // TODO: OsuGame or OsuGameBase?
-            beatmapUpdater = new BeatmapUpdater(BeatmapManager, difficultyCache, API, Storage);
-
+            dependencies.CacheAs(beatmapUpdater = new BeatmapUpdater(BeatmapManager, difficultyCache, API, Storage));
             dependencies.CacheAs(spectatorClient = new OnlineSpectatorClient(endpoints));
             dependencies.CacheAs(multiplayerClient = new OnlineMultiplayerClient(endpoints));
             dependencies.CacheAs(metadataClient = new OnlineMetadataClient(endpoints));

osu.Game/Rulesets/Difficulty/DifficultyCalculator.cs

@@ -34,6 +34,11 @@ namespace osu.Game.Rulesets.Difficulty
         private readonly IRulesetInfo ruleset;
         private readonly IWorkingBeatmap beatmap;
 
+        /// <summary>
+        /// A yymmdd version which is used to discern when reprocessing is required.
+        /// </summary>
+        public virtual int Version => 0;
+
         protected DifficultyCalculator(IRulesetInfo ruleset, IWorkingBeatmap beatmap)
         {
             this.ruleset = ruleset;

osu.Game/Rulesets/RulesetInfo.cs

@@ -4,6 +4,7 @@
 using System;
 using JetBrains.Annotations;
 using osu.Framework.Testing;
+using osu.Game.Rulesets.Difficulty;
 using Realms;
 
 namespace osu.Game.Rulesets
@@ -22,6 +23,11 @@ namespace osu.Game.Rulesets
 
         public string InstantiationInfo { get; set; } = string.Empty;
 
+        /// <summary>
+        /// Stores the last applied <see cref="DifficultyCalculator.Version"/>
+        /// </summary>
+        public int LastAppliedDifficultyVersion { get; set; }
+
         public RulesetInfo(string shortName, string name, string instantiationInfo, int onlineID)
         {
             ShortName = shortName;
@@ -86,7 +92,8 @@ namespace osu.Game.Rulesets
             Name = Name,
             ShortName = ShortName,
             InstantiationInfo = InstantiationInfo,
-            Available = Available
+            Available = Available,
+            LastAppliedDifficultyVersion = LastAppliedDifficultyVersion,
         };
 
         public Ruleset CreateInstance()