diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneFrameStabilityContainer.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneFrameStabilityContainer.cs
index 534348bed3..98a97e1d23 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneFrameStabilityContainer.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneFrameStabilityContainer.cs
@@ -129,10 +129,8 @@ namespace osu.Game.Tests.Visual.Gameplay
             checkRate(1);
         }
 
-        private const int max_frames_catchup = 50;
-
         private void createStabilityContainer(double gameplayStartTime = double.MinValue) => AddStep("create container", () =>
-            mainContainer.Child = new FrameStabilityContainer(gameplayStartTime) { MaxCatchUpFrames = max_frames_catchup }
+            mainContainer.Child = new FrameStabilityContainer(gameplayStartTime)
                 .WithChild(consumer = new ClockConsumingChild()));
 
         private void seekManualTo(double time) => AddStep($"seek manual clock to {time}", () => manualClock.CurrentTime = time);
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneSongProgress.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneSongProgress.cs
index e975a85401..99f0ffb9d0 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneSongProgress.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneSongProgress.cs
@@ -3,6 +3,7 @@
 
 using System;
 using System.Linq;
+using System.Threading;
 using NUnit.Framework;
 using osu.Framework.Allocation;
 using osu.Framework.Extensions.IEnumerableExtensions;
@@ -47,7 +48,7 @@ namespace osu.Game.Tests.Visual.Gameplay
                 {
                     Child = frameStabilityContainer = new FrameStabilityContainer
                     {
-                        MaxCatchUpFrames = 1
+                        Child = new FakeLoad()
                     }
                 }
             });
@@ -56,6 +57,15 @@ namespace osu.Game.Tests.Visual.Gameplay
             Dependencies.CacheAs<IFrameStableClock>(frameStabilityContainer);
         }
 
+        private partial class FakeLoad : Drawable
+        {
+            protected override void Update()
+            {
+                base.Update();
+                Thread.Sleep(1);
+            }
+        }
+
         [SetUpSteps]
         public void SetupSteps()
         {
diff --git a/osu.Game/Rulesets/UI/FrameStabilityContainer.cs b/osu.Game/Rulesets/UI/FrameStabilityContainer.cs
index ee19baaf65..8c9cb262af 100644
--- a/osu.Game/Rulesets/UI/FrameStabilityContainer.cs
+++ b/osu.Game/Rulesets/UI/FrameStabilityContainer.cs
@@ -25,9 +25,9 @@ namespace osu.Game.Rulesets.UI
         public ReplayInputHandler? ReplayInputHandler { get; set; }
 
         /// <summary>
-        /// The number of frames (per parent frame) which can be run in an attempt to catch-up to real-time.
+        /// The number of CPU milliseconds to spend at most during seek catch-up.
         /// </summary>
-        public int MaxCatchUpFrames { get; set; } = 5;
+        private const double max_catchup_milliseconds = 10;
 
         /// <summary>
         /// Whether to enable frame-stable playback.
@@ -59,6 +59,8 @@ namespace osu.Game.Rulesets.UI
         /// </summary>
         private readonly FramedClock framedClock;
 
+        private readonly Stopwatch stopwatch = new Stopwatch();
+
         /// <summary>
         /// The current direction of playback to be exposed to frame stable children.
         /// </summary>
@@ -97,7 +99,7 @@ namespace osu.Game.Rulesets.UI
 
         public override bool UpdateSubTree()
         {
-            int loops = MaxCatchUpFrames;
+            stopwatch.Restart();
 
             do
             {
@@ -110,7 +112,7 @@ namespace osu.Game.Rulesets.UI
 
                 base.UpdateSubTree();
                 UpdateSubTreeMasking(this, ScreenSpaceDrawQuad.AABBFloat);
-            } while (state == PlaybackState.RequiresCatchUp && loops-- > 0);
+            } while (state == PlaybackState.RequiresCatchUp && stopwatch.ElapsedMilliseconds < max_catchup_milliseconds);
 
             return true;
         }