diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs
index 6485cbdad3..a4df450db9 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs
@@ -61,13 +61,8 @@ namespace osu.Game.Tests.Visual.Gameplay
             finish();
         }
 
-        private OsuFramedReplayInputHandler replayHandler =>
-            (OsuFramedReplayInputHandler)Stack.ChildrenOfType<OsuInputManager>().First().ReplayInputHandler;
-
-        private Player player => Stack.CurrentScreen as Player;
-
         [Test]
-        public void TestBasicSpectatingFlow()
+        public void TestFrameStarvationAndResume()
         {
             loadSpectatingScreen();
 
@@ -82,11 +77,16 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddUntilStep("wait for frame starvation", () => replayHandler.NextFrame == null);
             checkPaused(true);
 
+            double? pausedTime = null;
+
+            AddStep("store time", () => pausedTime = currentFrameStableTime);
+
             sendFrames();
 
-            checkPaused(false);
             AddUntilStep("wait for frame starvation", () => replayHandler.NextFrame == null);
             checkPaused(true);
+
+            AddAssert("time advanced", () => currentFrameStableTime > pausedTime);
         }
 
         [Test]
@@ -98,7 +98,7 @@ namespace osu.Game.Tests.Visual.Gameplay
             waitForPlayer();
             checkPaused(true);
 
-            sendFrames();
+            sendFrames(1000); // send enough frames to ensure play won't be paused
 
             checkPaused(false);
         }
@@ -119,15 +119,23 @@ namespace osu.Game.Tests.Visual.Gameplay
         }
 
         [Test]
-        public void TestHostStartsPlayingWhileAlreadyWatching()
+        public void TestHostRetriesWhileWatching()
         {
             loadSpectatingScreen();
 
             start();
             sendFrames();
 
+            waitForPlayer();
+
+            Player lastPlayer = null;
+            AddStep("store first player", () => lastPlayer = player);
+
             start();
             sendFrames();
+
+            waitForPlayer();
+            AddAssert("player is different", () => lastPlayer != player);
         }
 
         [Test]
@@ -155,9 +163,25 @@ namespace osu.Game.Tests.Visual.Gameplay
             sendFrames();
             waitForPlayer();
 
-            // should immediately exit and unbind from streaming client
             AddStep("stop spectating", () => (Stack.CurrentScreen as Player)?.Exit());
-            AddUntilStep("spectating stopped", () => spectatorScreen.GetParentScreen() == null);
+            AddUntilStep("spectating stopped", () => spectatorScreen.GetChildScreen() == null);
+        }
+
+        [Test]
+        public void TestStopWatchingThenHostRetries()
+        {
+            loadSpectatingScreen();
+
+            start();
+            sendFrames();
+            waitForPlayer();
+
+            AddStep("stop spectating", () => (Stack.CurrentScreen as Player)?.Exit());
+            AddUntilStep("spectating stopped", () => spectatorScreen.GetChildScreen() == null);
+
+            // host starts playing a new session
+            start();
+            waitForPlayer();
         }
 
         [Test]
@@ -171,6 +195,14 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddAssert("screen didn't change", () => Stack.CurrentScreen is Spectator);
         }
 
+        private OsuFramedReplayInputHandler replayHandler =>
+            (OsuFramedReplayInputHandler)Stack.ChildrenOfType<OsuInputManager>().First().ReplayInputHandler;
+
+        private Player player => Stack.CurrentScreen as Player;
+
+        private double currentFrameStableTime
+            => player.ChildrenOfType<FrameStabilityContainer>().First().FrameStableClock.CurrentTime;
+
         private void waitForPlayer() => AddUntilStep("wait for player", () => Stack.CurrentScreen is Player);
 
         private void start(int? beatmapId = null) => AddStep("start play", () => testSpectatorStreamingClient.StartPlay(beatmapId ?? importedBeatmapId));
diff --git a/osu.Game/Rulesets/UI/FrameStabilityContainer.cs b/osu.Game/Rulesets/UI/FrameStabilityContainer.cs
index f8156164c1..75f3aa90ee 100644
--- a/osu.Game/Rulesets/UI/FrameStabilityContainer.cs
+++ b/osu.Game/Rulesets/UI/FrameStabilityContainer.cs
@@ -85,20 +85,31 @@ namespace osu.Game.Rulesets.UI
 
         public override bool UpdateSubTree()
         {
-            state = frameStableClock.IsPaused.Value ? PlaybackState.NotValid : PlaybackState.Valid;
+            double proposedTime = manualClock.CurrentTime;
 
             if (frameStableClock.WaitingOnFrames.Value)
             {
-                // for now, force one update loop to check if frames have arrived
-                // this may have to change in the future where we want stable user pausing during replay playback.
+                // when waiting on frames, the update loop still needs to be run (at least once) to check for newly arrived frames.
+                // time should not be sourced from the parent clock in this case.
                 state = PlaybackState.Valid;
             }
+            else if (!frameStableClock.IsPaused.Value)
+            {
+                state = PlaybackState.Valid;
+                proposedTime = parentGameplayClock.CurrentTime;
+            }
+            else
+            {
+                // time should not advance while paused, not should anything run.
+                state = PlaybackState.NotValid;
+                return true;
+            }
 
             int loops = MaxCatchUpFrames;
 
-            while (state != PlaybackState.NotValid && loops-- > 0)
+            while (loops-- > 0)
             {
-                updateClock();
+                updateClock(ref proposedTime);
 
                 if (state == PlaybackState.NotValid)
                     break;
@@ -110,7 +121,7 @@ namespace osu.Game.Rulesets.UI
             return true;
         }
 
-        private void updateClock()
+        private void updateClock(ref double proposedTime)
         {
             if (parentGameplayClock == null)
                 setClock(); // LoadComplete may not be run yet, but we still want the clock.
@@ -118,9 +129,6 @@ namespace osu.Game.Rulesets.UI
             // each update start with considering things in valid state.
             state = PlaybackState.Valid;
 
-            // our goal is to catch up to the time provided by the parent clock.
-            var proposedTime = parentGameplayClock.CurrentTime;
-
             if (FrameStablePlayback)
                 // if we require frame stability, the proposed time will be adjusted to move at most one known
                 // frame interval in the current direction.