diff --git a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/Sync/CatchUpSyncManager.cs b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/Sync/CatchUpSyncManager.cs
index f5c780e8b1..5912c0803b 100644
--- a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/Sync/CatchUpSyncManager.cs
+++ b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/Sync/CatchUpSyncManager.cs
@@ -78,15 +78,18 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate.Sync
             if (playerClocks.Count == 0)
                 return false;
 
-            firstStartAttemptTime ??= Time.Current;
-
             int readyCount = playerClocks.Count(s => !s.WaitingOnFrames.Value);
 
             if (readyCount == playerClocks.Count)
                 return hasStarted = true;
 
-            if (readyCount > 0 && (Time.Current - firstStartAttemptTime) > MAXIMUM_START_DELAY)
-                return hasStarted = true;
+            if (readyCount > 0)
+            {
+                firstStartAttemptTime ??= Time.Current;
+
+                if (Time.Current - firstStartAttemptTime > MAXIMUM_START_DELAY)
+                    return hasStarted = true;
+            }
 
             return false;
         }