diff --git a/osu.Game/Rulesets/Replays/FramedReplayInputHandler.cs b/osu.Game/Rulesets/Replays/FramedReplayInputHandler.cs
index 8a4451fdca..b43324bcfa 100644
--- a/osu.Game/Rulesets/Replays/FramedReplayInputHandler.cs
+++ b/osu.Game/Rulesets/Replays/FramedReplayInputHandler.cs
@@ -40,7 +40,7 @@ namespace osu.Game.Rulesets.Replays
                     return null;
 
                 if (!currentFrameIndex.HasValue)
-                    return (TFrame)Frames[0];
+                    return currentDirection > 0 ? (TFrame)Frames[0] : null;
 
                 int nextFrame = clampedNextFrameIndex;
 
@@ -109,30 +109,54 @@ namespace osu.Game.Rulesets.Replays
 
             Debug.Assert(currentDirection != 0);
 
-            TFrame next = NextFrame;
-
-            // check if the next frame is valid for the current playback direction.
-            // validity is if the next frame is equal or "earlier" than the current point in time (so we can change to it)
-            int compare = time.CompareTo(next?.Time);
-
-            if (next != null && (compare == 0 || compare == currentDirection))
+            if (!HasFrames)
             {
-                currentFrameIndex = clampedNextFrameIndex;
-                return CurrentTime = CurrentFrame.Time;
+                // in the case all frames are received, allow time to progress regardless.
+                if (replay.HasReceivedAllFrames)
+                    return CurrentTime = time;
+
+                return null;
             }
 
-            // at this point, the frame can't be advanced (in the replay).
-            // even so, we may be able to move the clock forward due to being at the end of the replay or
-            // moving towards the next valid frame.
+            TFrame next = NextFrame;
+
+            // if we have a next frame, check if it is before or at the current time in playback, and advance time to it if so.
+            if (next != null)
+            {
+                int compare = time.CompareTo(next.Time);
+
+                if (compare == 0 || compare == currentDirection)
+                {
+                    currentFrameIndex = clampedNextFrameIndex;
+                    return CurrentTime = CurrentFrame.Time;
+                }
+            }
+
+            // at this point, the frame index can't be advanced.
+            // even so, we may be able to propose the clock progresses forward due to being at an extent of the replay,
+            // or moving towards the next valid frame (ie. interpolating in a non-important section).
 
             // the exception is if currently in an important section, which is respected above all.
             if (inImportantSection)
+            {
+                Debug.Assert(next != null || !replay.HasReceivedAllFrames);
                 return null;
+            }
 
-            // in the case we have no next frames and haven't received the full replay, block.
-            if (next == null && !replay.HasReceivedAllFrames) return null;
+            // if a next frame does exist, allow interpolation.
+            if (next != null)
+                return CurrentTime = time;
 
-            return CurrentTime = time;
+            // if all frames have been received, allow playing beyond extents.
+            if (replay.HasReceivedAllFrames)
+                return CurrentTime = time;
+
+            // if not all frames are received but we are before the first frame, allow playing.
+            if (time < Frames[0].Time)
+                return CurrentTime = time;
+
+            // in the case we have no next frames and haven't received enough frame data, block.
+            return null;
         }
 
         private void updateDirection(double time)