diff --git a/osu.Game/Rulesets/UI/FrameStabilityContainer.cs b/osu.Game/Rulesets/UI/FrameStabilityContainer.cs
index 05d3c02381..e569bb8459 100644
--- a/osu.Game/Rulesets/UI/FrameStabilityContainer.cs
+++ b/osu.Game/Rulesets/UI/FrameStabilityContainer.cs
@@ -47,6 +47,11 @@ namespace osu.Game.Rulesets.UI
 
         private IFrameBasedClock parentGameplayClock;
 
+        /// <summary>
+        /// The current direction of playback to be exposed to frame stable children.
+        /// </summary>
+        private int direction;
+
         [BackgroundDependencyLoader(true)]
         private void load(GameplayClock clock)
         {
@@ -110,27 +115,22 @@ namespace osu.Game.Rulesets.UI
                 setClock(); // LoadComplete may not be run yet, but we still want the clock.
 
             validState = true;
-
-            manualClock.Rate = parentGameplayClock.Rate;
-            manualClock.IsRunning = parentGameplayClock.IsRunning;
+            requireMoreUpdateLoops = false;
 
             var newProposedTime = parentGameplayClock.CurrentTime;
 
             try
             {
                 if (!FrameStablePlayback)
-                {
-                    manualClock.CurrentTime = newProposedTime;
-                    requireMoreUpdateLoops = false;
                     return;
-                }
-                else if (firstConsumption)
+
+                if (firstConsumption)
                 {
                     // On the first update, frame-stability seeking would result in unexpected/unwanted behaviour.
                     // Instead we perform an initial seek to the proposed time.
-                    manualClock.CurrentTime = newProposedTime;
 
-                    // do a second process to clear out ElapsedTime
+                    // process frame (in addition to finally clause) to clear out ElapsedTime
+                    manualClock.CurrentTime = newProposedTime;
                     framedClock.ProcessFrame();
 
                     firstConsumption = false;
@@ -144,11 +144,7 @@ namespace osu.Game.Rulesets.UI
                         : Math.Max(newProposedTime, manualClock.CurrentTime - sixty_frame_time);
                 }
 
-                if (!isAttached)
-                {
-                    manualClock.CurrentTime = newProposedTime;
-                }
-                else
+                if (isAttached)
                 {
                     double? newTime = ReplayInputHandler.SetFrameFromTime(newProposedTime);
 
@@ -156,19 +152,24 @@ namespace osu.Game.Rulesets.UI
                     {
                         // we shouldn't execute for this time value. probably waiting on more replay data.
                         validState = false;
-
                         requireMoreUpdateLoops = true;
-                        manualClock.CurrentTime = newProposedTime;
                         return;
                     }
 
-                    manualClock.CurrentTime = newTime.Value;
+                    newProposedTime = newTime.Value;
                 }
-
-                requireMoreUpdateLoops = manualClock.CurrentTime != parentGameplayClock.CurrentTime;
             }
             finally
             {
+                if (newProposedTime != manualClock.CurrentTime)
+                    direction = newProposedTime > manualClock.CurrentTime ? 1 : -1;
+
+                manualClock.CurrentTime = newProposedTime;
+                manualClock.Rate = Math.Abs(parentGameplayClock.Rate) * direction;
+                manualClock.IsRunning = parentGameplayClock.IsRunning;
+
+                requireMoreUpdateLoops |= manualClock.CurrentTime != parentGameplayClock.CurrentTime;
+
                 // The manual clock time has changed in the above code. The framed clock now needs to be updated
                 // to ensure that the its time is valid for our children before input is processed
                 framedClock.ProcessFrame();
diff --git a/osu.Game/Rulesets/UI/RulesetInputManager.cs b/osu.Game/Rulesets/UI/RulesetInputManager.cs
index 98e27240d3..5cc213be41 100644
--- a/osu.Game/Rulesets/UI/RulesetInputManager.cs
+++ b/osu.Game/Rulesets/UI/RulesetInputManager.cs
@@ -137,9 +137,9 @@ namespace osu.Game.Rulesets.UI
             {
             }
 
-            public bool OnPressed(T action) => Target.Children.OfType<KeyCounterAction<T>>().Any(c => c.OnPressed(action, Clock.ElapsedFrameTime > 0));
+            public bool OnPressed(T action) => Target.Children.OfType<KeyCounterAction<T>>().Any(c => c.OnPressed(action, Clock.Rate >= 0));
 
-            public bool OnReleased(T action) => Target.Children.OfType<KeyCounterAction<T>>().Any(c => c.OnReleased(action, Clock.ElapsedFrameTime > 0));
+            public bool OnReleased(T action) => Target.Children.OfType<KeyCounterAction<T>>().Any(c => c.OnReleased(action, Clock.Rate >= 0));
         }
 
         #endregion