diff --git a/osu.Game/Screens/Play/GameplayClockContainer.cs b/osu.Game/Screens/Play/GameplayClockContainer.cs
index f2efbe6073..2f2028ff53 100644
--- a/osu.Game/Screens/Play/GameplayClockContainer.cs
+++ b/osu.Game/Screens/Play/GameplayClockContainer.cs
@@ -162,16 +162,12 @@ namespace osu.Game.Screens.Play
             if (sourceClock != beatmap.Track)
                 return;
 
+            removeSourceClockAdjustments();
+
             sourceClock = new TrackVirtual(beatmap.Track.Length);
             adjustableClock.ChangeSource(sourceClock);
         }
 
-        public void ResetLocalAdjustments()
-        {
-            // In the case of replays, we may have changed the playback rate.
-            UserPlaybackRate.Value = 1;
-        }
-
         protected override void Update()
         {
             if (!IsPaused.Value)
@@ -198,6 +194,14 @@ namespace osu.Game.Screens.Play
         protected override void Dispose(bool isDisposing)
         {
             base.Dispose(isDisposing);
+
+            removeSourceClockAdjustments();
+            sourceClock = null;
+        }
+
+        private void removeSourceClockAdjustments()
+        {
+            sourceClock.ResetSpeedAdjustments();
             (sourceClock as IAdjustableAudioComponent)?.RemoveAdjustment(AdjustableProperty.Frequency, pauseFreqAdjust);
         }
     }
diff --git a/osu.Game/Screens/Play/Player.cs b/osu.Game/Screens/Play/Player.cs
index a3c39d9cc1..a9b0649fab 100644
--- a/osu.Game/Screens/Play/Player.cs
+++ b/osu.Game/Screens/Play/Player.cs
@@ -536,8 +536,6 @@ namespace osu.Game.Screens.Play
                     return true;
             }
 
-            GameplayClockContainer.ResetLocalAdjustments();
-
             // GameplayClockContainer performs seeks / start / stop operations on the beatmap's track.
             // as we are no longer the current screen, we cannot guarantee the track is still usable.
             GameplayClockContainer.StopUsingBeatmapClock();