diff --git a/osu.Game.Tests/Visual/Gameplay/TestScenePause.cs b/osu.Game.Tests/Visual/Gameplay/TestScenePause.cs
index 64022b2410..6e8975f11b 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestScenePause.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestScenePause.cs
@@ -237,6 +237,7 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddUntilStep("player not exited", () => Player.IsCurrentScreen());
             AddStep("exit", () => Player.Exit());
             confirmExited();
+            confirmNoTrackAdjustments();
         }
 
         private void confirmPaused()
@@ -258,6 +259,11 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddUntilStep("player exited", () => !Player.IsCurrentScreen());
         }
 
+        private void confirmNoTrackAdjustments()
+        {
+            AddAssert("track has no adjustments", () => Beatmap.Value.Track.AggregateFrequency.Value == 1);
+        }
+
         private void restart() => AddStep("restart", () => Player.Restart());
         private void pause() => AddStep("pause", () => Player.Pause());
         private void resume() => AddStep("resume", () => Player.Resume());
diff --git a/osu.Game/Screens/Play/GameplayClockContainer.cs b/osu.Game/Screens/Play/GameplayClockContainer.cs
index f2efbe6073..2f2028ff53 100644
--- a/osu.Game/Screens/Play/GameplayClockContainer.cs
+++ b/osu.Game/Screens/Play/GameplayClockContainer.cs
@@ -162,16 +162,12 @@ namespace osu.Game.Screens.Play
             if (sourceClock != beatmap.Track)
                 return;
 
+            removeSourceClockAdjustments();
+
             sourceClock = new TrackVirtual(beatmap.Track.Length);
             adjustableClock.ChangeSource(sourceClock);
         }
 
-        public void ResetLocalAdjustments()
-        {
-            // In the case of replays, we may have changed the playback rate.
-            UserPlaybackRate.Value = 1;
-        }
-
         protected override void Update()
         {
             if (!IsPaused.Value)
@@ -198,6 +194,14 @@ namespace osu.Game.Screens.Play
         protected override void Dispose(bool isDisposing)
         {
             base.Dispose(isDisposing);
+
+            removeSourceClockAdjustments();
+            sourceClock = null;
+        }
+
+        private void removeSourceClockAdjustments()
+        {
+            sourceClock.ResetSpeedAdjustments();
             (sourceClock as IAdjustableAudioComponent)?.RemoveAdjustment(AdjustableProperty.Frequency, pauseFreqAdjust);
         }
     }
diff --git a/osu.Game/Screens/Play/Player.cs b/osu.Game/Screens/Play/Player.cs
index a3c39d9cc1..a9b0649fab 100644
--- a/osu.Game/Screens/Play/Player.cs
+++ b/osu.Game/Screens/Play/Player.cs
@@ -536,8 +536,6 @@ namespace osu.Game.Screens.Play
                     return true;
             }
 
-            GameplayClockContainer.ResetLocalAdjustments();
-
             // GameplayClockContainer performs seeks / start / stop operations on the beatmap's track.
             // as we are no longer the current screen, we cannot guarantee the track is still usable.
             GameplayClockContainer.StopUsingBeatmapClock();