diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneBeatmapOffsetControl.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneBeatmapOffsetControl.cs
index 67f5db548b..4b079cbb2c 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneBeatmapOffsetControl.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneBeatmapOffsetControl.cs
@@ -48,7 +48,7 @@ namespace osu.Game.Tests.Visual.Gameplay
         }
 
         [Test]
-        public void TestDisplay()
+        public void TestCalibrationFromZero()
         {
             const double average_error = -4.5;
 
@@ -70,5 +70,33 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddStep("Remove reference score", () => offsetControl.ReferenceScore.Value = null);
             AddAssert("No calibration button", () => !offsetControl.ChildrenOfType<SettingsButton>().Any());
         }
+
+        /// <summary>
+        /// When a beatmap offset was already set, the calibration should take it into account.
+        /// </summary>
+        [Test]
+        public void TestCalibrationFromNonZero()
+        {
+            const double average_error = -4.5;
+            const double initial_offset = -2;
+
+            AddStep("Set offset non-neutral", () => offsetControl.Current.Value = initial_offset);
+            AddAssert("No calibration button", () => !offsetControl.ChildrenOfType<SettingsButton>().Any());
+            AddStep("Set reference score", () =>
+            {
+                offsetControl.ReferenceScore.Value = new ScoreInfo
+                {
+                    HitEvents = TestSceneHitEventTimingDistributionGraph.CreateDistributedHitEvents(average_error)
+                };
+            });
+
+            AddAssert("Has calibration button", () => offsetControl.ChildrenOfType<SettingsButton>().Any());
+            AddStep("Press button", () => offsetControl.ChildrenOfType<SettingsButton>().Single().TriggerClick());
+            AddAssert("Offset is adjusted", () => offsetControl.Current.Value == initial_offset - average_error);
+
+            AddAssert("Button is disabled", () => !offsetControl.ChildrenOfType<SettingsButton>().Single().Enabled.Value);
+            AddStep("Remove reference score", () => offsetControl.ReferenceScore.Value = null);
+            AddAssert("No calibration button", () => !offsetControl.ChildrenOfType<SettingsButton>().Any());
+        }
     }
 }
diff --git a/osu.Game/Screens/Play/PlayerSettings/BeatmapOffsetControl.cs b/osu.Game/Screens/Play/PlayerSettings/BeatmapOffsetControl.cs
index 201e431367..1d6d0acb76 100644
--- a/osu.Game/Screens/Play/PlayerSettings/BeatmapOffsetControl.cs
+++ b/osu.Game/Screens/Play/PlayerSettings/BeatmapOffsetControl.cs
@@ -53,6 +53,7 @@ namespace osu.Game.Screens.Play.PlayerSettings
         private OsuColour colours { get; set; } = null!;
 
         private double lastPlayAverage;
+        private double lastPlayBeatmapOffset;
 
         private SettingsButton? useAverageButton;
 
@@ -156,7 +157,7 @@ namespace osu.Game.Screens.Play.PlayerSettings
                 }
 
                 if (useAverageButton != null)
-                    useAverageButton.Enabled.Value = !Precision.AlmostEquals(lastPlayAverage, -Current.Value, Current.Precision / 2);
+                    useAverageButton.Enabled.Value = !Precision.AlmostEquals(lastPlayAverage, -Current.Value + lastPlayBeatmapOffset, Current.Precision / 2);
 
                 realmWriteTask = realm.WriteAsync(r =>
                 {
@@ -213,6 +214,7 @@ namespace osu.Game.Screens.Play.PlayerSettings
             }
 
             lastPlayAverage = average;
+            lastPlayBeatmapOffset = Current.Value;
 
             referenceScoreContainer.AddRange(new Drawable[]
             {
@@ -225,7 +227,7 @@ namespace osu.Game.Screens.Play.PlayerSettings
                 useAverageButton = new SettingsButton
                 {
                     Text = BeatmapOffsetControlStrings.CalibrateUsingLastPlay,
-                    Action = () => Current.Value = -lastPlayAverage
+                    Action = () => Current.Value = lastPlayBeatmapOffset - lastPlayAverage
                 },
             });
         }