diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneBeatmapOffsetControl.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneBeatmapOffsetControl.cs
index 92a10628ff..68fd824d7c 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneBeatmapOffsetControl.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneBeatmapOffsetControl.cs
@@ -45,6 +45,44 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddAssert("No calibration button", () => !offsetControl.ChildrenOfType<SettingsButton>().Any());
         }
 
+        /// <summary>
+        /// When a beatmap offset was already set, the calibration should take it into account.
+        /// </summary>
+        [Test]
+        public void TestTooShortToDisplay_HasPreviousValidScore()
+        {
+            const double average_error = -4.5;
+            const double initial_offset = -2;
+
+            AddStep("Set offset non-neutral", () => offsetControl.Current.Value = initial_offset);
+            AddAssert("No calibration button", () => !offsetControl.ChildrenOfType<SettingsButton>().Any());
+
+            AddStep("Set reference score", () =>
+            {
+                offsetControl.ReferenceScore.Value = new ScoreInfo
+                {
+                    HitEvents = TestSceneHitEventTimingDistributionGraph.CreateDistributedHitEvents(average_error),
+                    BeatmapInfo = Beatmap.Value.BeatmapInfo,
+                };
+            });
+
+            AddUntilStep("Has calibration button", () => offsetControl.ChildrenOfType<SettingsButton>().Any());
+
+            AddStep("Set short reference score", () =>
+            {
+                offsetControl.ReferenceScore.Value = new ScoreInfo
+                {
+                    HitEvents = TestSceneHitEventTimingDistributionGraph.CreateDistributedHitEvents(0, 2),
+                    BeatmapInfo = Beatmap.Value.BeatmapInfo,
+                };
+            });
+
+            AddUntilStep("Still calibration button", () => offsetControl.ChildrenOfType<SettingsButton>().Any());
+
+            AddStep("Press button", () => offsetControl.ChildrenOfType<SettingsButton>().Single().TriggerClick());
+            AddAssert("Offset is adjusted", () => offsetControl.Current.Value == initial_offset - average_error);
+        }
+
         [Test]
         public void TestNotEnoughTimedHitEvents()
         {
diff --git a/osu.Game/Screens/Play/PlayerSettings/BeatmapOffsetControl.cs b/osu.Game/Screens/Play/PlayerSettings/BeatmapOffsetControl.cs
index b0b4f6cc5d..64ffe6d191 100644
--- a/osu.Game/Screens/Play/PlayerSettings/BeatmapOffsetControl.cs
+++ b/osu.Game/Screens/Play/PlayerSettings/BeatmapOffsetControl.cs
@@ -69,6 +69,7 @@ namespace osu.Game.Screens.Play.PlayerSettings
         private IDisposable? beatmapOffsetSubscription;
 
         private Task? realmWriteTask;
+        private ScoreInfo? lastValidScore;
 
         public BeatmapOffsetControl()
         {
@@ -177,8 +178,6 @@ namespace osu.Game.Screens.Play.PlayerSettings
 
         private void scoreChanged(ValueChangedEvent<ScoreInfo?> score)
         {
-            referenceScoreContainer.Clear();
-
             if (score.NewValue == null)
                 return;
 
@@ -196,6 +195,15 @@ namespace osu.Game.Screens.Play.PlayerSettings
             if (!(hitEvents.CalculateMedianHitError() is double median))
                 return;
 
+            // affecting unstable rate here is used as a substitute of determining if a hit event represents a *timed* hit event,
+            // i.e. an user input that the user had to *time to the track*,
+            // i.e. one that it *makes sense to use* when doing anything with timing and offsets.
+            bool hasEnoughUsableEvents = hitEvents.Count(HitEventExtensions.AffectsUnstableRate) >= 50;
+
+            // If we are already displaying a score, continue displaying it rather than showing the user "play too short" message.
+            if (lastValidScore != null && !hasEnoughUsableEvents)
+                return;
+
             referenceScoreContainer.Children = new Drawable[]
             {
                 new OsuSpriteText
@@ -204,10 +212,7 @@ namespace osu.Game.Screens.Play.PlayerSettings
                 },
             };
 
-            // affecting unstable rate here is used as a substitute of determining if a hit event represents a *timed* hit event,
-            // i.e. an user input that the user had to *time to the track*,
-            // i.e. one that it *makes sense to use* when doing anything with timing and offsets.
-            if (hitEvents.Count(HitEventExtensions.AffectsUnstableRate) < 50)
+            if (!hasEnoughUsableEvents)
             {
                 referenceScoreContainer.AddRange(new Drawable[]
                 {
@@ -223,6 +228,7 @@ namespace osu.Game.Screens.Play.PlayerSettings
                 return;
             }
 
+            lastValidScore = score.NewValue!;
             lastPlayMedian = median;
             lastPlayBeatmapOffset = Current.Value;
 
@@ -245,7 +251,7 @@ namespace osu.Game.Screens.Play.PlayerSettings
                             return;
 
                         Current.Value = lastPlayBeatmapOffset - lastPlayMedian;
-                        lastAppliedScore.Value = ReferenceScore.Value;
+                        lastAppliedScore.Value = lastValidScore;
                     },
                 },
                 globalOffsetText = new LinkFlowContainer