diff --git a/osu.Game/Configuration/SessionAverageHitErrorTracker.cs b/osu.Game/Configuration/SessionAverageHitErrorTracker.cs
index 13b1ea7d37..cd21eb6fa8 100644
--- a/osu.Game/Configuration/SessionAverageHitErrorTracker.cs
+++ b/osu.Game/Configuration/SessionAverageHitErrorTracker.cs
@@ -17,11 +17,14 @@ namespace osu.Game.Configuration
     [Cached]
     public partial class SessionAverageHitErrorTracker : Component
     {
-        public IBindableList<double> AverageHitErrorHistory => averageHitErrorHistory;
-        private readonly BindableList<double> averageHitErrorHistory = new BindableList<double>();
+        public IBindableList<DataPoint> AverageHitErrorHistory => averageHitErrorHistory;
+        private readonly BindableList<DataPoint> averageHitErrorHistory = new BindableList<DataPoint>();
 
         private readonly Bindable<ScoreInfo?> latestScore = new Bindable<ScoreInfo?>();
 
+        [Resolved]
+        private OsuConfigManager configManager { get; set; } = null!;
+
         [BackgroundDependencyLoader]
         private void load(SessionStatics statics)
         {
@@ -46,9 +49,25 @@ namespace osu.Game.Configuration
             // keep a sane maximum number of entries.
             if (averageHitErrorHistory.Count >= 50)
                 averageHitErrorHistory.RemoveAt(0);
-            averageHitErrorHistory.Add(averageError);
+
+            double globalOffset = configManager.Get<double>(OsuSetting.AudioOffset);
+            averageHitErrorHistory.Add(new DataPoint(averageError, globalOffset));
         }
 
         public void ClearHistory() => averageHitErrorHistory.Clear();
+
+        public readonly struct DataPoint
+        {
+            public double AverageHitError { get; }
+            public double GlobalAudioOffset { get; }
+
+            public double SuggestedGlobalAudioOffset => GlobalAudioOffset - AverageHitError;
+
+            public DataPoint(double averageHitError, double globalOffset)
+            {
+                AverageHitError = averageHitError;
+                GlobalAudioOffset = globalOffset;
+            }
+        }
     }
 }
diff --git a/osu.Game/Overlays/Settings/Sections/Audio/AudioOffsetAdjustControl.cs b/osu.Game/Overlays/Settings/Sections/Audio/AudioOffsetAdjustControl.cs
index 98acea1f2e..08bf4b0dad 100644
--- a/osu.Game/Overlays/Settings/Sections/Audio/AudioOffsetAdjustControl.cs
+++ b/osu.Game/Overlays/Settings/Sections/Audio/AudioOffsetAdjustControl.cs
@@ -42,7 +42,7 @@ namespace osu.Game.Overlays.Settings.Sections.Audio
 
             private readonly BindableNumberWithCurrent<double> current = new BindableNumberWithCurrent<double>();
 
-            private readonly IBindableList<double> averageHitErrorHistory = new BindableList<double>();
+            private readonly IBindableList<SessionAverageHitErrorTracker.DataPoint> averageHitErrorHistory = new BindableList<SessionAverageHitErrorTracker.DataPoint>();
 
             private readonly Bindable<double?> suggestedOffset = new Bindable<double?>();
 
@@ -112,7 +112,7 @@ namespace osu.Game.Overlays.Settings.Sections.Audio
                 switch (e.Action)
                 {
                     case NotifyCollectionChangedAction.Add:
-                        foreach (double average in e.NewItems!)
+                        foreach (SessionAverageHitErrorTracker.DataPoint dataPoint in e.NewItems!)
                         {
                             notchContainer.ForEach(n => n.Alpha *= 0.95f);
                             notchContainer.Add(new Box
@@ -122,16 +122,16 @@ namespace osu.Game.Overlays.Settings.Sections.Audio
                                 RelativePositionAxes = Axes.X,
                                 Anchor = Anchor.Centre,
                                 Origin = Anchor.Centre,
-                                X = getXPositionForAverage(average)
+                                X = getXPositionForOffset(dataPoint.SuggestedGlobalAudioOffset)
                             });
                         }
 
                         break;
 
                     case NotifyCollectionChangedAction.Remove:
-                        foreach (double average in e.OldItems!)
+                        foreach (SessionAverageHitErrorTracker.DataPoint dataPoint in e.OldItems!)
                         {
-                            var notch = notchContainer.FirstOrDefault(n => n.X == getXPositionForAverage(average));
+                            var notch = notchContainer.FirstOrDefault(n => n.X == getXPositionForOffset(dataPoint.SuggestedGlobalAudioOffset));
                             Debug.Assert(notch != null);
                             notchContainer.Remove(notch, true);
                         }
@@ -143,10 +143,10 @@ namespace osu.Game.Overlays.Settings.Sections.Audio
                         break;
                 }
 
-                suggestedOffset.Value = averageHitErrorHistory.Any() ? -averageHitErrorHistory.Average() : null;
+                suggestedOffset.Value = averageHitErrorHistory.Any() ? -averageHitErrorHistory.Average(dataPoint => dataPoint.SuggestedGlobalAudioOffset) : null;
             }
 
-            private float getXPositionForAverage(double average) => (float)(Math.Clamp(-average, current.MinValue, current.MaxValue) / (2 * current.MaxValue));
+            private float getXPositionForOffset(double offset) => (float)(Math.Clamp(offset, current.MinValue, current.MaxValue) / (2 * current.MaxValue));
 
             private void updateHintText()
             {