diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneAutoplay.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneAutoplay.cs
index 4b1c2ec256..0be949650e 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneAutoplay.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneAutoplay.cs
@@ -52,7 +52,7 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddStep($"seek to break {breakIndex}", () => Player.GameplayClockContainer.Seek(destBreak().StartTime));
             AddUntilStep("wait for seek to complete", () => Player.HUDOverlay.Progress.ReferenceClock.CurrentTime >= destBreak().StartTime);
 
-            BreakPeriod destBreak() => Player.ChildrenOfType<BreakTracker>().First().Breaks.ElementAt(breakIndex);
+            BreakPeriod destBreak() => Beatmap.Value.Beatmap.Breaks.ElementAt(breakIndex);
         }
     }
 }
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneBreakTracker.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneBreakTracker.cs
index 91d6f2f143..a6f996c30d 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneBreakTracker.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneBreakTracker.cs
@@ -97,8 +97,6 @@ namespace osu.Game.Tests.Visual.Gameplay
             loadBreaksStep("multiple breaks", testBreaks);
 
             seekAndAssertBreak("seek to break start", testBreaks[1].StartTime, true);
-            AddAssert("is skipped to break #2", () => breakTracker.CurrentBreakIndex == 1);
-
             seekAndAssertBreak("seek to break middle", testBreaks[1].StartTime + testBreaks[1].Duration / 2, true);
             seekAndAssertBreak("seek to break end", testBreaks[1].EndTime, false);
             seekAndAssertBreak("seek to break after end", testBreaks[1].EndTime + 500, false);
@@ -174,8 +172,6 @@ namespace osu.Game.Tests.Visual.Gameplay
             private readonly ManualClock manualClock;
             private IFrameBasedClock originalClock;
 
-            public new int CurrentBreakIndex => base.CurrentBreakIndex;
-
             public double ManualClockTime
             {
                 get => manualClock.CurrentTime;
diff --git a/osu.Game/Screens/Play/BreakTracker.cs b/osu.Game/Screens/Play/BreakTracker.cs
index 64262d52b5..c2eb069ee6 100644
--- a/osu.Game/Screens/Play/BreakTracker.cs
+++ b/osu.Game/Screens/Play/BreakTracker.cs
@@ -5,6 +5,7 @@ using System.Collections.Generic;
 using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Game.Beatmaps.Timing;
+using osu.Game.Lists;
 using osu.Game.Rulesets.Scoring;
 
 namespace osu.Game.Screens.Play
@@ -20,22 +21,24 @@ namespace osu.Game.Screens.Play
         /// </summary>
         public IBindable<bool> IsBreakTime => isBreakTime;
 
-        protected int CurrentBreakIndex;
-
         private readonly BindableBool isBreakTime = new BindableBool();
 
-        private IReadOnlyList<BreakPeriod> breaks;
+        private readonly IntervalList<double> breakIntervals = new IntervalList<double>();
 
         public IReadOnlyList<BreakPeriod> Breaks
         {
-            get => breaks;
             set
             {
-                breaks = value;
-
-                // reset index in case the new breaks list is smaller than last one
                 isBreakTime.Value = false;
-                CurrentBreakIndex = 0;
+                breakIntervals.Clear();
+
+                foreach (var b in value)
+                {
+                    if (!b.HasEffect)
+                        continue;
+
+                    breakIntervals.Add(b.StartTime, b.EndTime - BreakOverlay.BREAK_FADE_DURATION);
+                }
             }
         }
 
@@ -49,34 +52,11 @@ namespace osu.Game.Screens.Play
         {
             base.Update();
 
-            isBreakTime.Value = getCurrentBreak()?.HasEffect == true
-                                || Clock.CurrentTime < gameplayStartTime
+            var time = Clock.CurrentTime;
+
+            isBreakTime.Value = breakIntervals.IsInAnyInterval(time)
+                                || time < gameplayStartTime
                                 || scoreProcessor?.HasCompleted == true;
         }
-
-        private BreakPeriod getCurrentBreak()
-        {
-            if (breaks?.Count > 0)
-            {
-                var time = Clock.CurrentTime;
-
-                if (time > breaks[CurrentBreakIndex].EndTime)
-                {
-                    while (time > breaks[CurrentBreakIndex].EndTime && CurrentBreakIndex < breaks.Count - 1)
-                        CurrentBreakIndex++;
-                }
-                else
-                {
-                    while (time < breaks[CurrentBreakIndex].StartTime && CurrentBreakIndex > 0)
-                        CurrentBreakIndex--;
-                }
-
-                var closest = breaks[CurrentBreakIndex];
-
-                return closest.Contains(time) ? closest : null;
-            }
-
-            return null;
-        }
     }
 }