diff --git a/osu.Game/Screens/Play/BreakOverlay.cs b/osu.Game/Screens/Play/BreakOverlay.cs
index f3decb38fb..8f07b90b30 100644
--- a/osu.Game/Screens/Play/BreakOverlay.cs
+++ b/osu.Game/Screens/Play/BreakOverlay.cs
@@ -146,6 +146,16 @@ namespace osu.Game.Screens.Play
 
             var time = Clock.CurrentTime;
 
+            if (currentBreakIndex < breaks.Count - 1)
+            {
+                // Quick check if we're not in a break with our current index.
+                if (time > breaks[currentBreakIndex].EndTime && time < breaks[currentBreakIndex + 1].StartTime)
+                {
+                    isBreakTime.Value = false;
+                    return;
+                }
+            }
+
             if (time > breaks[currentBreakIndex].EndTime)
             {
                 for (int i = currentBreakIndex; i < breaks.Count; i++)