diff --git a/osu.Game/Screens/Edit/Compose/Components/Timeline/TimelineControlPointDisplay.cs b/osu.Game/Screens/Edit/Compose/Components/Timeline/TimelineControlPointDisplay.cs
index cd97b293ba..116a3ee105 100644
--- a/osu.Game/Screens/Edit/Compose/Components/Timeline/TimelineControlPointDisplay.cs
+++ b/osu.Game/Screens/Edit/Compose/Components/Timeline/TimelineControlPointDisplay.cs
@@ -1,10 +1,9 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
-using System.Collections.Specialized;
-using System.Diagnostics;
-using System.Linq;
+using osu.Framework.Allocation;
 using osu.Framework.Bindables;
+using osu.Framework.Caching;
 using osu.Game.Beatmaps.ControlPoints;
 using osu.Game.Screens.Edit.Components.Timelines.Summary.Parts;
 
@@ -15,6 +14,16 @@ namespace osu.Game.Screens.Edit.Compose.Components.Timeline
     /// </summary>
     public partial class TimelineControlPointDisplay : TimelinePart<TimelineControlPointGroup>
     {
+        [Resolved]
+        private Timeline timeline { get; set; } = null!;
+
+        /// <summary>
+        /// The visible time/position range of the timeline.
+        /// </summary>
+        private (float min, float max) visibleRange = (float.MinValue, float.MaxValue);
+
+        private readonly Cached groupCache = new Cached();
+
         private readonly IBindableList<ControlPointGroup> controlPointGroups = new BindableList<ControlPointGroup>();
 
         protected override void LoadBeatmap(EditorBeatmap beatmap)
@@ -23,34 +32,67 @@ namespace osu.Game.Screens.Edit.Compose.Components.Timeline
 
             controlPointGroups.UnbindAll();
             controlPointGroups.BindTo(beatmap.ControlPointInfo.Groups);
-            controlPointGroups.BindCollectionChanged((_, args) =>
-            {
-                switch (args.Action)
-                {
-                    case NotifyCollectionChangedAction.Reset:
-                        Clear();
-                        break;
-
-                    case NotifyCollectionChangedAction.Add:
-                        Debug.Assert(args.NewItems != null);
-
-                        foreach (var group in args.NewItems.OfType<ControlPointGroup>())
-                            Add(new TimelineControlPointGroup(group));
-                        break;
-
-                    case NotifyCollectionChangedAction.Remove:
-                        Debug.Assert(args.OldItems != null);
-
-                        foreach (var group in args.OldItems.OfType<ControlPointGroup>())
-                        {
-                            var matching = Children.SingleOrDefault(gv => ReferenceEquals(gv.Group, group));
-
-                            matching?.Expire();
-                        }
-
-                        break;
-                }
-            }, true);
+            controlPointGroups.BindCollectionChanged((_, _) => groupCache.Invalidate(), true);
         }
+
+        protected override void Update()
+        {
+            base.Update();
+
+            if (DrawWidth <= 0) return;
+
+            (float, float) newRange = (
+                (ToLocalSpace(timeline.ScreenSpaceDrawQuad.TopLeft).X - TopPointPiece.WIDTH) / DrawWidth * Content.RelativeChildSize.X,
+                (ToLocalSpace(timeline.ScreenSpaceDrawQuad.TopRight).X) / DrawWidth * Content.RelativeChildSize.X);
+
+            if (visibleRange != newRange)
+            {
+                visibleRange = newRange;
+                groupCache.Invalidate();
+            }
+
+            if (!groupCache.IsValid)
+            {
+                recreateDrawableGroups();
+                groupCache.Validate();
+            }
+        }
+
+        private void recreateDrawableGroups()
+        {
+            // Remove groups outside the visible range
+            foreach (TimelineControlPointGroup drawableGroup in this)
+            {
+                if (!shouldBeVisible(drawableGroup.Group))
+                    drawableGroup.Expire();
+            }
+
+            // Add remaining ones
+            for (int i = 0; i < controlPointGroups.Count; i++)
+            {
+                var group = controlPointGroups[i];
+
+                if (!shouldBeVisible(group))
+                    continue;
+
+                bool alreadyVisible = false;
+
+                foreach (var g in this)
+                {
+                    if (ReferenceEquals(g.Group, group))
+                    {
+                        alreadyVisible = true;
+                        break;
+                    }
+                }
+
+                if (alreadyVisible)
+                    continue;
+
+                Add(new TimelineControlPointGroup(group));
+            }
+        }
+
+        private bool shouldBeVisible(ControlPointGroup group) => group.Time >= visibleRange.min && group.Time <= visibleRange.max;
     }
 }
diff --git a/osu.Game/Screens/Edit/Compose/Components/Timeline/TopPointPiece.cs b/osu.Game/Screens/Edit/Compose/Components/Timeline/TopPointPiece.cs
index 243cdc6ddd..a40a805361 100644
--- a/osu.Game/Screens/Edit/Compose/Components/Timeline/TopPointPiece.cs
+++ b/osu.Game/Screens/Edit/Compose/Components/Timeline/TopPointPiece.cs
@@ -19,12 +19,12 @@ namespace osu.Game.Screens.Edit.Compose.Components.Timeline
 
         protected OsuSpriteText Label { get; private set; } = null!;
 
-        private const float width = 80;
+        public const float WIDTH = 80;
 
         public TopPointPiece(ControlPoint point)
         {
             Point = point;
-            Width = width;
+            Width = WIDTH;
             Height = 16;
             Margin = new MarginPadding { Vertical = 4 };
 
@@ -65,7 +65,7 @@ namespace osu.Game.Screens.Edit.Compose.Components.Timeline
                 new Container
                 {
                     RelativeSizeAxes = Axes.Y,
-                    Width = width - triangle_portion,
+                    Width = WIDTH - triangle_portion,
                     Anchor = Anchor.CentreLeft,
                     Origin = Anchor.CentreLeft,
                     Colour = Point.GetRepresentingColour(colours),