diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
index 4a6f261905..a24476418c 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
@@ -9,6 +9,7 @@ using osu.Framework.Bindables;
 using osu.Framework.Extensions.TypeExtensions;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Primitives;
+using osu.Framework.Threading;
 using osu.Game.Audio;
 using osu.Game.Rulesets.Judgements;
 using osu.Game.Rulesets.Objects.Types;
@@ -278,6 +279,14 @@ namespace osu.Game.Rulesets.Objects.Drawables
             UpdateResult(false);
         }
 
+        /// <summary>
+        /// Schedules an <see cref="Action"/> to this <see cref="DrawableHitObject"/>.
+        /// </summary>
+        /// <remarks>
+        /// Only provided temporarily until hitobject pooling is implemented.
+        /// </remarks>
+        protected internal new ScheduledDelegate Schedule(Action action) => base.Schedule(action);
+
         private double? lifetimeStart;
 
         public override double LifetimeStart
diff --git a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
index 1df8c8218f..107d55ff0d 100644
--- a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
@@ -1,6 +1,7 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System.Collections.Generic;
 using osu.Framework.Allocation;
 using osu.Framework.Bindables;
 using osu.Framework.Caching;
@@ -86,13 +87,34 @@ namespace osu.Game.Rulesets.UI.Scrolling
                 scrollingInfo.Algorithm.Reset();
 
                 foreach (var obj in Objects)
+                {
+                    computeLifetimeStartRecursive(obj);
                     computeInitialStateRecursive(obj);
+                }
+
                 initialStateCache.Validate();
             }
         }
 
-        private void computeInitialStateRecursive(DrawableHitObject hitObject)
+        private void computeLifetimeStartRecursive(DrawableHitObject hitObject)
         {
+            hitObject.LifetimeStart = scrollingInfo.Algorithm.GetDisplayStartTime(hitObject.HitObject.StartTime, timeRange.Value);
+
+            foreach (var obj in hitObject.NestedHitObjects)
+                computeLifetimeStartRecursive(obj);
+        }
+
+        private readonly Dictionary<DrawableHitObject, Cached> hitObjectInitialStateCache = new Dictionary<DrawableHitObject, Cached>();
+
+        // Cant use AddOnce() since the delegate is re-constructed every invocation
+        private void computeInitialStateRecursive(DrawableHitObject hitObject) => hitObject.Schedule(() =>
+        {
+            if (!hitObjectInitialStateCache.TryGetValue(hitObject, out var cached))
+                cached = hitObjectInitialStateCache[hitObject] = new Cached();
+
+            if (cached.IsValid)
+                return;
+
             double endTime = hitObject.HitObject.StartTime;
 
             if (hitObject.HitObject is IHasEndTime e)
@@ -113,7 +135,6 @@ namespace osu.Game.Rulesets.UI.Scrolling
                 }
             }
 
-            hitObject.LifetimeStart = scrollingInfo.Algorithm.GetDisplayStartTime(hitObject.HitObject.StartTime, timeRange.Value);
             hitObject.LifetimeEnd = scrollingInfo.Algorithm.TimeAt(scrollLength * safe_lifetime_end_multiplier, endTime, timeRange.Value, scrollLength);
 
             foreach (var obj in hitObject.NestedHitObjects)
@@ -123,7 +144,9 @@ namespace osu.Game.Rulesets.UI.Scrolling
                 // Nested hitobjects don't need to scroll, but they do need accurate positions
                 updatePosition(obj, hitObject.HitObject.StartTime);
             }
-        }
+
+            cached.Validate();
+        });
 
         protected override void UpdateAfterChildrenLife()
         {