diff --git a/osu.Game.Tests/Gameplay/TestSceneDrawableHitObject.cs b/osu.Game.Tests/Gameplay/TestSceneDrawableHitObject.cs
index da0d57f9d1..0ce71696bd 100644
--- a/osu.Game.Tests/Gameplay/TestSceneDrawableHitObject.cs
+++ b/osu.Game.Tests/Gameplay/TestSceneDrawableHitObject.cs
@@ -44,11 +44,9 @@ namespace osu.Game.Tests.Gameplay
         {
             TestDrawableHitObject dho = null;
             TestLifetimeEntry entry = null;
-            AddStep("Create DHO", () =>
+            AddStep("Create DHO", () => Child = dho = new TestDrawableHitObject
             {
-                dho = new TestDrawableHitObject(null);
-                dho.Apply(entry = new TestLifetimeEntry(new HitObject()));
-                Child = dho;
+                Entry = entry = new TestLifetimeEntry(new HitObject())
             });
 
             AddStep("KeepAlive = true", () =>
@@ -81,12 +79,10 @@ namespace osu.Game.Tests.Gameplay
             AddAssert("Lifetime is updated", () => entry.LifetimeStart == -TestLifetimeEntry.INITIAL_LIFETIME_OFFSET);
 
             TestDrawableHitObject dho = null;
-            AddStep("Create DHO", () =>
+            AddStep("Create DHO", () => Child = dho = new TestDrawableHitObject
             {
-                dho = new TestDrawableHitObject(null);
-                dho.Apply(entry);
-                Child = dho;
-                dho.SetLifetimeStartOnApply = true;
+                Entry = entry,
+                SetLifetimeStartOnApply = true
             });
             AddStep("ApplyDefaults", () => entry.HitObject.ApplyDefaults(new ControlPointInfo(), new BeatmapDifficulty()));
             AddAssert("Lifetime is correct", () => dho.LifetimeStart == TestDrawableHitObject.LIFETIME_ON_APPLY && entry.LifetimeStart == TestDrawableHitObject.LIFETIME_ON_APPLY);
@@ -97,11 +93,9 @@ namespace osu.Game.Tests.Gameplay
         {
             TestDrawableHitObject dho = null;
             TestLifetimeEntry entry = null;
-            AddStep("Create DHO", () =>
+            AddStep("Create DHO", () => Child = dho = new TestDrawableHitObject
             {
-                dho = new TestDrawableHitObject(null);
-                dho.Apply(entry = new TestLifetimeEntry(new HitObject()));
-                Child = dho;
+                Entry = entry = new TestLifetimeEntry(new HitObject())
             });
 
             AddStep("Set entry lifetime", () =>
@@ -135,7 +129,7 @@ namespace osu.Game.Tests.Gameplay
 
             public bool SetLifetimeStartOnApply;
 
-            public TestDrawableHitObject(HitObject hitObject)
+            public TestDrawableHitObject(HitObject hitObject = null)
                 : base(hitObject)
             {
             }
diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
index 7fc35fc778..a0717ec38e 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
@@ -156,10 +156,11 @@ namespace osu.Game.Rulesets.Objects.Drawables
         /// If <c>null</c>, a hitobject is expected to be later applied via <see cref="PoolableDrawableWithLifetime{TEntry}.Apply"/> (or automatically via pooling).
         /// </param>
         protected DrawableHitObject([CanBeNull] HitObject initialHitObject = null)
-            : base(initialHitObject != null ? new SyntheticHitObjectEntry(initialHitObject) : null)
         {
-            if (Entry != null)
-                ensureEntryHasResult();
+            if (initialHitObject == null) return;
+
+            Entry = new SyntheticHitObjectEntry(initialHitObject);
+            ensureEntryHasResult();
         }
 
         [BackgroundDependencyLoader]
diff --git a/osu.Game/Rulesets/Objects/Pooling/PoolableDrawableWithLifetime.cs b/osu.Game/Rulesets/Objects/Pooling/PoolableDrawableWithLifetime.cs
index 4440ca8d21..9c6097a048 100644
--- a/osu.Game/Rulesets/Objects/Pooling/PoolableDrawableWithLifetime.cs
+++ b/osu.Game/Rulesets/Objects/Pooling/PoolableDrawableWithLifetime.cs
@@ -5,6 +5,7 @@
 
 using System;
 using System.Diagnostics;
+using osu.Framework.Graphics;
 using osu.Framework.Graphics.Performance;
 using osu.Framework.Graphics.Pooling;
 
@@ -16,14 +17,32 @@ namespace osu.Game.Rulesets.Objects.Pooling
     /// <typeparam name="TEntry">The <see cref="LifetimeEntry"/> type storing state and controlling this drawable.</typeparam>
     public abstract class PoolableDrawableWithLifetime<TEntry> : PoolableDrawable where TEntry : LifetimeEntry
     {
+        private TEntry? entry;
+
         /// <summary>
         /// The entry holding essential state of this <see cref="PoolableDrawableWithLifetime{TEntry}"/>.
         /// </summary>
-        public TEntry? Entry { get; private set; }
+        /// <remarks>
+        /// If a non-null value is set before loading is started, the entry is applied when the loading is completed.
+        /// It is not valid to set an entry while this <see cref="PoolableDrawableWithLifetime{TEntry}"/> is loading.
+        /// </remarks>
+        public TEntry? Entry
+        {
+            get => entry;
+            set
+            {
+                if (LoadState == LoadState.NotLoaded)
+                    entry = value;
+                else if (value != null)
+                    Apply(value);
+                else if (HasEntryApplied)
+                    free();
+            }
+        }
 
         /// <summary>
         /// Whether <see cref="Entry"/> is applied to this <see cref="PoolableDrawableWithLifetime{TEntry}"/>.
-        /// When an initial entry is specified in the constructor, <see cref="Entry"/> is set but not applied until loading is completed.
+        /// When an <see cref="Entry"/> is set during initialization, it is not applied until loading is completed.
         /// </summary>
         protected bool HasEntryApplied { get; private set; }
 
@@ -65,9 +84,9 @@ namespace osu.Game.Rulesets.Objects.Pooling
         {
             base.LoadAsyncComplete();
 
-            // Apply the initial entry given in the constructor.
+            // Apply the initial entry.
             if (Entry != null && !HasEntryApplied)
-                Apply(Entry);
+                apply(Entry);
         }
 
         /// <summary>
@@ -76,16 +95,10 @@ namespace osu.Game.Rulesets.Objects.Pooling
         /// </summary>
         public void Apply(TEntry entry)
         {
-            if (HasEntryApplied)
-                free();
+            if (LoadState == LoadState.Loading)
+                throw new InvalidOperationException($"Cannot apply a new {nameof(TEntry)} while currently loading.");
 
-            Entry = entry;
-            entry.LifetimeChanged += setLifetimeFromEntry;
-            setLifetimeFromEntry(entry);
-
-            OnApply(entry);
-
-            HasEntryApplied = true;
+            apply(entry);
         }
 
         protected sealed override void FreeAfterUse()
@@ -111,6 +124,20 @@ namespace osu.Game.Rulesets.Objects.Pooling
         {
         }
 
+        private void apply(TEntry entry)
+        {
+            if (HasEntryApplied)
+                free();
+
+            this.entry = entry;
+            entry.LifetimeChanged += setLifetimeFromEntry;
+            setLifetimeFromEntry(entry);
+
+            OnApply(entry);
+
+            HasEntryApplied = true;
+        }
+
         private void free()
         {
             Debug.Assert(Entry != null && HasEntryApplied);
@@ -118,7 +145,7 @@ namespace osu.Game.Rulesets.Objects.Pooling
             OnFree(Entry);
 
             Entry.LifetimeChanged -= setLifetimeFromEntry;
-            Entry = null;
+            entry = null;
             base.LifetimeStart = double.MinValue;
             base.LifetimeEnd = double.MaxValue;