diff --git a/osu.Game.Rulesets.Osu/Objects/Drawables/DrawableOsuHitObject.cs b/osu.Game.Rulesets.Osu/Objects/Drawables/DrawableOsuHitObject.cs
index 2d0b939e30..0326f38439 100644
--- a/osu.Game.Rulesets.Osu/Objects/Drawables/DrawableOsuHitObject.cs
+++ b/osu.Game.Rulesets.Osu/Objects/Drawables/DrawableOsuHitObject.cs
@@ -52,14 +52,14 @@ namespace osu.Game.Rulesets.Osu.Objects.Drawables
             });
         }
 
-        protected override void FreeAfterUse()
+        public override void Free()
         {
             IndexInCurrentComboBindable.UnbindFrom(HitObject.IndexInCurrentComboBindable);
             PositionBindable.UnbindFrom(HitObject.PositionBindable);
             StackHeightBindable.UnbindFrom(HitObject.StackHeightBindable);
             ScaleBindable.UnbindFrom(HitObject.ScaleBindable);
 
-            base.FreeAfterUse();
+            base.Free();
         }
 
         public override void Apply(HitObject hitObject)
diff --git a/osu.Game.Rulesets.Osu/Objects/Drawables/DrawableSlider.cs b/osu.Game.Rulesets.Osu/Objects/Drawables/DrawableSlider.cs
index 72b932ea36..844e95aa7a 100644
--- a/osu.Game.Rulesets.Osu/Objects/Drawables/DrawableSlider.cs
+++ b/osu.Game.Rulesets.Osu/Objects/Drawables/DrawableSlider.cs
@@ -78,11 +78,11 @@ namespace osu.Game.Rulesets.Osu.Objects.Drawables
             Tracking.BindValueChanged(updateSlidingSample);
         }
 
-        protected override void FreeAfterUse()
+        public override void Free()
         {
             PathVersion.UnbindFrom(HitObject.Path.Version);
 
-            base.FreeAfterUse();
+            base.Free();
         }
 
         public override void Apply(HitObject hitObject)
diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
index 6e2967062c..8e56b07420 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
@@ -115,6 +115,11 @@ namespace osu.Game.Rulesets.Objects.Drawables
 
         public IBindable<ArmedState> State => state;
 
+        /// <summary>
+        /// Whether <see cref="HitObject"/> is currently applied.
+        /// </summary>
+        private bool hasHitObjectApplied;
+
         /// <summary>
         /// Creates a new <see cref="DrawableHitObject"/>.
         /// </summary>
@@ -151,50 +156,16 @@ namespace osu.Game.Rulesets.Objects.Drawables
             updateState(ArmedState.Idle, true);
         }
 
-        /// <summary>
-        /// Removes the <see cref="HitObject"/> currently applied to this <see cref="DrawableHitObject"/>,
-        /// </summary>
-        protected override void FreeAfterUse()
-        {
-            StartTimeBindable.UnbindFrom(HitObject.StartTimeBindable);
-            if (HitObject is IHasComboInformation combo)
-                comboIndexBindable.UnbindFrom(combo.ComboIndexBindable);
-
-            samplesBindable.UnbindFrom(HitObject.SamplesBindable);
-
-            // When a new hitobject is applied, the samples will be cleared before re-populating.
-            // In order to stop this needless update, the event is unbound and re-bound as late as possible in Apply().
-            samplesBindable.CollectionChanged -= onSamplesChanged;
-
-            if (nestedHitObjects.IsValueCreated)
-            {
-                foreach (var obj in nestedHitObjects.Value)
-                {
-                    obj.OnNewResult -= onNewResult;
-                    obj.OnRevertResult -= onRevertResult;
-                    obj.ApplyCustomUpdateState -= onApplyCustomUpdateState;
-                }
-
-                nestedHitObjects.Value.Clear();
-                ClearNestedHitObjects();
-            }
-
-            HitObject.DefaultsApplied -= onDefaultsApplied;
-            HitObject = null;
-
-            base.FreeAfterUse();
-        }
-
         /// <summary>
         /// Applies a new <see cref="HitObject"/> to be represented by this <see cref="DrawableHitObject"/>.
         /// </summary>
-        /// <param name="hitObject"></param>
+        /// <param name="hitObject">The <see cref="HitObject"/> to apply.</param>
         public virtual void Apply(HitObject hitObject)
         {
-            if (HitObject != null)
-                FreeAfterUse();
+            if (hasHitObjectApplied)
+                Free();
 
-            HitObject = hitObject;
+            HitObject = hitObject ?? throw new InvalidOperationException($"Cannot apply a null {nameof(HitObject)}.");
 
             // Copy any existing result from the hitobject (required for rewind / judgement revert).
             Result = HitObject.Result;
@@ -230,6 +201,52 @@ namespace osu.Game.Rulesets.Objects.Drawables
             // If not loaded, the state update happens in LoadComplete(). Otherwise, the update is scheduled to allow for lifetime updates.
             if (IsLoaded)
                 Schedule(() => updateState(ArmedState.Idle, true));
+
+            hasHitObjectApplied = true;
+        }
+
+        /// <summary>
+        /// Removes the currently applied <see cref="HitObject"/>
+        /// </summary>
+        public virtual void Free()
+        {
+            StartTimeBindable.UnbindFrom(HitObject.StartTimeBindable);
+            if (HitObject is IHasComboInformation combo)
+                comboIndexBindable.UnbindFrom(combo.ComboIndexBindable);
+
+            samplesBindable.UnbindFrom(HitObject.SamplesBindable);
+
+            // When a new hitobject is applied, the samples will be cleared before re-populating.
+            // In order to stop this needless update, the event is unbound and re-bound as late as possible in Apply().
+            samplesBindable.CollectionChanged -= onSamplesChanged;
+
+            if (nestedHitObjects.IsValueCreated)
+            {
+                foreach (var obj in nestedHitObjects.Value)
+                {
+                    obj.OnNewResult -= onNewResult;
+                    obj.OnRevertResult -= onRevertResult;
+                    obj.ApplyCustomUpdateState -= onApplyCustomUpdateState;
+                }
+
+                nestedHitObjects.Value.Clear();
+                ClearNestedHitObjects();
+            }
+
+            HitObject.DefaultsApplied -= onDefaultsApplied;
+            HitObject = null;
+
+            hasHitObjectApplied = false;
+        }
+
+        protected sealed override void FreeAfterUse()
+        {
+            base.FreeAfterUse();
+
+            if (!IsInPool)
+                return;
+
+            Free();
         }
 
         /// <summary>
@@ -268,7 +285,7 @@ namespace osu.Game.Rulesets.Objects.Drawables
 
         private void onDefaultsApplied(HitObject hitObject)
         {
-            FreeAfterUse();
+            Free();
             Apply(hitObject);
             DefaultsApplied?.Invoke(this);
         }