diff --git a/osu.Game.Rulesets.Catch/UI/CatchPlayfield.cs b/osu.Game.Rulesets.Catch/UI/CatchPlayfield.cs
index 39b7ffb387..8bb206543b 100644
--- a/osu.Game.Rulesets.Catch/UI/CatchPlayfield.cs
+++ b/osu.Game.Rulesets.Catch/UI/CatchPlayfield.cs
@@ -57,7 +57,6 @@ namespace osu.Game.Rulesets.Catch.UI
 
         public override void Add(DrawableHitObject h)
         {
-            h.Depth = (float)h.HitObject.StartTime;
             h.OnJudgement += onJudgement;
 
             base.Add(h);
diff --git a/osu.Game.Rulesets.Mania/UI/Column.cs b/osu.Game.Rulesets.Mania/UI/Column.cs
index 21f00c003b..109ffd0d25 100644
--- a/osu.Game.Rulesets.Mania/UI/Column.cs
+++ b/osu.Game.Rulesets.Mania/UI/Column.cs
@@ -205,7 +205,6 @@ namespace osu.Game.Rulesets.Mania.UI
         /// <param name="hitObject">The DrawableHitObject to add.</param>
         public override void Add(DrawableHitObject hitObject)
         {
-            hitObject.Depth = (float)hitObject.HitObject.StartTime;
             hitObject.AccentColour = AccentColour;
             hitObject.OnJudgement += OnJudgement;
 
diff --git a/osu.Game.Rulesets.Osu/UI/OsuPlayfield.cs b/osu.Game.Rulesets.Osu/UI/OsuPlayfield.cs
index 98a8096678..0c5d757474 100644
--- a/osu.Game.Rulesets.Osu/UI/OsuPlayfield.cs
+++ b/osu.Game.Rulesets.Osu/UI/OsuPlayfield.cs
@@ -55,8 +55,6 @@ namespace osu.Game.Rulesets.Osu.UI
 
         public override void Add(DrawableHitObject h)
         {
-            h.Depth = (float)h.HitObject.StartTime;
-
             h.OnJudgement += onJudgement;
 
             var c = h as IDrawableHitObjectWithProxiedApproach;
diff --git a/osu.Game.Rulesets.Taiko/UI/TaikoPlayfield.cs b/osu.Game.Rulesets.Taiko/UI/TaikoPlayfield.cs
index 75aaceaecb..effb9eb54f 100644
--- a/osu.Game.Rulesets.Taiko/UI/TaikoPlayfield.cs
+++ b/osu.Game.Rulesets.Taiko/UI/TaikoPlayfield.cs
@@ -208,8 +208,6 @@ namespace osu.Game.Rulesets.Taiko.UI
 
         public override void Add(DrawableHitObject h)
         {
-            h.Depth = (float)h.HitObject.StartTime;
-
             h.OnJudgement += OnJudgement;
 
             base.Add(h);
diff --git a/osu.Game/Rulesets/UI/HitObjectContainer.cs b/osu.Game/Rulesets/UI/HitObjectContainer.cs
index c26a6cdff0..ecb10dfba2 100644
--- a/osu.Game/Rulesets/UI/HitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/HitObjectContainer.cs
@@ -3,6 +3,7 @@
 
 using System.Collections.Generic;
 using System.Linq;
+using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 using osu.Game.Rulesets.Objects.Drawables;
 
@@ -15,5 +16,15 @@ namespace osu.Game.Rulesets.UI
 
         public virtual void Add(DrawableHitObject hitObject) => AddInternal(hitObject);
         public virtual bool Remove(DrawableHitObject hitObject) => RemoveInternal(hitObject);
+
+        protected override int Compare(Drawable x, Drawable y)
+        {
+            if (!(x is DrawableHitObject xObj) || !(y is DrawableHitObject yObj))
+                return base.Compare(x, y);
+
+            // Put earlier hitobjects towards the end of the list, so they handle input first
+            int i = yObj.HitObject.StartTime.CompareTo(xObj.HitObject.StartTime);
+            return i == 0 ? CompareReverseChildID(x, y) : i;
+        }
     }
 }