diff --git a/osu.Game.Tests/Gameplay/TestSceneHitObjectContainer.cs b/osu.Game.Tests/Gameplay/TestSceneHitObjectContainer.cs
new file mode 100644
index 0000000000..f2bfccb6de
--- /dev/null
+++ b/osu.Game.Tests/Gameplay/TestSceneHitObjectContainer.cs
@@ -0,0 +1,75 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using JetBrains.Annotations;
+using NUnit.Framework;
+using osu.Framework.Testing;
+using osu.Game.Rulesets.Objects;
+using osu.Game.Rulesets.Objects.Drawables;
+using osu.Game.Rulesets.UI;
+using osu.Game.Tests.Visual;
+
+namespace osu.Game.Tests.Gameplay
+{
+    [HeadlessTest]
+    public class TestSceneHitObjectContainer : OsuTestScene
+    {
+        private HitObjectContainer container;
+
+        [SetUp]
+        public void Setup() => Schedule(() =>
+        {
+            Child = container = new HitObjectContainer();
+        });
+
+        [Test]
+        public void TestLateHitObjectIsAddedEarlierInList()
+        {
+            DrawableHitObject hitObject = null;
+
+            AddStep("setup", () => container.Add(new TestDrawableHitObject(new HitObject { StartTime = 500 })));
+
+            AddStep("add late hitobject", () => container.Add(hitObject = new TestDrawableHitObject(new HitObject { StartTime = 1000 })));
+
+            AddAssert("hitobject index is 0", () => container.IndexOf(hitObject) == 0);
+        }
+
+        [Test]
+        public void TestEarlyHitObjectIsAddedLaterInList()
+        {
+            DrawableHitObject hitObject = null;
+
+            AddStep("setup", () => container.Add(new TestDrawableHitObject(new HitObject { StartTime = 500 })));
+
+            AddStep("add early hitobject", () => container.Add(hitObject = new TestDrawableHitObject(new HitObject())));
+
+            AddAssert("hitobject index is 0", () => container.IndexOf(hitObject) == 1);
+        }
+
+        [Test]
+        public void TestHitObjectsResortedAfterStartTimeChange()
+        {
+            DrawableHitObject firstObject = null;
+            DrawableHitObject secondObject = null;
+
+            AddStep("setup", () =>
+            {
+                container.Add(firstObject = new TestDrawableHitObject(new HitObject()));
+                container.Add(secondObject = new TestDrawableHitObject(new HitObject { StartTime = 1000 }));
+            });
+
+            AddStep("move first object after second", () => firstObject.HitObject.StartTime = 2000);
+
+            AddAssert("first object index is 1", () => container.IndexOf(firstObject) == 0);
+            AddAssert("second object index is 0", () => container.IndexOf(secondObject) == 1);
+        }
+
+        private class TestDrawableHitObject : DrawableHitObject
+        {
+            public TestDrawableHitObject([NotNull] HitObject hitObject)
+                : base(hitObject)
+            {
+            }
+        }
+    }
+}
diff --git a/osu.Game/Rulesets/UI/HitObjectContainer.cs b/osu.Game/Rulesets/UI/HitObjectContainer.cs
index 9485189433..dea981c3ad 100644
--- a/osu.Game/Rulesets/UI/HitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/HitObjectContainer.cs
@@ -3,6 +3,7 @@
 
 using System.Collections.Generic;
 using System.Linq;
+using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 using osu.Game.Rulesets.Objects.Drawables;
@@ -14,13 +15,45 @@ namespace osu.Game.Rulesets.UI
         public IEnumerable<DrawableHitObject> Objects => InternalChildren.Cast<DrawableHitObject>().OrderBy(h => h.HitObject.StartTime);
         public IEnumerable<DrawableHitObject> AliveObjects => AliveInternalChildren.Cast<DrawableHitObject>().OrderBy(h => h.HitObject.StartTime);
 
+        private readonly Dictionary<DrawableHitObject, (IBindable<double> bindable, double timeAtAdd)> startTimeMap = new Dictionary<DrawableHitObject, (IBindable<double>, double)>();
+
         public HitObjectContainer()
         {
             RelativeSizeAxes = Axes.Both;
         }
 
-        public virtual void Add(DrawableHitObject hitObject) => AddInternal(hitObject);
-        public virtual bool Remove(DrawableHitObject hitObject) => RemoveInternal(hitObject);
+        public virtual void Add(DrawableHitObject hitObject)
+        {
+            // Added first for the comparer to remain ordered during AddInternal
+            startTimeMap[hitObject] = (hitObject.HitObject.StartTimeBindable.GetBoundCopy(), hitObject.HitObject.StartTime);
+            startTimeMap[hitObject].bindable.BindValueChanged(_ => onStartTimeChanged(hitObject));
+
+            AddInternal(hitObject);
+        }
+
+        public virtual bool Remove(DrawableHitObject hitObject)
+        {
+            if (!RemoveInternal(hitObject))
+                return false;
+
+            // Removed last for the comparer to remain ordered during RemoveInternal
+            startTimeMap[hitObject].bindable.UnbindAll();
+            startTimeMap.Remove(hitObject);
+
+            return true;
+        }
+
+        public int IndexOf(DrawableHitObject hitObject) => IndexOfInternal(hitObject);
+
+        private void onStartTimeChanged(DrawableHitObject hitObject)
+        {
+            if (!RemoveInternal(hitObject))
+                return;
+
+            // Update the stored time, preserving the existing bindable
+            startTimeMap[hitObject] = (startTimeMap[hitObject].bindable, hitObject.HitObject.StartTime);
+            AddInternal(hitObject);
+        }
 
         protected override int Compare(Drawable x, Drawable y)
         {
@@ -28,7 +61,7 @@ namespace osu.Game.Rulesets.UI
                 return base.Compare(x, y);
 
             // Put earlier hitobjects towards the end of the list, so they handle input first
-            int i = yObj.HitObject.StartTime.CompareTo(xObj.HitObject.StartTime);
+            int i = startTimeMap[yObj].timeAtAdd.CompareTo(startTimeMap[xObj].timeAtAdd);
             return i == 0 ? CompareReverseChildID(x, y) : i;
         }