diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneOsuDistanceSnapGrid.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneOsuDistanceSnapGrid.cs
index eff4d919b0..75c57467c3 100644
--- a/osu.Game.Rulesets.Osu.Tests/TestSceneOsuDistanceSnapGrid.cs
+++ b/osu.Game.Rulesets.Osu.Tests/TestSceneOsuDistanceSnapGrid.cs
@@ -34,7 +34,7 @@ namespace osu.Game.Rulesets.Osu.Tests
         };
 
         [Cached(typeof(IEditorBeatmap))]
-        private readonly EditorBeatmap<OsuHitObject> editorBeatmap;
+        private readonly EditorBeatmap editorBeatmap;
 
         [Cached]
         private readonly BindableBeatDivisor beatDivisor = new BindableBeatDivisor();
@@ -46,7 +46,7 @@ namespace osu.Game.Rulesets.Osu.Tests
 
         public TestSceneOsuDistanceSnapGrid()
         {
-            editorBeatmap = new EditorBeatmap<OsuHitObject>(new OsuBeatmap());
+            editorBeatmap = new EditorBeatmap(new OsuBeatmap());
         }
 
         [SetUp]
diff --git a/osu.Game.Rulesets.Osu/Edit/OsuHitObjectComposer.cs b/osu.Game.Rulesets.Osu/Edit/OsuHitObjectComposer.cs
index a2c1a5f5f4..49624ea733 100644
--- a/osu.Game.Rulesets.Osu/Edit/OsuHitObjectComposer.cs
+++ b/osu.Game.Rulesets.Osu/Edit/OsuHitObjectComposer.cs
@@ -91,10 +91,10 @@ namespace osu.Game.Rulesets.Osu.Edit
             if (sourceIndex == -1)
                 return null;
 
-            OsuHitObject sourceObject = EditorBeatmap.HitObjects[sourceIndex];
+            HitObject sourceObject = EditorBeatmap.HitObjects[sourceIndex];
 
             int targetIndex = sourceIndex + targetOffset;
-            OsuHitObject targetObject = null;
+            HitObject targetObject = null;
 
             // Keep advancing the target object while its start time falls before the end time of the source object
             while (true)
@@ -111,7 +111,7 @@ namespace osu.Game.Rulesets.Osu.Edit
                 targetIndex++;
             }
 
-            return new OsuDistanceSnapGrid(sourceObject, targetObject);
+            return new OsuDistanceSnapGrid((OsuHitObject)sourceObject, (OsuHitObject)targetObject);
         }
     }
 }
diff --git a/osu.Game.Tests/Beatmaps/EditorBeatmapTest.cs b/osu.Game.Tests/Beatmaps/EditorBeatmapTest.cs
index 98e630abd2..12d729d09f 100644
--- a/osu.Game.Tests/Beatmaps/EditorBeatmapTest.cs
+++ b/osu.Game.Tests/Beatmaps/EditorBeatmapTest.cs
@@ -20,7 +20,7 @@ namespace osu.Game.Tests.Beatmaps
         [Test]
         public void TestHitObjectAddEvent()
         {
-            var editorBeatmap = new EditorBeatmap<OsuHitObject>(new OsuBeatmap());
+            var editorBeatmap = new EditorBeatmap(new OsuBeatmap());
 
             HitObject addedObject = null;
             editorBeatmap.HitObjectAdded += h => addedObject = h;
@@ -38,7 +38,7 @@ namespace osu.Game.Tests.Beatmaps
         public void HitObjectRemoveEvent()
         {
             var hitCircle = new HitCircle();
-            var editorBeatmap = new EditorBeatmap<OsuHitObject>(new OsuBeatmap { HitObjects = { hitCircle } });
+            var editorBeatmap = new EditorBeatmap(new OsuBeatmap { HitObjects = { hitCircle } });
 
             HitObject removedObject = null;
             editorBeatmap.HitObjectRemoved += h => removedObject = h;
@@ -55,7 +55,7 @@ namespace osu.Game.Tests.Beatmaps
         public void TestInitialHitObjectStartTimeChangeEvent()
         {
             var hitCircle = new HitCircle();
-            var editorBeatmap = new EditorBeatmap<OsuHitObject>(new OsuBeatmap { HitObjects = { hitCircle } });
+            var editorBeatmap = new EditorBeatmap(new OsuBeatmap { HitObjects = { hitCircle } });
 
             HitObject changedObject = null;
             editorBeatmap.StartTimeChanged += h => changedObject = h;
@@ -71,7 +71,7 @@ namespace osu.Game.Tests.Beatmaps
         [Test]
         public void TestAddedHitObjectStartTimeChangeEvent()
         {
-            var editorBeatmap = new EditorBeatmap<OsuHitObject>(new OsuBeatmap());
+            var editorBeatmap = new EditorBeatmap(new OsuBeatmap());
 
             HitObject changedObject = null;
             editorBeatmap.StartTimeChanged += h => changedObject = h;
@@ -92,7 +92,7 @@ namespace osu.Game.Tests.Beatmaps
         public void TestRemovedHitObjectStartTimeChangeEvent()
         {
             var hitCircle = new HitCircle();
-            var editorBeatmap = new EditorBeatmap<OsuHitObject>(new OsuBeatmap { HitObjects = { hitCircle } });
+            var editorBeatmap = new EditorBeatmap(new OsuBeatmap { HitObjects = { hitCircle } });
 
             HitObject changedObject = null;
             editorBeatmap.StartTimeChanged += h => changedObject = h;
@@ -110,7 +110,7 @@ namespace osu.Game.Tests.Beatmaps
         [Test]
         public void TestAddHitObjectInMiddle()
         {
-            var editorBeatmap = new EditorBeatmap<OsuHitObject>(new OsuBeatmap
+            var editorBeatmap = new EditorBeatmap(new OsuBeatmap
             {
                 HitObjects =
                 {
@@ -134,7 +134,7 @@ namespace osu.Game.Tests.Beatmaps
         public void TestResortWhenStartTimeChanged()
         {
             var hitCircle = new HitCircle { StartTime = 1000 };
-            var editorBeatmap = new EditorBeatmap<OsuHitObject>(new OsuBeatmap
+            var editorBeatmap = new EditorBeatmap(new OsuBeatmap
             {
                 HitObjects =
                 {
diff --git a/osu.Game.Tests/Editor/TestSceneHitObjectComposerDistanceSnapping.cs b/osu.Game.Tests/Editor/TestSceneHitObjectComposerDistanceSnapping.cs
index fe3cc375ea..b94f08aeaa 100644
--- a/osu.Game.Tests/Editor/TestSceneHitObjectComposerDistanceSnapping.cs
+++ b/osu.Game.Tests/Editor/TestSceneHitObjectComposerDistanceSnapping.cs
@@ -6,7 +6,6 @@ using osu.Framework.Testing;
 using osu.Game.Beatmaps.ControlPoints;
 using osu.Game.Rulesets.Osu;
 using osu.Game.Rulesets.Osu.Edit;
-using osu.Game.Rulesets.Osu.Objects;
 using osu.Game.Screens.Edit;
 using osu.Game.Tests.Visual;
 
@@ -183,7 +182,7 @@ namespace osu.Game.Tests.Editor
 
         private class TestHitObjectComposer : OsuHitObjectComposer
         {
-            public new EditorBeatmap<OsuHitObject> EditorBeatmap => base.EditorBeatmap;
+            public new EditorBeatmap EditorBeatmap => base.EditorBeatmap;
 
             public TestHitObjectComposer()
                 : base(new OsuRuleset())
diff --git a/osu.Game.Tests/Visual/Editor/TestSceneDistanceSnapGrid.cs b/osu.Game.Tests/Visual/Editor/TestSceneDistanceSnapGrid.cs
index 39b4bf7218..83b1f43f38 100644
--- a/osu.Game.Tests/Visual/Editor/TestSceneDistanceSnapGrid.cs
+++ b/osu.Game.Tests/Visual/Editor/TestSceneDistanceSnapGrid.cs
@@ -8,7 +8,6 @@ using osu.Framework.Graphics.Shapes;
 using osu.Game.Beatmaps.ControlPoints;
 using osu.Game.Rulesets.Edit;
 using osu.Game.Rulesets.Osu.Beatmaps;
-using osu.Game.Rulesets.Osu.Objects;
 using osu.Game.Screens.Edit;
 using osu.Game.Screens.Edit.Compose.Components;
 using osuTK;
@@ -22,14 +21,14 @@ namespace osu.Game.Tests.Visual.Editor
         private static readonly Vector2 grid_position = new Vector2(512, 384);
 
         [Cached(typeof(IEditorBeatmap))]
-        private readonly EditorBeatmap<OsuHitObject> editorBeatmap;
+        private readonly EditorBeatmap editorBeatmap;
 
         [Cached(typeof(IDistanceSnapProvider))]
         private readonly SnapProvider snapProvider = new SnapProvider();
 
         public TestSceneDistanceSnapGrid()
         {
-            editorBeatmap = new EditorBeatmap<OsuHitObject>(new OsuBeatmap());
+            editorBeatmap = new EditorBeatmap(new OsuBeatmap());
             editorBeatmap.ControlPointInfo.Add(0, new TimingControlPoint { BeatLength = beat_length });
         }
 
diff --git a/osu.Game.Tests/Visual/Editor/TestSceneEditorComposeTimeline.cs b/osu.Game.Tests/Visual/Editor/TestSceneEditorComposeTimeline.cs
index ed6bc5fe0c..29575cb42e 100644
--- a/osu.Game.Tests/Visual/Editor/TestSceneEditorComposeTimeline.cs
+++ b/osu.Game.Tests/Visual/Editor/TestSceneEditorComposeTimeline.cs
@@ -38,7 +38,7 @@ namespace osu.Game.Tests.Visual.Editor
         {
             Beatmap.Value = new WaveformTestBeatmap(audio);
 
-            var editorBeatmap = new EditorBeatmap<HitObject>((Beatmap<HitObject>)Beatmap.Value.Beatmap);
+            var editorBeatmap = new EditorBeatmap((Beatmap<HitObject>)Beatmap.Value.Beatmap);
 
             Children = new Drawable[]
             {
diff --git a/osu.Game/Rulesets/Edit/DrawableEditRulesetWrapper.cs b/osu.Game/Rulesets/Edit/DrawableEditRulesetWrapper.cs
index 4710465536..ab2332ea86 100644
--- a/osu.Game/Rulesets/Edit/DrawableEditRulesetWrapper.cs
+++ b/osu.Game/Rulesets/Edit/DrawableEditRulesetWrapper.cs
@@ -22,7 +22,7 @@ namespace osu.Game.Rulesets.Edit
         private readonly DrawableRuleset<TObject> drawableRuleset;
 
         [Resolved]
-        private IEditorBeatmap<TObject> beatmap { get; set; }
+        private IEditorBeatmap beatmap { get; set; }
 
         public DrawableEditRulesetWrapper(DrawableRuleset<TObject> drawableRuleset)
         {
diff --git a/osu.Game/Rulesets/Edit/HitObjectComposer.cs b/osu.Game/Rulesets/Edit/HitObjectComposer.cs
index 22d94abcb9..a550cf4a02 100644
--- a/osu.Game/Rulesets/Edit/HitObjectComposer.cs
+++ b/osu.Game/Rulesets/Edit/HitObjectComposer.cs
@@ -35,7 +35,7 @@ namespace osu.Game.Rulesets.Edit
     {
         protected IRulesetConfigManager Config { get; private set; }
 
-        protected new EditorBeatmap<TObject> EditorBeatmap { get; private set; }
+        protected new EditorBeatmap EditorBeatmap { get; private set; }
 
         protected readonly Ruleset Ruleset;
 
@@ -148,14 +148,13 @@ namespace osu.Game.Rulesets.Edit
 
             beatmapProcessor = Ruleset.CreateBeatmapProcessor(playableBeatmap);
 
-            base.EditorBeatmap = EditorBeatmap = new EditorBeatmap<TObject>(playableBeatmap);
+            base.EditorBeatmap = EditorBeatmap = new EditorBeatmap(playableBeatmap);
             EditorBeatmap.HitObjectAdded += addHitObject;
             EditorBeatmap.HitObjectRemoved += removeHitObject;
             EditorBeatmap.StartTimeChanged += UpdateHitObject;
 
             var dependencies = new DependencyContainer(parent);
             dependencies.CacheAs<IEditorBeatmap>(EditorBeatmap);
-            dependencies.CacheAs<IEditorBeatmap<TObject>>(EditorBeatmap);
 
             Config = dependencies.Get<RulesetConfigCache>().GetConfigFor(Ruleset);
 
diff --git a/osu.Game/Screens/Edit/EditorBeatmap.cs b/osu.Game/Screens/Edit/EditorBeatmap.cs
index c3a322ea36..b4177432e0 100644
--- a/osu.Game/Screens/Edit/EditorBeatmap.cs
+++ b/osu.Game/Screens/Edit/EditorBeatmap.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using osu.Framework.Bindables;
 using osu.Game.Beatmaps;
@@ -11,28 +12,27 @@ using osu.Game.Rulesets.Objects;
 
 namespace osu.Game.Screens.Edit
 {
-    public class EditorBeatmap<T> : IEditorBeatmap<T>
-        where T : HitObject
+    public class EditorBeatmap : IEditorBeatmap
     {
         /// <summary>
-        /// Invoked when a <see cref="HitObject"/> is added to this <see cref="EditorBeatmap{T}"/>.
+        /// Invoked when a <see cref="HitObject"/> is added to this <see cref="EditorBeatmap"/>.
         /// </summary>
         public event Action<HitObject> HitObjectAdded;
 
         /// <summary>
-        /// Invoked when a <see cref="HitObject"/> is removed from this <see cref="EditorBeatmap{T}"/>.
+        /// Invoked when a <see cref="HitObject"/> is removed from this <see cref="EditorBeatmap"/>.
         /// </summary>
         public event Action<HitObject> HitObjectRemoved;
 
         /// <summary>
-        /// Invoked when the start time of a <see cref="HitObject"/> in this <see cref="EditorBeatmap{T}"/> was changed.
+        /// Invoked when the start time of a <see cref="HitObject"/> in this <see cref="EditorBeatmap"/> was changed.
         /// </summary>
         public event Action<HitObject> StartTimeChanged;
 
-        private readonly Dictionary<T, Bindable<double>> startTimeBindables = new Dictionary<T, Bindable<double>>();
-        private readonly Beatmap<T> beatmap;
+        private readonly Dictionary<HitObject, Bindable<double>> startTimeBindables = new Dictionary<HitObject, Bindable<double>>();
+        private readonly IBeatmap beatmap;
 
-        public EditorBeatmap(Beatmap<T> beatmap)
+        public EditorBeatmap(IBeatmap beatmap)
         {
             this.beatmap = beatmap;
 
@@ -54,70 +54,73 @@ namespace osu.Game.Screens.Edit
 
         public double TotalBreakTime => beatmap.TotalBreakTime;
 
-        public IReadOnlyList<T> HitObjects => beatmap.HitObjects;
+        public IReadOnlyList<HitObject> HitObjects => beatmap.HitObjects;
 
         IReadOnlyList<HitObject> IBeatmap.HitObjects => beatmap.HitObjects;
 
         public IEnumerable<BeatmapStatistic> GetStatistics() => beatmap.GetStatistics();
 
-        public IBeatmap Clone() => (EditorBeatmap<T>)MemberwiseClone();
+        public IBeatmap Clone() => (EditorBeatmap)MemberwiseClone();
+
+        private IList mutableHitObjects => (IList)beatmap.HitObjects;
 
         /// <summary>
-        /// Adds a <see cref="HitObject"/> to this <see cref="EditorBeatmap{T}"/>.
+        /// Adds a <see cref="HitObject"/> to this <see cref="EditorBeatmap"/>.
         /// </summary>
         /// <param name="hitObject">The <see cref="HitObject"/> to add.</param>
-        public void Add(T hitObject)
+        public void Add(HitObject hitObject)
         {
             trackStartTime(hitObject);
 
             // Preserve existing sorting order in the beatmap
-            var insertionIndex = beatmap.HitObjects.FindLastIndex(h => h.StartTime <= hitObject.StartTime);
-            beatmap.HitObjects.Insert(insertionIndex + 1, hitObject);
+            var insertionIndex = findInsertionIndex(beatmap.HitObjects, hitObject.StartTime);
+            mutableHitObjects.Insert(insertionIndex + 1, hitObject);
 
             HitObjectAdded?.Invoke(hitObject);
         }
 
         /// <summary>
-        /// Removes a <see cref="HitObject"/> from this <see cref="EditorBeatmap{T}"/>.
+        /// Removes a <see cref="HitObject"/> from this <see cref="EditorBeatmap"/>.
         /// </summary>
         /// <param name="hitObject">The <see cref="HitObject"/> to add.</param>
-        public void Remove(T hitObject)
+        public void Remove(HitObject hitObject)
         {
-            if (beatmap.HitObjects.Remove(hitObject))
-            {
-                var bindable = startTimeBindables[hitObject];
-                bindable.UnbindAll();
+            if (!mutableHitObjects.Contains(hitObject))
+                return;
 
-                startTimeBindables.Remove(hitObject);
-                HitObjectRemoved?.Invoke(hitObject);
-            }
+            mutableHitObjects.Remove(hitObject);
+
+            var bindable = startTimeBindables[hitObject];
+            bindable.UnbindAll();
+
+            startTimeBindables.Remove(hitObject);
+            HitObjectRemoved?.Invoke(hitObject);
         }
 
-        private void trackStartTime(T hitObject)
+        private void trackStartTime(HitObject hitObject)
         {
             startTimeBindables[hitObject] = hitObject.StartTimeBindable.GetBoundCopy();
             startTimeBindables[hitObject].ValueChanged += _ =>
             {
                 // For now we'll remove and re-add the hitobject. This is not optimal and can be improved if required.
-                beatmap.HitObjects.Remove(hitObject);
+                mutableHitObjects.Remove(hitObject);
 
-                var insertionIndex = beatmap.HitObjects.FindLastIndex(h => h.StartTime <= hitObject.StartTime);
-                beatmap.HitObjects.Insert(insertionIndex + 1, hitObject);
+                var insertionIndex = findInsertionIndex(beatmap.HitObjects, hitObject.StartTime);
+                mutableHitObjects.Insert(insertionIndex + 1, hitObject);
 
                 StartTimeChanged?.Invoke(hitObject);
             };
         }
 
-        /// <summary>
-        /// Adds a <see cref="HitObject"/> to this <see cref="EditorBeatmap{T}"/>.
-        /// </summary>
-        /// <param name="hitObject">The <see cref="HitObject"/> to add.</param>
-        public void Add(HitObject hitObject) => Add((T)hitObject);
+        private int findInsertionIndex(IReadOnlyList<HitObject> list, double startTime)
+        {
+            for (int i = 0; i < list.Count; i++)
+            {
+                if (list[i].StartTime > startTime)
+                    return i - 1;
+            }
 
-        /// <summary>
-        /// Removes a <see cref="HitObject"/> from this <see cref="EditorBeatmap{T}"/>.
-        /// </summary>
-        /// <param name="hitObject">The <see cref="HitObject"/> to add.</param>
-        public void Remove(HitObject hitObject) => Remove((T)hitObject);
+            return list.Count - 1;
+        }
     }
 }
diff --git a/osu.Game/Screens/Edit/IEditorBeatmap.cs b/osu.Game/Screens/Edit/IEditorBeatmap.cs
index 3e3418ef79..df4efd289e 100644
--- a/osu.Game/Screens/Edit/IEditorBeatmap.cs
+++ b/osu.Game/Screens/Edit/IEditorBeatmap.cs
@@ -10,7 +10,7 @@ namespace osu.Game.Screens.Edit
 {
     /// <summary>
     /// Interface for the <see cref="IBeatmap"/> contained by the see <see cref="HitObjectComposer"/>.
-    /// Children of <see cref="HitObjectComposer"/> may resolve the beatmap via <see cref="IEditorBeatmap"/> or <see cref="IEditorBeatmap{T}"/>.
+    /// Children of <see cref="HitObjectComposer"/> may resolve the beatmap via <see cref="IEditorBeatmap"/>.
     /// </summary>
     public interface IEditorBeatmap : IBeatmap
     {
@@ -25,17 +25,8 @@ namespace osu.Game.Screens.Edit
         event Action<HitObject> HitObjectRemoved;
 
         /// <summary>
-        /// Invoked when the start time of a <see cref="HitObject"/> in this <see cref="EditorBeatmap{T}"/> was changed.
+        /// Invoked when the start time of a <see cref="HitObject"/> in this <see cref="EditorBeatmap"/> was changed.
         /// </summary>
         event Action<HitObject> StartTimeChanged;
     }
-
-    /// <summary>
-    /// Interface for the <see cref="IBeatmap"/> contained by the see <see cref="HitObjectComposer"/>.
-    /// Children of <see cref="HitObjectComposer"/> may resolve the beatmap via <see cref="IEditorBeatmap"/> or <see cref="IEditorBeatmap{T}"/>.
-    /// </summary>
-    public interface IEditorBeatmap<out T> : IEditorBeatmap, IBeatmap<T>
-        where T : HitObject
-    {
-    }
 }