diff --git a/osu.Game/Beatmaps/ControlPoints/ControlPointInfo.cs b/osu.Game/Beatmaps/ControlPoints/ControlPointInfo.cs
index 3ff40fe194..dd88d0b539 100644
--- a/osu.Game/Beatmaps/ControlPoints/ControlPointInfo.cs
+++ b/osu.Game/Beatmaps/ControlPoints/ControlPointInfo.cs
@@ -33,14 +33,6 @@ namespace osu.Game.Beatmaps.ControlPoints
 
         private readonly SortedList<TimingControlPoint> timingPoints = new SortedList<TimingControlPoint>(Comparer<TimingControlPoint>.Default);
 
-        /// <summary>
-        /// All difficulty points.
-        /// </summary>
-        [JsonProperty]
-        public IReadOnlyList<DifficultyControlPoint> DifficultyPoints => difficultyPoints;
-
-        private readonly SortedList<DifficultyControlPoint> difficultyPoints = new SortedList<DifficultyControlPoint>(Comparer<DifficultyControlPoint>.Default);
-
         /// <summary>
         /// All effect points.
         /// </summary>
@@ -55,14 +47,6 @@ namespace osu.Game.Beatmaps.ControlPoints
         [JsonIgnore]
         public IEnumerable<ControlPoint> AllControlPoints => Groups.SelectMany(g => g.ControlPoints).ToArray();
 
-        /// <summary>
-        /// Finds the difficulty control point that is active at <paramref name="time"/>.
-        /// </summary>
-        /// <param name="time">The time to find the difficulty control point at.</param>
-        /// <returns>The difficulty control point.</returns>
-        [NotNull]
-        public DifficultyControlPoint DifficultyPointAt(double time) => BinarySearchWithFallback(DifficultyPoints, time, DifficultyControlPoint.DEFAULT);
-
         /// <summary>
         /// Finds the effect control point that is active at <paramref name="time"/>.
         /// </summary>
@@ -100,7 +84,6 @@ namespace osu.Game.Beatmaps.ControlPoints
         {
             groups.Clear();
             timingPoints.Clear();
-            difficultyPoints.Clear();
             effectPoints.Clear();
         }
 
@@ -277,10 +260,6 @@ namespace osu.Game.Beatmaps.ControlPoints
                 case EffectControlPoint _:
                     existing = EffectPointAt(time);
                     break;
-
-                case DifficultyControlPoint _:
-                    existing = DifficultyPointAt(time);
-                    break;
             }
 
             return newPoint?.IsRedundant(existing) == true;
@@ -297,10 +276,6 @@ namespace osu.Game.Beatmaps.ControlPoints
                 case EffectControlPoint typed:
                     effectPoints.Add(typed);
                     break;
-
-                case DifficultyControlPoint typed:
-                    difficultyPoints.Add(typed);
-                    break;
             }
         }
 
@@ -315,10 +290,6 @@ namespace osu.Game.Beatmaps.ControlPoints
                 case EffectControlPoint typed:
                     effectPoints.Remove(typed);
                     break;
-
-                case DifficultyControlPoint typed:
-                    difficultyPoints.Remove(typed);
-                    break;
             }
         }
 
diff --git a/osu.Game/Beatmaps/Legacy/LegacyControlPointInfo.cs b/osu.Game/Beatmaps/Legacy/LegacyControlPointInfo.cs
index ff0ca5ebe1..d3c0559ec8 100644
--- a/osu.Game/Beatmaps/Legacy/LegacyControlPointInfo.cs
+++ b/osu.Game/Beatmaps/Legacy/LegacyControlPointInfo.cs
@@ -1,9 +1,11 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System.Collections.Generic;
 using JetBrains.Annotations;
 using Newtonsoft.Json;
 using osu.Framework.Bindables;
+using osu.Framework.Lists;
 using osu.Game.Beatmaps.ControlPoints;
 
 namespace osu.Game.Beatmaps.Legacy
@@ -26,35 +28,72 @@ namespace osu.Game.Beatmaps.Legacy
         [NotNull]
         public SampleControlPoint SamplePointAt(double time) => BinarySearchWithFallback(SamplePoints, time, SamplePoints.Count > 0 ? SamplePoints[0] : SampleControlPoint.DEFAULT);
 
+        /// <summary>
+        /// All difficulty points.
+        /// </summary>
+        [JsonProperty]
+        public IReadOnlyList<DifficultyControlPoint> DifficultyPoints => difficultyPoints;
+
+        private readonly SortedList<DifficultyControlPoint> difficultyPoints = new SortedList<DifficultyControlPoint>(Comparer<DifficultyControlPoint>.Default);
+
+        /// <summary>
+        /// Finds the difficulty control point that is active at <paramref name="time"/>.
+        /// </summary>
+        /// <param name="time">The time to find the difficulty control point at.</param>
+        /// <returns>The difficulty control point.</returns>
+        [NotNull]
+        public DifficultyControlPoint DifficultyPointAt(double time) => BinarySearchWithFallback(DifficultyPoints, time, DifficultyControlPoint.DEFAULT);
+
         public override void Clear()
         {
             base.Clear();
             samplePoints.Clear();
+            difficultyPoints.Clear();
         }
 
         protected override bool CheckAlreadyExisting(double time, ControlPoint newPoint)
         {
-            if (newPoint is SampleControlPoint)
+            switch (newPoint)
             {
-                var existing = BinarySearch(SamplePoints, time);
-                return newPoint.IsRedundant(existing);
-            }
+                case SampleControlPoint _:
+                    return newPoint.IsRedundant(SamplePointAt(time));
 
-            return base.CheckAlreadyExisting(time, newPoint);
+                case DifficultyControlPoint _:
+                    return newPoint.IsRedundant(DifficultyPointAt(time));
+
+                default:
+                    return base.CheckAlreadyExisting(time, newPoint);
+            }
         }
 
         protected override void GroupItemAdded(ControlPoint controlPoint)
         {
-            if (controlPoint is SampleControlPoint typed)
-                samplePoints.Add(typed);
+            switch (controlPoint)
+            {
+                case SampleControlPoint typed:
+                    samplePoints.Add(typed);
+                    break;
+
+                case DifficultyControlPoint typed:
+                    difficultyPoints.Add(typed);
+                    break;
+            }
 
             base.GroupItemAdded(controlPoint);
         }
 
         protected override void GroupItemRemoved(ControlPoint controlPoint)
         {
-            if (controlPoint is SampleControlPoint typed)
-                samplePoints.Remove(typed);
+            switch (controlPoint)
+            {
+                case SampleControlPoint typed:
+                    samplePoints.Remove(typed);
+                    break;
+
+                case DifficultyControlPoint typed:
+                    difficultyPoints.Remove(typed);
+                    break;
+            }
 
             base.GroupItemRemoved(controlPoint);
         }