diff --git a/osu.Game/Rulesets/Difficulty/DifficultyCalculator.cs b/osu.Game/Rulesets/Difficulty/DifficultyCalculator.cs
index d37cfc28b9..4644a457bf 100644
--- a/osu.Game/Rulesets/Difficulty/DifficultyCalculator.cs
+++ b/osu.Game/Rulesets/Difficulty/DifficultyCalculator.cs
@@ -109,26 +109,23 @@ namespace osu.Game.Rulesets.Difficulty
             var progressiveBeatmap = new ProgressiveCalculationBeatmap(Beatmap);
             var difficultyObjects = getDifficultyHitObjects().ToArray();
 
-            foreach (var obj in difficultyObjects)
+            int currentIndex = 0;
+
+            foreach (var obj in Beatmap.HitObjects)
             {
-                // Implementations expect the progressive beatmap to only contain top-level objects from the original beatmap.
-                // At the same time, we also need to consider the possibility DHOs may not be generated for any given object,
-                // so we'll add all remaining objects up to the current point in time to the progressive beatmap.
-                for (int i = progressiveBeatmap.HitObjects.Count; i < Beatmap.HitObjects.Count; i++)
-                {
-                    if (obj != difficultyObjects[^1] && Beatmap.HitObjects[i].StartTime > obj.BaseObject.StartTime)
-                        break;
+                progressiveBeatmap.HitObjects.Add(obj);
 
-                    progressiveBeatmap.HitObjects.Add(Beatmap.HitObjects[i]);
+                while (currentIndex < difficultyObjects.Length && difficultyObjects[currentIndex].BaseObject.GetEndTime() <= obj.GetEndTime())
+                {
+                    foreach (var skill in skills)
+                    {
+                        cancellationToken.ThrowIfCancellationRequested();
+                        skill.Process(difficultyObjects[currentIndex]);
+                    }
+                    currentIndex++;
                 }
 
-                foreach (var skill in skills)
-                {
-                    cancellationToken.ThrowIfCancellationRequested();
-                    skill.Process(obj);
-                }
-
-                attribs.Add(new TimedDifficultyAttributes(obj.EndTime * clockRate, CreateDifficultyAttributes(progressiveBeatmap, playableMods, skills, clockRate)));
+                attribs.Add(new TimedDifficultyAttributes(obj.GetEndTime(), CreateDifficultyAttributes(progressiveBeatmap, playableMods, skills, clockRate)));
             }
 
             return attribs;