diff --git a/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs b/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs
index 19637f321b..b41a5e0612 100644
--- a/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs
+++ b/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs
@@ -33,13 +33,13 @@ namespace osu.Game.Rulesets.Catch.Replays
             if (lastFrame != null)
             {
                 if (Position > lastFrame.Position)
-                    Actions.Add(CatchAction.MoveRight);
+                    lastFrame.Actions.Add(CatchAction.MoveRight);
                 else if (Position < lastFrame.Position)
-                    Actions.Add(CatchAction.MoveLeft);
+                    lastFrame.Actions.Add(CatchAction.MoveLeft);
             }
         }
 
-        public void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null)
+        public void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, ReplayFrame lastFrame = null)
         {
             Position = currentFrame.Position.X / CatchPlayfield.BASE_WIDTH;
             Dashing = currentFrame.ButtonState == ReplayButtonState.Left1;
@@ -47,11 +47,12 @@ namespace osu.Game.Rulesets.Catch.Replays
             if (Dashing)
                 Actions.Add(CatchAction.Dash);
 
-            if (lastFrame != null)
+            // this probably needs some cross-checking with osu-stable to ensure it is actually correct.
+            if (lastFrame is CatchReplayFrame lastCatchFrame)
             {
-                if (currentFrame.Position.X > lastFrame.Position.X)
-                    Actions.Add(CatchAction.MoveRight);
-                else if (currentFrame.Position.X < lastFrame.Position.X)
+                if (Position > lastCatchFrame.Position)
+                    lastCatchFrame.Actions.Add(CatchAction.MoveRight);
+                else if (Position < lastCatchFrame.Position)
                     Actions.Add(CatchAction.MoveLeft);
             }
         }
diff --git a/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs b/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs
index b2901f46c0..70ba5cd938 100644
--- a/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs
+++ b/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs
@@ -24,7 +24,7 @@ namespace osu.Game.Rulesets.Mania.Replays
             Actions.AddRange(actions);
         }
 
-        public void ConvertFrom(LegacyReplayFrame legacyFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null)
+        public void ConvertFrom(LegacyReplayFrame legacyFrame, IBeatmap beatmap, ReplayFrame lastFrame = null)
         {
             // We don't need to fully convert, just create the converter
             var converter = new ManiaBeatmapConverter(beatmap);
diff --git a/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs b/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs
index 441b69ef2d..e6c6db5e61 100644
--- a/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs
+++ b/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs
@@ -26,7 +26,7 @@ namespace osu.Game.Rulesets.Osu.Replays
             Actions.AddRange(actions);
         }
 
-        public void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null)
+        public void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, ReplayFrame lastFrame = null)
         {
             Position = currentFrame.Position;
             if (currentFrame.MouseLeft) Actions.Add(OsuAction.LeftButton);
diff --git a/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs b/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs
index 6e43892777..c5ebefc397 100644
--- a/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs
+++ b/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs
@@ -23,7 +23,7 @@ namespace osu.Game.Rulesets.Taiko.Replays
             Actions.AddRange(actions);
         }
 
-        public void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null)
+        public void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, ReplayFrame lastFrame = null)
         {
             if (currentFrame.MouseRight1) Actions.Add(TaikoAction.LeftRim);
             if (currentFrame.MouseRight2) Actions.Add(TaikoAction.RightRim);
diff --git a/osu.Game/Rulesets/Replays/Types/IConvertibleReplayFrame.cs b/osu.Game/Rulesets/Replays/Types/IConvertibleReplayFrame.cs
index 8fcdec6630..c2947c0aca 100644
--- a/osu.Game/Rulesets/Replays/Types/IConvertibleReplayFrame.cs
+++ b/osu.Game/Rulesets/Replays/Types/IConvertibleReplayFrame.cs
@@ -16,7 +16,7 @@ namespace osu.Game.Rulesets.Replays.Types
         /// </summary>
         /// <param name="currentFrame">The <see cref="LegacyReplayFrame"/> to extract values from.</param>
         /// <param name="beatmap">The beatmap.</param>
-        /// <param name="lastFrame">The last <see cref="LegacyReplayFrame"/>, used to fill in missing delta information. May be null.</param>
-        void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, LegacyReplayFrame lastFrame = null);
+        /// <param name="lastFrame">The last post-conversion <see cref="ReplayFrame"/>, used to fill in missing delta information. May be null.</param>
+        void ConvertFrom(LegacyReplayFrame currentFrame, IBeatmap beatmap, ReplayFrame lastFrame = null);
     }
 }
diff --git a/osu.Game/Scoring/Legacy/LegacyScoreParser.cs b/osu.Game/Scoring/Legacy/LegacyScoreParser.cs
index 5a90daa045..2cdd0c4b5e 100644
--- a/osu.Game/Scoring/Legacy/LegacyScoreParser.cs
+++ b/osu.Game/Scoring/Legacy/LegacyScoreParser.cs
@@ -218,7 +218,7 @@ namespace osu.Game.Scoring.Legacy
         private void readLegacyReplay(Replay replay, StreamReader reader)
         {
             float lastTime = 0;
-            LegacyReplayFrame currentFrame = null;
+            ReplayFrame currentFrame = null;
 
             foreach (var l in reader.ReadToEnd().Split(','))
             {
@@ -241,18 +241,16 @@ namespace osu.Game.Scoring.Legacy
                 if (diff < 0)
                     continue;
 
-                var lastFrame = currentFrame;
-
-                currentFrame = new LegacyReplayFrame(lastTime,
+                currentFrame = convertFrame(new LegacyReplayFrame(lastTime,
                     Parsing.ParseFloat(split[1], Parsing.MAX_COORDINATE_VALUE),
                     Parsing.ParseFloat(split[2], Parsing.MAX_COORDINATE_VALUE),
-                    (ReplayButtonState)Parsing.ParseInt(split[3]));
+                    (ReplayButtonState)Parsing.ParseInt(split[3])), currentFrame);
 
-                replay.Frames.Add(convertFrame(currentFrame, lastFrame));
+                replay.Frames.Add(currentFrame);
             }
         }
 
-        private ReplayFrame convertFrame(LegacyReplayFrame currentFrame, LegacyReplayFrame lastFrame)
+        private ReplayFrame convertFrame(LegacyReplayFrame currentFrame, ReplayFrame lastFrame)
         {
             var convertible = currentRuleset.CreateConvertibleReplayFrame();
             if (convertible == null)