diff --git a/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform/Replays/EmptyFreeformReplayFrame.cs b/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform/Replays/EmptyFreeformReplayFrame.cs
index c84101ca70..c6be5d6861 100644
--- a/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform/Replays/EmptyFreeformReplayFrame.cs
+++ b/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform/Replays/EmptyFreeformReplayFrame.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
+using System.Linq;
 using osu.Game.Rulesets.Replays;
 using osuTK;
 
@@ -17,5 +18,8 @@ namespace osu.Game.Rulesets.EmptyFreeform.Replays
             if (button.HasValue)
                 Actions.Add(button.Value);
         }
+
+        public override bool IsEquivalentTo(ReplayFrame other)
+            => other is EmptyFreeformReplayFrame freeformFrame && Time == freeformFrame.Time && Position == freeformFrame.Position && Actions.SequenceEqual(freeformFrame.Actions);
     }
 }
diff --git a/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon/Replays/PippidonReplayFrame.cs b/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon/Replays/PippidonReplayFrame.cs
index 949ca160be..c434b62257 100644
--- a/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon/Replays/PippidonReplayFrame.cs
+++ b/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon/Replays/PippidonReplayFrame.cs
@@ -9,5 +9,8 @@ namespace osu.Game.Rulesets.Pippidon.Replays
     public class PippidonReplayFrame : ReplayFrame
     {
         public Vector2 Position;
+
+        public override bool IsEquivalentTo(ReplayFrame other)
+            => other is PippidonReplayFrame pippidonFrame && Time == pippidonFrame.Time && Position == pippidonFrame.Position;
     }
 }
diff --git a/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling/Replays/EmptyScrollingReplayFrame.cs b/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling/Replays/EmptyScrollingReplayFrame.cs
index 2f19cffd2a..722eff6f05 100644
--- a/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling/Replays/EmptyScrollingReplayFrame.cs
+++ b/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling/Replays/EmptyScrollingReplayFrame.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
+using System.Linq;
 using osu.Game.Rulesets.Replays;
 
 namespace osu.Game.Rulesets.EmptyScrolling.Replays
@@ -15,5 +16,8 @@ namespace osu.Game.Rulesets.EmptyScrolling.Replays
             if (button.HasValue)
                 Actions.Add(button.Value);
         }
+
+        public override bool IsEquivalentTo(ReplayFrame other)
+            => other is EmptyScrollingReplayFrame scrollingFrame && Time == scrollingFrame.Time && Actions.SequenceEqual(scrollingFrame.Actions);
     }
 }
diff --git a/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon/Replays/PippidonReplayFrame.cs b/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon/Replays/PippidonReplayFrame.cs
index 468ac9c725..c8df06f6d7 100644
--- a/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon/Replays/PippidonReplayFrame.cs
+++ b/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon/Replays/PippidonReplayFrame.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
+using System.Linq;
 using osu.Game.Rulesets.Replays;
 
 namespace osu.Game.Rulesets.Pippidon.Replays
@@ -15,5 +16,8 @@ namespace osu.Game.Rulesets.Pippidon.Replays
             if (button.HasValue)
                 Actions.Add(button.Value);
         }
+
+        public override bool IsEquivalentTo(ReplayFrame other)
+            => other is PippidonReplayFrame pippidonFrame && Time == pippidonFrame.Time && Actions.SequenceEqual(pippidonFrame.Actions);
     }
 }
diff --git a/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs b/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs
index e30e535e9b..dd1ada21bd 100644
--- a/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs
+++ b/osu.Game.Rulesets.Catch/Replays/CatchReplayFrame.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
+using System.Linq;
 using osu.Game.Beatmaps;
 using osu.Game.Replays.Legacy;
 using osu.Game.Rulesets.Replays;
@@ -64,5 +65,12 @@ namespace osu.Game.Rulesets.Catch.Replays
 
             return new LegacyReplayFrame(Time, Position, null, state);
         }
+
+        public override bool IsEquivalentTo(ReplayFrame other)
+            => other is CatchReplayFrame catchFrame
+               && Time == catchFrame.Time
+               && Position == catchFrame.Position
+               && Dashing == catchFrame.Dashing
+               && Actions.SequenceEqual(catchFrame.Actions);
     }
 }
diff --git a/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs b/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs
index f80c442025..abbaa374f0 100644
--- a/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs
+++ b/osu.Game.Rulesets.Mania/Replays/ManiaReplayFrame.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
+using System.Linq;
 using osu.Game.Beatmaps;
 using osu.Game.Replays.Legacy;
 using osu.Game.Rulesets.Replays;
@@ -47,5 +48,8 @@ namespace osu.Game.Rulesets.Mania.Replays
 
             return new LegacyReplayFrame(Time, keys, null, ReplayButtonState.None);
         }
+
+        public override bool IsEquivalentTo(ReplayFrame other)
+            => other is ManiaReplayFrame maniaFrame && Time == maniaFrame.Time && Actions.SequenceEqual(maniaFrame.Actions);
     }
 }
diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneReplayRecording.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneReplayRecording.cs
index 6b867a7729..8608ea1dfc 100644
--- a/osu.Game.Rulesets.Osu.Tests/TestSceneReplayRecording.cs
+++ b/osu.Game.Rulesets.Osu.Tests/TestSceneReplayRecording.cs
@@ -78,6 +78,16 @@ namespace osu.Game.Rulesets.Osu.Tests
             AddAssert("smoke button press recorded to replay", () => Player.Score.Replay.Frames.OfType<OsuReplayFrame>().Any(f => f.Actions.SequenceEqual([OsuAction.Smoke])));
         }
 
+        [Test]
+        public void TestPressAndReleaseOnSameFrame()
+        {
+            seekTo(0);
+            AddStep("move cursor to circle", () => InputManager.MoveMouseTo(Player.DrawableRuleset.Playfield.HitObjectContainer.AliveObjects.Single()));
+            AddStep("press X", () => InputManager.PressKey(Key.X));
+            AddStep("release X", () => InputManager.ReleaseKey(Key.X));
+            AddAssert("right button press recorded to replay", () => Player.Score.Replay.Frames.OfType<OsuReplayFrame>().Any(f => f.Actions.SequenceEqual([OsuAction.RightButton])));
+        }
+
         private void seekTo(double time)
         {
             AddStep($"seek to {time}ms", () => Player.GameplayClockContainer.Seek(time));
diff --git a/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs b/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs
index 8082c5aef4..db2d9eaeda 100644
--- a/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs
+++ b/osu.Game.Rulesets.Osu/Replays/OsuReplayFrame.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
+using System.Linq;
 using osu.Game.Beatmaps;
 using osu.Game.Replays.Legacy;
 using osu.Game.Rulesets.Replays;
@@ -47,5 +48,8 @@ namespace osu.Game.Rulesets.Osu.Replays
 
             return new LegacyReplayFrame(Time, Position.X, Position.Y, state);
         }
+
+        public override bool IsEquivalentTo(ReplayFrame other)
+            => other is OsuReplayFrame osuFrame && Time == osuFrame.Time && Position == osuFrame.Position && Actions.SequenceEqual(osuFrame.Actions);
     }
 }
diff --git a/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs b/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs
index a0a687dca6..6f10c03a96 100644
--- a/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs
+++ b/osu.Game.Rulesets.Taiko/Replays/TaikoReplayFrame.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
+using System.Linq;
 using osu.Game.Beatmaps;
 using osu.Game.Replays.Legacy;
 using osu.Game.Rulesets.Replays;
@@ -42,5 +43,8 @@ namespace osu.Game.Rulesets.Taiko.Replays
 
             return new LegacyReplayFrame(Time, null, null, state);
         }
+
+        public override bool IsEquivalentTo(ReplayFrame other)
+            => other is TaikoReplayFrame taikoFrame && Time == taikoFrame.Time && Actions.SequenceEqual(taikoFrame.Actions);
     }
 }
diff --git a/osu.Game.Tests/NonVisual/FramedReplayInputHandlerTest.cs b/osu.Game.Tests/NonVisual/FramedReplayInputHandlerTest.cs
index 541ad1e8bb..ffb21f124c 100644
--- a/osu.Game.Tests/NonVisual/FramedReplayInputHandlerTest.cs
+++ b/osu.Game.Tests/NonVisual/FramedReplayInputHandlerTest.cs
@@ -383,6 +383,9 @@ namespace osu.Game.Tests.NonVisual
                 IsImportant = isImportant;
                 FrameIndex = frameIndex;
             }
+
+            public override bool IsEquivalentTo(ReplayFrame other)
+                => other is TestReplayFrame testFrame && Time == testFrame.Time && IsImportant == testFrame.IsImportant && FrameIndex == testFrame.FrameIndex;
         }
 
         private class TestInputHandler : FramedReplayInputHandler<TestReplayFrame>
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecorder.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecorder.cs
index 4ad6bc66e3..8ada550174 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecorder.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecorder.cs
@@ -317,6 +317,9 @@ namespace osu.Game.Tests.Visual.Gameplay
                 Position = position;
                 Actions.AddRange(actions);
             }
+
+            public override bool IsEquivalentTo(ReplayFrame other)
+                => other is TestReplayFrame testFrame && Time == testFrame.Time && Position == testFrame.Position && Actions.SequenceEqual(testFrame.Actions);
         }
 
         public enum TestAction
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectatorPlayback.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectatorPlayback.cs
index dd5bbf70b4..062d73abbf 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectatorPlayback.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectatorPlayback.cs
@@ -353,6 +353,9 @@ namespace osu.Game.Tests.Visual.Gameplay
 
                 return new LegacyReplayFrame(Time, Position.X, Position.Y, state);
             }
+
+            public override bool IsEquivalentTo(ReplayFrame other)
+                => other is TestReplayFrame testFrame && Time == testFrame.Time && Position == testFrame.Position && Actions.SequenceEqual(testFrame.Actions);
         }
 
         public enum TestAction
diff --git a/osu.Game/Online/Spectator/SpectatorClient.cs b/osu.Game/Online/Spectator/SpectatorClient.cs
index dd0e03463c..7f09fbdc9e 100644
--- a/osu.Game/Online/Spectator/SpectatorClient.cs
+++ b/osu.Game/Online/Spectator/SpectatorClient.cs
@@ -247,12 +247,10 @@ namespace osu.Game.Online.Spectator
 
                 var convertedFrame = convertible.ToLegacy(currentBeatmap);
 
-                // only keep the last recorded frame for a given timestamp.
                 // this reduces redundancy of frames in the resulting replay.
-                //
-                // this is also done at `ReplayRecorded`, but needs to be done here as well
+                // it is also done at `ReplayRecorder`, but needs to be done here as well
                 // due to the flow being handled differently.
-                if (pendingFrames.LastOrDefault()?.Time == convertedFrame.Time)
+                if (pendingFrames.LastOrDefault()?.IsEquivalentTo(convertedFrame) == true)
                     pendingFrames[^1] = convertedFrame;
                 else
                     pendingFrames.Add(convertedFrame);
diff --git a/osu.Game/Replays/Legacy/LegacyReplayFrame.cs b/osu.Game/Replays/Legacy/LegacyReplayFrame.cs
index b48fc44963..bfc8ad5df8 100644
--- a/osu.Game/Replays/Legacy/LegacyReplayFrame.cs
+++ b/osu.Game/Replays/Legacy/LegacyReplayFrame.cs
@@ -64,5 +64,12 @@ namespace osu.Game.Replays.Legacy
         {
             return $"{Time}\t({MouseX},{MouseY})\t{MouseLeft}\t{MouseRight}\t{MouseLeft1}\t{MouseRight1}\t{MouseLeft2}\t{MouseRight2}\t{ButtonState}";
         }
+
+        public override bool IsEquivalentTo(ReplayFrame other)
+            => other is LegacyReplayFrame legacyFrame
+               && Time == legacyFrame.Time
+               && MouseX == legacyFrame.MouseX
+               && MouseY == legacyFrame.MouseY
+               && ButtonState == legacyFrame.ButtonState;
     }
 }
diff --git a/osu.Game/Rulesets/Replays/ReplayFrame.cs b/osu.Game/Rulesets/Replays/ReplayFrame.cs
index 433be6e4b7..269de228b1 100644
--- a/osu.Game/Rulesets/Replays/ReplayFrame.cs
+++ b/osu.Game/Rulesets/Replays/ReplayFrame.cs
@@ -30,5 +30,10 @@ namespace osu.Game.Rulesets.Replays
         {
             Time = time;
         }
+
+        /// <summary>
+        /// Whether this frame is equivalent to <paramref name="other"/> with respect to replay recording.
+        /// </summary>
+        public virtual bool IsEquivalentTo(ReplayFrame other) => Time == other.Time;
     }
 }
diff --git a/osu.Game/Rulesets/UI/ReplayRecorder.cs b/osu.Game/Rulesets/UI/ReplayRecorder.cs
index c2187b0634..8829c15a21 100644
--- a/osu.Game/Rulesets/UI/ReplayRecorder.cs
+++ b/osu.Game/Rulesets/UI/ReplayRecorder.cs
@@ -86,9 +86,8 @@ namespace osu.Game.Rulesets.UI
 
             if (frame != null)
             {
-                // only keep the last recorded frame for a given timestamp.
                 // this reduces redundancy of frames in the resulting replay.
-                if (last?.Time == frame.Time)
+                if (last?.IsEquivalentTo(frame) == true)
                     target.Replay.Frames[^1] = frame;
                 else
                     target.Replay.Frames.Add(frame);