diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneAutoplay.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneAutoplay.cs
index bc7cf8eee2..fdc3916c47 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneAutoplay.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneAutoplay.cs
@@ -11,7 +11,6 @@ using osu.Game.Rulesets.Scoring;
 using osu.Game.Screens.Play;
 using osu.Game.Screens.Play.Break;
 using osu.Game.Screens.Ranking;
-using osuTK.Input;
 
 namespace osu.Game.Tests.Visual.Gameplay
 {
@@ -36,18 +35,6 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddStep("rewind", () => Player.GameplayClockContainer.Seek(-80000));
             AddUntilStep("key counter reset", () => Player.HUDOverlay.KeyCounter.Children.All(kc => kc.CountPresses == 0));
 
-            double? time = null;
-
-            AddStep("store time", () => time = Player.GameplayClockContainer.GameplayClock.CurrentTime);
-
-            // test seek via keyboard
-            AddStep("seek with right arrow key", () => InputManager.Key(Key.Right));
-            AddAssert("time seeked forward", () => Player.GameplayClockContainer.GameplayClock.CurrentTime > time + 2000);
-
-            AddStep("store time", () => time = Player.GameplayClockContainer.GameplayClock.CurrentTime);
-            AddStep("seek with left arrow key", () => InputManager.Key(Key.Left));
-            AddAssert("time seeked backward", () => Player.GameplayClockContainer.GameplayClock.CurrentTime < time);
-
             seekToBreak(0);
             seekToBreak(1);
 
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneReplayPlayer.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneReplayPlayer.cs
new file mode 100644
index 0000000000..fcd65eaff3
--- /dev/null
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneReplayPlayer.cs
@@ -0,0 +1,87 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using NUnit.Framework;
+using osu.Game.Rulesets;
+using osu.Game.Rulesets.Osu;
+using osuTK.Input;
+
+namespace osu.Game.Tests.Visual.Gameplay
+{
+    public class TestSceneReplayPlayer : RateAdjustedBeatmapTestScene
+    {
+        protected TestReplayPlayer Player;
+
+        public override void SetUpSteps()
+        {
+            base.SetUpSteps();
+
+            AddStep("Initialise player", () => Player = CreatePlayer(new OsuRuleset()));
+            AddStep("Load player", () => LoadScreen(Player));
+            AddUntilStep("player loaded", () => Player.IsLoaded);
+        }
+
+        [Test]
+        public void TestPause()
+        {
+            double? lastTime = null;
+
+            AddUntilStep("wait for first hit", () => Player.ScoreProcessor.TotalScore.Value > 0);
+
+            AddStep("Pause playback", () => InputManager.Key(Key.Space));
+
+            AddUntilStep("Time stopped progressing", () =>
+            {
+                double current = Player.GameplayClockContainer.CurrentTime;
+                bool changed = lastTime != current;
+                lastTime = current;
+
+                return !changed;
+            });
+
+            AddWaitStep("wait some", 10);
+
+            AddAssert("Time still stopped", () => lastTime == Player.GameplayClockContainer.CurrentTime);
+        }
+
+        [Test]
+        public void TestSeekBackwards()
+        {
+            double? lastTime = null;
+
+            AddUntilStep("wait for first hit", () => Player.ScoreProcessor.TotalScore.Value > 0);
+
+            AddStep("Seek backwards", () =>
+            {
+                lastTime = Player.GameplayClockContainer.CurrentTime;
+                InputManager.Key(Key.Left);
+            });
+
+            AddAssert("Jumped backwards", () => Player.GameplayClockContainer.CurrentTime - lastTime < 0);
+        }
+
+        [Test]
+        public void TestSeekForwards()
+        {
+            double? lastTime = null;
+
+            AddUntilStep("wait for first hit", () => Player.ScoreProcessor.TotalScore.Value > 0);
+
+            AddStep("Seek forwards", () =>
+            {
+                lastTime = Player.GameplayClockContainer.CurrentTime;
+                InputManager.Key(Key.Right);
+            });
+
+            AddAssert("Jumped forwards", () => Player.GameplayClockContainer.CurrentTime - lastTime > 500);
+        }
+
+        protected TestReplayPlayer CreatePlayer(Ruleset ruleset)
+        {
+            Beatmap.Value = CreateWorkingBeatmap(ruleset.RulesetInfo);
+            SelectedMods.Value = new[] { ruleset.GetAutoplayMod() };
+
+            return new TestReplayPlayer(false);
+        }
+    }
+}
diff --git a/osu.Game/Input/Bindings/GlobalActionContainer.cs b/osu.Game/Input/Bindings/GlobalActionContainer.cs
index 2482be90ee..d3cc90ef99 100644
--- a/osu.Game/Input/Bindings/GlobalActionContainer.cs
+++ b/osu.Game/Input/Bindings/GlobalActionContainer.cs
@@ -87,6 +87,8 @@ namespace osu.Game.Input.Bindings
             new KeyBinding(new[] { InputKey.Shift, InputKey.Tab }, GlobalAction.ToggleInGameInterface),
             new KeyBinding(InputKey.MouseMiddle, GlobalAction.PauseGameplay),
             new KeyBinding(InputKey.Space, GlobalAction.TogglePauseReplay),
+            new KeyBinding(InputKey.Left, GlobalAction.SeekReplayBackward),
+            new KeyBinding(InputKey.Right, GlobalAction.SeekReplayForward),
             new KeyBinding(InputKey.Control, GlobalAction.HoldForHUD),
         };
 
@@ -272,5 +274,11 @@ namespace osu.Game.Input.Bindings
 
         [Description("Next volume meter")]
         NextVolumeMeter,
+
+        [Description("Seek replay forward")]
+        SeekReplayForward,
+
+        [Description("Seek replay backward")]
+        SeekReplayBackward,
     }
 }
diff --git a/osu.Game/Screens/Play/ReplayPlayer.cs b/osu.Game/Screens/Play/ReplayPlayer.cs
index f70c05c2ff..adbb5a53f6 100644
--- a/osu.Game/Screens/Play/ReplayPlayer.cs
+++ b/osu.Game/Screens/Play/ReplayPlayer.cs
@@ -3,11 +3,15 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using osu.Framework.Input.Bindings;
+using osu.Framework.Threading;
 using osu.Game.Beatmaps;
+using osu.Game.Extensions;
 using osu.Game.Input.Bindings;
 using osu.Game.Rulesets.Mods;
+using osu.Game.Rulesets.Objects;
 using osu.Game.Scoring;
 using osu.Game.Screens.Ranking;
 
@@ -43,10 +47,24 @@ namespace osu.Game.Screens.Play
 
         protected override ResultsScreen CreateResults(ScoreInfo score) => new SoloResultsScreen(score, false);
 
+        private ScheduledDelegate keyboardSeekDelegate;
+
         public bool OnPressed(GlobalAction action)
         {
+            const double keyboard_seek_amount = 5000;
+
             switch (action)
             {
+                case GlobalAction.SeekReplayBackward:
+                    keyboardSeekDelegate?.Cancel();
+                    keyboardSeekDelegate = this.BeginKeyRepeat(Scheduler, () => keyboardSeek(-1));
+                    return true;
+
+                case GlobalAction.SeekReplayForward:
+                    keyboardSeekDelegate?.Cancel();
+                    keyboardSeekDelegate = this.BeginKeyRepeat(Scheduler, () => keyboardSeek(1));
+                    return true;
+
                 case GlobalAction.TogglePauseReplay:
                     if (GameplayClockContainer.IsPaused.Value)
                         GameplayClockContainer.Start();
@@ -56,10 +74,24 @@ namespace osu.Game.Screens.Play
             }
 
             return false;
+
+            void keyboardSeek(int direction)
+            {
+                double target = Math.Clamp(GameplayClockContainer.CurrentTime + direction * keyboard_seek_amount, 0, GameplayBeatmap.HitObjects.Last().GetEndTime());
+
+                Seek(target);
+            }
         }
 
         public void OnReleased(GlobalAction action)
         {
+            switch (action)
+            {
+                case GlobalAction.SeekReplayBackward:
+                case GlobalAction.SeekReplayForward:
+                    keyboardSeekDelegate?.Cancel();
+                    break;
+            }
         }
     }
 }
diff --git a/osu.Game/Screens/Play/SongProgressBar.cs b/osu.Game/Screens/Play/SongProgressBar.cs
index 939b5fad1f..5052b32335 100644
--- a/osu.Game/Screens/Play/SongProgressBar.cs
+++ b/osu.Game/Screens/Play/SongProgressBar.cs
@@ -57,8 +57,6 @@ namespace osu.Game.Screens.Play
             set => CurrentNumber.Value = value;
         }
 
-        protected override bool AllowKeyboardInputWhenNotHovered => true;
-
         public SongProgressBar(float barHeight, float handleBarHeight, Vector2 handleSize)
         {
             CurrentNumber.MinValue = 0;