diff --git a/osu.Desktop.VisualTests/Tests/TestCasePauseOverlay.cs b/osu.Desktop.VisualTests/Tests/TestCasePauseOverlay.cs
index 93dfc6cd72..c1f81810ab 100644
--- a/osu.Desktop.VisualTests/Tests/TestCasePauseOverlay.cs
+++ b/osu.Desktop.VisualTests/Tests/TestCasePauseOverlay.cs
@@ -38,7 +38,7 @@ namespace osu.Desktop.VisualTests.Tests
                 Width = 100,
                 Height = 50,
                 Colour = Color4.Black,
-                Action = (() => pauseOverlay.Pause())
+                Action = (() => pauseOverlay.Pause()),
             });
 
             pauseOverlay.OnPause += (() => Logger.Log(@"Pause"));
diff --git a/osu.Game/Overlays/Pause/PauseButton.cs b/osu.Game/Overlays/Pause/PauseButton.cs
index c7f35dcaaa..ddf2d49c97 100644
--- a/osu.Game/Overlays/Pause/PauseButton.cs
+++ b/osu.Game/Overlays/Pause/PauseButton.cs
@@ -2,8 +2,6 @@
 using OpenTK.Graphics;
 using osu.Framework.Graphics.UserInterface;
 using osu.Framework.Graphics.Transformations;
-using osu.Framework.Allocation;
-using osu.Framework.Audio;
 using osu.Framework.Audio.Sample;
 
 namespace osu.Game.Overlays.Pause
@@ -14,16 +12,8 @@ namespace osu.Game.Overlays.Pause
         private float width = 300;
         private float expandedWidth = 350;
 
-        private AudioSample sampleClick;
-        private AudioSample sampleHover;
-
-        [BackgroundDependencyLoader]
-        private void load(AudioManager audio)
-        {
-            // Placeholder till the actual samples are added to osu-resources
-            sampleClick = audio.Sample.Get(@"Menu/menuhit");
-            sampleHover = audio.Sample.Get(@"Menu/menuclick");
-        }
+        public AudioSample sampleClick;
+        public AudioSample sampleHover;
 
         protected override bool OnMouseDown(Framework.Input.InputState state, Framework.Graphics.MouseDownEventArgs args)
         {
diff --git a/osu.Game/Overlays/Pause/PauseOverlay.cs b/osu.Game/Overlays/Pause/PauseOverlay.cs
index e64e9a85b8..ecdf48c4ff 100644
--- a/osu.Game/Overlays/Pause/PauseOverlay.cs
+++ b/osu.Game/Overlays/Pause/PauseOverlay.cs
@@ -3,6 +3,7 @@ using OpenTK;
 using OpenTK.Input;
 using OpenTK.Graphics;
 using osu.Game.Graphics;
+using osu.Framework.Audio;
 using osu.Framework.Input;
 using osu.Framework.Allocation;
 using osu.Framework.Graphics;
@@ -15,17 +16,28 @@ namespace osu.Game.Overlays.Pause
 {
     public class PauseOverlay : OverlayContainer
     {
-        private bool paused = false;
-        private int fadeDuration = 100;
-
         public event Action OnPause;
         public event Action OnResume;
         public event Action OnRetry;
         public event Action OnQuit;
 
+        public bool isPaused = false;
+
+        private int fadeDuration = 100;
+        private double pauseDisableTime = 1000;
+        private double lastActionTime = -1000;
+
+        private PauseButton resumeButton;
+        private PauseButton retryButton;
+        private PauseButton quitButton;
+
         [BackgroundDependencyLoader]
-        private void load(OsuColour colours)
-        {
+        private void load(AudioManager audio, OsuColour colours)
+        {
+            var sampleHover = audio.Sample.Get(@"Menu/menuclick");
+            var sampleBack = audio.Sample.Get(@"Menu/menuback");
+            var samplePlayClick = audio.Sample.Get(@"Menu/menu-play-click");
+
             Children = new Drawable[]
             {
                 new Box
@@ -34,42 +46,51 @@ namespace osu.Game.Overlays.Pause
                     Colour = Color4.Black,
                     Alpha = 0.6f,
                 },
-                new PauseButton
-                {
+                resumeButton = new PauseButton
+                {
                     Text = @"Resume",
                     Origin = Anchor.Centre,
                     Anchor = Anchor.Centre,
                     Position = new Vector2(0, -200),
-                    Action = Resume
+                    Action = Resume,
                 },
-                new PauseButton
+                retryButton = new PauseButton
                 {
                     Text = @"Retry",
                     Origin = Anchor.Centre,
                     Anchor = Anchor.Centre,
-                    Action = Retry
+                    Action = Retry,
                 },
-                new PauseButton
+                quitButton = new PauseButton
                 {
                     Text = @"Quit",
                     Origin = Anchor.Centre,
                     Anchor = Anchor.Centre,
                     Position = new Vector2(0, 200),
-                    Action = Quit
-                }
-            };
+                    Action = Quit,
+                },
+            };
+
+            resumeButton.sampleHover = sampleHover;
+            resumeButton.sampleClick = sampleBack;
+
+            retryButton.sampleHover = sampleHover;
+            retryButton.sampleClick = samplePlayClick;
+
+            quitButton.sampleHover = sampleHover;
+            quitButton.sampleClick = sampleBack;
         }
 
         protected override void PopIn()
         {
-            this.FadeTo(1, fadeDuration, EasingTypes.In);
-            paused = true;
+            FadeTo(1, fadeDuration, EasingTypes.In);
+            isPaused = true;
         }
 
         protected override void PopOut()
         {
-            this.FadeTo(0, fadeDuration, EasingTypes.In);
-            paused = false;
+            FadeTo(0, fadeDuration, EasingTypes.In);
+            isPaused = false;
         }
 
         protected override bool OnKeyDown(InputState state, KeyDownEventArgs args)
@@ -84,21 +105,31 @@ namespace osu.Game.Overlays.Pause
         }
 
         public void Pause()
-        {
-            Show();
-            OnPause?.Invoke();
+        {
+            // Only allow pausing once a second
+            if (Time.Current >= (lastActionTime + pauseDisableTime))
+            {
+                lastActionTime = Time.Current;
+                Show();
+                OnPause?.Invoke();
+            }
+            else
+            {
+                isPaused = false;
+            }
         }
 
         public void Resume()
-        {
+        {
+            lastActionTime = Time.Current;
             Hide();
             OnResume?.Invoke();
         }
 
         public void TogglePaused()
-        {
-            ToggleVisibility();
-            (paused ? (Action)Pause : Resume)?.Invoke();
+        {
+            isPaused = !isPaused;
+            (isPaused ? (Action)Pause : Resume)?.Invoke();
         }
 
         private void Retry()