diff --git a/osu.Game.Tests/Visual/Gameplay/TestCasePause.cs b/osu.Game.Tests/Visual/Gameplay/TestCasePause.cs
index a52e84ed62..b9c7fd14bd 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestCasePause.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestCasePause.cs
@@ -196,9 +196,10 @@ namespace osu.Game.Tests.Visual.Gameplay
 
             public bool PauseOverlayVisible => PauseOverlay.State == Visibility.Visible;
 
-            public PausePlayer()
+            protected override void LoadComplete()
             {
-                PauseOnFocusLost = false;
+                base.LoadComplete();
+                HUDOverlay.HoldToQuit.PauseOnFocusLost = false;
             }
         }
     }
diff --git a/osu.Game/Screens/Play/HUD/HoldForMenuButton.cs b/osu.Game/Screens/Play/HUD/HoldForMenuButton.cs
index c0ee5e6142..446df94aca 100644
--- a/osu.Game/Screens/Play/HUD/HoldForMenuButton.cs
+++ b/osu.Game/Screens/Play/HUD/HoldForMenuButton.cs
@@ -4,6 +4,7 @@
 using System;
 using System.Linq;
 using osu.Framework.Allocation;
+using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 using osu.Framework.Graphics.Shapes;
@@ -70,6 +71,11 @@ namespace osu.Game.Screens.Play.HUD
             return base.OnMouseMove(e);
         }
 
+        public bool PauseOnFocusLost
+        {
+            set => button.PauseOnFocusLost = value;
+        }
+
         protected override void Update()
         {
             base.Update();
@@ -93,8 +99,10 @@ namespace osu.Game.Screens.Play.HUD
             public Action HoverGained;
             public Action HoverLost;
 
+            private readonly IBindable<bool> gameActive = new Bindable<bool>(true);
+
             [BackgroundDependencyLoader]
-            private void load(OsuColour colours)
+            private void load(OsuColour colours, Framework.Game game)
             {
                 Size = new Vector2(60);
 
@@ -135,6 +143,14 @@ namespace osu.Game.Screens.Play.HUD
                 };
 
                 bind();
+
+                gameActive.BindTo(game.IsActive);
+            }
+
+            protected override void LoadComplete()
+            {
+                base.LoadComplete();
+                gameActive.BindValueChanged(_ => updateActive(), true);
             }
 
             private void bind()
@@ -184,6 +200,31 @@ namespace osu.Game.Screens.Play.HUD
                 base.OnHoverLost(e);
             }
 
+            private bool pauseOnFocusLost = true;
+
+            public bool PauseOnFocusLost
+            {
+                set
+                {
+                    if (pauseOnFocusLost == value)
+                        return;
+
+                    pauseOnFocusLost = value;
+                    if (IsLoaded)
+                        updateActive();
+                }
+            }
+
+            private void updateActive()
+            {
+                if (!pauseOnFocusLost) return;
+
+                if (gameActive.Value)
+                    AbortConfirm();
+                else
+                    BeginConfirm();
+            }
+
             public bool OnPressed(GlobalAction action)
             {
                 switch (action)
diff --git a/osu.Game/Screens/Play/Player.cs b/osu.Game/Screens/Play/Player.cs
index 81fa348d72..fd9ddec314 100644
--- a/osu.Game/Screens/Play/Player.cs
+++ b/osu.Game/Screens/Play/Player.cs
@@ -44,8 +44,6 @@ namespace osu.Game.Screens.Play
 
         public bool HasFailed { get; private set; }
 
-        public bool PauseOnFocusLost { get; set; } = true;
-
         private Bindable<bool> mouseWheelDisabled;
 
         private readonly Bindable<bool> storyboardReplacesBackground = new Bindable<bool>();
@@ -388,15 +386,6 @@ namespace osu.Game.Screens.Play
             // already resuming
             && !IsResuming;
 
-        protected override void Update()
-        {
-            base.Update();
-
-            // eagerly pause when we lose window focus (if we are locally playing).
-            if (PauseOnFocusLost && !Game.IsActive.Value)
-                Pause();
-        }
-
         public void Pause()
         {
             if (!canPause) return;