diff --git a/osu.Game/Graphics/Containers/HoldToConfirmContainer.cs b/osu.Game/Graphics/Containers/HoldToConfirmContainer.cs
index c74245461d..078721ec77 100644
--- a/osu.Game/Graphics/Containers/HoldToConfirmContainer.cs
+++ b/osu.Game/Graphics/Containers/HoldToConfirmContainer.cs
@@ -12,6 +12,13 @@ namespace osu.Game.Graphics.Containers
 {
     public abstract class HoldToConfirmContainer : Container
     {
+        public const double DANGEROUS_HOLD_ACTIVATION_DELAY = 500;
+
+        /// <summary>
+        /// Whether the associated action is considered dangerous, warranting a longer hold.
+        /// </summary>
+        public bool IsDangerousAction { get; }
+
         public Action Action;
 
         private const int fadeout_delay = 200;
@@ -29,12 +36,9 @@ namespace osu.Game.Graphics.Containers
         protected virtual bool AllowMultipleFires => false;
 
         /// <summary>
-        /// Specify a custom activation delay, overriding the game-wide user setting.
+        /// The current activation delay for this control.
         /// </summary>
-        /// <remarks>
-        /// This should be used in special cases where we want to be extra sure the user knows what they are doing. An example is when changes would be lost.
-        /// </remarks>
-        protected virtual double? HoldActivationDelay => null;
+        protected IBindable<double> HoldActivationDelay => holdActivationDelay;
 
         public Bindable<double> Progress = new BindableDouble();
 
@@ -43,13 +47,25 @@ namespace osu.Game.Graphics.Containers
         [Resolved]
         private OsuConfigManager config { get; set; }
 
+        protected HoldToConfirmContainer(bool isDangerousAction = false)
+        {
+            IsDangerousAction = isDangerousAction;
+        }
+
         protected override void LoadComplete()
         {
             base.LoadComplete();
 
-            holdActivationDelay = HoldActivationDelay != null
-                ? new Bindable<double>(HoldActivationDelay.Value)
-                : config.GetBindable<double>(OsuSetting.UIHoldActivationDelay);
+            if (IsDangerousAction)
+            {
+                holdActivationDelay.Value = DANGEROUS_HOLD_ACTIVATION_DELAY;
+            }
+            else
+            {
+                holdActivationDelay = HoldActivationDelay != null
+                    ? new Bindable<double>(HoldActivationDelay.Value)
+                    : config.GetBindable<double>(OsuSetting.UIHoldActivationDelay);
+            }
         }
 
         protected void BeginConfirm()
diff --git a/osu.Game/Overlays/Dialog/PopupDialogDangerousButton.cs b/osu.Game/Overlays/Dialog/PopupDialogDangerousButton.cs
index 6c775f44f8..6239c5e409 100644
--- a/osu.Game/Overlays/Dialog/PopupDialogDangerousButton.cs
+++ b/osu.Game/Overlays/Dialog/PopupDialogDangerousButton.cs
@@ -12,8 +12,6 @@ namespace osu.Game.Overlays.Dialog
 {
     public class PopupDialogDangerousButton : PopupDialogButton
     {
-        public const double DANGEROUS_HOLD_ACTIVATION_DELAY = 500;
-
         private Box progressBox;
         private DangerousConfirmContainer confirmContainer;
 
@@ -44,7 +42,10 @@ namespace osu.Game.Overlays.Dialog
 
         private class DangerousConfirmContainer : HoldToConfirmContainer
         {
-            protected override double? HoldActivationDelay => DANGEROUS_HOLD_ACTIVATION_DELAY;
+            public DangerousConfirmContainer()
+                : base(isDangerousAction: true)
+            {
+            }
 
             protected override bool OnMouseDown(MouseDownEvent e)
             {
diff --git a/osu.Game/Screens/Play/HUD/HoldForMenuButton.cs b/osu.Game/Screens/Play/HUD/HoldForMenuButton.cs
index c6aa3fbe08..9d280a1737 100644
--- a/osu.Game/Screens/Play/HUD/HoldForMenuButton.cs
+++ b/osu.Game/Screens/Play/HUD/HoldForMenuButton.cs
@@ -13,12 +13,10 @@ using osu.Framework.Graphics.UserInterface;
 using osu.Framework.Input.Bindings;
 using osu.Framework.Input.Events;
 using osu.Framework.Utils;
-using osu.Game.Configuration;
 using osu.Game.Graphics;
 using osu.Game.Graphics.Containers;
 using osu.Game.Graphics.Sprites;
 using osu.Game.Input.Bindings;
-using osu.Game.Overlays.Dialog;
 using osuTK;
 
 namespace osu.Game.Screens.Play.HUD
@@ -29,20 +27,11 @@ namespace osu.Game.Screens.Play.HUD
 
         public readonly Bindable<bool> IsPaused = new Bindable<bool>();
 
-        private readonly Button button;
+        private Button button;
 
-        public Action Action
-        {
-            set => button.Action = value;
-        }
+        public Action Action { get; set; }
 
-        private readonly OsuSpriteText text;
-
-        [Resolved]
-        private OsuConfigManager config { get; set; }
-
-        [Resolved(canBeNull: true)]
-        private Player player { get; set; }
+        private OsuSpriteText text;
 
         private readonly Bindable<double> activationDelay = new Bindable<double>();
 
@@ -51,6 +40,11 @@ namespace osu.Game.Screens.Play.HUD
             Direction = FillDirection.Horizontal;
             Spacing = new Vector2(20, 0);
             Margin = new MarginPadding(10);
+        }
+
+        [BackgroundDependencyLoader(true)]
+        private void load(Player player)
+        {
             Children = new Drawable[]
             {
                 text = new OsuSpriteText
@@ -59,11 +53,12 @@ namespace osu.Game.Screens.Play.HUD
                     Anchor = Anchor.CentreLeft,
                     Origin = Anchor.CentreLeft
                 },
-                button = new Button
+                button = new Button(player?.Configuration.AllowRestart == false)
                 {
                     HoverGained = () => text.FadeIn(500, Easing.OutQuint),
                     HoverLost = () => text.FadeOut(500, Easing.OutQuint),
-                    IsPaused = { BindTarget = IsPaused }
+                    IsPaused = { BindTarget = IsPaused },
+                    Action = () => Action(),
                 }
             };
             AutoSizeAxes = Axes.Both;
@@ -71,13 +66,6 @@ namespace osu.Game.Screens.Play.HUD
 
         protected override void LoadComplete()
         {
-            if (player?.Configuration.AllowRestart == false)
-            {
-                activationDelay.Value = PopupDialogDangerousButton.DANGEROUS_HOLD_ACTIVATION_DELAY;
-            }
-            else
-                config.BindWith(OsuSetting.UIHoldActivationDelay, activationDelay);
-
             activationDelay.BindValueChanged(v =>
             {
                 text.Text = v.NewValue > 0
@@ -125,10 +113,10 @@ namespace osu.Game.Screens.Play.HUD
             public Action HoverGained;
             public Action HoverLost;
 
-            [Resolved(canBeNull: true)]
-            private Player player { get; set; }
-
-            protected override double? HoldActivationDelay => player?.Configuration.AllowRestart == false ? PopupDialogDangerousButton.DANGEROUS_HOLD_ACTIVATION_DELAY : (double?)null;
+            public Button(bool isDangerousAction)
+                : base(isDangerousAction)
+            {
+            }
 
             [BackgroundDependencyLoader]
             private void load(OsuColour colours)