diff --git a/osu.Game.Rulesets.Catch/Mods/CatchModDifficultyAdjust.cs b/osu.Game.Rulesets.Catch/Mods/CatchModDifficultyAdjust.cs
index 4c0f5d510e..8377b3786a 100644
--- a/osu.Game.Rulesets.Catch/Mods/CatchModDifficultyAdjust.cs
+++ b/osu.Game.Rulesets.Catch/Mods/CatchModDifficultyAdjust.cs
@@ -34,8 +34,8 @@ namespace osu.Game.Rulesets.Catch.Mods
         {
             base.TransferSettings(difficulty);
 
-            CircleSize.Value = CircleSize.Default = difficulty.CircleSize;
-            ApproachRate.Value = ApproachRate.Default = difficulty.ApproachRate;
+            TransferSetting(CircleSize, difficulty.CircleSize);
+            TransferSetting(ApproachRate, difficulty.ApproachRate);
         }
 
         protected override void ApplySettings(BeatmapDifficulty difficulty)
diff --git a/osu.Game.Rulesets.Osu/Mods/OsuModDifficultyAdjust.cs b/osu.Game.Rulesets.Osu/Mods/OsuModDifficultyAdjust.cs
index 0514e2ab34..7eee71be81 100644
--- a/osu.Game.Rulesets.Osu/Mods/OsuModDifficultyAdjust.cs
+++ b/osu.Game.Rulesets.Osu/Mods/OsuModDifficultyAdjust.cs
@@ -34,8 +34,8 @@ namespace osu.Game.Rulesets.Osu.Mods
         {
             base.TransferSettings(difficulty);
 
-            CircleSize.Value = CircleSize.Default = difficulty.CircleSize;
-            ApproachRate.Value = ApproachRate.Default = difficulty.ApproachRate;
+            TransferSetting(CircleSize, difficulty.CircleSize);
+            TransferSetting(ApproachRate, difficulty.ApproachRate);
         }
 
         protected override void ApplySettings(BeatmapDifficulty difficulty)
diff --git a/osu.Game/Rulesets/Mods/ModDifficultyAdjust.cs b/osu.Game/Rulesets/Mods/ModDifficultyAdjust.cs
index 7b8d4bb3df..1c26e5a720 100644
--- a/osu.Game/Rulesets/Mods/ModDifficultyAdjust.cs
+++ b/osu.Game/Rulesets/Mods/ModDifficultyAdjust.cs
@@ -5,6 +5,7 @@ using osu.Game.Beatmaps;
 using osu.Framework.Bindables;
 using osu.Framework.Graphics.Sprites;
 using System;
+using System.Collections.Generic;
 using osu.Game.Configuration;
 
 namespace osu.Game.Rulesets.Mods
@@ -51,8 +52,8 @@ namespace osu.Game.Rulesets.Mods
         {
             if (this.difficulty == null || this.difficulty.ID != difficulty.ID)
             {
-                this.difficulty = difficulty;
                 TransferSettings(difficulty);
+                this.difficulty = difficulty;
             }
         }
 
@@ -64,8 +65,31 @@ namespace osu.Game.Rulesets.Mods
         /// <param name="difficulty">The beatmap's initial values.</param>
         protected virtual void TransferSettings(BeatmapDifficulty difficulty)
         {
-            DrainRate.Value = DrainRate.Default = difficulty.DrainRate;
-            OverallDifficulty.Value = OverallDifficulty.Default = difficulty.OverallDifficulty;
+            TransferSetting(DrainRate, difficulty.DrainRate);
+            TransferSetting(OverallDifficulty, difficulty.OverallDifficulty);
+        }
+
+        private readonly Dictionary<IBindable, bool> userChangedSettings = new Dictionary<IBindable, bool>();
+
+        /// <summary>
+        /// Transfer a setting from <see cref="BeatmapDifficulty"/> to a configuration bindable.
+        /// Only performs the transfer if the user it not currently overriding..
+        /// </summary>
+        protected void TransferSetting<T>(BindableNumber<T> bindable, T beatmapDefault)
+            where T : struct, IComparable<T>, IConvertible, IEquatable<T>
+        {
+            bindable.UnbindEvents();
+
+            userChangedSettings.TryAdd(bindable, false);
+
+            // users generally choose a difficulty setting and want it to stick across multiple beatmap changes.
+            // we only want to value transfer if the user hasn't changed the value previously.
+            if (!userChangedSettings[bindable])
+            {
+                bindable.Value = bindable.Default = beatmapDefault;
+            }
+
+            bindable.ValueChanged += _ => userChangedSettings[bindable] = !bindable.IsDefault;
         }
 
         /// <summary>