diff --git a/osu.Game.Tests/WaveformTestBeatmap.cs b/osu.Game.Tests/WaveformTestBeatmap.cs
index 9c85fa0c9c..ab7bf7fb73 100644
--- a/osu.Game.Tests/WaveformTestBeatmap.cs
+++ b/osu.Game.Tests/WaveformTestBeatmap.cs
@@ -59,6 +59,13 @@ namespace osu.Game.Tests
 
         protected override Track GetBeatmapTrack() => trackStore.Get(firstAudioFile);
 
+        public override bool TryTransferTrack(WorkingBeatmap target)
+        {
+            // Our track comes from a local track store that's disposed on finalizer,
+            // therefore it's unsafe to transfer it to another working beatmap.
+            return false;
+        }
+
         private string firstAudioFile
         {
             get
diff --git a/osu.Game/Beatmaps/WorkingBeatmap.cs b/osu.Game/Beatmaps/WorkingBeatmap.cs
index bb64ec796c..09072ec897 100644
--- a/osu.Game/Beatmaps/WorkingBeatmap.cs
+++ b/osu.Game/Beatmaps/WorkingBeatmap.cs
@@ -126,11 +126,19 @@ namespace osu.Game.Beatmaps
         }
 
         /// <summary>
-        /// Transfer a valid audio track into this working beatmap. Used as an optimisation to avoid reload / track swap
-        /// across difficulties in the same beatmap set.
+        /// Attempts to transfer the audio track to a target working beatmap, if valid for transferring.
+        /// Used as an optimisation to avoid reload / track swap across difficulties in the same beatmap set.
         /// </summary>
-        /// <param name="track">The track to transfer.</param>
-        public void TransferTrack([NotNull] Track track) => this.track = track ?? throw new ArgumentNullException(nameof(track));
+        /// <param name="target">The target working beatmap to transfer this track to.</param>
+        /// <returns>Whether the track has been transferred to the <paramref name="target"/>.</returns>
+        public virtual bool TryTransferTrack([NotNull] WorkingBeatmap target)
+        {
+            if (BeatmapInfo?.AudioEquals(target.BeatmapInfo) != true || Track.IsDummyDevice)
+                return false;
+
+            target.track = Track;
+            return true;
+        }
 
         /// <summary>
         /// Get the loaded audio track instance. <see cref="LoadTrack"/> must have first been called.
diff --git a/osu.Game/Graphics/UserInterface/ShearedButton.cs b/osu.Game/Graphics/UserInterface/ShearedButton.cs
index dea44e6d99..66c6eedd0c 100644
--- a/osu.Game/Graphics/UserInterface/ShearedButton.cs
+++ b/osu.Game/Graphics/UserInterface/ShearedButton.cs
@@ -5,6 +5,7 @@
 using osu.Framework.Allocation;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Colour;
+using osu.Framework.Graphics.Containers;
 using osu.Framework.Graphics.Shapes;
 using osu.Framework.Input.Events;
 using osu.Framework.Localisation;
@@ -68,6 +69,7 @@ namespace osu.Game.Graphics.UserInterface
         private Colour4? lighterColour;
         private Colour4? textColour;
 
+        private readonly Container backgroundLayer;
         private readonly Box flashLayer;
 
         /// <summary>
@@ -85,24 +87,35 @@ namespace osu.Game.Graphics.UserInterface
             Height = 50;
             Padding = new MarginPadding { Horizontal = shear * 50 };
 
-            Content.CornerRadius = 7;
+            const float corner_radius = 7;
+
+            Content.CornerRadius = corner_radius;
             Content.Shear = new Vector2(shear, 0);
             Content.Masking = true;
-            Content.BorderThickness = 2;
             Content.Anchor = Content.Origin = Anchor.Centre;
 
             Children = new Drawable[]
             {
-                background = new Box
+                backgroundLayer = new Container
                 {
-                    RelativeSizeAxes = Axes.Both
-                },
-                text = new OsuSpriteText
-                {
-                    Anchor = Anchor.Centre,
-                    Origin = Anchor.Centre,
-                    Font = OsuFont.TorusAlternate.With(size: 17),
-                    Shear = new Vector2(-shear, 0)
+                    RelativeSizeAxes = Axes.Both,
+                    CornerRadius = corner_radius,
+                    Masking = true,
+                    BorderThickness = 2,
+                    Children = new Drawable[]
+                    {
+                        background = new Box
+                        {
+                            RelativeSizeAxes = Axes.Both
+                        },
+                        text = new OsuSpriteText
+                        {
+                            Anchor = Anchor.Centre,
+                            Origin = Anchor.Centre,
+                            Font = OsuFont.TorusAlternate.With(size: 17),
+                            Shear = new Vector2(-shear, 0)
+                        },
+                    }
                 },
                 flashLayer = new Box
                 {
@@ -186,7 +199,7 @@ namespace osu.Game.Graphics.UserInterface
             }
 
             background.FadeColour(colourDark, 150, Easing.OutQuint);
-            Content.TransformTo(nameof(BorderColour), ColourInfo.GradientVertical(colourDark, colourLight), 150, Easing.OutQuint);
+            backgroundLayer.TransformTo(nameof(BorderColour), ColourInfo.GradientVertical(colourDark, colourLight), 150, Easing.OutQuint);
 
             if (!Enabled.Value)
                 colourText = colourText.Opacity(0.6f);
diff --git a/osu.Game/Overlays/MusicController.cs b/osu.Game/Overlays/MusicController.cs
index 65b06eb864..aa09ff6b97 100644
--- a/osu.Game/Overlays/MusicController.cs
+++ b/osu.Game/Overlays/MusicController.cs
@@ -267,7 +267,7 @@ namespace osu.Game.Overlays
 
             TrackChangeDirection direction = TrackChangeDirection.None;
 
-            bool audioEquals = newWorking?.BeatmapInfo?.AudioEquals(current?.BeatmapInfo) ?? false;
+            bool audioEquals = newWorking?.BeatmapInfo?.AudioEquals(current?.BeatmapInfo) == true;
 
             if (current != null)
             {
@@ -290,15 +290,8 @@ namespace osu.Game.Overlays
 
             current = newWorking;
 
-            if (!audioEquals || CurrentTrack.IsDummyDevice)
-            {
+            if (lastWorking == null || !lastWorking.TryTransferTrack(current))
                 changeTrack();
-            }
-            else
-            {
-                // transfer still valid track to new working beatmap
-                current.TransferTrack(lastWorking.Track);
-            }
 
             TrackChanged?.Invoke(current, direction);
 
diff --git a/osu.Game/Tests/Visual/OsuTestScene.cs b/osu.Game/Tests/Visual/OsuTestScene.cs
index f2d280417e..1582bdfca4 100644
--- a/osu.Game/Tests/Visual/OsuTestScene.cs
+++ b/osu.Game/Tests/Visual/OsuTestScene.cs
@@ -373,6 +373,13 @@ namespace osu.Game.Tests.Visual
 
             protected override Track GetBeatmapTrack() => track;
 
+            public override bool TryTransferTrack(WorkingBeatmap target)
+            {
+                // Our track comes from a local track store that's disposed on finalizer,
+                // therefore it's unsafe to transfer it to another working beatmap.
+                return false;
+            }
+
             public class TrackVirtualStore : AudioCollectionManager<Track>, ITrackStore
             {
                 private readonly IFrameBasedClock referenceClock;