Drank some coffee and figured out the fix

osu.Game/Screens/Play/PauseOverlay.cs

@@ -28,7 +28,6 @@ namespace osu.Game.Screens.Play
 
         private SkinnableSound pauseLoop;
 
-        public void FlushPendingSkinChanges() => pauseLoop.FlushPendingSkinChanges();
         protected override Action BackAction => () => InternalButtons.First().TriggerClick();
 
         [BackgroundDependencyLoader]

osu.Game/Screens/Play/Player.cs

@@ -316,11 +316,6 @@ namespace osu.Game.Screens.Play
             // we may want to limit this in the future to disallow rulesets from outright replacing elements the user expects to be there.
             failAnimationLayer.Add(createOverlayComponents(Beatmap.Value));
 
-            rulesetSkinProvider.SourceChanged += () =>
-            {
-                PauseOverlay.FlushPendingSkinChanges();
-            };
-
             if (!DrawableRuleset.AllowGameplayOverlays)
             {
                 HUDOverlay.ShowHud.Value = false;

osu.Game/Skinning/PoolableSkinnableSample.cs

@@ -132,6 +132,8 @@ namespace osu.Game.Skinning
             if (Sample == null)
                 return;
 
+            FlushPendingSkinChanges();
+
             activeChannel = Sample.GetChannel();
             activeChannel.Looping = Looping;
             activeChannel.Play();

osu.Game/Skinning/SkinReloadableDrawable.cs

@@ -27,6 +27,15 @@ namespace osu.Game.Skinning
         /// </summary>
         protected ISkinSource CurrentSkin { get; private set; } = null!;
 
+        protected void FlushPendingSkinChanges()
+        {
+            if (pendingSkinChange == null)
+                return;
+
+            pendingSkinChange.RunTask();
+            pendingSkinChange = null;
+        }
+
         [BackgroundDependencyLoader]
         private void load(ISkinSource source)
         {
@@ -42,15 +51,6 @@ namespace osu.Game.Skinning
             pendingSkinChange = Scheduler.Add(skinChanged);
         }
 
-        public virtual void FlushPendingSkinChanges()
-        {
-            if (pendingSkinChange == null)
-                return;
-
-            pendingSkinChange.RunTask();
-            pendingSkinChange = null;
-        }
-
         protected override void LoadAsyncComplete()
         {
             base.LoadAsyncComplete();
@@ -61,6 +61,8 @@ namespace osu.Game.Skinning
         {
             SkinChanged(CurrentSkin);
             OnSkinChanged?.Invoke();
+
+            pendingSkinChange = null;
         }
 
         /// <summary>

osu.Game/Skinning/SkinnableSound.cs

@@ -110,13 +110,6 @@ namespace osu.Game.Skinning
             }
         }
 
-        public override void FlushPendingSkinChanges()
-        {
-            base.FlushPendingSkinChanges();
-
-            samplesContainer.ForEach(c => c.FlushPendingSkinChanges());
-        }
-
         /// <summary>
         /// Plays the samples.
         /// </summary>