diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs
index be054495cc..a7020b6534 100644
--- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs
+++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs
@@ -16,6 +16,7 @@ using osu.Framework.Platform;
 using osu.Framework.Screens;
 using osu.Game.Beatmaps;
 using osu.Game.Configuration;
+using osu.Game.Overlays;
 using osu.Game.Rulesets;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Osu;
@@ -34,6 +35,8 @@ namespace osu.Game.Tests.Visual.SongSelect
 
         private RulesetStore rulesets;
 
+        private MusicController music;
+
         private WorkingBeatmap defaultBeatmap;
 
         public override IReadOnlyList<Type> RequiredTypes => new[]
@@ -79,6 +82,11 @@ namespace osu.Game.Tests.Visual.SongSelect
             Dependencies.Cache(rulesets = new RulesetStore(ContextFactory));
             Dependencies.Cache(manager = new BeatmapManager(LocalStorage, ContextFactory, rulesets, null, audio, host, defaultBeatmap = Beatmap.Default));
 
+            Dependencies.Cache(music = new MusicController());
+
+            // required to get bindables attached
+            Add(music);
+
             Beatmap.SetDefault();
 
             Dependencies.Cache(config = new OsuConfigManager(LocalStorage));
@@ -93,6 +101,57 @@ namespace osu.Game.Tests.Visual.SongSelect
             manager?.Delete(manager.GetAllUsableBeatmapSets());
         });
 
+        [Test]
+        public void TestAudioResuming()
+        {
+            createSongSelect();
+
+            addRulesetImportStep(0);
+            addRulesetImportStep(0);
+
+            checkMusicPlaying(true);
+            AddStep("select first", () => songSelect.Carousel.SelectBeatmap(songSelect.Carousel.BeatmapSets.First().Beatmaps.First()));
+            checkMusicPlaying(true);
+
+            AddStep("manual pause", () => music.TogglePause());
+            checkMusicPlaying(false);
+            AddStep("select next difficulty", () => songSelect.Carousel.SelectNext(skipDifficulties: false));
+            checkMusicPlaying(false);
+
+            AddStep("select next set", () => songSelect.Carousel.SelectNext());
+            checkMusicPlaying(true);
+        }
+
+        [TestCase(false)]
+        [TestCase(true)]
+        public void TestAudioRemainsCorrectOnRulesetChange(bool rulesetsInSameBeatmap)
+        {
+            createSongSelect();
+
+            // start with non-osu! to avoid convert confusion
+            changeRuleset(1);
+
+            if (rulesetsInSameBeatmap)
+                AddStep("import multi-ruleset map", () =>
+                {
+                    var usableRulesets = rulesets.AvailableRulesets.Where(r => r.ID != 2).ToArray();
+                    manager.Import(createTestBeatmapSet(0, usableRulesets)).Wait();
+                });
+            else
+            {
+                addRulesetImportStep(1);
+                addRulesetImportStep(0);
+            }
+
+            checkMusicPlaying(true);
+
+            AddStep("manual pause", () => music.TogglePause());
+            checkMusicPlaying(false);
+
+            changeRuleset(0);
+            checkMusicPlaying(!rulesetsInSameBeatmap);
+        }
+
         [Test]
         public void TestDummy()
         {
@@ -222,6 +281,9 @@ namespace osu.Game.Tests.Visual.SongSelect
         private static int importId;
         private int getImportId() => ++importId;
 
+        private void checkMusicPlaying(bool playing) =>
+            AddUntilStep($"music {(playing ? "" : "not ")}playing", () => music.IsPlaying == playing);
+
         private void changeMods(params Mod[] mods) => AddStep($"change mods to {string.Join(", ", mods.Select(m => m.Acronym))}", () => Mods.Value = mods);
 
         private void changeRuleset(int id) => AddStep($"change ruleset to {id}", () => Ruleset.Value = rulesets.AvailableRulesets.First(r => r.ID == id));
diff --git a/osu.Game/Screens/Select/SongSelect.cs b/osu.Game/Screens/Select/SongSelect.cs
index d9ddfa2a94..6c5f64ed6c 100644
--- a/osu.Game/Screens/Select/SongSelect.cs
+++ b/osu.Game/Screens/Select/SongSelect.cs
@@ -412,9 +412,6 @@ namespace osu.Game.Screens.Select
                     WorkingBeatmap previous = Beatmap.Value;
                     Beatmap.Value = beatmaps.GetWorkingBeatmap(beatmap, previous);
 
-                    if (this.IsCurrentScreen() && Beatmap.Value?.Track != previous?.Track)
-                        ensurePlayingSelected(true);
-
                     if (beatmap != null)
                     {
                         if (beatmap.BeatmapSetInfoID == beatmapNoDebounce?.BeatmapSetInfoID)
@@ -424,6 +421,9 @@ namespace osu.Game.Screens.Select
                     }
                 }
 
+                if (this.IsCurrentScreen())
+                    ensurePlayingSelected();
+
                 UpdateBeatmap(Beatmap.Value);
             }
         }
@@ -581,19 +581,24 @@ namespace osu.Game.Screens.Select
                 beatmap.Track.Looping = true;
         }
 
-        private void ensurePlayingSelected(bool restart = false)
+        private readonly WeakReference<Track> lastTrack = new WeakReference<Track>(null);
+
+        /// <summary>
+        /// Ensures some music is playing for the current track.
+        /// Will resume playback from a manual user pause if the track has changed.
+        /// </summary>
+        private void ensurePlayingSelected()
         {
             Track track = Beatmap.Value.Track;
 
-            if (!track.IsRunning)
-            {
-                track.RestartPoint = Beatmap.Value.Metadata.PreviewTime;
+            bool isNewTrack = !lastTrack.TryGetTarget(out var last) || last != track;
 
-                if (restart)
-                    track.Restart();
-                else
-                    track.Start();
-            }
+            track.RestartPoint = Beatmap.Value.Metadata.PreviewTime;
+
+            if (!track.IsRunning && (music?.IsUserPaused != true || isNewTrack))
+                track.Restart();
+
+            lastTrack.SetTarget(track);
         }
 
         private void onBeatmapSetAdded(BeatmapSetInfo s) => Carousel.UpdateBeatmapSet(s);