diff --git a/osu.Game.Tests/Visual/SongSelectV2/BeatmapCarouselTestScene.cs b/osu.Game.Tests/Visual/SongSelectV2/BeatmapCarouselTestScene.cs
index d961946c04..dc3a49c068 100644
--- a/osu.Game.Tests/Visual/SongSelectV2/BeatmapCarouselTestScene.cs
+++ b/osu.Game.Tests/Visual/SongSelectV2/BeatmapCarouselTestScene.cs
@@ -44,6 +44,8 @@ namespace osu.Game.Tests.Visual.SongSelectV2
 
         protected TestBeatmapCarousel Carousel = null!;
 
+        protected bool RetainSelection { get; set; }
+
         protected OsuScrollContainer<Drawable> Scroll => Carousel.ChildrenOfType<OsuScrollContainer<Drawable>>().Single();
 
         [Cached(typeof(BeatmapStore))]
@@ -78,7 +80,7 @@ namespace osu.Game.Tests.Visual.SongSelectV2
             Dependencies.Cache(Realm);
         }
 
-        protected void CreateCarousel()
+        protected void CreateCarousel(bool retainSelection = false)
         {
             AddStep("create components", () =>
             {
@@ -87,6 +89,8 @@ namespace osu.Game.Tests.Visual.SongSelectV2
                 BeatmapRecommendationFunction = null;
                 NewItemsPresentedInvocationCount = 0;
 
+                GroupedBeatmap? previousSelection = retainSelection ? Carousel.CurrentGroupedBeatmap : null;
+
                 Box topBox;
                 Children = new Drawable[]
                 {
@@ -120,6 +124,7 @@ namespace osu.Game.Tests.Visual.SongSelectV2
                             {
                                 Carousel = new TestBeatmapCarousel
                                 {
+                                    CurrentGroupedBeatmap = previousSelection,
                                     NewItemsPresented = _ => NewItemsPresentedInvocationCount++,
                                     RequestSelection = b =>
                                     {
diff --git a/osu.Game.Tests/Visual/SongSelectV2/TestSceneBeatmapCarouselSetsSplitApart.cs b/osu.Game.Tests/Visual/SongSelectV2/TestSceneBeatmapCarouselSetsSplitApart.cs
index fa635f9bde..31488e399a 100644
--- a/osu.Game.Tests/Visual/SongSelectV2/TestSceneBeatmapCarouselSetsSplitApart.cs
+++ b/osu.Game.Tests/Visual/SongSelectV2/TestSceneBeatmapCarouselSetsSplitApart.cs
@@ -23,6 +23,33 @@ namespace osu.Game.Tests.Visual.SongSelectV2
             SortAndGroupBy(SortMode.Title, GroupMode.Length);
         }
 
+        [Test]
+        public void TestInitialVisualState()
+        {
+            AddBeatmaps(3, splitApart: true);
+
+            WaitForDrawablePanels();
+            SelectNextSet();
+            WaitForSetSelection(set: 0, diff: 0);
+
+            AddAssert("selected item is visible", () => GetSelectedPanel()?.Item?.IsVisible, () => Is.True);
+            AddAssert("has visually expanded set", () => Carousel.GetCarouselItems()!.Count(item => item.Model is GroupedBeatmapSet && item.IsExpanded && item.IsVisible), () => Is.EqualTo(1));
+
+            CreateCarousel(retainSelection: true);
+            WaitForDrawablePanels();
+            WaitForSetSelection(set: 0, diff: 0);
+
+            AddAssert("selected item is visible", () => GetSelectedPanel()?.Item?.IsVisible, () => Is.True);
+            AddAssert("has visually expanded set", () => Carousel.GetCarouselItems()!.Count(item => item.Model is GroupedBeatmapSet && item.IsExpanded && item.IsVisible), () => Is.EqualTo(1));
+
+            CreateCarousel(retainSelection: true);
+            WaitForDrawablePanels();
+            WaitForSetSelection(set: 0, diff: 0);
+
+            AddAssert("selected item is visible", () => GetSelectedPanel()?.Item?.IsVisible, () => Is.True);
+            AddAssert("has visually expanded set", () => Carousel.GetCarouselItems()!.Count(item => item.Model is GroupedBeatmapSet && item.IsExpanded && item.IsVisible), () => Is.EqualTo(1));
+        }
+
         [Test]
         public void TestSetTraversal()
         {
diff --git a/osu.Game/Screens/SelectV2/BeatmapCarousel.cs b/osu.Game/Screens/SelectV2/BeatmapCarousel.cs
index ba54fd269c..adf6588727 100644
--- a/osu.Game/Screens/SelectV2/BeatmapCarousel.cs
+++ b/osu.Game/Screens/SelectV2/BeatmapCarousel.cs
@@ -408,8 +408,7 @@ namespace osu.Game.Screens.SelectV2
                 case GroupedBeatmap groupedBeatmap:
                     setExpandedGroup(groupedBeatmap.Group);
 
-                    if (grouping.BeatmapSetsGroupedTogether)
-                        setExpandedSet(new GroupedBeatmapSet(groupedBeatmap.Group, groupedBeatmap.Beatmap.BeatmapSet!));
+                    setExpandedSet(new GroupedBeatmapSet(groupedBeatmap.Group, groupedBeatmap.Beatmap.BeatmapSet!));
                     break;
             }
         }
@@ -647,14 +646,28 @@ namespace osu.Game.Screens.SelectV2
 
         private void setExpandedSet(GroupedBeatmapSet set)
         {
-            if (ExpandedBeatmapSet != null)
-                setExpansionStateOfSetItems(ExpandedBeatmapSet, false);
+            GroupedBeatmapSet? lastExpandedSet = ExpandedBeatmapSet;
+
+            // It's important that we update the stored ExpandedBeatmapSet even when
+            // sets are not grouped together.
+            //
+            // This is stored when selection is changed and used later to ensure correct
+            // visual states are achieved (see call of this method in `HandleFilterCompleted`
+            // for an important case).
             ExpandedBeatmapSet = set;
+
+            if (!grouping.BeatmapSetsGroupedTogether)
+                return;
+
+            setExpansionStateOfSetItems(lastExpandedSet, false);
             setExpansionStateOfSetItems(ExpandedBeatmapSet, true);
         }
 
-        private void setExpansionStateOfSetItems(GroupedBeatmapSet set, bool expanded)
+        private void setExpansionStateOfSetItems(GroupedBeatmapSet? set, bool expanded)
         {
+            if (set == null)
+                return;
+
             bool canMakeVisible = !grouping.GroupItems.Any() || ExpandedGroup == set.Group;
 
             if (grouping.SetItems.TryGetValue(set, out var items))