diff --git a/osu.Game/Screens/Select/BeatmapCarousel.cs b/osu.Game/Screens/Select/BeatmapCarousel.cs
index adaaddaf78..b3db33401b 100644
--- a/osu.Game/Screens/Select/BeatmapCarousel.cs
+++ b/osu.Game/Screens/Select/BeatmapCarousel.cs
@@ -252,7 +252,7 @@ namespace osu.Game.Screens.Select
                 var notYetVisitedSets = visibleSets.Except(previouslyVisitedRandomSets).ToList();
                 if (!notYetVisitedSets.Any())
                 {
-                    previouslyVisitedRandomSets.Clear();
+                    previouslyVisitedRandomSets.RemoveAll(s => visibleSets.Contains(s));
                     notYetVisitedSets = visibleSets;
                 }