diff --git a/osu.Game.Tests/Visual/UserInterface/TestSceneModColumn.cs b/osu.Game.Tests/Visual/UserInterface/TestSceneModColumn.cs
index 01afde8f7e..e47ae860c6 100644
--- a/osu.Game.Tests/Visual/UserInterface/TestSceneModColumn.cs
+++ b/osu.Game.Tests/Visual/UserInterface/TestSceneModColumn.cs
@@ -54,17 +54,20 @@ namespace osu.Game.Tests.Visual.UserInterface
         [Test]
         public void TestMultiSelection()
         {
+            ModColumn column = null;
             AddStep("create content", () => Child = new Container
             {
                 RelativeSizeAxes = Axes.Both,
                 Padding = new MarginPadding(30),
-                Child = new ModColumn(ModType.DifficultyIncrease, true)
+                Child = column = new ModColumn(ModType.DifficultyIncrease, true)
                 {
                     Anchor = Anchor.Centre,
                     Origin = Anchor.Centre
                 }
             });
 
+            AddUntilStep("wait for panel load", () => column.IsLoaded && column.ItemsLoaded);
+
             clickToggle();
             AddUntilStep("all panels selected", () => this.ChildrenOfType<ModPanel>().All(panel => panel.Active.Value));
 
@@ -140,13 +143,15 @@ namespace osu.Game.Tests.Visual.UserInterface
             {
                 RelativeSizeAxes = Axes.Both,
                 Padding = new MarginPadding(30),
-                Child = column = new ModColumn(ModType.DifficultyReduction, true, new Key[] { Key.Q, Key.W, Key.E, Key.R, Key.T, Key.Y, Key.U, Key.I, Key.O, Key.P })
+                Child = column = new ModColumn(ModType.DifficultyReduction, true, new[] { Key.Q, Key.W, Key.E, Key.R, Key.T, Key.Y, Key.U, Key.I, Key.O, Key.P })
                 {
                     Anchor = Anchor.Centre,
                     Origin = Anchor.Centre
                 }
             });
 
+            AddUntilStep("wait for panel load", () => column.IsLoaded && column.ItemsLoaded);
+
             AddStep("press W", () => InputManager.Key(Key.W));
             AddAssert("NF panel selected", () => this.ChildrenOfType<ModPanel>().Single(panel => panel.Mod.Acronym == "NF").Active.Value);
 
diff --git a/osu.Game/Overlays/Mods/ModColumn.cs b/osu.Game/Overlays/Mods/ModColumn.cs
index 7f3cc8249f..b615b7bd32 100644
--- a/osu.Game/Overlays/Mods/ModColumn.cs
+++ b/osu.Game/Overlays/Mods/ModColumn.cs
@@ -5,6 +5,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
+using System.Threading.Tasks;
 using Humanizer;
 using osu.Framework.Allocation;
 using osu.Framework.Bindables;
@@ -56,6 +57,9 @@ namespace osu.Game.Overlays.Mods
 
         private Colour4 accentColour;
 
+        private Task? latestLoadTask;
+        internal bool ItemsLoaded => latestLoadTask == null;
+
         private const float header_height = 42;
 
         public ModColumn(ModType modType, bool allowBulkSelection, Key[]? toggleKeys = null)
@@ -234,7 +238,9 @@ namespace osu.Game.Overlays.Mods
                 Shear = new Vector2(-ModPanel.SHEAR_X, 0)
             });
 
-            LoadComponentsAsync(panels, loaded =>
+            Task? loadTask;
+
+            latestLoadTask = loadTask = LoadComponentsAsync(panels, loaded =>
             {
                 panelFlow.ChildrenEnumerable = loaded;
 
@@ -244,6 +250,11 @@ namespace osu.Game.Overlays.Mods
 
                 updateFilter();
             }, (cancellationTokenSource = new CancellationTokenSource()).Token);
+            loadTask.ContinueWith(_ =>
+            {
+                if (loadTask == latestLoadTask)
+                    latestLoadTask = null;
+            });
         }
 
         #region Bulk select / deselect