diff --git a/osu.Game.Tests/Visual/Ranking/TestSceneUserTagControl.cs b/osu.Game.Tests/Visual/Ranking/TestSceneUserTagControl.cs
index c546c9727c..b63f8ca31c 100644
--- a/osu.Game.Tests/Visual/Ranking/TestSceneUserTagControl.cs
+++ b/osu.Game.Tests/Visual/Ranking/TestSceneUserTagControl.cs
@@ -2,8 +2,10 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Linq;
+using NUnit.Framework;
 using osu.Framework.Allocation;
 using osu.Framework.Graphics;
+using osu.Framework.Graphics.Containers;
 using osu.Framework.Graphics.Cursor;
 using osu.Framework.Testing;
 using osu.Game.Beatmaps;
@@ -14,10 +16,12 @@ using osu.Game.Overlays;
 using osu.Game.Rulesets.Osu;
 using osu.Game.Rulesets.Taiko;
 using osu.Game.Screens.Ranking;
+using osuTK;
+using osuTK.Input;
 
 namespace osu.Game.Tests.Visual.Ranking
 {
-    public partial class TestSceneUserTagControl : OsuTestScene
+    public partial class TestSceneUserTagControl : OsuManualInputManagerTestScene
     {
         [Cached]
         private OverlayColourProvider colourProvider = new OverlayColourProvider(OverlayColourScheme.Aquamarine);
@@ -63,6 +67,8 @@ namespace osu.Game.Tests.Visual.Ranking
                             beatmapSet.Beatmaps.Single().TopTags =
                             [
                                 new APIBeatmapTag { TagId = 3, VoteCount = 9 },
+                                new APIBeatmapTag { TagId = 2, VoteCount = 8 },
+                                new APIBeatmapTag { TagId = 0, VoteCount = 7 },
                             ];
                             Scheduler.AddDelayed(() => getBeatmapSetRequest.TriggerSuccess(beatmapSet), 500);
                             return true;
@@ -79,6 +85,11 @@ namespace osu.Game.Tests.Visual.Ranking
                     return false;
                 };
             });
+        }
+
+        [Test]
+        public void TestRulesetSupport()
+        {
             AddStep("show for osu! beatmap", () =>
             {
                 var working = CreateWorkingBeatmap(new OsuRuleset().RulesetInfo);
@@ -86,6 +97,7 @@ namespace osu.Game.Tests.Visual.Ranking
                 Beatmap.Value = working;
                 recreateControl();
             });
+
             AddStep("show for taiko beatmap", () =>
             {
                 var working = CreateWorkingBeatmap(new TaikoRuleset().RulesetInfo);
@@ -95,6 +107,47 @@ namespace osu.Game.Tests.Visual.Ranking
             });
         }
 
+        [Test]
+        public void TestTagsDoNotMoveUntilMouseMovesAway()
+        {
+            AddStep("show", () =>
+            {
+                var working = CreateWorkingBeatmap(new OsuRuleset().RulesetInfo);
+                working.BeatmapInfo.OnlineID = 42;
+                Beatmap.Value = working;
+                recreateControl();
+            });
+            AddUntilStep("wait for ready", () => getTagFlow().Count, () => Is.EqualTo(4));
+            AddAssert("tag 2 is second", () => getTagFlow().GetLayoutPosition(getDrawableTagById(2)), () => Is.EqualTo(1));
+            AddStep("vote for tag 2", () =>
+            {
+                InputManager.MoveMouseTo(getDrawableTagById(2));
+                InputManager.Click(MouseButton.Left);
+            });
+            AddUntilStep("tag 2 voted for", () => getDrawableTagById(2).UserTag.VoteCount.Value, () => Is.EqualTo(9));
+
+            AddStep("remove vote for tag 2", () =>
+            {
+                InputManager.MoveMouseTo(getDrawableTagById(2));
+                InputManager.Click(MouseButton.Left);
+            });
+            AddUntilStep("tag 2 not voted for", () => getDrawableTagById(2).UserTag.VoteCount.Value, () => Is.EqualTo(8));
+            AddAssert("tag 2 is still second", () => getTagFlow().GetLayoutPosition(getDrawableTagById(2)), () => Is.EqualTo(1));
+
+            AddStep("vote for tag 2", () =>
+            {
+                InputManager.MoveMouseTo(getDrawableTagById(2));
+                InputManager.Click(MouseButton.Left);
+            });
+            AddUntilStep("tag 2 voted for", () => getDrawableTagById(2).UserTag.VoteCount.Value, () => Is.EqualTo(9));
+            AddStep("move mouse away", () => InputManager.MoveMouseTo(Vector2.Zero));
+            AddAssert("tag 2 reordered to first", () => getTagFlow().GetLayoutPosition(getDrawableTagById(2)), () => Is.EqualTo(0));
+
+            FillFlowContainer<UserTagControl.DrawableUserTag> getTagFlow() => this.ChildrenOfType<FillFlowContainer<UserTagControl.DrawableUserTag>>().Single();
+
+            UserTagControl.DrawableUserTag getDrawableTagById(long id) => getTagFlow().Single(t => t.UserTag.Id == id);
+        }
+
         private void recreateControl()
         {
             Child = new PopoverContainer
diff --git a/osu.Game/Screens/Ranking/UserTagControl.cs b/osu.Game/Screens/Ranking/UserTagControl.cs
index e323107783..5618dd2490 100644
--- a/osu.Game/Screens/Ranking/UserTagControl.cs
+++ b/osu.Game/Screens/Ranking/UserTagControl.cs
@@ -13,6 +13,7 @@ using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 using osu.Framework.Graphics.Cursor;
 using osu.Framework.Graphics.UserInterface;
+using osu.Framework.Input;
 using osu.Framework.Input.Events;
 using osu.Game.Beatmaps;
 using osu.Game.Configuration;
@@ -44,6 +45,8 @@ namespace osu.Game.Screens.Ranking
 
         private AddNewTagUserTag addNewTagUserTag = null!;
 
+        private InputManager inputManager = null!;
+
         [Resolved]
         private IAPIProvider api { get; set; } = null!;
 
@@ -124,6 +127,8 @@ namespace osu.Game.Screens.Ranking
             updateTags();
 
             displayedTags.BindCollectionChanged(displayTags, true);
+
+            inputManager = GetContainingInputManager()!;
         }
 
         private void updateTags()
@@ -251,7 +256,7 @@ namespace osu.Game.Screens.Ranking
         {
             base.Update();
 
-            if (!layout.IsValid && !IsHovered)
+            if (!layout.IsValid && !Contains(inputManager.CurrentState.Mouse.Position))
             {
                 var sortedTags = new Dictionary<UserTag, int>(
                     displayedTags.OrderByDescending(t => t.VoteCount.Value)
diff --git a/osu.Game/Screens/Ranking/UserTagControl_DrawableUserTag.cs b/osu.Game/Screens/Ranking/UserTagControl_DrawableUserTag.cs
index e54d88bca2..ff3c0711c0 100644
--- a/osu.Game/Screens/Ranking/UserTagControl_DrawableUserTag.cs
+++ b/osu.Game/Screens/Ranking/UserTagControl_DrawableUserTag.cs
@@ -18,7 +18,7 @@ namespace osu.Game.Screens.Ranking
 {
     public partial class UserTagControl
     {
-        private partial class DrawableUserTag : OsuAnimatedButton
+        public partial class DrawableUserTag : OsuAnimatedButton
         {
             public readonly UserTag UserTag;