diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarouselV2.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarouselV2.cs
index 1d7d6041ae..f99e0a418a 100644
--- a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarouselV2.cs
+++ b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarouselV2.cs
@@ -168,7 +168,33 @@ namespace osu.Game.Tests.Visual.SongSelect
         }
 
         [Test]
-        public void TestScrollPositionVelocityMaintained()
+        public void TestScrollPositionMaintainedOnAddSecondSelected()
+        {
+            Quad positionBefore = default;
+
+            AddStep("add 10 beatmaps", () =>
+            {
+                for (int i = 0; i < 10; i++)
+                    beatmapSets.Add(TestResources.CreateTestBeatmapSetInfo(RNG.Next(1, 4)));
+            });
+
+            AddUntilStep("visual item added", () => carousel.ChildrenOfType<BeatmapCarouselPanel>().Count(), () => Is.GreaterThan(0));
+
+            AddStep("select middle beatmap", () => carousel.CurrentSelection = beatmapSets.ElementAt(beatmapSets.Count - 2));
+            AddStep("scroll to selected item", () => scroll.ScrollTo(scroll.ChildrenOfType<BeatmapCarouselPanel>().Single(p => p.Item!.Selected.Value)));
+
+            AddUntilStep("wait for scroll finished", () => scroll.Current, () => Is.EqualTo(scroll.Target));
+
+            AddStep("save selected screen position", () => positionBefore = carousel.ChildrenOfType<BeatmapCarouselPanel>().FirstOrDefault(p => p.Item!.Selected.Value)!.ScreenSpaceDrawQuad);
+
+            AddStep("remove first beatmap", () => beatmapSets.Remove(beatmapSets.Last()));
+            AddUntilStep("sorting finished", () => carousel.IsFiltering, () => Is.False);
+            AddAssert("select screen position unchanged", () => carousel.ChildrenOfType<BeatmapCarouselPanel>().Single(p => p.Item!.Selected.Value).ScreenSpaceDrawQuad,
+                () => Is.EqualTo(positionBefore));
+        }
+
+        [Test]
+        public void TestScrollPositionMaintainedOnAddLastSelected()
         {
             Quad positionBefore = default;
 
diff --git a/osu.Game/Screens/SelectV2/BeatmapCarouselPanel.cs b/osu.Game/Screens/SelectV2/BeatmapCarouselPanel.cs
index 5b8ae211d1..27023b50be 100644
--- a/osu.Game/Screens/SelectV2/BeatmapCarouselPanel.cs
+++ b/osu.Game/Screens/SelectV2/BeatmapCarouselPanel.cs
@@ -9,7 +9,6 @@ using osu.Framework.Graphics;
 using osu.Framework.Graphics.Pooling;
 using osu.Framework.Graphics.Shapes;
 using osu.Framework.Input.Events;
-using osu.Framework.Utils;
 using osu.Game.Beatmaps;
 using osu.Game.Graphics.Sprites;
 using osuTK;
@@ -98,18 +97,6 @@ namespace osu.Game.Screens.SelectV2
             return true;
         }
 
-        protected override void Update()
-        {
-            base.Update();
-
-            Debug.Assert(Item != null);
-
-            if (DrawYPosition != Item.CarouselYPosition)
-            {
-                DrawYPosition = Interpolation.DampContinuously(DrawYPosition, Item.CarouselYPosition, 50, Time.Elapsed);
-            }
-        }
-
-        public double DrawYPosition { get; private set; }
+        public double DrawYPosition { get; set; }
     }
 }
diff --git a/osu.Game/Screens/SelectV2/Carousel.cs b/osu.Game/Screens/SelectV2/Carousel.cs
index aeab6a96d0..12a86be7b9 100644
--- a/osu.Game/Screens/SelectV2/Carousel.cs
+++ b/osu.Game/Screens/SelectV2/Carousel.cs
@@ -14,6 +14,7 @@ using osu.Framework.Graphics.Containers;
 using osu.Framework.Graphics.Pooling;
 using osu.Framework.Graphics.Shapes;
 using osu.Framework.Logging;
+using osu.Framework.Utils;
 using osu.Game.Graphics.Containers;
 using osuTK;
 using osuTK.Graphics;
@@ -107,7 +108,7 @@ namespace osu.Game.Screens.SelectV2
 
         private List<CarouselItem>? displayedCarouselItems;
 
-        private readonly DoublePrecisionScroll scroll;
+        private readonly CarouselScrollContainer scroll;
 
         protected Carousel()
         {
@@ -118,7 +119,7 @@ namespace osu.Game.Screens.SelectV2
                     Colour = Color4.Black,
                     RelativeSizeAxes = Axes.Both,
                 },
-                scroll = new DoublePrecisionScroll
+                scroll = new CarouselScrollContainer
                 {
                     RelativeSizeAxes = Axes.Both,
                     Masking = false,
@@ -389,13 +390,13 @@ namespace osu.Game.Screens.SelectV2
         /// Implementation of scroll container which handles very large vertical lists by internally using <c>double</c> precision
         /// for pre-display Y values.
         /// </summary>
-        private partial class DoublePrecisionScroll : OsuScrollContainer
+        private partial class CarouselScrollContainer : OsuScrollContainer
         {
             public readonly Container Panels;
 
             public void SetLayoutHeight(float height) => Panels.Height = height;
 
-            public DoublePrecisionScroll()
+            public CarouselScrollContainer()
             {
                 // Managing our own custom layout within ScrollContent causes feedback with public ScrollContainer calculations,
                 // so we must maintain one level of separation from ScrollContent.
@@ -406,6 +407,33 @@ namespace osu.Game.Screens.SelectV2
                 });
             }
 
+            public override void OffsetScrollPosition(double offset)
+            {
+                base.OffsetScrollPosition(offset);
+
+                foreach (var panel in Panels)
+                {
+                    var c = (ICarouselPanel)panel;
+                    Debug.Assert(c.Item != null);
+
+                    c.DrawYPosition += offset;
+                }
+            }
+
+            protected override void Update()
+            {
+                base.Update();
+
+                foreach (var panel in Panels)
+                {
+                    var c = (ICarouselPanel)panel;
+                    Debug.Assert(c.Item != null);
+
+                    if (c.DrawYPosition != c.Item.CarouselYPosition)
+                        c.DrawYPosition = Interpolation.DampContinuously(c.DrawYPosition, c.Item.CarouselYPosition, 50, Time.Elapsed);
+                }
+            }
+
             public override void Clear(bool disposeChildren)
             {
                 Panels.Height = 0;
diff --git a/osu.Game/Screens/SelectV2/ICarouselPanel.cs b/osu.Game/Screens/SelectV2/ICarouselPanel.cs
index d729df7876..117feab621 100644
--- a/osu.Game/Screens/SelectV2/ICarouselPanel.cs
+++ b/osu.Game/Screens/SelectV2/ICarouselPanel.cs
@@ -11,9 +11,9 @@ namespace osu.Game.Screens.SelectV2
     public interface ICarouselPanel
     {
         /// <summary>
-        /// The Y position which should be used for displaying this item within the carousel.
+        /// The Y position which should be used for displaying this item within the carousel. This is managed by <see cref="Carousel{T}"/> and should not be set manually.
         /// </summary>
-        double DrawYPosition { get; }
+        double DrawYPosition { get; set; }
 
         /// <summary>
         /// The carousel item this drawable is representing. This is managed by <see cref="Carousel{T}"/> and should not be set manually.