diff --git a/osu.Game.Tests/Visual/UserInterface/TestSceneOverlayScrollContainer.cs b/osu.Game.Tests/Visual/UserInterface/TestSceneOverlayScrollContainer.cs
index 926bc01aea..77e7178c9e 100644
--- a/osu.Game.Tests/Visual/UserInterface/TestSceneOverlayScrollContainer.cs
+++ b/osu.Game.Tests/Visual/UserInterface/TestSceneOverlayScrollContainer.cs
@@ -61,6 +61,18 @@ namespace osu.Game.Tests.Visual.UserInterface
             AddStep("scroll to 500", () => scroll.ScrollTo(500));
             AddUntilStep("scrolled to 500", () => Precision.AlmostEquals(scroll.Current, 500, 0.1f));
             AddAssert("button is visible", () => scroll.Button.State == Visibility.Visible);
+
+            AddStep("click button", () =>
+            {
+                InputManager.MoveMouseTo(scroll.Button);
+                InputManager.Click(MouseButton.Left);
+            });
+
+            AddAssert("button is visible", () => scroll.Button.State == Visibility.Visible);
+
+            AddStep("user scroll down by 1", () => InputManager.ScrollVerticalBy(-1));
+
+            AddAssert("button is hidden", () => scroll.Button.State == Visibility.Hidden);
         }
 
         [Test]
@@ -71,6 +83,10 @@ namespace osu.Game.Tests.Visual.UserInterface
             AddStep("invoke action", () => scroll.Button.Action.Invoke());
 
             AddUntilStep("scrolled back to start", () => Precision.AlmostEquals(scroll.Current, 0, 0.1f));
+
+            AddStep("invoke action", () => scroll.Button.Action.Invoke());
+
+            AddAssert("scrolled to end", () => scroll.IsScrolledToEnd());
         }
 
         [Test]
@@ -85,6 +101,14 @@ namespace osu.Game.Tests.Visual.UserInterface
             });
 
             AddUntilStep("scrolled back to start", () => Precision.AlmostEquals(scroll.Current, 0, 0.1f));
+
+            AddStep("click button", () =>
+            {
+                InputManager.MoveMouseTo(scroll.Button);
+                InputManager.Click(MouseButton.Left);
+            });
+
+            AddAssert("scrolled to end", () => scroll.IsScrolledToEnd());
         }
 
         [Test]
@@ -97,12 +121,12 @@ namespace osu.Game.Tests.Visual.UserInterface
             AddStep("hover button", () => InputManager.MoveMouseTo(scroll.Button));
             AddRepeatStep("click button", () => InputManager.Click(MouseButton.Left), 3);
 
-            AddAssert("invocation count is 1", () => invocationCount == 1);
+            AddAssert("invocation count is 3", () => invocationCount == 3);
         }
 
         private partial class TestScrollContainer : OverlayScrollContainer
         {
-            public new ScrollToTopButton Button => base.Button;
+            public new ScrollBackButton Button => base.Button;
         }
     }
 }
diff --git a/osu.Game/Overlays/OverlayScrollContainer.cs b/osu.Game/Overlays/OverlayScrollContainer.cs
index 5bd7f014a9..9752e04f44 100644
--- a/osu.Game/Overlays/OverlayScrollContainer.cs
+++ b/osu.Game/Overlays/OverlayScrollContainer.cs
@@ -5,6 +5,7 @@
 
 using System.Collections.Generic;
 using osu.Framework.Allocation;
+using osu.Framework.Bindables;
 using osu.Framework.Extensions.Color4Extensions;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
@@ -21,25 +22,29 @@ using osuTK.Graphics;
 namespace osu.Game.Overlays
 {
     /// <summary>
-    /// <see cref="UserTrackingScrollContainer"/> which provides <see cref="ScrollToTopButton"/>. Mostly used in <see cref="FullscreenOverlay{T}"/>.
+    /// <see cref="UserTrackingScrollContainer"/> which provides <see cref="ScrollBackButton"/>. Mostly used in <see cref="FullscreenOverlay{T}"/>.
     /// </summary>
     public partial class OverlayScrollContainer : UserTrackingScrollContainer
     {
         /// <summary>
-        /// Scroll position at which the <see cref="ScrollToTopButton"/> will be shown.
+        /// Scroll position at which the <see cref="ScrollBackButton"/> will be shown.
         /// </summary>
         private const int button_scroll_position = 200;
 
-        protected readonly ScrollToTopButton Button;
+        protected ScrollBackButton Button;
 
-        public OverlayScrollContainer()
+        private readonly Bindable<float?> lastScrollTarget = new Bindable<float?>();
+
+        [BackgroundDependencyLoader]
+        private void load()
         {
-            AddInternal(Button = new ScrollToTopButton
+            AddInternal(Button = new ScrollBackButton
             {
                 Anchor = Anchor.BottomRight,
                 Origin = Anchor.BottomRight,
                 Margin = new MarginPadding(20),
-                Action = scrollToTop
+                Action = scrollBack,
+                LastScrollTarget = { BindTarget = lastScrollTarget }
             });
         }
 
@@ -53,16 +58,31 @@ namespace osu.Game.Overlays
                 return;
             }
 
-            Button.State = Target > button_scroll_position ? Visibility.Visible : Visibility.Hidden;
+            Button.State = Target > button_scroll_position || lastScrollTarget.Value != null ? Visibility.Visible : Visibility.Hidden;
         }
 
-        private void scrollToTop()
+        protected override void OnUserScroll(float value, bool animated = true, double? distanceDecay = default)
         {
-            ScrollToStart();
-            Button.State = Visibility.Hidden;
+            base.OnUserScroll(value, animated, distanceDecay);
+
+            lastScrollTarget.Value = null;
         }
 
-        public partial class ScrollToTopButton : OsuHoverContainer
+        private void scrollBack()
+        {
+            if (lastScrollTarget.Value == null)
+            {
+                lastScrollTarget.Value = Target;
+                ScrollToStart();
+            }
+            else
+            {
+                ScrollTo(lastScrollTarget.Value.Value);
+                lastScrollTarget.Value = null;
+            }
+        }
+
+        public partial class ScrollBackButton : OsuHoverContainer
         {
             private const int fade_duration = 500;
 
@@ -88,8 +108,11 @@ namespace osu.Game.Overlays
 
             private readonly Container content;
             private readonly Box background;
+            private readonly SpriteIcon spriteIcon;
 
-            public ScrollToTopButton()
+            public Bindable<float?> LastScrollTarget = new Bindable<float?>();
+
+            public ScrollBackButton()
                 : base(HoverSampleSet.ScrollToTop)
             {
                 Size = new Vector2(50);
@@ -113,7 +136,7 @@ namespace osu.Game.Overlays
                         {
                             RelativeSizeAxes = Axes.Both
                         },
-                        new SpriteIcon
+                        spriteIcon = new SpriteIcon
                         {
                             Anchor = Anchor.Centre,
                             Origin = Anchor.Centre,
@@ -134,6 +157,17 @@ namespace osu.Game.Overlays
                 flashColour = colourProvider.Light1;
             }
 
+            protected override void LoadComplete()
+            {
+                base.LoadComplete();
+
+                LastScrollTarget.BindValueChanged(target =>
+                {
+                    spriteIcon.RotateTo(target.NewValue != null ? 180 : 0, fade_duration, Easing.OutQuint);
+                    TooltipText = target.NewValue != null ? CommonStrings.ButtonsBackToPrevious : CommonStrings.ButtonsBackToTop;
+                }, true);
+            }
+
             protected override bool OnClick(ClickEvent e)
             {
                 background.FlashColour(flashColour, 800, Easing.OutQuint);