diff --git a/osu.Game/Overlays/ChatOverlay.cs b/osu.Game/Overlays/ChatOverlay.cs
index 90068eb170..dccbc18f30 100644
--- a/osu.Game/Overlays/ChatOverlay.cs
+++ b/osu.Game/Overlays/ChatOverlay.cs
@@ -135,17 +135,22 @@ namespace osu.Game.Overlays
             channelTabs.Current.ValueChanged += newChannel => CurrentChannel = newChannel;
         }
 
+        private double startDragChatHeight;
+
         protected override bool OnDragStart(InputState state)
         {
-            if (channelTabs.Hovering)
-                return true;
+            if (!channelTabs.Hovering)
+                return base.OnDragStart(state);
 
-            return base.OnDragStart(state);
+            startDragChatHeight = chatHeight.Value;
+            return true;
         }
 
         protected override bool OnDrag(InputState state)
         {
-            chatHeight.Value = Height - state.Mouse.Delta.Y / Parent.DrawSize.Y;
+            Trace.Assert(state.Mouse.PositionMouseDown != null);
+
+            chatHeight.Value = startDragChatHeight - (state.Mouse.Position.Y - state.Mouse.PositionMouseDown.Value.Y) / Parent.DrawSize.Y;
             return base.OnDrag(state);
         }