diff --git a/osu.Game/Overlays/SkinEditor/SkinSelectionHandler.cs b/osu.Game/Overlays/SkinEditor/SkinSelectionHandler.cs
index c90a1d8edf..ae0a3d0635 100644
--- a/osu.Game/Overlays/SkinEditor/SkinSelectionHandler.cs
+++ b/osu.Game/Overlays/SkinEditor/SkinSelectionHandler.cs
@@ -31,8 +31,6 @@ namespace osu.Game.Overlays.SkinEditor
             UpdatePosition = updateDrawablePosition
         };
 
-        private float accumulatedNegativeScaling;
-
         public override bool HandleScale(Vector2 scale, Anchor anchor)
         {
             // convert scale to screen space
@@ -73,32 +71,25 @@ namespace osu.Game.Overlays.SkinEditor
                     scale.Y = scale.X / selectionRect.Width * selectionRect.Height;
             }
 
-            // If scaling reverses the selection, don't scale and accumulate the amount of scaling.
-            if (adjustedRect.Width + scale.X < 0 || adjustedRect.Height + scale.Y < 0)
-            {
-                accumulatedNegativeScaling += scale.Length; // - new Vector2(selectionRect.Width, selectionRect.Height).Length;
-
-                return true;
-            }
-
-            // Compensate for accumulated negative scaling.
-            if (Precision.AlmostBigger(accumulatedNegativeScaling, 0) && !Precision.AlmostEquals(accumulatedNegativeScaling, 0))
-            {
-                float length = scale.Length;
-                accumulatedNegativeScaling -= length;
-
-                // If the accumulated negative scaling is still positive, don't scale.
-                if (Precision.AlmostBigger(accumulatedNegativeScaling, 0)) return true;
-                scale *= Math.Abs(accumulatedNegativeScaling) / length;
-                accumulatedNegativeScaling = 0;
-            }
-
             if (anchor.HasFlagFast(Anchor.x0)) adjustedRect.X -= scale.X;
             if (anchor.HasFlagFast(Anchor.y0)) adjustedRect.Y -= scale.Y;
 
             adjustedRect.Width += scale.X;
             adjustedRect.Height += scale.Y;
 
+            if (adjustedRect.Width < 0)
+            {
+                SelectionBox.ScaleHandlesFlip(Direction.Horizontal);
+                HandleFlip(Direction.Horizontal, false);
+            }
+            if (adjustedRect.Height < 0)
+            {
+                SelectionBox.ScaleHandlesFlip(Direction.Vertical);
+                HandleFlip(Direction.Vertical, false);
+            }
+            if (adjustedRect.Width < 0 || adjustedRect.Height < 0)
+                return true;
+
             // scale adjust applied to each individual item should match that of the quad itself.
             var scaledDelta = new Vector2(
                 adjustedRect.Width / selectionRect.Width,
@@ -169,12 +160,6 @@ namespace osu.Game.Overlays.SkinEditor
 
         public static void ApplyClosestAnchor(Drawable drawable) => applyAnchor(drawable, getClosestAnchor(drawable));
 
-        protected override void OnOperationEnded()
-        {
-            base.OnOperationEnded();
-            accumulatedNegativeScaling = 0;
-        }
-
         protected override void OnSelectionChanged()
         {
             base.OnSelectionChanged();
diff --git a/osu.Game/Screens/Edit/Compose/Components/SelectionBox.cs b/osu.Game/Screens/Edit/Compose/Components/SelectionBox.cs
index 876e8ccbe9..a261b635b3 100644
--- a/osu.Game/Screens/Edit/Compose/Components/SelectionBox.cs
+++ b/osu.Game/Screens/Edit/Compose/Components/SelectionBox.cs
@@ -307,6 +307,8 @@ namespace osu.Game.Screens.Edit.Compose.Components
             return button;
         }
 
+        public void ScaleHandlesFlip(Direction direction) => dragHandles.ScaleHandlesFlip(direction);
+
         private void addScaleHandle(Anchor anchor)
         {
             var handle = new SelectionBoxScaleHandle
diff --git a/osu.Game/Screens/Edit/Compose/Components/SelectionBoxDragHandleContainer.cs b/osu.Game/Screens/Edit/Compose/Components/SelectionBoxDragHandleContainer.cs
index 5c87271493..4fd2e9aba9 100644
--- a/osu.Game/Screens/Edit/Compose/Components/SelectionBoxDragHandleContainer.cs
+++ b/osu.Game/Screens/Edit/Compose/Components/SelectionBoxDragHandleContainer.cs
@@ -7,6 +7,7 @@ using System.Collections.Generic;
 using System.Linq;
 using JetBrains.Annotations;
 using osu.Framework.Allocation;
+using osu.Framework.Extensions.EnumExtensions;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 
@@ -69,6 +70,17 @@ namespace osu.Game.Screens.Edit.Compose.Components
             allDragHandles.Add(handle);
         }
 
+        public void ScaleHandlesFlip(Direction direction)
+        {
+            foreach (var handle in scaleHandles)
+            {
+                if (direction == Direction.Horizontal && !handle.Anchor.HasFlagFast(Anchor.x1))
+                    handle.Anchor ^= Anchor.x0 | Anchor.x2;
+                if (direction == Direction.Vertical && !handle.Anchor.HasFlagFast(Anchor.y1))
+                    handle.Anchor ^= Anchor.y0 | Anchor.y2;
+            }
+        }
+
         private SelectionBoxRotationHandle displayedRotationHandle;
         private SelectionBoxDragHandle activeHandle;