diff --git a/osu.Game.Rulesets.Catch/UI/CatcherSprite.cs b/osu.Game.Rulesets.Catch/UI/CatcherSprite.cs
index 52eb8d597e..ef69e3d2d1 100644
--- a/osu.Game.Rulesets.Catch/UI/CatcherSprite.cs
+++ b/osu.Game.Rulesets.Catch/UI/CatcherSprite.cs
@@ -16,7 +16,7 @@ namespace osu.Game.Rulesets.Catch.UI
 
         public CatcherSprite(CatcherAnimationState state)
             : base(new CatchSkinComponent(componentFromState(state)), _ =>
-                new DefaultCatcherSprite(state), confineMode: ConfineMode.ScaleDownToFit)
+                new DefaultCatcherSprite(state), confineMode: ConfineMode.ScaleToFit)
         {
             RelativeSizeAxes = Axes.None;
             Size = new Vector2(CatcherArea.CATCHER_SIZE);
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneSkinnableDrawable.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneSkinnableDrawable.cs
index ec94053679..3b91243fee 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneSkinnableDrawable.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneSkinnableDrawable.cs
@@ -43,16 +43,15 @@ namespace osu.Game.Tests.Visual.Gameplay
                         {
                             new ExposedSkinnableDrawable("default", _ => new DefaultBox(), _ => true),
                             new ExposedSkinnableDrawable("available", _ => new DefaultBox(), _ => true),
-                            new ExposedSkinnableDrawable("available", _ => new DefaultBox(), _ => true, ConfineMode.ScaleToFit),
                             new ExposedSkinnableDrawable("available", _ => new DefaultBox(), _ => true, ConfineMode.NoScaling)
                         }
                     },
                 };
             });
 
-            AddAssert("check sizes", () => fill.Children.Select(c => c.Drawable.DrawWidth).SequenceEqual(new float[] { 30, 30, 30, 50 }));
+            AddAssert("check sizes", () => fill.Children.Select(c => c.Drawable.DrawWidth).SequenceEqual(new float[] { 30, 30, 50 }));
             AddStep("adjust scale", () => fill.Scale = new Vector2(2));
-            AddAssert("check sizes unchanged by scale", () => fill.Children.Select(c => c.Drawable.DrawWidth).SequenceEqual(new float[] { 30, 30, 30, 50 }));
+            AddAssert("check sizes unchanged by scale", () => fill.Children.Select(c => c.Drawable.DrawWidth).SequenceEqual(new float[] { 30, 30, 50 }));
         }
 
         [Test]
@@ -74,7 +73,6 @@ namespace osu.Game.Tests.Visual.Gameplay
                         Children = new[]
                         {
                             new ExposedSkinnableDrawable("default", _ => new DefaultBox(), _ => true),
-                            new ExposedSkinnableDrawable("available", _ => new DefaultBox(), _ => true),
                             new ExposedSkinnableDrawable("available", _ => new DefaultBox(), _ => true, ConfineMode.ScaleToFit),
                             new ExposedSkinnableDrawable("available", _ => new DefaultBox(), _ => true, ConfineMode.NoScaling)
                         }
@@ -82,9 +80,9 @@ namespace osu.Game.Tests.Visual.Gameplay
                 };
             });
 
-            AddAssert("check sizes", () => fill.Children.Select(c => c.Drawable.DrawWidth).SequenceEqual(new float[] { 50, 30, 50, 30 }));
+            AddAssert("check sizes", () => fill.Children.Select(c => c.Drawable.DrawWidth).SequenceEqual(new float[] { 50, 50, 30 }));
             AddStep("adjust scale", () => fill.Scale = new Vector2(2));
-            AddAssert("check sizes unchanged by scale", () => fill.Children.Select(c => c.Drawable.DrawWidth).SequenceEqual(new float[] { 50, 30, 50, 30 }));
+            AddAssert("check sizes unchanged by scale", () => fill.Children.Select(c => c.Drawable.DrawWidth).SequenceEqual(new float[] { 50, 50, 30 }));
         }
 
         [Test]
@@ -182,7 +180,7 @@ namespace osu.Game.Tests.Visual.Gameplay
             public new Drawable Drawable => base.Drawable;
 
             public ExposedSkinnableDrawable(string name, Func<ISkinComponent, Drawable> defaultImplementation, Func<ISkinSource, bool> allowFallback = null,
-                                            ConfineMode confineMode = ConfineMode.ScaleDownToFit)
+                                            ConfineMode confineMode = ConfineMode.ScaleToFit)
                 : base(new TestSkinComponent(name), defaultImplementation, allowFallback, confineMode)
             {
             }
diff --git a/osu.Game/Skinning/SkinnableDrawable.cs b/osu.Game/Skinning/SkinnableDrawable.cs
index fda031e6cb..68a7a8c159 100644
--- a/osu.Game/Skinning/SkinnableDrawable.cs
+++ b/osu.Game/Skinning/SkinnableDrawable.cs
@@ -92,20 +92,13 @@ namespace osu.Game.Skinning
 
                     switch (confineMode)
                     {
-                        case ConfineMode.NoScaling:
-                            return;
-
-                        case ConfineMode.ScaleDownToFit:
-                            if (Drawable.DrawSize.X <= DrawSize.X && Drawable.DrawSize.Y <= DrawSize.Y)
-                                return;
-
+                        case ConfineMode.ScaleToFit:
+                            Drawable.RelativeSizeAxes = Axes.Both;
+                            Drawable.Size = Vector2.One;
+                            Drawable.Scale = Vector2.One;
+                            Drawable.FillMode = FillMode.Fit;
                             break;
                     }
-
-                    Drawable.RelativeSizeAxes = Axes.Both;
-                    Drawable.Size = Vector2.One;
-                    Drawable.Scale = Vector2.One;
-                    Drawable.FillMode = FillMode.Fit;
                 }
                 finally
                 {
@@ -121,7 +114,6 @@ namespace osu.Game.Skinning
         /// Don't apply any scaling. This allows the user element to be of any size, exceeding specified bounds.
         /// </summary>
         NoScaling,
-        ScaleDownToFit,
         ScaleToFit,
     }
 }