diff --git a/osu.Game/Rulesets/Edit/HitObjectComposer.cs b/osu.Game/Rulesets/Edit/HitObjectComposer.cs
index 3c38a7258e..c2a7bec9f9 100644
--- a/osu.Game/Rulesets/Edit/HitObjectComposer.cs
+++ b/osu.Game/Rulesets/Edit/HitObjectComposer.cs
@@ -305,7 +305,9 @@ namespace osu.Game.Rulesets.Edit
                 PlayfieldContentContainer.X = TOOLBOX_CONTRACTED_SIZE_LEFT;
             }
 
-            composerFocusMode.Value = PlayfieldContentContainer.Contains(InputManager.CurrentState.Mouse.Position);
+            composerFocusMode.Value = PlayfieldContentContainer.Contains(InputManager.CurrentState.Mouse.Position)
+                                      && !LeftToolbox.Contains(InputManager.CurrentState.Mouse.Position)
+                                      && !RightToolbox.Contains(InputManager.CurrentState.Mouse.Position);
         }
 
         public override Playfield Playfield => drawableRulesetWrapper.Playfield;