diff --git a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableSwell.cs b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableSwell.cs
index 1f571544d8..df36a475d6 100644
--- a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableSwell.cs
+++ b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableSwell.cs
@@ -34,7 +34,6 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
         /// </summary>
         private int userHits;
 
-        private bool hasProxied;
         private readonly SwellSymbolPiece symbol;
 
         public DrawableSwell(Swell swell)
@@ -172,7 +171,6 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
             switch (state)
             {
                 case ArmedState.Idle:
-                    hasProxied = false;
                     UnproxyContent();
                     break;
                 case ArmedState.Hit:
@@ -193,11 +191,8 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
             X = Math.Max(0, X);
 
             double t = Math.Min(HitObject.StartTime, Time.Current);
-            if (t == HitObject.StartTime && !hasProxied)
-            {
+            if (t == HitObject.StartTime)
                 ProxyContent();
-                hasProxied = true;
-            }
         }
 
         private bool? lastWasCentre;
diff --git a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableTaikoHitObject.cs b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableTaikoHitObject.cs
index e712d62a26..a6d61f1a5a 100644
--- a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableTaikoHitObject.cs
+++ b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableTaikoHitObject.cs
@@ -40,23 +40,31 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
         /// </summary>
         protected override bool ComputeIsMaskedAway(RectangleF maskingBounds) => false;
 
+        private bool isProxied;
+
         /// <summary>
         /// Moves <see cref="Content"/> to a layer proxied above the playfield.
+        /// Does nothing is content is already proxied.
         /// </summary>
         protected void ProxyContent()
         {
+            if (isProxied) return;
+            isProxied = true;
+
             nonProxiedContent.Remove(Content);
-            proxiedContent.Remove(Content);
             proxiedContent.Add(Content);
         }
 
         /// <summary>
         /// Moves <see cref="Content"/> to the normal hitobject layer.
+        /// Does nothing is content is not currently proxied.
         /// </summary>
         protected void UnproxyContent()
         {
+            if (!isProxied) return;
+            isProxied = false;
+
             proxiedContent.Remove(Content);
-            nonProxiedContent.Remove(Content);
             nonProxiedContent.Add(Content);
         }