Merge pull request #22375 from peppy/osu-ruleset-touch-support

Add empty space tap-streaming support for osu! ruleset on touchscreen devices

osu.Game.Rulesets.Osu.Tests/TestSceneOsuTouchInput.cs

@@ -13,7 +13,14 @@ using osu.Framework.Input.Bindings;
 using osu.Framework.Input.Events;
 using osu.Framework.Input.States;
 using osu.Framework.Testing;
+using osu.Framework.Timing;
+using osu.Framework.Utils;
+using osu.Game.Beatmaps;
+using osu.Game.Beatmaps.ControlPoints;
 using osu.Game.Configuration;
+using osu.Game.Rulesets.Osu.Objects;
+using osu.Game.Rulesets.Osu.Objects.Drawables;
+using osu.Game.Rulesets.Osu.UI.Cursor;
 using osu.Game.Screens.Play;
 using osu.Game.Tests.Visual;
 using osuTK;
@@ -22,7 +29,7 @@ using osuTK.Graphics;
 namespace osu.Game.Rulesets.Osu.Tests
 {
     [TestFixture]
-    public partial class TestSceneTouchInput : OsuManualInputManagerTestScene
+    public partial class TestSceneOsuTouchInput : OsuManualInputManagerTestScene
     {
         [Resolved]
         private OsuConfigManager config { get; set; } = null!;
@@ -33,6 +40,8 @@ namespace osu.Game.Rulesets.Osu.Tests
 
         private OsuInputManager osuInputManager = null!;
 
+        private Container mainContent = null!;
+
         [SetUpSteps]
         public void SetUpSteps()
         {
@@ -44,7 +53,7 @@ namespace osu.Game.Rulesets.Osu.Tests
                 {
                     osuInputManager = new OsuInputManager(new OsuRuleset().RulesetInfo)
                     {
-                        Child = new Container
+                        Child = mainContent = new Container
                         {
                             Anchor = Anchor.Centre,
                             Origin = Anchor.Centre,
@@ -54,13 +63,19 @@ namespace osu.Game.Rulesets.Osu.Tests
                                 {
                                     Anchor = Anchor.Centre,
                                     Origin = Anchor.CentreRight,
+                                    Depth = float.MinValue,
                                     X = -100,
                                 },
                                 rightKeyCounter = new TestActionKeyCounter(OsuAction.RightButton)
                                 {
                                     Anchor = Anchor.Centre,
                                     Origin = Anchor.CentreLeft,
+                                    Depth = float.MinValue,
                                     X = 100,
+                                },
+                                new OsuCursorContainer
+                                {
+                                    Depth = float.MinValue,
                                 }
                             },
                         }
@@ -70,6 +85,40 @@ namespace osu.Game.Rulesets.Osu.Tests
             });
         }
 
+        [Test]
+        public void TestStreamInputVisual()
+        {
+            addHitCircleAt(TouchSource.Touch1);
+            addHitCircleAt(TouchSource.Touch2);
+
+            beginTouch(TouchSource.Touch1);
+            beginTouch(TouchSource.Touch2);
+
+            endTouch(TouchSource.Touch1);
+
+            int i = 0;
+
+            AddRepeatStep("Alternate", () =>
+            {
+                TouchSource down = i % 2 == 0 ? TouchSource.Touch3 : TouchSource.Touch4;
+                TouchSource up = i % 2 == 0 ? TouchSource.Touch4 : TouchSource.Touch3;
+
+                // sometimes the user will end the previous touch before touching again, sometimes not.
+                if (RNG.NextBool())
+                {
+                    InputManager.BeginTouch(new Touch(down, getSanePositionForSource(down)));
+                    InputManager.EndTouch(new Touch(up, getSanePositionForSource(up)));
+                }
+                else
+                {
+                    InputManager.EndTouch(new Touch(up, getSanePositionForSource(up)));
+                    InputManager.BeginTouch(new Touch(down, getSanePositionForSource(down)));
+                }
+
+                i++;
+            }, 100);
+        }
+
         [Test]
         public void TestSimpleInput()
         {
@@ -116,9 +165,224 @@ namespace osu.Game.Rulesets.Osu.Tests
             endTouch(TouchSource.Touch2);
             checkPosition(TouchSource.Touch2);
 
-            // note that touch1 was never ended, but becomes active for tracking again.
+            // note that touch1 was never ended, but is no longer valid for touch input due to touch 2 occurring.
             beginTouch(TouchSource.Touch1);
+            checkPosition(TouchSource.Touch2);
+        }
+
+        [Test]
+        public void TestStreamInput()
+        {
+            // In this scenario, the user is tapping on the first object in a stream,
+            // then using one or two fingers in empty space to continue the stream.
+
+            addHitCircleAt(TouchSource.Touch1);
+            beginTouch(TouchSource.Touch1);
+
+            // The first touch is handled as normal.
+            assertKeyCounter(1, 0);
+            checkPressed(OsuAction.LeftButton);
             checkPosition(TouchSource.Touch1);
+
+            // The second touch should release the first, and also act as a right button.
+            beginTouch(TouchSource.Touch2);
+
+            assertKeyCounter(1, 1);
+            // Importantly, this is different from the simple case because an object was interacted with in the first touch, but not the second touch.
+            // left button is automatically released.
+            checkNotPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            // Also importantly, the positional part of the second touch is ignored.
+            checkPosition(TouchSource.Touch1);
+
+            // In this scenario, a third touch should be allowed, and handled similarly to the second.
+            beginTouch(TouchSource.Touch3);
+
+            assertKeyCounter(2, 1);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            // Position is still ignored.
+            checkPosition(TouchSource.Touch1);
+
+            endTouch(TouchSource.Touch2);
+
+            checkPressed(OsuAction.LeftButton);
+            checkNotPressed(OsuAction.RightButton);
+            // Position is still ignored.
+            checkPosition(TouchSource.Touch1);
+
+            // User continues streaming
+            beginTouch(TouchSource.Touch2);
+
+            assertKeyCounter(2, 2);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            // Position is still ignored.
+            checkPosition(TouchSource.Touch1);
+
+            // In this mode a maximum of three touches should be supported.
+            // A fourth touch should result in no changes anywhere.
+            beginTouch(TouchSource.Touch4);
+            assertKeyCounter(2, 2);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch1);
+            endTouch(TouchSource.Touch4);
+        }
+
+        [Test]
+        public void TestStreamInputWithInitialTouchDownLeft()
+        {
+            // In this scenario, the user is wanting to use stream input but we start with one finger still on the screen.
+            // That finger is mapped to a left action.
+
+            addHitCircleAt(TouchSource.Touch2);
+
+            beginTouch(TouchSource.Touch1);
+            assertKeyCounter(1, 0);
+            checkPressed(OsuAction.LeftButton);
+            checkPosition(TouchSource.Touch1);
+
+            // hits circle as right action
+            beginTouch(TouchSource.Touch2);
+            assertKeyCounter(1, 1);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch2);
+
+            endTouch(TouchSource.Touch1);
+            checkNotPressed(OsuAction.LeftButton);
+
+            // stream using other two fingers while touch2 tracks
+            beginTouch(TouchSource.Touch1);
+            assertKeyCounter(2, 1);
+            checkPressed(OsuAction.LeftButton);
+            // right button is automatically released
+            checkNotPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch2);
+
+            beginTouch(TouchSource.Touch3);
+            assertKeyCounter(2, 2);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch2);
+
+            endTouch(TouchSource.Touch1);
+            checkNotPressed(OsuAction.LeftButton);
+
+            beginTouch(TouchSource.Touch1);
+            assertKeyCounter(3, 2);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch2);
+        }
+
+        [Test]
+        public void TestStreamInputWithInitialTouchDownRight()
+        {
+            // In this scenario, the user is wanting to use stream input but we start with one finger still on the screen.
+            // That finger is mapped to a right action.
+
+            beginTouch(TouchSource.Touch1);
+            beginTouch(TouchSource.Touch2);
+
+            assertKeyCounter(1, 1);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+
+            endTouch(TouchSource.Touch1);
+
+            addHitCircleAt(TouchSource.Touch1);
+
+            // hits circle as left action
+            beginTouch(TouchSource.Touch1);
+            assertKeyCounter(2, 1);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch1);
+
+            endTouch(TouchSource.Touch2);
+
+            // stream using other two fingers while touch1 tracks
+            beginTouch(TouchSource.Touch2);
+            assertKeyCounter(2, 2);
+            checkPressed(OsuAction.RightButton);
+            // left button is automatically released
+            checkNotPressed(OsuAction.LeftButton);
+            checkPosition(TouchSource.Touch1);
+
+            beginTouch(TouchSource.Touch3);
+            assertKeyCounter(3, 2);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch1);
+
+            endTouch(TouchSource.Touch2);
+            checkNotPressed(OsuAction.RightButton);
+
+            beginTouch(TouchSource.Touch2);
+            assertKeyCounter(3, 3);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch1);
+        }
+
+        [Test]
+        public void TestNonStreamOverlappingDirectTouchesWithRelease()
+        {
+            // In this scenario, the user is tapping on three circles directly while correctly releasing the first touch.
+            // All three should be recognised.
+
+            addHitCircleAt(TouchSource.Touch1);
+            addHitCircleAt(TouchSource.Touch2);
+            addHitCircleAt(TouchSource.Touch3);
+
+            beginTouch(TouchSource.Touch1);
+            assertKeyCounter(1, 0);
+            checkPressed(OsuAction.LeftButton);
+            checkPosition(TouchSource.Touch1);
+
+            beginTouch(TouchSource.Touch2);
+            assertKeyCounter(1, 1);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch2);
+
+            endTouch(TouchSource.Touch1);
+
+            beginTouch(TouchSource.Touch3);
+            assertKeyCounter(2, 1);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch3);
+        }
+
+        [Test]
+        public void TestNonStreamOverlappingDirectTouchesWithoutRelease()
+        {
+            // In this scenario, the user is tapping on three circles directly without releasing any touches.
+            // The first two should be recognised, but a third should not (as the user already has two fingers down).
+
+            addHitCircleAt(TouchSource.Touch1);
+            addHitCircleAt(TouchSource.Touch2);
+            addHitCircleAt(TouchSource.Touch3);
+
+            beginTouch(TouchSource.Touch1);
+            assertKeyCounter(1, 0);
+            checkPressed(OsuAction.LeftButton);
+            checkPosition(TouchSource.Touch1);
+
+            beginTouch(TouchSource.Touch2);
+            assertKeyCounter(1, 1);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch2);
+
+            beginTouch(TouchSource.Touch3);
+            assertKeyCounter(1, 1);
+            checkPressed(OsuAction.LeftButton);
+            checkPressed(OsuAction.RightButton);
+            checkPosition(TouchSource.Touch3);
         }
 
         [Test]
@@ -263,6 +527,22 @@ namespace osu.Game.Rulesets.Osu.Tests
             assertKeyCounter(1, 1);
         }
 
+        private void addHitCircleAt(TouchSource source)
+        {
+            AddStep($"Add circle at {source}", () =>
+            {
+                var hitCircle = new HitCircle();
+
+                hitCircle.ApplyDefaults(new ControlPointInfo(), new BeatmapDifficulty());
+
+                mainContent.Add(new DrawableHitCircle(hitCircle)
+                {
+                    Clock = new FramedClock(new ManualClock()),
+                    Position = mainContent.ToLocalSpace(getSanePositionForSource(source)),
+                });
+            });
+        }
+
         private void beginTouch(TouchSource source, Vector2? screenSpacePosition = null) =>
             AddStep($"Begin touch for {source}", () => InputManager.BeginTouch(new Touch(source, screenSpacePosition ??= getSanePositionForSource(source))));
 

osu.Game.Rulesets.Osu/OsuInputManager.cs

@@ -9,8 +9,10 @@ using osu.Framework.Graphics;
 using osu.Framework.Input.Bindings;
 using osu.Framework.Input.Events;
 using osu.Game.Input.Bindings;
+using osu.Game.Rulesets.Osu.Objects.Drawables;
 using osu.Game.Rulesets.Osu.UI;
 using osu.Game.Rulesets.UI;
+using osuTK;
 
 namespace osu.Game.Rulesets.Osu
 {
@@ -40,6 +42,13 @@ namespace osu.Game.Rulesets.Osu
         protected override KeyBindingContainer<OsuAction> CreateKeyBindingContainer(RulesetInfo ruleset, int variant, SimultaneousBindingMode unique)
             => new OsuKeyBindingContainer(ruleset, variant, unique);
 
+        public bool CheckScreenSpaceActionPressJudgeable(Vector2 screenSpacePosition) =>
+            // This is a very naive but simple approach.
+            //
+            // Based on user feedback of more nuanced scenarios (where touch doesn't behave as expected),
+            // this can be expanded to a more complex implementation, but I'd still want to keep it as simple as we can.
+            NonPositionalInputQueue.OfType<DrawableHitCircle.HitReceptor>().Any(c => c.ReceivePositionalInputAt(screenSpacePosition));
+
         public OsuInputManager(RulesetInfo ruleset)
             : base(ruleset, 0, SimultaneousBindingMode.Unique)
         {

osu.Game.Rulesets.Osu/UI/OsuTouchInputMapper.cs

@@ -22,6 +22,8 @@ namespace osu.Game.Rulesets.Osu.UI
         /// </summary>
         private readonly List<TrackedTouch> trackedTouches = new List<TrackedTouch>();
 
+        private TrackedTouch? positionTrackingTouch;
+
         private readonly OsuInputManager osuInputManager;
 
         private Bindable<bool> mouseDisabled = null!;
@@ -57,7 +59,15 @@ namespace osu.Game.Rulesets.Osu.UI
             // Ignore any taps which trigger an action which is already handled. But track them for potential positional input in the future.
             bool shouldResultInAction = osuInputManager.AllowGameplayInputs && !mouseDisabled.Value && trackedTouches.All(t => t.Action != action);
 
-            trackedTouches.Add(new TrackedTouch(e.Touch.Source, shouldResultInAction ? action : null));
+            // If we can actually accept as an action, check whether this tap was on a circle's receptor.
+            // This case gets special handling to allow for empty-space stream tapping.
+            bool isDirectCircleTouch = osuInputManager.CheckScreenSpaceActionPressJudgeable(e.ScreenSpaceTouchDownPosition);
+
+            var newTouch = new TrackedTouch(e.Touch.Source, shouldResultInAction ? action : null, isDirectCircleTouch);
+
+            updatePositionTracking(newTouch);
+
+            trackedTouches.Add(newTouch);
 
             // Important to update position before triggering the pressed action.
             handleTouchMovement(e);
@@ -68,10 +78,47 @@ namespace osu.Game.Rulesets.Osu.UI
             return true;
         }
 
+        /// <summary>
+        /// Given a new touch, update the positional tracking state and any related operations.
+        /// </summary>
+        private void updatePositionTracking(TrackedTouch newTouch)
+        {
+            // If the new touch directly interacted with a circle's receptor, it always becomes the current touch for positional tracking.
+            if (newTouch.DirectTouch)
+            {
+                positionTrackingTouch = newTouch;
+                return;
+            }
+
+            // Otherwise, we only want to use the new touch for position tracking if no other touch is tracking position yet..
+            if (positionTrackingTouch == null)
+            {
+                positionTrackingTouch = newTouch;
+                return;
+            }
+
+            // ..or if the current position tracking touch was not a direct touch (this one is debatable and may be change in the future, but it's the simplest way to handle)
+            if (!positionTrackingTouch.DirectTouch)
+            {
+                positionTrackingTouch = newTouch;
+                return;
+            }
+
+            // In the case the new touch was not used for position tracking, we should also check the previous position tracking touch.
+            // If it was a direct touch and still has its action pressed, that action should be released.
+            //
+            // This is done to allow tracking with the initial touch while still having both Left/Right actions available for alternating with two more touches.
+            if (positionTrackingTouch.DirectTouch && positionTrackingTouch.Action is OsuAction directTouchAction)
+            {
+                osuInputManager.KeyBindingContainer.TriggerReleased(directTouchAction);
+                positionTrackingTouch.Action = null;
+            }
+        }
+
         private void handleTouchMovement(TouchEvent touchEvent)
         {
             // Movement should only be tracked for the most recent touch.
-            if (touchEvent.Touch.Source != trackedTouches.Last().Source)
+            if (touchEvent.Touch.Source != positionTrackingTouch?.Source)
                 return;
 
             if (!osuInputManager.AllowUserCursorMovement)
@@ -87,6 +134,9 @@ namespace osu.Game.Rulesets.Osu.UI
             if (tracked.Action is OsuAction action)
                 osuInputManager.KeyBindingContainer.TriggerReleased(action);
 
+            if (positionTrackingTouch == tracked)
+                positionTrackingTouch = null;
+
             trackedTouches.Remove(tracked);
 
             base.OnTouchUp(e);
@@ -96,12 +146,15 @@ namespace osu.Game.Rulesets.Osu.UI
         {
             public readonly TouchSource Source;
 
-            public readonly OsuAction? Action;
+            public OsuAction? Action;
 
-            public TrackedTouch(TouchSource source, OsuAction? action)
+            public readonly bool DirectTouch;
+
+            public TrackedTouch(TouchSource source, OsuAction? action, bool directTouch)
             {
                 Source = source;
                 Action = action;
+                DirectTouch = directTouch;
             }
         }
     }