Merge branch 'master' into skin-editor-depth-changing

osu.Game.Rulesets.Osu/Mods/InputBlockingMod.cs

@@ -11,6 +11,7 @@ using osu.Game.Beatmaps.Timing;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Objects;
 using osu.Game.Rulesets.Osu.Objects;
+using osu.Game.Rulesets.Osu.UI;
 using osu.Game.Rulesets.Scoring;
 using osu.Game.Rulesets.UI;
 using osu.Game.Screens.Play;
@@ -26,7 +27,7 @@ namespace osu.Game.Rulesets.Osu.Mods
 
         private const double flash_duration = 1000;
 
-        private DrawableRuleset<OsuHitObject> ruleset = null!;
+        private DrawableOsuRuleset ruleset = null!;
 
         protected OsuAction? LastAcceptedAction { get; private set; }
 
@@ -42,8 +43,8 @@ namespace osu.Game.Rulesets.Osu.Mods
 
         public void ApplyToDrawableRuleset(DrawableRuleset<OsuHitObject> drawableRuleset)
         {
-            ruleset = drawableRuleset;
+            ruleset = (DrawableOsuRuleset)drawableRuleset;
-            drawableRuleset.KeyBindingInputManager.Add(new InputInterceptor(this));
+            ruleset.KeyBindingInputManager.Add(new InputInterceptor(this));
 
             var periods = new List<Period>();
 

osu.Game.Rulesets.Osu/Mods/OsuModAutopilot.cs

@@ -11,6 +11,7 @@ using osu.Game.Graphics;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Osu.Objects;
 using osu.Game.Rulesets.Osu.Replays;
+using osu.Game.Rulesets.Osu.UI;
 using osu.Game.Rulesets.UI;
 
 namespace osu.Game.Rulesets.Osu.Mods
@@ -55,7 +56,7 @@ namespace osu.Game.Rulesets.Osu.Mods
         public void ApplyToDrawableRuleset(DrawableRuleset<OsuHitObject> drawableRuleset)
         {
             // Grab the input manager to disable the user's cursor, and for future use
-            inputManager = (OsuInputManager)drawableRuleset.KeyBindingInputManager;
+            inputManager = ((DrawableOsuRuleset)drawableRuleset).KeyBindingInputManager;
             inputManager.AllowUserCursorMovement = false;
 
             // Generate the replay frames the cursor should follow

osu.Game.Rulesets.Osu/Mods/OsuModRelax.cs

@@ -10,6 +10,7 @@ using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Objects.Types;
 using osu.Game.Rulesets.Osu.Objects;
 using osu.Game.Rulesets.Osu.Objects.Drawables;
+using osu.Game.Rulesets.Osu.UI;
 using osu.Game.Rulesets.Replays;
 using osu.Game.Rulesets.UI;
 using osu.Game.Screens.Play;
@@ -42,7 +43,7 @@ namespace osu.Game.Rulesets.Osu.Mods
         public void ApplyToDrawableRuleset(DrawableRuleset<OsuHitObject> drawableRuleset)
         {
             // grab the input manager for future use.
-            osuInputManager = (OsuInputManager)drawableRuleset.KeyBindingInputManager;
+            osuInputManager = ((DrawableOsuRuleset)drawableRuleset).KeyBindingInputManager;
         }
 
         public void ApplyToPlayer(Player player)

osu.Game.Rulesets.Osu/UI/DrawableOsuRuleset.cs

@@ -26,6 +26,8 @@ namespace osu.Game.Rulesets.Osu.UI
     {
         protected new OsuRulesetConfigManager Config => (OsuRulesetConfigManager)base.Config;
 
+        public new OsuInputManager KeyBindingInputManager => (OsuInputManager)base.KeyBindingInputManager;
+
         public new OsuPlayfield Playfield => (OsuPlayfield)base.Playfield;
 
         public DrawableOsuRuleset(Ruleset ruleset, IBeatmap beatmap, IReadOnlyList<Mod> mods = null)

osu.Game.Tests/NonVisual/FirstAvailableHitWindowsTest.cs

@@ -7,6 +7,7 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using NUnit.Framework;
+using osu.Framework.Audio;
 using osu.Framework.Graphics.Containers;
 using osu.Game.Rulesets.Judgements;
 using osu.Game.Rulesets.Mods;
@@ -93,6 +94,7 @@ namespace osu.Game.Tests.NonVisual
                 remove => throw new InvalidOperationException($"{nameof(RevertResult)} operations not supported in test context");
             }
 
+            public override IAdjustableAudioComponent Audio { get; }
             public override Playfield Playfield { get; }
             public override Container Overlays { get; }
             public override Container FrameStableComponents { get; }

osu.Game.Tests/Visual/Gameplay/TestSceneGameplaySampleTriggerSource.cs

@@ -7,16 +7,20 @@ using NUnit.Framework;
 using osu.Framework.Allocation;
 using osu.Framework.Audio;
 using osu.Framework.Timing;
+using osu.Framework.Utils;
 using osu.Game.Audio;
 using osu.Game.Beatmaps;
 using osu.Game.Beatmaps.ControlPoints;
+using osu.Game.Beatmaps.Legacy;
 using osu.Game.Rulesets;
 using osu.Game.Rulesets.Objects;
 using osu.Game.Rulesets.Objects.Drawables;
+using osu.Game.Rulesets.Objects.Types;
 using osu.Game.Rulesets.Osu;
 using osu.Game.Rulesets.Osu.Objects;
 using osu.Game.Rulesets.UI;
 using osu.Game.Storyboards;
+using osuTK;
 using osuTK.Input;
 
 namespace osu.Game.Tests.Visual.Gameplay
@@ -36,13 +40,16 @@ namespace osu.Game.Tests.Visual.Gameplay
 
         protected override IBeatmap CreateBeatmap(RulesetInfo ruleset)
         {
+            ControlPointInfo controlPointInfo = new LegacyControlPointInfo();
+
             beatmap = new Beatmap
             {
                 BeatmapInfo = new BeatmapInfo
                 {
                     Difficulty = new BeatmapDifficulty { CircleSize = 6, SliderMultiplier = 3 },
                     Ruleset = ruleset
-                }
+                },
+                ControlPointInfo = controlPointInfo
             };
 
             const double start_offset = 8000;
@@ -51,7 +58,7 @@ namespace osu.Game.Tests.Visual.Gameplay
             // intentionally start objects a bit late so we can test the case of no alive objects.
             double t = start_offset;
 
-            beatmap.HitObjects.AddRange(new[]
+            beatmap.HitObjects.AddRange(new HitObject[]
             {
                 new HitCircle
                 {
@@ -71,12 +78,24 @@ namespace osu.Game.Tests.Visual.Gameplay
                 },
                 new HitCircle
                 {
-                    StartTime = t + spacing,
+                    StartTime = t += spacing,
+                },
+                new Slider
+                {
+                    StartTime = t += spacing,
+                    Path = new SliderPath(PathType.Linear, new[] { Vector2.Zero, Vector2.UnitY * 200 }),
                     Samples = new[] { new HitSampleInfo(HitSampleInfo.HIT_WHISTLE) },
                     SampleControlPoint = new SampleControlPoint { SampleBank = "soft" },
                 },
             });
 
+            // Add a change in volume halfway through final slider.
+            controlPointInfo.Add(t, new SampleControlPoint
+            {
+                SampleBank = "normal",
+                SampleVolume = 20,
+            });
+
             return beatmap;
         }
 
@@ -129,14 +148,36 @@ namespace osu.Game.Tests.Visual.Gameplay
             waitForAliveObjectIndex(3);
             checkValidObjectIndex(3);
 
-            AddStep("Seek into future", () => Beatmap.Value.Track.Seek(beatmap.HitObjects.Last().GetEndTime() + 10000));
+            seekBeforeIndex(4);
+            waitForAliveObjectIndex(4);
 
+            // Even before the object, we should prefer the first nested object's sample.
+            // This is because the (parent) object will only play its sample at the final EndTime.
+            AddAssert("check valid object is slider's first nested", () => sampleTriggerSource.GetMostValidObject(), () => Is.EqualTo(beatmap.HitObjects[4].NestedHitObjects.First()));
+
+            AddStep("seek to just before slider ends", () => Player.GameplayClockContainer.Seek(beatmap.HitObjects[4].GetEndTime() - 100));
+            waitForCatchUp();
+            AddUntilStep("wait until valid object is slider's last nested", () => sampleTriggerSource.GetMostValidObject(), () => Is.EqualTo(beatmap.HitObjects[4].NestedHitObjects.Last()));
+
+            // After we get far enough away, the samples of the object itself should be used, not any nested object.
+            AddStep("seek to further after slider", () => Player.GameplayClockContainer.Seek(beatmap.HitObjects[4].GetEndTime() + 1000));
+            waitForCatchUp();
+            AddUntilStep("wait until valid object is slider itself", () => sampleTriggerSource.GetMostValidObject(), () => Is.EqualTo(beatmap.HitObjects[4]));
+
+            AddStep("Seek into future", () => Player.GameplayClockContainer.Seek(beatmap.HitObjects.Last().GetEndTime() + 10000));
+            waitForCatchUp();
             waitForAliveObjectIndex(null);
-            checkValidObjectIndex(3);
+            checkValidObjectIndex(4);
         }
 
-        private void seekBeforeIndex(int index) =>
+        private void seekBeforeIndex(int index)
-            AddStep($"seek to just before object {index}", () => Beatmap.Value.Track.Seek(beatmap.HitObjects[index].StartTime - 100));
+        {
+            AddStep($"seek to just before object {index}", () => Player.GameplayClockContainer.Seek(beatmap.HitObjects[index].StartTime - 100));
+            waitForCatchUp();
+        }
+
+        private void waitForCatchUp() =>
+            AddUntilStep("wait for frame stable clock to catch up", () => Precision.AlmostEquals(Player.GameplayClockContainer.CurrentTime, Player.DrawableRuleset.FrameStableClock.CurrentTime));
 
         private void waitForAliveObjectIndex(int? index)
         {

osu.Game.Tests/Visual/Gameplay/TestSceneHitErrorMeter.cs

@@ -9,6 +9,7 @@ using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using NUnit.Framework;
 using osu.Framework.Allocation;
+using osu.Framework.Audio;
 using osu.Framework.Extensions.IEnumerableExtensions;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
@@ -281,6 +282,7 @@ namespace osu.Game.Tests.Visual.Gameplay
                 remove => throw new InvalidOperationException($"{nameof(RevertResult)} operations not supported in test context");
             }
 
+            public override IAdjustableAudioComponent Audio { get; }
             public override Playfield Playfield { get; }
             public override Container Overlays { get; }
             public override Container FrameStableComponents { get; }

osu.Game/Rulesets/Mods/ModFlashlight.cs

@@ -82,7 +82,7 @@ namespace osu.Game.Rulesets.Mods
             flashlight.Colour = Color4.Black;
 
             flashlight.Combo.BindTo(Combo);
-            drawableRuleset.KeyBindingInputManager.Add(flashlight);
+            drawableRuleset.Overlays.Add(flashlight);
         }
 
         protected abstract Flashlight CreateFlashlight();

osu.Game/Rulesets/Mods/ModMuted.cs

@@ -67,7 +67,8 @@ namespace osu.Game.Rulesets.Mods
             {
                 MetronomeBeat metronomeBeat;
 
-                drawableRuleset.Overlays.Add(metronomeBeat = new MetronomeBeat(drawableRuleset.Beatmap.HitObjects.First().StartTime));
+                // Importantly, this is added to FrameStableComponents and not Overlays as the latter would cause it to be self-muted by the mod's volume adjustment.
+                drawableRuleset.FrameStableComponents.Add(metronomeBeat = new MetronomeBeat(drawableRuleset.Beatmap.HitObjects.First().StartTime));
 
                 metronomeBeat.AddAdjustment(AdjustableProperty.Volume, metronomeVolumeAdjust);
             }

osu.Game/Rulesets/UI/DrawableRuleset.cs

@@ -53,7 +53,7 @@ namespace osu.Game.Rulesets.UI
         /// <summary>
         /// The key conversion input manager for this DrawableRuleset.
         /// </summary>
-        public PassThroughInputManager KeyBindingInputManager;
+        protected PassThroughInputManager KeyBindingInputManager;
 
         public override double GameplayStartTime => Objects.FirstOrDefault()?.StartTime - 2000 ?? 0;
 
@@ -66,6 +66,10 @@ namespace osu.Game.Rulesets.UI
 
         public override Container Overlays { get; } = new Container { RelativeSizeAxes = Axes.Both };
 
+        public override IAdjustableAudioComponent Audio => audioContainer;
+
+        private readonly AudioContainer audioContainer = new AudioContainer { RelativeSizeAxes = Axes.Both };
+
         public override Container FrameStableComponents { get; } = new Container { RelativeSizeAxes = Axes.Both };
 
         public override IFrameStableClock FrameStableClock => frameStabilityContainer;
@@ -102,14 +106,6 @@ namespace osu.Game.Rulesets.UI
 
         private DrawableRulesetDependencies dependencies;
 
-        /// <summary>
-        /// Audio adjustments which are applied to the playfield.
-        /// </summary>
-        /// <remarks>
-        /// Does not affect <see cref="Overlays"/>.
-        /// </remarks>
-        public IAdjustableAudioComponent Audio { get; private set; }
-
         /// <summary>
         /// Creates a ruleset visualisation for the provided ruleset and beatmap.
         /// </summary>
@@ -172,28 +168,22 @@ namespace osu.Game.Rulesets.UI
         [BackgroundDependencyLoader]
         private void load(CancellationToken? cancellationToken)
         {
-            AudioContainer audioContainer;
-
             InternalChild = frameStabilityContainer = new FrameStabilityContainer(GameplayStartTime)
             {
                 FrameStablePlayback = FrameStablePlayback,
                 Children = new Drawable[]
                 {
                     FrameStableComponents,
-                    audioContainer = new AudioContainer
+                    audioContainer.WithChild(KeyBindingInputManager
-                    {
+                        .WithChildren(new Drawable[]
-                        RelativeSizeAxes = Axes.Both,
+                        {
-                        Child = KeyBindingInputManager
+                            CreatePlayfieldAdjustmentContainer()
-                            .WithChild(CreatePlayfieldAdjustmentContainer()
+                                .WithChild(Playfield),
-                                .WithChild(Playfield)
+                            Overlays
-                            ),
+                        })),
-                    },
-                    Overlays,
                 }
             };
 
-            Audio = audioContainer;
-
             if ((ResumeOverlay = CreateResumeOverlay()) != null)
             {
                 AddInternal(CreateInputManager()
@@ -436,13 +426,18 @@ namespace osu.Game.Rulesets.UI
         /// </summary>
         public readonly BindableBool IsPaused = new BindableBool();
 
+        /// <summary>
+        /// Audio adjustments which are applied to the playfield.
+        /// </summary>
+        public abstract IAdjustableAudioComponent Audio { get; }
+
         /// <summary>
         /// The playfield.
         /// </summary>
         public abstract Playfield Playfield { get; }
 
         /// <summary>
-        /// Content to be placed above hitobjects. Will be affected by frame stability.
+        /// Content to be placed above hitobjects. Will be affected by frame stability and adjustments applied to <see cref="Audio"/>.
         /// </summary>
         public abstract Container Overlays { get; }
 

osu.Game/Rulesets/UI/GameplaySampleTriggerSource.cs

@@ -7,6 +7,7 @@ using System.Linq;
 using osu.Framework.Graphics.Containers;
 using osu.Game.Audio;
 using osu.Game.Rulesets.Objects;
+using osu.Game.Rulesets.Objects.Drawables;
 using osu.Game.Skinning;
 
 namespace osu.Game.Rulesets.UI
@@ -68,27 +69,61 @@ namespace osu.Game.Rulesets.UI
         protected HitObject GetMostValidObject()
         {
             // The most optimal lookup case we have is when an object is alive. There are usually very few alive objects so there's no drawbacks in attempting this lookup each time.
-            var hitObject = hitObjectContainer.AliveObjects.FirstOrDefault(h => h.Result?.HasResult != true)?.HitObject;
+            var drawableHitObject = hitObjectContainer.AliveObjects.FirstOrDefault(h => h.Result?.HasResult != true);
 
-            // In the case a next object isn't available in drawable form, we need to do a somewhat expensive traversal to get a valid sound to play.
+            if (drawableHitObject != null)
-            if (hitObject == null)
             {
-                // This lookup can be skipped if the last entry is still valid (in the future and not yet hit).
+                // A hit object may have a more valid nested object.
-                if (fallbackObject == null || fallbackObject.Result?.HasResult == true)
+                drawableHitObject = getMostValidNestedDrawable(drawableHitObject);
-                {
-                    // We need to use lifetime entries to find the next object (we can't just use `hitObjectContainer.Objects` due to pooling - it may even be empty).
-                    // If required, we can make this lookup more efficient by adding support to get next-future-entry in LifetimeEntryManager.
-                    fallbackObject = hitObjectContainer.Entries
-                                                       .Where(e => e.Result?.HasResult != true).MinBy(e => e.HitObject.StartTime);
 
-                    // In the case there are no unjudged objects, the last hit object should be used instead.
+                return drawableHitObject.HitObject;
-                    fallbackObject ??= hitObjectContainer.Entries.LastOrDefault();
-                }
-
-                hitObject = fallbackObject?.HitObject;
             }
 
-            return hitObject;
+            // In the case a next object isn't available in drawable form, we need to do a somewhat expensive traversal to get a valid sound to play.
+            // This lookup can be skipped if the last entry is still valid (in the future and not yet hit).
+            if (fallbackObject == null || fallbackObject.Result?.HasResult == true)
+            {
+                // We need to use lifetime entries to find the next object (we can't just use `hitObjectContainer.Objects` due to pooling - it may even be empty).
+                // If required, we can make this lookup more efficient by adding support to get next-future-entry in LifetimeEntryManager.
+                fallbackObject = hitObjectContainer.Entries
+                                                   .Where(e => e.Result?.HasResult != true).MinBy(e => e.HitObject.StartTime);
+
+                if (fallbackObject != null)
+                    return getEarliestNestedObject(fallbackObject.HitObject);
+
+                // In the case there are no non-judged objects, the last hit object should be used instead.
+                fallbackObject ??= hitObjectContainer.Entries.LastOrDefault();
+            }
+
+            if (fallbackObject == null)
+                return null;
+
+            bool fallbackHasResult = fallbackObject.Result?.HasResult == true;
+
+            // If the fallback has been judged then we want the sample from the object itself.
+            if (fallbackHasResult)
+                return fallbackObject.HitObject;
+
+            // Else we want the earliest (including nested).
+            // In cases of nested objects, they will always have earlier sample data than their parent object.
+            return getEarliestNestedObject(fallbackObject.HitObject);
+        }
+
+        private DrawableHitObject getMostValidNestedDrawable(DrawableHitObject o)
+        {
+            var nestedWithoutResult = o.NestedHitObjects.FirstOrDefault(n => n.Result?.HasResult != true);
+
+            if (nestedWithoutResult == null)
+                return o;
+
+            return getMostValidNestedDrawable(nestedWithoutResult);
+        }
+
+        private HitObject getEarliestNestedObject(HitObject hitObject)
+        {
+            var nested = hitObject.NestedHitObjects.FirstOrDefault();
+
+            return nested != null ? getEarliestNestedObject(nested) : hitObject;
         }
 
         private SkinnableSound getNextSample()