Merge pull request #25315 from peppy/fix-flaky-tests

Address some flaky tests from recent times

osu.Game.Tests/Visual/Gameplay/TestScenePause.cs

@@ -70,7 +70,7 @@ namespace osu.Game.Tests.Visual.Gameplay
         [Test]
         public void TestPauseWithLargeOffset()
         {
-            double lastTime;
+            double lastStopTime;
             bool alwaysGoingForward = true;
 
             AddStep("force large offset", () =>
@@ -84,20 +84,24 @@ namespace osu.Game.Tests.Visual.Gameplay
 
             AddStep("add time forward check hook", () =>
             {
-                lastTime = double.MinValue;
+                lastStopTime = double.MinValue;
                 alwaysGoingForward = true;
 
                 Player.OnUpdate += _ =>
                 {
-                    double currentTime = Player.GameplayClockContainer.CurrentTime;
+                    var masterClock = (MasterGameplayClockContainer)Player.GameplayClockContainer;
-                    bool goingForward = currentTime >= lastTime - 500;
+
+                    double currentTime = masterClock.CurrentTime;
+
+                    bool goingForward = currentTime >= (masterClock.LastStopTime ?? lastStopTime);
 
                     alwaysGoingForward &= goingForward;
 
                     if (!goingForward)
-                        Logger.Log($"Backwards time occurred ({currentTime:N1} -> {lastTime:N1})");
+                        Logger.Log($"Went too far backwards (last stop: {lastStopTime:N1} current: {currentTime:N1})");
 
-                    lastTime = currentTime;
+                    if (masterClock.LastStopTime != null)
+                        lastStopTime = masterClock.LastStopTime.Value;
                 };
             });
 

osu.Game.Tests/Visual/Gameplay/TestScenePlayerLocalScoreImport.cs

@@ -214,10 +214,18 @@ namespace osu.Game.Tests.Visual.Gameplay
 
             // Files starting with _ are temporary, created by CreateFileSafely call.
             AddUntilStep("wait for export file", () => filePath = LocalStorage.GetFiles("exports").SingleOrDefault(f => !Path.GetFileName(f).StartsWith("_", StringComparison.Ordinal)), () => Is.Not.Null);
-            AddAssert("filesize is non-zero", () =>
+            AddUntilStep("filesize is non-zero", () =>
             {
-                using (var stream = LocalStorage.GetStream(filePath))
+                try
-                    return stream.Length;
+                {
+                    using (var stream = LocalStorage.GetStream(filePath))
+                        return stream.Length;
+                }
+                catch (IOException)
+                {
+                    // file move may still be in progress.
+                    return 0;
+                }
             }, () => Is.Not.Zero);
         }
 

osu.Game.Tests/Visual/Gameplay/TestSceneStoryboardSamplePlayback.cs

@@ -41,6 +41,7 @@ namespace osu.Game.Tests.Visual.Gameplay
             backgroundLayer.Add(new StoryboardSampleInfo("Intro/welcome.mp3", time: -7000, volume: 20));
             backgroundLayer.Add(new StoryboardSampleInfo("Intro/welcome.mp3", time: -5000, volume: 20));
             backgroundLayer.Add(new StoryboardSampleInfo("Intro/welcome.mp3", time: 0, volume: 20));
+            backgroundLayer.Add(new StoryboardSampleInfo("Intro/welcome.mp3", time: 2000, volume: 20));
         }
 
         [SetUp]

osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayer.cs

@@ -693,7 +693,9 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [Test]
-        [FlakyTest] // See above
+        [Ignore("Failing too often, needs revisiting in some future.")]
+        // This test is failing even after 10 retries (see https://github.com/ppy/osu/actions/runs/6700910613/job/18208272419)
+        // Something is stopping the ready button from changing states, over multiple runs.
         public void TestGameplayExitFlow()
         {
             Bindable<double>? holdDelay = null;

osu.Game.Tests/Visual/Navigation/TestSceneScreenNavigation.cs

@@ -175,7 +175,7 @@ namespace osu.Game.Tests.Visual.Navigation
             double scrollPosition = 0;
 
             AddStep("set game volume to max", () => Game.Dependencies.Get<FrameworkConfigManager>().SetValue(FrameworkSetting.VolumeUniversal, 1d));
-            AddUntilStep("wait for volume overlay to hide", () => Game.ChildrenOfType<VolumeOverlay>().Single().State.Value, () => Is.EqualTo(Visibility.Hidden));
+            AddUntilStep("wait for volume overlay to hide", () => Game.ChildrenOfType<VolumeOverlay>().SingleOrDefault()?.State.Value, () => Is.EqualTo(Visibility.Hidden));
             PushAndConfirm(() => songSelect = new TestPlaySongSelect());
             AddUntilStep("wait for song select", () => songSelect.BeatmapSetsLoaded);
             AddStep("import beatmap", () => BeatmapImportHelper.LoadQuickOszIntoOsu(Game).WaitSafely());

osu.Game.Tests/Visual/UserInterface/TestSceneModSelectOverlay.cs

@@ -799,8 +799,11 @@ namespace osu.Game.Tests.Visual.UserInterface
             AddUntilStep("difficulty multiplier display shows correct value", () => modSelectOverlay.ChildrenOfType<ScoreMultiplierDisplay>().Single().Current.Value, () => Is.EqualTo(0.7));
         }
 
-        private void waitForColumnLoad() => AddUntilStep("all column content loaded",
+        private void waitForColumnLoad() => AddUntilStep("all column content loaded", () =>
-            () => modSelectOverlay.ChildrenOfType<ModColumn>().Any() && modSelectOverlay.ChildrenOfType<ModColumn>().All(column => column.IsLoaded && column.ItemsLoaded));
+            modSelectOverlay.ChildrenOfType<ModColumn>().Any()
+            && modSelectOverlay.ChildrenOfType<ModColumn>().All(column => column.IsLoaded && column.ItemsLoaded)
+            && modSelectOverlay.ChildrenOfType<ModPresetColumn>().Any()
+            && modSelectOverlay.ChildrenOfType<ModPresetColumn>().All(column => column.IsLoaded));
 
         private void changeRuleset(int id)
         {

osu.Game/Screens/Play/MasterGameplayClockContainer.cs

@@ -54,7 +54,7 @@ namespace osu.Game.Screens.Play
         ///
         /// In the future I want to change this.
         /// </summary>
-        private double? actualStopTime;
+        internal double? LastStopTime;
 
         [Resolved]
         private MusicController musicController { get; set; } = null!;
@@ -100,7 +100,7 @@ namespace osu.Game.Screens.Play
 
         protected override void StopGameplayClock()
         {
-            actualStopTime = GameplayClock.CurrentTime;
+            LastStopTime = GameplayClock.CurrentTime;
 
             if (IsLoaded)
             {
@@ -127,17 +127,17 @@ namespace osu.Game.Screens.Play
         public override void Seek(double time)
         {
             // Safety in case the clock is seeked while stopped.
-            actualStopTime = null;
+            LastStopTime = null;
 
             base.Seek(time);
         }
 
         protected override void PrepareStart()
         {
-            if (actualStopTime != null)
+            if (LastStopTime != null)
             {
-                Seek(actualStopTime.Value);
+                Seek(LastStopTime.Value);
-                actualStopTime = null;
+                LastStopTime = null;
             }
             else
                 base.PrepareStart();