Mark and document remaining flaky tests

osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs

@@ -24,6 +24,7 @@ using osu.Game.Rulesets.Osu.Objects;
 using osu.Game.Rulesets.Osu.Objects.Drawables;
 using osu.Game.Rulesets.Osu.Skinning.Default;
 using osu.Game.Storyboards;
+using osu.Game.Tests;
 using osuTK;
 
 namespace osu.Game.Rulesets.Osu.Tests
@@ -71,6 +72,16 @@ namespace osu.Game.Rulesets.Osu.Tests
         [TestCase(0)]
         [TestCase(1)]
         [TestCase(2)]
+        [FlakyTest]
+        /*
+         * Fail rate around 0.15%
+         *
+         * TearDown : System.TimeoutException : "wait for seek to finish" timed out
+         *   --TearDown
+         *      at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0()
+         *      at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered)
+         *      at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition)
+         */
         public void TestSnakingEnabled(int sliderIndex)
         {
             AddStep("enable autoplay", () => autoplay = true);
@@ -95,6 +106,16 @@ namespace osu.Game.Rulesets.Osu.Tests
         [TestCase(0)]
         [TestCase(1)]
         [TestCase(2)]
+        [FlakyTest]
+        /*
+         * Fail rate around 0.15%
+         *
+         * TearDown : System.TimeoutException : "wait for seek to finish" timed out
+         *   --TearDown
+         *      at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0()
+         *      at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered)
+         *      at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition)
+         */
         public void TestSnakingDisabled(int sliderIndex)
         {
             AddStep("have autoplay", () => autoplay = true);

osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs

@@ -78,6 +78,16 @@ namespace osu.Game.Tests.Gameplay
         }
 
         [Test]
+        [FlakyTest]
+        /*
+         * Fail rate around 0.15%
+         *
+         * TearDown : osu.Framework.Testing.Drawables.Steps.AssertButton+TracedException : gameplay clock time = 2500
+         * --TearDown
+         *    at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal()
+         *    at osu.Framework.Threading.Scheduler.Update()
+         *    at osu.Framework.Graphics.Drawable.UpdateSubTree()
+         */
         public void TestSeekPerformsInGameplayTime(
             [Values(1.0, 0.5, 2.0)] double clockRate,
             [Values(0.0, 200.0, -200.0)] double userOffset,

osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs

@@ -82,6 +82,25 @@ namespace osu.Game.Tests.Visual.Editing
         }
 
         [Test]
+        [FlakyTest]
+        /*
+         * Fail rate around 1.2%.
+         *
+         * Failing with realm refetch occasionally being null.
+         * My only guess is that the WorkingBeatmap at SetupScreen is dummy instead of the true one.
+         * If it's something else, we have larger issues with realm, but I don't think that's the case.
+         *
+         * at osu.Framework.Logging.ThrowingTraceListener.Fail(String message1, String message2)
+         * at System.Diagnostics.TraceInternal.Fail(String message, String detailMessage)
+         * at System.Diagnostics.TraceInternal.TraceProvider.Fail(String message, String detailMessage)
+         * at System.Diagnostics.Debug.Fail(String message, String detailMessage)
+         * at osu.Game.Database.ModelManager`1.<>c__DisplayClass8_0.<performFileOperation>b__0(Realm realm) ModelManager.cs:line 50
+         * at osu.Game.Database.RealmExtensions.Write(Realm realm, Action`1 function) RealmExtensions.cs:line 14
+         * at osu.Game.Database.ModelManager`1.performFileOperation(TModel item, Action`1 operation) ModelManager.cs:line 47
+         * at osu.Game.Database.ModelManager`1.AddFile(TModel item, Stream contents, String filename) ModelManager.cs:line 37
+         * at osu.Game.Screens.Edit.Setup.ResourcesSection.ChangeAudioTrack(FileInfo source) ResourcesSection.cs:line 115
+         * at osu.Game.Tests.Visual.Editing.TestSceneEditorBeatmapCreation.<TestAddAudioTrack>b__11_0() TestSceneEditorBeatmapCreation.cs:line 101
+         */
         public void TestAddAudioTrack()
         {
             AddAssert("switch track to real track", () =>

osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs

@@ -14,6 +14,16 @@ namespace osu.Game.Tests.Visual.Editing
         public override Drawable CreateTestComponent() => Empty();
 
         [Test]
+        [FlakyTest]
+        /*
+         * Fail rate around 0.3%
+         *
+         * TearDown : osu.Framework.Testing.Drawables.Steps.AssertButton+TracedException : range halved
+         *   --TearDown
+         *      at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal()
+         *      at osu.Framework.Threading.Scheduler.Update()
+         *      at osu.Framework.Graphics.Drawable.UpdateSubTree()
+         */
         public void TestVisibleRangeUpdatesOnZoomChange()
         {
             double initialVisibleRange = 0;

osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs

@@ -61,6 +61,16 @@ namespace osu.Game.Tests.Visual.Gameplay
         /// Tests whether can still pause after cancelling completion by reverting <see cref="IScreen.ValidForResume"/> back to true.
         /// </summary>
         [Test]
+        [FlakyTest]
+        /*
+         * Fail rate around 0.45%
+         *
+         * TearDown : System.TimeoutException : "completion set by processor" timed out
+         * --TearDown
+         *    at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0()
+         *    at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered)
+         *    at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition)
+         */
         public void TestCanPauseAfterCancellation()
         {
             complete();

osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs

@@ -74,6 +74,25 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [Test]
+        [FlakyTest]
+        /*
+         * Fail rate around 1.5%
+         *
+         * TearDown : System.AggregateException : One or more errors occurred. (Index was out of range. Must be non-negative and less than the size of the collection. (Parameter 'index'))
+  ----> System.ArgumentOutOfRangeException : Index was out of range. Must be non-negative and less than the size of the collection. (Parameter 'index')
+         * --TearDown
+         *    at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)
+         *   at System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken cancellationToken)
+         *   at osu.Framework.Extensions.TaskExtensions.WaitSafely(Task task)
+         *   at osu.Framework.Testing.TestScene.checkForErrors()
+         *   at osu.Framework.Testing.TestScene.RunTestsFromNUnit()
+         *--ArgumentOutOfRangeException
+         *   at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller)
+         *   at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller)
+         *   at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller)
+         *   at osu.Game.Online.Multiplayer.MultiplayerClient.<>c__DisplayClass106_0.<PlaylistItemChanged>b__0() in C:\BuildAgent\work\ecd860037212ac52\osu.Game\Online\Multiplayer\MultiplayerClient     .cs:line 702
+         *   at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal()
+         */
         public void TestCreatedRoom()
         {
             AddStep("add playlist item", () =>
@@ -90,6 +109,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [Test]
+        [FlakyTest]
         public void TestTaikoOnlyMod()
         {
             AddStep("add playlist item", () =>
@@ -110,6 +130,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [Test]
+        [FlakyTest]
         public void TestSettingValidity()
         {
             AddAssert("create button not enabled", () => !this.ChildrenOfType<MultiplayerMatchSettingsOverlay.CreateOrUpdateButton>().Single().Enabled.Value);
@@ -126,6 +147,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [Test]
+        [FlakyTest]
         public void TestStartMatchWhileSpectating()
         {
             AddStep("set playlist", () =>
@@ -156,6 +178,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [Test]
+        [FlakyTest]
         public void TestFreeModSelectionHasAllowedMods()
         {
             AddStep("add playlist item with allowed mod", () =>
@@ -183,6 +206,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [Test]
+        [FlakyTest]
         public void TestModSelectKeyWithAllowedMods()
         {
             AddStep("add playlist item with allowed mod", () =>
@@ -204,6 +228,19 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [Test]
+        [FlakyTest]
+        /*
+         * Fail rate around 0.3%
+         *
+         * Somehow there are two mod select overlays?
+         *
+         * TearDown : System.InvalidOperationException : Sequence contains more than one element
+         *  --TearDown
+         *     at System.Linq.ThrowHelper.ThrowMoreThanOneElementException()
+         *     at System.Linq.Enumerable.TryGetSingle[TSource](IEnumerable`1 source, Boolean& found)
+         *     at System.Linq.Enumerable.Single[TSource](IEnumerable`1 source)
+         *     at osu.Game.Tests.Visual.Multiplayer.TestSceneMultiplayerMatchSubScreen.<TestModSelectKeyWithNoAllowedMods>b__14_3() in /opt/buildagent/work/ecd860037212ac52/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs:line 223
+         */
         public void TestModSelectKeyWithNoAllowedMods()
         {
             AddStep("add playlist item with no allowed mods", () =>
@@ -224,6 +261,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [Test]
+        [FlakyTest]
         public void TestNextPlaylistItemSelectedAfterCompletion()
         {
             AddStep("add two playlist items", () =>