diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs index 366793058d..0118ed6513 100644 --- a/osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs +++ b/osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs @@ -24,6 +24,7 @@ using osu.Game.Rulesets.Osu.Objects; using osu.Game.Rulesets.Osu.Objects.Drawables; using osu.Game.Rulesets.Osu.Skinning.Default; using osu.Game.Storyboards; +using osu.Game.Tests; using osuTK; namespace osu.Game.Rulesets.Osu.Tests @@ -71,6 +72,16 @@ namespace osu.Game.Rulesets.Osu.Tests [TestCase(0)] [TestCase(1)] [TestCase(2)] + [FlakyTest] + /* + * Fail rate around 0.15% + * + * TearDown : System.TimeoutException : "wait for seek to finish" timed out + * --TearDown + * at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0() + * at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered) + * at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition) + */ public void TestSnakingEnabled(int sliderIndex) { AddStep("enable autoplay", () => autoplay = true); @@ -95,6 +106,16 @@ namespace osu.Game.Rulesets.Osu.Tests [TestCase(0)] [TestCase(1)] [TestCase(2)] + [FlakyTest] + /* + * Fail rate around 0.15% + * + * TearDown : System.TimeoutException : "wait for seek to finish" timed out + * --TearDown + * at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0() + * at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered) + * at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition) + */ public void TestSnakingDisabled(int sliderIndex) { AddStep("have autoplay", () => autoplay = true); diff --git a/osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs b/osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs index ced397921c..ae431e77ae 100644 --- a/osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs +++ b/osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs @@ -78,6 +78,16 @@ namespace osu.Game.Tests.Gameplay } [Test] + [FlakyTest] + /* + * Fail rate around 0.15% + * + * TearDown : osu.Framework.Testing.Drawables.Steps.AssertButton+TracedException : gameplay clock time = 2500 + * --TearDown + * at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal() + * at osu.Framework.Threading.Scheduler.Update() + * at osu.Framework.Graphics.Drawable.UpdateSubTree() + */ public void TestSeekPerformsInGameplayTime( [Values(1.0, 0.5, 2.0)] double clockRate, [Values(0.0, 200.0, -200.0)] double userOffset, diff --git a/osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs b/osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs index b711d55e15..2707682b4c 100644 --- a/osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs +++ b/osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs @@ -82,6 +82,25 @@ namespace osu.Game.Tests.Visual.Editing } [Test] + [FlakyTest] + /* + * Fail rate around 1.2%. + * + * Failing with realm refetch occasionally being null. + * My only guess is that the WorkingBeatmap at SetupScreen is dummy instead of the true one. + * If it's something else, we have larger issues with realm, but I don't think that's the case. + * + * at osu.Framework.Logging.ThrowingTraceListener.Fail(String message1, String message2) + * at System.Diagnostics.TraceInternal.Fail(String message, String detailMessage) + * at System.Diagnostics.TraceInternal.TraceProvider.Fail(String message, String detailMessage) + * at System.Diagnostics.Debug.Fail(String message, String detailMessage) + * at osu.Game.Database.ModelManager`1.<>c__DisplayClass8_0.b__0(Realm realm) ModelManager.cs:line 50 + * at osu.Game.Database.RealmExtensions.Write(Realm realm, Action`1 function) RealmExtensions.cs:line 14 + * at osu.Game.Database.ModelManager`1.performFileOperation(TModel item, Action`1 operation) ModelManager.cs:line 47 + * at osu.Game.Database.ModelManager`1.AddFile(TModel item, Stream contents, String filename) ModelManager.cs:line 37 + * at osu.Game.Screens.Edit.Setup.ResourcesSection.ChangeAudioTrack(FileInfo source) ResourcesSection.cs:line 115 + * at osu.Game.Tests.Visual.Editing.TestSceneEditorBeatmapCreation.b__11_0() TestSceneEditorBeatmapCreation.cs:line 101 + */ public void TestAddAudioTrack() { AddAssert("switch track to real track", () => diff --git a/osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs b/osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs index fd103ff70f..2cada1989e 100644 --- a/osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs +++ b/osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs @@ -14,6 +14,16 @@ namespace osu.Game.Tests.Visual.Editing public override Drawable CreateTestComponent() => Empty(); [Test] + [FlakyTest] + /* + * Fail rate around 0.3% + * + * TearDown : osu.Framework.Testing.Drawables.Steps.AssertButton+TracedException : range halved + * --TearDown + * at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal() + * at osu.Framework.Threading.Scheduler.Update() + * at osu.Framework.Graphics.Drawable.UpdateSubTree() + */ public void TestVisibleRangeUpdatesOnZoomChange() { double initialVisibleRange = 0; diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs index 6aedc64370..13ceb05aff 100644 --- a/osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs +++ b/osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs @@ -61,6 +61,16 @@ namespace osu.Game.Tests.Visual.Gameplay /// Tests whether can still pause after cancelling completion by reverting back to true. /// [Test] + [FlakyTest] + /* + * Fail rate around 0.45% + * + * TearDown : System.TimeoutException : "completion set by processor" timed out + * --TearDown + * at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0() + * at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered) + * at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition) + */ public void TestCanPauseAfterCancellation() { complete(); diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs index dc6f18285c..5d6a6c8104 100644 --- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs +++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs @@ -74,6 +74,25 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] + /* + * Fail rate around 1.5% + * + * TearDown : System.AggregateException : One or more errors occurred. (Index was out of range. Must be non-negative and less than the size of the collection. (Parameter 'index')) + ----> System.ArgumentOutOfRangeException : Index was out of range. Must be non-negative and less than the size of the collection. (Parameter 'index') + * --TearDown + * at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions) + * at System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken cancellationToken) + * at osu.Framework.Extensions.TaskExtensions.WaitSafely(Task task) + * at osu.Framework.Testing.TestScene.checkForErrors() + * at osu.Framework.Testing.TestScene.RunTestsFromNUnit() + *--ArgumentOutOfRangeException + * at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller) + * at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller) + * at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller) + * at osu.Game.Online.Multiplayer.MultiplayerClient.<>c__DisplayClass106_0.b__0() in C:\BuildAgent\work\ecd860037212ac52\osu.Game\Online\Multiplayer\MultiplayerClient .cs:line 702 + * at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal() + */ public void TestCreatedRoom() { AddStep("add playlist item", () => @@ -90,6 +109,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] // See above public void TestTaikoOnlyMod() { AddStep("add playlist item", () => @@ -110,6 +130,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] // See above public void TestSettingValidity() { AddAssert("create button not enabled", () => !this.ChildrenOfType().Single().Enabled.Value); @@ -126,6 +147,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] // See above public void TestStartMatchWhileSpectating() { AddStep("set playlist", () => @@ -156,6 +178,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] // See above public void TestFreeModSelectionHasAllowedMods() { AddStep("add playlist item with allowed mod", () => @@ -182,6 +205,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] // See above public void TestModSelectKeyWithAllowedMods() { AddStep("add playlist item with allowed mod", () => @@ -203,6 +227,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] // See above public void TestModSelectKeyWithNoAllowedMods() { AddStep("add playlist item with no allowed mods", () => @@ -223,6 +248,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] // See above public void TestNextPlaylistItemSelectedAfterCompletion() { AddStep("add two playlist items", () => diff --git a/osu.Game/Tests/FlakyTestAttribute.cs b/osu.Game/Tests/FlakyTestAttribute.cs new file mode 100644 index 0000000000..299dbb89a2 --- /dev/null +++ b/osu.Game/Tests/FlakyTestAttribute.cs @@ -0,0 +1,25 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using NUnit.Framework; + +namespace osu.Game.Tests +{ + /// + /// An attribute to mark any flaky tests. + /// Will add a retry count unless environment variable `FAIL_FLAKY_TESTS` is set to `1`. + /// + public class FlakyTestAttribute : RetryAttribute + { + public FlakyTestAttribute() + : this(10) + { + } + + public FlakyTestAttribute(int tryCount) + : base(Environment.GetEnvironmentVariable("OSU_TESTS_FAIL_FLAKY") == "1" ? 0 : tryCount) + { + } + } +}