h1or4dme8/osu-lazer
1
0
Fork 0
mirror of https://github.com/ppy/osu.git synced 2025-01-27 00:23:01 +08:00
Code Issues Releases Wiki Activity

Merge pull request #18965 from peppy/mark-flaky-tests

Browse Source 
Mark, document and silence remaining flaky tests on standard CI runs
This commit is contained in:
Dan Balasescu 2022-07-03 21:41:14 +09:00 committed by GitHub
commit 385bd5a1b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 121 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs
View File

@ -24,6 +24,7 @@ using osu.Game.Rulesets.Osu.Objects;
using osu.Game.Rulesets.Osu.Objects.Drawables;
using osu.Game.Rulesets.Osu.Skinning.Default;
using osu.Game.Storyboards;
using osu.Game.Tests;
using osuTK;
namespace osu.Game.Rulesets.Osu.Tests
@ -71,6 +72,16 @@ namespace osu.Game.Rulesets.Osu.Tests
[TestCase(0)]
[TestCase(1)]
[TestCase(2)]
[FlakyTest]
/*
* Fail rate around 0.15%
*
* TearDown : System.TimeoutException : "wait for seek to finish" timed out
* --TearDown
* at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0()
* at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered)
* at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition)
*/
public void TestSnakingEnabled(int sliderIndex)
{
AddStep("enable autoplay", () => autoplay = true);
@ -95,6 +106,16 @@ namespace osu.Game.Rulesets.Osu.Tests
[TestCase(0)]
[TestCase(1)]
[TestCase(2)]
[FlakyTest]
/*
* Fail rate around 0.15%
*
* TearDown : System.TimeoutException : "wait for seek to finish" timed out
* --TearDown
* at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0()
* at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered)
* at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition)
*/
public void TestSnakingDisabled(int sliderIndex)
{
AddStep("have autoplay", () => autoplay = true);

10
osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs
View File

@ -78,6 +78,16 @@ namespace osu.Game.Tests.Gameplay
}
[Test]
[FlakyTest]
/*
* Fail rate around 0.15%
*
* TearDown : osu.Framework.Testing.Drawables.Steps.AssertButton+TracedException : gameplay clock time = 2500
* --TearDown
* at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal()
* at osu.Framework.Threading.Scheduler.Update()
* at osu.Framework.Graphics.Drawable.UpdateSubTree()
*/
public void TestSeekPerformsInGameplayTime(
[Values(1.0, 0.5, 2.0)] double clockRate,
[Values(0.0, 200.0, -200.0)] double userOffset,

19
osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs
View File

@ -82,6 +82,25 @@ namespace osu.Game.Tests.Visual.Editing
}
[Test]
[FlakyTest]
/*
* Fail rate around 1.2%.
*
* Failing with realm refetch occasionally being null.
* My only guess is that the WorkingBeatmap at SetupScreen is dummy instead of the true one.
* If it's something else, we have larger issues with realm, but I don't think that's the case.
*
* at osu.Framework.Logging.ThrowingTraceListener.Fail(String message1, String message2)
* at System.Diagnostics.TraceInternal.Fail(String message, String detailMessage)
* at System.Diagnostics.TraceInternal.TraceProvider.Fail(String message, String detailMessage)
* at System.Diagnostics.Debug.Fail(String message, String detailMessage)
* at osu.Game.Database.ModelManager`1.<>c__DisplayClass8_0.<performFileOperation>b__0(Realm realm) ModelManager.cs:line 50
* at osu.Game.Database.RealmExtensions.Write(Realm realm, Action`1 function) RealmExtensions.cs:line 14
* at osu.Game.Database.ModelManager`1.performFileOperation(TModel item, Action`1 operation) ModelManager.cs:line 47
* at osu.Game.Database.ModelManager`1.AddFile(TModel item, Stream contents, String filename) ModelManager.cs:line 37
* at osu.Game.Screens.Edit.Setup.ResourcesSection.ChangeAudioTrack(FileInfo source) ResourcesSection.cs:line 115
* at osu.Game.Tests.Visual.Editing.TestSceneEditorBeatmapCreation.<TestAddAudioTrack>b__11_0() TestSceneEditorBeatmapCreation.cs:line 101
*/
public void TestAddAudioTrack()
{
AddAssert("switch track to real track", () =>

10
osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs
View File

@ -14,6 +14,16 @@ namespace osu.Game.Tests.Visual.Editing
public override Drawable CreateTestComponent() => Empty();
[Test]
[FlakyTest]
/*
* Fail rate around 0.3%
*
* TearDown : osu.Framework.Testing.Drawables.Steps.AssertButton+TracedException : range halved
* --TearDown
* at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal()
* at osu.Framework.Threading.Scheduler.Update()
* at osu.Framework.Graphics.Drawable.UpdateSubTree()
*/
public void TestVisibleRangeUpdatesOnZoomChange()
{
double initialVisibleRange = 0;

10
osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs
View File

@ -61,6 +61,16 @@ namespace osu.Game.Tests.Visual.Gameplay
/// Tests whether can still pause after cancelling completion by reverting <see cref="IScreen.ValidForResume"/> back to true.
/// </summary>
[Test]
[FlakyTest]
/*
* Fail rate around 0.45%
*
* TearDown : System.TimeoutException : "completion set by processor" timed out
* --TearDown
* at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0()
* at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered)
* at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition)
*/
public void TestCanPauseAfterCancellation()
{
complete();

26
osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs
View File

@ -74,6 +74,25 @@ namespace osu.Game.Tests.Visual.Multiplayer
}
[Test]
[FlakyTest]
/*
* Fail rate around 1.5%
*
* TearDown : System.AggregateException : One or more errors occurred. (Index was out of range. Must be non-negative and less than the size of the collection. (Parameter 'index'))
----> System.ArgumentOutOfRangeException : Index was out of range. Must be non-negative and less than the size of the collection. (Parameter 'index')
* --TearDown
* at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)
* at System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken cancellationToken)
* at osu.Framework.Extensions.TaskExtensions.WaitSafely(Task task)
* at osu.Framework.Testing.TestScene.checkForErrors()
* at osu.Framework.Testing.TestScene.RunTestsFromNUnit()
*--ArgumentOutOfRangeException
* at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller)
* at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller)
* at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller)
* at osu.Game.Online.Multiplayer.MultiplayerClient.<>c__DisplayClass106_0.<PlaylistItemChanged>b__0() in C:\BuildAgent\work\ecd860037212ac52\osu.Game\Online\Multiplayer\MultiplayerClient .cs:line 702
* at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal()
*/
public void TestCreatedRoom()
{
AddStep("add playlist item", () =>
@ -90,6 +109,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
}
[Test]
[FlakyTest] // See above
public void TestTaikoOnlyMod()
{
AddStep("add playlist item", () =>
@ -110,6 +130,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
}
[Test]
[FlakyTest] // See above
public void TestSettingValidity()
{
AddAssert("create button not enabled", () => !this.ChildrenOfType<MultiplayerMatchSettingsOverlay.CreateOrUpdateButton>().Single().Enabled.Value);
@ -126,6 +147,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
}
[Test]
[FlakyTest] // See above
public void TestStartMatchWhileSpectating()
{
AddStep("set playlist", () =>
@ -156,6 +178,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
}
[Test]
[FlakyTest] // See above
public void TestFreeModSelectionHasAllowedMods()
{
AddStep("add playlist item with allowed mod", () =>
@ -182,6 +205,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
}
[Test]
[FlakyTest] // See above
public void TestModSelectKeyWithAllowedMods()
{
AddStep("add playlist item with allowed mod", () =>
@ -203,6 +227,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
}
[Test]
[FlakyTest] // See above
public void TestModSelectKeyWithNoAllowedMods()
{
AddStep("add playlist item with no allowed mods", () =>
@ -223,6 +248,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
}
[Test]
[FlakyTest] // See above
public void TestNextPlaylistItemSelectedAfterCompletion()
{
AddStep("add two playlist items", () =>

25
osu.Game/Tests/FlakyTestAttribute.cs Normal file
View File

@ -0,0 +1,25 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using NUnit.Framework;
namespace osu.Game.Tests
{
/// <summary>
/// An attribute to mark any flaky tests.
/// Will add a retry count unless environment variable `FAIL_FLAKY_TESTS` is set to `1`.
/// </summary>
public class FlakyTestAttribute : RetryAttribute
{
public FlakyTestAttribute()
: this(10)
{
}
public FlakyTestAttribute(int tryCount)
: base(Environment.GetEnvironmentVariable("OSU_TESTS_FAIL_FLAKY") == "1" ? 0 : tryCount)
{
}
}
}