From 1777a6d24abffb933105e6fdd148368a4c841d50 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Jul 2022 16:06:18 +0900 Subject: [PATCH 1/5] Add attribute to retry flaky tests on normal CI runs --- osu.Game/Tests/FlakyTestAttribute.cs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 osu.Game/Tests/FlakyTestAttribute.cs diff --git a/osu.Game/Tests/FlakyTestAttribute.cs b/osu.Game/Tests/FlakyTestAttribute.cs new file mode 100644 index 0000000000..a5646f5d00 --- /dev/null +++ b/osu.Game/Tests/FlakyTestAttribute.cs @@ -0,0 +1,26 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +#nullable disable +using System; +using NUnit.Framework; + +namespace osu.Game.Tests +{ + /// + /// An attribute to mark any flaky tests. + /// Will add a retry count unless environment variable `FAIL_FLAKY_TESTS` is set to `1`. + /// + public class FlakyTestAttribute : RetryAttribute + { + public FlakyTestAttribute() + : this(10) + { + } + + public FlakyTestAttribute(int tryCount) + : base(Environment.GetEnvironmentVariable("FAIL_FLAKY_TESTS") == "1" ? 0 : tryCount) + { + } + } +} From b597843579c3a10fbf043b91bac12b1db3ac7b47 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Jul 2022 16:17:40 +0900 Subject: [PATCH 2/5] Mark and document remaining flaky tests --- .../TestSceneSliderSnaking.cs | 21 ++++++++++ .../TestSceneMasterGameplayClockContainer.cs | 10 +++++ .../Editing/TestSceneEditorBeatmapCreation.cs | 19 ++++++++++ .../Visual/Editing/TestSceneTimelineZoom.cs | 10 +++++ .../TestSceneCompletionCancellation.cs | 10 +++++ .../TestSceneMultiplayerMatchSubScreen.cs | 38 +++++++++++++++++++ 6 files changed, 108 insertions(+) diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs index 366793058d..0118ed6513 100644 --- a/osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs +++ b/osu.Game.Rulesets.Osu.Tests/TestSceneSliderSnaking.cs @@ -24,6 +24,7 @@ using osu.Game.Rulesets.Osu.Objects; using osu.Game.Rulesets.Osu.Objects.Drawables; using osu.Game.Rulesets.Osu.Skinning.Default; using osu.Game.Storyboards; +using osu.Game.Tests; using osuTK; namespace osu.Game.Rulesets.Osu.Tests @@ -71,6 +72,16 @@ namespace osu.Game.Rulesets.Osu.Tests [TestCase(0)] [TestCase(1)] [TestCase(2)] + [FlakyTest] + /* + * Fail rate around 0.15% + * + * TearDown : System.TimeoutException : "wait for seek to finish" timed out + * --TearDown + * at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0() + * at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered) + * at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition) + */ public void TestSnakingEnabled(int sliderIndex) { AddStep("enable autoplay", () => autoplay = true); @@ -95,6 +106,16 @@ namespace osu.Game.Rulesets.Osu.Tests [TestCase(0)] [TestCase(1)] [TestCase(2)] + [FlakyTest] + /* + * Fail rate around 0.15% + * + * TearDown : System.TimeoutException : "wait for seek to finish" timed out + * --TearDown + * at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0() + * at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered) + * at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition) + */ public void TestSnakingDisabled(int sliderIndex) { AddStep("have autoplay", () => autoplay = true); diff --git a/osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs b/osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs index ced397921c..ae431e77ae 100644 --- a/osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs +++ b/osu.Game.Tests/Gameplay/TestSceneMasterGameplayClockContainer.cs @@ -78,6 +78,16 @@ namespace osu.Game.Tests.Gameplay } [Test] + [FlakyTest] + /* + * Fail rate around 0.15% + * + * TearDown : osu.Framework.Testing.Drawables.Steps.AssertButton+TracedException : gameplay clock time = 2500 + * --TearDown + * at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal() + * at osu.Framework.Threading.Scheduler.Update() + * at osu.Framework.Graphics.Drawable.UpdateSubTree() + */ public void TestSeekPerformsInGameplayTime( [Values(1.0, 0.5, 2.0)] double clockRate, [Values(0.0, 200.0, -200.0)] double userOffset, diff --git a/osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs b/osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs index b711d55e15..2707682b4c 100644 --- a/osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs +++ b/osu.Game.Tests/Visual/Editing/TestSceneEditorBeatmapCreation.cs @@ -82,6 +82,25 @@ namespace osu.Game.Tests.Visual.Editing } [Test] + [FlakyTest] + /* + * Fail rate around 1.2%. + * + * Failing with realm refetch occasionally being null. + * My only guess is that the WorkingBeatmap at SetupScreen is dummy instead of the true one. + * If it's something else, we have larger issues with realm, but I don't think that's the case. + * + * at osu.Framework.Logging.ThrowingTraceListener.Fail(String message1, String message2) + * at System.Diagnostics.TraceInternal.Fail(String message, String detailMessage) + * at System.Diagnostics.TraceInternal.TraceProvider.Fail(String message, String detailMessage) + * at System.Diagnostics.Debug.Fail(String message, String detailMessage) + * at osu.Game.Database.ModelManager`1.<>c__DisplayClass8_0.b__0(Realm realm) ModelManager.cs:line 50 + * at osu.Game.Database.RealmExtensions.Write(Realm realm, Action`1 function) RealmExtensions.cs:line 14 + * at osu.Game.Database.ModelManager`1.performFileOperation(TModel item, Action`1 operation) ModelManager.cs:line 47 + * at osu.Game.Database.ModelManager`1.AddFile(TModel item, Stream contents, String filename) ModelManager.cs:line 37 + * at osu.Game.Screens.Edit.Setup.ResourcesSection.ChangeAudioTrack(FileInfo source) ResourcesSection.cs:line 115 + * at osu.Game.Tests.Visual.Editing.TestSceneEditorBeatmapCreation.b__11_0() TestSceneEditorBeatmapCreation.cs:line 101 + */ public void TestAddAudioTrack() { AddAssert("switch track to real track", () => diff --git a/osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs b/osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs index fd103ff70f..2cada1989e 100644 --- a/osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs +++ b/osu.Game.Tests/Visual/Editing/TestSceneTimelineZoom.cs @@ -14,6 +14,16 @@ namespace osu.Game.Tests.Visual.Editing public override Drawable CreateTestComponent() => Empty(); [Test] + [FlakyTest] + /* + * Fail rate around 0.3% + * + * TearDown : osu.Framework.Testing.Drawables.Steps.AssertButton+TracedException : range halved + * --TearDown + * at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal() + * at osu.Framework.Threading.Scheduler.Update() + * at osu.Framework.Graphics.Drawable.UpdateSubTree() + */ public void TestVisibleRangeUpdatesOnZoomChange() { double initialVisibleRange = 0; diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs index 6aedc64370..13ceb05aff 100644 --- a/osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs +++ b/osu.Game.Tests/Visual/Gameplay/TestSceneCompletionCancellation.cs @@ -61,6 +61,16 @@ namespace osu.Game.Tests.Visual.Gameplay /// Tests whether can still pause after cancelling completion by reverting back to true. /// [Test] + [FlakyTest] + /* + * Fail rate around 0.45% + * + * TearDown : System.TimeoutException : "completion set by processor" timed out + * --TearDown + * at osu.Framework.Testing.Drawables.Steps.UntilStepButton.<>c__DisplayClass11_0.<.ctor>b__0() + * at osu.Framework.Testing.Drawables.Steps.StepButton.PerformStep(Boolean userTriggered) + * at osu.Framework.Testing.TestScene.runNextStep(Action onCompletion, Action`1 onError, Func`2 stopCondition) + */ public void TestCanPauseAfterCancellation() { complete(); diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs index bb7587ac56..3a48f68f1d 100644 --- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs +++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs @@ -74,6 +74,25 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] + /* + * Fail rate around 1.5% + * + * TearDown : System.AggregateException : One or more errors occurred. (Index was out of range. Must be non-negative and less than the size of the collection. (Parameter 'index')) + ----> System.ArgumentOutOfRangeException : Index was out of range. Must be non-negative and less than the size of the collection. (Parameter 'index') + * --TearDown + * at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions) + * at System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken cancellationToken) + * at osu.Framework.Extensions.TaskExtensions.WaitSafely(Task task) + * at osu.Framework.Testing.TestScene.checkForErrors() + * at osu.Framework.Testing.TestScene.RunTestsFromNUnit() + *--ArgumentOutOfRangeException + * at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller) + * at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller) + * at osu.Framework.Bindables.BindableList`1.removeAt(Int32 index, BindableList`1 caller) + * at osu.Game.Online.Multiplayer.MultiplayerClient.<>c__DisplayClass106_0.b__0() in C:\BuildAgent\work\ecd860037212ac52\osu.Game\Online\Multiplayer\MultiplayerClient .cs:line 702 + * at osu.Framework.Threading.ScheduledDelegate.RunTaskInternal() + */ public void TestCreatedRoom() { AddStep("add playlist item", () => @@ -90,6 +109,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] public void TestTaikoOnlyMod() { AddStep("add playlist item", () => @@ -110,6 +130,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] public void TestSettingValidity() { AddAssert("create button not enabled", () => !this.ChildrenOfType().Single().Enabled.Value); @@ -126,6 +147,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] public void TestStartMatchWhileSpectating() { AddStep("set playlist", () => @@ -156,6 +178,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] public void TestFreeModSelectionHasAllowedMods() { AddStep("add playlist item with allowed mod", () => @@ -183,6 +206,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] public void TestModSelectKeyWithAllowedMods() { AddStep("add playlist item with allowed mod", () => @@ -204,6 +228,19 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] + /* + * Fail rate around 0.3% + * + * Somehow there are two mod select overlays? + * + * TearDown : System.InvalidOperationException : Sequence contains more than one element + * --TearDown + * at System.Linq.ThrowHelper.ThrowMoreThanOneElementException() + * at System.Linq.Enumerable.TryGetSingle[TSource](IEnumerable`1 source, Boolean& found) + * at System.Linq.Enumerable.Single[TSource](IEnumerable`1 source) + * at osu.Game.Tests.Visual.Multiplayer.TestSceneMultiplayerMatchSubScreen.b__14_3() in /opt/buildagent/work/ecd860037212ac52/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs:line 223 + */ public void TestModSelectKeyWithNoAllowedMods() { AddStep("add playlist item with no allowed mods", () => @@ -224,6 +261,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] + [FlakyTest] public void TestNextPlaylistItemSelectedAfterCompletion() { AddStep("add two playlist items", () => From c4f166084133f129d1f0195473836943db4dff5f Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Jul 2022 16:20:34 +0900 Subject: [PATCH 3/5] Rename ENVVAR in line with previous one (`OSU_TESTS_NO_TIMEOUT`) --- osu.Game/Tests/FlakyTestAttribute.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Tests/FlakyTestAttribute.cs b/osu.Game/Tests/FlakyTestAttribute.cs index a5646f5d00..a1c16284c8 100644 --- a/osu.Game/Tests/FlakyTestAttribute.cs +++ b/osu.Game/Tests/FlakyTestAttribute.cs @@ -19,7 +19,7 @@ namespace osu.Game.Tests } public FlakyTestAttribute(int tryCount) - : base(Environment.GetEnvironmentVariable("FAIL_FLAKY_TESTS") == "1" ? 0 : tryCount) + : base(Environment.GetEnvironmentVariable("OSU_TESTS_FAIL_FLAKY") == "1" ? 0 : tryCount) { } } From 615c3234d8df6339635f6ae7dcbca6158b282e26 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Jul 2022 16:25:21 +0900 Subject: [PATCH 4/5] Remove non-required NRT hint --- osu.Game/Tests/FlakyTestAttribute.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/osu.Game/Tests/FlakyTestAttribute.cs b/osu.Game/Tests/FlakyTestAttribute.cs index a1c16284c8..299dbb89a2 100644 --- a/osu.Game/Tests/FlakyTestAttribute.cs +++ b/osu.Game/Tests/FlakyTestAttribute.cs @@ -1,7 +1,6 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. -#nullable disable using System; using NUnit.Framework; From 8f8c2a8c9aa1e87a2468bed6408a33939e6d9b85 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 1 Jul 2022 16:55:47 +0900 Subject: [PATCH 5/5] Remove one flaky test description (fixed via #18966) --- .../TestSceneMultiplayerMatchSubScreen.cs | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs index 3a48f68f1d..75a011b6eb 100644 --- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs +++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs @@ -109,7 +109,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] - [FlakyTest] + [FlakyTest] // See above public void TestTaikoOnlyMod() { AddStep("add playlist item", () => @@ -130,7 +130,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] - [FlakyTest] + [FlakyTest] // See above public void TestSettingValidity() { AddAssert("create button not enabled", () => !this.ChildrenOfType().Single().Enabled.Value); @@ -147,7 +147,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] - [FlakyTest] + [FlakyTest] // See above public void TestStartMatchWhileSpectating() { AddStep("set playlist", () => @@ -178,7 +178,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] - [FlakyTest] + [FlakyTest] // See above public void TestFreeModSelectionHasAllowedMods() { AddStep("add playlist item with allowed mod", () => @@ -206,7 +206,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] - [FlakyTest] + [FlakyTest] // See above public void TestModSelectKeyWithAllowedMods() { AddStep("add playlist item with allowed mod", () => @@ -228,19 +228,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] - [FlakyTest] - /* - * Fail rate around 0.3% - * - * Somehow there are two mod select overlays? - * - * TearDown : System.InvalidOperationException : Sequence contains more than one element - * --TearDown - * at System.Linq.ThrowHelper.ThrowMoreThanOneElementException() - * at System.Linq.Enumerable.TryGetSingle[TSource](IEnumerable`1 source, Boolean& found) - * at System.Linq.Enumerable.Single[TSource](IEnumerable`1 source) - * at osu.Game.Tests.Visual.Multiplayer.TestSceneMultiplayerMatchSubScreen.b__14_3() in /opt/buildagent/work/ecd860037212ac52/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSubScreen.cs:line 223 - */ + [FlakyTest] // See above public void TestModSelectKeyWithNoAllowedMods() { AddStep("add playlist item with no allowed mods", () => @@ -261,7 +249,7 @@ namespace osu.Game.Tests.Visual.Multiplayer } [Test] - [FlakyTest] + [FlakyTest] // See above public void TestNextPlaylistItemSelectedAfterCompletion() { AddStep("add two playlist items", () =>