Skip to content

Commit

Permalink
Disable tests on Debug CoreClr that are already disabled on Checked C…
Browse files Browse the repository at this point in the history
…oreClr (#64891)

* Disable tests on Debug CoreClr that are already disabled on Checked CoreClr

* Increase Process timeout

* Extract timeout modifier and make Debug only

* Update src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs
  • Loading branch information
danmoseley committed Feb 12, 2022
1 parent 7aae884 commit 348e06c
Show file tree
Hide file tree
Showing 25 changed files with 51 additions and 31 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,18 @@ public static partial class PlatformDetection
public static bool Is64BitProcess => IntPtr.Size == 8;
public static bool IsNotWindows => !IsWindows;

private static Lazy<bool> s_isCheckedRuntime => new Lazy<bool>(() => AssemblyConfigurationEquals("Checked"));
private static Lazy<bool> s_isReleaseRuntime => new Lazy<bool>(() => AssemblyConfigurationEquals("Release"));
private static Lazy<bool> s_isDebugRuntime => new Lazy<bool>(() => AssemblyConfigurationEquals("Debug"));

public static bool IsCheckedRuntime => s_isCheckedRuntime.Value;
public static bool IsReleaseRuntime => s_isReleaseRuntime.Value;
public static bool IsDebugRuntime => s_isDebugRuntime.Value;

// For use as needed on tests that time out when run on a Debug runtime.
// Not relevant for timeouts on external activities, such as network timeouts.
public static int SlowRuntimeTimeoutModifier = (PlatformDetection.IsDebugRuntime ? 5 : 1);

public static bool IsCaseInsensitiveOS => IsWindows || IsOSX || IsMacCatalyst;

#if NETCOREAPP
Expand All @@ -68,7 +80,7 @@ public static partial class PlatformDetection
#else
public static bool IsCaseSensitiveOS => !IsCaseInsensitiveOS;
#endif

public static bool IsThreadingSupported => !IsBrowser;
public static bool IsBinaryFormatterSupported => IsNotMobile && !IsNativeAot;
public static bool IsSymLinkSupported => !IsiOS && !IstvOS;
Expand All @@ -86,9 +98,9 @@ public static partial class PlatformDetection
public static bool IsUsingLimitedCultures => !IsNotMobile;
public static bool IsNotUsingLimitedCultures => IsNotMobile;

public static bool IsLinqExpressionsBuiltWithIsInterpretingOnly => s_LinqExpressionsBuiltWithIsInterpretingOnly.Value;
public static bool IsLinqExpressionsBuiltWithIsInterpretingOnly => s_linqExpressionsBuiltWithIsInterpretingOnly.Value;
public static bool IsNotLinqExpressionsBuiltWithIsInterpretingOnly => !IsLinqExpressionsBuiltWithIsInterpretingOnly;
private static readonly Lazy<bool> s_LinqExpressionsBuiltWithIsInterpretingOnly = new Lazy<bool>(GetLinqExpressionsBuiltWithIsInterpretingOnly);
private static readonly Lazy<bool> s_linqExpressionsBuiltWithIsInterpretingOnly = new Lazy<bool>(GetLinqExpressionsBuiltWithIsInterpretingOnly);
private static bool GetLinqExpressionsBuiltWithIsInterpretingOnly()
{
return !(bool)typeof(LambdaExpression).GetMethod("get_CanCompileToIL").Invoke(null, Array.Empty<object>());
Expand Down Expand Up @@ -203,7 +215,7 @@ private static bool GetAlpnSupport()
{
return true;
}

return OpenSslVersion.Major == 1 && (OpenSslVersion.Minor >= 1 || OpenSslVersion.Build >= 2);
}

Expand Down Expand Up @@ -513,5 +525,13 @@ private static bool IsEnvironmentVariableTrue(string variableName)
var val = Environment.GetEnvironmentVariable(variableName);
return (val != null && val == "true");
}

private static bool AssemblyConfigurationEquals(string configuration)
{
AssemblyConfigurationAttribute assemblyConfigurationAttribute = typeof(string).Assembly.GetCustomAttribute<AssemblyConfigurationAttribute>();

return assemblyConfigurationAttribute != null &&
string.Equals(assemblyConfigurationAttribute.Configuration, configuration, StringComparison.InvariantCulture);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

namespace System.ComponentModel.Composition.Registration.Tests
{
[SkipOnCoreClr("Test failures on stress tests", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("Test failures on stress tests", ~RuntimeConfiguration.Release)]
[SkipOnMono("Test failures on stress tests")]
public class RegistrationBuilderAttributedOverrideUnitTests
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace System.Diagnostics.Tests
{
public partial class ProcessTestBase : FileCleanupTestBase
{
protected const int WaitInMS = 30 * 1000;
protected readonly int WaitInMS = 30 * 1000 * PlatformDetection.SlowRuntimeTimeoutModifier;
protected Process _process;
protected readonly List<Process> _processes = new List<Process>();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
namespace MonoTests.System.Drawing
{

[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/37082", TestPlatforms.AnyUnix, RuntimeConfiguration.Checked)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/37082", TestPlatforms.AnyUnix, ~RuntimeConfiguration.Release)]
public class TestBitmap
{

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@

using Xunit;

[assembly: SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/12927", RuntimeConfiguration.Checked)]
[assembly: SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/12927", ~RuntimeConfiguration.Release)]
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@
using System;
using Xunit;

[assembly: SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131", RuntimeConfiguration.Checked)]
[assembly: SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131", ~RuntimeConfiguration.Release)]
[assembly: ActiveIssue("https://github.com/dotnet/runtime/issues/131", ~(TestPlatforms.Android | TestPlatforms.Browser), TargetFrameworkMonikers.Any, TestRuntimes.Mono)] // System.Net.Tests are flaky and/or long running

Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

namespace System.Net.Tests
{
[SkipOnCoreClr("System.Net.Tests may timeout in stress configurations", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("System.Net.Tests may timeout in stress configurations", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/2391", TestRuntimes.Mono)]
[ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
public class HttpListenerAuthenticationTests : IDisposable
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

namespace System.Net.Tests
{
[SkipOnCoreClr("System.Net.Tests may timeout in stress configurations", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("System.Net.Tests may timeout in stress configurations", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/2391", TestRuntimes.Mono)]
[ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
public class HttpListenerContextTests : IDisposable
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

namespace System.Net.Tests
{
[SkipOnCoreClr("System.Net.Tests may timeout in stress configurations", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("System.Net.Tests may timeout in stress configurations", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/2391", TestRuntimes.Mono)]
[ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
public class HttpListenerResponseCookiesTests : HttpListenerResponseTestBase
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ protected async Task<HttpListenerResponse> GetResponse(string httpVersion = "1.1
}
}

[SkipOnCoreClr("System.Net.Tests may timeout in stress configurations", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("System.Net.Tests may timeout in stress configurations", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/2391", TestRuntimes.Mono)]
[ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
public class HttpListenerResponseTests : HttpListenerResponseTestBase
Expand Down
2 changes: 1 addition & 1 deletion src/libraries/System.Net.Requests/tests/LoggingTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace System.Net.Tests
public class LoggingTest
{
[Fact]
[SkipOnCoreClr("System.Net.Tests are flaky", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("System.Net.Tests are flaky", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/37087", TestPlatforms.Android)]
public void EventSource_ExistsWithCorrectId()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ public async Task SslStream_ServerCallbackNotSet_UsesLocalCertificateSelection(s
}

[Fact]
[SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/131", TestRuntimes.Mono)] // System.Net.Tests are flaky and/or long running
public async Task SslStream_NoSniFromClient_CallbackReturnsNull()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@

using Xunit;

[assembly: SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131", RuntimeConfiguration.Checked)]
[assembly: SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131", ~RuntimeConfiguration.Release)]
[assembly: ActiveIssue("https://github.com/dotnet/runtime/issues/34690", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[assembly: SkipOnPlatform(TestPlatforms.Browser, "System.Net.Sockets is not supported on Browser")]
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ public static void RunStandardFormatToStringTests()
}

[Fact]
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/11980", TestPlatforms.Linux, RuntimeConfiguration.Checked)]
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/11980", TestPlatforms.Linux, ~RuntimeConfiguration.Release)]
public static void RunRegionSpecificStandardFormatToStringTests()
{
CultureInfo[] cultures = new CultureInfo[] { new CultureInfo("en-US"), new CultureInfo("en-GB"), new CultureInfo("fr-CA"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public partial class BinaryFormatterTests : FileCleanupTestBase
{
// On 32-bit we can't test these high inputs as they cause OutOfMemoryExceptions.
[ConditionalTheory(typeof(Environment), nameof(Environment.Is64BitProcess))]
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/11191", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/11191", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/35915", typeof(PlatformDetection), nameof(PlatformDetection.IsMonoInterpreter))]
[InlineData(2 * 6_584_983 - 2)] // previous limit
[InlineData(2 * 7_199_369 - 2)] // last pre-computed prime number
Expand All @@ -47,7 +47,7 @@ public void SerializeHugeObjectGraphs(int limit)
}

[Theory]
[SkipOnCoreClr("Takes too long on Checked", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("Takes too long on Checked", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34008", TestPlatforms.Linux, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34753", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[SkipOnPlatform(TestPlatforms.Browser, "Takes too long on Browser.")]
Expand All @@ -65,14 +65,14 @@ public void ValidateBasicObjectsRoundtrip(object obj, FormatterAssemblyStyle ass
}

[Theory]
[SkipOnCoreClr("Takes too long on Checked", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("Takes too long on Checked", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/mono/mono/issues/15115", TestRuntimes.Mono)]
[MemberData(nameof(SerializableObjects_MemberData))]
public void ValidateAgainstBlobs(object obj, TypeSerializableValue[] blobs)
=> ValidateAndRoundtrip(obj, blobs, false);

[Theory]
[SkipOnCoreClr("Takes too long on Checked", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("Takes too long on Checked", ~RuntimeConfiguration.Release)]
[MemberData(nameof(SerializableEqualityComparers_MemberData))]
public void ValidateEqualityComparersAgainstBlobs(object obj, TypeSerializableValue[] blobs)
=> ValidateAndRoundtrip(obj, blobs, true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ public async Task PropertyCacheWithMinInputsLast()
private JsonSerializerOptions s_options = new JsonSerializerOptions();

[Fact]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", ~RuntimeConfiguration.Release)]
public async Task MultipleTypes()
{
async Task Serialize<T>(object[] args)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ namespace System.Text.Json.Serialization.Tests
public abstract partial class ConstructorTests
{
[Fact]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", ~RuntimeConfiguration.Release)]
public async Task ReadSimpleObjectAsync()
{
async Task RunTestAsync<T>(byte[] testData)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,7 @@ public static void ParseJson_UnseekableStream_BadBOM(string json)

[Theory]
[MemberData(nameof(BadBOMCases))]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", ~RuntimeConfiguration.Release)]
public static Task ParseJson_UnseekableStream_Async_BadBOM(string json)
{
byte[] data = Encoding.UTF8.GetBytes(json);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ public static async Task InvalidJsonShouldFailAtAnyPosition_Stream(
[Theory]
[MemberData(nameof(TestData), /* enumeratePayloadTweaks: */ false)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/42677", platforms: TestPlatforms.Windows, runtimes: TestRuntimes.Mono)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", ~RuntimeConfiguration.Release)]
public static void ShouldWorkAtAnyPosition_Sequence(
string json,
int bufferSize,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -602,7 +602,7 @@ private static void AssertDictionaryElements_StringValues(string serialized)

[Fact]
[ActiveIssue("https://github.com/dotnet/runtime/issues/39674", typeof(PlatformDetection), nameof(PlatformDetection.IsMonoInterpreter))]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", ~RuntimeConfiguration.Release)]
public static void DictionariesRoundTrip()
{
RunAllDictionariessRoundTripTest(JsonNumberTestData.ULongs);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public async Task NullObjectValue()
}

[Fact]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", ~RuntimeConfiguration.Release)]
public async Task RoundTripAsync()
{
byte[] buffer;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1839,7 +1839,7 @@ private static void TrailingCommasHelper(ReadOnlySequence<byte> utf8, JsonReader
[InlineData("{\"Property1\": {\"Property1.1\": 42} // comment\n,5}")]
[InlineData("{\"Property1\": {\"Property1.1\": 42}, // comment\n // comment\n5}")]
[InlineData("{// comment\n5}")]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", ~RuntimeConfiguration.Release)]
public static void ReadInvalidJsonStringsWithComments(string jsonString)
{
byte[] input = Encoding.UTF8.GetBytes(jsonString);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3196,7 +3196,7 @@ public void WritingLargestPossibleBase64Bytes(bool formatted, bool skipValidatio
[InlineData(true, false)]
[InlineData(false, true)]
[InlineData(false, false)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("https://github.com/dotnet/runtime/issues/45464", ~RuntimeConfiguration.Release)]
public void Writing3MBBase64Bytes(bool formatted, bool skipValidation)
{
byte[] value = new byte[3 * 1024 * 1024];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1549,7 +1549,7 @@ public async Task Match_SpecialUnicodeCharacters_Invariant(RegexEngine engine)

[ConditionalTheory(nameof(IsNotArmProcessAndRemoteExecutorSupported))] // times out on ARM
[SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, ".NET Framework does not have fix for https://github.com/dotnet/runtime/issues/24749")]
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/10680", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/10680", ~RuntimeConfiguration.Release)]
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/10680", RuntimeTestModes.JitMinOpts)]
[MemberData(nameof(RegexHelpers.AvailableEngines_MemberData), MemberType = typeof(RegexHelpers))]
public void Match_ExcessPrefix(RegexEngine engine)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public static void RunContinueWithAsyncStateCheckTests()

// Stresses on multiple continuations from a single antecedent
[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))]
[SkipOnCoreClr("Test timing out: https://github.com/dotnet/runtime/issues/2271", RuntimeConfiguration.Checked)]
[SkipOnCoreClr("Test timing out: https://github.com/dotnet/runtime/issues/2271", ~RuntimeConfiguration.Release)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/2271")]
public static void RunContinueWithStressTestsNoState()
{
Expand Down

0 comments on commit 348e06c

Please sign in to comment.