Dynamially detect stubbed libraries to ignore tests (#4059)

Fixes #4017

packages/http-client-csharp/eng/scripts/Generate.ps1

@@ -1,5 +1,8 @@
 #Requires -Version 7.0
-param($filter)
+param(
+    $filter,
+    [bool]$Stubbed = $true
+)
 
 Import-Module "$PSScriptRoot\Generation.psm1" -DisableNameChecking -Force;
 
@@ -132,7 +135,7 @@ foreach ($directory in $directories) {
 
     $cadlRanchLaunchProjects.Add(($folders -join "-"), ("TestProjects/CadlRanch/$($subPath.Replace([System.IO.Path]::DirectorySeparatorChar, '/'))"))
     Write-Host "Generating $subPath" -ForegroundColor Cyan
-    Invoke (Get-TspCommand $specFile $generationDir $true)
+    Invoke (Get-TspCommand $specFile $generationDir $stubbed)
 
     # exit if the generation failed
     if ($LASTEXITCODE -ne 0) {

packages/http-client-csharp/eng/scripts/Get-CadlRanch-Coverage.ps1

@@ -1,5 +1,3 @@
-# cspell:ignore cadlranch
-
 #Requires -Version 7.0
 
 Import-Module "$PSScriptRoot\Generation.psm1" -DisableNameChecking -Force;
@@ -13,7 +11,6 @@ $specsDirectory = Join-Path $packageRoot 'node_modules' '@azure-tools' 'cadl-ran
 $cadlRanchRoot = Join-Path $packageRoot 'generator' 'TestProjects' 'CadlRanch'
 $directories = Get-ChildItem -Path "$cadlRanchRoot" -Directory -Recurse
 $cadlRanchCsproj = Join-Path $packageRoot 'generator' 'TestProjects' 'CadlRanch.Tests' 'TestProjects.CadlRanch.Tests.csproj'
-$runSettings = Join-Path $packageRoot 'eng' 'test-configurations' 'cadlranch.runsettings'
 
 $coverageDir = Join-Path $packageRoot 'generator' 'artifacts' 'coverage'
 
@@ -44,7 +41,7 @@ foreach ($directory in $directories) {
 
 # test all
 Write-Host "Testing $subPath" -ForegroundColor Cyan
-$command  = "dotnet test $cadlRanchCsproj --settings $runSettings"
+$command  = "dotnet test $cadlRanchCsproj"
 Invoke $command
 # exit if the testing failed
 if ($LASTEXITCODE -ne 0) {

packages/http-client-csharp/eng/scripts/Test-CadlRanch.ps1

@@ -1,6 +1,5 @@
-# cspell:ignore cadlranch
-
 #Requires -Version 7.0
+
 param($filter)
 
 Import-Module "$PSScriptRoot\Generation.psm1" -DisableNameChecking -Force;
@@ -14,7 +13,6 @@ $specsDirectory = "$packageRoot/node_modules/@azure-tools/cadl-ranch-specs"
 $cadlRanchRoot = Join-Path $packageRoot 'generator' 'TestProjects' 'CadlRanch'
 $directories = Get-ChildItem -Path "$cadlRanchRoot" -Directory -Recurse
 $cadlRanchCsproj = Join-Path $packageRoot 'generator' 'TestProjects' 'CadlRanch.Tests' 'TestProjects.CadlRanch.Tests.csproj'
-$runSettings = Join-Path $packageRoot 'eng' 'test-configurations' 'cadlranch.runsettings'
 
 $coverageDir = Join-Path $packageRoot 'generator' 'artifacts' 'coverage'
 
@@ -52,7 +50,7 @@ foreach ($directory in $directories) {
     }
 
     Write-Host "Testing $subPath" -ForegroundColor Cyan
-    $command  = "dotnet test $cadlRanchCsproj --filter `"FullyQualifiedName~$testFilter`" --settings $runSettings"
+    $command  = "dotnet test $cadlRanchCsproj --filter `"FullyQualifiedName~$testFilter`""
     Invoke $command
     # exit if the testing failed
     if ($LASTEXITCODE -ne 0) {

packages/http-client-csharp/generator/TestProjects/CadlRanch.Tests/Infrastructure/CadlRanchTestAttribute.cs

@@ -2,23 +2,85 @@
 // Licensed under the MIT License.
 
 using System;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.CSharp.Syntax;
 using NUnit.Framework;
 using NUnit.Framework.Interfaces;
 using NUnit.Framework.Internal;
 
 namespace TestProjects.CadlRanch.Tests
 {
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    internal class CadlRanchTestAttribute : TestAttribute, IApplyToTest
+    internal partial class CadlRanchTestAttribute : TestAttribute, IApplyToTest
     {
+        [GeneratedRegex("(?<=[a-z])([A-Z])")]
+        private static partial Regex ToKebabCase();
+
         public new void ApplyToTest(Test test)
         {
-            var runCadlRanchParam = TestContext.Parameters.Get("RunCadlRanch", "false");
-            if (!bool.TryParse(runCadlRanchParam, out var runCadlRanch) || !runCadlRanch)
+            string clientCodeDirectory = GetGeneratedDirectory(test);
+
+            var clientCsFile = GetClientCsFile(clientCodeDirectory);
+            if (clientCsFile is null || IsLibraryStubbed(clientCsFile))
             {
-                test.RunState = RunState.Ignored;
-                test.Properties.Set(PropertyNames.SkipReason, "Test skipped because RunCadlRanch parameter is not set to true.");
+                SkipTest(test);
             }
         }
+
+        private static bool IsLibraryStubbed(string clientCsFile)
+        {
+            SyntaxTree tree = CSharpSyntaxTree.ParseText(File.ReadAllText(clientCsFile));
+            CompilationUnitSyntax root = tree.GetCompilationUnitRoot();
+
+            var constructors = root.DescendantNodes()
+                .OfType<ConstructorDeclarationSyntax>()
+                .Where(c => c.Modifiers.Any(SyntaxKind.PublicKeyword))
+                .ToList();
+
+            if (constructors.Count != 0)
+            {
+                ConstructorDeclarationSyntax? constructorWithMostParameters = constructors
+                    .OrderByDescending(c => c.ParameterList.Parameters.Count)
+                    .FirstOrDefault();
+
+                return constructorWithMostParameters?.ExpressionBody != null;
+            }
+
+            return true;
+        }
+
+        private static void SkipTest(Test test)
+        {
+            test.RunState = RunState.Ignored;
+            test.Properties.Set(PropertyNames.SkipReason, $"Test skipped because {test.FullName} is currently a stubbed implementation.");
+        }
+
+        private static string? GetClientCsFile(string clientCodeDirectory)
+        {
+            return Directory.GetFiles(clientCodeDirectory, "*.cs", SearchOption.TopDirectoryOnly)
+                .Where(f => f.EndsWith("Client.cs", StringComparison.Ordinal))
+                .FirstOrDefault();
+        }
+
+        private static string GetGeneratedDirectory(Test test)
+        {
+            var namespaceParts = test.FullName.Split('.').Skip(3);
+            namespaceParts = namespaceParts.Take(namespaceParts.Count() - 2);
+            var clientCodeDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, "..", "..", "..", "..", "..", "TestProjects", "CadlRanch");
+            foreach (var part in namespaceParts)
+            {
+                clientCodeDirectory = Path.Combine(clientCodeDirectory, FixName(part));
+            }
+            return Path.Combine(clientCodeDirectory, "src", "Generated");
+        }
+
+        private static string FixName(string part)
+        {
+            return ToKebabCase().Replace(part, "-$1").ToLower();
+        }
     }
 }

packages/http-client-csharp/generator/TestProjects/CadlRanch.Tests/TestProjects.CadlRanch.Tests.csproj

@@ -3,6 +3,7 @@
   <ItemGroup>
     <PackageReference Include="System.Memory.Data" />
     <PackageReference Include="System.ClientModel" />
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" />
   </ItemGroup>
 
   <ItemGroup>
@@ -15,5 +16,5 @@
       <_Parameter2>$(ArtifactsDir)</_Parameter2>
     </AssemblyAttribute>
   </ItemGroup>
-  
+
 </Project>

packages/http-client-csharp/generator/docs/cadl-ranch.md

@@ -13,6 +13,13 @@ will live in the following folder inside the CadlRanch test projects
 
 The files that get generated here will only be stubbed public APIs. This is done to minimize the size of the repo and reduce PR diff noise when an internal implementation is modified which can potentially effect every model. Seeing the same diff in hundreds of files doesn't provide value it only introduces noise.
 
+If you want to manually generate the non stubbed version you can call Generate.ps1 with the name of the project you want to generate and set Stubbed to false.
+For example if you want to generate the non stubbed version of `http/authentication/api-key` you can do the following.
+
+```powershell
+./eng/scripts/Generate.ps1 http/authentication/api-key -Stubbed $false
+```
+
 ## Writing CadlRanchTests
 
 Generating the stubs allows us write tests against the public API surface that will compile. To do this we add a test class in same folder structure although this time we will modify the casing slightly to match dotnet standards.
@@ -32,9 +39,11 @@ public Task Valid() => Test(async (host) =>
 
 This test validates that we can successfully call the mock cadl ranch service and verifies that we receive a successful response with code 204. Notice that we are using `CadlRanchTest` attribute instead of the standard NUnit `Test` attribute. This is because the test will not complete successfully against the stub and we must generate the full library before running the test.
 
+The `CadlRanchTest` attribute will dynamically determine if the library is stubbed and mark the step as `Ignore` if it is. If it is not stubbed it will run the full test.
+
 ## Testing CadlRanch Scenarios
 
-All of this is automated into a script called `./eng/scripts/Test-CadlRanch.ps1`. This script will find all generated cadl ranch projects and regenerate each of them without using the `StubLibraryPlugin`. It will then run dotnet test using `./eng/test-configurations/cadlranch.runsettings` which will cause tests using the `CadlRanchTest` attribute to no longer be skipped. Finally it will restore the files back to the stubs if everything was successful and if not it will leave the files in place so you can investigate.
+All of this is automated into a script called `./eng/scripts/Test-CadlRanch.ps1`. This script will find all generated cadl ranch projects and regenerate each of them without using the `StubLibraryPlugin`. It will then run dotnet test which will cause tests using the `CadlRanchTest` attribute to no longer be skipped. Finally it will restore the files back to the stubs if everything was successful and if not it will leave the files in place so you can investigate.
 
 <details>
 <Summary>Here is an example output for one library</Summary>
@@ -96,9 +105,13 @@ The plugin does not skip generating the methods bodies and xml docs it simply re
 
 ## Debugging CadlRanch Tests
 
-To debug one of the `CadlRanchTest` you will need to set your launch settings in the TestExplorer to the cadlranch.runsettings otherwise the test will simply be skipped.
+To debug one of the `CadlRanchTest` you will need to generate the non stubbed library first by calling Generate.ps1 with Stubbed set to false.
+
+```powershell
+./eng/scripts/Generate.ps1 http/authentication/api-key -Stubbed $false
+```
 
-![alt text](runsettings.png)
+If you don't do this the test will be ignored by `CadlRanchTest` attribute since you cannot run a test when the library has no implementation.
 
 ## Problematic Specs