Revert "Revert "Helix proof of concept for unit tests (#224)""

This reverts commit 8c8e04e.
dotnet · Mar 8, 2019 · 24eed41 · 24eed41
1 parent 8c8e04e
commit 24eed41
Show file tree

Hide file tree

Showing 11 changed files with 118 additions and 59 deletions.
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -57,5 +57,9 @@ Note: if the Uri is a new place, you will need to add a subscription from that p
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>1d2edb248bc2dc37fa0669053b0c88f3bffffb26</Sha>
     </Dependency>
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19156.20">
+      <Uri>https://github.com/dotnet/arcade</Uri>
+      <Sha>b94a591377451bda3ac80e597d7faff03a06c29e</Sha>
+    </Dependency>
   </ToolsetDependencies>
 </Dependencies>
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -24,10 +24,18 @@
     <MicrosoftDotNetGenFacadesPackageVersion>1.0.0-beta.19156.20</MicrosoftDotNetGenFacadesPackageVersion>
     <MicrosoftDotNetXUnitExtensionsPackageVersion>2.4.0-beta.19156.20</MicrosoftDotNetXUnitExtensionsPackageVersion>
   </PropertyGroup>
-  <!-- Additional dependencies -->
+  <!-- XUnit-related unchanging dependencies -->
   <PropertyGroup>
-    <AccessibilityPackageVersion>4.6.0-alpha-27122-5</AccessibilityPackageVersion>
+    <XUnitVersion>2.4.1-pre.build.4059</XUnitVersion>
+    <XUnitAssertVersion>$(XUnitVersion)</XUnitAssertVersion>
+    <XUnitRunnerConsoleVersion>$(XUnitVersion)</XUnitRunnerConsoleVersion>
+    <XUnitRunnerVisualStudioVersion>$(XUnitVersion)</XUnitRunnerVisualStudioVersion>
+    <XUnitExtensibilityExecutionVersion>$(XUnitVersion)</XUnitExtensibilityExecutionVersion>
     <XUnitStaFactPackageVersion>0.3.5</XUnitStaFactPackageVersion>
+  </PropertyGroup>
+  <!-- Additional unchanging dependencies -->
+  <PropertyGroup>
+    <AccessibilityPackageVersion>4.6.0-alpha-27122-5</AccessibilityPackageVersion>
     <MoqPackageVersion>4.10.0</MoqPackageVersion>
     <MicrosoftTargetingPackNETFrameworkv472PackageVersion>1.0.0</MicrosoftTargetingPackNETFrameworkv472PackageVersion>
   </PropertyGroup>

diff --git a/eng/ci-helix-test.yml b/eng/ci-helix-test.yml
@@ -0,0 +1,30 @@
+parameters:
+  # All parameters are required for our purposes (except HelixType); see send-to-helix.yml for details
+  RepoName: ''
+  HelixType: 'tests/default'            
+  HelixTargetQueues: ''    
+  HelixAccessToken: ''          
+  XUnitProjects: ''
+  DisplayNamePrefix: ''        
+
+steps:
+- template: /eng/common/templates/steps/send-to-helix.yml
+  parameters:
+    HelixSource: ${{ parameters.RepoName }}
+    HelixType: ${{ parameters.HelixType }} 
+    HelixBuild: $(Build.BuildNumber)
+    HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+    HelixAccessToken: ${{ parameters.HelixAccessToken }}            # only defined for internal CI
+    Creator: ${{ parameters.RepoName }}         # required for public / external (when there is no access token)
+    XUnitProjects: ${{ parameters.XUnitProjects }} 
+    XUnitPublishTargetFramework: netcoreapp3.0  # Whatever tfm will work for our test projects for `dotnet publish`
+    XUnitRuntimeTargetFramework: netcoreapp2.0  # Whatever tfm to pick from the xunit package, it must exist in that package or the helix job will fail
+    XUnitRunnerVersion: '2.4.1'                 # Should match XUnitRunnerConsoleVersion in eng/Versions.props
+    IncludeDotNetCli: true
+    DotNetCliPackageType: 'sdk'
+    DotNetCliVersion: '3.0.100-preview-010184'  # MUST be official release: https://dotnet.microsoft.com/download/dotnet-core/3.0 ; does not need to match sdk we build against
+    EnableXUnitReporter: true
+    WaitForWorkItemCompletion: true
+    DisplayNamePrefix: ${{ parameters.DisplayNamePrefix }}
+    condition: succeeded()
+    continueOnError: false
diff --git a/eng/ci.yml b/eng/ci.yml
@@ -1,10 +1,8 @@
 parameters:
-
-  # Needed because agent pool can't be read from a user-defined variable (Azure DevOps limitation)
-  agentPool: dotnet-external-temp-vs2019
-
-  # Needed because runAsPublic is used in template expressions, which can't read from user-defined variables
-  runAsPublic: true
+  # This section is required because agent pool can't be read from a user-defined variable (Azure DevOps limitation)
+  agentPool: dotnet-external-temp-vs2019 
+  runAsPublic: true                      
+  repoName: dotnet/winforms              
 
 jobs:
 - template: /eng/common/templates/jobs/jobs.yml
@@ -15,7 +13,7 @@ jobs:
     enablePublishTestResults: true
     enablePublishBuildAssets: true
     enableTelemetry: true
-    helixRepo: dotnet/winforms
+    helixRepo: $(repoName)
 
     jobs:
     - job: Windows_NT
@@ -44,16 +42,19 @@ jobs:
           value: ''
 
         # needed for helix jobs
-        - name: _HelixAgentPool
+        - name: _UnitTestHelixAgentPool
           value: 'Windows.10.Amd64.Open'
         - name: _WinformsControlsTestBinDir
           value: $(BUILD.SOURCESDIRECTORY)\artifacts\bin\WinformsControlsTest\$(_BuildConfig)\netcoreapp3.0
         - name: _WinformsFuncTestBinDir
           value: $(BUILD.SOURCESDIRECTORY)\artifacts\bin\System.Windows.Forms.Func.Tests\$(_BuildConfig)\netcoreapp3.0
         - name: _HelixStagingDir
           value: $(BUILD.STAGINGDIRECTORY)\helix\functests
+        - name: _HelixToken
+          value: ''
 
-        # Override some values if we're building internally
+
+        # Override some values if we're building internally (not public)
         - ${{ if eq(parameters.runAsPublic, 'false') }}:
 
           # note: You have to use list syntax here (-name: value) or you will get errors about declaring the same variable multiple times
@@ -68,9 +69,8 @@ jobs:
           - name: _PublishBlobFeedUrl
             value: https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json
 
-          # note: Even though they are referenced here, user defined variables (like $(_SignType)) are not resolved 
-          # until the agent is running on the machine. They can be overridden any time before they are resolved,
-          # like in the job matrix below (see Build_Debug)
+          # note: Even though they are referenced here, user defined variables (like $(_SignType)) are not resolved until the agent
+          # is running on the machine. They can be overridden any time before they are resolved, like in the job matrix below (see Build_Debug)
           - name: _SignArgs
             value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:Sign=$(_Sign)
           - name: _PublishArgs
@@ -82,8 +82,11 @@ jobs:
           - name: _OfficialBuildIdArgs
             value: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
 
-          - name: _HelixAgentPool
+          # Windows.81.Amd64 will fail on Debug.Assert in TryGetSystemMetricsForDpi in UnsafeNativeMethods
+          - name: _UnitTestHelixAgentPool
             value: 'Windows.10.Amd64'
+          - name: _HelixToken
+            value: '$(helix-token)'
 
       strategy:
         matrix:
@@ -109,15 +112,21 @@ jobs:
           /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\Build.binlog          
         displayName: Build
 
-      # Run unit tests and rename binlog
-      - script: eng\cibuild.cmd
-          -configuration $(_BuildConfig)
-          $(_OfficialBuildIdArgs)
-          -test
-          /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\Test.binlog
-        displayName: Run Unit Tests     
-
+      # Run unit tests on Helix
+      - template: /eng/ci-helix-test.yml
+        parameters:
+          RepoName: ${{ parameters.repoName }}
+          HelixType: 'tests/unit'
+          HelixTargetQueues: $(_UnitTestHelixAgentPool)
+          HelixAccessToken: $(_HelixToken)            # only defined for internal CI
+          # Should be $(BUILD.SOURCESDIRECTOR)System.Windows.Forms/src/**/*.Tests.csproj, but cannot until https://github.com/dotnet/arcade/issues/2156 is fixed
+          XUnitProjects: '$(BUILD.SOURCESDIRECTORY)\src\System.Windows.Forms\tests\UnitTests\System.Windows.Forms.Tests.csproj;$(BUILD.SOURCESDIRECTORY)\src\System.Windows.Forms.Design\tests\UnitTests\System.Windows.Forms.Design.Tests.csproj;$(BUILD.SOURCESDIRECTORY)\src\System.Windows.Forms.Design.Editors\tests\UnitTests\System.Windows.Forms.Design.Editors.Tests.csproj'
+          DisplayNamePrefix: 'Send Unit Tests (Windows.10) to Helix'
+
       # Run integration tests and rename binlog
+      # These are not ready to run on Helix because:
+        # 1. Helix doesn't build projects (it only uses publish); we need .dotnet and also bin from build
+        # 2. we need an interactive pool for SendKeys to work
       - script: eng\cibuild.cmd
           -configuration $(_BuildConfig)
           $(_OfficialBuildIdArgs)
@@ -150,4 +159,4 @@ jobs:
         # Run component governance detection (only for release; only for internal)
         - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
           displayName: Component Governance Detection
-          condition: and(succeeded(), eq(variables['_BuildConfig'], 'Release'))
+          condition: and(succeeded(), eq(variables['_BuildConfig'], 'Release'))
diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml
@@ -1,32 +1,33 @@
 parameters:
-  HelixSource: 'pr/default'          # required -- sources must start with pr/, official/, prodcon/, or agent/
-  HelixType: 'tests/default/'        # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
-  HelixBuild: $(Build.BuildNumber)   # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
-  HelixTargetQueues: ''              # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
-  HelixAccessToken: ''               # required -- access token to make Helix API requests; should be provided by the appropriate variable group
-  HelixPreCommands: ''               # optional -- commands to run before Helix work item execution
-  HelixPostCommands: ''              # optional -- commands to run after Helix work item execution
-  WorkItemDirectory: ''              # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
-  WorkItemCommand: ''                # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
-  WorkItemTimeout: ''                # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
-  CorrelationPayloadDirectory: ''    # optional -- a directory to zip up and send to Helix as a correlation payload
-  XUnitProjects: ''                  # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
-  XUnitPublishTargetFramework: ''    # optional -- framework to use to publish your xUnit projects
-  XUnitRuntimeTargetFramework: ''    # optional -- framework to use for the xUnit console runner
-  XUnitRunnerVersion: ''             # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
-  IncludeDotNetCli: false            # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
-  DotNetCliPackageType: ''           # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
-  DotNetCliVersion: ''               # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
-  EnableXUnitReporter: false         # optional -- true enables XUnit result reporting to Mission Control
-  WaitForWorkItemCompletion: true    # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
-  IsExternal: false                  # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
-  Creator: ''                        # optional -- if the build is external, use this to specify who is sending the job
-  condition: succeeded()             # optional -- condition for step to execute; defaults to succeeded()
-  continueOnError: false             # optional -- determines whether to continue the build if the step errors; defaults to false
+  HelixSource: 'pr/default'              # required -- sources must start with pr/, official/, prodcon/, or agent/
+  HelixType: 'tests/default/'            # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+  HelixBuild: $(Build.BuildNumber)       # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+  HelixTargetQueues: ''                  # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
+  HelixAccessToken: ''                   # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+  HelixPreCommands: ''                   # optional -- commands to run before Helix work item execution
+  HelixPostCommands: ''                  # optional -- commands to run after Helix work item execution
+  WorkItemDirectory: ''                  # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+  WorkItemCommand: ''                    # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
+  WorkItemTimeout: ''                    # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
+  CorrelationPayloadDirectory: ''        # optional -- a directory to zip up and send to Helix as a correlation payload
+  XUnitProjects: ''                      # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+  XUnitPublishTargetFramework: ''        # optional -- framework to use to publish your xUnit projects
+  XUnitRuntimeTargetFramework: ''        # optional -- framework to use for the xUnit console runner
+  XUnitRunnerVersion: ''                 # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+  IncludeDotNetCli: false                # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+  DotNetCliPackageType: ''               # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  EnableXUnitReporter: false             # optional -- true enables XUnit result reporting to Mission Control
+  WaitForWorkItemCompletion: true        # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+  IsExternal: false                      # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
+  Creator: ''                            # optional -- if the build is external, use this to specify who is sending the job
+  DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO 
+  condition: succeeded()                 # optional -- condition for step to execute; defaults to succeeded()
+  continueOnError: false                 # optional -- determines whether to continue the build if the step errors; defaults to false
 
 steps:
   - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
-    displayName: Send job to Helix (Windows)
+    displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
     env:
       BuildConfig: $(_BuildConfig)
       HelixSource: ${{ parameters.HelixSource }}
@@ -54,7 +55,7 @@ steps:
     condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
     continueOnError: ${{ parameters.continueOnError }}
   - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
-    displayName: Send job to Helix (Unix)
+    displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
     env:
       BuildConfig: $(_BuildConfig)
       HelixSource: ${{ parameters.HelixSource }}

diff --git a/global.json b/global.json
@@ -9,6 +9,7 @@
     "version": "3.0.100-preview-010024"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19156.20"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19156.20",
+    "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19156.20"
   }
 }
diff --git a/src/Common/tests/InternalUtilitiesForTests/InternalUtilitiesForTests.csproj b/src/Common/tests/InternalUtilitiesForTests/InternalUtilitiesForTests.csproj
@@ -10,7 +10,7 @@
   <ItemGroup>
     <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
     <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
-    <PackageReference Include= "Microsoft.DotNet.XUnitExtensions" Version="$(MicrosoftDotNetXUnitExtensionsPackageVersion)" />
+    <PackageReference Include="Microsoft.DotNet.XUnitExtensions" Version="$(MicrosoftDotNetXUnitExtensionsPackageVersion)" />
     <PackageReference Include="Microsoft.Win32.Registry" Version="$(MicrosoftWin32RegistryPackageVersion)" />
   </ItemGroup>
 </Project>
diff --git a/...ows.Forms.Design.Editors/tests/UnitTests/System.Windows.Forms.Design.Editors.Tests.csproj b/...ows.Forms.Design.Editors/tests/UnitTests/System.Windows.Forms.Design.Editors.Tests.csproj
@@ -5,8 +5,8 @@
     <AssemblyName>System.Windows.Forms.Design.Editors.Tests</AssemblyName>
   </PropertyGroup>
 
-  <ItemGroup>
-    <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />    
+  <ItemGroup>    
+    <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
     <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
   </ItemGroup>
 

diff --git a/src/System.Windows.Forms.Design/tests/UnitTests/System.Windows.Forms.Design.Tests.csproj b/src/System.Windows.Forms.Design/tests/UnitTests/System.Windows.Forms.Design.Tests.csproj
@@ -6,6 +6,11 @@
     <LangVersion>7.3</LangVersion>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
+    <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\src\System.Windows.Forms.Design.csproj" />
     <Compile Include="..\..\..\Common\tests\CommonTestHelper.cs" Link="Common\CommonTestHelper.cs" />

diff --git a/...nTests/System.Windows.Forms.IntegrationTests/System.Windows.Forms.IntegrationTests.csproj b/...nTests/System.Windows.Forms.IntegrationTests/System.Windows.Forms.IntegrationTests.csproj
@@ -13,6 +13,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\..\src\System.Windows.Forms.csproj" />
+    <ProjectReference Include="..\WinformsControlsTest\WinformsControlsTest.csproj" />
   </ItemGroup>
 
 </Project>
diff --git a/...Windows.Forms/tests/IntegrationTests/System.Windows.Forms.IntegrationTests/TestHelpers.cs b/...Windows.Forms/tests/IntegrationTests/System.Windows.Forms.IntegrationTests/TestHelpers.cs
@@ -34,15 +34,15 @@ public static Process StartProcess(string byPathFromBinToExe)
                 throw new ArgumentException(nameof(byPathFromBinToExe) + " must end in a .exe");
             }
 
-            var dotnetPath = DotNetPath();
-            if (!Directory.Exists(dotnetPath))
-            {
-                throw new DirectoryNotFoundException(dotnetPath + " directory cannot be found.");
+            var dotnetPath = DotNetPath();	
+            if (!Directory.Exists(dotnetPath))	
+            {	
+                throw new DirectoryNotFoundException(dotnetPath + " directory cannot be found.");	
             }
 
             ProcessStartInfo startInfo = new ProcessStartInfo();
             startInfo.FileName = Path.Combine(BinPath(), byPathFromBinToExe.Trim('\\'));
-            startInfo.EnvironmentVariables["DOTNET_ROOT"] = dotnetPath;
+            startInfo.EnvironmentVariables["DOTNET_ROOT"] = dotnetPath;	// required
             // ...
 
             return StartProcess(startInfo);