From 7785fea63767cbdf20de2fc56523cb15362c49f9 Mon Sep 17 00:00:00 2001 From: Gerald Versluis Date: Fri, 5 Jul 2024 20:44:33 +0200 Subject: [PATCH] Run UI tests per groups of categories in CI (#22609) * Run tests by category groups * Some changes * Fix * Update ui-tests-compatibility-steps.yml * No categories for compatibility project * Update ui-tests-compatibility-steps.yml --- eng/devices/android.cake | 6 +- eng/devices/catalyst.cake | 18 ++--- eng/devices/devices-shared.cake | 9 ++- eng/devices/ios.cake | 6 +- eng/devices/windows.cake | 14 +++- .../common/ui-tests-build-sample.yml | 69 ++++++++++++++++ .../common/ui-tests-compatibility-steps.yml | 4 +- eng/pipelines/common/ui-tests-steps.yml | 76 ++++++++---------- eng/pipelines/common/ui-tests.yml | 78 +++++++++++++++++-- eng/pipelines/ui-tests.yml | 2 +- .../Tests/ScrollViewUITests.cs | 4 - .../UITestCategories.cs | 2 + 12 files changed, 209 insertions(+), 79 deletions(-) create mode 100644 eng/pipelines/common/ui-tests-build-sample.yml diff --git a/eng/devices/android.cake b/eng/devices/android.cake index d91c5ff6198c..4ddbcab4aaf9 100644 --- a/eng/devices/android.cake +++ b/eng/devices/android.cake @@ -18,9 +18,6 @@ var testAppInstrumentation = Argument("instrumentation", EnvironmentVariable("AN var testResultsPath = Argument("results", EnvironmentVariable("ANDROID_TEST_RESULTS") ?? GetTestResultsDirectory()?.FullPath); var deviceCleanupEnabled = Argument("cleanup", true); -// Test where clause -string testWhere = Argument("where", EnvironmentVariable("NUNIT_TEST_WHERE") ?? ""); - // Device details var deviceSkin = Argument("skin", EnvironmentVariable("ANDROID_TEST_SKIN") ?? "Nexus 5X"); var androidAvd = "DEVICE_TESTS_EMULATOR"; @@ -96,7 +93,6 @@ Task("uitest-build") }); Task("uitest") - .IsDependentOn("uitest-build") .Does(() => { ExecuteUITests(projectPath, testAppProjectPath, testAppPackageName, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, "", androidVersion, dotnetToolPath, testAppInstrumentation); @@ -305,7 +301,7 @@ void ExecuteUITests(string project, string app, string appPackageName, string de var name = System.IO.Path.GetFileNameWithoutExtension(project); var binlog = $"{binDir}/{name}-{config}-{platform}.binlog"; var appiumLog = $"{binDir}/appium_{platform}.log"; - var resultsFileName = $"{name}-{config}-{platform}"; + var resultsFileName = SanitizeTestResultsFilename($"{name}-{config}-{platform}-{testFilter}"); DotNetBuild(project, new DotNetBuildSettings { diff --git a/eng/devices/catalyst.cake b/eng/devices/catalyst.cake index 6b5edb096671..5fe0fb1a2a2b 100644 --- a/eng/devices/catalyst.cake +++ b/eng/devices/catalyst.cake @@ -58,7 +58,6 @@ Task("uitest-build") }); Task("uitest") - .IsDependentOn("uitest-build") .Does(() => { ExecuteUITests(projectPath, testAppProjectPath, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, runtimeIdentifier, dotnetToolPath); @@ -131,21 +130,20 @@ void ExecuteUITests(string project, string app, string device, string resultsDir } // Launch the app so it can be found by the test runner - DotNetBuild(app, new DotNetBuildSettings - { - Configuration = config, - Framework = tfm, - ToolPath = toolPath, - ArgumentCustomization = args => args - .Append("/t:Run") - }); + StartProcess("chmod", $"+x {testApp}/Contents/MacOS/Controls.TestCases.HostApp"); + + var p = new System.Diagnostics.Process(); + p.StartInfo.UseShellExecute = true; + p.StartInfo.FileName = "open"; + p.StartInfo.Arguments = testApp; + p.Start(); Information("Build UITests project {0}", project); var name = System.IO.Path.GetFileNameWithoutExtension(project); var binlog = $"{binDir}/{name}-{config}-mac.binlog"; var appiumLog = $"{binDir}/appium_mac.log"; - var resultsFileName = $"{name}-{config}-catalyst"; + var resultsFileName = SanitizeTestResultsFilename($"{name}-{config}-catalyst-{testFilter}"); DotNetBuild(project, new DotNetBuildSettings { diff --git a/eng/devices/devices-shared.cake b/eng/devices/devices-shared.cake index dc2d61ab9de4..6b99a2d483b7 100644 --- a/eng/devices/devices-shared.cake +++ b/eng/devices/devices-shared.cake @@ -45,7 +45,7 @@ IEnumerable GetTestApplications(string project, string device, string co const string artifactsDir = "../../artifacts/bin/"; bool isAndroid = tfm.Contains("android"); - var binDir = new DirectoryPath(project).Combine($"{binDirBase}/{config}/{tfm}/{rid}"); + var binDir = new DirectoryPath(project).Combine($"{config}/{tfm}/{rid}"); IEnumerable applications; if (isAndroid) @@ -222,3 +222,10 @@ void ExecuteWithRetries(Func action, int retries) System.Threading.Thread.Sleep(1000); } } + +string SanitizeTestResultsFilename(string input) +{ + string resultFilename = input.Replace("|", "_").Replace("TestCategory=", ""); + + return resultFilename; +} diff --git a/eng/devices/ios.cake b/eng/devices/ios.cake index 2d83da80853d..9e9871fcc5ae 100644 --- a/eng/devices/ios.cake +++ b/eng/devices/ios.cake @@ -18,9 +18,6 @@ var platform = testDevice.ToLower().Contains("simulator") ? "iPhoneSimulator" : var runtimeIdentifier = Argument("rid", EnvironmentVariable("IOS_RUNTIME_IDENTIFIER") ?? GetDefaultRuntimeIdentifier(testDevice)); var deviceCleanupEnabled = Argument("cleanup", true); -// Test where clause -string testWhere = Argument("where", EnvironmentVariable("NUNIT_TEST_WHERE") ?? ""); - // Device details var udid = Argument("udid", EnvironmentVariable("IOS_SIMULATOR_UDID") ?? ""); var iosVersion = Argument("apiversion", EnvironmentVariable("IOS_PLATFORM_VERSION") ?? DefaultVersion); @@ -87,7 +84,6 @@ Task("uitest-build") }); Task("uitest") - .IsDependentOn("uitest-build") .Does(() => { ExecuteUITests(projectPath, testAppProjectPath, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, runtimeIdentifier, iosVersion, dotnetToolPath); @@ -212,7 +208,7 @@ void ExecuteUITests(string project, string app, string device, string resultsDir var name = System.IO.Path.GetFileNameWithoutExtension(project); var binlog = $"{binDir}/{name}-{config}-ios.binlog"; var appiumLog = $"{binDir}/appium_ios.log"; - var resultsFileName = $"{name}-{config}-ios"; + var resultsFileName = SanitizeTestResultsFilename($"{name}-{config}-ios-{testFilter}"); DotNetBuild(project, new DotNetBuildSettings { diff --git a/eng/devices/windows.cake b/eng/devices/windows.cake index 449926f74ffc..17a3aba88b34 100644 --- a/eng/devices/windows.cake +++ b/eng/devices/windows.cake @@ -212,7 +212,15 @@ Task("Test") Information("Cleaned directories"); var testResultsPath = MakeAbsolute((DirectoryPath)TEST_RESULTS).FullPath.Replace("/", "\\"); - var testResultsFile = testResultsPath + $"\\TestResults-{PACKAGEID.Replace(".", "_")}.xml"; + var testResultsFile = testResultsPath + $"\\TestResults-{PACKAGEID.Replace(".", "_")}"; + + if (!string.IsNullOrWhiteSpace(testFilter)) + { + testResultsFile += SanitizeTestResultsFilename($"-{testFilter}"); + } + + testResultsFile += ".xml"; + var testsToRunFile = MakeAbsolute((DirectoryPath)TEST_RESULTS).FullPath.Replace("/", "\\") + $"\\devicetestcategories.txt"; Information($"Test Results File: {testResultsFile}"); @@ -432,7 +440,7 @@ Task("SetupTestPaths") } var winVersion = $"{dotnetVersion}-windows{windowsVersion}"; - var binDir = TEST_APP_PROJECT.GetDirectory().Combine("bin").Combine(CONFIGURATION + "/" + winVersion).Combine(DOTNET_PLATFORM); + var binDir = TEST_APP_PROJECT.GetDirectory().Combine("Controls.TestCases.HostApp").Combine(CONFIGURATION + "/" + winVersion).Combine(DOTNET_PLATFORM); Information("BinDir: {0}", binDir); var apps = GetFiles(binDir + "/*.exe").Where(c => !c.FullPath.EndsWith("createdump.exe")); if (apps.Count() == 0) @@ -464,7 +472,7 @@ Task("SetupTestPaths") binDir = MakeAbsolute(new DirectoryPath(arcadeBin + "/Essentials.DeviceTests/" + CONFIGURATION + "/" + winVersion).Combine(DOTNET_PLATFORM)); } - Information("Looking for .app in arcade binDir {0}", binDir); + Information("Looking for .exe in arcade binDir {0}", binDir); apps = GetFiles(binDir + "/*.exe").Where(c => !c.FullPath.EndsWith("createdump.exe")); if(apps.Count() == 0 ) { diff --git a/eng/pipelines/common/ui-tests-build-sample.yml b/eng/pipelines/common/ui-tests-build-sample.yml new file mode 100644 index 000000000000..c9d411062eda --- /dev/null +++ b/eng/pipelines/common/ui-tests-build-sample.yml @@ -0,0 +1,69 @@ +parameters: + platform: '' # [ android, ios, windows, catalyst ] + path: '' # path to csproj + device: '' # the xharness device to use + cakeArgs: '' # additional cake args + app: '' #path to app to test + version: '' #the iOS version' + provisionatorChannel: 'latest' + agentPoolAccessToken: '' + configuration : "Release" + testFilter: '' + +steps: + - ${{ if eq(parameters.platform, 'ios')}}: + - bash: | + chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh + $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh + displayName: 'Clean bot' + continueOnError: true + timeoutInMinutes: 60 + + - template: provision.yml + parameters: + skipProvisioning: ${{ eq(parameters.platform, 'windows') }} + skipAndroidSdks: ${{ ne(parameters.platform, 'android') }} + skipXcode: ${{ or(eq(parameters.platform, 'android'), eq(parameters.platform, 'windows')) }} + provisionatorChannel: ${{ parameters.provisionatorChannel }} + + - task: PowerShell@2 + condition: ne('${{ parameters.platform }}' , 'windows') + inputs: + targetType: 'inline' + script: | + defaults write -g NSAutomaticCapitalizationEnabled -bool false + defaults write -g NSAutomaticTextCompletionEnabled -bool false + defaults write -g NSAutomaticSpellingCorrectionEnabled -bool false + displayName: "Modify defaults" + continueOnError: true + + - pwsh: ./build.ps1 --target=dotnet --configuration="${{ parameters.configuration }}" --verbosity=diagnostic + displayName: 'Install .NET' + retryCountOnTaskFailure: 2 + env: + DOTNET_TOKEN: $(dotnetbuilds-internal-container-read-token) + PRIVATE_BUILD: $(PrivateBuild) + + - pwsh: echo "##vso[task.prependpath]$(DotNet.Dir)" + displayName: 'Add .NET to PATH' + + - pwsh: ./build.ps1 --target=dotnet-buildtasks --configuration="${{ parameters.configuration }}" + displayName: 'Build the MSBuild Tasks' + + - pwsh: ./build.ps1 --target=dotnet-samples --configuration="${{ parameters.configuration }}" --${{ parameters.platform }} --verbosity=diagnostic --usenuget=false + displayName: 'Build the samples' + + - bash: | + if [ -f "$HOME/Library/Logs/CoreSimulator/*" ]; then rm -r $HOME/Library/Logs/CoreSimulator/*; fi + if [ -f "$HOME/Library/Logs/DiagnosticReports/*" ]; then rm -r $HOME/Library/Logs/DiagnosticReports/*; fi + displayName: Delete Old Simulator Logs + condition: ${{ eq(parameters.platform, 'ios') }} + continueOnError: true + + - publish: $(System.DefaultWorkingDirectory)/artifacts/bin + condition: ne('${{ parameters.platform }}' , 'windows') + artifact: ui-tests-samples + + - publish: $(System.DefaultWorkingDirectory)/artifacts/bin + condition: eq('${{ parameters.platform }}' , 'windows') + artifact: ui-tests-samples-windows \ No newline at end of file diff --git a/eng/pipelines/common/ui-tests-compatibility-steps.yml b/eng/pipelines/common/ui-tests-compatibility-steps.yml index 8885cb1466fe..28f29d1eaa6e 100644 --- a/eng/pipelines/common/ui-tests-compatibility-steps.yml +++ b/eng/pipelines/common/ui-tests-compatibility-steps.yml @@ -8,9 +8,9 @@ parameters: provisionatorChannel: 'latest' agentPoolAccessToken: '' configuration : "Release" - where : "cat==Issues" targetSample: "dotnet-legacy-controlgallery" provisionPlatform: "windows" + testFilter: '' steps: - bash: | @@ -56,7 +56,7 @@ steps: condition: ${{ eq(parameters.platform, 'ios') }} continueOnError: true - - pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=cg-uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic --where="${{ parameters.where }}" + - pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=cg-uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic displayName: $(Agent.JobName) # retryCountOnTaskFailure: 2 diff --git a/eng/pipelines/common/ui-tests-steps.yml b/eng/pipelines/common/ui-tests-steps.yml index 37edc0526431..9e28c5579b6e 100644 --- a/eng/pipelines/common/ui-tests-steps.yml +++ b/eng/pipelines/common/ui-tests-steps.yml @@ -8,33 +8,28 @@ parameters: provisionatorChannel: 'latest' agentPoolAccessToken: '' configuration : "Release" + testFilter: '' steps: - - ${{ if eq(parameters.platform, 'ios')}}: - - bash: | - chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh - $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh - displayName: 'Clean bot' - continueOnError: true - timeoutInMinutes: 60 - - - template: provision.yml - parameters: - skipProvisioning: ${{ eq(parameters.platform, 'windows') }} - skipAndroidSdks: ${{ ne(parameters.platform, 'android') }} - skipXcode: ${{ or(eq(parameters.platform, 'android'), eq(parameters.platform, 'windows')) }} - provisionatorChannel: ${{ parameters.provisionatorChannel }} - - - task: PowerShell@2 + - task: DownloadPipelineArtifact@2 condition: ne('${{ parameters.platform }}' , 'windows') inputs: - targetType: 'inline' - script: | - defaults write -g NSAutomaticCapitalizationEnabled -bool false - defaults write -g NSAutomaticTextCompletionEnabled -bool false - defaults write -g NSAutomaticSpellingCorrectionEnabled -bool false - displayName: "Modify defaults" - continueOnError: true + artifact: ui-tests-samples + + - task: DownloadPipelineArtifact@2 + condition: eq('${{ parameters.platform }}' , 'windows') + inputs: + artifact: ui-tests-samples-windows + + - pwsh: ./build.ps1 --target=dotnet --configuration="${{ parameters.configuration }}" --verbosity=diagnostic + displayName: 'Install .NET' + retryCountOnTaskFailure: 2 + env: + DOTNET_TOKEN: $(dotnetbuilds-internal-container-read-token) + PRIVATE_BUILD: $(PrivateBuild) + + - pwsh: echo "##vso[task.prependpath]$(DotNet.Dir)" + displayName: 'Add .NET to PATH' # AzDO hosted agents default to 1024x768; set something bigger for Windows UI tests - task: ScreenResolutionUtility@1 @@ -60,30 +55,25 @@ steps: env: APPIUM_HOME: $(APPIUM_HOME) - - pwsh: ./build.ps1 --target=dotnet --configuration="${{ parameters.configuration }}" --verbosity=diagnostic - displayName: 'Install .NET' - retryCountOnTaskFailure: 2 - env: - DOTNET_TOKEN: $(dotnetbuilds-internal-container-read-token) - PRIVATE_BUILD: $(PrivateBuild) - - - pwsh: echo "##vso[task.prependpath]$(DotNet.Dir)" - displayName: 'Add .NET to PATH' + - pwsh: | + $command = "./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=uitest --project=""${{ parameters.path }}""" + $command += " --appproject=""${{ parameters.app }}"" --device=""${{ parameters.device }}"" --apiversion=""${{ parameters.version }}"" --configuration=""${{ parameters.configuration }}""" + $command += " --results=""$(TestResultsDirectory)"" --binlog=""$(LogDirectory)"" ${{ parameters.cakeArgs }} --verbosity=diagnostic" + + $testFilter = "" - - pwsh: ./build.ps1 --target=dotnet-buildtasks --configuration="${{ parameters.configuration }}" - displayName: 'Build the MSBuild Tasks' + "${{ parameters.testFilter }}".Split(",") | ForEach { + $testFilter += "TestCategory=" + $_ + "|" + } - - pwsh: ./build.ps1 --target=dotnet-samples --configuration="${{ parameters.configuration }}" --${{ parameters.platform }} --verbosity=diagnostic --usenuget=false - displayName: 'Build the samples' + $testFilter = $testFilter.TrimEnd("|") - - bash: | - if [ -f "$HOME/Library/Logs/CoreSimulator/*" ]; then rm -r $HOME/Library/Logs/CoreSimulator/*; fi - if [ -f "$HOME/Library/Logs/DiagnosticReports/*" ]; then rm -r $HOME/Library/Logs/DiagnosticReports/*; fi - displayName: Delete Old Simulator Logs - condition: ${{ eq(parameters.platform, 'ios') }} - continueOnError: true + # Cake does not allow empty parameters, so check if our filter is empty before adding it + if ($testFilter) { + $command += " --test-filter ""$testFilter""" + } - - pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic + Invoke-Expression $command displayName: $(Agent.JobName) ${{ if ne(parameters.platform, 'android')}}: retryCountOnTaskFailure: 1 diff --git a/eng/pipelines/common/ui-tests.yml b/eng/pipelines/common/ui-tests.yml index 9ae31afb61cb..2cc19a42d6ac 100644 --- a/eng/pipelines/common/ui-tests.yml +++ b/eng/pipelines/common/ui-tests.yml @@ -10,6 +10,23 @@ parameters: provisionatorChannel: 'latest' agentPoolAccessToken: '' runCompatibilityTests: false + categoryGroupsToTest: + # Make sure that this list is always up-to-date with src/Controls/tests/TestCases.Shared.Tests/UITestCategories.cs + # we might want to improve this somehow depending on how much the categories change over time + - 'Accessibility,ActionSheet,ActivityIndicator,Animation,AppLinks' + - 'Border,BoxView,Brush,Button' + - 'CarouselView,Cells,CheckBox,CollectionView,ContextActions,CustomRenderers' + - 'DatePicker,Dispatcher,DisplayAlert,DisplayPrompt,DragAndDrop' + - 'Editor,Effects,Entry,FlyoutPage,Focus,Frame,Gestures' + - 'Image,ImageButton,IndicatorView,InputTransparent,IsEnabled,IsVisible' + - 'Label,Layout,Lifecycle,ListView' + - 'ManualReview,Maps,Navigation' + - 'Page,Performance,Picker,ProgressBar' + - 'RadioButton,RefreshView' + - 'ScrollView,SearchBar,Shape,Shell,Slider,SoftInput,Stepper,Switch,SwipeView' + - 'TabbedPage,TableView,TimePicker,TitleView,ToolbarItem' + - 'ViewBaseTests,Visual,WebView,Window' + projects: - name: name desc: Human Description @@ -24,16 +41,48 @@ parameters: compatibilityiOSApp: /optional/path/to/app.csproj stages: + - stage: build_ui_tests + displayName: Build UITests Sample App + dependsOn: [] + jobs: + - job: build_ui_tests + displayName: Build Sample App + pool: ${{ parameters.androidPool }} + variables: + REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) + APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ + steps: + - template: ui-tests-build-sample.yml + + - stage: build_ui_tests_windows + displayName: Build UITests Windows Sample App + dependsOn: [] + jobs: + - job: build_ui_tests + displayName: Build Sample App (Windows) + pool: ${{ parameters.windowsPool }} + variables: + REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE) + APPIUM_HOME: $(System.DefaultWorkingDirectory)/.appium/ + steps: + - template: ui-tests-build-sample.yml + parameters: + platform: windows - stage: android_ui_tests displayName: Android UITests - dependsOn: [] + dependsOn: build_ui_tests jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.android, '') }}: - ${{ each api in parameters.androidApiLevels }}: - ${{ if not(containsValue(project.androidApiLevelsExclude, api)) }}: - job: android_ui_tests_${{ project.name }}_${{ api }} + strategy: + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: 240 # how long to run the job before automatically cancelling workspace: clean: all @@ -55,16 +104,22 @@ stages: device: android-emulator-64_${{ api }} provisionatorChannel: ${{ parameters.provisionatorChannel }} agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} + testFilter: $(CATEGORYGROUP) - stage: ios_ui_tests displayName: iOS UITests - dependsOn: [] + dependsOn: build_ui_tests jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.ios, '') }}: - ${{ each version in parameters.iosVersions }}: - ${{ if not(containsValue(project.iosVersionsExclude, version)) }}: - job: ios_ui_tests_${{ project.name }}_${{ replace(version, '.', '_') }} + strategy: + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: 240 # how long to run the job before automatically cancelling workspace: clean: all @@ -89,14 +144,20 @@ stages: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} + testFilter: $(CATEGORYGROUP) - stage: winui_ui_tests displayName: WinUI UITests - dependsOn: [] + dependsOn: build_ui_tests_windows jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.winui, '') }}: - job: winui_ui_tests_${{ project.name }} + strategy: + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: 240 # how long to run the job before automatically cancelling workspace: clean: all @@ -114,14 +175,20 @@ stages: app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} + testFilter: $(CATEGORYGROUP) - stage: mac_ui_tests displayName: macOS UITests - dependsOn: [] + dependsOn: build_ui_tests jobs: - ${{ each project in parameters.projects }}: - ${{ if ne(project.mac, '') }}: - job: mac_ui_tests_${{ project.name }} + strategy: + matrix: + ${{ each categoryGroup in parameters.categoryGroupsToTest }}: + ${{ categoryGroup }}: + CATEGORYGROUP: ${{ categoryGroup }} timeoutInMinutes: 240 # how long to run the job before automatically cancelling workspace: clean: all @@ -140,6 +207,7 @@ stages: app: ${{ project.app }} provisionatorChannel: ${{ parameters.provisionatorChannel }} agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} + testFilter: $(CATEGORYGROUP) - ${{ if eq(parameters.runCompatibilityTests, true) }}: - stage: android_compatibility_ui_tests @@ -209,4 +277,4 @@ stages: ${{ if ne(version, 'latest') }}: device: ios-simulator-64_${{ version }} provisionatorChannel: ${{ parameters.provisionatorChannel }} - agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} + agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }} \ No newline at end of file diff --git a/eng/pipelines/ui-tests.yml b/eng/pipelines/ui-tests.yml index 217c21551e6c..c7e09bfb3ffc 100644 --- a/eng/pipelines/ui-tests.yml +++ b/eng/pipelines/ui-tests.yml @@ -143,7 +143,7 @@ stages: desc: Controls androidApiLevelsExclude: [25] # Ignore for now API25 since the runs's are not stable android: $(System.DefaultWorkingDirectory)/src/Controls/tests/TestCases.Android.Tests/Controls.TestCases.Android.Tests.csproj - app: $(System.DefaultWorkingDirectory)/src/Controls/tests/TestCases.HostApp/Controls.TestCases.HostApp.csproj + app: $(Pipeline.Workspace)/Controls.TestCases.HostApp/ iosVersionsExclude: [ '12.4'] # Ignore iOS 12.4 while we can't make it work on CI ios: $(System.DefaultWorkingDirectory)/src/Controls/tests/TestCases.iOS.Tests/Controls.TestCases.iOS.Tests.csproj winui: $(System.DefaultWorkingDirectory)/src/Controls/tests/TestCases.WinUI.Tests/Controls.TestCases.WinUI.Tests.csproj diff --git a/src/Controls/tests/TestCases.Shared.Tests/Tests/ScrollViewUITests.cs b/src/Controls/tests/TestCases.Shared.Tests/Tests/ScrollViewUITests.cs index 6e41a367f975..e159d6989467 100644 --- a/src/Controls/tests/TestCases.Shared.Tests/Tests/ScrollViewUITests.cs +++ b/src/Controls/tests/TestCases.Shared.Tests/Tests/ScrollViewUITests.cs @@ -21,7 +21,6 @@ protected override void FixtureSetup() } [Test] - [Category(UITestCategories.ScrollView)] [Description("Scroll element to the start")] public void ScrollToElement1Start() { @@ -43,7 +42,6 @@ public void ScrollToElement1Start() } [Test] - [Category(UITestCategories.ScrollView)] [Description("Scroll element to the center")] public void ScrollToElement2Center() { @@ -68,7 +66,6 @@ public void ScrollToElement2Center() } [Test] - [Category(UITestCategories.ScrollView)] [Description("Scroll element to the end")] public void ScrollToElement3End() { @@ -130,7 +127,6 @@ public void ScrollToYTwice() } #if ANDROID || IOS [Test] - [Category(UITestCategories.ScrollView)] [Description("Scroll down the ScrollView using a gesture")] public void ScrollUpAndDownWithGestures() { diff --git a/src/Controls/tests/TestCases.Shared.Tests/UITestCategories.cs b/src/Controls/tests/TestCases.Shared.Tests/UITestCategories.cs index 0547136f43f8..53e0956279a1 100644 --- a/src/Controls/tests/TestCases.Shared.Tests/UITestCategories.cs +++ b/src/Controls/tests/TestCases.Shared.Tests/UITestCategories.cs @@ -1,5 +1,7 @@ namespace Microsoft.Maui.TestCases.Tests { + // Make sure that this list is always up-to-date with eng/pipelines/common/ui-tests.yml + // we might want to improve this somehow depending on how much the categories change over time internal static class UITestCategories { public const string ViewBaseTests = "ViewBaseTests";