From 2cfff8776613b41808cc5155b4129ac64ba232a9 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Wed, 27 May 2026 12:51:05 -0400 Subject: [PATCH 01/16] [CI] Use ACES for CI builds --- tools/devops/automation/build-pipeline.yml | 6 +++++ .../templates/build/build-stage.yml | 27 +++++++++++++------ .../automation/templates/main-stage.yml | 5 ++++ .../automation/templates/variables/common.yml | 6 +++++ 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/tools/devops/automation/build-pipeline.yml b/tools/devops/automation/build-pipeline.yml index d3e3f8b9389..5b7c186be1f 100644 --- a/tools/devops/automation/build-pipeline.yml +++ b/tools/devops/automation/build-pipeline.yml @@ -36,6 +36,11 @@ parameters: displayName: 'Push Nugets (Maestro)' default: true + - name: useACES + type: boolean + displayName: 'Use ACES shared pool' + default: true + resources: repositories: - repository: self @@ -132,6 +137,7 @@ extends: forceInsertion: ${{ parameters.forceInsertion }} pushNugets: ${{ parameters.pushNugets }} pushNugetsToMaestro: ${{ parameters.pushNugetsToMaestro }} + useACES: ${{ parameters.useACES }} - template: localization/v1.yml@yaml-templates parameters: diff --git a/tools/devops/automation/templates/build/build-stage.yml b/tools/devops/automation/templates/build/build-stage.yml index c03d3abe9a3..3bd8bc1f799 100644 --- a/tools/devops/automation/templates/build/build-stage.yml +++ b/tools/devops/automation/templates/build/build-stage.yml @@ -38,6 +38,10 @@ parameters: - name: use1ES type: boolean + - name: useACES + type: boolean + default: false + jobs: # This job performs the build of the nuget pkgs and the framework pkgs. There are two interesting dependencies in this job: - job: build @@ -66,14 +70,21 @@ jobs: BRANCH_NAME: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] XHARNESS_LABELS: $[ stageDependencies.configure_build.configure.outputs['labels.xharness_labels'] ] RUN_MAC_TESTS: $[ stageDependencies.configure_build.configure.outputs['decisions.RUN_MAC_TESTS'] ] - pool: - os: macOS - name: ${{ parameters.pool }} - demands: - - Agent.OS -equals Darwin - - Agent.OSVersion -gtVersion $(minimumMacOSVersion) - - macOS.Name -equals ${{ parameters.macOSName }} - - XcodeChannel -equals ${{ parameters.xcodeChannel }} + ${{ if parameters.useACES }}: + pool: + os: macOS + name: $(CIBuildPoolACES) + demands: + - ImageOverride -equals $(CIBuildPoolACESImage) + ${{ else }}: + pool: + os: macOS + name: ${{ parameters.pool }} + demands: + - Agent.OS -equals Darwin + - Agent.OSVersion -gtVersion $(minimumMacOSVersion) + - macOS.Name -equals ${{ parameters.macOSName }} + - XcodeChannel -equals ${{ parameters.xcodeChannel }} steps: - template: build-pkgs.yml diff --git a/tools/devops/automation/templates/main-stage.yml b/tools/devops/automation/templates/main-stage.yml index c06cfb6e682..671be873a5d 100644 --- a/tools/devops/automation/templates/main-stage.yml +++ b/tools/devops/automation/templates/main-stage.yml @@ -56,6 +56,10 @@ parameters: type: string default: '' + - name: useACES + type: boolean + default: false + stages: - stage: configure_build @@ -103,6 +107,7 @@ stages: xqaCertPass: $(xqa--certificates--password) pool: ${{ parameters.pool }} use1ES: true + useACES: ${{ parameters.useACES }} # .NET Release Prep and VS Insertion Stages, only execute them when the build comes from an official branch and is not a schedule build from OneLoc # setting the stage at this level makes the graph of the UI look better, else the lines overlap and is not clear. diff --git a/tools/devops/automation/templates/variables/common.yml b/tools/devops/automation/templates/variables/common.yml index c596f2b2893..2c6e6f019fb 100644 --- a/tools/devops/automation/templates/variables/common.yml +++ b/tools/devops/automation/templates/variables/common.yml @@ -44,6 +44,12 @@ variables: value: 'VSEng-Xamarin-RedmondMacBuildPool-iOS-Trusted' - name: CIBuildPoolUrl value: 'https://devdiv.visualstudio.com/_settings/agentpools?poolId=367&view=agents' +- name: CIBuildPoolACES + value: 'AcesShared' +- name: CIBuildPoolACESUrl + value: 'https://dev.azure.com/devdiv/DevDiv/_settings/agentqueues?queueId=8259&view=jobs' +- name: CIBuildPoolACESImage + value: 'ACES_VM_SharedPool_Tahoe' # override the default build revision - name: BUILD_REVISION From 99e26041130ffced365d715f4fca0927d79feec0 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Wed, 27 May 2026 14:10:13 -0400 Subject: [PATCH 02/16] [CI] Use ACES for CI API Diff --- tools/devops/automation/run-ci-api-diff.yml | 1 + .../automation/templates/api-diff-stage.yml | 4 ++++ .../templates/build/api-diff-stage.yml | 24 +++++++++++++------ .../templates/pipelines/api-diff-pipeline.yml | 5 ++++ 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/tools/devops/automation/run-ci-api-diff.yml b/tools/devops/automation/run-ci-api-diff.yml index 8ccd511ab0c..3a84fa82c11 100644 --- a/tools/devops/automation/run-ci-api-diff.yml +++ b/tools/devops/automation/run-ci-api-diff.yml @@ -17,3 +17,4 @@ extends: parameters: isPR: false pool: $(CIBuildPool) + useACES: true diff --git a/tools/devops/automation/templates/api-diff-stage.yml b/tools/devops/automation/templates/api-diff-stage.yml index a81ac79c8bd..e13e9957a7c 100644 --- a/tools/devops/automation/templates/api-diff-stage.yml +++ b/tools/devops/automation/templates/api-diff-stage.yml @@ -24,6 +24,9 @@ parameters: - name: macOSName type: string +- name: useACES + type: boolean + default: false stages: @@ -69,3 +72,4 @@ stages: gitHubToken: $(Github.Token) xqaCertPass: $(xqa--certificates--password) pool: ${{ parameters.pool }} + useACES: ${{ parameters.useACES }} diff --git a/tools/devops/automation/templates/build/api-diff-stage.yml b/tools/devops/automation/templates/build/api-diff-stage.yml index 03cc9d7f34d..edac9d79ca9 100644 --- a/tools/devops/automation/templates/build/api-diff-stage.yml +++ b/tools/devops/automation/templates/build/api-diff-stage.yml @@ -34,6 +34,10 @@ parameters: - name: macOSName type: string +- name: useACES + type: boolean + default: false + jobs: # Detect changes - job: api_diff @@ -44,13 +48,19 @@ jobs: # set the branch variable name, this is required by jenkins and we have a lot of scripts that depend on it BRANCH_NAME: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] XHARNESS_LABELS: $[ stageDependencies.configure_build.configure.outputs['labels.xharness_labels'] ] - pool: - name: ${{ parameters.pool }} - demands: - - Agent.OS -equals Darwin - - Agent.OSVersion -gtVersion $(minimumMacOSVersion) - - macOS.Name -equals ${{ parameters.macOSName }} - - XcodeChannel -equals ${{ parameters.xcodeChannel }} + ${{ if parameters.useACES }}: + pool: + name: $(CIBuildPoolACES) + demands: + - ImageOverride -equals $(CIBuildPoolACESImage) + ${{ else }}: + pool: + name: ${{ parameters.pool }} + demands: + - Agent.OS -equals Darwin + - Agent.OSVersion -gtVersion $(minimumMacOSVersion) + - macOS.Name -equals ${{ parameters.macOSName }} + - XcodeChannel -equals ${{ parameters.xcodeChannel }} workspace: clean: all diff --git a/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml index 5b81d42282d..76d8275b16c 100644 --- a/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml @@ -22,6 +22,10 @@ parameters: type: boolean default: false +- name: useACES + type: boolean + default: false + resources: repositories: - repository: self @@ -49,3 +53,4 @@ stages: isPR: ${{ parameters.isPR }} provisionatorChannel: ${{ parameters.provisionatorChannel }} pool: ${{ parameters.pool }} + useACES: ${{ parameters.useACES }} From a98508ad3c612dfcc5767c5e9181120d664fa096 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Wed, 27 May 2026 16:11:20 -0400 Subject: [PATCH 03/16] [CI] Run CI simulator tests on ACES --- .../automation/run-post-ci-build-tests.yml | 1 + .../pipelines/run-tests-pipeline.yml | 6 ++++ .../automation/templates/tests-stage.yml | 5 +++ .../automation/templates/tests/stage.yml | 34 +++++++++++++------ 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/tools/devops/automation/run-post-ci-build-tests.yml b/tools/devops/automation/run-post-ci-build-tests.yml index af8677d8b4e..6ec3d492f83 100644 --- a/tools/devops/automation/run-post-ci-build-tests.yml +++ b/tools/devops/automation/run-post-ci-build-tests.yml @@ -17,3 +17,4 @@ extends: template: templates/pipelines/run-tests-pipeline.yml parameters: isPR: false + useACES: true diff --git a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml index a3ef20748f6..02efa1c1a51 100644 --- a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml @@ -36,6 +36,11 @@ parameters: type: boolean default: false + - name: useACES + displayName: Use ACES shared pool for simulator tests + type: boolean + default: false + resources: repositories: - repository: self @@ -62,3 +67,4 @@ stages: runTests: ${{ parameters.runTests }} runWindowsIntegration: ${{ parameters.runWindowsIntegration }} buildPackages: ${{ parameters.buildPackages }} + useACES: ${{ parameters.useACES }} diff --git a/tools/devops/automation/templates/tests-stage.yml b/tools/devops/automation/templates/tests-stage.yml index 41f6fec21dd..ec2f1966c25 100644 --- a/tools/devops/automation/templates/tests-stage.yml +++ b/tools/devops/automation/templates/tests-stage.yml @@ -125,6 +125,10 @@ parameters: type: boolean default: false +- name: useACES + type: boolean + default: false + stages: - template: ./build/linux-build-verification.yml @@ -234,6 +238,7 @@ stages: xqaCertPass: $(xqa--certificates--password) condition: ${{ parameters.runTests }} postPipeline: ${{ not(parameters.buildPackages) }} + useACES: ${{ parameters.useACES }} - template: ./tests/publish-results.yml parameters: diff --git a/tools/devops/automation/templates/tests/stage.yml b/tools/devops/automation/templates/tests/stage.yml index 7301307f3f1..a3becddf176 100644 --- a/tools/devops/automation/templates/tests/stage.yml +++ b/tools/devops/automation/templates/tests/stage.yml @@ -66,6 +66,10 @@ parameters: type: boolean default: false +- name: useACES + type: boolean + default: false + stages: - stage: ${{ parameters.stageName }} displayName: ${{ parameters.displayName }} @@ -89,17 +93,25 @@ stages: BRANCH_NAME: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] XHARNESS_LABELS: $[ stageDependencies.configure_build.configure.outputs['labels.xharness_labels'] ] DOTNET_PLATFORMS: $[ stageDependencies.configure_build.configure.outputs['configure_platforms.DOTNET_PLATFORMS'] ] - pool: - name: ${{ parameters.testPool }} - demands: - - Agent.OS -equals Darwin - - macOS.Name -equals ${{ parameters.macOSName }} - - macOS.Architecture -equals arm64 - - XcodeChannel -equals ${{ parameters.XcodeChannel }} - - ${{ each demand in parameters.extraBotDemands }}: - - demand - workspace: - clean: all + ${{ if parameters.useACES }}: + pool: + name: $(CIBuildPoolACES) + demands: + - ImageOverride -equals $(CIBuildPoolACESImage) + workspace: + clean: all + ${{ else }}: + pool: + name: ${{ parameters.testPool }} + demands: + - Agent.OS -equals Darwin + - macOS.Name -equals ${{ parameters.macOSName }} + - macOS.Architecture -equals arm64 + - XcodeChannel -equals ${{ parameters.XcodeChannel }} + - ${{ each demand in parameters.extraBotDemands }}: + - demand + workspace: + clean: all strategy: matrix: $[ stageDependencies.configure_build.configure.outputs['test_matrix.SIMULATOR_TEST_MATRIX'] ] condition: ne(stageDependencies.configure_build.configure.outputs['labels.skip_all_tests'], 'True') From cf6bde10cea4fdb7def11fc6b321260817246bda Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Wed, 27 May 2026 16:30:25 -0400 Subject: [PATCH 04/16] [CI] Prepare PR chain pipelines for future ACES use --- tools/devops/automation/build-pull-request.yml | 6 ++++++ tools/devops/automation/run-post-pr-build-tests.yml | 1 + tools/devops/automation/run-pr-api-diff.yml | 1 + 3 files changed, 8 insertions(+) diff --git a/tools/devops/automation/build-pull-request.yml b/tools/devops/automation/build-pull-request.yml index b6e7f76620c..a30507eb881 100644 --- a/tools/devops/automation/build-pull-request.yml +++ b/tools/devops/automation/build-pull-request.yml @@ -27,6 +27,11 @@ parameters: type: boolean default: false +- name: useACES + displayName: 'Use ACES shared pool' + type: boolean + default: false + resources: repositories: - repository: self @@ -113,3 +118,4 @@ extends: forceInsertion: ${{ parameters.forceInsertion }} pushNugets: false pushNugetsToMaestro: false + useACES: ${{ parameters.useACES }} diff --git a/tools/devops/automation/run-post-pr-build-tests.yml b/tools/devops/automation/run-post-pr-build-tests.yml index b83f2d8e95c..733320bbef5 100644 --- a/tools/devops/automation/run-post-pr-build-tests.yml +++ b/tools/devops/automation/run-post-pr-build-tests.yml @@ -27,3 +27,4 @@ extends: parameters: isPR: true buildPackages: true + useACES: false diff --git a/tools/devops/automation/run-pr-api-diff.yml b/tools/devops/automation/run-pr-api-diff.yml index 07401bdf077..5ed437efeba 100644 --- a/tools/devops/automation/run-pr-api-diff.yml +++ b/tools/devops/automation/run-pr-api-diff.yml @@ -26,3 +26,4 @@ extends: parameters: isPR: true pool: $(PRBuildPool) + useACES: false From 9446c3fe996965609cbd42c8ef397cca71a58b61 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Wed, 27 May 2026 18:41:39 -0400 Subject: [PATCH 05/16] [CI] Create a global ACES switch for PR and CI builds --- tools/devops/automation/build-pipeline.yml | 7 +---- .../devops/automation/build-pull-request.yml | 7 +---- tools/devops/automation/run-ci-api-diff.yml | 2 +- .../automation/run-post-ci-build-tests.yml | 2 +- .../automation/run-post-pr-build-tests.yml | 2 +- tools/devops/automation/run-pr-api-diff.yml | 2 +- .../automation/templates/variables/common.yml | 31 +++++++++++++++++++ 7 files changed, 37 insertions(+), 16 deletions(-) diff --git a/tools/devops/automation/build-pipeline.yml b/tools/devops/automation/build-pipeline.yml index 5b7c186be1f..182b11a2929 100644 --- a/tools/devops/automation/build-pipeline.yml +++ b/tools/devops/automation/build-pipeline.yml @@ -36,11 +36,6 @@ parameters: displayName: 'Push Nugets (Maestro)' default: true - - name: useACES - type: boolean - displayName: 'Use ACES shared pool' - default: true - resources: repositories: - repository: self @@ -137,7 +132,7 @@ extends: forceInsertion: ${{ parameters.forceInsertion }} pushNugets: ${{ parameters.pushNugets }} pushNugetsToMaestro: ${{ parameters.pushNugetsToMaestro }} - useACES: ${{ parameters.useACES }} + useACES: ${{ eq(variables.UseACES_CI, 'true') }} - template: localization/v1.yml@yaml-templates parameters: diff --git a/tools/devops/automation/build-pull-request.yml b/tools/devops/automation/build-pull-request.yml index a30507eb881..866efecc52b 100644 --- a/tools/devops/automation/build-pull-request.yml +++ b/tools/devops/automation/build-pull-request.yml @@ -27,11 +27,6 @@ parameters: type: boolean default: false -- name: useACES - displayName: 'Use ACES shared pool' - type: boolean - default: false - resources: repositories: - repository: self @@ -118,4 +113,4 @@ extends: forceInsertion: ${{ parameters.forceInsertion }} pushNugets: false pushNugetsToMaestro: false - useACES: ${{ parameters.useACES }} + useACES: ${{ eq(variables.UseACES_PR, 'true') }} diff --git a/tools/devops/automation/run-ci-api-diff.yml b/tools/devops/automation/run-ci-api-diff.yml index 3a84fa82c11..336e4fdae83 100644 --- a/tools/devops/automation/run-ci-api-diff.yml +++ b/tools/devops/automation/run-ci-api-diff.yml @@ -17,4 +17,4 @@ extends: parameters: isPR: false pool: $(CIBuildPool) - useACES: true + useACES: ${{ eq(variables.UseACES_CI, 'true') }} diff --git a/tools/devops/automation/run-post-ci-build-tests.yml b/tools/devops/automation/run-post-ci-build-tests.yml index 6ec3d492f83..79976885771 100644 --- a/tools/devops/automation/run-post-ci-build-tests.yml +++ b/tools/devops/automation/run-post-ci-build-tests.yml @@ -17,4 +17,4 @@ extends: template: templates/pipelines/run-tests-pipeline.yml parameters: isPR: false - useACES: true + useACES: ${{ eq(variables.UseACES_CI, 'true') }} diff --git a/tools/devops/automation/run-post-pr-build-tests.yml b/tools/devops/automation/run-post-pr-build-tests.yml index 733320bbef5..5eb7deffba7 100644 --- a/tools/devops/automation/run-post-pr-build-tests.yml +++ b/tools/devops/automation/run-post-pr-build-tests.yml @@ -27,4 +27,4 @@ extends: parameters: isPR: true buildPackages: true - useACES: false + useACES: ${{ eq(variables.UseACES_PR, 'true') }} diff --git a/tools/devops/automation/run-pr-api-diff.yml b/tools/devops/automation/run-pr-api-diff.yml index 5ed437efeba..2dfacbce30d 100644 --- a/tools/devops/automation/run-pr-api-diff.yml +++ b/tools/devops/automation/run-pr-api-diff.yml @@ -26,4 +26,4 @@ extends: parameters: isPR: true pool: $(PRBuildPool) - useACES: false + useACES: ${{ eq(variables.UseACES_PR, 'true') }} diff --git a/tools/devops/automation/templates/variables/common.yml b/tools/devops/automation/templates/variables/common.yml index 2c6e6f019fb..ee74f009a5a 100644 --- a/tools/devops/automation/templates/variables/common.yml +++ b/tools/devops/automation/templates/variables/common.yml @@ -51,6 +51,37 @@ variables: - name: CIBuildPoolACESImage value: 'ACES_VM_SharedPool_Tahoe' +# Global switches that route build / api-diff / simulator-test jobs onto the +# ACES shared pool ($(CIBuildPoolACES) with image $(CIBuildPoolACESImage)) +# instead of the traditional Redmond Mac build pools ($(CIBuildPool) / $(PRBuildPool)). +# +# There is one switch per "side" of the pipeline so CI and PR can be migrated +# independently: +# - UseACES_CI: consumed by build-pipeline.yml, run-ci-api-diff.yml and +# run-post-ci-build-tests.yml. +# - UseACES_PR: consumed by build-pull-request.yml, run-pr-api-diff.yml and +# run-post-pr-build-tests.yml. +# +# Each entry pipeline reads the matching variable with +# ${{ eq(variables.UseACES_XX, 'true') }} +# and forwards the resulting boolean as the `useACES` template parameter. That +# parameter is threaded through main-stage.yml / build-stage.yml / +# api-diff-stage.yml / tests/stage.yml, where the `pool:` block is emitted at +# template-expansion time as either: +# - ACES: name: $(CIBuildPoolACES), demands: ImageOverride -equals $(CIBuildPoolACESImage) +# - default: the original pool + Darwin / macOS.Name / XcodeChannel demands +# +# Because the value is used inside a `${{ }}` template expression, it must stay +# a literal string ('true' / 'false') defined here — variable groups or runtime +# expressions would not work. Flip a value and queue a new run to switch sides. +# +# We may want to switch back to traditional pools for example when the ACES pool is not available +# or when we need beta versions of Xcode or macOS that are not available in the ACES pool. +- name: UseACES_CI + value: 'true' +- name: UseACES_PR + value: 'false' + # override the default build revision - name: BUILD_REVISION value: azure-devops-$(Build.SourceVersion) From 18ef1a85fa3475bda7a445ebf3478ed6131744c1 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Wed, 27 May 2026 20:47:51 -0400 Subject: [PATCH 06/16] [Tests] Fix SpecialFolder tests on macOS CI VMs --- .../linker/link sdk/LinkSdkRegressionTest.cs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/linker/link sdk/LinkSdkRegressionTest.cs b/tests/linker/link sdk/LinkSdkRegressionTest.cs index b75d223ed24..97b17f2f66a 100644 --- a/tests/linker/link sdk/LinkSdkRegressionTest.cs +++ b/tests/linker/link sdk/LinkSdkRegressionTest.cs @@ -789,11 +789,11 @@ void SpecialFolderImpl () #else var myExists = false; #endif - path = TestFolder (Environment.SpecialFolder.MyMusic, exists: myExists); + path = TestFolderIfAvailableInCI (Environment.SpecialFolder.MyMusic, myExists); - path = TestFolder (Environment.SpecialFolder.MyVideos, exists: myExists); + path = TestFolderIfAvailableInCI (Environment.SpecialFolder.MyVideos, myExists); - path = TestFolder (Environment.SpecialFolder.DesktopDirectory, exists: myExists); + path = TestFolderIfAvailableInCI (Environment.SpecialFolder.DesktopDirectory, myExists); #if __TVOS__ path = TestFolder (Environment.SpecialFolder.Fonts, exists: null, supported: true); @@ -811,7 +811,7 @@ void SpecialFolderImpl () path = TestFolder (Environment.SpecialFolder.Templates, exists: false); #endif - path = TestFolder (Environment.SpecialFolder.MyPictures, exists: myExists); + path = TestFolderIfAvailableInCI (Environment.SpecialFolder.MyPictures, myExists); #if __MACOS__ path = TestFolder (Environment.SpecialFolder.CommonTemplates, supported: false); @@ -897,6 +897,17 @@ void SpecialFolderImpl () path = TestFolder (Environment.SpecialFolder.Resources, readOnly: tvos && device); Assert.That (path.EndsWith ("/Library", StringComparison.Ordinal), Is.True, "Resources"); #endif + // Some CI VM images don't initialize all standard user directories, so keep this + // tolerance CI-only and preserve the stricter check for local runs. + string TestFolderIfAvailableInCI (Environment.SpecialFolder folder, bool exists) + { +#if __MACOS__ + var path = Environment.GetFolderPath (folder); + if (string.IsNullOrEmpty (path) && TestRuntime.IsInCI) + return path; +#endif + return TestFolder (folder, exists: exists); + } } #if !__MACOS__ From 60feeedf82105766e9cc5c524829fef7ef91154f Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Wed, 27 May 2026 21:16:52 -0400 Subject: [PATCH 07/16] [Tests] remove retry in order to get results from CI monotouch-tests run --- tools/devops/automation/templates/tests/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/automation/templates/tests/run-tests.yml b/tools/devops/automation/templates/tests/run-tests.yml index 483f3afbb30..0272f5e2c4f 100644 --- a/tools/devops/automation/templates/tests/run-tests.yml +++ b/tools/devops/automation/templates/tests/run-tests.yml @@ -32,7 +32,7 @@ parameters: - name: retryCount type: number - default: 3 + default: 0 steps: - bash: | From c58dfa2ca7fbbd8862c65cb344ac3dbdb690bf10 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Wed, 27 May 2026 22:47:00 -0400 Subject: [PATCH 08/16] [CI] Fix global usage of UseACES_XX --- tools/devops/automation/build-pipeline.yml | 1 - .../devops/automation/build-pull-request.yml | 1 - tools/devops/automation/run-ci-api-diff.yml | 1 - .../automation/run-post-ci-build-tests.yml | 1 - .../automation/run-post-pr-build-tests.yml | 1 - tools/devops/automation/run-pr-api-diff.yml | 1 - .../automation/templates/main-stage.yml | 9 ++++----- .../templates/pipelines/api-diff-pipeline.yml | 9 ++++----- .../pipelines/run-tests-pipeline.yml | 10 ++++------ .../automation/templates/variables/common.yml | 19 +++++++++++++------ 10 files changed, 25 insertions(+), 28 deletions(-) diff --git a/tools/devops/automation/build-pipeline.yml b/tools/devops/automation/build-pipeline.yml index 182b11a2929..d3e3f8b9389 100644 --- a/tools/devops/automation/build-pipeline.yml +++ b/tools/devops/automation/build-pipeline.yml @@ -132,7 +132,6 @@ extends: forceInsertion: ${{ parameters.forceInsertion }} pushNugets: ${{ parameters.pushNugets }} pushNugetsToMaestro: ${{ parameters.pushNugetsToMaestro }} - useACES: ${{ eq(variables.UseACES_CI, 'true') }} - template: localization/v1.yml@yaml-templates parameters: diff --git a/tools/devops/automation/build-pull-request.yml b/tools/devops/automation/build-pull-request.yml index 866efecc52b..b6e7f76620c 100644 --- a/tools/devops/automation/build-pull-request.yml +++ b/tools/devops/automation/build-pull-request.yml @@ -113,4 +113,3 @@ extends: forceInsertion: ${{ parameters.forceInsertion }} pushNugets: false pushNugetsToMaestro: false - useACES: ${{ eq(variables.UseACES_PR, 'true') }} diff --git a/tools/devops/automation/run-ci-api-diff.yml b/tools/devops/automation/run-ci-api-diff.yml index 336e4fdae83..8ccd511ab0c 100644 --- a/tools/devops/automation/run-ci-api-diff.yml +++ b/tools/devops/automation/run-ci-api-diff.yml @@ -17,4 +17,3 @@ extends: parameters: isPR: false pool: $(CIBuildPool) - useACES: ${{ eq(variables.UseACES_CI, 'true') }} diff --git a/tools/devops/automation/run-post-ci-build-tests.yml b/tools/devops/automation/run-post-ci-build-tests.yml index 79976885771..af8677d8b4e 100644 --- a/tools/devops/automation/run-post-ci-build-tests.yml +++ b/tools/devops/automation/run-post-ci-build-tests.yml @@ -17,4 +17,3 @@ extends: template: templates/pipelines/run-tests-pipeline.yml parameters: isPR: false - useACES: ${{ eq(variables.UseACES_CI, 'true') }} diff --git a/tools/devops/automation/run-post-pr-build-tests.yml b/tools/devops/automation/run-post-pr-build-tests.yml index 5eb7deffba7..b83f2d8e95c 100644 --- a/tools/devops/automation/run-post-pr-build-tests.yml +++ b/tools/devops/automation/run-post-pr-build-tests.yml @@ -27,4 +27,3 @@ extends: parameters: isPR: true buildPackages: true - useACES: ${{ eq(variables.UseACES_PR, 'true') }} diff --git a/tools/devops/automation/run-pr-api-diff.yml b/tools/devops/automation/run-pr-api-diff.yml index 2dfacbce30d..07401bdf077 100644 --- a/tools/devops/automation/run-pr-api-diff.yml +++ b/tools/devops/automation/run-pr-api-diff.yml @@ -26,4 +26,3 @@ extends: parameters: isPR: true pool: $(PRBuildPool) - useACES: ${{ eq(variables.UseACES_PR, 'true') }} diff --git a/tools/devops/automation/templates/main-stage.yml b/tools/devops/automation/templates/main-stage.yml index 671be873a5d..ab8b0d50d3e 100644 --- a/tools/devops/automation/templates/main-stage.yml +++ b/tools/devops/automation/templates/main-stage.yml @@ -56,10 +56,6 @@ parameters: type: string default: '' - - name: useACES - type: boolean - default: false - stages: - stage: configure_build @@ -107,7 +103,10 @@ stages: xqaCertPass: $(xqa--certificates--password) pool: ${{ parameters.pool }} use1ES: true - useACES: ${{ parameters.useACES }} + ${{ if parameters.isPR }}: + useACES: ${{ eq(variables['UseACES_PR'], 'true') }} + ${{ else }}: + useACES: ${{ eq(variables['UseACES_CI'], 'true') }} # .NET Release Prep and VS Insertion Stages, only execute them when the build comes from an official branch and is not a schedule build from OneLoc # setting the stage at this level makes the graph of the UI look better, else the lines overlap and is not clear. diff --git a/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml index 76d8275b16c..13c309b007a 100644 --- a/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml @@ -22,10 +22,6 @@ parameters: type: boolean default: false -- name: useACES - type: boolean - default: false - resources: repositories: - repository: self @@ -53,4 +49,7 @@ stages: isPR: ${{ parameters.isPR }} provisionatorChannel: ${{ parameters.provisionatorChannel }} pool: ${{ parameters.pool }} - useACES: ${{ parameters.useACES }} + ${{ if parameters.isPR }}: + useACES: ${{ eq(variables['UseACES_PR'], 'true') }} + ${{ else }}: + useACES: ${{ eq(variables['UseACES_CI'], 'true') }} diff --git a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml index 02efa1c1a51..f13a6e786a5 100644 --- a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml @@ -36,11 +36,6 @@ parameters: type: boolean default: false - - name: useACES - displayName: Use ACES shared pool for simulator tests - type: boolean - default: false - resources: repositories: - repository: self @@ -67,4 +62,7 @@ stages: runTests: ${{ parameters.runTests }} runWindowsIntegration: ${{ parameters.runWindowsIntegration }} buildPackages: ${{ parameters.buildPackages }} - useACES: ${{ parameters.useACES }} + ${{ if parameters.isPR }}: + useACES: ${{ eq(variables['UseACES_PR'], 'true') }} + ${{ else }}: + useACES: ${{ eq(variables['UseACES_CI'], 'true') }} diff --git a/tools/devops/automation/templates/variables/common.yml b/tools/devops/automation/templates/variables/common.yml index ee74f009a5a..33ce7395267 100644 --- a/tools/devops/automation/templates/variables/common.yml +++ b/tools/devops/automation/templates/variables/common.yml @@ -62,12 +62,19 @@ variables: # - UseACES_PR: consumed by build-pull-request.yml, run-pr-api-diff.yml and # run-post-pr-build-tests.yml. # -# Each entry pipeline reads the matching variable with -# ${{ eq(variables.UseACES_XX, 'true') }} -# and forwards the resulting boolean as the `useACES` template parameter. That -# parameter is threaded through main-stage.yml / build-stage.yml / -# api-diff-stage.yml / tests/stage.yml, where the `pool:` block is emitted at -# template-expansion time as either: +# Each entry pipeline reads the matching variable via +# ${{ eq(variables['UseACES_XX'], 'true') }} +# inside the template chain heads that actually import this file as +# variables (main-stage.yml, pipelines/api-diff-pipeline.yml and +# pipelines/run-tests-pipeline.yml). The lookup must happen there — the leaf +# `run-*.yml` entry pipelines do not include common.yml themselves, so the +# variable would be undefined if read at that level. +# +# Each template head picks the right variable based on the `isPR` parameter +# and forwards the resulting boolean as the `useACES` template parameter down +# to the job-level templates (build-stage.yml, build/api-diff-stage.yml and +# tests/stage.yml), where the `pool:` block is emitted at template-expansion +# time as either: # - ACES: name: $(CIBuildPoolACES), demands: ImageOverride -equals $(CIBuildPoolACESImage) # - default: the original pool + Darwin / macOS.Name / XcodeChannel demands # From ae929aa7793ffd182563e608f0c955fa517e9dcf Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Wed, 27 May 2026 23:17:37 -0400 Subject: [PATCH 09/16] [CI] Unfortunately no global switch so going back to docs and switches --- tools/devops/automation/build-pipeline.yml | 1 + .../devops/automation/build-pull-request.yml | 1 + tools/devops/automation/run-ci-api-diff.yml | 1 + .../automation/run-post-ci-build-tests.yml | 1 + .../automation/run-post-pr-build-tests.yml | 1 + tools/devops/automation/run-pr-api-diff.yml | 1 + .../automation/templates/main-stage.yml | 9 +-- .../templates/pipelines/api-diff-pipeline.yml | 9 +-- .../pipelines/run-tests-pipeline.yml | 10 +-- .../automation/templates/variables/common.yml | 72 +++++++++---------- 10 files changed, 56 insertions(+), 50 deletions(-) diff --git a/tools/devops/automation/build-pipeline.yml b/tools/devops/automation/build-pipeline.yml index d3e3f8b9389..ba07f97f56e 100644 --- a/tools/devops/automation/build-pipeline.yml +++ b/tools/devops/automation/build-pipeline.yml @@ -132,6 +132,7 @@ extends: forceInsertion: ${{ parameters.forceInsertion }} pushNugets: ${{ parameters.pushNugets }} pushNugetsToMaestro: ${{ parameters.pushNugetsToMaestro }} + useACES: true # flip to false to opt CI out of ACES; see templates/variables/common.yml - template: localization/v1.yml@yaml-templates parameters: diff --git a/tools/devops/automation/build-pull-request.yml b/tools/devops/automation/build-pull-request.yml index b6e7f76620c..812d2d33944 100644 --- a/tools/devops/automation/build-pull-request.yml +++ b/tools/devops/automation/build-pull-request.yml @@ -113,3 +113,4 @@ extends: forceInsertion: ${{ parameters.forceInsertion }} pushNugets: false pushNugetsToMaestro: false + useACES: false # flip to true to opt PR into ACES; see templates/variables/common.yml diff --git a/tools/devops/automation/run-ci-api-diff.yml b/tools/devops/automation/run-ci-api-diff.yml index 8ccd511ab0c..caf428a3087 100644 --- a/tools/devops/automation/run-ci-api-diff.yml +++ b/tools/devops/automation/run-ci-api-diff.yml @@ -17,3 +17,4 @@ extends: parameters: isPR: false pool: $(CIBuildPool) + useACES: true # flip to false to opt CI api-diff out of ACES; see templates/variables/common.yml diff --git a/tools/devops/automation/run-post-ci-build-tests.yml b/tools/devops/automation/run-post-ci-build-tests.yml index af8677d8b4e..f09d70d3e7b 100644 --- a/tools/devops/automation/run-post-ci-build-tests.yml +++ b/tools/devops/automation/run-post-ci-build-tests.yml @@ -17,3 +17,4 @@ extends: template: templates/pipelines/run-tests-pipeline.yml parameters: isPR: false + useACES: true # flip to false to opt CI simulator tests out of ACES; see templates/variables/common.yml diff --git a/tools/devops/automation/run-post-pr-build-tests.yml b/tools/devops/automation/run-post-pr-build-tests.yml index b83f2d8e95c..81be17692d3 100644 --- a/tools/devops/automation/run-post-pr-build-tests.yml +++ b/tools/devops/automation/run-post-pr-build-tests.yml @@ -27,3 +27,4 @@ extends: parameters: isPR: true buildPackages: true + useACES: false # flip to true to opt PR simulator tests into ACES; see templates/variables/common.yml diff --git a/tools/devops/automation/run-pr-api-diff.yml b/tools/devops/automation/run-pr-api-diff.yml index 07401bdf077..03f693b4b5e 100644 --- a/tools/devops/automation/run-pr-api-diff.yml +++ b/tools/devops/automation/run-pr-api-diff.yml @@ -26,3 +26,4 @@ extends: parameters: isPR: true pool: $(PRBuildPool) + useACES: false # flip to true to opt PR api-diff into ACES; see templates/variables/common.yml diff --git a/tools/devops/automation/templates/main-stage.yml b/tools/devops/automation/templates/main-stage.yml index ab8b0d50d3e..671be873a5d 100644 --- a/tools/devops/automation/templates/main-stage.yml +++ b/tools/devops/automation/templates/main-stage.yml @@ -56,6 +56,10 @@ parameters: type: string default: '' + - name: useACES + type: boolean + default: false + stages: - stage: configure_build @@ -103,10 +107,7 @@ stages: xqaCertPass: $(xqa--certificates--password) pool: ${{ parameters.pool }} use1ES: true - ${{ if parameters.isPR }}: - useACES: ${{ eq(variables['UseACES_PR'], 'true') }} - ${{ else }}: - useACES: ${{ eq(variables['UseACES_CI'], 'true') }} + useACES: ${{ parameters.useACES }} # .NET Release Prep and VS Insertion Stages, only execute them when the build comes from an official branch and is not a schedule build from OneLoc # setting the stage at this level makes the graph of the UI look better, else the lines overlap and is not clear. diff --git a/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml index 13c309b007a..76d8275b16c 100644 --- a/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/api-diff-pipeline.yml @@ -22,6 +22,10 @@ parameters: type: boolean default: false +- name: useACES + type: boolean + default: false + resources: repositories: - repository: self @@ -49,7 +53,4 @@ stages: isPR: ${{ parameters.isPR }} provisionatorChannel: ${{ parameters.provisionatorChannel }} pool: ${{ parameters.pool }} - ${{ if parameters.isPR }}: - useACES: ${{ eq(variables['UseACES_PR'], 'true') }} - ${{ else }}: - useACES: ${{ eq(variables['UseACES_CI'], 'true') }} + useACES: ${{ parameters.useACES }} diff --git a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml index f13a6e786a5..02efa1c1a51 100644 --- a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml @@ -36,6 +36,11 @@ parameters: type: boolean default: false + - name: useACES + displayName: Use ACES shared pool for simulator tests + type: boolean + default: false + resources: repositories: - repository: self @@ -62,7 +67,4 @@ stages: runTests: ${{ parameters.runTests }} runWindowsIntegration: ${{ parameters.runWindowsIntegration }} buildPackages: ${{ parameters.buildPackages }} - ${{ if parameters.isPR }}: - useACES: ${{ eq(variables['UseACES_PR'], 'true') }} - ${{ else }}: - useACES: ${{ eq(variables['UseACES_CI'], 'true') }} + useACES: ${{ parameters.useACES }} diff --git a/tools/devops/automation/templates/variables/common.yml b/tools/devops/automation/templates/variables/common.yml index 33ce7395267..db4b945f053 100644 --- a/tools/devops/automation/templates/variables/common.yml +++ b/tools/devops/automation/templates/variables/common.yml @@ -44,6 +44,40 @@ variables: value: 'VSEng-Xamarin-RedmondMacBuildPool-iOS-Trusted' - name: CIBuildPoolUrl value: 'https://devdiv.visualstudio.com/_settings/agentpools?poolId=367&view=agents' + +# ACES shared pool wiring. +# +# The infrastructure for routing build / api-diff / simulator-test jobs onto +# the ACES shared pool ($(CIBuildPoolACES), image $(CIBuildPoolACESImage)) +# instead of the traditional Redmond Mac build pools ($(CIBuildPool) / +# $(PRBuildPool)) is plumbed through the `useACES` template parameter, which +# flows from each entry pipeline down through main-stage.yml / +# build-stage.yml, api-diff-stage.yml and tests/stage.yml. When `useACES` is +# true, those job-level templates emit: +# pool: +# name: $(CIBuildPoolACES) +# demands: ImageOverride -equals $(CIBuildPoolACESImage) +# Otherwise they emit the original pool + Darwin / macOS.Name / XcodeChannel +# demands. +# +# The switch MUST be a literal in the entry pipeline. Azure Pipelines only +# resolves `${{ }}` expressions at template-expansion time, and variables +# brought in via `- template:` (like this file) are not visible at that +# point — they are runtime-only. So there is no working "single global +# variable" we can read here; the literal lives in each entry pipeline: +# +# CI (uses CIBuildPool / ACES): +# - build-pipeline.yml -> useACES: true +# - run-ci-api-diff.yml -> useACES: true +# - run-post-ci-build-tests.yml -> useACES: true +# PR (uses PRBuildPool): +# - build-pull-request.yml -> useACES: false +# - run-pr-api-diff.yml -> useACES: false +# - run-post-pr-build-tests.yml -> useACES: false +# +# Flip the literal in the three files on the side you want to move (e.g. when +# the ACES pool is unavailable, or when you need an Xcode/macOS beta that is +# only present on the traditional pools) and queue a new run. - name: CIBuildPoolACES value: 'AcesShared' - name: CIBuildPoolACESUrl @@ -51,44 +85,6 @@ variables: - name: CIBuildPoolACESImage value: 'ACES_VM_SharedPool_Tahoe' -# Global switches that route build / api-diff / simulator-test jobs onto the -# ACES shared pool ($(CIBuildPoolACES) with image $(CIBuildPoolACESImage)) -# instead of the traditional Redmond Mac build pools ($(CIBuildPool) / $(PRBuildPool)). -# -# There is one switch per "side" of the pipeline so CI and PR can be migrated -# independently: -# - UseACES_CI: consumed by build-pipeline.yml, run-ci-api-diff.yml and -# run-post-ci-build-tests.yml. -# - UseACES_PR: consumed by build-pull-request.yml, run-pr-api-diff.yml and -# run-post-pr-build-tests.yml. -# -# Each entry pipeline reads the matching variable via -# ${{ eq(variables['UseACES_XX'], 'true') }} -# inside the template chain heads that actually import this file as -# variables (main-stage.yml, pipelines/api-diff-pipeline.yml and -# pipelines/run-tests-pipeline.yml). The lookup must happen there — the leaf -# `run-*.yml` entry pipelines do not include common.yml themselves, so the -# variable would be undefined if read at that level. -# -# Each template head picks the right variable based on the `isPR` parameter -# and forwards the resulting boolean as the `useACES` template parameter down -# to the job-level templates (build-stage.yml, build/api-diff-stage.yml and -# tests/stage.yml), where the `pool:` block is emitted at template-expansion -# time as either: -# - ACES: name: $(CIBuildPoolACES), demands: ImageOverride -equals $(CIBuildPoolACESImage) -# - default: the original pool + Darwin / macOS.Name / XcodeChannel demands -# -# Because the value is used inside a `${{ }}` template expression, it must stay -# a literal string ('true' / 'false') defined here — variable groups or runtime -# expressions would not work. Flip a value and queue a new run to switch sides. -# -# We may want to switch back to traditional pools for example when the ACES pool is not available -# or when we need beta versions of Xcode or macOS that are not available in the ACES pool. -- name: UseACES_CI - value: 'true' -- name: UseACES_PR - value: 'false' - # override the default build revision - name: BUILD_REVISION value: azure-devops-$(Build.SourceVersion) From cb53e402619d3af2e720624e041da7fa2a67f63f Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Thu, 28 May 2026 00:29:36 -0400 Subject: [PATCH 10/16] [CI] onboard Build macOS tests into ACES too --- .../templates/build/build-mac-tests-stage.yml | 27 +++++++++++++------ .../automation/templates/tests-stage.yml | 1 + 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tools/devops/automation/templates/build/build-mac-tests-stage.yml b/tools/devops/automation/templates/build/build-mac-tests-stage.yml index 0622c8741b5..be7ea2efb3b 100644 --- a/tools/devops/automation/templates/build/build-mac-tests-stage.yml +++ b/tools/devops/automation/templates/build/build-mac-tests-stage.yml @@ -34,6 +34,10 @@ parameters: - name: macOSName type: string +- name: useACES + type: boolean + default: false + jobs: # This job builds the macOS tests. @@ -49,14 +53,21 @@ jobs: BRANCH_NAME: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] RUN_MAC_TESTS: $[ stageDependencies.configure_build.configure.outputs['decisions.RUN_MAC_TESTS'] ] condition: ne(stageDependencies.configure_build.configure.outputs['decisions.RUN_MAC_TESTS'],'') - pool: - os: macOS - name: ${{ parameters.pool }} - demands: - - Agent.OS -equals Darwin - - Agent.OSVersion -gtVersion $(minimumMacOSVersion) - - macOS.Name -equals ${{ parameters.macOSName }} - - XcodeChannel -equals ${{ parameters.xcodeChannel }} + ${{ if parameters.useACES }}: + pool: + os: macOS + name: $(CIBuildPoolACES) + demands: + - ImageOverride -equals $(CIBuildPoolACESImage) + ${{ else }}: + pool: + os: macOS + name: ${{ parameters.pool }} + demands: + - Agent.OS -equals Darwin + - Agent.OSVersion -gtVersion $(minimumMacOSVersion) + - macOS.Name -equals ${{ parameters.macOSName }} + - XcodeChannel -equals ${{ parameters.xcodeChannel }} steps: - template: build-mac-tests.yml diff --git a/tools/devops/automation/templates/tests-stage.yml b/tools/devops/automation/templates/tests-stage.yml index ec2f1966c25..25bf39e8065 100644 --- a/tools/devops/automation/templates/tests-stage.yml +++ b/tools/devops/automation/templates/tests-stage.yml @@ -287,6 +287,7 @@ stages: gitHubToken: $(Github.Token) xqaCertPass: $(xqa--certificates--password) pool: $(PRBuildPool) + useACES: ${{ parameters.useACES }} - ${{ each config in parameters.macTestsConfigurations }}: - template: ./mac/stage.yml From 8489ecec07f6842d7779ab2923a01ec3692bb4a6 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Thu, 28 May 2026 08:20:51 -0400 Subject: [PATCH 11/16] [CI] Fix monotouch-tests by propagating that we are running under a VM when using ACES --- tests/common/TestRuntime.cs | 2 -- .../monotouch-test/AudioToolbox/AudioQueueTest.cs | 2 ++ tests/monotouch-test/Metal/MTLDeviceTests.cs | 2 ++ tests/monotouch-test/Network/NWConnectionTest.cs | 2 +- tests/monotouch-test/Network/NWParametersTest.cs | 14 +++++++------- .../System.Net.Http/NetworkResources.cs | 1 + tests/xharness/AppRunner.cs | 7 +++++++ tools/devops/automation/templates/tests/stage.yml | 4 ++++ 8 files changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/common/TestRuntime.cs b/tests/common/TestRuntime.cs index 296c348ac2c..4e1ab21b3b0 100644 --- a/tests/common/TestRuntime.cs +++ b/tests/common/TestRuntime.cs @@ -299,12 +299,10 @@ public static void AssertSimulatorOrDesktop (string message = "This test only wo public static void AssertNotVirtualMachine () { -#if MONOMAC || __MACCATALYST__ // enviroment variable set by the CI when running on a VM var vmVendor = Environment.GetEnvironmentVariable ("VM_VENDOR"); if (!string.IsNullOrEmpty (vmVendor)) NUnit.Framework.Assert.Ignore ($"This test only runs on device. Found vm vendor: {vmVendor}"); -#endif } public static bool IsVSTS => diff --git a/tests/monotouch-test/AudioToolbox/AudioQueueTest.cs b/tests/monotouch-test/AudioToolbox/AudioQueueTest.cs index a5249b42ea5..abe7b02f326 100644 --- a/tests/monotouch-test/AudioToolbox/AudioQueueTest.cs +++ b/tests/monotouch-test/AudioToolbox/AudioQueueTest.cs @@ -30,6 +30,8 @@ public void Properties () [Test] public void ChannelAssignments () { + TestRuntime.AssertNotVirtualMachine (); + var aq = new OutputAudioQueue (AudioStreamBasicDescription.CreateLinearPCM ()); var route = global::AVFoundation.AVAudioSession.SharedInstance ().CurrentRoute; diff --git a/tests/monotouch-test/Metal/MTLDeviceTests.cs b/tests/monotouch-test/Metal/MTLDeviceTests.cs index 2a6a38b5113..948737e2749 100644 --- a/tests/monotouch-test/Metal/MTLDeviceTests.cs +++ b/tests/monotouch-test/Metal/MTLDeviceTests.cs @@ -86,6 +86,8 @@ public void ReturnReleaseTest () if (device is null) Assert.Inconclusive ("Metal is not supported"); + TestRuntime.AssertNotVirtualMachine (); + // Apple claims that "Indirect command buffers" are available with MTLGPUFamilyCommon2, but it crashes on at least one machine. // Log what the current device supports, just to have it in the log. foreach (MTLFeatureSet fs in Enum.GetValues ()) { diff --git a/tests/monotouch-test/Network/NWConnectionTest.cs b/tests/monotouch-test/Network/NWConnectionTest.cs index 0d6b7832e8d..e19a053507d 100644 --- a/tests/monotouch-test/Network/NWConnectionTest.cs +++ b/tests/monotouch-test/Network/NWConnectionTest.cs @@ -72,7 +72,7 @@ public void TestForceCancel () } class ConnectionManager : IDisposable { - string host = NetworkResources.MicrosoftUri.Host; + string host = NetworkResources.MicrosoftHttpUri.Host; AutoResetEvent connectedEvent = new AutoResetEvent (false); // used to let us know when the connection was established so that we can access the Report NWConnection? connection; NWParameters? parameters; diff --git a/tests/monotouch-test/Network/NWParametersTest.cs b/tests/monotouch-test/Network/NWParametersTest.cs index 7d59a66f895..c20d892b5b1 100644 --- a/tests/monotouch-test/Network/NWParametersTest.cs +++ b/tests/monotouch-test/Network/NWParametersTest.cs @@ -87,7 +87,7 @@ public void CreateSecureUpdTest () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureUdp (configureTls: setUpTls, configureUdp: setUpProtocol)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { secureEvent.WaitOne (); configureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.True, "Configure TLS handler was not called."); @@ -101,7 +101,7 @@ public void CreateSecureUpdTestDoNotSetUpProtocol () var setUpTls = CreateTlsHandler (); using (var parameters = NWParameters.CreateSecureUdp (configureTls: setUpTls)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { secureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.True, "Configure TLS handler was not called."); Assert.That (protocolConfigured, Is.False, "Protocol configure handler was called."); @@ -114,7 +114,7 @@ public void CreateSecureUpdTestDoNotSetUpTls () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureUdp (configureTls: null, configureUdp: setUpProtocol)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { configureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.False, "Configure TLS handler was not called."); Assert.That (protocolConfigured, Is.True, "Protocol configure handler was not called."); @@ -128,7 +128,7 @@ public void CreateSecureTcpTest () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureTcp (configureTls: setUpTls, configureTcp: setUpProtocol)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { secureEvent.WaitOne (); configureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.True, "Configure TLS handler was not called."); @@ -143,7 +143,7 @@ public void CreateSecureTcpTestDoNotSetUpProtocol () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureTcp (configureTls: setUpTls)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { secureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.True, "Configure TLS handler was not called."); Assert.That (protocolConfigured, Is.False, "Protocol configure handler was called."); @@ -156,7 +156,7 @@ public void CreateSecureTcpTestDoNotSetUpTls () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureTcp (configureTls: null, configureTcp: setUpProtocol)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { configureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.False, "Configure TLS handler was called."); Assert.That (protocolConfigured, Is.True, "Protocol configure handler was not called."); @@ -317,7 +317,7 @@ public void LocalEndpointPropertyTest () { Assert.Ignore ("nw_parameters_copy_local_endpoint always return null. Rdar filled 44095278."); using (var parameters = NWParameters.CreateUdp ()) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { var defaultValue = parameters.LocalEndpoint; Assert.That (defaultValue, Is.Null, "Default value changed."); parameters.LocalEndpoint = endpoint; diff --git a/tests/monotouch-test/System.Net.Http/NetworkResources.cs b/tests/monotouch-test/System.Net.Http/NetworkResources.cs index 6fa21fe5850..2a887a52a3d 100644 --- a/tests/monotouch-test/System.Net.Http/NetworkResources.cs +++ b/tests/monotouch-test/System.Net.Http/NetworkResources.cs @@ -10,6 +10,7 @@ public static class NetworkResources { public static string MicrosoftUrl => AssertNetworkConnection ("https://www.microsoft.com"); public static Uri MicrosoftUri => new Uri (MicrosoftUrl); public static string MicrosoftHttpUrl => AssertNetworkConnection ("http://www.microsoft.com"); + public static Uri MicrosoftHttpUri => new Uri (MicrosoftHttpUrl); public static string XamarinUrl => AssertNetworkConnection ("https://dotnet.microsoft.com/apps/xamarin"); public static string XamarinHttpUrl => AssertNetworkConnection ("http://dotnet.microsoft.com/apps/xamarin"); public static Uri XamarinUri => new Uri (XamarinUrl); diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 4cc1a11b269..8f79f259d1a 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -225,6 +225,13 @@ public async Task RunAsync () if (harness.InCI) { // We use the 'BUILD_REVISION' variable to detect whether we're running CI or not. args.Add (new SetEnvVariableArgument ("BUILD_REVISION", Environment.GetEnvironmentVariable ("BUILD_REVISION"))); + + // Forward VM_VENDOR (set by the pipeline when running on a VM-backed + // pool such as ACES) so TestRuntime.AssertNotVirtualMachine works + // inside the iOS/tvOS simulator process. + var vmVendor = Environment.GetEnvironmentVariable ("VM_VENDOR"); + if (!string.IsNullOrEmpty (vmVendor)) + args.Add (new SetEnvVariableArgument ("VM_VENDOR", vmVendor)); } if (!harness.GetIncludeSystemPermissionTests (TestPlatform.iOS, !isSimulator)) diff --git a/tools/devops/automation/templates/tests/stage.yml b/tools/devops/automation/templates/tests/stage.yml index a3becddf176..cbe4c4c7388 100644 --- a/tools/devops/automation/templates/tests/stage.yml +++ b/tools/devops/automation/templates/tests/stage.yml @@ -93,6 +93,10 @@ stages: BRANCH_NAME: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] XHARNESS_LABELS: $[ stageDependencies.configure_build.configure.outputs['labels.xharness_labels'] ] DOTNET_PLATFORMS: $[ stageDependencies.configure_build.configure.outputs['configure_platforms.DOTNET_PLATFORMS'] ] + # The ACES pool is a virtual-machine pool; expose VM_VENDOR so tests + # gated by TestRuntime.AssertNotVirtualMachine get ignored there. + ${{ if parameters.useACES }}: + VM_VENDOR: ACES ${{ if parameters.useACES }}: pool: name: $(CIBuildPoolACES) From c0475b0e22ef4f42379d8bf6e55c9526d91c390c Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Thu, 28 May 2026 16:44:26 -0400 Subject: [PATCH 12/16] [tests] A couple of more VM checks on VideoToolbox --- .../VideoToolbox/VTCompressionSessionTests.cs | 3 +++ .../VideoToolbox/VTMotionEstimationSessionTest.cs | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/tests/monotouch-test/VideoToolbox/VTCompressionSessionTests.cs b/tests/monotouch-test/VideoToolbox/VTCompressionSessionTests.cs index d72f372a878..0c5a4f57af5 100644 --- a/tests/monotouch-test/VideoToolbox/VTCompressionSessionTests.cs +++ b/tests/monotouch-test/VideoToolbox/VTCompressionSessionTests.cs @@ -236,6 +236,9 @@ public void TestCallbackBackground (bool stronglyTyped) public void TestMultiImage (bool stronglyTyped, bool customCallback) { TestRuntime.AssertXcodeVersion (26, 0); +#if __MACOS__ || __MACCATALYST__ + TestRuntime.AssertNotVirtualMachine (); +#endif if (!VTCompressionSession.IsStereoMvHevcEncodeSupported ()) Assert.Ignore ("Stereo MV-HEVC encoding is not supported on the current system."); diff --git a/tests/monotouch-test/VideoToolbox/VTMotionEstimationSessionTest.cs b/tests/monotouch-test/VideoToolbox/VTMotionEstimationSessionTest.cs index a8f49139e6f..9e338d9d6ef 100644 --- a/tests/monotouch-test/VideoToolbox/VTMotionEstimationSessionTest.cs +++ b/tests/monotouch-test/VideoToolbox/VTMotionEstimationSessionTest.cs @@ -24,6 +24,9 @@ public class VTMotionEstimationSessionTest { public void CreateTest () { TestRuntime.AssertXcodeVersion (26, 0); +#if __MACOS__ || __MACCATALYST__ + TestRuntime.AssertNotVirtualMachine (); +#endif // VTMotionEstimationSessionCreate just returns in the simulator (a single 'ret' instruction), // which means it returns with status=VTStatus.Ok, but no session actually created. So ignore // this test in the simulator. @@ -49,6 +52,9 @@ public void CreateTest () public void CreateStronglyTypedTest () { TestRuntime.AssertXcodeVersion (26, 0); +#if __MACOS__ || __MACCATALYST__ + TestRuntime.AssertNotVirtualMachine (); +#endif // VTMotionEstimationSessionCreate just returns in the simulator (a single 'ret' instruction), // which means it returns with status=VTStatus.Ok, but no session actually created. So ignore // this test in the simulator. From 541d5bf6cec2c2862c99ed530e39ad24c1643067 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Thu, 28 May 2026 18:12:05 -0400 Subject: [PATCH 13/16] [CI] Retry the provisioning if it fails --- tools/devops/automation/templates/tests/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/devops/automation/templates/tests/build.yml b/tools/devops/automation/templates/tests/build.yml index 1eacbac94c7..a844ccfaec8 100644 --- a/tools/devops/automation/templates/tests/build.yml +++ b/tools/devops/automation/templates/tests/build.yml @@ -208,6 +208,7 @@ steps: provisioning_script: $(System.DefaultWorkingDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/build-provisioning.csx displayName: 'Provisionator dependencies' provisionatorChannel: $(PROVISIONATOR_CHANNEL) + retryCount: 3 - bash: | set -x From c9f4987d5b6add68817a90d74ddde33f85b9a614 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Thu, 28 May 2026 21:14:29 -0400 Subject: [PATCH 14/16] [Tests] bring back retry on tests --- tools/devops/automation/templates/tests/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/automation/templates/tests/run-tests.yml b/tools/devops/automation/templates/tests/run-tests.yml index 0272f5e2c4f..483f3afbb30 100644 --- a/tools/devops/automation/templates/tests/run-tests.yml +++ b/tools/devops/automation/templates/tests/run-tests.yml @@ -32,7 +32,7 @@ parameters: - name: retryCount type: number - default: 0 + default: 3 steps: - bash: | From 1adb3588e5dbd0cd05700f7eac294c295fb91dc8 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Fri, 29 May 2026 08:21:50 -0400 Subject: [PATCH 15/16] [tests] refine CI-only tolerance check for user directories in LinkSdkRegressionTest --- tests/linker/link sdk/LinkSdkRegressionTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/linker/link sdk/LinkSdkRegressionTest.cs b/tests/linker/link sdk/LinkSdkRegressionTest.cs index 97b17f2f66a..0f147186f70 100644 --- a/tests/linker/link sdk/LinkSdkRegressionTest.cs +++ b/tests/linker/link sdk/LinkSdkRegressionTest.cs @@ -898,12 +898,12 @@ void SpecialFolderImpl () Assert.That (path.EndsWith ("/Library", StringComparison.Ordinal), Is.True, "Resources"); #endif // Some CI VM images don't initialize all standard user directories, so keep this - // tolerance CI-only and preserve the stricter check for local runs. + // tolerance limited to CI VMs and preserve the stricter check for other runs. string TestFolderIfAvailableInCI (Environment.SpecialFolder folder, bool exists) { #if __MACOS__ var path = Environment.GetFolderPath (folder); - if (string.IsNullOrEmpty (path) && TestRuntime.IsInCI) + if (string.IsNullOrEmpty (path) && TestRuntime.IsInCI && TestRuntime.IsVM) return path; #endif return TestFolder (folder, exists: exists); From a6425cb41403d77f9fbe74329bf1d4829db21b50 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Mon, 1 Jun 2026 15:42:05 -0400 Subject: [PATCH 16/16] [tests] Revert to https and disable retry to diagnose furrther --- tests/monotouch-test/Network/NWConnectionTest.cs | 2 +- tests/monotouch-test/Network/NWParametersTest.cs | 14 +++++++------- .../automation/templates/tests/run-tests.yml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/monotouch-test/Network/NWConnectionTest.cs b/tests/monotouch-test/Network/NWConnectionTest.cs index e19a053507d..0d6b7832e8d 100644 --- a/tests/monotouch-test/Network/NWConnectionTest.cs +++ b/tests/monotouch-test/Network/NWConnectionTest.cs @@ -72,7 +72,7 @@ public void TestForceCancel () } class ConnectionManager : IDisposable { - string host = NetworkResources.MicrosoftHttpUri.Host; + string host = NetworkResources.MicrosoftUri.Host; AutoResetEvent connectedEvent = new AutoResetEvent (false); // used to let us know when the connection was established so that we can access the Report NWConnection? connection; NWParameters? parameters; diff --git a/tests/monotouch-test/Network/NWParametersTest.cs b/tests/monotouch-test/Network/NWParametersTest.cs index c20d892b5b1..7d59a66f895 100644 --- a/tests/monotouch-test/Network/NWParametersTest.cs +++ b/tests/monotouch-test/Network/NWParametersTest.cs @@ -87,7 +87,7 @@ public void CreateSecureUpdTest () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureUdp (configureTls: setUpTls, configureUdp: setUpProtocol)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { secureEvent.WaitOne (); configureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.True, "Configure TLS handler was not called."); @@ -101,7 +101,7 @@ public void CreateSecureUpdTestDoNotSetUpProtocol () var setUpTls = CreateTlsHandler (); using (var parameters = NWParameters.CreateSecureUdp (configureTls: setUpTls)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { secureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.True, "Configure TLS handler was not called."); Assert.That (protocolConfigured, Is.False, "Protocol configure handler was called."); @@ -114,7 +114,7 @@ public void CreateSecureUpdTestDoNotSetUpTls () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureUdp (configureTls: null, configureUdp: setUpProtocol)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { configureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.False, "Configure TLS handler was not called."); Assert.That (protocolConfigured, Is.True, "Protocol configure handler was not called."); @@ -128,7 +128,7 @@ public void CreateSecureTcpTest () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureTcp (configureTls: setUpTls, configureTcp: setUpProtocol)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { secureEvent.WaitOne (); configureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.True, "Configure TLS handler was not called."); @@ -143,7 +143,7 @@ public void CreateSecureTcpTestDoNotSetUpProtocol () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureTcp (configureTls: setUpTls)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { secureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.True, "Configure TLS handler was not called."); Assert.That (protocolConfigured, Is.False, "Protocol configure handler was called."); @@ -156,7 +156,7 @@ public void CreateSecureTcpTestDoNotSetUpTls () var setUpProtocol = CreateConfigureProtocolHandler (); using (var parameters = NWParameters.CreateSecureTcp (configureTls: null, configureTcp: setUpProtocol)) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { configureEvent.WaitOne (); Assert.That (secureConnectionWasSet, Is.False, "Configure TLS handler was called."); Assert.That (protocolConfigured, Is.True, "Protocol configure handler was not called."); @@ -317,7 +317,7 @@ public void LocalEndpointPropertyTest () { Assert.Ignore ("nw_parameters_copy_local_endpoint always return null. Rdar filled 44095278."); using (var parameters = NWParameters.CreateUdp ()) - using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftHttpUri.Host, "80")) { + using (var endpoint = NWEndpoint.Create (NetworkResources.MicrosoftUri.Host, "80")) { var defaultValue = parameters.LocalEndpoint; Assert.That (defaultValue, Is.Null, "Default value changed."); parameters.LocalEndpoint = endpoint; diff --git a/tools/devops/automation/templates/tests/run-tests.yml b/tools/devops/automation/templates/tests/run-tests.yml index 483f3afbb30..0272f5e2c4f 100644 --- a/tools/devops/automation/templates/tests/run-tests.yml +++ b/tools/devops/automation/templates/tests/run-tests.yml @@ -32,7 +32,7 @@ parameters: - name: retryCount type: number - default: 3 + default: 0 steps: - bash: |