Combine official-build artifact uploads for perf (#12931)

Scans injected by policy have dramatically increased our official build
time. I consulted with some folks internally, the highest-impact
approach we identified was to reduce the number of invocations of the
artifact-upload tasks, since each one in the normal flow of the job
incurs a scan cost.

To do this we had to:

1. Convert all of the `1ES.PublishBuildArtifacts` task calls to
declarative template `outputs:`
2. define an `outputParentDirectory` that can be scanned exactly once
(for all uploaded artifacts) instead of per-upload
3. Restrict that `outputParentDirectory` to contain only things that
will be uploaded (`artifacts/` was too broad and caused the scan to find
some false positives).

Example successful official build:
https://dev.azure.com/devdiv/DevDiv/_build/results?buildId=12895565
<img width="503" height="346" alt="image"
src="https://github.com/user-attachments/assets/b79838d2-bba1-43e0-bd15-00584c4ca858"
/>

(compare that 1h 21m to 2h 34m from a recent `main` branch official
build)

Example exp build showing log upload on (artificially injected) failure:
https://dev.azure.com/devdiv/DevDiv/_build/results?buildId=12941708

Author: rainersigwald

azure-pipelines/.vsts-dotnet-build-jobs.yml

@@ -22,6 +22,51 @@ jobs:
 
   timeoutInMinutes: 180
 
+  templateContext:
+    outputParentDirectory: '$(Build.SourcesDirectory)\artifacts\official'
+    outputs:
+    # Publish OptProf configuration files
+    - output: artifactsDrop
+      sourcePath: '$(Build.SourcesDirectory)\artifacts\official\OptProf\$(BuildConfiguration)\Data'
+      dropServiceURI: 'https://devdiv.artifacts.visualstudio.com'
+      dropMetadataContainerName: 'ProfilingInputs/DevDiv/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber)'
+      condition: and(succeeded(), ${{ parameters.enableOptProf }})
+
+    # Publish bootstrapper info for OptProf data collection run to consume
+    - output: buildArtifacts
+      PathtoPublish: '$(Build.SourcesDirectory)\artifacts\official\MicroBuild\Output'
+      ArtifactName: MicroBuildOutputs
+
+    # RAINES TODO: switch to pipeline?
+    - output: buildArtifacts
+      PathtoPublish: '$(Build.SourcesDirectory)\artifacts\official\log\$(BuildConfiguration)'
+      ArtifactName: logs
+      condition: always()
+
+    # Publish test configuration for OptProf data collection
+    - output: buildArtifacts
+      PathtoPublish: '$(Build.SourcesDirectory)\artifacts\official\VSSetup\$(BuildConfiguration)'
+      ArtifactName: VSSetup
+      condition: and(succeeded(), ${{ parameters.enableOptProf }})
+
+    # Archive NuGet packages to DevOps.
+    # Publish our NuPkgs as an artifact. The name of this artifact must be PackageArtifacts as the
+    # arcade templates depend on the name.
+    - output: buildArtifacts
+      PathtoPublish: '$(Build.SourcesDirectory)\artifacts\official\packages\$(BuildConfiguration)'
+      ArtifactName: PackageArtifacts
+
+    # Publish "IntelliSense" XSD files to their own artifact
+    # so it can be consumed by the insertion-to-VS job
+    - output: pipelineArtifact
+      targetPath: '$(Build.SourcesDirectory)\artifacts\official\xsd'
+      artifactName: xsd
+
+    # Publish Asset Manifests for Build Asset Registry job
+    - output: buildArtifacts
+      PathtoPublish: '$(Build.SourcesDirectory)\artifacts\official\log\$(BuildConfiguration)\AssetManifest'
+      ArtifactName: AssetManifests
+
   variables:
   - group: Publish-Build-Assets
   - name: TeamName
@@ -106,17 +151,6 @@ jobs:
   # Required by Microsoft policy
   - template: /eng/common/templates-official/steps/generate-sbom.yml@self
 
-  # Publish OptProf configuration files
-  - task: 1ES.PublishArtifactsDrop@1
-    inputs:
-      dropServiceURI: 'https://devdiv.artifacts.visualstudio.com'
-      buildNumber: 'ProfilingInputs/DevDiv/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber)'
-      sourcePath: '$(Build.SourcesDirectory)\artifacts\OptProf\$(BuildConfiguration)\Data'
-      toLowerCase: false
-      usePat: true
-    displayName: 'OptProf - Publish to Artifact Services - ProfilingInputs'
-    condition: and(succeeded(), ${{ parameters.enableOptProf }})
-
   # Build VS bootstrapper
   # Generates $(Build.StagingDirectory)\MicroBuild\Output\BootstrapperInfo.json
   - task: MicroBuildBuildVSBootstrapper@3
@@ -140,22 +174,6 @@ jobs:
     displayName: 'OptProf - Build IBC training settings'
     condition: and(succeeded(), ${{ parameters.enableOptProf }})
 
-  # Publish bootstrapper info
-  - task: 1ES.PublishBuildArtifacts@1
-    inputs:
-      PathtoPublish: $(Build.StagingDirectory)\MicroBuild\Output
-      ArtifactName: MicroBuildOutputs
-      ArtifactType: Container
-    displayName: 'OptProf - Publish Artifact: MicroBuildOutputs'
-    condition: succeeded()
-
-  - task: 1ES.PublishBuildArtifacts@1
-    displayName: 'Publish Artifact: logs'
-    inputs:
-      PathtoPublish: 'artifacts\log\$(BuildConfiguration)'
-      ArtifactName: logs
-    condition: succeededOrFailed()
-
   # Publishes setup VSIXes to a drop.
   # Note: The insertion tool looks for the display name of this task in the logs.
   - task: 1ES.MicroBuildVstsDrop@1
@@ -169,40 +187,69 @@ jobs:
       vsDropServiceUri: 'https://vsdrop.corp.microsoft.com/file/v1'
     condition: succeeded()
 
-  # Publish an artifact that the RoslynInsertionTool is able to find by its name.
-  - task: 1ES.PublishBuildArtifacts@1
-    displayName: 'Publish Artifact: VSSetup'
-    inputs:
-      PathtoPublish: 'artifacts\VSSetup\$(BuildConfiguration)'
-      ArtifactName: VSSetup
-    condition: succeeded()
+  - pwsh: |
+      $ErrorActionPreference = 'Stop'
 
-  # Archive NuGet packages to DevOps.
-  # Publish our NuPkgs as an artifact. The name of this artifact must be PackageArtifacts as the
-  # arcade templates depend on the name.
-  - task: 1ES.PublishBuildArtifacts@1
-    displayName: 'Publish Artifact: packages'
-    inputs:
-      PathtoPublish: 'artifacts\packages\$(BuildConfiguration)'
-      ArtifactName: PackageArtifacts
-    condition: succeeded()
+      $sourceRoot = Join-Path '$(Build.SourcesDirectory)' 'artifacts'
+      $targetRoot = Join-Path $sourceRoot 'official'
 
-  # Publish "IntelliSense" XSD files to their own artifact
-  # so it can be consumed by the insertion-to-VS job
-  - task: 1ES.PublishPipelineArtifact@1
-    displayName: 'Publish Artifact: xsd'
-    inputs:
-      path: 'artifacts\xsd'
-      artifactName: xsd
-    condition: succeeded()
+      if (-not (Test-Path $targetRoot)) {
+        New-Item -Path $targetRoot -ItemType Directory | Out-Null
+      }
 
-  # Publish Asset Manifests for Build Asset Registry job
-  - task: 1ES.PublishBuildArtifacts@1
-    displayName: Publish Asset Manifests
-    inputs:
-      PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)/AssetManifest'
-      ArtifactName: AssetManifests
-    condition: succeeded()
+      $paths = @(
+        "log/$(BuildConfiguration)",
+        "$(Build.StagingDirectory)/MicroBuild/Output",
+        "packages/$(BuildConfiguration)",
+        "xsd"
+      )
+
+      if ("${{ parameters.enableOptProf }}" -eq "true") {  
+        $paths += "OptProf/$(BuildConfiguration)/Data"  
+        $paths += "VSSetup/$(BuildConfiguration)"  
+      }
+
+      $allpresent = $true
+
+      foreach ($path in $paths) {
+        $from = [System.IO.Path]::Combine($sourceRoot, $path)
+
+        $destinationSubPath = if ($path -eq "$(Build.StagingDirectory)/MicroBuild/Output") {
+          "MicroBuild/Output"
+        } else {
+          $path
+        }
+
+        $to = Join-Path $targetRoot $destinationSubPath
+
+        Write-Host "Moving '$from' -> '$to'"
+
+        $parent = Split-Path $to -Parent
+        if (-not (Test-Path $parent)) {
+          New-Item -Path $parent -ItemType Directory -Force | Out-Null
+        }
+
+        if (-not (Test-Path $from)) {
+          Write-Host "##vso[task.logissue type=error]Source '$from' does not exist; skipping move for '$path'."
+          $allpresent = $false
+          continue
+        }
+
+        $robocopyResult = & robocopy $from $to /E /MOVE /NFL /NDL /NJH /NJS /NP /MT:8
+        $exitCode = $LASTEXITCODE
+        if ($exitCode -ge 8) {
+          $robocopyResult | Out-String | Write-Host
+          throw "Robocopy failed with exit code $exitCode while moving '$path' to '$destinationSubPath'."
+        }
+      }
+
+      if (-not $allpresent) {
+        throw "An expected output was not present."
+      }
+
+      $global:LASTEXITCODE = 0
+    displayName: 'Move build artifacts into official folder'
+    condition: always()
 
   # Tag the build at the very end when we know it's been successful.
   - task: colinsalmcorner.colinsalmcorner-buildtasks.tag-build-task.tagBuildOrRelease@0