From 7a49c8b9d49e1acd5c427df9fd0eefe14ad6feb0 Mon Sep 17 00:00:00 2001
From: Ewerton Scaboro da Silva <ewertons@microsoft.com>
Date: Sat, 10 Oct 2026 00:36:08 +0000
Subject: [PATCH 1/8] [CI] Rebuild the shared mqttv5 e2e ADR namespace when it
 disappears

The ADR namespace of the shared mqttv5 environment can disappear from ARM,
taking its role assignments with it; DPS registrations then fail.

c/eng/e2e-mqttv5-adr-namespace.ps1 re-creates the namespace (or re-asserts
its endpoints when ADR still holds it), re-grants the role assignments,
re-links hub and DPS, and pushes the namespace to DPS. A healthy namespace
costs one ARM read. ci-c-e2e-mqttv5-adr-namespace.yml runs it hourly, on
demand, and via workflow_call.
---
 .../ci-c-e2e-mqttv5-adr-namespace.yml         |  64 ++++++
 c/docs/eng/end-to-end-tests.md                |  11 +
 c/eng/e2e-mqttv5-adr-namespace.ps1            | 192 ++++++++++++++++++
 3 files changed, 267 insertions(+)
 create mode 100644 .github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
 create mode 100644 c/eng/e2e-mqttv5-adr-namespace.ps1

diff --git a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
new file mode 100644
index 000000000..c97f7fb7d
--- /dev/null
+++ b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
@@ -0,0 +1,64 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+# Keeps the shared mqttv5 e2e environment's ADR namespace present and linked.
+#
+# Workaround: the namespace can disappear from ARM (taking its role assignments
+# with it), after which DPS registrations fail. c/eng/e2e-mqttv5-adr-namespace.ps1
+# rebuilds and re-links it; when it is healthy the run is one ARM read.
+#
+# Runs hourly, on demand, and as the first job of the mqttv5 e2e workflow
+# (workflow_call), so tests start against a linked namespace.
+#
+#   vars:    E2E_MQTTV5_SHARED_SUBSCRIPTION_ID, E2E_MQTTV5_SHARED_RESOURCE_GROUP,
+#            E2E_MQTTV5_SHARED_HUB_NAME, E2E_MQTTV5_SHARED_DPS_NAME,
+#            E2E_MQTTV5_SHARED_ADR_NAMESPACE
+#   secrets: AZURE_CLIENT_ID, AZURE_TENANT_ID (OIDC). On the resource group the
+#            identity needs Contributor, and role assignment write for
+#            Contributor, IoT Hub Data Contributor and Azure Device Registry
+#            Contributor.
+
+name: ci-c-e2e-mqttv5-adr-namespace
+
+on:
+  workflow_dispatch:
+  workflow_call:
+  schedule:
+    - cron: '17 * * * *'
+
+permissions:
+  contents: read
+  id-token: write   # azure/login OIDC
+
+jobs:
+  ensure:
+    name: ensure ADR namespace
+    if: ${{ vars.E2E_MQTTV5_SHARED_ADR_NAMESPACE != '' }}
+    runs-on: ubuntu-24.04
+    # Namespace create + role replication + link (up to 4 attempts).
+    timeout-minutes: 75
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+          sparse-checkout: c/eng
+
+      - name: Azure Login via OIDC
+        uses: azure/login@7184910d9eb2b1c5e48f7073824a90609bb9b6d6 # v2.3.1
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ vars.E2E_MQTTV5_SHARED_SUBSCRIPTION_ID }}
+
+      - name: Ensure ADR namespace
+        shell: pwsh
+        env:
+          SUBSCRIPTION_ID: ${{ vars.E2E_MQTTV5_SHARED_SUBSCRIPTION_ID }}
+          RESOURCE_GROUP: ${{ vars.E2E_MQTTV5_SHARED_RESOURCE_GROUP }}
+          HUB_NAME: ${{ vars.E2E_MQTTV5_SHARED_HUB_NAME }}
+          DPS_NAME: ${{ vars.E2E_MQTTV5_SHARED_DPS_NAME }}
+          ADR_NAMESPACE: ${{ vars.E2E_MQTTV5_SHARED_ADR_NAMESPACE }}
+        run: |
+          ./c/eng/e2e-mqttv5-adr-namespace.ps1 -SubscriptionId $env:SUBSCRIPTION_ID `
+            -ResourceGroup $env:RESOURCE_GROUP -HubName $env:HUB_NAME `
+            -DpsName $env:DPS_NAME -NamespaceName $env:ADR_NAMESPACE
diff --git a/c/docs/eng/end-to-end-tests.md b/c/docs/eng/end-to-end-tests.md
index aeddc0a3d..a986cb1a0 100644
--- a/c/docs/eng/end-to-end-tests.md
+++ b/c/docs/eng/end-to-end-tests.md
@@ -235,6 +235,17 @@ CA certificate and key, then its root). `E2E_CSR_SHARED_DPS_HOST` optionally set
 device endpoint. `E2E_CSR_SHARED_SAS_GROUP_KEY` is the key of a symmetric-key group linked
 to the same certificate policy.
 
+The shared mqttv5 environment (ADR-linked mqttv5 hub and DPS) is kept usable by
+[`ci-c-e2e-mqttv5-adr-namespace.yml`](../../../.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml)
+(hourly, manual, or `workflow_call` as the first job of a mqttv5 e2e workflow). The ADR namespace
+can disappear from ARM, removing its role assignments; DPS registrations then fail.
+[`c/eng/e2e-mqttv5-adr-namespace.ps1`](../../eng/e2e-mqttv5-adr-namespace.ps1) re-creates it
+(same name, hub region), re-grants the 8 role assignments, re-links hub and DPS, and pushes the
+namespace to DPS; when the namespace is healthy it makes one ARM read. Inputs are repository
+variables `E2E_MQTTV5_SHARED_SUBSCRIPTION_ID`, `_RESOURCE_GROUP`, `_HUB_NAME`, `_DPS_NAME`
+and `_ADR_NAMESPACE` (unset `_ADR_NAMESPACE` skips the job). The OIDC identity needs Contributor
+and role assignment write on the resource group.
+
 > **Software updates e2e** runs in its own workflow
 > ([`ci-c-e2e-adu.yml`](../../../.github/workflows/ci-c-e2e-adu.yml), Linux, manual dispatch). See [Software updates e2e](#software-updates-e2e).
 
diff --git a/c/eng/e2e-mqttv5-adr-namespace.ps1 b/c/eng/e2e-mqttv5-adr-namespace.ps1
new file mode 100644
index 000000000..8ccd9e068
--- /dev/null
+++ b/c/eng/e2e-mqttv5-adr-namespace.ps1
@@ -0,0 +1,192 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+<#
+.SYNOPSIS
+Ensures the ADR namespace of the shared mqttv5 e2e environment exists and links its hub and DPS.
+
+.DESCRIPTION
+Workaround: in some regions the ADR namespace can disappear from ARM while the ADR service still
+holds it; its role assignments go with it, and DPS registrations then fail. When the namespace
+is missing, or an endpoint is not linked, this script:
+  1. re-creates it (same name/region; if ADR still holds it, re-asserts its endpoints),
+  2. re-grants the 8 role assignments the link needs (idempotent),
+  3. links the hub (messaging/hub-1) and DPS (provisioning/dps-1) and waits for Succeeded,
+  4. pushes the namespace to the DPS data plane (tags-only DPS update).
+If the namespace is present with both endpoints Succeeded it makes one ARM read and exits.
+
+Requires `az` logged in with, on the resource group: Contributor, and
+Microsoft.Authorization/roleAssignments/write (e.g. Role Based Access Control Administrator)
+for Contributor (b24988ac-...), IoT Hub Data Contributor (4fc6c259-...) and
+Azure Device Registry Contributor (a5c3590a-...).
+Hub and DPS must have system-assigned identities.
+#>
+[CmdletBinding(PositionalBinding = $false)]
+param(
+    [Parameter(Mandatory)][string]$SubscriptionId,
+    [Parameter(Mandatory)][string]$ResourceGroup,
+    [Parameter(Mandatory)][string]$HubName,
+    [Parameter(Mandatory)][string]$DpsName,
+    [Parameter(Mandatory)][string]$NamespaceName,
+    [string]$AdrApiVersion = '2026-11-02-preview',
+    [string]$HubApiVersion = '2026-10-01-preview',
+    [string]$DpsApiVersion = '2026-11-02-preview',
+    [int]$LinkTimeoutMinutes = 12,
+    [int]$LinkAttempts = 4
+)
+
+$ErrorActionPreference = 'Stop'
+
+$Arm = 'https://management.azure.com'
+$Rg = "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroup"
+$HubId = "$Rg/providers/Microsoft.Devices/IotHubs/$HubName"
+$DpsId = "$Rg/providers/Microsoft.Devices/provisioningServices/$DpsName"
+$NsId = "$Rg/providers/Microsoft.DeviceRegistry/namespaces/$NamespaceName"
+$HubUrl = "$Arm${HubId}?api-version=$HubApiVersion"
+$DpsUrl = "$Arm${DpsId}?api-version=$DpsApiVersion"
+$NsUrl = "$Arm${NsId}?api-version=$AdrApiVersion"
+$RoleContributor = 'b24988ac-6180-42a0-ab88-20f7382dd24c'
+$RoleHubData = '4fc6c259-987e-4a07-842e-c321cc9d413f'
+$RoleAdr = 'a5c3590a-3a1a-4cd4-9648-ea0a32b15137'
+# Link failures that clear once new role assignments replicate.
+$PropagationCodes = 'AdrMiNotAuthorized|LinkableResourceNotReady|AuthorizationFailed|LinkInitiateFailed|NamespaceMiTokenAcquisitionFailed|OutboundIdentityUnavailable'
+
+# Returns $null on 404. Response bodies are not logged: DPS reads can carry keys.
+function Invoke-Arm([string]$Method, [string]$Url, $Body) {
+    $a = @('rest', '--method', $Method, '--url', $Url, '-o', 'json')
+    $tmp = $null
+    if ($null -ne $Body) {
+        $tmp = New-TemporaryFile
+        $Body | ConvertTo-Json -Depth 20 -Compress | Set-Content -LiteralPath $tmp -Encoding utf8NoBOM
+        $a += @('--headers', 'Content-Type=application/json', '--body', "@$tmp")
+    }
+    try {
+        $out = ((& az @a 2>&1) | Out-String).Trim()
+        if ($LASTEXITCODE -ne 0) {
+            if ($Method -eq 'get' -and $out -match 'ResourceNotFound|\(NotFound\)|Not Found') { return $null }
+            $msg = $out -replace '(?i)("(?:primaryKey|secondaryKey|accessToken)"\s*:\s*")[^"]*', '$1<redacted>'
+            throw "ARM $($Method.ToUpper()) $($Url -replace '\?.*') failed: $msg"
+        }
+        if ($out) { $out | ConvertFrom-Json -Depth 50 }
+    }
+    finally { if ($tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } }
+}
+
+function Get-EndpointStates($Ns) {
+    foreach ($g in 'messaging', 'provisioning') {
+        $eps = $Ns.properties.$g.endpoints
+        if (-not $eps) { continue }
+        foreach ($p in $eps.PSObject.Properties) {
+            [pscustomobject]@{ Name = "$g/$($p.Name)"; State = [string]$p.Value.linkingState; Code = [string]$p.Value.linkingError.code; Message = [string]$p.Value.linkingError.message }
+        }
+    }
+}
+
+function Test-Linked($Ns) {
+    if (-not $Ns -or $Ns.properties.provisioningState -ne 'Succeeded') { return $false }
+    $s = @(Get-EndpointStates $Ns)
+    $s.Count -ge 2 -and -not ($s | Where-Object State -ne 'Succeeded')
+}
+
+function Wait-For([string]$What, [int]$Minutes, [scriptblock]$Probe) {
+    $deadline = (Get-Date).AddMinutes($Minutes)
+    while ($true) {
+        $r = & $Probe
+        if ($r.Done) { return $r }
+        if ($r.ContainsKey('Failed') -and $r.Failed) { throw "$What failed: $($r.Detail)" }
+        if ((Get-Date) -ge $deadline) { throw "$What did not finish within $Minutes min (last: $($r.Detail))." }
+        Write-Host "  $What : $($r.Detail)"
+        Start-Sleep -Seconds 15
+    }
+}
+
+function Grant-Role([string]$Scope, [string]$PrincipalId, [string]$RoleId, [string]$What) {
+    for ($i = 1; $i -le 8; $i++) {
+        $out = ((& az role assignment create --assignee-object-id $PrincipalId --assignee-principal-type ServicePrincipal --role $RoleId --scope $Scope -o none 2>&1) | Out-String).Trim()
+        if ($LASTEXITCODE -eq 0 -or $out -match 'RoleAssignmentExists|already exists') { Write-Host "  role: $What"; return }
+        if ($i -eq 8) { throw "Role assignment '$What' failed: $out" }
+        Start-Sleep -Seconds 15
+    }
+}
+
+[void](& az account set --subscription $SubscriptionId)
+if ($LASTEXITCODE -ne 0) { throw "az account set --subscription $SubscriptionId failed." }
+
+$ns = Invoke-Arm get $NsUrl
+if (Test-Linked $ns) { Write-Host "ADR namespace $NamespaceName present and linked."; return }
+Write-Host "::warning::ADR namespace $NamespaceName $(if ($ns) { 'not linked' } else { 'missing from ARM' }); rebuilding it."
+
+$hub = Invoke-Arm get $HubUrl
+$dps = Invoke-Arm get $DpsUrl
+if (-not $hub -or -not $dps) { throw 'Hub or DPS not found.' }
+$hubPid = $hub.identity.principalId; $dpsPid = $dps.identity.principalId
+if (-not $hubPid -or -not $dpsPid) { throw 'Hub and DPS need system-assigned identities.' }
+# ADR requires the namespace in the hub's region.
+$location = $hub.location
+
+$tags = @{ purpose = 'mqttv5-e2e'; restoredBy = 'e2e-mqttv5-adr-namespace' }
+$linkBody = @{
+    location = $location; identity = @{ type = 'SystemAssigned' }; tags = $tags
+    properties = @{
+        messaging    = @{ endpoints = @{ 'hub-1' = @{ endpointType = 'Microsoft.Devices/IotHubs'; resourceId = $HubId
+                        inboundCallerIdentity = @{ type = 'SystemAssigned' }; provisioning = @{ availability = 'Available'; allocationWeight = 1 } } } }
+        provisioning = @{ endpoints = @{ 'dps-1' = @{ endpointType = 'Microsoft.Devices/provisioningServices'; resourceId = $DpsId
+                        inboundCallerIdentity = @{ type = 'SystemAssigned' } } } }
+    }
+}
+
+# 1. Namespace. A bare create fails with EndpointUsedByNestedDevices when ADR still holds it; then re-assert its endpoints.
+if (-not $ns) {
+    try { [void](Invoke-Arm put $NsUrl @{ location = $location; identity = @{ type = 'SystemAssigned' }; tags = $tags }) }
+    catch {
+        if ($_.Exception.Message -notmatch 'EndpointUsedByNestedDevices') { throw }
+        Write-Host '  ADR still holds the namespace; re-asserting its endpoints.'
+        [void](Invoke-Arm put $NsUrl $linkBody)
+    }
+}
+$ns = (Wait-For 'namespace' 10 {
+        $n = Invoke-Arm get $NsUrl
+        $ps = if ($n) { $n.properties.provisioningState } else { 'absent' }
+        @{ Done = ($ps -in 'Succeeded', 'Failed' -and $n.identity.principalId); Detail = $ps; Ns = $n } }).Ns
+$nsPid = $ns.identity.principalId
+
+# 2. Role assignments (removed with the namespace; the namespace identity may be new).
+Grant-Role $HubId $nsPid $RoleContributor 'namespace -> hub (Contributor)'
+Grant-Role $NsId $hubPid $RoleContributor 'hub -> namespace (Contributor)'
+Grant-Role $DpsId $nsPid $RoleContributor 'namespace -> DPS (Contributor)'
+Grant-Role $NsId $dpsPid $RoleContributor 'DPS -> namespace (Contributor)'
+Grant-Role $NsId $dpsPid $RoleAdr 'DPS -> namespace (Azure Device Registry Contributor)'
+Grant-Role $HubId $dpsPid $RoleHubData 'DPS -> hub (IoT Hub Data Contributor)'
+Grant-Role $NsId $nsPid $RoleContributor 'namespace -> namespace (Contributor)'
+Grant-Role $HubId $nsPid $RoleHubData 'namespace -> hub (IoT Hub Data Contributor)'
+
+# 3. Link hub + DPS in one write (ADR refuses a messaging endpoint alone).
+for ($attempt = 1; ; $attempt++) {
+    Start-Sleep -Seconds 60   # role assignment replication
+    Write-Host "  link attempt $attempt/$LinkAttempts"
+    [void](Invoke-Arm put $NsUrl $linkBody)
+    $r = Wait-For 'link' $LinkTimeoutMinutes {
+        $n = Invoke-Arm get $NsUrl
+        $s = @(if ($n) { Get-EndpointStates $n })
+        $failed = @($s | Where-Object State -eq 'Failed')
+        $done = (Test-Linked $n) -or $failed.Count -gt 0 -or ($n -and $n.properties.provisioningState -in 'Failed', 'Canceled')
+        @{ Done = $done; Linked = (Test-Linked $n); Failed = $false; States = $failed
+           Detail = if ($s) { ($s | ForEach-Object { "$($_.Name)=$($_.State)" }) -join ', ' } else { '<no endpoints>' } } }
+    if ($r.Linked) { break }
+    $why = ($r.States | ForEach-Object { "$($_.Name): $($_.Code) $($_.Message)" }) -join '; '
+    $retryable = $r.States.Count -gt 0 -and -not ($r.States | Where-Object { $_.Code -notmatch "^($PropagationCodes)$" })
+    if (-not $retryable -or $attempt -ge $LinkAttempts) { throw "ADR link failed: $(if ($why) { $why } else { 'namespace Failed' })" }
+    Write-Host "  role replication pending ($why); retrying"
+}
+
+# 4. Push the namespace to the DPS data plane (tags-only update).
+$dpsTags = @{}
+if ($dps.tags) { $dps.tags.PSObject.Properties | ForEach-Object { $dpsTags[$_.Name] = $_.Value } }
+$dpsTags['adrNamespaceRestored'] = [DateTime]::UtcNow.ToString('o')
+[void](Invoke-Arm patch $DpsUrl @{ tags = $dpsTags })
+[void](Wait-For 'DPS update' 5 {
+        $d = Invoke-Arm get $DpsUrl
+        @{ Done = ($d.properties.provisioningState -eq 'Succeeded' -and $d.properties.state -eq 'Active'); Detail = "$($d.properties.provisioningState) / $($d.properties.state)" } })
+
+Write-Host "ADR namespace $NamespaceName rebuilt and linked."
+if ($env:GITHUB_STEP_SUMMARY) { "ADR namespace ``$NamespaceName`` was rebuilt and re-linked by this run." | Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY }

From ffeb6596b94bc96d547ca117ded04c00902e683a Mon Sep 17 00:00:00 2001
From: Ewerton Scaboro da Silva <ewertons@microsoft.com>
Date: Sat, 10 Oct 2026 00:43:31 +0000
Subject: [PATCH 2/8] Check hub-1/dps-1 and their targets; raise job timeout to
 100 min

---
 .github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml | 5 +++--
 c/eng/e2e-mqttv5-adr-namespace.ps1                  | 8 ++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
index c97f7fb7d..f616220bc 100644
--- a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
+++ b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
@@ -35,8 +35,9 @@ jobs:
     name: ensure ADR namespace
     if: ${{ vars.E2E_MQTTV5_SHARED_ADR_NAMESPACE != '' }}
     runs-on: ubuntu-24.04
-    # Namespace create + role replication + link (up to 4 attempts).
-    timeout-minutes: 75
+    # Above the script's worst case: namespace 10 + role grants 14 + 4 link
+    # attempts x 13 + DPS 5 = 81 min, plus ARM latency.
+    timeout-minutes: 100
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
         with:
diff --git a/c/eng/e2e-mqttv5-adr-namespace.ps1 b/c/eng/e2e-mqttv5-adr-namespace.ps1
index 8ccd9e068..783368eea 100644
--- a/c/eng/e2e-mqttv5-adr-namespace.ps1
+++ b/c/eng/e2e-mqttv5-adr-namespace.ps1
@@ -82,10 +82,14 @@ function Get-EndpointStates($Ns) {
     }
 }
 
+# Healthy only if hub-1 and dps-1 are Succeeded and point at this hub and DPS.
 function Test-Linked($Ns) {
     if (-not $Ns -or $Ns.properties.provisioningState -ne 'Succeeded') { return $false }
-    $s = @(Get-EndpointStates $Ns)
-    $s.Count -ge 2 -and -not ($s | Where-Object State -ne 'Succeeded')
+    foreach ($e in @(@($Ns.properties.messaging.endpoints.'hub-1', $HubId), @($Ns.properties.provisioning.endpoints.'dps-1', $DpsId))) {
+        if (-not $e[0] -or $e[0].linkingState -ne 'Succeeded') { return $false }
+        if ($e[0].resourceId -and $e[0].resourceId -ne $e[1]) { return $false }
+    }
+    $true
 }
 
 function Wait-For([string]$What, [int]$Minutes, [scriptblock]$Probe) {

From f0d8a1a8eedb3f14c2b6f6f48638ec1d70cfea6a Mon Sep 17 00:00:00 2001
From: Ewerton Scaboro da Silva <ewertons@microsoft.com>
Date: Sat, 10 Oct 2026 00:51:36 +0000
Subject: [PATCH 3/8] Serialize repair runs; re-read DPS tags before the PATCH

---
 .github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml | 5 +++++
 c/eng/e2e-mqttv5-adr-namespace.ps1                  | 2 ++
 2 files changed, 7 insertions(+)

diff --git a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
index f616220bc..a247c1395 100644
--- a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
+++ b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
@@ -35,6 +35,11 @@ jobs:
     name: ensure ADR namespace
     if: ${{ vars.E2E_MQTTV5_SHARED_ADR_NAMESPACE != '' }}
     runs-on: ubuntu-24.04
+    # One repair at a time across schedule, dispatch and callers. GitHub keeps
+    # one pending run per group; a third queued run cancels the pending one.
+    concurrency:
+      group: mqttv5-adr-namespace
+      cancel-in-progress: false
     # Above the script's worst case: namespace 10 + role grants 14 + 4 link
     # attempts x 13 + DPS 5 = 81 min, plus ARM latency.
     timeout-minutes: 100
diff --git a/c/eng/e2e-mqttv5-adr-namespace.ps1 b/c/eng/e2e-mqttv5-adr-namespace.ps1
index 783368eea..c74b7094b 100644
--- a/c/eng/e2e-mqttv5-adr-namespace.ps1
+++ b/c/eng/e2e-mqttv5-adr-namespace.ps1
@@ -184,6 +184,8 @@ for ($attempt = 1; ; $attempt++) {
 }
 
 # 4. Push the namespace to the DPS data plane (tags-only update).
+# Fresh read: recovery can take over an hour and the PATCH replaces the whole tag map.
+$dps = Invoke-Arm get $DpsUrl
 $dpsTags = @{}
 if ($dps.tags) { $dps.tags.PSObject.Properties | ForEach-Object { $dpsTags[$_.Name] = $_.Value } }
 $dpsTags['adrNamespaceRestored'] = [DateTime]::UtcNow.ToString('o')

From 84040b1c344c84c2eb17534ec71013632425914b Mon Sep 17 00:00:00 2001
From: Ewerton Scaboro da Silva <ewertons@microsoft.com>
Date: Sat, 10 Oct 2026 01:01:38 +0000
Subject: [PATCH 4/8] Declare workflow_call secrets; wait for the DPS tag
 marker

---
 .github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml | 5 +++++
 c/eng/e2e-mqttv5-adr-namespace.ps1                  | 6 ++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
index a247c1395..1ef9dac18 100644
--- a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
+++ b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
@@ -23,6 +23,11 @@ name: ci-c-e2e-mqttv5-adr-namespace
 on:
   workflow_dispatch:
   workflow_call:
+    secrets:
+      AZURE_CLIENT_ID:
+        required: true
+      AZURE_TENANT_ID:
+        required: true
   schedule:
     - cron: '17 * * * *'
 
diff --git a/c/eng/e2e-mqttv5-adr-namespace.ps1 b/c/eng/e2e-mqttv5-adr-namespace.ps1
index c74b7094b..0a918d6dd 100644
--- a/c/eng/e2e-mqttv5-adr-namespace.ps1
+++ b/c/eng/e2e-mqttv5-adr-namespace.ps1
@@ -188,11 +188,13 @@ for ($attempt = 1; ; $attempt++) {
 $dps = Invoke-Arm get $DpsUrl
 $dpsTags = @{}
 if ($dps.tags) { $dps.tags.PSObject.Properties | ForEach-Object { $dpsTags[$_.Name] = $_.Value } }
-$dpsTags['adrNamespaceRestored'] = [DateTime]::UtcNow.ToString('o')
+$marker = [guid]::NewGuid().ToString('N')
+$dpsTags['adrNamespaceRestored'] = $marker
 [void](Invoke-Arm patch $DpsUrl @{ tags = $dpsTags })
 [void](Wait-For 'DPS update' 5 {
         $d = Invoke-Arm get $DpsUrl
-        @{ Done = ($d.properties.provisioningState -eq 'Succeeded' -and $d.properties.state -eq 'Active'); Detail = "$($d.properties.provisioningState) / $($d.properties.state)" } })
+        # Succeeded/Active is also the pre-update state; require the new marker.
+        @{ Done = ($d.tags.adrNamespaceRestored -eq $marker -and $d.properties.provisioningState -eq 'Succeeded' -and $d.properties.state -eq 'Active'); Detail = "$($d.properties.provisioningState) / $($d.properties.state)" } })
 
 Write-Host "ADR namespace $NamespaceName rebuilt and linked."
 if ($env:GITHUB_STEP_SUMMARY) { "ADR namespace ``$NamespaceName`` was rebuilt and re-linked by this run." | Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY }

From 5e301b44077ebddc7595931a581b7536bc18780e Mon Sep 17 00:00:00 2001
From: Ewerton Scaboro da Silva <ewertons@microsoft.com>
Date: Sat, 10 Oct 2026 01:10:08 +0000
Subject: [PATCH 5/8] Keep useMiSdk tag; retry rejected link PUTs; reconcile
 Failed namespace with linked endpoints

---
 .../ci-c-e2e-mqttv5-adr-namespace.yml         |  5 +-
 c/eng/e2e-mqttv5-adr-namespace.ps1            | 52 ++++++++++++++++---
 2 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
index 1ef9dac18..1963cb62f 100644
--- a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
+++ b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
@@ -46,8 +46,9 @@ jobs:
       group: mqttv5-adr-namespace
       cancel-in-progress: false
     # Above the script's worst case: namespace 10 + role grants 14 + 4 link
-    # attempts x 13 + DPS 5 = 81 min, plus ARM latency.
-    timeout-minutes: 100
+    # attempts x 13 + 3 settle waits x 5 + reconcile 5 + DPS 5 = 101 min,
+    # plus ARM latency.
+    timeout-minutes: 120
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
         with:
diff --git a/c/eng/e2e-mqttv5-adr-namespace.ps1 b/c/eng/e2e-mqttv5-adr-namespace.ps1
index 0a918d6dd..7ab930cec 100644
--- a/c/eng/e2e-mqttv5-adr-namespace.ps1
+++ b/c/eng/e2e-mqttv5-adr-namespace.ps1
@@ -50,6 +50,10 @@ $RoleHubData = '4fc6c259-987e-4a07-842e-c321cc9d413f'
 $RoleAdr = 'a5c3590a-3a1a-4cd4-9648-ea0a32b15137'
 # Link failures that clear once new role assignments replicate.
 $PropagationCodes = 'AdrMiNotAuthorized|LinkableResourceNotReady|AuthorizationFailed|LinkInitiateFailed|NamespaceMiTokenAcquisitionFailed|OutboundIdentityUnavailable'
+# Link submissions that may be rejected outright yet succeed on a later attempt.
+$RetryableSubmit = "$PropagationCodes|ResourceProvisioningInProgress|Service ?Unavailable|Gateway ?Timeout|Bad ?Gateway|Too ?Many ?Requests|Internal ?Server ?Error|ServerTimeout|ServerBusy"
+# ADR feature flag: without it the link takes another path and reports the linked resource as unreadable.
+$FeatureTags = @{ useMiSdk = 'true' }
 
 # Returns $null on 404. Response bodies are not logged: DPS reads can carry keys.
 function Invoke-Arm([string]$Method, [string]$Url, $Body) {
@@ -82,9 +86,9 @@ function Get-EndpointStates($Ns) {
     }
 }
 
-# Healthy only if hub-1 and dps-1 are Succeeded and point at this hub and DPS.
-function Test-Linked($Ns) {
-    if (-not $Ns -or $Ns.properties.provisioningState -ne 'Succeeded') { return $false }
+# hub-1 and dps-1 Succeeded and pointing at this hub and DPS.
+function Test-EndpointsLinked($Ns) {
+    if (-not $Ns) { return $false }
     foreach ($e in @(@($Ns.properties.messaging.endpoints.'hub-1', $HubId), @($Ns.properties.provisioning.endpoints.'dps-1', $DpsId))) {
         if (-not $e[0] -or $e[0].linkingState -ne 'Succeeded') { return $false }
         if ($e[0].resourceId -and $e[0].resourceId -ne $e[1]) { return $false }
@@ -92,6 +96,15 @@ function Test-Linked($Ns) {
     $true
 }
 
+function Test-Linked($Ns) { $Ns -and $Ns.properties.provisioningState -eq 'Succeeded' -and (Test-EndpointsLinked $Ns) }
+
+function Get-NsTags($Ns) {
+    $t = @{}
+    if ($Ns -and $Ns.tags) { $Ns.tags.PSObject.Properties | ForEach-Object { $t[$_.Name] = $_.Value } }
+    $FeatureTags.GetEnumerator() | ForEach-Object { $t[$_.Key] = $_.Value }
+    $t
+}
+
 function Wait-For([string]$What, [int]$Minutes, [scriptblock]$Probe) {
     $deadline = (Get-Date).AddMinutes($Minutes)
     while ($true) {
@@ -128,7 +141,9 @@ if (-not $hubPid -or -not $dpsPid) { throw 'Hub and DPS need system-assigned ide
 # ADR requires the namespace in the hub's region.
 $location = $hub.location
 
-$tags = @{ purpose = 'mqttv5-e2e'; restoredBy = 'e2e-mqttv5-adr-namespace' }
+# Full PUTs replace tags: keep existing ones, re-assert the feature flag.
+$tags = Get-NsTags $ns
+$tags['restoredBy'] = 'e2e-mqttv5-adr-namespace'
 $linkBody = @{
     location = $location; identity = @{ type = 'SystemAssigned' }; tags = $tags
     properties = @{
@@ -167,16 +182,41 @@ Grant-Role $HubId $nsPid $RoleHubData 'namespace -> hub (IoT Hub Data Contributo
 # 3. Link hub + DPS in one write (ADR refuses a messaging endpoint alone).
 for ($attempt = 1; ; $attempt++) {
     Start-Sleep -Seconds 60   # role assignment replication
+    if ($attempt -gt 1) {
+        # ARM rejects writes (ResourceProvisioningInProgress) until a failed link settles.
+        try { [void](Wait-For 'namespace settle' 5 {
+                    $ps = (Invoke-Arm get $NsUrl).properties.provisioningState
+                    @{ Done = (-not $ps -or $ps -in 'Succeeded', 'Failed', 'Canceled'); Detail = $ps } }) }
+        catch { Write-Host "  $($_.Exception.Message)" }
+    }
     Write-Host "  link attempt $attempt/$LinkAttempts"
-    [void](Invoke-Arm put $NsUrl $linkBody)
+    try { [void](Invoke-Arm put $NsUrl $linkBody) }
+    catch {
+        $msg = $_.Exception.Message
+        if ($msg -notmatch $RetryableSubmit -or $attempt -ge $LinkAttempts) { throw }
+        Write-Host "  link rejected, retrying: $msg"
+        continue
+    }
     $r = Wait-For 'link' $LinkTimeoutMinutes {
         $n = Invoke-Arm get $NsUrl
         $s = @(if ($n) { Get-EndpointStates $n })
         $failed = @($s | Where-Object State -eq 'Failed')
         $done = (Test-Linked $n) -or $failed.Count -gt 0 -or ($n -and $n.properties.provisioningState -in 'Failed', 'Canceled')
-        @{ Done = $done; Linked = (Test-Linked $n); Failed = $false; States = $failed
+        @{ Done = $done; Linked = (Test-Linked $n); Failed = $false; States = $failed; Ns = $n
            Detail = if ($s) { ($s | ForEach-Object { "$($_.Name)=$($_.State)" }) -join ', ' } else { '<no endpoints>' } } }
     if ($r.Linked) { break }
+    # Endpoints linked but namespace Failed: a tags-only update re-runs reconciliation
+    # (re-sending endpoints is rejected as immutable). Failed can persist briefly after it is accepted.
+    if ((Test-EndpointsLinked $r.Ns) -and $r.Ns.properties.provisioningState -eq 'Failed') {
+        Write-Host '  endpoints linked, namespace Failed; reconciling (tags-only update)'
+        $t = Get-NsTags $r.Ns
+        $t['adrReconcile'] = [guid]::NewGuid().ToString('N')
+        [void](Invoke-Arm patch $NsUrl @{ tags = $t })
+        [void](Wait-For 'namespace reconcile' 5 {
+                $ps = (Invoke-Arm get $NsUrl).properties.provisioningState
+                @{ Done = ($ps -eq 'Succeeded'); Detail = $ps } })
+        break
+    }
     $why = ($r.States | ForEach-Object { "$($_.Name): $($_.Code) $($_.Message)" }) -join '; '
     $retryable = $r.States.Count -gt 0 -and -not ($r.States | Where-Object { $_.Code -notmatch "^($PropagationCodes)$" })
     if (-not $retryable -or $attempt -ge $LinkAttempts) { throw "ADR link failed: $(if ($why) { $why } else { 'namespace Failed' })" }

From 671a6116fd7ec26e6e809256412d551afc2dd097 Mon Sep 17 00:00:00 2001
From: Ewerton Scaboro da Silva <ewertons@microsoft.com>
Date: Sat, 10 Oct 2026 01:17:43 +0000
Subject: [PATCH 6/8] Pick the DPS api-version from what ARM serves

---
 c/eng/e2e-mqttv5-adr-namespace.ps1 | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/c/eng/e2e-mqttv5-adr-namespace.ps1 b/c/eng/e2e-mqttv5-adr-namespace.ps1
index 7ab930cec..f9f3265bc 100644
--- a/c/eng/e2e-mqttv5-adr-namespace.ps1
+++ b/c/eng/e2e-mqttv5-adr-namespace.ps1
@@ -30,7 +30,8 @@ param(
     [Parameter(Mandatory)][string]$NamespaceName,
     [string]$AdrApiVersion = '2026-11-02-preview',
     [string]$HubApiVersion = '2026-10-01-preview',
-    [string]$DpsApiVersion = '2026-11-02-preview',
+    # Empty: the first of 2026-11-02-preview / 2026-03-01-preview that ARM serves, else its newest.
+    [string]$DpsApiVersion = '',
     [int]$LinkTimeoutMinutes = 12,
     [int]$LinkAttempts = 4
 )
@@ -43,7 +44,6 @@ $HubId = "$Rg/providers/Microsoft.Devices/IotHubs/$HubName"
 $DpsId = "$Rg/providers/Microsoft.Devices/provisioningServices/$DpsName"
 $NsId = "$Rg/providers/Microsoft.DeviceRegistry/namespaces/$NamespaceName"
 $HubUrl = "$Arm${HubId}?api-version=$HubApiVersion"
-$DpsUrl = "$Arm${DpsId}?api-version=$DpsApiVersion"
 $NsUrl = "$Arm${NsId}?api-version=$AdrApiVersion"
 $RoleContributor = 'b24988ac-6180-42a0-ab88-20f7382dd24c'
 $RoleHubData = '4fc6c259-987e-4a07-842e-c321cc9d413f'
@@ -133,6 +133,14 @@ $ns = Invoke-Arm get $NsUrl
 if (Test-Linked $ns) { Write-Host "ADR namespace $NamespaceName present and linked."; return }
 Write-Host "::warning::ADR namespace $NamespaceName $(if ($ns) { 'not linked' } else { 'missing from ARM' }); rebuilding it."
 
+if (-not $DpsApiVersion) {
+    $served = @(((& az provider show --namespace Microsoft.Devices -o json 2>$null) | Out-String | ConvertFrom-Json).resourceTypes |
+        Where-Object resourceType -eq 'provisioningServices' | ForEach-Object apiVersions)
+    if (-not $served) { throw 'Could not read the Microsoft.Devices/provisioningServices api-versions.' }
+    $DpsApiVersion = @(@('2026-11-02-preview', '2026-03-01-preview') | Where-Object { $_ -in $served }) + @($served | Sort-Object -Descending) | Select-Object -First 1
+}
+Write-Host "  DPS api-version: $DpsApiVersion"
+$DpsUrl = "$Arm${DpsId}?api-version=$DpsApiVersion"
 $hub = Invoke-Arm get $HubUrl
 $dps = Invoke-Arm get $DpsUrl
 if (-not $hub -or -not $dps) { throw 'Hub or DPS not found.' }

From e13239c6d13a8d77220b033e6298f88059773dad Mon Sep 17 00:00:00 2001
From: Ewerton Scaboro da Silva <ewertons@microsoft.com>
Date: Sat, 10 Oct 2026 01:24:43 +0000
Subject: [PATCH 7/8] Wait out a Deleting namespace; accept Canceled as
 terminal

---
 .github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml |  8 ++++----
 c/eng/e2e-mqttv5-adr-namespace.ps1                  | 12 +++++++++++-
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
index 1963cb62f..b084049a8 100644
--- a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
+++ b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
@@ -45,10 +45,10 @@ jobs:
     concurrency:
       group: mqttv5-adr-namespace
       cancel-in-progress: false
-    # Above the script's worst case: namespace 10 + role grants 14 + 4 link
-    # attempts x 13 + 3 settle waits x 5 + reconcile 5 + DPS 5 = 101 min,
-    # plus ARM latency.
-    timeout-minutes: 120
+    # Above the script's worst case: delete wait 15 + namespace 10 + role
+    # grants 14 + 4 link attempts x 13 + 3 settle waits x 5 + reconcile 5 +
+    # DPS 5 = 116 min, plus ARM latency.
+    timeout-minutes: 135
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
         with:
diff --git a/c/eng/e2e-mqttv5-adr-namespace.ps1 b/c/eng/e2e-mqttv5-adr-namespace.ps1
index f9f3265bc..0f96d7d79 100644
--- a/c/eng/e2e-mqttv5-adr-namespace.ps1
+++ b/c/eng/e2e-mqttv5-adr-namespace.ps1
@@ -162,6 +162,14 @@ $linkBody = @{
     }
 }
 
+# A namespace being deleted is waited out, then created again.
+if ($ns -and $ns.properties.provisioningState -eq 'Deleting') {
+    [void](Wait-For 'namespace delete' 15 {
+            $n = Invoke-Arm get $NsUrl
+            @{ Done = (-not $n); Detail = if ($n) { $n.properties.provisioningState } else { 'absent' } } })
+    $ns = $null
+}
+
 # 1. Namespace. A bare create fails with EndpointUsedByNestedDevices when ADR still holds it; then re-assert its endpoints.
 if (-not $ns) {
     try { [void](Invoke-Arm put $NsUrl @{ location = $location; identity = @{ type = 'SystemAssigned' }; tags = $tags }) }
@@ -174,7 +182,9 @@ if (-not $ns) {
 $ns = (Wait-For 'namespace' 10 {
         $n = Invoke-Arm get $NsUrl
         $ps = if ($n) { $n.properties.provisioningState } else { 'absent' }
-        @{ Done = ($ps -in 'Succeeded', 'Failed' -and $n.identity.principalId); Detail = $ps; Ns = $n } }).Ns
+        # Any terminal state with an identity can go on: the link PUT below re-drives the namespace.
+        $terminal = $ps -in 'Succeeded', 'Failed', 'Canceled'
+        @{ Done = ($terminal -and $n.identity.principalId); Failed = ($terminal -and -not $n.identity.principalId); Detail = $ps; Ns = $n } }).Ns
 $nsPid = $ns.identity.principalId
 
 # 2. Role assignments (removed with the namespace; the namespace identity may be new).

From 54ccb0ecb0b1d0632a756f4de4a5bf8a120e19c1 Mon Sep 17 00:00:00 2001
From: Ewerton Scaboro da Silva <ewertons@microsoft.com>
Date: Sat, 10 Oct 2026 01:34:42 +0000
Subject: [PATCH 8/8] Allow overriding the ADR api-version via
 E2E_MQTTV5_SHARED_ADR_API_VERSION

---
 .github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml | 8 ++++++--
 c/docs/eng/end-to-end-tests.md                      | 3 ++-
 c/eng/e2e-mqttv5-adr-namespace.ps1                  | 2 ++
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
index b084049a8..b32f9ba79 100644
--- a/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
+++ b/.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml
@@ -12,7 +12,8 @@
 #
 #   vars:    E2E_MQTTV5_SHARED_SUBSCRIPTION_ID, E2E_MQTTV5_SHARED_RESOURCE_GROUP,
 #            E2E_MQTTV5_SHARED_HUB_NAME, E2E_MQTTV5_SHARED_DPS_NAME,
-#            E2E_MQTTV5_SHARED_ADR_NAMESPACE
+#            E2E_MQTTV5_SHARED_ADR_NAMESPACE,
+#            E2E_MQTTV5_SHARED_ADR_API_VERSION (optional; default 2026-11-02-preview)
 #   secrets: AZURE_CLIENT_ID, AZURE_TENANT_ID (OIDC). On the resource group the
 #            identity needs Contributor, and role assignment write for
 #            Contributor, IoT Hub Data Contributor and Azure Device Registry
@@ -70,7 +71,10 @@ jobs:
           HUB_NAME: ${{ vars.E2E_MQTTV5_SHARED_HUB_NAME }}
           DPS_NAME: ${{ vars.E2E_MQTTV5_SHARED_DPS_NAME }}
           ADR_NAMESPACE: ${{ vars.E2E_MQTTV5_SHARED_ADR_NAMESPACE }}
+          ADR_API_VERSION: ${{ vars.E2E_MQTTV5_SHARED_ADR_API_VERSION }}
         run: |
+          $extra = @{}
+          if ($env:ADR_API_VERSION) { $extra.AdrApiVersion = $env:ADR_API_VERSION }
           ./c/eng/e2e-mqttv5-adr-namespace.ps1 -SubscriptionId $env:SUBSCRIPTION_ID `
             -ResourceGroup $env:RESOURCE_GROUP -HubName $env:HUB_NAME `
-            -DpsName $env:DPS_NAME -NamespaceName $env:ADR_NAMESPACE
+            -DpsName $env:DPS_NAME -NamespaceName $env:ADR_NAMESPACE @extra
diff --git a/c/docs/eng/end-to-end-tests.md b/c/docs/eng/end-to-end-tests.md
index a986cb1a0..01311f467 100644
--- a/c/docs/eng/end-to-end-tests.md
+++ b/c/docs/eng/end-to-end-tests.md
@@ -243,7 +243,8 @@ can disappear from ARM, removing its role assignments; DPS registrations then fa
 (same name, hub region), re-grants the 8 role assignments, re-links hub and DPS, and pushes the
 namespace to DPS; when the namespace is healthy it makes one ARM read. Inputs are repository
 variables `E2E_MQTTV5_SHARED_SUBSCRIPTION_ID`, `_RESOURCE_GROUP`, `_HUB_NAME`, `_DPS_NAME`
-and `_ADR_NAMESPACE` (unset `_ADR_NAMESPACE` skips the job). The OIDC identity needs Contributor
+and `_ADR_NAMESPACE` (unset `_ADR_NAMESPACE` skips the job); optional `_ADR_API_VERSION`
+(default `2026-11-02-preview`) must match the version the namespace was linked with. The OIDC identity needs Contributor
 and role assignment write on the resource group.
 
 > **Software updates e2e** runs in its own workflow
diff --git a/c/eng/e2e-mqttv5-adr-namespace.ps1 b/c/eng/e2e-mqttv5-adr-namespace.ps1
index 0f96d7d79..16f4aa110 100644
--- a/c/eng/e2e-mqttv5-adr-namespace.ps1
+++ b/c/eng/e2e-mqttv5-adr-namespace.ps1
@@ -28,6 +28,8 @@ param(
     [Parameter(Mandatory)][string]$HubName,
     [Parameter(Mandatory)][string]$DpsName,
     [Parameter(Mandatory)][string]$NamespaceName,
+    # Use the version the namespace was linked with: the namespace principalId differs per
+    # api-version, so grants and link must share one. 2026-11-01-preview where 11-02 is not enabled.
     [string]$AdrApiVersion = '2026-11-02-preview',
     [string]$HubApiVersion = '2026-10-01-preview',
     # Empty: the first of 2026-11-02-preview / 2026-03-01-preview that ARM serves, else its newest.