[C] e2e: mqttv5 custom topics, and a ci-c-e2e-mqttv5 workflow

.github/workflows/ci-c-e2e-mqttv5.yml

@@ -0,0 +1,126 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+# C e2e tests on an mqttv5 IoT Hub.
+#
+# Runs against the shared, long-lived mqttv5 environment (hub, DPS linked through
+# an ADR namespace); nothing is provisioned per run. Each job issues its own
+# device certificate from the DPS X.509 enrollment group's CA
+# (c/eng/e2e-shared-device.ps1 -Prefix E2E_MQTTV5_SHARED) and reads the events
+# endpoint through its own consumer group. Add a job per mqttv5 client.
+#
+#   vars:    E2E_MQTTV5_SHARED_ID_SCOPE (unset: the workflow skips),
+#            E2E_MQTTV5_SHARED_DPS_HOST, plus those of
+#            ci-c-e2e-mqttv5-adr-namespace.yml
+#   secrets: E2E_MQTTV5_SHARED_GROUP_CA, E2E_MQTTV5_SHARED_IOTHUB_CS,
+#            E2E_MQTTV5_SHARED_EVENTHUB_CS, E2E_MQTTV5_SHARED_SAS_GROUP_KEY,
+#            AZURE_CLIENT_ID, AZURE_TENANT_ID
+#
+# Pull requests from forks and from Dependabot get no secrets, so they skip.
+
+name: ci-c-e2e-mqttv5
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'c/src/core/**'
+      - 'c/src/mqttv5/**'
+      - 'c/inc/azure/iot/*.h'
+      - 'c/inc/azure/iot/mqttv5/**'
+      - 'c/adapters/paho/**'
+      - 'c/tests/e2e/**'
+      - 'c/eng/e2e-shared-device.ps1'
+      - 'c/eng/e2e-mqttv5-adr-namespace.ps1'
+      - '.github/workflows/ci-c-e2e-mqttv5.yml'
+      - '.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml'
+  schedule:
+    - cron: '30 11 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  # The namespace can drop out of Azure; DPS registrations fail until it is back.
+  adr-namespace:
+    name: ADR namespace
+    if: >-
+      ${{ vars.E2E_MQTTV5_SHARED_ID_SCOPE != '' &&
+          (github.event_name != 'pull_request' ||
+           (github.event.pull_request.head.repo.full_name == github.repository &&
+            github.actor != 'dependabot[bot]')) }}
+    permissions:
+      contents: read
+      id-token: write
+    # actionlint 1.7.12 does not accept the $/ self-repository form yet.
+    uses: ./.github/workflows/ci-c-e2e-mqttv5-adr-namespace.yml # zizmor: ignore[self-repository]
+    secrets:
+      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+
+  custom-topics:
+    name: custom topics / linux
+    needs: adr-namespace
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+
+      - name: Install ninja
+        run: sudo apt-get update && sudo apt-get install -y ninja-build
+
+      - name: Build
+        working-directory: c
+        run: |
+          cmake --preset linux-gcc-debug -DAZ_IOT_BUILD_E2E=ON -DAZ_IOT_BUILD_E2E_MQTTV5=ON -DAZ_IOT_WITH_PAHO=ON
+          cmake --build --preset linux-gcc-debug --target az_iot_tests_e2e_mqttv5_custom_topic
+
+      - name: Issue a device
+        shell: pwsh
+        env:
+          E2E_MQTTV5_SHARED_GROUP_CA: ${{ secrets.E2E_MQTTV5_SHARED_GROUP_CA }}
+          E2E_MQTTV5_SHARED_IOTHUB_CS: ${{ secrets.E2E_MQTTV5_SHARED_IOTHUB_CS }}
+          E2E_MQTTV5_SHARED_EVENTHUB_CS: ${{ secrets.E2E_MQTTV5_SHARED_EVENTHUB_CS }}
+          E2E_MQTTV5_SHARED_SAS_GROUP_KEY: ${{ secrets.E2E_MQTTV5_SHARED_SAS_GROUP_KEY }}
+          E2E_MQTTV5_SHARED_ID_SCOPE: ${{ vars.E2E_MQTTV5_SHARED_ID_SCOPE }}
+          E2E_MQTTV5_SHARED_DPS_HOST: ${{ vars.E2E_MQTTV5_SHARED_DPS_HOST }}
+          E2E_REGISTRATION_ID: ci-mqttv5-ct-${{ github.run_id }}-${{ github.run_attempt }}
+          E2E_RUN_ID: ${{ github.run_id }}
+        # Consumer groups e2e-0..e2e-9; Event Hubs allows 5 readers per partition per group.
+        run: |
+          $group = 'e2e-' + ([long]$env:E2E_RUN_ID % 10)
+          ./c/eng/e2e-shared-device.ps1 -Prefix E2E_MQTTV5_SHARED -RegistrationId $env:E2E_REGISTRATION_ID `
+            -ConsumerGroup $group -OutFile test_config/set_test_env_vars.ps1
+
+      - name: Run
+        shell: pwsh
+        env:
+          # The hub acknowledges custom topic publishes but does not deliver them to the
+          # events endpoint yet; the routed cases skip until it does. Remove when fixed.
+          AZ_IOT_E2E_SKIP_CUSTOM_TOPIC_ROUTING: '1'
+        run: |
+          . ./test_config/set_test_env_vars.ps1
+          $dir = Join-Path ([System.IO.Path]::GetTempPath()) ('az-iot-c-e2e-' + [Guid]::NewGuid().ToString('N'))
+          New-Item -ItemType Directory -Path $dir -Force | Out-Null
+          $cert = Join-Path $dir 'device-cert.pem'
+          $key = Join-Path $dir 'device-key.pem'
+          [System.IO.File]::WriteAllText($cert,
+            [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($env:IOT_DPS_INDIVIDUAL_X509_CERTIFICATE_0)))
+          [System.IO.File]::WriteAllText($key,
+            [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($env:IOT_DPS_INDIVIDUAL_X509_KEY_0)))
+
+          $env:AZ_IOT_DPS_ID_SCOPE = $env:IOT_DPS_ID_SCOPE
+          $env:AZ_IOT_DPS_REGISTRATION_ID = $env:IOT_DPS_INDIVIDUAL_REGISTRATION_ID_0
+          $env:AZ_IOT_CLIENT_CERT = $cert
+          $env:AZ_IOT_CLIENT_KEY = $key
+          $env:AZ_IOT_TRUSTED_CA = '/etc/ssl/certs/ca-certificates.crt'
+          if ($env:IOT_DPS_GLOBAL_ENDPOINT) { $env:AZ_IOT_DPS_GLOBAL_ENDPOINT = $env:IOT_DPS_GLOBAL_ENDPOINT }
+
+          ctest --test-dir c/build/linux-gcc-debug -R e2e_mqttv5_custom_topic `
+            --verbose --no-tests=error --timeout 900

c/cmake/az_iot_options.cmake

@@ -58,6 +58,9 @@ option(AZ_IOT_BUILD_CONFORMANCE_TESTS_TLS "Include the TLS certificate-validatio
 # only the dedicated ci-c-e2e-csr workflow provisions. Same rule as above: the
 # test is built when it is going to be run, not built-and-skipped.
 option(AZ_IOT_BUILD_E2E_CSR    "Build the CSR enrollment e2e test (needs a CA-linked DPS enrollment)" OFF)
+# The mqttv5 e2e suites need an mqttv5 hub configured for them (custom topic
+# templates), which only the ci-c-e2e-mqttv5 workflow has. Same rule as above.
+option(AZ_IOT_BUILD_E2E_MQTTV5 "Build the mqttv5 e2e tests (needs the mqttv5 e2e environment)" OFF)
 # The PKCS#11 custody tests drive a REAL token (SoftHSM2 in CI) through an
 # OpenSSL 3.x pkcs11 provider. Same rule as the conformance suites: the test is
 # built when it is going to be run, not built-and-skipped. The token URI comes

c/docs/eng/end-to-end-tests.md

@@ -125,6 +125,14 @@ shared; env `AZ_IOT_DPS_SAS_GROUP_KEY`, `AZ_IOT_DPS_SAS_REGISTRATION_ID`,
   with a CSR; the hub over the DPS-issued certificate, then telemetry. Also needs
   `AZ_IOT_BUILD_E2E_CSR` (group linked to the signing CA). `ci-c-e2e-csr`.
 
+**mqttv5 suites** (`AZ_IOT_BUILD_E2E_MQTTV5`; DPS assigns the device to an mqttv5 hub):
+
+- [`e2e_mqttv5_custom_topic_test.c`](../../tests/e2e/tests/e2e_mqttv5_custom_topic_test.c) — needs the
+  hub's custom topic templates `e2e/{deviceId}/#` and `e2e/shared/#`. Telemetry as a baseline;
+  QoS 1 and QoS 0 publishes to those templates reach the events endpoint; a topic no template
+  allows and another device's id are refused without disconnecting. `ci-c-e2e-mqttv5`.
+  `AZ_IOT_E2E_SKIP_CUSTOM_TOPIC_ROUTING` turns the routed checks into skips when nothing arrives.
+
 > The Windows reference transport keeps a single TLS connection at a time, so the
 > telemetry watcher is closed before the c2d/method/twin scenarios open theirs.
 > The device uses Paho's own independent TLS stack, so the two never collide.
@@ -249,6 +257,14 @@ on the resource group. Grant the link's role assignments (hub, DPS and namespace
 resource group once: those survive the namespace, so the job creates none and needs no role
 assignment write unless the namespace identity changes.
 
+[`ci-c-e2e-mqttv5.yml`](../../../.github/workflows/ci-c-e2e-mqttv5.yml) runs the mqttv5 suites on
+that environment, one job per client, after the namespace job. Each job issues a device with
+`e2e-shared-device.ps1 -Prefix E2E_MQTTV5_SHARED` from secret `E2E_MQTTV5_SHARED_GROUP_CA`, and
+reads secrets `E2E_MQTTV5_SHARED_IOTHUB_CS`, `_EVENTHUB_CS` (`service` policy) and
+`_SAS_GROUP_KEY`, and variables `E2E_MQTTV5_SHARED_ID_SCOPE` (unset skips the workflow) and
+`_DPS_HOST`. Triggers: pull requests touching mqttv5 or core device code, the e2e harness or the
+workflow; nightly; manual. Pull requests from forks and from Dependabot skip.
+
 > **Software updates e2e** runs in its own workflow
 > ([`ci-c-e2e-adu.yml`](../../../.github/workflows/ci-c-e2e-adu.yml), Linux, manual dispatch). See [Software updates e2e](#software-updates-e2e).
 

c/eng/e2e-shared-device.ps1

@@ -18,6 +18,9 @@ Default (ci-c-e2e), inputs from the environment (repository secrets/variables):
   E2E_SHARED_ID_SCOPE        DPS ID scope
   E2E_SHARED_SAS_GROUP_KEY   primary key of the DPS symmetric-key enrollment group
 
+-Prefix E2E_MQTTV5_SHARED (ci-c-e2e-mqttv5) reads the same five variables under that prefix, and
+optionally E2E_MQTTV5_SHARED_DPS_HOST, the DPS device endpoint.
+
 -Csr (ci-c-e2e-csr), writes the IOT_DPS_GROUP_X509_* bootstrap identity instead:
   E2E_CSR_SHARED_GROUP_CA    base64 of a PEM holding the group's issuing CA certificate and its
                              private key, then any CA certificates above it (issuer first)
@@ -30,31 +33,34 @@ param(
     [Parameter(Mandatory)][ValidatePattern('^[a-z0-9][a-z0-9-]{0,127}$')][string]$RegistrationId,
     [ValidateRange(0, 99)][int]$DeviceIndex = 0,
     [ValidatePattern('^[A-Za-z0-9$._-]{1,50}$')][string]$ConsumerGroup = '$Default',
+    # Variable prefix for the default (non -Csr) mode.
+    [ValidateSet('E2E_SHARED', 'E2E_MQTTV5_SHARED')][string]$Prefix = 'E2E_SHARED',
     [switch]$Csr,
     [string]$OutFile = 'test_config/set_test_env_vars.ps1'
 )
 
 $ErrorActionPreference = 'Stop'
 Set-StrictMode -Version Latest
 
-$Prefix = if ($Csr) { 'E2E_CSR_SHARED' } else { 'E2E_SHARED' }
+$Vars = if ($Csr) { 'E2E_CSR_SHARED' } else { $Prefix }
+function Get-Shared([string]$Name) { [Environment]::GetEnvironmentVariable("${Vars}_$Name") }
 $Required = if ($Csr) { @('GROUP_CA', 'ID_SCOPE', 'SAS_GROUP_KEY') } else { @('GROUP_CA', 'IOTHUB_CS', 'EVENTHUB_CS', 'ID_SCOPE', 'SAS_GROUP_KEY') }
-$Missing = $Required | ForEach-Object { "${Prefix}_$_" } |
+$Missing = $Required | ForEach-Object { "${Vars}_$_" } |
     Where-Object { [string]::IsNullOrWhiteSpace([Environment]::GetEnvironmentVariable($_)) }
 if ($Missing) { throw "Shared e2e environment not configured; missing: $($Missing -join ', ')." }
 
 if (-not $Csr) {
-    foreach ($Name in 'E2E_SHARED_IOTHUB_CS', 'E2E_SHARED_EVENTHUB_CS') {
+    foreach ($Name in "${Vars}_IOTHUB_CS", "${Vars}_EVENTHUB_CS") {
         if ([Environment]::GetEnvironmentVariable($Name) -match 'SharedAccessKeyName=iothubowner') {
             throw "$Name uses the iothubowner policy; use the 'service' policy."
         }
     }
 }
 
-$CaPem = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String([Environment]::GetEnvironmentVariable("${Prefix}_GROUP_CA")))
+$CaPem = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String([Environment]::GetEnvironmentVariable("${Vars}_GROUP_CA")))
 # The first certificate is the issuer; the key must match it.
 $Ca = [System.Security.Cryptography.X509Certificates.X509Certificate2]::CreateFromPem($CaPem, $CaPem)
-if (-not $Ca.HasPrivateKey) { throw "${Prefix}_GROUP_CA has no private key." }
+if (-not $Ca.HasPrivateKey) { throw "${Vars}_GROUP_CA has no private key." }
 $CaChain = [System.Security.Cryptography.X509Certificates.X509Certificate2Collection]::new()
 $CaChain.ImportFromPem($CaPem)
 
@@ -104,14 +110,15 @@ $Lines = if ($Csr) {
     )
 } else {
     @(
-        Format-Assignment 'IOTHUB_CONNECTION_STRING' $env:E2E_SHARED_IOTHUB_CS
-        Format-Assignment 'IOTHUB_EVENTHUB_CONNECTION_STRING' $env:E2E_SHARED_EVENTHUB_CS
+        Format-Assignment 'IOTHUB_CONNECTION_STRING' (Get-Shared 'IOTHUB_CS')
+        Format-Assignment 'IOTHUB_EVENTHUB_CONNECTION_STRING' (Get-Shared 'EVENTHUB_CS')
         Format-Assignment 'IOTHUB_EVENTHUB_CONSUMER_GROUP' $ConsumerGroup
-        Format-Assignment 'IOT_DPS_ID_SCOPE' $env:E2E_SHARED_ID_SCOPE
+        Format-Assignment 'IOT_DPS_ID_SCOPE' (Get-Shared 'ID_SCOPE')
+        if (Get-Shared 'DPS_HOST') { Format-Assignment 'IOT_DPS_GLOBAL_ENDPOINT' (Get-Shared 'DPS_HOST') }
         Format-Assignment "IOT_DPS_INDIVIDUAL_REGISTRATION_ID_$DeviceIndex" $RegistrationId
         Format-Assignment "IOT_DPS_INDIVIDUAL_X509_CERTIFICATE_$DeviceIndex" (ConvertTo-B64 $Chain)
         Format-Assignment "IOT_DPS_INDIVIDUAL_X509_KEY_$DeviceIndex" $KeyB64
-        Format-Assignment 'IOT_DPS_SYMM_KEY_GROUP_PRIMARY_KEY' $env:E2E_SHARED_SAS_GROUP_KEY
+        Format-Assignment 'IOT_DPS_SYMM_KEY_GROUP_PRIMARY_KEY' (Get-Shared 'SAS_GROUP_KEY')
     )
 }
 $OutDir = Split-Path -Parent $OutFile

c/tests/e2e/CMakeLists.txt

@@ -222,3 +222,17 @@ endif()
 
 
 endif() # AZ_IOT_WITH_PAHO
+
+# ---------------------------------------------------------------------------
+# az_iot_tests_e2e_mqttv5_custom_topic
+#
+# mqttv5 custom topics. Needs an mqttv5 hub with the custom topic templates
+# e2e/{deviceId}/# and e2e/shared/#, which only the ci-c-e2e-mqttv5 workflow's
+# environment has, so it is built only when AZ_IOT_BUILD_E2E_MQTTV5 is set.
+# ---------------------------------------------------------------------------
+if(AZ_IOT_BUILD_E2E_MQTTV5)
+    az_iot_add_cmocka_test(az_iot_tests_e2e_mqttv5_custom_topic
+        tests/e2e_mqttv5_custom_topic_test.c tests/e2e_device.c)
+    target_link_libraries(az_iot_tests_e2e_mqttv5_custom_topic PRIVATE
+        az_iot_e2e_service az_iot_mqttv5 az_iot_adapter_paho)
+endif()