Skip to content

Commit 1835cf1

Browse files
cartertinneycartertinney
committed
optimized azure pipelines workflows
1 parent f7f95a6 commit 1835cf1

6 files changed

Lines changed: 161 additions & 44 deletions

File tree

‎vsts/build.yaml‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,6 @@ jobs:
5151
alertWarningLevel: 'High'
5252

5353
- job: 'Publish'
54-
dependsOn:
55-
- 'Static_Analysis'
56-
5754
pool:
5855
vmImage: 'ubuntu-24.04'
5956

@@ -66,8 +63,20 @@ jobs:
6663
- script: 'python -m pip install uv'
6764
displayName: 'Install uv'
6865

66+
# Cache downloads without reusing the job environment.
67+
- task: Cache@2
68+
displayName: 'Cache uv'
69+
inputs:
70+
key: '"uv" | "$(Agent.OS)" | "build" | uv.lock'
71+
restoreKeys: |
72+
"uv" | "$(Agent.OS)" | "build"
73+
"uv" | "$(Agent.OS)"
74+
path: $(Pipeline.Workspace)/.uv-cache
75+
6976
- script: 'uv build'
7077
displayName: Build source and wheel distribution
78+
env:
79+
UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
7180

7281
- task: PublishBuildArtifacts@1
7382
displayName: 'Publish artifact dist'

‎vsts/dps-e2e.yaml‎

Lines changed: 35 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@ stages:
99
pool:
1010
vmImage: 'ubuntu-24.04'
1111
steps:
12+
- checkout: none
1213
- script: |
13-
sudo apt-get install -y uuid
14-
AZURE_RESOURCE_GROUP="iotpythondpse2e$(uuid | tr -d '-')"
15-
AZURE_IOTHUB_NAME="iotpythondpse2e$(uuid | tr -d '-')"
16-
AZURE_DPS_NAME="iotpythondpse2e$(uuid | tr -d '-')"
14+
RESOURCE_SUFFIX=$(tr -d '-' < /proc/sys/kernel/random/uuid)
15+
AZURE_RESOURCE_GROUP="iotpythondpse2e$RESOURCE_SUFFIX"
16+
AZURE_IOTHUB_NAME="iotpythondpse2e$RESOURCE_SUFFIX"
17+
AZURE_DPS_NAME="iotpythondpse2e$RESOURCE_SUFFIX"
1718
1819
# Export variables to other stages/jobs/steps.
1920
echo "##vso[task.setvariable variable=AZURE_RESOURCE_GROUP;isOutput=true]$AZURE_RESOURCE_GROUP"
@@ -178,8 +179,8 @@ stages:
178179
echo "##vso[task.setvariable variable=DPS_ROOT_CA_CERT_PATH;isOutput=true]$DPS_ROOT_CA_CERT_PATH"
179180
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_PATH;isOutput=true]$DPS_ROOT_CA_PK_PATH"
180181
echo "##vso[task.setvariable variable=DPS_ROOT_CA_CERT_BASE64;isOutput=true]$DPS_ROOT_CA_CERT_BASE64"
181-
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_BASE64;isOutput=true]$DPS_ROOT_CA_PK_BASE64"
182-
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PASSWORD;isOutput=true]$DPS_ROOT_CA_PASSWORD"
182+
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_BASE64;isOutput=true;isSecret=true]$DPS_ROOT_CA_PK_BASE64"
183+
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PASSWORD;isOutput=true;isSecret=true]$DPS_ROOT_CA_PASSWORD"
183184
echo "##vso[task.setvariable variable=DPS_ROOT_CA_CONFIG_PATH;isOutput=true]$DPS_ROOT_CA_CONFIG_PATH"
184185
name: testCertificates
185186
displayName: Create Test Certificates
@@ -202,19 +203,22 @@ stages:
202203
# Create Resource Group
203204
az group create -g $AZURE_RESOURCE_GROUP -l $(AZURE_REGION)
204205
205-
# Create Azure IoT Hub
206-
time az iot hub create -g $AZURE_RESOURCE_GROUP -n $AZURE_IOTHUB_NAME --dla false --mintls 1.2 --partition-count 4 --sku S1
206+
# Create independent resources concurrently, then wait before linking them.
207+
time az iot hub create -g $AZURE_RESOURCE_GROUP -n $AZURE_IOTHUB_NAME --dla false --mintls 1.2 --partition-count 4 --sku S1 &
208+
iothub_create_pid=$!
209+
time az iot dps create -g $AZURE_RESOURCE_GROUP --name $AZURE_DPS_NAME -l $(AZURE_REGION) &
210+
dps_create_pid=$!
211+
212+
create_failed=0
213+
wait "$iothub_create_pid" || create_failed=1
214+
wait "$dps_create_pid" || create_failed=1
215+
if (( create_failed )); then
216+
exit 1
217+
fi
207218
208219
export IOTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --pn iothubowner --query "connectionString" --output tsv)
209220
export EVENTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --eh --query "connectionString" --output tsv)
210221
211-
for i in $(seq 1 11); do
212-
cg_name="cg$i"
213-
az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name"
214-
done
215-
216-
# Create Device Provisioning service
217-
time az iot dps create -g $AZURE_RESOURCE_GROUP --name $AZURE_DPS_NAME -l $(AZURE_REGION)
218222
time az iot dps linked-hub create -g $AZURE_RESOURCE_GROUP --dps-name $AZURE_DPS_NAME --connection-string "$IOTHUB_CONNECTION_STRING"
219223
220224
export DPS_CONNECTION_STRING=$(az iot dps connection-string show -g $AZURE_RESOURCE_GROUP --dps-name $AZURE_DPS_NAME --policy-name provisioningserviceowner --query "connectionString" --output tsv)
@@ -245,9 +249,9 @@ stages:
245249
time az iot dps certificate verify --dps-name $AZURE_DPS_NAME --resource-group $AZURE_RESOURCE_GROUP --name $DPS_ROOT_CA_CERT_NAME --path $verification_code_pem_path --etag $etag
246250
247251
# Export variables to other stages/jobs/steps.
248-
echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true]$IOTHUB_CONNECTION_STRING"
249-
echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true]$EVENTHUB_CONNECTION_STRING"
250-
echo "##vso[task.setvariable variable=DPS_CONNECTION_STRING;isOutput=true]$DPS_CONNECTION_STRING"
252+
echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$IOTHUB_CONNECTION_STRING"
253+
echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$EVENTHUB_CONNECTION_STRING"
254+
echo "##vso[task.setvariable variable=DPS_CONNECTION_STRING;isOutput=true;isSecret=true]$DPS_CONNECTION_STRING"
251255
echo "##vso[task.setvariable variable=DPS_ID_SCOPE;isOutput=true]$DPS_ID_SCOPE"
252256
echo "##vso[task.setvariable variable=DPS_HOSTNAME;isOutput=true]$DPS_HOSTNAME"
253257
displayName: Create IoT Hub & DPS
@@ -273,11 +277,23 @@ stages:
273277
versionSpec: '3.10'
274278
architecture: 'x64'
275279

280+
# Cache downloads without reusing the job environment.
281+
- task: Cache@2
282+
displayName: 'Cache uv'
283+
inputs:
284+
key: '"uv" | "$(Agent.OS)" | "3.10" | uv.lock'
285+
restoreKeys: |
286+
"uv" | "$(Agent.OS)" | "3.10"
287+
"uv" | "$(Agent.OS)"
288+
path: $(Pipeline.Workspace)/.uv-cache
289+
276290
- script: 'python -m pip install uv'
277291
displayName: 'Install uv'
278292

279293
- script: 'uv sync --upgrade --python 3.10 --no-dev --group test --group e2e'
280294
displayName: 'Sync E2E environment'
295+
env:
296+
UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
281297

282298
- script: |
283299
cd $(Agent.WorkFolder)
@@ -322,6 +338,7 @@ stages:
322338
variables:
323339
AZURE_RESOURCE_GROUP: $[stageDependencies.setup.create_azure_resources.outputs['setResourceNames.AZURE_RESOURCE_GROUP']]
324340
steps:
341+
- checkout: none
325342
- task: AzureCLI@2
326343
inputs:
327344
azureSubscription: 'iot hub sdk service connection'

‎vsts/horton-e2e.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1+
trigger:
2+
branches:
3+
include:
4+
- main
5+
6+
pr:
7+
branches:
8+
include:
9+
- main
10+
111
variables:
212
Horton.FrameworkRoot: $(Agent.BuildDirectory)/e2e-fx
313
Horton.FrameworkRef: master

‎vsts/python-canary.yaml‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ jobs:
44
- job: 'Test'
55

66
strategy:
7-
maxParallel: 4
7+
# Each leg has an isolated consumer group, so all legs can run together.
8+
maxParallel: 7
89
matrix:
910
py310_windows_mqtt:
1011
pv: '3.10'
@@ -19,27 +20,27 @@ jobs:
1920
py310_linux_mqttws:
2021
pv: '3.10'
2122
transport: 'mqttws'
22-
imageName: 'Ubuntu 20.04'
23+
imageName: 'ubuntu-24.04'
2324
consumerGroup: 'cg4'
2425
py310_linux_mqtt:
2526
pv: '3.10'
2627
transport: 'mqtt'
27-
imageName: 'Ubuntu 20.04'
28+
imageName: 'ubuntu-24.04'
2829
consumerGroup: 'cg6'
2930
py311_linux_mqtt:
3031
pv: '3.11'
3132
transport: 'mqtt'
32-
imageName: 'Ubuntu 20.04'
33+
imageName: 'ubuntu-24.04'
3334
consumerGroup: 'cg7'
3435
py312_linux_mqtt:
3536
pv: '3.12'
3637
transport: 'mqtt'
37-
imageName: 'Ubuntu 20.04'
38+
imageName: 'ubuntu-24.04'
3839
consumerGroup: 'cg8'
3940
py312_linux_mqttws:
4041
pv: '3.12'
4142
transport: 'mqttws'
42-
imageName: 'Ubuntu 20.04'
43+
imageName: 'ubuntu-24.04'
4344
consumerGroup: 'cg9'
4445

4546
pool:
@@ -51,11 +52,23 @@ jobs:
5152
versionSpec: $(pv)
5253
architecture: 'x64'
5354

55+
# Cache downloads without reusing the job environment.
56+
- task: Cache@2
57+
displayName: 'Cache uv'
58+
inputs:
59+
key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock'
60+
restoreKeys: |
61+
"uv" | "$(Agent.OS)" | "$(pv)"
62+
"uv" | "$(Agent.OS)"
63+
path: $(Pipeline.Workspace)/.uv-cache
64+
5465
- script: 'python -m pip install uv'
5566
displayName: 'Install uv'
5667

5768
- script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e'
5869
displayName: 'Sync E2E environment'
70+
env:
71+
UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
5972

6073
- script: |
6174
cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e

‎vsts/python-e2e.yaml‎

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,11 @@ stages:
1515
pool:
1616
vmImage: 'ubuntu-24.04'
1717
steps:
18+
- checkout: none
1819
- script: |
19-
sudo apt-get install -y uuid
20-
AZURE_RESOURCE_GROUP="aziotsdkpythone2e$(uuid | tr -d '-')"
21-
AZURE_IOTHUB_NAME="aziotsdkpythone2e$(uuid | tr -d '-')"
20+
RESOURCE_SUFFIX=$(tr -d '-' < /proc/sys/kernel/random/uuid)
21+
AZURE_RESOURCE_GROUP="aziotsdkpythone2e$RESOURCE_SUFFIX"
22+
AZURE_IOTHUB_NAME="aziotsdkpythone2e$RESOURCE_SUFFIX"
2223
2324
# Export variables to other stages/jobs/steps.
2425
echo "##vso[task.setvariable variable=AZURE_RESOURCE_GROUP;isOutput=true]$AZURE_RESOURCE_GROUP"
@@ -43,10 +44,26 @@ stages:
4344
export IOTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --pn iothubowner --query "connectionString" --output tsv)
4445
export EVENTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --eh --query "connectionString" --output tsv)
4546
47+
# Limit Azure API concurrency while creating independent groups faster.
48+
wait_for_consumer_groups() {
49+
local failed=0
50+
for pid in "$@"; do
51+
wait "$pid" || failed=1
52+
done
53+
return "$failed"
54+
}
55+
56+
consumer_group_pids=()
4657
for i in $(seq 1 5); do
4758
cg_name="cg$i"
48-
az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name"
59+
az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name" &
60+
consumer_group_pids+=("$!")
61+
if (( ${#consumer_group_pids[@]} == 4 )); then
62+
wait_for_consumer_groups "${consumer_group_pids[@]}"
63+
consumer_group_pids=()
64+
fi
4965
done
66+
wait_for_consumer_groups "${consumer_group_pids[@]}"
5067
5168
# Route twin change events to the built-in Event Hubs-compatible endpoint
5269
# so reported-properties E2E tests can observe service-side patch arrivals.
@@ -63,8 +80,8 @@ stages:
6380
--route-name "twin-change-events"
6481
6582
# Export variables to other stages/jobs/steps.
66-
echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true]$IOTHUB_CONNECTION_STRING"
67-
echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true]$EVENTHUB_CONNECTION_STRING"
83+
echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$IOTHUB_CONNECTION_STRING"
84+
echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$EVENTHUB_CONNECTION_STRING"
6885
displayName: Create IoT Hub & DPS
6986
name: createAzureResources
7087
- stage: build_and_tests
@@ -87,11 +104,23 @@ stages:
87104
versionSpec: $(pv)
88105
architecture: 'x64'
89106

107+
# Cache downloads without reusing the job environment.
108+
- task: Cache@2
109+
displayName: 'Cache uv'
110+
inputs:
111+
key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock'
112+
restoreKeys: |
113+
"uv" | "$(Agent.OS)" | "$(pv)"
114+
"uv" | "$(Agent.OS)"
115+
path: $(Pipeline.Workspace)/.uv-cache
116+
90117
- script: 'python -m pip install uv'
91118
displayName: 'Install uv'
92119

93120
- script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e'
94121
displayName: 'Sync E2E environment'
122+
env:
123+
UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
95124

96125
- script: |
97126
cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e
@@ -126,11 +155,23 @@ stages:
126155
versionSpec: $(pv)
127156
architecture: 'x64'
128157

158+
# Cache downloads without reusing the job environment.
159+
- task: Cache@2
160+
displayName: 'Cache uv'
161+
inputs:
162+
key: '"uv" | "$(Agent.OS)" | "$(pv)" | uv.lock'
163+
restoreKeys: |
164+
"uv" | "$(Agent.OS)" | "$(pv)"
165+
"uv" | "$(Agent.OS)"
166+
path: $(Pipeline.Workspace)/.uv-cache
167+
129168
- script: 'python -m pip install uv'
130169
displayName: 'Install uv'
131170

132171
- script: 'uv sync --upgrade --python "$(pv)" --no-dev --group test --group e2e'
133172
displayName: 'Sync E2E environment'
173+
env:
174+
UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
134175

135176
- script: |
136177
cd $(Build.SourcesDirectory)/tests/e2e/iothub_e2e
@@ -156,6 +197,7 @@ stages:
156197
variables:
157198
AZURE_RESOURCE_GROUP: $[stageDependencies.setup.create_iot_hub_and_dps.outputs['setResourceNames.AZURE_RESOURCE_GROUP']]
158199
steps:
200+
- checkout: none
159201
- task: AzureCLI@2
160202
inputs:
161203
azureSubscription: 'iot hub sdk service connection'
@@ -164,8 +206,6 @@ stages:
164206
inlineScript: |
165207
# set -e
166208
167-
printenv | sort | grep -i azure
168-
169209
time az group delete --name $(AZURE_RESOURCE_GROUP) --yes
170210
displayName: Destroy Azure Resource Group
171211
condition: always() # Run step even if the pipeline run is cancelled.

0 commit comments

Comments
 (0)