Skip to content

Commit feedb22

Browse files
cartertinneycartertinney
authored
ci: move unit tests to GitHub Actions and optimize Azure Pipelines (#1246)
* Extract unittests * optimized azure pipelines workflows
1 parent 09af59e commit feedb22

9 files changed

Lines changed: 268 additions & 105 deletions

File tree

‎.github/dependabot.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
version: 2
3+
updates:
4+
- package-ecosystem: github-actions
5+
directory: /
6+
schedule:
7+
interval: weekly

‎.github/workflows/ci.yml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
---
2+
name: CI
3+
4+
"on":
5+
pull_request:
6+
push:
7+
branches:
8+
- main
9+
workflow_dispatch:
10+
11+
permissions:
12+
contents: read
13+
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.event.number || github.run_id }}
16+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
17+
18+
jobs:
19+
lint:
20+
name: Ruff
21+
runs-on: ubuntu-24.04
22+
steps:
23+
- name: Check out repository
24+
# v7.0.1
25+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
26+
with:
27+
persist-credentials: false
28+
- name: Set up Python
29+
# v7.0.0
30+
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
31+
with:
32+
python-version: 3.x
33+
- name: Set up uv
34+
# v10.0.1
35+
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d
36+
with:
37+
enable-cache: true
38+
cache-suffix: lint-upgrade
39+
- name: Sync lint environment
40+
run: uv sync --upgrade --no-dev --group lint
41+
- name: Run Ruff
42+
run: uv run --no-sync ruff check .
43+
44+
test:
45+
name: Python ${{ matrix.python-version }}
46+
runs-on: ubuntu-24.04
47+
strategy:
48+
fail-fast: false
49+
matrix:
50+
python-version:
51+
- "3.10"
52+
- "3.11"
53+
- "3.12"
54+
- "3.13"
55+
- "3.14"
56+
steps:
57+
- name: Check out repository
58+
# v7.0.1
59+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
60+
with:
61+
persist-credentials: false
62+
- name: Set up Python ${{ matrix.python-version }}
63+
# v7.0.0
64+
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
65+
with:
66+
python-version: ${{ matrix.python-version }}
67+
- name: Set up uv
68+
# v10.0.1
69+
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d
70+
with:
71+
enable-cache: true
72+
cache-suffix: test-${{ matrix.python-version }}-upgrade
73+
- name: Sync test environment
74+
run: >-
75+
uv sync --upgrade --python "${{ matrix.python-version }}"
76+
--no-dev --group test
77+
- name: Run unit tests
78+
run: >-
79+
uv run --no-sync pytest tests
80+
--junitxml=junit/azure-iot-device-test-results.xml
81+
--cov=azure
82+
--cov-report=xml:coverage.xml
83+
--cov-report=html:coverage
84+
--cov-append
85+
- name: Upload test results and coverage
86+
if: ${{ always() }}
87+
# v7.0.1
88+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
89+
with:
90+
name: test-results-python-${{ matrix.python-version }}
91+
path: |
92+
junit/
93+
coverage.xml
94+
coverage/
95+
if-no-files-found: error

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<h1> azure-iot-device </h1>
55
</div>
66

7-
![Build Status](https://azure-iot-sdks.visualstudio.com/azure-iot-sdks/_apis/build/status/Azure.azure-iot-sdk-python)
7+
[![Build Status](https://github.com/Azure/azure-iot-sdk-python/actions/workflows/ci.yml/badge.svg)](https://github.com/Azure/azure-iot-sdk-python/actions/workflows/ci.yml)
88

99
The Azure IoT Device SDK for Python enables Python developers to easily create IoT device solutions that seamlessly connect to the Azure IoT Hub ecosystem.
1010

‎vsts/build.yaml‎

Lines changed: 16 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ resources:
33

44
variables:
55
# CodeQL is auto-injected into every job by the 1ES policy decorator. Off by
6-
# default so the python test matrix (5 parallel jobs) does not each build and
7-
# upload a competing database -- S360 records arg_max(SnapshotDate) per
8-
# (repo, language). The dedicated 'CodeQL' job below opts back in.
6+
# default so other jobs do not build and upload competing databases -- S360
7+
# records arg_max(SnapshotDate) per (repo, language). The dedicated 'CodeQL'
8+
# job below opts back in.
99
# See https://aka.ms/codeql3000 and https://aka.ms/sdlfaqcodeql.
1010
Codeql.Enabled: false
1111

@@ -39,77 +39,18 @@ jobs:
3939
condition: always()
4040

4141
- job: 'Static_Analysis'
42+
displayName: 'Component Governance'
4243
pool:
4344
vmImage: 'ubuntu-24.04'
4445

4546
steps:
46-
- task: UsePythonVersion@0
47-
inputs:
48-
versionSpec: '3.x'
49-
architecture: 'x64'
5047
- task: ComponentGovernanceComponentDetection@0
5148
inputs:
5249
scanType: 'Register'
5350
verbosity: 'Verbose'
5451
alertWarningLevel: 'High'
55-
56-
- script: 'python -m pip install uv'
57-
displayName: 'Install uv'
58-
59-
- script: 'uv sync --upgrade --no-dev --group lint'
60-
displayName: 'Sync lint environment'
61-
62-
- script: 'uv run --no-sync ruff check .'
63-
displayName: 'Ruff check'
64-
65-
- job: 'Test'
66-
pool:
67-
vmImage: 'ubuntu-24.04'
68-
strategy:
69-
matrix:
70-
Python310:
71-
python.version: '3.10'
72-
Python311:
73-
python.version: '3.11'
74-
Python312:
75-
python.version: '3.12'
76-
Python313:
77-
python.version: '3.13'
78-
Python314:
79-
python.version: '3.14'
80-
steps:
81-
- task: UsePythonVersion@0
82-
displayName: 'Use Python $(python.version)'
83-
inputs:
84-
versionSpec: '$(python.version)'
85-
architecture: 'x64'
86-
87-
- script: 'python -m pip install uv'
88-
displayName: 'Install uv'
89-
90-
- script: 'uv sync --upgrade --python "$(python.version)" --no-dev --group test'
91-
displayName: 'Sync test environment'
92-
93-
- script: 'uv run --no-sync pytest tests --junitxml=junit/azure-iot-device-test-results.xml --cov=azure --cov-report=xml:coverage.xml --cov-report=html:coverage --cov-append'
94-
displayName: pytest
95-
96-
- task: PublishTestResults@2
97-
displayName: 'Publish Test Results **/*-test-results.xml'
98-
inputs:
99-
testResultsFiles: '**/*-test-results.xml'
100-
testRunTitle: 'Python $(python.version)'
101-
condition: always()
102-
103-
- task: PublishCodeCoverageResults@2
104-
inputs:
105-
summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml'
106-
failIfCoverageEmpty: true
10752

10853
- job: 'Publish'
109-
dependsOn:
110-
- 'Test'
111-
- 'Static_Analysis'
112-
11354
pool:
11455
vmImage: 'ubuntu-24.04'
11556

@@ -122,8 +63,20 @@ jobs:
12263
- script: 'python -m pip install uv'
12364
displayName: 'Install uv'
12465

66+
# Cache downloads without reusing the job environment.
67+
- task: Cache@2
68+
displayName: 'Cache uv'
69+
inputs:
70+
key: '"uv" | "$(Agent.OS)" | "build" | uv.lock'
71+
restoreKeys: |
72+
"uv" | "$(Agent.OS)" | "build"
73+
"uv" | "$(Agent.OS)"
74+
path: $(Pipeline.Workspace)/.uv-cache
75+
12576
- script: 'uv build'
12677
displayName: Build source and wheel distribution
78+
env:
79+
UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
12780

12881
- task: PublishBuildArtifacts@1
12982
displayName: 'Publish artifact dist'

‎vsts/dps-e2e.yaml‎

Lines changed: 35 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@ stages:
99
pool:
1010
vmImage: 'ubuntu-24.04'
1111
steps:
12+
- checkout: none
1213
- script: |
13-
sudo apt-get install -y uuid
14-
AZURE_RESOURCE_GROUP="iotpythondpse2e$(uuid | tr -d '-')"
15-
AZURE_IOTHUB_NAME="iotpythondpse2e$(uuid | tr -d '-')"
16-
AZURE_DPS_NAME="iotpythondpse2e$(uuid | tr -d '-')"
14+
RESOURCE_SUFFIX=$(tr -d '-' < /proc/sys/kernel/random/uuid)
15+
AZURE_RESOURCE_GROUP="iotpythondpse2e$RESOURCE_SUFFIX"
16+
AZURE_IOTHUB_NAME="iotpythondpse2e$RESOURCE_SUFFIX"
17+
AZURE_DPS_NAME="iotpythondpse2e$RESOURCE_SUFFIX"
1718
1819
# Export variables to other stages/jobs/steps.
1920
echo "##vso[task.setvariable variable=AZURE_RESOURCE_GROUP;isOutput=true]$AZURE_RESOURCE_GROUP"
@@ -178,8 +179,8 @@ stages:
178179
echo "##vso[task.setvariable variable=DPS_ROOT_CA_CERT_PATH;isOutput=true]$DPS_ROOT_CA_CERT_PATH"
179180
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_PATH;isOutput=true]$DPS_ROOT_CA_PK_PATH"
180181
echo "##vso[task.setvariable variable=DPS_ROOT_CA_CERT_BASE64;isOutput=true]$DPS_ROOT_CA_CERT_BASE64"
181-
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_BASE64;isOutput=true]$DPS_ROOT_CA_PK_BASE64"
182-
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PASSWORD;isOutput=true]$DPS_ROOT_CA_PASSWORD"
182+
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PK_BASE64;isOutput=true;isSecret=true]$DPS_ROOT_CA_PK_BASE64"
183+
echo "##vso[task.setvariable variable=DPS_ROOT_CA_PASSWORD;isOutput=true;isSecret=true]$DPS_ROOT_CA_PASSWORD"
183184
echo "##vso[task.setvariable variable=DPS_ROOT_CA_CONFIG_PATH;isOutput=true]$DPS_ROOT_CA_CONFIG_PATH"
184185
name: testCertificates
185186
displayName: Create Test Certificates
@@ -202,19 +203,22 @@ stages:
202203
# Create Resource Group
203204
az group create -g $AZURE_RESOURCE_GROUP -l $(AZURE_REGION)
204205
205-
# Create Azure IoT Hub
206-
time az iot hub create -g $AZURE_RESOURCE_GROUP -n $AZURE_IOTHUB_NAME --dla false --mintls 1.2 --partition-count 4 --sku S1
206+
# Create independent resources concurrently, then wait before linking them.
207+
time az iot hub create -g $AZURE_RESOURCE_GROUP -n $AZURE_IOTHUB_NAME --dla false --mintls 1.2 --partition-count 4 --sku S1 &
208+
iothub_create_pid=$!
209+
time az iot dps create -g $AZURE_RESOURCE_GROUP --name $AZURE_DPS_NAME -l $(AZURE_REGION) &
210+
dps_create_pid=$!
211+
212+
create_failed=0
213+
wait "$iothub_create_pid" || create_failed=1
214+
wait "$dps_create_pid" || create_failed=1
215+
if (( create_failed )); then
216+
exit 1
217+
fi
207218
208219
export IOTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --pn iothubowner --query "connectionString" --output tsv)
209220
export EVENTHUB_CONNECTION_STRING=$(az iot hub connection-string show -g $AZURE_RESOURCE_GROUP --hub-name $AZURE_IOTHUB_NAME --eh --query "connectionString" --output tsv)
210221
211-
for i in $(seq 1 11); do
212-
cg_name="cg$i"
213-
az iot hub consumer-group create -g "$AZURE_RESOURCE_GROUP" --hub-name "$AZURE_IOTHUB_NAME" --name "$cg_name"
214-
done
215-
216-
# Create Device Provisioning service
217-
time az iot dps create -g $AZURE_RESOURCE_GROUP --name $AZURE_DPS_NAME -l $(AZURE_REGION)
218222
time az iot dps linked-hub create -g $AZURE_RESOURCE_GROUP --dps-name $AZURE_DPS_NAME --connection-string "$IOTHUB_CONNECTION_STRING"
219223
220224
export DPS_CONNECTION_STRING=$(az iot dps connection-string show -g $AZURE_RESOURCE_GROUP --dps-name $AZURE_DPS_NAME --policy-name provisioningserviceowner --query "connectionString" --output tsv)
@@ -245,9 +249,9 @@ stages:
245249
time az iot dps certificate verify --dps-name $AZURE_DPS_NAME --resource-group $AZURE_RESOURCE_GROUP --name $DPS_ROOT_CA_CERT_NAME --path $verification_code_pem_path --etag $etag
246250
247251
# Export variables to other stages/jobs/steps.
248-
echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true]$IOTHUB_CONNECTION_STRING"
249-
echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true]$EVENTHUB_CONNECTION_STRING"
250-
echo "##vso[task.setvariable variable=DPS_CONNECTION_STRING;isOutput=true]$DPS_CONNECTION_STRING"
252+
echo "##vso[task.setvariable variable=IOTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$IOTHUB_CONNECTION_STRING"
253+
echo "##vso[task.setvariable variable=EVENTHUB_CONNECTION_STRING;isOutput=true;isSecret=true]$EVENTHUB_CONNECTION_STRING"
254+
echo "##vso[task.setvariable variable=DPS_CONNECTION_STRING;isOutput=true;isSecret=true]$DPS_CONNECTION_STRING"
251255
echo "##vso[task.setvariable variable=DPS_ID_SCOPE;isOutput=true]$DPS_ID_SCOPE"
252256
echo "##vso[task.setvariable variable=DPS_HOSTNAME;isOutput=true]$DPS_HOSTNAME"
253257
displayName: Create IoT Hub & DPS
@@ -273,11 +277,23 @@ stages:
273277
versionSpec: '3.10'
274278
architecture: 'x64'
275279

280+
# Cache downloads without reusing the job environment.
281+
- task: Cache@2
282+
displayName: 'Cache uv'
283+
inputs:
284+
key: '"uv" | "$(Agent.OS)" | "3.10" | uv.lock'
285+
restoreKeys: |
286+
"uv" | "$(Agent.OS)" | "3.10"
287+
"uv" | "$(Agent.OS)"
288+
path: $(Pipeline.Workspace)/.uv-cache
289+
276290
- script: 'python -m pip install uv'
277291
displayName: 'Install uv'
278292

279293
- script: 'uv sync --upgrade --python 3.10 --no-dev --group test --group e2e'
280294
displayName: 'Sync E2E environment'
295+
env:
296+
UV_CACHE_DIR: $(Pipeline.Workspace)/.uv-cache
281297

282298
- script: |
283299
cd $(Agent.WorkFolder)
@@ -322,6 +338,7 @@ stages:
322338
variables:
323339
AZURE_RESOURCE_GROUP: $[stageDependencies.setup.create_azure_resources.outputs['setResourceNames.AZURE_RESOURCE_GROUP']]
324340
steps:
341+
- checkout: none
325342
- task: AzureCLI@2
326343
inputs:
327344
azureSubscription: 'iot hub sdk service connection'

‎vsts/horton-e2e.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1+
trigger:
2+
branches:
3+
include:
4+
- main
5+
6+
pr:
7+
branches:
8+
include:
9+
- main
10+
111
variables:
212
Horton.FrameworkRoot: $(Agent.BuildDirectory)/e2e-fx
313
Horton.FrameworkRef: master

0 commit comments

Comments
 (0)