Skip to content

Commit 623bfae

Browse files
barnabasbusabarnabasbusa
authored
Merge branch 'master' into bbusa/update-images
Signed-off-by: Barnabas Busa <barnabas.busa@ethereum.org>
2 parents 8e50bf1 + 105f599 commit 623bfae

19 files changed

Lines changed: 210 additions & 300 deletions

‎.github/workflows/_shared-prepare-docker-cache.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
- name: Cache Docker images.
2929
id: cache
3030
if: ${{ steps.check_cache.outputs.exists == 'false' }}
31-
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # 4.2.3
31+
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # 4.3.0
3232
with:
3333
path: ./docker-cache
3434
key: kurtosis-docker-${{ runner.os }}-${{ inputs.kurtosis_version }}

‎.github/workflows/_shared-run-test.yaml‎

Lines changed: 132 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ on:
3333
required: true
3434
clients:
3535
type: string
36-
description: "The clients config to use (default: clients/latest.yaml)."
37-
default: "clients/latest.yaml"
36+
description: "The clients config to use (default: clients/latest-dev.yaml)."
37+
default: "clients/latest-dev.yaml"
3838
kurtosis:
3939
type: string
4040
description: "The kurtosis network config to use (default: kurtosis-config/default.yaml)."
@@ -51,6 +51,10 @@ on:
5151
type: string
5252
description: "The list of assertoor test files to run."
5353
required: true
54+
assertoor_image:
55+
type: string
56+
description: "The assertoor image to use (default: ethpandaops/assertoor:master)."
57+
default: "ethpandaops/assertoor:master"
5458
send_notification:
5559
type: string
5660
description: "Send discord notification on test failure (default: false)."
@@ -85,6 +89,9 @@ on:
8589
CHATGPT_KEY:
8690
description: 'ChatGPT Key'
8791
required: false
92+
OPENROUTER_API_KEY:
93+
description: 'OpenRouter API Key'
94+
required: false
8895
S3_ACCESS_KEY:
8996
description: 'S3 Access Key'
9097
required: false
@@ -99,7 +106,7 @@ jobs:
99106
timeout-minutes: 1440
100107
steps:
101108
- name: Checkout Repository
102-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
109+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
103110

104111
- name: "Show Inputs"
105112
shell: bash
@@ -246,6 +253,7 @@ jobs:
246253
fi
247254
fi
248255
256+
echo " supernode: true" >> ./temp/participants.yaml
249257
echo " count: 1" >> ./temp/participants.yaml
250258
251259
echo "Client $client_idx"
@@ -288,6 +296,14 @@ jobs:
288296
done <<< $(echo "$tests_json" | jq -rc '.[]')
289297
echo "$kurtosis_cfg" > ./temp/test-network.yaml
290298
299+
# Override assertoor image if provided
300+
assertoor_image="${{ inputs.assertoor_image }}"
301+
if ! [ -z "$assertoor_image" ]; then
302+
echo "Overriding assertoor image to: $assertoor_image"
303+
kurtosis_cfg="$(cat ./temp/test-network.yaml | yq ".assertoor_params.image = \"$assertoor_image\"")"
304+
echo "$kurtosis_cfg" > ./temp/test-network.yaml
305+
fi
306+
291307
echo ""
292308
cat ./temp/test-network.yaml
293309
@@ -441,6 +457,98 @@ jobs:
441457
442458
env:
443459
CHATGPT_KEY: ${{ secrets.CHATGPT_KEY }}
460+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
461+
462+
- name: Check OpenRouter API key availability
463+
if: ${{ steps.test_result.outputs.result == 'failure' }}
464+
id: check_openrouter
465+
shell: bash
466+
run: |
467+
if [ ! -z "$OPENROUTER_API_KEY" ]; then
468+
echo "has_key=true" >> $GITHUB_OUTPUT
469+
else
470+
echo "has_key=false" >> $GITHUB_OUTPUT
471+
fi
472+
env:
473+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
474+
475+
- name: Generate failure summary via OpenRouter/Gemini (on failure)
476+
if: ${{ steps.test_result.outputs.result == 'failure' && steps.check_openrouter.outputs.has_key == 'true' }}
477+
id: openrouter
478+
shell: bash
479+
run: |
480+
test_status=$(
481+
cat <<"EOF"
482+
${{ steps.test_result.outputs.test_overview }}
483+
EOF
484+
)
485+
failed_test_status=$(
486+
cat <<"EOF"
487+
${{ steps.test_result.outputs.failed_test_details }}
488+
EOF
489+
)
490+
491+
touch ./temp/openrouter_query.txt
492+
echo "You will receive the test status & the last 200 lines of logs from the ethereum testnet testing tool assertoor." >> ./temp/openrouter_query.txt
493+
echo "The status & logs you get are from failed tests. Please find the error, which clients caused it and try summarizing what went wrong for a first technical insight." >> ./temp/openrouter_query.txt
494+
echo "Assume that assertoor and the test are working fine, so it is probably one or multiple clients causing problems." >> ./temp/openrouter_query.txt
495+
echo "Your summary should not exceed 500 characters. Use easy language with technical details like client names. Avoid listing block numbers or timestamps." >> ./temp/openrouter_query.txt
496+
echo "" >> ./temp/openrouter_query.txt
497+
498+
echo "Client Pairs: (Format: <execution layer client>-<consensus layer client>[-<validator client>], validator client may be left out)" >> ./temp/openrouter_query.txt
499+
cat ./temp/clients_summary.txt >> ./temp/openrouter_query.txt
500+
echo "" >> ./temp/openrouter_query.txt
501+
502+
echo "$test_status" >> ./temp/openrouter_query.txt
503+
echo "" >> ./temp/openrouter_query.txt
504+
echo "Failed Test Task Status:" >> ./temp/openrouter_query.txt
505+
echo "$failed_test_status" >> ./temp/openrouter_query.txt
506+
echo "" >> ./temp/openrouter_query.txt
507+
508+
tempdir="${{ steps.enclave_dump.outputs.dump_dir }}"
509+
tail -n 200 $tempdir/assertoor-*/assertoor--*/output.log >> ./temp/openrouter_query.txt
510+
511+
query_text="$(cat ./temp/openrouter_query.txt)"
512+
cp ./temp/openrouter_query.txt $tempdir/openrouter_query.txt
513+
514+
export QUERY_TEXT=$(echo "$query_text" | jq -R -s '.')
515+
query_json=$(
516+
cat <<"EOF" | envsubst | jq -c
517+
{
518+
"model": "anthropic/claude-sonnet-4.5",
519+
"messages": [
520+
{
521+
"role": "user",
522+
"content": $QUERY_TEXT
523+
}
524+
]
525+
}
526+
EOF
527+
)
528+
529+
result=$(curl https://openrouter.ai/api/v1/chat/completions \
530+
-H "Content-Type: application/json" \
531+
-H "Authorization: Bearer $OPENROUTER_API_KEY" \
532+
-H "HTTP-Referer: https://github.com/ethpandaops/assertoor-test" \
533+
-H "X-Title: Assertoor Test Runner" \
534+
-d "$query_json"
535+
)
536+
537+
echo "OpenRouter/Gemini Summary:"
538+
summary="$(echo $result | jq -r ".choices[0].message.content")"
539+
if [ "$summary" == "null" ]; then
540+
echo "OpenRouter/Gemini failed to generate a summary"
541+
echo $result | jq
542+
else
543+
echo "$summary" > ./temp/openrouter_summary.txt
544+
fi
545+
546+
echo "summary<<EOF" >> $GITHUB_OUTPUT
547+
echo $result | jq -r ".choices[0].message.content" >> $GITHUB_OUTPUT
548+
echo "EOF" >> $GITHUB_OUTPUT
549+
550+
env:
551+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
444552

445553
- name: Upload dump artifact
446554
if: ${{ steps.enclave_dump.outputs.dump_dir != '' }}
@@ -495,6 +603,17 @@ jobs:
495603
echo "$chatgpt_summary" >> $GITHUB_STEP_SUMMARY
496604
fi
497605
606+
openrouter_summary=$(
607+
cat <<"EOF"
608+
${{ steps.openrouter.outputs.summary }}
609+
EOF
610+
)
611+
if ! [ -z "$openrouter_summary" ]; then
612+
echo "" >> $GITHUB_STEP_SUMMARY
613+
echo "## Summary (OpenRouter/Gemini)" >> $GITHUB_STEP_SUMMARY
614+
echo "$openrouter_summary" >> $GITHUB_STEP_SUMMARY
615+
fi
616+
498617
echo "" >> $GITHUB_STEP_SUMMARY
499618
echo "See job logs and artifacts for more details about this failure." >> $GITHUB_STEP_SUMMARY
500619
@@ -545,6 +664,16 @@ jobs:
545664
echo "$chatgpt_summary" >> ./temp/notification_text.txt
546665
fi
547666
667+
openrouter_summary=$(
668+
cat <<"EOF"
669+
${{ steps.openrouter.outputs.summary }}
670+
EOF
671+
)
672+
if ! [ -z "$openrouter_summary" ]; then
673+
echo "**Summary: (OpenRouter/Gemini)**" >> ./temp/notification_text.txt
674+
echo "$openrouter_summary" >> ./temp/notification_text.txt
675+
fi
676+
548677
echo "Resolve job_id..."
549678
job_id=$(gh run view ${{ github.run_id }} --json jobs --jq '.jobs[] | select(.name=="${{ github.job }}") | .id')
550679

‎.github/workflows/_shared-run.yaml‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,14 @@ on:
3030
type: string
3131
description: "The S3 path to use for the cache."
3232
default: ""
33+
clients:
34+
type: string
35+
description: "Override client images to use."
36+
default: "clients/latest-dev.yaml"
37+
assertoor_image:
38+
type: string
39+
description: "Override assertoor image to use."
40+
default: "ethpandaops/assertoor:master"
3341
secrets:
3442
RANCHER_TOKEN:
3543
description: "The rancher token to login to the rancher server."
@@ -41,6 +49,9 @@ on:
4149
CHATGPT_KEY:
4250
description: 'ChatGPT Key'
4351
required: false
52+
OPENROUTER_API_KEY:
53+
description: 'OpenRouter API Key'
54+
required: false
4455
S3_ACCESS_KEY:
4556
description: 'S3 Access Key'
4657
required: false
@@ -182,18 +193,20 @@ jobs:
182193
backend: ${{ needs.get_test.outputs.backend }}
183194
kubeCluster: ${{ needs.get_test.outputs.kubernetes_cluster }}
184195
kubeStorageClass: ${{ needs.get_test.outputs.kubernetes_storageclass }}
185-
clients: ${{ needs.get_test.outputs.clients }}
186196
assertoor_tests: ${{ needs.get_test.outputs.assertoor_tests }}
187197
send_notification: ${{ needs.get_test.outputs.send_notification }}
188198
use_chatgpt: ${{ inputs.use_chatgpt }}
189199
s3_bucket: ${{ inputs.s3_bucket }}
190200
s3_endpoint: ${{ inputs.s3_endpoint }}
191201
s3_insecure: ${{ inputs.s3_insecure }}
192202
s3_path: ${{ inputs.s3_path }}
203+
clients: ${{ inputs.clients }}
204+
assertoor_image: ${{ inputs.assertoor_image }}
193205
secrets:
194206
KUBECONFIG: ${{ needs.get_test.outputs.kubeconfig }}
195207
DISCORD_HOOK: ${{ secrets.DISCORD_HOOK }}
196208
CHATGPT_KEY: ${{ secrets.CHATGPT_KEY }}
209+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
197210
S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
198211
S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
199212

@@ -204,7 +217,7 @@ jobs:
204217
needs: [get_test, run_test]
205218
steps:
206219
- name: Checkout Repository
207-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
220+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
208221
- name: Get summary artifacts
209222
id: "summary_ids"
210223
run: |
@@ -215,7 +228,7 @@ jobs:
215228
env:
216229
GH_TOKEN: ${{ github.token }}
217230
- name: Download summary artifacts
218-
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
231+
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
219232
with:
220233
artifact-ids: ${{ steps.summary_ids.outputs.artifact_ids }}
221234
path: "./summaries"

‎.github/workflows/run-manual.yml‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,18 @@ on:
1919
description: 'Override kurtosis ethereum package branch to run this test with (eg. main)'
2020
default: ""
2121
type: string
22+
overrideClientImages:
23+
description: 'Override client images to use (eg. clients/latest-stable)'
24+
default: "clients/latest-dev.yaml"
25+
type: string
26+
overrideAssertoorImage:
27+
description: 'Override assertoor image to use (eg. ethpandaops/assertoor:latest)'
28+
default: "ethpandaops/assertoor:master"
29+
type: string
30+
sendNotification:
31+
description: 'Send Discord notification on test failure'
32+
default: "false"
33+
type: string
2234

2335
jobs:
2436
get_tests:
@@ -29,7 +41,7 @@ jobs:
2941
kurtosis_versions: ${{ steps.tests.outputs.kurtosis_versions }}
3042
steps:
3143
- name: Checkout Repository
32-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
44+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
3345
- name: "Load test configurations from tests.yaml"
3446
id: tests
3547
shell: bash
@@ -59,15 +71,15 @@ jobs:
5971
fi
6072
6173
if ! [ -z "$override_pairs" ]; then
62-
test_configs="$(echo "$test_configs" | jq -c "(.[]).clientPairs |= [\"$override_pairs\"]")"
74+
test_configs="$(echo "$test_configs" | jq -c "map(.clientPairs = [\"$override_pairs\"])")"
6375
fi
6476
6577
if ! [ -z "$override_kurtosis_config" ]; then
66-
test_configs="$(echo "$test_configs" | jq -c "(.[]).kurtosis = \"$override_kurtosis_config\"")"
78+
test_configs="$(echo "$test_configs" | jq -c "map(.kurtosis = \"$override_kurtosis_config\")")"
6779
fi
6880
6981
if ! [ -z "$override_kurtosis_branch" ]; then
70-
test_configs="$(echo "$test_configs" | jq -c "(.[]).kurtosis_branch = \"$override_kurtosis_branch\"")"
82+
test_configs="$(echo "$test_configs" | jq -c "map(.kurtosis_branch = \"$override_kurtosis_branch\")")"
7183
fi
7284
7385
kurtosis_versions="$(echo "$test_configs" | jq -c "[.[] | select(.backend == \"docker\") | .kurtosis_version // \"latest\"] | unique")"
@@ -95,6 +107,7 @@ jobs:
95107
needs: get_tests
96108
name: "Warmup docker cache"
97109
runs-on: ubuntu-latest
110+
if: ${{ needs.get_tests.outputs.kurtosis_versions != '[]' }}
98111
strategy:
99112
fail-fast: false
100113
matrix:
@@ -116,6 +129,7 @@ jobs:
116129

117130
run_tests:
118131
needs: [get_tests, prepare_cache]
132+
if: ${{ !cancelled() && needs.get_tests.outputs.test_configs != '[]' }}
119133
uses: ./.github/workflows/_shared-run.yaml
120134
name: "${{ matrix.config.name }}"
121135
strategy:
@@ -124,13 +138,17 @@ jobs:
124138
config: ${{ fromJson(needs.get_tests.outputs.test_configs) }}
125139
with:
126140
config: ${{ toJSON(matrix.config) }}
141+
send_notification: ${{ inputs.sendNotification || 'false' }}
127142
use_chatgpt: '{"url": "${{ vars.CHATGPT_URL }}", "model": "${{ vars.CHATGPT_MODEL }}", "extra_cfg": ${{ vars.CHATGPT_EXTRA_CFG }}}'
128143
s3_bucket: ${{ vars.S3CACHE_BUCKET }}
129144
s3_endpoint: ${{ vars.S3CACHE_ENDPOINT }}
145+
clients: ${{ inputs.overrideClientImages || 'clients/latest-dev.yaml' }}
146+
assertoor_image: ${{ inputs.overrideAssertoorImage || 'ethpandaops/assertoor:master' }}
130147
secrets:
131148
RANCHER_URL: ${{ secrets.RANCHER_URL }}
132149
RANCHER_TOKEN: ${{ secrets.RANCHER_TOKEN }}
133150
DISCORD_HOOK: ${{ secrets.DISCORD_WEBHOOK }}
134151
CHATGPT_KEY: ${{ secrets.CHATGPT_KEY }}
152+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
135153
S3_ACCESS_KEY: ${{ secrets.S3CACHE_ACCESS_KEY }}
136154
S3_SECRET_KEY: ${{ secrets.S3CACHE_ACCESS_SECRET }}

‎.github/workflows/run-scheduled.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
kurtosis_versions: ${{ steps.tests.outputs.kurtosis_versions }}
1919
steps:
2020
- name: Checkout Repository
21-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
21+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
2222
- name: "Load test configurations from tests.yaml"
2323
id: tests
2424
shell: bash
@@ -51,6 +51,7 @@ jobs:
5151
needs: get_tests
5252
name: "Warmup docker cache"
5353
runs-on: ubuntu-latest
54+
if: ${{ needs.get_tests.outputs.kurtosis_versions != '[]' }}
5455
strategy:
5556
fail-fast: false
5657
matrix:
@@ -72,6 +73,7 @@ jobs:
7273

7374
run_tests:
7475
needs: [get_tests, prepare_cache]
76+
if: ${{ !cancelled() && needs.get_tests.outputs.test_configs != '[]' }}
7577
uses: ./.github/workflows/_shared-run.yaml
7678
name: "${{ matrix.config.name }}"
7779
strategy:
@@ -89,5 +91,6 @@ jobs:
8991
RANCHER_TOKEN: ${{ secrets.RANCHER_TOKEN }}
9092
DISCORD_HOOK: ${{ secrets.DISCORD_WEBHOOK }}
9193
CHATGPT_KEY: ${{ secrets.CHATGPT_KEY }}
94+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
9295
S3_ACCESS_KEY: ${{ secrets.S3CACHE_ACCESS_KEY }}
9396
S3_SECRET_KEY: ${{ secrets.S3CACHE_ACCESS_SECRET }}

0 commit comments

Comments
 (0)