Repository navigation
datasets-online-checks #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: datasets-online-checks | |
| # Weekly: check the datasets against live sources (CVE Program, GitHub | |
| # Advisory Database, every cited URL, CISA KEV, FIRST EPSS) with | |
| # data_validation/qc_tools/online_check.py. Runs on a schedule, not on pull | |
| # requests, so an outage at any of those services never blocks a | |
| # contribution. On failure it opens (or comments on) one tracking issue. | |
| on: | |
| schedule: | |
| - cron: '23 5 * * 1' # 05:23 UTC every Monday | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| online: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Set up Python | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: '3.11' | |
| - name: Install requirements | |
| run: python -m pip install --quiet -r data_validation/requirements.txt | |
| - name: Online checks | |
| id: online | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| set -o pipefail | |
| python data_validation/qc_tools/online_check.py --summary online-report.md | tee online-findings.txt | |
| - name: Report (job summary) | |
| if: always() | |
| run: | | |
| { | |
| if [ -f online-report.md ]; then cat online-report.md; fi | |
| echo | |
| echo '## Findings' | |
| echo | |
| echo '```' | |
| cat online-findings.txt 2>/dev/null || true | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Open or update the tracking issue | |
| if: failure() && steps.online.outcome == 'failure' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| title="Online dataset checks are failing" | |
| { | |
| echo "The weekly online checks found errors (rejected CVEs, withdrawn advisories, dead links, or KEV mismatches)." | |
| echo | |
| echo '```' | |
| grep '^ERROR' online-findings.txt || true | |
| echo '```' | |
| echo | |
| echo "Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| } > body.md | |
| number=$(gh issue list --state open --search "in:title \"$title\"" --json number --jq '.[0].number // empty') | |
| if [ -n "$number" ]; then | |
| gh issue comment "$number" --body-file body.md | |
| else | |
| gh issue create --title "$title" --body-file body.md | |
| fi |