diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index f0364023f7..9b8add2ed5 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -18,11 +18,32 @@ env: # state_db tests run in their own workflow (sei-db-tests.yml); exclude # that subtree everywhere in this workflow to avoid double-running them. STATE_DB_PKG_PREFIX: github.com/sei-protocol/sei-chain/sei-db/state_db + # Number of race-detection shards. Matches `NUM_SPLIT` passed to + # `make test-group-N` so shards are locally reproducible. + NUM_SPLIT: 3 jobs: + shard-indexes: + name: Generate race shard indexes + runs-on: ubuntu-latest + outputs: + json: ${{ steps.generate-index-list.outputs.json }} + steps: + - id: generate-index-list + run: | + MAX_INDEX=$((${{ env.NUM_SPLIT }}-1)) + INDEX_LIST=$(seq 0 ${MAX_INDEX}) + INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) + echo "json=${INDEX_JSON}" >> "$GITHUB_OUTPUT" + test: - name: Race Detection + name: "Race Detection (shard ${{ matrix.shard }})" runs-on: uci-default + needs: shard-indexes + strategy: + fail-fast: false + matrix: + shard: ${{ fromJson(needs.shard-indexes.outputs.json) }} env: GOFLAGS: -race -tags=ledger,test_ledger_mock steps: @@ -52,9 +73,27 @@ jobs: - name: Go test run: | set -euo pipefail - PKGS=$(go list ./... | grep -v "^${STATE_DB_PKG_PREFIX}") - echo "$PKGS" | xargs go test \ - -timeout=${{ env.GO_TEST_TIMEOUT }} + make split-test-packages NUM_SPLIT=${{ env.NUM_SPLIT }} + SHARD_FILE=build/packages.txt.${{ matrix.shard }} + PARALLEL=() + if grep -qx "github.com/sei-protocol/sei-chain/occ_tests" "$SHARD_FILE"; then + echo "occ_tests present in this shard; forcing -parallel=1" + PARALLEL=(-parallel=1) + fi + xargs go test "${PARALLEL[@]}" -timeout=${{ env.GO_TEST_TIMEOUT }} < "$SHARD_FILE" + + test-check: + name: Race Detection + runs-on: ubuntu-latest + needs: test + if: always() + steps: + - name: Check shard results + run: | + if [[ "${{ needs.test.result }}" != "success" ]]; then + echo "One or more Race Detection shards failed" + exit 1 + fi coverage: name: Coverage diff --git a/AGENTS.md b/AGENTS.md index fa3426f844..01434ebba1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,12 +99,15 @@ make build # build the seid binary into ./build/seid make install # install seid into $GOBIN ``` -Tests run with the race detector and coverage. CI shards them into groups; while -iterating, run a single package directly: +Tests run with the race detector and coverage. `go-test.yml`'s Race Detection job +shards into `NUM_SPLIT` (currently 3) parallel matrix jobs, round-robin split +(package `i` goes to shard `i % NUM_SPLIT`, not a contiguous chunk — see the +`split-test-packages` Makefile target). `make test-group-N` reproduces a given +shard locally with the same package split: ```bash -make test-group-0 # one CI test shard (race + coverage) -go test .//... # run a single package +NUM_SPLIT=3 make test-group-0 # reproduce CI race shard 0 locally +go test .//... # run a single package ``` CI mirrors these checks: `.github/workflows/golangci.yml` runs golangci-lint diff --git a/Makefile b/Makefile index 82448449f0..255e4508a3 100644 --- a/Makefile +++ b/Makefile @@ -569,19 +569,60 @@ GO_TEST_FILES != find $(CURDIR) -name "*_test.go" # default to four splits by default NUM_SPLIT ?= 4 +# state_db tests run in their own workflow (sei-db-tests.yml); exclude that +# subtree here too so local shards match the CI shards exactly. +STATE_DB_PKG_PREFIX := github.com/sei-protocol/sei-chain/sei-db/state_db + $(BUILDDIR): mkdir -p $@ -# The format statement filters out all packages that don't have tests. -# Note we need to check for both in-package tests (.TestGoFiles) and -# out-of-package tests (.XTestGoFiles). +# Includes every package, not just ones with test files: `go test` on a +# package with no tests still compiles it (reported as "no test files"), +# which is how go-test.yml's Race Detection job also acts as a compile +# check under -race -tags=ledger,test_ledger_mock for the whole tree. +# Filtering to test-only packages here would silently drop that coverage. $(BUILDDIR)/packages.txt:$(GO_TEST_FILES) $(BUILDDIR) - go list -f "{{ if (or .TestGoFiles .XTestGoFiles) }}{{ .ImportPath }}{{ end }}" ./... | sort > $@ + go list ./... > $@.tmp + grep -v "^$(STATE_DB_PKG_PREFIX)" $@.tmp | sort > $@ + @rm -f $@.tmp TARGET_PACKAGE := github.com/sei-protocol/sei-chain/occ_tests +# Packages whose test suite alone regularly runs 1-4+ minutes under -race. +# Plain i%N round-robin assigns these by their position in the full, +# alphabetically-sorted package list, so several of them can land on the +# same shard by coincidence. Splitting them into their own round-robin +# pass, ahead of the rest of the list, guarantees consecutive heavy +# packages rotate across shards instead of clustering. Re-derive this list +# occasionally from a race job's `ok s` log lines. +HEAVY_TEST_PACKAGES := \ + github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable \ + github.com/sei-protocol/sei-chain/sei-cosmos/storev2/rootmulti \ + github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/test \ + github.com/sei-protocol/sei-chain/sei-ibc-go/modules/core/04-channel/keeper \ + github.com/sei-protocol/sei-chain/giga/tests \ + github.com/sei-protocol/sei-chain/sei-cosmos/x/staking/keeper \ + github.com/sei-protocol/sei-chain/evmrpc/tests \ + github.com/sei-protocol/sei-chain/sei-ibc-go/modules/core/03-connection/keeper \ + github.com/sei-protocol/sei-chain/sei-ibc-go/modules/apps/transfer/keeper \ + github.com/sei-protocol/sei-chain/sei-cosmos/x/bank/keeper + +# Round-robin split: package i goes to shard i%N. +# Interleaving avoids dumping a whole cluster of alphabetically-adjacent +# (and often runtime-correlated, e.g. a module's many keeper packages) +# packages into one shard, unlike a straight `split -d -n l/N` chunk split. +# Pre-touch all N files first so a shard with zero packages (NUM_SPLIT > +# package count) still gets an (empty) file instead of breaking test-group-%. +# HEAVY_TEST_PACKAGES is round-robined separately, and first, so its own +# i%N indexing can't collide with the coincidental clustering above. split-test-packages:$(BUILDDIR)/packages.txt - split -d -n l/$(NUM_SPLIT) $< $<. + @for i in $$(seq 0 $$(($(NUM_SPLIT)-1))); do : > $(BUILDDIR)/packages.txt.$$i; done + @printf '%s\n' $(HEAVY_TEST_PACKAGES) > $(BUILDDIR)/heavy-packages.txt + @grep -Fxf $(BUILDDIR)/heavy-packages.txt $< > $(BUILDDIR)/packages.txt.heavy || true + @grep -Fxvf $(BUILDDIR)/heavy-packages.txt $< > $(BUILDDIR)/packages.txt.rest || true + @awk -v n=$(NUM_SPLIT) -v dir=$(BUILDDIR) '{print >> (dir "/packages.txt." (NR-1)%n)}' $(BUILDDIR)/packages.txt.heavy + @awk -v n=$(NUM_SPLIT) -v dir=$(BUILDDIR) '{print >> (dir "/packages.txt." (NR-1)%n)}' $(BUILDDIR)/packages.txt.rest + @rm -f $(BUILDDIR)/heavy-packages.txt $(BUILDDIR)/packages.txt.heavy $(BUILDDIR)/packages.txt.rest test-group-%:split-test-packages @echo "🔍 Checking for special package: $(TARGET_PACKAGE)" @if grep -q "$(TARGET_PACKAGE)" $(BUILDDIR)/packages.txt.$*; then \