diff --git a/.github/workflows/load-generator-image.yml b/.github/workflows/load-generator-image.yml new file mode 100644 index 0000000000..db35bc0fb1 --- /dev/null +++ b/.github/workflows/load-generator-image.yml @@ -0,0 +1,62 @@ +name: Load Generator Image + +on: + push: + branches: + - main + paths: + - integration_test/load_generator/** + - .github/workflows/load-generator-image.yml + pull_request: + branches: + - main + paths: + - integration_test/load_generator/** + - .github/workflows/load-generator-image.yml + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: integration_test/load_generator/package-lock.json + - name: Test load generator + working-directory: integration_test/load_generator + run: | + npm ci + npm run typecheck + npm test + npm run test:fixtures + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 + - name: Build load generator + uses: docker/build-push-action@v6 + with: + context: integration_test/load_generator + file: integration_test/load_generator/Dockerfile + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: ghcr.io/sei-protocol/sei-load-generator:sha-${{ github.sha }} + cache-from: type=gha,scope=sei-load-generator + cache-to: type=gha,mode=max,scope=sei-load-generator diff --git a/.gitignore b/.gitignore index b8223e9da9..f0deb83282 100644 --- a/.gitignore +++ b/.gitignore @@ -68,4 +68,4 @@ sei-db/state_db/bench/cryptosim/bin/ sei-db/state_db/bench/cryptosim/logs/ sei-db/ledger_db/block/blocksim/bin/ sei-db/seiwal/walsim/bin/ -sei-db/db_engine/litt/bin/ +sei-db/db_engine/litt/bin/ \ No newline at end of file diff --git a/integration_test/load_generator/.dockerignore b/integration_test/load_generator/.dockerignore new file mode 100644 index 0000000000..77b91ad6c0 --- /dev/null +++ b/integration_test/load_generator/.dockerignore @@ -0,0 +1,11 @@ +node_modules +artifacts +cache +dist +typechain-types +runtime +test +hardhat-test +observability +.env +*.log diff --git a/integration_test/load_generator/.env.example b/integration_test/load_generator/.env.example index eb49a496bd..e07f6c5e7b 100644 --- a/integration_test/load_generator/.env.example +++ b/integration_test/load_generator/.env.example @@ -17,23 +17,38 @@ TRACE_MAX_FRAMES=64 TRACE_TIMEOUT_MS=30000 TRACE_MAX_RETRIES=3 -# Target (arctic-1 or atlantic-2) -TARGET_NETWORK=arctic-1 +# Every target must set its name, chain IDs, and RPC URLs explicitly. +TARGET_NETWORK="" +TARGET_EVM_CHAIN_ID="" +TARGET_COSMOS_CHAIN_ID="" TARGET_EVM_RPC="" TARGET_COSMOS_RPC="" # Put the mnemonic only in your ignored .env file. Never commit it. TARGET_MNEMONIC="" -LOAD_USERS=runtime/replay-users/arctic-1-100.json -# Schema v4 adds bounded synthetic CREATE/CREATE2 replay. -LOAD_DEPLOYMENT=runtime/replay-deployments/arctic-1-v4.json +# One-time isolated runner account preparation. +# RUNNER_MNEMONIC_PATH=runtime/runner.mnemonic +# RUNNER_ACCOUNT_FUND_SEI=2000001000 +# LOAD_USERS=runtime/replay-users/arctic-1-100.json # Optional path override. +# Schema v5 adds token-operation fixtures. +LOAD_DEPLOYMENT=runtime/replay-deployments/arctic-1-v5.json # Safety: set to 1 only when deployment, funding, or replay mutations are intended. EXECUTE=0 USER_COUNT=100 +# Target balance per worker. Large exact decimal values are supported. FUND_SEI=100 TIME_SCALE=1 MAX_TPS=25 -WORKER_COUNT=20 +# Generated load derives workers as ceil(TXS_PER_SECOND * USERS_PER_TPS). +USERS_PER_TPS=2 +MAX_WORKER_COUNT=200 +# WORKER_COUNT=20 # Optional explicit override; replay defaults to 20. +# Reserve a fixed range so concurrent processes never share accounts. +USERS_PER_PARTITION=200 +# Each process uses a disjoint range: offset = PARTITION_INDEX * USERS_PER_PARTITION. +PARTITION_INDEX=0 +# WORKER_INDEX_OFFSET=0 # Optional explicit range override. +# USER_COUNT=2000 # One-time provision command size for a shared user pool. MAX_PENDING_PER_LANE=2 MAX_GAS_PER_TX=5000000 MAX_CALLDATA_BYTES=131072 @@ -49,6 +64,21 @@ FIXTURE_PREPARE_GAS_LIMIT=2000000 # Set only when intentionally replaying the corpus again from its first block. # REPLAY_FROM_START=1 +# Unified generated load. RUN_ID must be unique per executing process or pod. +LOAD_TYPE=defi +TXS_PER_SECOND=10 +# Per-process safety ceiling. Raise explicitly for intentional high-rate tests. +MAX_SYNTHETIC_TPS=100 +RUN_ID="" +LOAD_RUNTIME_DIR=runtime/load-runs/local +# Synthetic transaction audits rotate at this size and retain this many old files. +LOAD_AUDIT_MAX_BYTES=104857600 +LOAD_AUDIT_RETAIN_FILES=5 +# Optional comma-separated operation weights, for example: +# LOAD_MIX=swap_a_to_b:40,swap_b_to_a:40,lend_supply:20 +# CW1155_CONTRACT=sei1... +# SIMULATE_MODE=buffered + # Continuous mode and metrics BUFFER_START_MODE=latest INITIAL_BUFFER_BLOCKS=200 diff --git a/integration_test/load_generator/.gitignore b/integration_test/load_generator/.gitignore index c245465c6b..5c8ebf56f0 100644 --- a/integration_test/load_generator/.gitignore +++ b/integration_test/load_generator/.gitignore @@ -4,7 +4,10 @@ node_modules/ /cache/ typechain-types/ .env +# Runner mnemonics are written wherever RUNNER_MNEMONIC_PATH points, so this is unanchored. +*.mnemonic runtime/replay/ runtime/replay-deployments/ runtime/replay-users/ +runtime/load-runs/ *.log diff --git a/integration_test/load_generator/Dockerfile b/integration_test/load_generator/Dockerfile new file mode 100644 index 0000000000..970d039c24 --- /dev/null +++ b/integration_test/load_generator/Dockerfile @@ -0,0 +1,31 @@ +FROM node:22-bookworm-slim AS build + +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci + +COPY hardhat.config.ts tsconfig.json tsconfig.build.json ./ +COPY contracts ./contracts +COPY vendor ./vendor +COPY src ./src +RUN npm run build + +FROM node:22-bookworm-slim AS runtime + +ENV NODE_ENV=production \ + LOADGEN_COMPILED=1 \ + METRICS_HOST=0.0.0.0 \ + METRICS_PORT=9465 +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force +COPY --from=build --chown=node:node /app/dist ./dist +COPY --from=build --chown=node:node /app/artifacts ./artifacts +COPY --from=build --chown=node:node /app/vendor ./vendor + +USER node +EXPOSE 9465 +HEALTHCHECK --interval=15s --timeout=3s --start-period=30s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:9465/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" +ENTRYPOINT ["node", "dist/runLoad.js"] +CMD ["run"] diff --git a/integration_test/load_generator/README.md b/integration_test/load_generator/README.md index 329ecf59f9..67088c419c 100644 --- a/integration_test/load_generator/README.md +++ b/integration_test/load_generator/README.md @@ -1,6 +1,6 @@ -# Pacific replay load generator +# Sei multi-mode load generator -Standalone TypeScript package for capturing canonical Pacific-1 blocks and replaying equivalent, newly signed traffic on Arctic-1 or Atlantic-2. It never rebroadcasts Pacific signatures or assumes Pacific addresses and state exist on the target. +Standalone TypeScript package for generated DeFi, token, and chain-native load, plus capture and replay of canonical Pacific-1 traffic on configurable Sei networks. It never rebroadcasts Pacific signatures or assumes Pacific addresses and state exist on the target. ## Safety @@ -10,36 +10,171 @@ Standalone TypeScript package for capturing canonical Pacific-1 blocks and repla - Wrapped EVM transactions are correlated by reconstructing their signed hash from `MsgEVMTransaction`; Cosmos and EVM indexes are never assumed to align. - Privileged module traffic becomes labelled bank-shaped load by default (`PRIVILEGED_REPLAY_MODE=skip` omits it). - Successful unknown contract creations use `SyntheticCreationHarness` for bounded safe CREATE/CREATE2 load. Unknown traced calls use the allowlisted `CallGraphHarness`; untraced calls use `ProfileLoadHarness`. None executes source-selected target addresses or untrusted Pacific initcode. -- Mnemonics come only from `TARGET_MNEMONIC` or `SEI_ADMIN_MNEMONIC` and are never persisted. +- Mnemonics come only from `TARGET_MNEMONIC` or `SEI_ADMIN_MNEMONIC` and are not included in generated manifests. `load:prepare-account` writes only to the explicitly selected mode-`0600` mnemonic file. -## Setup +## Local quick start ```bash cd integration_test/load_generator npm install npm run compile +cp .env.example .env ``` -All configuration is parsed in `src/config.ts`. Copy the tracked template to the ignored `.env` file and set target endpoints and the mnemonic there: +Edit `.env` and set `TARGET_NETWORK`, both target RPC URLs, and a funded +`TARGET_MNEMONIC`. The mnemonic's account 0 pays for deployments and worker funding; +workers are derived from accounts 1 through `USER_COUNT`. ```bash -cp .env.example .env -$EDITOR .env +TARGET_NETWORK=arctic-1 +TARGET_EVM_CHAIN_ID=713715 +TARGET_COSMOS_CHAIN_ID=arctic-1 +TARGET_EVM_RPC=https://... +TARGET_COSMOS_RPC=https://... +TARGET_MNEMONIC="" +LOAD_DEPLOYMENT=runtime/replay-deployments/arctic-1-v5.json +``` + +Then deploy the shared fixtures and provision the workers. Both commands are idempotent: + +```bash +EXECUTE=1 npm run load:setup +EXECUTE=1 TXS_PER_SECOND=20 USERS_PER_TPS=2 FUND_SEI=1000 npm run load:provision +EXECUTE=1 RUN_ID=defi-local npm run load -- run --type defi --tps 20 --duration 600 ``` -Every command loads `.env` automatically. To use a different file, set `DOTENV_CONFIG_PATH`, for example `DOTENV_CONFIG_PATH=.env.arctic npm run replay:deploy`. `.env` is gitignored; never put a real mnemonic in `.env.example` or another tracked file. Explicit shell environment variables override values loaded from the file. +`--duration` is seconds; omit it to run until SIGINT/SIGTERM. `--tps` is the offered +rate for this process. If the target cannot keep up, excess operations are marked +`skipped` instead of building an unbounded queue. + +Every command loads `.env` automatically. Use another file with +`DOTENV_CONFIG_PATH=.env.arctic`. Explicit shell variables override `.env`. Never put a +real mnemonic in `.env.example` or another tracked file. -At minimum, confirm these values before an executed run: +Every target is explicit and supports any path-safe network name: ```bash -TARGET_NETWORK=arctic-1 -TARGET_MNEMONIC="" -EXECUTE=1 -RUN_DURATION_HOURS=2 -BUFFER_START_MODE=latest -LOAD_DEPLOYMENT=runtime/replay-deployments/arctic-1-v4.json +TARGET_NETWORK=devnet-7 +TARGET_EVM_CHAIN_ID=7007 +TARGET_COSMOS_CHAIN_ID=devnet-7 +TARGET_EVM_RPC=https://evm.devnet.example +TARGET_COSMOS_RPC=https://rpc.devnet.example +``` + +Chain IDs remain mandatory so the generator can refuse a misconfigured RPC before +submitting transactions. Network-specific values belong in the deployment repository +or an untracked local env file, not application code. + +## Load modes + +The unified entry point is `npm run load -- run --type --tps `. +`LOAD_TYPE`, `TXS_PER_SECOND`, and `RUN_DURATION_SECONDS` are equivalent environment +variables. Generated workloads reject rates above the per-process +`MAX_SYNTHETIC_TPS` safety ceiling (default `100`); raise both values explicitly for +intentional higher-rate tests. Worker count defaults to +`ceil(TXS_PER_SECOND * USERS_PER_TPS)` with `USERS_PER_TPS=2`; `WORKER_COUNT` is an +explicit override. Aggregate TPS across processes is the sum of their configured rates. + +Concurrent processes can share one pre-provisioned user pool without sharing accounts. +Each process reserves `USERS_PER_PARTITION` users beginning at +`PARTITION_INDEX * USERS_PER_PARTITION`, then activates the first `WORKER_COUNT` users in +that range; `WORKER_INDEX_OFFSET` overrides the calculated offset. Keeping the reserved +range fixed prevents rate changes from moving accounts between processes. Provision the +full pool separately, for example: + +```bash +EXECUTE=1 USER_COUNT=2000 FUND_SEI=1000 npm run load:provision +``` + +For a 2,000-user pool and `USERS_PER_PARTITION=200`, partition indexes 0–9 receive stable, +non-overlapping ranges. `WORKER_COUNT` may vary from 1 to 200 without changing ownership. +Scaling beyond the prepared pool or activating more than the reserved range fails before +load starts. + +- `defi`: bidirectional swaps, farming, lending/borrowing, liquid staking, and vault operations against the shared fixture state. +- `tokenops`: ERC20, ERC1155, and ERC721 mint/transfer traffic, including repeatable + cross-worker ERC721 round trips. Set `CW1155_CONTRACT` only for an existing CW1155 + contract whose worker token IDs are already funded. +- `nativetransfers`: native EVM transfers, Cosmos bank sends, and EVM bank-precompile + sends between workers. +- `simulate`: existing Pacific capture/replay. `SIMULATE_MODE=buffered` is the default; use `corpus` for an already captured finite corpus. + +Examples: + +```bash +EXECUTE=1 RUN_ID=defi-20 npm run load -- run --type defi --tps 20 +EXECUTE=1 RUN_ID=tokens-50 npm run load -- run --type tokenops --tps 50 +EXECUTE=1 RUN_ID=native-10 LOAD_MIX=cosmos_bank_send:1 \ + npm run load -- run --type nativetransfers --tps 10 --duration 3600 +``` + +Generated runs require a unique `RUN_ID` when `EXECUTE=1`. `LOAD_MIX` enables only the listed +operations and sets their relative weights, for example +`LOAD_MIX=swap_a_to_b:40,swap_b_to_a:40,lend_supply:20`. + +### Funding long runs + +`FUND_SEI` is the target balance for each worker, not the total funding budget. +Provisioning tops up existing users and accepts large exact decimal values without +JavaScript number rounding: + +```bash +EXECUTE=1 TXS_PER_SECOND=20 USERS_PER_TPS=2 FUND_SEI=1000000 npm run load:provision +``` + +This example derives 40 workers. Account 0 must hold at least +`workerCount * FUND_SEI` plus deployment, association, and funding fees. Re-run +`load:provision` with the same TPS and worker settings to top workers up before another +run. Size the target from measured cost: + +```text +SEI per worker ~= duration_seconds * tps * average_fee_SEI / worker_count ``` +Add headroom for uneven operation weights and fee spikes. Synthetic audit files rotate +at 100 MiB and retain five old files by default; tune `LOAD_AUDIT_MAX_BYTES` and +`LOAD_AUDIT_RETAIN_FILES` for longer runs. + +## Docker + +Build one image for fixture deployment, pool provisioning, and runners: + +```bash +docker build -t sei-load-generator:local . +docker run --rm --env-file .env -v "$PWD/runtime:/runtime" \ + -e EXECUTE=1 \ + -e LOAD_DEPLOYMENT=/runtime/deployment.json \ + sei-load-generator:local setup +docker run --rm --env-file .env -v "$PWD/runtime:/runtime" \ + -e EXECUTE=1 -e LOAD_USERS=/runtime/users.json \ + sei-load-generator:local provision +docker run --rm --env-file .env -p 9465:9465 \ + -e EXECUTE=1 -e RUN_ID=defi-docker \ + -e LOAD_DEPLOYMENT=/runtime/deployment.json -e LOAD_USERS=/runtime/users.json \ + -v "$PWD/runtime:/runtime" \ + sei-load-generator:local run --type defi --tps 10 +``` + +For isolated runner stacks, prepare account 0 from a separate treasury before using +that runner mnemonic for fixture deployment and user provisioning: + +```bash +TARGET_MNEMONIC="$TREASURY_MNEMONIC" \ +RUNNER_MNEMONIC_PATH=runtime/runner.mnemonic \ +RUNNER_ACCOUNT_FUND_SEI=2000001000 \ +EXECUTE=1 npm run load:prepare-account +``` + +The command creates the mnemonic file with mode `0600`, tops the account up to the +requested balance, and verifies its EVM association. It has no Kubernetes or secret +store dependency. + +Kubernetes manifests, SOPS Secrets, fixture and user-pool ConfigMaps, replica counts, +cluster RPC endpoints, resource limits, and PodMonitor configuration are owned by the +`sei-protocol/platform` repository. Platform maps StatefulSet ordinals to the generic +partition variables above. This package intentionally contains no Helm chart or direct +cluster orchestration. + ## Single-command run After configuring `.env`, the recommended one-command continuous run is: @@ -48,7 +183,7 @@ After configuring `.env`, the recommended one-command continuous run is: npm run replay:start:buffered ``` -This command compiles contracts, verifies or deploys schema-v4 fixtures, idempotently associates and funds users, captures the latest safe Pacific window, starts the replay follower, continuously appends new blocks, emits metrics/audits, and cleans consumed segment files. It runs for `RUN_DURATION_HOURS` from `.env` and mutates the target only when `EXECUTE=1`. +This command compiles contracts, verifies or deploys schema-v5 fixtures, idempotently associates and funds users, captures the latest safe Pacific window, starts the replay follower, continuously appends new blocks, emits metrics/audits, and cleans consumed segment files. It runs for `RUN_DURATION_HOURS` from `.env` and mutates the target only when `EXECUTE=1`. For a finite capture followed by one bounded replay instead: @@ -56,7 +191,7 @@ For a finite capture followed by one bounded replay instead: npm run replay:start ``` -A valid schema-v4 deployment is reused after its chain ID, bytecode hashes, SushiSwap provenance, creation harness, and protocol wiring are checked. Only `FORCE_DEPLOY=1` replaces it. Existing users are only topped up to `FUND_SEI`. Buffered mode starts from a fresh latest window by default; use `BUFFER_START_MODE=resume` only when intentionally continuing a live, non-pruned corpus. +A valid schema-v5 deployment is reused after its chain ID, bytecode hashes, SushiSwap provenance, creation harness, and protocol wiring are checked. Only `FORCE_DEPLOY=1` replaces it. Existing users are only topped up to `FUND_SEI`. Buffered mode starts from a fresh latest window by default; use `BUFFER_START_MODE=resume` only when intentionally continuing a live, non-pruned corpus. ## Capture and validate @@ -89,7 +224,7 @@ The package deploys canonical production SushiSwap V2 factory/router bytecode, W TARGET_NETWORK=arctic-1 EXECUTE=1 npm run replay:deploy ``` -Existing schema-v4 manifests are verified and reused. Set `FORCE_DEPLOY=1` to replace one. Defaults are `runtime/replay-deployments/-v4.json`; older manifests are intentionally not reused. Canonical source, artifact checksums, compiler settings, and deployment provenance are recorded in `vendor/sushiswap-v2/PROVENANCE.json`. +Existing schema-v5 manifests are verified and reused. Set `FORCE_DEPLOY=1` to replace one. Defaults are `runtime/replay-deployments/-v5.json`; older manifests are intentionally not reused. Canonical source, artifact checksums, compiler settings, and deployment provenance are recorded in `vendor/sushiswap-v2/PROVENANCE.json`. The vendored SushiSwap Solidity source and executable artifacts are GPL-3.0-covered third-party material. See `THIRD_PARTY_NOTICES.md` and the retained `contracts/uniswapv2/LICENSE`; the surrounding deterministic fixture implementations remain separate. @@ -159,7 +294,7 @@ The production-shaped fixtures reproduce corresponding protocol operations and n ## Buffered continuous mode -If schema-v4 fixtures and funded users already exist, start replay without deployment or provisioning: +If schema-v5 fixtures and funded users already exist, start replay without deployment or provisioning: ```bash TARGET_NETWORK=arctic-1 RUN_DURATION_HOURS=2 EXECUTE=1 npm run replay:buffered @@ -191,7 +326,13 @@ Executed replay exposes Prometheus metrics on `127.0.0.1:9465/metrics` and `/hea GRAFANA_ADMIN_PASSWORD='' npm run dashboard:up ``` -Grafana is at `http://localhost:3000/d/pacific-replay`; Prometheus is at `http://localhost:9090`. Both ports bind to localhost. Prometheus scrapes the host runner at `host.docker.internal:9465`, so keep `METRICS_HOST=0.0.0.0` and `METRICS_PORT=9465` when using the bundled stack. The dashboard shows scrape/process health, throughput, successful and failed inclusion, latency, pending work, adapter/fidelity mix, source-vs-target bytes and gas, calldata-size fidelity, trace availability and operation pressure, skips, and source cursors. The Grafana user defaults to `admin`; `GRAFANA_ADMIN_PASSWORD` is required. Stop the stack with `npm run dashboard:down`. +The local replay dashboard is at `http://localhost:3000/d/pacific-replay`; Prometheus is +at `http://localhost:9090`. Both ports bind to localhost. Prometheus scrapes the host +runner at `host.docker.internal:9465`, so keep `METRICS_HOST=0.0.0.0` and +`METRICS_PORT=9465` when using the bundled stack. The Grafana user is `admin`; +`GRAFANA_ADMIN_PASSWORD` is required. Production generated-load dashboards and aggregate +reports are owned by `sei-protocol/platform`. Stop the local stack with +`npm run dashboard:down`. Each executed run writes: @@ -201,6 +342,20 @@ Each executed run writes: Override paths with `BUCKET_AUDIT_PATH`, `UNBUCKETED_AUDIT_PATH`, and `REPLAY_REPORT`. +Each executed `defi`, `tokenops`, or `nativetransfers` run writes into +`runtime/load-runs//`, or `LOAD_RUNTIME_DIR` when set: + +- `run.json`: the configuration the process started with, including its worker partition. +- `transactions.jsonl`: one line per transaction with operation, lane, outcome, hash, and error. +- `summary.json`: the aggregate report, also printed to stdout when the run ends. + +The summary covers one process. It reports offered, submitted, and included counts, the +success rate over transactions that reached the chain, achieved against target TPS, and +p50/p90/p95/p99 inclusion latency interpolated from the metric histogram, so its resolution is +bounded by the histogram bucket edges. `stopReason` distinguishes a run that finished from one +cut short by a signal or an error. To aggregate across a multi-pod deployment, use the +Prometheus report in `sei-protocol/platform`. + ## Verification ```bash diff --git a/integration_test/load_generator/contracts/TestERC1155.sol b/integration_test/load_generator/contracts/TestERC1155.sol new file mode 100644 index 0000000000..f74b1af018 --- /dev/null +++ b/integration_test/load_generator/contracts/TestERC1155.sol @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.28; + +contract TestERC1155 { + mapping(uint256 => mapping(address => uint256)) public balanceOf; + mapping(address => mapping(address => bool)) public isApprovedForAll; + + event TransferSingle( + address indexed operator, + address indexed from, + address indexed to, + uint256 id, + uint256 value + ); + event TransferBatch( + address indexed operator, + address indexed from, + address indexed to, + uint256[] ids, + uint256[] values + ); + event ApprovalForAll(address indexed account, address indexed operator, bool approved); + + function mint(address to, uint256 id, uint256 amount) external { + balanceOf[id][to] += amount; + emit TransferSingle(msg.sender, address(0), to, id, amount); + } + + function mintBatch(address to, uint256[] calldata ids, uint256[] calldata amounts) external { + require(ids.length == amounts.length, "length mismatch"); + for (uint256 i; i < ids.length; i++) { + balanceOf[ids[i]][to] += amounts[i]; + } + emit TransferBatch(msg.sender, address(0), to, ids, amounts); + } + + function setApprovalForAll(address operator, bool approved) external { + isApprovedForAll[msg.sender][operator] = approved; + emit ApprovalForAll(msg.sender, operator, approved); + } + + function safeTransferFrom( + address from, + address to, + uint256 id, + uint256 amount, + bytes calldata + ) external { + require(msg.sender == from || isApprovedForAll[from][msg.sender], "not approved"); + require(balanceOf[id][from] >= amount, "insufficient balance"); + balanceOf[id][from] -= amount; + balanceOf[id][to] += amount; + emit TransferSingle(msg.sender, from, to, id, amount); + } + + function safeBatchTransferFrom( + address from, + address to, + uint256[] calldata ids, + uint256[] calldata amounts, + bytes calldata + ) external { + require(msg.sender == from || isApprovedForAll[from][msg.sender], "not approved"); + require(ids.length == amounts.length, "length mismatch"); + for (uint256 i; i < ids.length; i++) { + require(balanceOf[ids[i]][from] >= amounts[i], "insufficient balance"); + balanceOf[ids[i]][from] -= amounts[i]; + balanceOf[ids[i]][to] += amounts[i]; + } + emit TransferBatch(msg.sender, from, to, ids, amounts); + } +} diff --git a/integration_test/load_generator/contracts/TestERC20.sol b/integration_test/load_generator/contracts/TestERC20.sol index 9714dea717..32a6da7db5 100644 --- a/integration_test/load_generator/contracts/TestERC20.sol +++ b/integration_test/load_generator/contracts/TestERC20.sol @@ -54,6 +54,15 @@ contract TestERC20 { emit Transfer(address(0), to, value); } + function burn(uint256 value) external { + require(balanceOf[msg.sender] >= value, "ERC20: insufficient balance"); + unchecked { + balanceOf[msg.sender] -= value; + totalSupply -= value; + } + emit Transfer(msg.sender, address(0), value); + } + function _transfer(address from, address to, uint256 value) internal { require(balanceOf[from] >= value, "ERC20: insufficient balance"); unchecked { diff --git a/integration_test/load_generator/contracts/TestNFT.sol b/integration_test/load_generator/contracts/TestNFT.sol index eda521644e..a738f56e4f 100644 --- a/integration_test/load_generator/contracts/TestNFT.sol +++ b/integration_test/load_generator/contracts/TestNFT.sol @@ -7,6 +7,7 @@ contract TestNFT { address public immutable owner; mapping(uint256 => address) public ownerOf; mapping(address => uint256) public balanceOf; + mapping(uint256 => address) public getApproved; event Transfer(address indexed from, address indexed to, uint256 indexed tokenId); @@ -21,4 +22,32 @@ contract TestNFT { balanceOf[to]++; emit Transfer(address(0), to, tokenId); } + + function approve(address spender, uint256 tokenId) external { + require(ownerOf[tokenId] == msg.sender, "not owner"); + getApproved[tokenId] = spender; + } + + function transferFrom(address from, address to, uint256 tokenId) external { + require(ownerOf[tokenId] == from, "wrong owner"); + require(msg.sender == from || getApproved[tokenId] == msg.sender, "not approved"); + _transfer(from, to, tokenId); + } + + function roundTripTransfer(address to, uint256 tokenId) external { + address from = ownerOf[tokenId]; + require(from == msg.sender, "not owner"); + require(to != from, "same recipient"); + _transfer(from, to, tokenId); + _transfer(to, from, tokenId); + } + + function _transfer(address from, address to, uint256 tokenId) private { + require(to != address(0), "zero recipient"); + delete getApproved[tokenId]; + ownerOf[tokenId] = to; + balanceOf[from]--; + balanceOf[to]++; + emit Transfer(from, to, tokenId); + } } diff --git a/integration_test/load_generator/hardhat-test/tokenOps.spec.ts b/integration_test/load_generator/hardhat-test/tokenOps.spec.ts new file mode 100644 index 0000000000..9a2cddab16 --- /dev/null +++ b/integration_test/load_generator/hardhat-test/tokenOps.spec.ts @@ -0,0 +1,40 @@ +import { expect } from 'chai'; +import { ethers } from 'hardhat'; + +describe('token operation fixtures', () => { + it('supports repeated ERC20 and ERC721 state changes', async () => { + const [owner, recipient] = await ethers.getSigners(); + const token = await ethers.deployContract('TestERC20', [owner.address]); + await token.mint(owner.address, 100); + await token.transfer(recipient.address, 25); + await token.burn(10); + expect(await token.balanceOf(owner.address)).to.equal(65); + expect(await token.balanceOf(recipient.address)).to.equal(25); + + const nft = await ethers.deployContract('TestNFT', [owner.address]); + await nft.safeMint(owner.address, 1); + await nft.transferFrom(owner.address, recipient.address, 1); + expect(await nft.ownerOf(1)).to.equal(recipient.address); + + await nft.safeMint(owner.address, 2); + await expect(nft.roundTripTransfer(recipient.address, 2)) + .to.emit(nft, 'Transfer') + .withArgs(owner.address, recipient.address, 2) + .and.to.emit(nft, 'Transfer') + .withArgs(recipient.address, owner.address, 2); + expect(await nft.ownerOf(2)).to.equal(owner.address); + expect(await nft.balanceOf(recipient.address)).to.equal(1); + }); + + it('supports ERC1155 single and batch operations', async () => { + const [owner, recipient] = await ethers.getSigners(); + const token = await ethers.deployContract('TestERC1155'); + await token.mint(owner.address, 1, 10); + await token.safeTransferFrom(owner.address, recipient.address, 1, 3, '0x'); + await token.mintBatch(owner.address, [2, 3], [5, 6]); + await token.safeBatchTransferFrom(owner.address, recipient.address, [2, 3], [1, 2], '0x'); + expect(await token.balanceOf(1, recipient.address)).to.equal(3); + expect(await token.balanceOf(2, recipient.address)).to.equal(1); + expect(await token.balanceOf(3, recipient.address)).to.equal(2); + }); +}); diff --git a/integration_test/load_generator/package-lock.json b/integration_test/load_generator/package-lock.json index 076c70160c..88c3173830 100644 --- a/integration_test/load_generator/package-lock.json +++ b/integration_test/load_generator/package-lock.json @@ -1,11 +1,11 @@ { - "name": "sei-pacific-replay-load-generator", + "name": "sei-multi-mode-load-generator", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "sei-pacific-replay-load-generator", + "name": "sei-multi-mode-load-generator", "version": "1.0.0", "license": "MIT", "dependencies": { @@ -55,38 +55,6 @@ "protobufjs": "^6.8.8" } }, - "node_modules/@confio/ics23/node_modules/long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", - "license": "Apache-2.0" - }, - "node_modules/@confio/ics23/node_modules/protobufjs": { - "version": "6.11.6", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.6.tgz", - "integrity": "sha512-k8BHqgPBOtrlougZZqF2uUk5Z7bN8f0wj+3e8M3hvtSv0NBAz4VBy5f6R5Nxq/l+i7mRFTgNZb2trxqTpHNY/A==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": ">=13.7.0", - "long": "^4.0.0" - }, - "bin": { - "pbjs": "bin/pbjs", - "pbts": "bin/pbts" - } - }, "node_modules/@cosmjs/amino": { "version": "0.32.4", "resolved": "https://registry.npmjs.org/@cosmjs/amino/-/amino-0.32.4.tgz", @@ -2158,69 +2126,6 @@ "node": ">=14" } }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", - "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", - "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", - "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.2.tgz", - "integrity": "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", - "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", - "license": "BSD-3-Clause" - }, "node_modules/@scure/base": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.2.6.tgz", @@ -2639,12 +2544,6 @@ "@types/node": "*" } }, - "node_modules/@types/long": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", - "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", - "license": "MIT" - }, "node_modules/@types/minimatch": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", diff --git a/integration_test/load_generator/package.json b/integration_test/load_generator/package.json index bdae52b0bb..7e54f3cd41 100644 --- a/integration_test/load_generator/package.json +++ b/integration_test/load_generator/package.json @@ -1,11 +1,12 @@ { - "name": "sei-pacific-replay-load-generator", + "name": "sei-multi-mode-load-generator", "version": "1.0.0", "private": true, - "description": "Capture Pacific traffic and safely replay equivalent load on Sei testnets.", + "description": "Generate synthetic workloads or safely replay Pacific traffic on Sei testnets.", "license": "MIT", "scripts": { "compile": "hardhat compile", + "build": "npm run compile && tsc -p tsconfig.build.json", "typecheck": "tsc --noEmit", "test": "mocha --require tsx/cjs \"test/**/*.spec.ts\"", "test:fixtures": "hardhat test hardhat-test/*.spec.ts", @@ -15,6 +16,10 @@ "replay:users": "tsx src/provisionUsers.ts", "replay:run": "tsx src/runReplay.ts", "replay:buffered": "tsx src/runBuffered.ts", + "load": "tsx src/runLoad.ts", + "load:setup": "tsx src/runLoad.ts setup", + "load:provision": "tsx src/runLoad.ts provision", + "load:prepare-account": "tsx src/prepareRunnerAccount.ts", "replay:setup": "npm run compile && npm run replay:deploy && npm run replay:users", "replay:start": "npm run replay:setup && npm run replay:capture && npm run replay:validate && npm run replay:run", "replay:start:buffered": "npm run replay:setup && npm run replay:buffered", @@ -43,5 +48,8 @@ "solc": "0.6.12", "tsx": "^4.19.0", "typescript": "^5.5.4" + }, + "overrides": { + "protobufjs": "$protobufjs" } } diff --git a/integration_test/load_generator/src/config.ts b/integration_test/load_generator/src/config.ts index 2b761c188f..3d352f21ce 100644 --- a/integration_test/load_generator/src/config.ts +++ b/integration_test/load_generator/src/config.ts @@ -8,21 +8,6 @@ const PACIFIC_EVM_RPC = 'https://node-wave-0-evm.pacific-1.platform.sei.io/'; const PACIFIC_COSMOS_RPC = 'https://node-wave-0-rpc.pacific-1.platform.sei.io/'; const DEFAULT_REPLAY_DIR = 'runtime/replay/pacific-1/pacific-1-20m'; -const TARGETS = { - 'arctic-1': { - evmChainId: 713715n, - cosmosChainId: 'arctic-1', - evmRpcUrl: 'https://node-wave-0-evm.arctic-1.platform.sei.io/', - cosmosRpcUrl: 'https://node-wave-0-rpc.arctic-1.platform.sei.io/', - }, - 'atlantic-2': { - evmChainId: 1328n, - cosmosChainId: 'atlantic-2', - evmRpcUrl: 'https://node-wave-0-evm.atlantic-2.platform.sei.io/', - cosmosRpcUrl: 'https://node-wave-0-rpc.atlantic-2.platform.sei.io/', - }, -} as const; - export interface TargetConfig { network: ReplayTargetNetwork; evmChainId: bigint; @@ -35,17 +20,17 @@ export interface TargetConfig { } export function loadTargetConfig(env: Environment = process.env): TargetConfig { - const network = string(env, 'TARGET_NETWORK', 'arctic-1'); - if (network !== 'arctic-1' && network !== 'atlantic-2') { - throw new Error('TARGET_NETWORK must be "arctic-1" or "atlantic-2"'); + const network = requiredString(env, 'TARGET_NETWORK'); + if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(network)) { + throw new Error('TARGET_NETWORK must be a path-safe network name'); } - const defaults = TARGETS[network]; const userCount = positiveInteger(env, 'USER_COUNT', 100); return { network, - ...defaults, - evmRpcUrl: string(env, 'TARGET_EVM_RPC', defaults.evmRpcUrl), - cosmosRpcUrl: string(env, 'TARGET_COSMOS_RPC', defaults.cosmosRpcUrl), + evmChainId: targetEvmChainId(env), + cosmosChainId: requiredString(env, 'TARGET_COSMOS_CHAIN_ID'), + evmRpcUrl: requiredString(env, 'TARGET_EVM_RPC'), + cosmosRpcUrl: requiredString(env, 'TARGET_COSMOS_RPC'), usersPath: resolvePath( env, 'LOAD_USERS', @@ -54,12 +39,23 @@ export function loadTargetConfig(env: Environment = process.env): TargetConfig { deploymentPath: resolvePath( env, 'LOAD_DEPLOYMENT', - `runtime/replay-deployments/${network}-v4.json`, + `runtime/replay-deployments/${network}-v5.json`, ), mnemonic: string(env, 'TARGET_MNEMONIC', string(env, 'SEI_ADMIN_MNEMONIC', '')), }; } +function targetEvmChainId(env: Environment): bigint { + requiredString(env, 'TARGET_EVM_CHAIN_ID'); + return positiveBigInt(env, 'TARGET_EVM_CHAIN_ID', 1n); +} + +function requiredString(env: Environment, key: string): string { + const value = env[key]?.trim(); + if (value) return value; + throw new Error(`${key} is required`); +} + export function loadCaptureConfig(env: Environment = process.env) { const captureId = string( env, @@ -166,12 +162,12 @@ export function loadBufferedConfig(env: Environment = process.env) { } export function loadProvisionConfig(env: Environment = process.env) { - const fundSei = positiveNumber(env, 'FUND_SEI', 100); + const fundSei = string(env, 'FUND_SEI', '100'); return { execute: flag(env, 'EXECUTE'), userCount: positiveInteger(env, 'USER_COUNT', 100), fundSei, - targetUsei: BigInt(Math.round(fundSei * 1_000_000)), + targetUsei: seiToUsei(fundSei, 'FUND_SEI'), }; } @@ -295,6 +291,16 @@ function positiveBigInt(env: Environment, key: string, fallback: bigint): bigint return value; } +export function seiToUsei(value: string, key = 'SEI amount'): bigint { + const match = /^(\d+)(?:\.(\d{1,6}))?$/.exec(value.trim()); + if (!match) { + throw new Error(`${key} must be a positive decimal with at most 6 decimal places`); + } + const usei = BigInt(match[1]) * 1_000_000n + BigInt((match[2] ?? '').padEnd(6, '0') || '0'); + if (usei <= 0n) throw new Error(`${key} must be greater than zero`); + return usei; +} + function traceConfig(env: Environment): { traceCaptureMode: 'off' | 'calls' | 'full'; traceConcurrency: number; diff --git a/integration_test/load_generator/src/deployFixtures.ts b/integration_test/load_generator/src/deployFixtures.ts index 165e14e3d3..08f72a2403 100644 --- a/integration_test/load_generator/src/deployFixtures.ts +++ b/integration_test/load_generator/src/deployFixtures.ts @@ -1,17 +1,16 @@ /** * Deploy the allowlisted contracts used by Pacific semantic replay. * - * TARGET_NETWORK=arctic-1 EXECUTE=1 npm run replay:deploy - * TARGET_NETWORK=atlantic-2 EXECUTE=1 npm run replay:deploy + * TARGET_NETWORK= EXECUTE=1 npm run replay:deploy */ import fs from 'fs/promises'; import path from 'path'; import { ethers } from 'ethers'; -import { loadDeployConfig, loadTargetConfig, verifyTargetRpc } from './config'; +import { loadDeployConfig, loadTargetConfig, TargetConfig, verifyTargetRpc } from './config'; import { minBigInt } from './numeric'; import { REPLAY_DEPLOYMENT_SCHEMA_VERSION, - REPLAY_V4_CONTRACT_KEYS, + REPLAY_CONTRACT_KEYS, ReplayDeploymentManifest, } from './replay/replayTypes'; import { writeJsonAtomic } from './io'; @@ -28,7 +27,7 @@ interface Artifact { const deployConfig = loadDeployConfig(); const { execute: EXECUTE, forceDeploy: FORCE_DEPLOY } = deployConfig; -async function main(): Promise { +export async function deployFixturesMain(): Promise { const target = loadTargetConfig(); console.log(`Replay fixtures: ${target.network} (${target.evmChainId})`); console.log(`Manifest: ${target.deploymentPath}`); @@ -42,6 +41,17 @@ async function main(): Promise { const provider = new ethers.JsonRpcProvider(target.evmRpcUrl); provider.pollingInterval = 200; + try { + await deployFixtures(target, provider); + } finally { + provider.destroy(); + } +} + +async function deployFixtures( + target: TargetConfig, + provider: ethers.JsonRpcProvider, +): Promise { await verifyTargetRpc(target, provider); const artifacts = await loadArtifacts(); validateSushiArtifacts(artifacts); @@ -94,6 +104,7 @@ async function main(): Promise { [adminAddress], ); const nft = await deploy('ReplayNFT', admin, artifacts.nft, takeNonce(), [adminAddress]); + const erc1155 = await deploy('ReplayERC1155', admin, artifacts.erc1155, takeNonce()); const profileHarness = await deploy( 'ProfileLoadHarness', admin, @@ -461,6 +472,7 @@ async function main(): Promise { tokenB: await tokenB.getAddress(), pair, nft: await nft.getAddress(), + erc1155: await erc1155.getAddress(), profileHarness: await profileHarness.getAddress(), callGraphHarness: await callGraphHarness.getAddress(), callGraphNode, @@ -518,6 +530,7 @@ async function loadArtifacts(): Promise> { weth: 'artifacts/contracts/mocks/WETH9.sol/WETH9.json', token: 'artifacts/contracts/TestERC20.sol/TestERC20.json', nft: 'artifacts/contracts/TestNFT.sol/TestNFT.json', + erc1155: 'artifacts/contracts/TestERC1155.sol/TestERC1155.json', harness: 'artifacts/contracts/ProfileLoadHarness.sol/ProfileLoadHarness.json', callGraph: 'artifacts/contracts/CallGraphHarness.sol/CallGraphHarness.json', syntheticCreation: @@ -753,7 +766,7 @@ async function verifyExisting( ); } validateSushiV2Provenance(manifest); - for (const name of REPLAY_V4_CONTRACT_KEYS) { + for (const name of REPLAY_CONTRACT_KEYS) { if (!manifest.contracts[name]) throw new Error(`Existing deployment is missing ${name}`); } if (manifest.network !== network || BigInt(manifest.chainId) !== chainId) { @@ -773,7 +786,9 @@ async function verifyExisting( await verifyWiring(manifest.contracts as Record, artifacts, provider); } -main().catch(error => { - console.error('Fatal:', error instanceof Error ? error.message : error); - process.exit(1); -}); +if (require.main === module) { + deployFixturesMain().catch(error => { + console.error('Fatal:', error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/integration_test/load_generator/src/fixturePreparation.ts b/integration_test/load_generator/src/fixturePreparation.ts index 7d9794ea16..8f20484cc5 100644 --- a/integration_test/load_generator/src/fixturePreparation.ts +++ b/integration_test/load_generator/src/fixturePreparation.ts @@ -33,6 +33,7 @@ export async function prepareSemanticFixtures( workers: FixturePreparationWorker[], deployment: ReplayDeploymentManifest, gasLimit: bigint, + receiptTimeoutMs = 60_000, ): Promise { const contracts = requiredPreparationContracts(deployment); const targetBalance = ethers.parseEther('1000'); @@ -50,7 +51,7 @@ export async function prepareSemanticFixtures( ): Promise => { const response = await transaction; try { - await response.wait(); + await response.wait(1, receiptTimeoutMs); nonce++; } catch (error) { throw new Error( diff --git a/integration_test/load_generator/src/funding.ts b/integration_test/load_generator/src/funding.ts new file mode 100644 index 0000000000..d578d3901f --- /dev/null +++ b/integration_test/load_generator/src/funding.ts @@ -0,0 +1,44 @@ +import { coins } from '@cosmjs/amino'; +import { EncodeObject } from '@cosmjs/proto-signing'; +import { SigningStargateClient } from '@cosmjs/stargate'; + +export async function sendFundingBatches( + client: SigningStargateClient, + sender: string, + amounts: Array<{ address: string; amount: bigint }>, + memo: string, +): Promise { + const nonZero = amounts.filter(item => item.amount > 0n); + const fees = Array.from({ length: Math.ceil(nonZero.length / 10) }, (_, batch) => + BigInt(30_000 + Math.min(10, nonZero.length - batch * 10) * 5_000), + ).reduce((sum, fee) => sum + fee, 0n); + const required = nonZero.reduce((sum, item) => sum + item.amount, fees); + const available = BigInt((await client.getBalance(sender, 'usei')).amount); + if (available < required) { + throw new Error( + `Funding account has ${available} usei but this pass requires at least ${required} usei`, + ); + } + for (let start = 0; start < nonZero.length; start += 10) { + const batch = nonZero.slice(start, start + 10); + const messages: EncodeObject[] = batch.map(item => ({ + typeUrl: '/cosmos.bank.v1beta1.MsgSend', + value: { + fromAddress: sender, + toAddress: item.address, + amount: coins(item.amount.toString(), 'usei'), + }, + })); + const result = await client.signAndBroadcast( + sender, + messages, + { + amount: coins(String(30_000 + batch.length * 5_000), 'usei'), + gas: String(200_000 + batch.length * 100_000), + }, + memo, + ); + if (result.code !== 0) throw new Error(`Funding batch failed: ${result.rawLog}`); + console.log(` funded batch ${start / 10 + 1}/${Math.ceil(nonZero.length / 10)}`); + } +} diff --git a/integration_test/load_generator/src/loadAudit.ts b/integration_test/load_generator/src/loadAudit.ts new file mode 100644 index 0000000000..f613f99889 --- /dev/null +++ b/integration_test/load_generator/src/loadAudit.ts @@ -0,0 +1,62 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { LoadLane, LoadOutcome } from './loadMetrics'; + +export interface LoadAuditRecord { + timestamp: string; + runId: string; + loadType: string; + sequence: number; + worker: number; + operation: string; + lane: LoadLane; + outcome: LoadOutcome; + hash?: string; + error?: string; +} + +export class LoadAuditWriter { + private queue = Promise.resolve(); + private size = 0; + + constructor( + readonly file: string, + private readonly maxBytes = 100 * 1024 * 1024, + private readonly retainFiles = 5, + ) {} + + async initialize(): Promise { + await fs.mkdir(path.dirname(this.file), { recursive: true }); + await fs.writeFile(this.file, '', { flag: 'a' }); + this.size = (await fs.stat(this.file)).size; + } + + record(record: LoadAuditRecord): Promise { + const write = this.queue.then(() => this.append(`${JSON.stringify(record)}\n`)); + this.queue = write.catch(() => undefined); + return write; + } + + flush(): Promise { + return this.queue; + } + + private async append(line: string): Promise { + const bytes = Buffer.byteLength(line); + if (this.size > 0 && this.size + bytes > this.maxBytes) await this.rotate(); + await fs.appendFile(this.file, line); + this.size += bytes; + } + + private async rotate(): Promise { + await fs.rm(`${this.file}.${this.retainFiles}`, { force: true }); + for (let index = this.retainFiles - 1; index >= 1; index--) { + await fs.rename(`${this.file}.${index}`, `${this.file}.${index + 1}`).catch(error => { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + }); + } + await fs.rename(this.file, `${this.file}.1`); + await fs.writeFile(this.file, ''); + this.size = 0; + } +} diff --git a/integration_test/load_generator/src/loadConfig.ts b/integration_test/load_generator/src/loadConfig.ts new file mode 100644 index 0000000000..1b059b24d0 --- /dev/null +++ b/integration_test/load_generator/src/loadConfig.ts @@ -0,0 +1,219 @@ +import path from 'node:path'; +import { Environment } from './config'; + +export const LOAD_TYPES = ['defi', 'tokenops', 'nativetransfers', 'simulate'] as const; +export type LoadType = (typeof LOAD_TYPES)[number]; +export type LoadCommand = 'run' | 'setup' | 'provision'; + +export interface LoadGeneratorConfig { + command: LoadCommand; + type: LoadType; + tps: number; + maxTps: number; + durationSeconds?: number; + runId: string; + runtimeDirectory: string; + workerCount: number; + partitionIndex: number; + usersPerPartition: number; + workerIndexOffset: number; + usersPerTps: number; + maxWorkerCount: number; + maxPendingPerWorker: number; + metricsHost: string; + metricsPort: number; + receiptTimeoutMs: number; + fixturePrepareGasLimit: bigint; + auditMaxBytes: number; + auditRetainFiles: number; + operationWeights: Record; + cw1155Contract?: string; + execute: boolean; +} + +export function loadGeneratorConfig( + argv: string[] = process.argv.slice(2), + env: Environment = process.env, +): LoadGeneratorConfig { + const args = parseArguments(argv); + const command = commandValue(args.command); + const type = loadType(args.type ?? env.LOAD_TYPE ?? 'simulate'); + const tps = positiveNumber(args.tps ?? env.TXS_PER_SECOND ?? env.MAX_TPS ?? '10', 'tps'); + const maxTps = positiveNumber(env.MAX_SYNTHETIC_TPS ?? '100', 'MAX_SYNTHETIC_TPS'); + if (type !== 'simulate' && tps > maxTps) { + throw new Error( + `tps ${tps} exceeds MAX_SYNTHETIC_TPS ${maxTps}; raise the safety limit explicitly`, + ); + } + const usersPerTps = positiveNumber(env.USERS_PER_TPS ?? '2', 'USERS_PER_TPS'); + const maxWorkerCount = positiveInteger(env.MAX_WORKER_COUNT ?? '200', 'MAX_WORKER_COUNT'); + const workerCount = env.WORKER_COUNT?.trim() + ? positiveInteger(env.WORKER_COUNT, 'WORKER_COUNT') + : Math.ceil(tps * usersPerTps); + if (workerCount > maxWorkerCount) { + throw new Error( + `worker count ${workerCount} exceeds MAX_WORKER_COUNT ${maxWorkerCount}; ` + + `raise the safety limit explicitly`, + ); + } + const usersPerPartition = positiveInteger( + env.USERS_PER_PARTITION ?? String(workerCount), + 'USERS_PER_PARTITION', + ); + if (workerCount > usersPerPartition) { + throw new Error( + `worker count ${workerCount} exceeds USERS_PER_PARTITION ${usersPerPartition}; ` + + `increase the reserved range before increasing active workers`, + ); + } + const partitionIndex = nonNegativeInteger( + env.PARTITION_INDEX ?? '0', + 'PARTITION_INDEX', + ); + const workerIndexOffset = env.WORKER_INDEX_OFFSET?.trim() + ? nonNegativeInteger(env.WORKER_INDEX_OFFSET, 'WORKER_INDEX_OFFSET') + : partitionIndex * usersPerPartition; + if (!Number.isSafeInteger(workerIndexOffset + usersPerPartition)) { + throw new Error('worker index range exceeds the safe integer limit'); + } + const durationRaw = args.duration ?? env.RUN_DURATION_SECONDS; + const durationSeconds = durationRaw ? positiveNumber(durationRaw, 'duration') : undefined; + const execute = env.EXECUTE === '1'; + const runId = (args.runId ?? env.RUN_ID ?? '').trim(); + if (execute && command === 'run' && !runId) { + throw new Error('RUN_ID or --run-id is required when EXECUTE=1'); + } + const effectiveRunId = runId || 'dry-run'; + return { + command, + type, + tps, + maxTps, + durationSeconds, + runId: effectiveRunId, + runtimeDirectory: path.resolve( + args.runtimeDirectory ?? env.LOAD_RUNTIME_DIR ?? `runtime/load-runs/${effectiveRunId}`, + ), + workerCount, + partitionIndex, + usersPerPartition, + workerIndexOffset, + usersPerTps, + maxWorkerCount, + maxPendingPerWorker: positiveInteger( + env.MAX_PENDING_PER_LANE ?? '2', + 'MAX_PENDING_PER_LANE', + ), + metricsHost: (env.METRICS_HOST ?? '127.0.0.1').trim(), + metricsPort: nonNegativeInteger(env.METRICS_PORT ?? '9465', 'METRICS_PORT'), + receiptTimeoutMs: positiveInteger( + env.EVM_RECEIPT_TIMEOUT_MS ?? '60000', + 'EVM_RECEIPT_TIMEOUT_MS', + ), + fixturePrepareGasLimit: BigInt( + positiveInteger( + env.FIXTURE_PREPARE_GAS_LIMIT ?? '2000000', + 'FIXTURE_PREPARE_GAS_LIMIT', + ), + ), + auditMaxBytes: positiveInteger( + env.LOAD_AUDIT_MAX_BYTES ?? String(100 * 1024 * 1024), + 'LOAD_AUDIT_MAX_BYTES', + ), + auditRetainFiles: positiveInteger( + env.LOAD_AUDIT_RETAIN_FILES ?? '5', + 'LOAD_AUDIT_RETAIN_FILES', + ), + operationWeights: parseWeights(env.LOAD_MIX), + cw1155Contract: optional(env.CW1155_CONTRACT), + execute, + }; +} + +interface ParsedArguments { + command?: string; + type?: string; + tps?: string; + duration?: string; + runId?: string; + runtimeDirectory?: string; +} + +function parseArguments(argv: string[]): ParsedArguments { + const result: ParsedArguments = {}; + const values = new Map>([ + ['--type', 'type'], + ['--tps', 'tps'], + ['--duration', 'duration'], + ['--run-id', 'runId'], + ['--runtime-dir', 'runtimeDirectory'], + ]); + for (let index = 0; index < argv.length; index++) { + const arg = argv[index]; + if (!arg.startsWith('-') && !result.command) { + result.command = arg; + continue; + } + const [flag, inline] = arg.split('=', 2); + const key = values.get(flag); + if (!key) throw new Error(`Unknown argument ${arg}`); + const value = inline ?? argv[++index]; + if (!value || value.startsWith('--')) throw new Error(`${flag} requires a value`); + result[key] = value; + } + return result; +} + +function commandValue(value?: string): LoadCommand { + const command = value ?? 'run'; + if (command !== 'run' && command !== 'setup' && command !== 'provision') { + throw new Error('Command must be run, setup, or provision'); + } + return command; +} + +function loadType(value: string): LoadType { + if (!LOAD_TYPES.includes(value as LoadType)) { + throw new Error(`type must be one of ${LOAD_TYPES.join(', ')}`); + } + return value as LoadType; +} + +function parseWeights(value?: string): Record { + if (!value?.trim()) return {}; + return Object.fromEntries( + value.split(',').map(entry => { + const [name, weight] = entry.split(':').map(part => part.trim()); + if (!name || !weight) throw new Error('LOAD_MIX must use operation:weight entries'); + return [name, positiveNumber(weight, `LOAD_MIX ${name}`)]; + }), + ); +} + +function positiveNumber(value: string, name: string): number { + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) + throw new Error(`${name} must be greater than zero`); + return parsed; +} + +function positiveInteger(value: string, name: string): number { + const parsed = positiveNumber(value, name); + if (!Number.isInteger(parsed)) throw new Error(`${name} must be an integer`); + return parsed; +} + +function nonNegativeInteger(value: string, name: string): number { + // An empty value reaches here when the downward API cannot resolve a field, so it must + // fail rather than parse as zero: PARTITION_INDEX 0 puts every pod on the same accounts. + const parsed = value.trim() ? Number(value) : Number.NaN; + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`${name} must be a non-negative integer`); + } + return parsed; +} + +function optional(value?: string): string | undefined { + const trimmed = value?.trim(); + return trimmed || undefined; +} diff --git a/integration_test/load_generator/src/loadMetrics.ts b/integration_test/load_generator/src/loadMetrics.ts new file mode 100644 index 0000000000..3ade59cc45 --- /dev/null +++ b/integration_test/load_generator/src/loadMetrics.ts @@ -0,0 +1,170 @@ +import http, { Server } from 'node:http'; +import { collectDefaultMetrics, Counter, Gauge, Histogram, Registry } from 'prom-client'; +import { LoadType } from './loadConfig'; + +export type LoadLane = 'evm' | 'cosmos'; +export type LoadOutcome = + | 'submitted' + | 'included' + | 'included_failed' + | 'poll_timeout' + | 'rejected' + | 'skipped'; + +export interface LoadTransactionCount { + lane: LoadLane; + operation: string; + outcome: LoadOutcome; + count: number; +} + +export interface LoadLatencyHistogram { + buckets: { upperBound: number; count: number }[]; + count: number; + sum: number; +} + +export interface LoadMetricsSnapshot { + transactions: LoadTransactionCount[]; + includedLatency: LoadLatencyHistogram; +} + +type HistogramValues = Awaited['get']>>['values']; + +export class LoadMetrics { + private readonly registry = new Registry(); + private readonly transactions = new Counter({ + name: 'sei_loadgen_transactions_total', + help: 'Load-generator transactions by operation and lifecycle outcome.', + labelNames: ['load_type', 'lane', 'operation', 'outcome'] as const, + registers: [this.registry], + }); + private readonly latency = new Histogram({ + name: 'sei_loadgen_transaction_seconds', + help: 'Transaction build, broadcast, and inclusion latency.', + labelNames: ['load_type', 'lane', 'operation', 'outcome'] as const, + buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 40, 60], + registers: [this.registry], + }); + private readonly pending = new Gauge({ + name: 'sei_loadgen_pending_transactions', + help: 'Transactions currently queued or awaiting a result.', + labelNames: ['load_type', 'lane'] as const, + registers: [this.registry], + }); + private readonly targetTps = new Gauge({ + name: 'sei_loadgen_target_tps', + help: 'Configured transaction rate for this process.', + labelNames: ['load_type'] as const, + registers: [this.registry], + }); + private server?: Server; + private ready = false; + + constructor(private readonly loadType: LoadType, tps: number) { + this.registry.setDefaultLabels({ load_type: loadType }); + collectDefaultMetrics({ register: this.registry, prefix: 'sei_loadgen_' }); + this.targetTps.set({ load_type: loadType }, tps); + } + + record(lane: LoadLane, operation: string, outcome: LoadOutcome): void { + this.transactions.inc({ + load_type: this.loadType, + lane, + operation, + outcome, + }); + } + + observe(lane: LoadLane, operation: string, outcome: LoadOutcome, seconds: number): void { + this.latency.observe({ load_type: this.loadType, lane, operation, outcome }, seconds); + } + + setPending(lane: LoadLane, value: number): void { + this.pending.set({ load_type: this.loadType, lane }, value); + } + + setReady(ready: boolean): void { + this.ready = ready; + } + + // Reading the counters back keeps the end-of-run report and the Prometheus dashboards on + // one source of truth, and it works when the metrics server is disabled. + async snapshot(): Promise { + const [transactions, latency] = await Promise.all([ + this.transactions.get(), + this.latency.get(), + ]); + return { + transactions: transactions.values.map(value => ({ + lane: String(value.labels.lane) as LoadLane, + operation: String(value.labels.operation), + outcome: String(value.labels.outcome) as LoadOutcome, + count: value.value, + })), + includedLatency: includedLatency(latency.values), + }; + } + + async listen(port: number, host: string): Promise { + this.server = http.createServer(async (request, response) => { + if (request.url === '/healthz') { + response.statusCode = 200; + response.end('ok\n'); + return; + } + if (request.url === '/readyz') { + response.statusCode = this.ready ? 200 : 503; + response.end(this.ready ? 'ready\n' : 'not ready\n'); + return; + } + if (request.url !== '/metrics') { + response.statusCode = 404; + response.end('not found\n'); + return; + } + response.setHeader('content-type', this.registry.contentType); + response.end(await this.registry.metrics()); + }); + await new Promise((resolve, reject) => { + this.server!.once('error', reject); + this.server!.listen(port, host, () => { + this.server!.off('error', reject); + resolve(); + }); + }); + } + + async close(): Promise { + if (!this.server) return; + await new Promise((resolve, reject) => { + this.server!.close(error => (error ? reject(error) : resolve())); + }); + this.server = undefined; + } +} + +function includedLatency(values: HistogramValues): LoadLatencyHistogram { + const buckets = new Map(); + let count = 0; + let sum = 0; + for (const value of values) { + if (value.labels.outcome !== 'included') continue; + if (value.metricName?.endsWith('_bucket')) { + const upperBound = + value.labels.le === '+Inf' ? Number.POSITIVE_INFINITY : Number(value.labels.le); + buckets.set(upperBound, (buckets.get(upperBound) ?? 0) + value.value); + } else if (value.metricName?.endsWith('_sum')) { + sum += value.value; + } else if (value.metricName?.endsWith('_count')) { + count += value.value; + } + } + return { + buckets: [...buckets] + .sort(([left], [right]) => left - right) + .map(([upperBound, bucketCount]) => ({ upperBound, count: bucketCount })), + count, + sum, + }; +} diff --git a/integration_test/load_generator/src/prepareRunnerAccount.ts b/integration_test/load_generator/src/prepareRunnerAccount.ts new file mode 100644 index 0000000000..22d86439fe --- /dev/null +++ b/integration_test/load_generator/src/prepareRunnerAccount.ts @@ -0,0 +1,149 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { coins } from '@cosmjs/amino'; +import { SigningStargateClient } from '@cosmjs/stargate'; +import { Encoder } from '@sei-js/cosmos/encoding'; +import { ethers } from 'ethers'; +import { queryEvmAssociation } from './association'; +import { loadTargetConfig, seiToUsei, verifyTargetCosmosRpc } from './config'; +import { sendFundingBatches } from './funding'; +import { cosmosWalletAt, privateKeyAt, replayRegistry } from './keys'; + +const ASSOCIATION_BUFFER_USEI = 100_000n; + +export async function prepareRunnerAccountMain(): Promise { + const target = loadTargetConfig(); + const mnemonicPathValue = ( + process.env.RUNNER_MNEMONIC_PATH ?? 'runtime/runner.mnemonic' + ).trim(); + if (!mnemonicPathValue) throw new Error('RUNNER_MNEMONIC_PATH must not be empty'); + const mnemonicPath = path.resolve(mnemonicPathValue); + const targetUsei = seiToUsei( + process.env.RUNNER_ACCOUNT_FUND_SEI ?? '', + 'RUNNER_ACCOUNT_FUND_SEI', + ); + if (targetUsei < ASSOCIATION_BUFFER_USEI) { + throw new Error( + `RUNNER_ACCOUNT_FUND_SEI must provide at least ${ASSOCIATION_BUFFER_USEI} usei`, + ); + } + if (process.env.EXECUTE !== '1') { + console.log( + `Dry-run: prepare a runner account at ${mnemonicPath} with target balance ` + + `${targetUsei} usei. Set EXECUTE=1 to continue.`, + ); + return; + } + if (!target.mnemonic) { + throw new Error('TARGET_MNEMONIC or SEI_ADMIN_MNEMONIC is required as the treasury'); + } + + const runnerMnemonic = await loadOrCreateMnemonic(mnemonicPath); + const treasuryWallet = await cosmosWalletAt(target.mnemonic, 0); + const runnerWallet = await cosmosWalletAt(runnerMnemonic, 0); + const treasuryAddress = (await treasuryWallet.getAccounts())[0].address; + const runnerAddress = (await runnerWallet.getAccounts())[0].address; + if (treasuryAddress === runnerAddress) { + throw new Error('Runner account must not be the treasury account'); + } + + const client = await SigningStargateClient.connectWithSigner( + target.cosmosRpcUrl, + treasuryWallet, + { + registry: replayRegistry(), + broadcastPollIntervalMs: 200, + }, + ); + try { + await verifyTargetCosmosRpc(target, client); + const balance = BigInt((await client.getBalance(runnerAddress, 'usei')).amount); + await sendFundingBatches( + client, + treasuryAddress, + [ + { + address: runnerAddress, + amount: balance < targetUsei ? targetUsei - balance : 0n, + }, + ], + `fund ${target.network} load runner`, + ); + } finally { + client.disconnect(); + } + + await associateRunner(target.cosmosRpcUrl, target.network, runnerMnemonic, runnerAddress); + console.log(`Runner account ready: ${runnerAddress}`); + console.log(`Mnemonic file: ${mnemonicPath}`); +} + +async function loadOrCreateMnemonic(mnemonicPath: string): Promise { + try { + const existing = (await fs.readFile(mnemonicPath, 'utf8')).trim(); + if (!existing) throw new Error(`Mnemonic file is empty: ${mnemonicPath}`); + return existing; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + + const mnemonic = ethers.Wallet.createRandom().mnemonic?.phrase; + if (!mnemonic) throw new Error('Failed to generate runner mnemonic'); + await fs.mkdir(path.dirname(mnemonicPath), { recursive: true }); + const handle = await fs.open(mnemonicPath, 'wx', 0o600); + try { + await handle.writeFile(`${mnemonic}\n`, 'utf8'); + } finally { + await handle.close(); + } + console.log(`Created runner mnemonic file: ${mnemonicPath}`); + return mnemonic; +} + +async function associateRunner( + cosmosRpcUrl: string, + network: string, + mnemonic: string, + seiAddress: string, +): Promise { + const expectedEvmAddress = new ethers.Wallet(privateKeyAt(mnemonic, 0)).address; + const mapping = await queryEvmAssociation(cosmosRpcUrl, seiAddress); + if (mapping.associated) { + if (mapping.evmAddress.toLowerCase() !== expectedEvmAddress.toLowerCase()) { + throw new Error( + `Runner is associated with ${mapping.evmAddress}, expected ${expectedEvmAddress}`, + ); + } + return; + } + + const wallet = await cosmosWalletAt(mnemonic, 0); + const client = await SigningStargateClient.connectWithSigner(cosmosRpcUrl, wallet, { + registry: replayRegistry(), + broadcastPollIntervalMs: 200, + }); + try { + const message = Encoder.evm.MsgAssociate.fromPartial({ + sender: seiAddress, + custom_message: `${network} load runner`, + }); + const result = await client.signAndBroadcast( + seiAddress, + [{ typeUrl: `/${Encoder.evm.MsgAssociate.$type}`, value: message }], + { amount: coins('21000', 'usei'), gas: '200000' }, + `associate ${network} load runner`, + ); + if (result.code !== 0) { + throw new Error(`Runner association failed: ${result.rawLog}`); + } + } finally { + client.disconnect(); + } +} + +if (require.main === module) { + prepareRunnerAccountMain().catch(error => { + console.error('Fatal:', error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/integration_test/load_generator/src/provisionUsers.ts b/integration_test/load_generator/src/provisionUsers.ts index b25df4fbe2..e76a919307 100644 --- a/integration_test/load_generator/src/provisionUsers.ts +++ b/integration_test/load_generator/src/provisionUsers.ts @@ -1,13 +1,11 @@ /** * Derive, associate, and fund replay users on Arctic-1 or Atlantic-2. * - * TARGET_NETWORK=arctic-1 EXECUTE=1 npm run replay:users - * TARGET_NETWORK=atlantic-2 EXECUTE=1 npm run replay:users + * TARGET_NETWORK= EXECUTE=1 npm run replay:users */ import fs from 'fs/promises'; import path from 'path'; import { coins } from '@cosmjs/amino'; -import { EncodeObject } from '@cosmjs/proto-signing'; import { SigningStargateClient } from '@cosmjs/stargate'; import { Encoder } from '@sei-js/cosmos/encoding'; import { ethers } from 'ethers'; @@ -21,6 +19,7 @@ import { writeJsonAtomic } from './io'; import { mapConcurrent } from './concurrency'; import { cosmosWalletAt, privateKeyAt, replayRegistry } from './keys'; import { queryEvmAssociation } from './association'; +import { sendFundingBatches } from './funding'; const provisionConfig = loadProvisionConfig(); const { @@ -30,6 +29,7 @@ const { execute: EXECUTE, } = provisionConfig; const ASSOCIATION_BUFFER_USEI = 100_000n; +const PROVISION_CONCURRENCY = 5; interface LoadUser { index: number; @@ -38,149 +38,180 @@ interface LoadUser { evmAddress: string; } -async function main(): Promise { +export async function provisionUsersMain(): Promise { const target = loadTargetConfig(); console.log( `Replay users: ${target.network}, ${USER_COUNT} users, target ${FUND_SEI} SEI each`, ); console.log(`Manifest: ${target.usersPath}`); + if (!target.mnemonic) { + if (!EXECUTE) { + console.log('Dry-run only. Set TARGET_MNEMONIC to print the funding account.'); + return; + } + throw new Error('TARGET_MNEMONIC or SEI_ADMIN_MNEMONIC is required for provisioning'); + } + const adminWallet = await cosmosWalletAt(target.mnemonic, 0); + const adminAddress = (await adminWallet.getAccounts())[0].address; + const workerFundingUsei = TARGET_USEI * BigInt(USER_COUNT); + console.log(`Funding account: ${adminAddress}`); + console.log( + `Worker funding target: ${formatUsei(workerFundingUsei)} SEI ` + + `(${workerFundingUsei} usei), excluding fees`, + ); if (!EXECUTE) { console.log('Dry-run only. Set EXECUTE=1 to associate and fund users.'); return; } - if (!target.mnemonic) { - throw new Error('TARGET_MNEMONIC or SEI_ADMIN_MNEMONIC is required for provisioning'); - } const provider = new ethers.JsonRpcProvider(target.evmRpcUrl); - await verifyTargetRpc(target, provider); - const users = await deriveUsers(target.mnemonic); - const adminWallet = await cosmosWalletAt(target.mnemonic, 0); - const adminAddress = (await adminWallet.getAccounts())[0].address; - const admin = await SigningStargateClient.connectWithSigner(target.cosmosRpcUrl, adminWallet, { - registry: replayRegistry(), - broadcastPollIntervalMs: 200, - }); try { - await verifyTargetCosmosRpc(target, admin); - } catch (error) { - admin.disconnect(); - throw error; - } + await verifyTargetRpc(target, provider); + const users = await deriveUsers(target.mnemonic); + const admin = await SigningStargateClient.connectWithSigner( + target.cosmosRpcUrl, + adminWallet, + { + registry: replayRegistry(), + broadcastPollIntervalMs: 200, + }, + ); + try { + await verifyTargetCosmosRpc(target, admin); - const associations = await mapConcurrent(users, 5, async user => { - const mapping = await queryEvmAssociation(target.cosmosRpcUrl, user.seiAddress); - if ( - mapping.associated && - mapping.evmAddress.toLowerCase() !== user.evmAddress.toLowerCase() - ) { - throw new Error( - `User ${user.index} is associated with ${mapping.evmAddress}, ` + - `expected ${user.evmAddress}`, + const associations = await mapConcurrent(users, PROVISION_CONCURRENCY, async user => { + const mapping = await queryEvmAssociation(target.cosmosRpcUrl, user.seiAddress); + if ( + mapping.associated && + mapping.evmAddress.toLowerCase() !== user.evmAddress.toLowerCase() + ) { + throw new Error( + `User ${user.index} is associated with ${mapping.evmAddress}, ` + + `expected ${user.evmAddress}`, + ); + } + return mapping; + }); + const associationByAddress = new Map( + users.map((user, index) => [user.seiAddress, associations[index]]), ); - } - return mapping; - }); - const associationByAddress = new Map( - users.map((user, index) => [user.seiAddress, associations[index]]), - ); - const initialBalances = await Promise.all( - users.map(async user => BigInt((await admin.getBalance(user.seiAddress, 'usei')).amount)), - ); - await sendFundingBatches( - admin, - adminAddress, - users.map((user, index) => ({ - address: user.seiAddress, - amount: - associations[index].associated || initialBalances[index] >= ASSOCIATION_BUFFER_USEI - ? 0n - : ASSOCIATION_BUFFER_USEI - initialBalances[index], - })), - `fund ${target.network} replay association buffer`, - ); - - let associated = 0; - let newAssociations = 0; - await mapConcurrent(users, 5, async user => { - const mapping = associationByAddress.get(user.seiAddress)!; - if (mapping.associated) { - associated++; - return; - } - const wallet = await cosmosWalletAt(target.mnemonic, user.index); - const client = await SigningStargateClient.connectWithSigner(target.cosmosRpcUrl, wallet, { - registry: replayRegistry(), - broadcastPollIntervalMs: 200, - }); - const message = Encoder.evm.MsgAssociate.fromPartial({ - sender: user.seiAddress, - custom_message: `${target.network} Pacific replay`, - }); - try { - const result = await client.signAndBroadcast( - user.seiAddress, - [{ typeUrl: `/${Encoder.evm.MsgAssociate.$type}`, value: message }], - { amount: coins('21000', 'usei'), gas: '200000' }, - `associate ${target.network} replay user`, + const initialBalances = await readBalances(admin, users); + await sendFundingBatches( + admin, + adminAddress, + users.map((user, index) => ({ + address: user.seiAddress, + amount: + associations[index].associated || + initialBalances[index] >= ASSOCIATION_BUFFER_USEI + ? 0n + : ASSOCIATION_BUFFER_USEI - initialBalances[index], + })), + `fund ${target.network} replay association buffer`, ); - if (result.code !== 0 && !/already|associated/i.test(result.rawLog ?? '')) { - throw new Error( - `Association failed for user ${user.index}: ${result.rawLog}`, + + let associated = 0; + let newAssociations = 0; + await mapConcurrent(users, PROVISION_CONCURRENCY, async user => { + const mapping = associationByAddress.get(user.seiAddress)!; + if (mapping.associated) { + associated++; + return; + } + const wallet = await cosmosWalletAt(target.mnemonic, user.index); + const client = await SigningStargateClient.connectWithSigner( + target.cosmosRpcUrl, + wallet, + { + registry: replayRegistry(), + broadcastPollIntervalMs: 200, + }, ); - } + const message = Encoder.evm.MsgAssociate.fromPartial({ + sender: user.seiAddress, + custom_message: `${target.network} Pacific replay`, + }); + try { + const result = await client.signAndBroadcast( + user.seiAddress, + [{ typeUrl: `/${Encoder.evm.MsgAssociate.$type}`, value: message }], + { amount: coins('21000', 'usei'), gas: '200000' }, + `associate ${target.network} replay user`, + ); + if (result.code !== 0 && !/already|associated/i.test(result.rawLog ?? '')) { + throw new Error( + `Association failed for user ${user.index}: ${result.rawLog}`, + ); + } + } finally { + client.disconnect(); + } + associated++; + newAssociations++; + if (associated % 10 === 0 || associated === users.length) { + console.log(` associated ${associated}/${users.length}`); + } + }); + console.log( + `Association check complete: ${associated}/${users.length} valid, ` + + `${newAssociations} newly associated`, + ); + + const balancesAfterAssociation = await readBalances(admin, users); + await sendFundingBatches( + admin, + adminAddress, + users.map((user, index) => ({ + address: user.seiAddress, + amount: + balancesAfterAssociation[index] >= TARGET_USEI + ? 0n + : TARGET_USEI - balancesAfterAssociation[index], + })), + `fund ${target.network} replay users`, + ); + + const finalBalances = await readBalances(admin, users); + const manifest = { + schemaVersion: 1, + network: target.network, + chainId: Number(target.evmChainId), + generatedAt: new Date().toISOString(), + targetBalanceSei: FUND_SEI, + targetBalanceUsei: TARGET_USEI.toString(), + derivation: { + source: 'TARGET_MNEMONIC or SEI_ADMIN_MNEMONIC', + privateKeysPersisted: false, + }, + users: users.map((user, index) => ({ + ...user, + balanceUsei: finalBalances[index].toString(), + })), + }; + await fs.mkdir(path.dirname(target.usersPath), { recursive: true }); + await writeJsonAtomic(target.usersPath, manifest); + console.log(`Saved ${users.length} users to ${target.usersPath}`); } finally { - client.disconnect(); - } - associated++; - newAssociations++; - if (associated % 10 === 0 || associated === users.length) { - console.log(` associated ${associated}/${users.length}`); + admin.disconnect(); } - }); - console.log( - `Association check complete: ${associated}/${users.length} valid, ` + - `${newAssociations} newly associated`, - ); + } finally { + provider.destroy(); + } +} - const balancesAfterAssociation = await Promise.all( - users.map(async user => BigInt((await admin.getBalance(user.seiAddress, 'usei')).amount)), - ); - await sendFundingBatches( - admin, - adminAddress, - users.map((user, index) => ({ - address: user.seiAddress, - amount: - balancesAfterAssociation[index] >= TARGET_USEI - ? 0n - : TARGET_USEI - balancesAfterAssociation[index], - })), - `fund ${target.network} replay users`, +async function readBalances( + client: SigningStargateClient, + users: LoadUser[], +): Promise { + return mapConcurrent(users, PROVISION_CONCURRENCY, async user => + BigInt((await client.getBalance(user.seiAddress, 'usei')).amount), ); +} - const finalBalances = await Promise.all( - users.map(async user => BigInt((await admin.getBalance(user.seiAddress, 'usei')).amount)), - ); - const manifest = { - schemaVersion: 1, - network: target.network, - chainId: Number(target.evmChainId), - generatedAt: new Date().toISOString(), - targetBalanceSei: FUND_SEI, - derivation: { - source: 'TARGET_MNEMONIC or SEI_ADMIN_MNEMONIC', - privateKeysPersisted: false, - }, - users: users.map((user, index) => ({ - ...user, - balanceUsei: finalBalances[index].toString(), - })), - }; - await fs.mkdir(path.dirname(target.usersPath), { recursive: true }); - await writeJsonAtomic(target.usersPath, manifest); - admin.disconnect(); - console.log(`Saved ${users.length} users to ${target.usersPath}`); +function formatUsei(usei: bigint): string { + const whole = usei / 1_000_000n; + const fraction = (usei % 1_000_000n).toString().padStart(6, '0').replace(/0+$/, ''); + return fraction ? `${whole}.${fraction}` : whole.toString(); } async function deriveUsers(mnemonic: string): Promise { @@ -197,38 +228,9 @@ async function deriveUsers(mnemonic: string): Promise { ); } -async function sendFundingBatches( - client: SigningStargateClient, - sender: string, - amounts: Array<{ address: string; amount: bigint }>, - memo: string, -): Promise { - const nonZero = amounts.filter(item => item.amount > 0n); - for (let start = 0; start < nonZero.length; start += 10) { - const batch = nonZero.slice(start, start + 10); - const messages: EncodeObject[] = batch.map(item => ({ - typeUrl: '/cosmos.bank.v1beta1.MsgSend', - value: { - fromAddress: sender, - toAddress: item.address, - amount: coins(item.amount.toString(), 'usei'), - }, - })); - const result = await client.signAndBroadcast( - sender, - messages, - { - amount: coins(String(30_000 + batch.length * 5_000), 'usei'), - gas: String(200_000 + batch.length * 100_000), - }, - memo, - ); - if (result.code !== 0) throw new Error(`Funding batch failed: ${result.rawLog}`); - console.log(` funded batch ${start / 10 + 1}/${Math.ceil(nonZero.length / 10)}`); - } +if (require.main === module) { + provisionUsersMain().catch(error => { + console.error('Fatal:', error instanceof Error ? error.message : error); + process.exitCode = 1; + }); } - -main().catch(error => { - console.error('Fatal:', error instanceof Error ? error.message : error); - process.exit(1); -}); diff --git a/integration_test/load_generator/src/replay/replayTypes.ts b/integration_test/load_generator/src/replay/replayTypes.ts index 27e7e9e97f..775996d956 100644 --- a/integration_test/load_generator/src/replay/replayTypes.ts +++ b/integration_test/load_generator/src/replay/replayTypes.ts @@ -2,8 +2,8 @@ export const REPLAY_SCHEMA_VERSION = 1; export const PACIFIC_EVM_CHAIN_ID = 1329; export const PACIFIC_COSMOS_CHAIN_ID = 'pacific-1'; export const EVM_MESSAGE_TYPE = '/seiprotocol.seichain.evm.MsgEVMTransaction'; -export const REPLAY_DEPLOYMENT_SCHEMA_VERSION = 4 as const; -export const REPLAY_V4_CONTRACT_KEYS = [ +export const REPLAY_DEPLOYMENT_SCHEMA_VERSION = 5 as const; +export const REPLAY_CONTRACT_KEYS = [ 'weth', 'factory', 'router', @@ -11,6 +11,7 @@ export const REPLAY_V4_CONTRACT_KEYS = [ 'tokenB', 'pair', 'nft', + 'erc1155', 'profileHarness', 'callGraphHarness', 'callGraphNode', @@ -37,7 +38,7 @@ export const REPLAY_V4_CONTRACT_KEYS = [ 'strategyVaultProxy', ] as const; -export type ReplayTargetNetwork = 'arctic-1' | 'atlantic-2'; +export type ReplayTargetNetwork = string; export type ReplayTransactionKind = 'transfer' | 'contractCall' | 'contractCreation'; export interface ReplayCallFrame { @@ -255,6 +256,7 @@ export interface ReplayDeploymentManifest { tokenB?: string; pair?: string; nft?: string; + erc1155?: string; profileHarness?: string; callGraphHarness?: string; callGraphNode?: string; diff --git a/integration_test/load_generator/src/runBuffered.ts b/integration_test/load_generator/src/runBuffered.ts index 02d42c4b35..1fef4104a2 100644 --- a/integration_test/load_generator/src/runBuffered.ts +++ b/integration_test/load_generator/src/runBuffered.ts @@ -6,7 +6,7 @@ * REPLAY_DIR=runtime/replay/pacific-1/pacific-1-20m npm run replay:buffered * * Continuous execution: - * TARGET_NETWORK=arctic-1 REPLAY_DIR=... EXECUTE=1 npm run replay:buffered + * TARGET_NETWORK= REPLAY_DIR=... EXECUTE=1 npm run replay:buffered */ import fs from 'fs/promises'; import path from 'path'; @@ -45,7 +45,7 @@ let stopping = false; let currentChild: ChildProcess | undefined; const segmentCache = new Map(); -async function main(): Promise { +export async function runBufferedMain(): Promise { if (TIME_SCALE > 1 && !bufferedConfig.allowBufferDrain) { throw new Error( 'Buffered continuous replay cannot sustain TIME_SCALE > 1; ' + @@ -290,11 +290,21 @@ async function runContinuousReplay(): Promise { async function spawnNpm(script: string, extraEnvironment: Record): Promise { await new Promise((resolve, reject) => { - const child = spawn('npm', ['run', script], { - cwd: process.cwd(), - env: { ...process.env, ...extraEnvironment }, - stdio: 'inherit', - }); + const compiledEntries: Record = { + 'replay:capture': path.join(process.cwd(), 'dist', 'capture.js'), + 'replay:run': path.join(process.cwd(), 'dist', 'runReplay.js'), + }; + const compiledEntry = + process.env.LOADGEN_COMPILED === '1' ? compiledEntries[script] : undefined; + const child = spawn( + compiledEntry ? process.execPath : 'npm', + compiledEntry ? [compiledEntry] : ['run', script], + { + cwd: process.cwd(), + env: { ...process.env, ...extraEnvironment }, + stdio: 'inherit', + }, + ); currentChild = child; child.once('error', reject); child.once('exit', (code, signal) => { @@ -322,9 +332,11 @@ async function sleep(milliseconds: number): Promise { await new Promise(resolve => setTimeout(resolve, milliseconds)); } -main().catch(error => { - stopping = true; - currentChild?.kill('SIGTERM'); - console.error('Fatal:', error instanceof Error ? error.message : error); - process.exit(1); -}); +if (require.main === module) { + runBufferedMain().catch(error => { + stopping = true; + currentChild?.kill('SIGTERM'); + console.error('Fatal:', error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/integration_test/load_generator/src/runLoad.ts b/integration_test/load_generator/src/runLoad.ts new file mode 100644 index 0000000000..19dd7911ef --- /dev/null +++ b/integration_test/load_generator/src/runLoad.ts @@ -0,0 +1,61 @@ +import { LoadGeneratorConfig, loadGeneratorConfig } from './loadConfig'; +import { runSynthetic } from './runSynthetic'; + +export async function runLoadMain(): Promise { + const config = loadGeneratorConfig(); + configureProvisioningEnvironment(config); + if (config.command === 'setup') { + const { deployFixturesMain } = await import('./deployFixtures'); + await deployFixturesMain(); + return; + } + if (config.command === 'provision') { + const { provisionUsersMain } = await import('./provisionUsers'); + await provisionUsersMain(); + return; + } + if (config.type !== 'simulate') { + await runSynthetic(config); + return; + } + if (!config.execute) { + console.log( + `Dry-run simulate: ${config.tps} max tx/s, ` + + `${config.durationSeconds ?? 'unbounded'} seconds`, + ); + return; + } + + process.env.MAX_TPS = String(config.tps); + if (config.durationSeconds) { + process.env.RUN_DURATION_SECONDS = String(config.durationSeconds); + } + const simulateMode = (process.env.SIMULATE_MODE ?? 'buffered').trim(); + if (simulateMode === 'buffered') { + const { runBufferedMain } = await import('./runBuffered'); + await runBufferedMain(); + return; + } + if (simulateMode === 'corpus') { + const { runReplayMain } = await import('./runReplay'); + await runReplayMain(); + return; + } + throw new Error('SIMULATE_MODE must be buffered or corpus'); +} + +export function configureProvisioningEnvironment( + config: Pick, + env: NodeJS.ProcessEnv = process.env, +): void { + if (config.command !== 'provision') return; + env.USER_COUNT = env.USER_COUNT?.trim() || String(config.workerCount); + env.WORKER_COUNT = env.USER_COUNT; +} + +if (require.main === module) { + runLoadMain().catch(error => { + console.error('Fatal:', error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/integration_test/load_generator/src/runReplay.ts b/integration_test/load_generator/src/runReplay.ts index 2ae126a088..e629e430cd 100644 --- a/integration_test/load_generator/src/runReplay.ts +++ b/integration_test/load_generator/src/runReplay.ts @@ -5,7 +5,7 @@ * REPLAY_DIR=runtime/replay/pacific-1/ npm run replay:run * * Execute: - * TARGET_NETWORK=arctic-1 REPLAY_DIR=... EXECUTE=1 npm run replay:run + * TARGET_NETWORK= REPLAY_DIR=... EXECUTE=1 npm run replay:run */ import path from 'path'; import { SigningStargateClient, StargateClient, TimeoutError } from '@cosmjs/stargate'; @@ -21,7 +21,7 @@ import { } from './replay/evmAdapters'; import { REPLAY_DEPLOYMENT_SCHEMA_VERSION, - REPLAY_V4_CONTRACT_KEYS, + REPLAY_CONTRACT_KEYS, ReplayBlock, ReplayDeploymentManifest, ReplayEvmTransaction, @@ -117,7 +117,7 @@ interface ReplayBlockCheckpoint { updatedAt: string; } -async function main(): Promise { +export async function runReplayMain(): Promise { if (FOLLOW_SEGMENTS && (MAX_SEGMENTS || REPLAY_THROUGH_BLOCK)) { throw new Error( 'FOLLOW_SEGMENTS cannot be combined with MAX_SEGMENTS or REPLAY_THROUGH_BLOCK', @@ -164,32 +164,16 @@ async function main(): Promise { ]); validateTargetManifests(manifest, deployment, target.network, target.evmChainId); if (manifest.users.length < 2) throw new Error('At least two replay users are required'); + const users = { ...manifest, users: manifest.users.slice(0, WORKER_COUNT) }; const provider = new ethers.JsonRpcProvider(target.evmRpcUrl); provider.pollingInterval = 200; - await verifyTargetRpc(target, provider); - const cosmosVerifier = await StargateClient.connect(target.cosmosRpcUrl); - try { - await verifyTargetCosmosRpc(target, cosmosVerifier); - } finally { - cosmosVerifier.disconnect(); - } - await verifyDeploymentCode(deployment, provider); - const users = { ...manifest, users: manifest.users.slice(0, WORKER_COUNT) }; - const workers = await createWorkers(users, target.mnemonic, provider); const liveMetrics = new ReplayMetrics( 'pacific-1', target.network, TIME_SCALE, PRIVILEGED_REPLAY_MODE, ); - if (METRICS_PORT > 0) { - await liveMetrics.listen(METRICS_PORT, METRICS_HOST); - console.log( - `Prometheus metrics: http://${METRICS_HOST}:${METRICS_PORT}/metrics ` + - `(health: /healthz)`, - ); - } const auditRunId = Date.now(); const bucketAudit = new BucketAuditWriter( path.resolve( @@ -202,7 +186,29 @@ async function main(): Promise { ), LOG_BUCKETS, ); - await bucketAudit.initialize(); + let workers: Worker[] = []; + try { + await verifyTargetRpc(target, provider); + const cosmosVerifier = await StargateClient.connect(target.cosmosRpcUrl); + try { + await verifyTargetCosmosRpc(target, cosmosVerifier); + } finally { + cosmosVerifier.disconnect(); + } + await verifyDeploymentCode(deployment, provider); + workers = await createWorkers(users, target.mnemonic, provider); + if (METRICS_PORT > 0) { + await liveMetrics.listen(METRICS_PORT, METRICS_HOST); + console.log( + `Prometheus metrics: http://${METRICS_HOST}:${METRICS_PORT}/metrics ` + + `(health: /healthz)`, + ); + } + await bucketAudit.initialize(); + } catch (error) { + await closeReplayResources(provider, liveMetrics, workers); + throw error; + } console.log(`Bucket audit: ${bucketAudit.auditPath}`); console.log(`No-semantic-bucket list: ${bucketAudit.unmatchedPath}`); let stopRequested = false; @@ -216,7 +222,12 @@ async function main(): Promise { try { if (!replayConfig.skipFixturePrepare) { - await prepareSemanticFixtures(workers, deployment, FIXTURE_PREPARE_GAS_LIMIT); + await prepareSemanticFixtures( + workers, + deployment, + FIXTURE_PREPARE_GAS_LIMIT, + EVM_RECEIPT_TIMEOUT_MS, + ); } let fees = await readFees(provider); let feeUpdatedAt = Date.now(); @@ -541,9 +552,23 @@ async function main(): Promise { console.log(`Bucket summary: ${JSON.stringify(bucketAudit.summary())}`); console.log(`Report: ${reportPath}`); } finally { - await bucketAudit.flush(); - for (const worker of workers) worker.cosmosClient?.disconnect(); + try { + await bucketAudit.flush(); + } finally { + await closeReplayResources(provider, liveMetrics, workers); + } + } +} + +async function closeReplayResources( + provider: ethers.JsonRpcProvider, + liveMetrics: ReplayMetrics, + workers: Worker[], +): Promise { + for (const worker of workers) worker.cosmosClient?.disconnect(); + try { await liveMetrics.close(); + } finally { provider.destroy(); } } @@ -1074,7 +1099,7 @@ function validateTargetManifests( ); } validateSushiV2Provenance(deployment); - for (const name of REPLAY_V4_CONTRACT_KEYS) { + for (const name of REPLAY_CONTRACT_KEYS) { if (!deployment.contracts[name]) throw new Error(`Deployment is missing ${name}`); } } @@ -1329,7 +1354,9 @@ function skipReasonLabel(reason: string): string { return 'adapter_skipped'; } -main().catch(error => { - console.error('Fatal:', error instanceof Error ? error.message : error); - process.exit(1); -}); +if (require.main === module) { + runReplayMain().catch(error => { + console.error('Fatal:', error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/integration_test/load_generator/src/runSynthetic.ts b/integration_test/load_generator/src/runSynthetic.ts new file mode 100644 index 0000000000..aa7ee887c0 --- /dev/null +++ b/integration_test/load_generator/src/runSynthetic.ts @@ -0,0 +1,611 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { coins } from '@cosmjs/amino'; +import { SigningStargateClient, StargateClient, TimeoutError } from '@cosmjs/stargate'; +import { ethers } from 'ethers'; +import { loadTargetConfig, verifyTargetCosmosRpc, verifyTargetRpc } from './config'; +import { prepareSemanticFixtures } from './fixturePreparation'; +import { readJson, writeJsonAtomic } from './io'; +import { cosmosWalletAt, privateKeyAt, replayRegistry } from './keys'; +import { LoadAuditWriter } from './loadAudit'; +import { LoadGeneratorConfig } from './loadConfig'; +import { LoadMetrics, LoadOutcome } from './loadMetrics'; +import { formatRunSummary, LoadStopReason, summarizeRun } from './loadSummary'; +import { + ReplayDeploymentManifest, + REPLAY_DEPLOYMENT_SCHEMA_VERSION, + ReplayUserManifest, +} from './replay/replayTypes'; +import { defiOperations } from './workloads/defi'; +import { nativeTransferOperations } from './workloads/nativetransfers'; +import { + applyOperationWeights, + chooseOperation, + nextScheduleAt, + paceUntil, + seededRandom, +} from './workloads/scheduler'; +import { prepareTokenFixtures, tokenOperations } from './workloads/tokenops'; +import { BuiltLoad, LoadOperation, LoadWorker, WorkloadContext } from './workloads/types'; + +interface SyntheticWorker extends LoadWorker { + mnemonic: string; + evmPending: number; + cosmosPending: number; + evmQueue: Promise; + cosmosQueue: Promise; + cosmosClient?: SigningStargateClient; +} + +class FeeOracle { + private updatedAt = 0; + private chainId?: bigint; + private fees?: { maxFeePerGas: bigint; maxPriorityFeePerGas: bigint }; + + constructor(private readonly provider: ethers.Provider) {} + + async transactionFields(): Promise<{ + chainId: bigint; + maxFeePerGas: bigint; + maxPriorityFeePerGas: bigint; + }> { + if (!this.chainId) this.chainId = (await this.provider.getNetwork()).chainId; + if (!this.fees || Date.now() - this.updatedAt >= 60_000) { + const fees = await this.provider.getFeeData(); + const gasPrice = fees.gasPrice ?? 50_000_000_000n; + this.fees = { + maxFeePerGas: fees.maxFeePerGas ?? gasPrice * 2n, + maxPriorityFeePerGas: fees.maxPriorityFeePerGas ?? 1_000_000_000n, + }; + this.updatedAt = Date.now(); + } + return { chainId: this.chainId, ...this.fees }; + } + + invalidate(): void { + this.fees = undefined; + } +} + +export async function runSynthetic(config: LoadGeneratorConfig): Promise { + if (config.type === 'simulate') throw new Error('simulate must use the replay runner'); + if (!config.execute) { + console.log( + `Dry-run ${config.type}: ${config.tps} tx/s, ${config.workerCount} workers, ` + + `pool users ${config.workerIndexOffset + 1}-` + + `${config.workerIndexOffset + config.workerCount} from reserved range ` + + `${config.workerIndexOffset + 1}-` + + `${config.workerIndexOffset + config.usersPerPartition}, ` + + `${config.durationSeconds ?? 'unbounded'} seconds ` + + `(safety ceiling ${config.maxTps} tx/s)`, + ); + return; + } + const target = loadTargetConfig(); + if (!target.mnemonic) throw new Error('TARGET_MNEMONIC is required'); + await fs.mkdir(config.runtimeDirectory, { recursive: true }); + const executionId = randomUUID(); + const startedAt = new Date().toISOString(); + await writeJsonAtomic(path.join(config.runtimeDirectory, 'run.json'), { + runId: config.runId, + executionId, + type: config.type, + tps: config.tps, + maxTps: config.maxTps, + workerCount: config.workerCount, + partitionIndex: config.partitionIndex, + usersPerPartition: config.usersPerPartition, + workerIndexOffset: config.workerIndexOffset, + workerIndexStart: config.workerIndexOffset + 1, + workerIndexEnd: config.workerIndexOffset + config.workerCount, + reservedUserIndexEnd: config.workerIndexOffset + config.usersPerPartition, + usersPerTps: config.usersPerTps, + startedAt, + }); + const [usersManifest, deployment] = await Promise.all([ + readJson(target.usersPath), + readJson(target.deploymentPath), + ]); + validateManifests(usersManifest, deployment, target.network, target.evmChainId); + const selectedUsers = selectWorkerUsers( + usersManifest.users, + config.workerIndexOffset, + config.workerCount, + ); + + const provider = new ethers.JsonRpcProvider(target.evmRpcUrl); + provider.pollingInterval = 200; + const metrics = new LoadMetrics(config.type, config.tps); + const audit = new LoadAuditWriter( + path.join(config.runtimeDirectory, 'transactions.jsonl'), + config.auditMaxBytes, + config.auditRetainFiles, + ); + const abort = new AbortController(); + let stopReason: LoadStopReason = 'completed'; + const requestStop = (signal: string) => { + if (abort.signal.aborted) return; + stopReason = 'signal'; + console.log(`Received ${signal}; stopping new submissions...`); + abort.abort(); + }; + process.once('SIGINT', () => requestStop('SIGINT')); + process.once('SIGTERM', () => requestStop('SIGTERM')); + + const workers: SyntheticWorker[] = []; + try { + await audit.initialize(); + if (config.metricsPort > 0) { + await metrics.listen(config.metricsPort, config.metricsHost); + console.log(`Metrics: http://${config.metricsHost}:${config.metricsPort}/metrics`); + } + await verifyTargetRpc(target, provider); + const verifier = await StargateClient.connect(target.cosmosRpcUrl); + try { + await verifyTargetCosmosRpc(target, verifier); + } finally { + verifier.disconnect(); + } + workers.push( + ...(await createWorkers(selectedUsers, target.mnemonic, provider)), + ); + if (workers.length < 2) throw new Error('At least two provisioned workers are required'); + const context: WorkloadContext = { + runId: config.runId, + executionId, + deployment, + provider, + workers, + cw1155Contract: config.cw1155Contract, + }; + if (config.type === 'defi') { + await prepareSemanticFixtures( + workers, + deployment, + config.fixturePrepareGasLimit, + config.receiptTimeoutMs, + ); + } else if (config.type === 'tokenops') { + await prepareTokenFixtures(workers, context, config.receiptTimeoutMs); + } + const operations = applyOperationWeights( + operationsFor(config.type, context), + config.operationWeights, + ); + metrics.setReady(true); + console.log( + `Starting ${config.type} load at ${config.tps} tx/s with ${workers.length} workers ` + + `(safety ceiling ${config.maxTps} tx/s)`, + ); + await runSchedule(config, target.cosmosRpcUrl, workers, operations, metrics, audit, abort); + } catch (error) { + stopReason = 'error'; + throw error; + } finally { + metrics.setReady(false); + abort.abort(); + await Promise.allSettled( + workers.map(async worker => { + await Promise.allSettled([worker.evmQueue, worker.cosmosQueue]); + worker.cosmosClient?.disconnect(); + }), + ); + await audit.flush(); + await reportRun(config, executionId, startedAt, stopReason, metrics); + await metrics.close(); + provider.destroy(); + } +} + +/** + * Writes this process's aggregate report next to the run manifest and prints it. A reporting + * failure is logged rather than thrown, so it cannot replace the error that ended the run. + */ +async function reportRun( + config: LoadGeneratorConfig, + executionId: string, + startedAt: string, + stopReason: LoadStopReason, + metrics: LoadMetrics, +): Promise { + try { + const completedAt = new Date(); + const summary = summarizeRun( + { + runId: config.runId, + executionId, + loadType: config.type, + targetTps: config.tps, + workerCount: config.workerCount, + partitionIndex: config.partitionIndex, + }, + { + startedAt, + completedAt: completedAt.toISOString(), + durationSeconds: (completedAt.getTime() - Date.parse(startedAt)) / 1_000, + }, + stopReason, + await metrics.snapshot(), + ); + await writeJsonAtomic(path.join(config.runtimeDirectory, 'summary.json'), summary); + console.log(formatRunSummary(summary)); + } catch (error) { + console.error(`Run summary unavailable: ${error instanceof Error ? error.message : error}`); + } +} + +async function runSchedule( + config: LoadGeneratorConfig, + cosmosRpcUrl: string, + workers: SyntheticWorker[], + operations: LoadOperation[], + metrics: LoadMetrics, + audit: LoadAuditWriter, + abort: AbortController, +): Promise { + const active = new Set>(); + const random = seededRandom(config.runId); + const startedAt = Date.now(); + const feeOracle = new FeeOracle(workers[0].wallet.provider!); + const deadline = config.durationSeconds + ? startedAt + config.durationSeconds * 1_000 + : undefined; + const deadlineTimer = deadline + ? setTimeout(() => abort.abort(), Math.max(0, deadline - Date.now())) + : undefined; + let sequence = 0; + let scheduledAt = startedAt; + try { + while (!abort.signal.aborted && (!deadline || Date.now() < deadline)) { + if (!(await paceUntil(scheduledAt, abort.signal))) break; + scheduledAt = nextScheduleAt(scheduledAt, config.tps); + const operationSequence = sequence; + const operation = chooseOperation(operations, random()); + const worker = workers[operationSequence % workers.length]; + const pending = operation.lane === 'evm' ? worker.evmPending : worker.cosmosPending; + if (pending >= config.maxPendingPerWorker) { + metrics.record(operation.lane, operation.name, 'skipped'); + await audit.record( + auditRecord(config, worker, operation, operationSequence, 'skipped'), + ); + sequence++; + continue; + } + if (operation.lane === 'evm') worker.evmPending++; + else worker.cosmosPending++; + metrics.setPending( + operation.lane, + workers.reduce( + (sum, item) => + sum + (operation.lane === 'evm' ? item.evmPending : item.cosmosPending), + 0, + ), + ); + const previous = operation.lane === 'evm' ? worker.evmQueue : worker.cosmosQueue; + const task = previous.then(() => + executeOperation( + config, + cosmosRpcUrl, + worker, + operation, + operationSequence, + metrics, + audit, + feeOracle, + ), + ); + const settled = task + .catch(error => { + console.error( + `${operation.name} worker ${worker.index}: ` + + `${error instanceof Error ? error.message : error}`, + ); + }) + .finally(() => { + if (operation.lane === 'evm') worker.evmPending--; + else worker.cosmosPending--; + metrics.setPending( + operation.lane, + workers.reduce( + (sum, item) => + sum + + (operation.lane === 'evm' ? item.evmPending : item.cosmosPending), + 0, + ), + ); + active.delete(settled); + }); + if (operation.lane === 'evm') worker.evmQueue = settled; + else worker.cosmosQueue = settled; + active.add(settled); + sequence++; + } + } finally { + if (deadlineTimer) clearTimeout(deadlineTimer); + } + abort.abort(); + await Promise.allSettled(active); +} + +async function executeOperation( + config: LoadGeneratorConfig, + cosmosRpcUrl: string, + worker: SyntheticWorker, + operation: LoadOperation, + sequence: number, + metrics: LoadMetrics, + audit: LoadAuditWriter, + feeOracle: FeeOracle, +): Promise { + const startedAt = process.hrtime.bigint(); + let hash: string | undefined; + let outcome: LoadOutcome = 'rejected'; + let errorMessage: string | undefined; + try { + const built = await operation.build(worker, sequence); + if (built.lane !== operation.lane) { + throw new Error(`${operation.name} built the wrong lane`); + } + if (built.lane === 'evm') { + hash = await executeEvm( + worker, + built, + config.receiptTimeoutMs, + metrics, + operation.name, + feeOracle, + ); + } else { + hash = await executeCosmos(worker, built, cosmosRpcUrl, metrics, operation.name); + } + outcome = 'included'; + } catch (error) { + const submittedHash = (error as { transactionHash?: string }).transactionHash; + hash = hash ?? submittedHash; + outcome = (error as { loadOutcome?: LoadOutcome }).loadOutcome ?? 'rejected'; + errorMessage = error instanceof Error ? error.message : String(error); + metrics.record(operation.lane, operation.name, outcome); + } + metrics.observe( + operation.lane, + operation.name, + outcome, + Number(process.hrtime.bigint() - startedAt) / 1e9, + ); + await audit.record({ + ...auditRecord(config, worker, operation, sequence, outcome), + hash, + error: errorMessage, + }); +} + +async function executeEvm( + worker: SyntheticWorker, + built: Extract, + timeoutMs: number, + metrics: LoadMetrics, + operation: string, + feeOracle: FeeOracle, +): Promise { + const fees = await feeOracle.transactionFields(); + const signed = await worker.wallet.signTransaction({ + ...built.transaction, + chainId: fees.chainId, + nonce: worker.evmNonce, + type: 2, + maxFeePerGas: fees.maxFeePerGas, + maxPriorityFeePerGas: fees.maxPriorityFeePerGas, + }); + let response: ethers.TransactionResponse; + try { + response = await worker.wallet.provider!.broadcastTransaction(signed); + } catch (error) { + feeOracle.invalidate(); + worker.evmNonce = await worker.wallet.provider!.getTransactionCount( + worker.evmAddress, + 'pending', + ); + throw error; + } + worker.evmNonce++; + metrics.record('evm', operation, 'submitted'); + let receipt: ethers.TransactionReceipt | null; + try { + receipt = await response.wait(1, timeoutMs); + } catch (error) { + const failedReceipt = (error as { receipt?: ethers.TransactionReceipt }).receipt; + if (failedReceipt?.status === 0) { + throw outcomeError('EVM transaction reverted', 'included_failed', response.hash); + } + feeOracle.invalidate(); + await resyncEvmNonce(worker); + throw outcomeError( + error instanceof Error ? error.message : String(error), + 'poll_timeout', + response.hash, + ); + } + if (!receipt) throw outcomeError('EVM receipt unavailable', 'poll_timeout', response.hash); + if (receipt.status === 0) { + throw outcomeError('EVM transaction reverted', 'included_failed', response.hash); + } + metrics.record('evm', operation, 'included'); + return response.hash; +} + +async function resyncEvmNonce(worker: SyntheticWorker): Promise { + try { + worker.evmNonce = await worker.wallet.provider!.getTransactionCount( + worker.evmAddress, + 'pending', + ); + } catch { + // Keep the local nonce; the next broadcast rejection will retry synchronization. + } +} + +async function executeCosmos( + worker: SyntheticWorker, + built: Extract, + cosmosRpcUrl: string, + metrics: LoadMetrics, + operation: string, +): Promise { + const client = await cosmosClient(worker, cosmosRpcUrl); + try { + const result = await client.signAndBroadcast( + worker.seiAddress, + [...built.messages], + { + amount: coins(built.feeUsei ?? '25000', 'usei'), + gas: built.gas ?? '250000', + }, + built.memo ?? `loadgen ${operation}`, + ); + metrics.record('cosmos', operation, 'submitted'); + if (result.code !== 0) { + throw outcomeError( + `Cosmos transaction failed: ${result.rawLog}`, + 'included_failed', + result.transactionHash, + ); + } + metrics.record('cosmos', operation, 'included'); + return result.transactionHash; + } catch (error) { + if (error instanceof TimeoutError) { + metrics.record('cosmos', operation, 'submitted'); + worker.cosmosClient?.disconnect(); + worker.cosmosClient = undefined; + throw outcomeError(error.message, 'poll_timeout'); + } + throw error; + } +} + +async function cosmosClient( + worker: SyntheticWorker, + rpcUrl: string, +): Promise { + if (worker.cosmosClient) return worker.cosmosClient; + const wallet = await cosmosWalletAt(worker.mnemonic, worker.index); + worker.cosmosClient = await SigningStargateClient.connectWithSigner(rpcUrl, wallet, { + registry: replayRegistry(), + broadcastPollIntervalMs: 200, + broadcastTimeoutMs: 60_000, + }); + return worker.cosmosClient; +} + +export function selectWorkerUsers( + users: ReplayUserManifest['users'], + offset: number, + count: number, +): ReplayUserManifest['users'] { + const end = offset + count; + if (end > users.length) { + throw new Error( + `User pool has ${users.length} workers but this pod requires indexes ` + + `${offset + 1}-${end}; provision a larger pool or reduce replicas`, + ); + } + const selected = users.slice(offset, end); + selected.forEach((user, index) => { + const expected = offset + index + 1; + if (user.index !== expected) { + throw new Error( + `User pool entry ${offset + index} has derivation index ${user.index}, ` + + `expected ${expected}`, + ); + } + }); + return selected; +} + +async function createWorkers( + users: ReplayUserManifest['users'], + mnemonic: string, + provider: ethers.JsonRpcProvider, +): Promise { + return Promise.all( + users.map(async (user, slot) => { + const wallet = new ethers.Wallet(privateKeyAt(mnemonic, user.index), provider); + const cosmosWallet = await cosmosWalletAt(mnemonic, user.index); + const seiAddress = (await cosmosWallet.getAccounts())[0].address; + if ( + wallet.address.toLowerCase() !== user.evmAddress.toLowerCase() || + seiAddress !== user.seiAddress + ) { + throw new Error(`Derived address mismatch for worker ${user.index}`); + } + return { + slot, + index: user.index, + mnemonic, + seiAddress, + evmAddress: wallet.address, + wallet, + evmNonce: await provider.getTransactionCount(wallet.address, 'pending'), + evmPending: 0, + cosmosPending: 0, + evmQueue: Promise.resolve(), + cosmosQueue: Promise.resolve(), + }; + }), + ); +} + +function operationsFor( + type: Exclude, + context: WorkloadContext, +): LoadOperation[] { + if (type === 'defi') return defiOperations(context); + if (type === 'tokenops') return tokenOperations(context); + return nativeTransferOperations(context); +} + +function validateManifests( + users: ReplayUserManifest, + deployment: ReplayDeploymentManifest, + network: string, + chainId: bigint, +): void { + if ( + users.schemaVersion !== 1 || + users.network !== network || + users.chainId !== Number(chainId) + ) { + throw new Error('User manifest does not match the target'); + } + if ( + deployment.schemaVersion !== REPLAY_DEPLOYMENT_SCHEMA_VERSION || + deployment.network !== network || + deployment.chainId !== Number(chainId) + ) { + throw new Error('Deployment manifest does not match the target'); + } +} + +function auditRecord( + config: LoadGeneratorConfig, + worker: SyntheticWorker, + operation: LoadOperation, + sequence: number, + outcome: LoadOutcome, +) { + return { + timestamp: new Date().toISOString(), + runId: config.runId, + loadType: config.type, + sequence, + worker: worker.index, + operation: operation.name, + lane: operation.lane, + outcome, + }; +} + +function outcomeError(message: string, loadOutcome: LoadOutcome, transactionHash?: string): Error { + return Object.assign(new Error(message), { loadOutcome, transactionHash }); +} diff --git a/integration_test/load_generator/src/workloads/common.ts b/integration_test/load_generator/src/workloads/common.ts new file mode 100644 index 0000000000..105bae1e88 --- /dev/null +++ b/integration_test/load_generator/src/workloads/common.ts @@ -0,0 +1,52 @@ +import { ethers } from 'ethers'; +import { LoadOperation, LoadWorker, WorkloadContext } from './types'; + +export function nextWorker(context: WorkloadContext, slot: number): LoadWorker { + return context.workers[(slot + 1) % context.workers.length]; +} + +export function requiredContract(context: WorkloadContext, name: string): string { + const address = context.deployment.contracts[name]; + if (!address) throw new Error(`Deployment is missing ${name}`); + return address; +} + +export function requiredContracts( + context: WorkloadContext, + names: readonly T[], +): Record { + return Object.fromEntries(names.map(name => [name, requiredContract(context, name)])) as Record< + T, + string + >; +} + +export function evmCall(to: string, data: string, gasLimit: bigint, value = 0n) { + return { + lane: 'evm' as const, + transaction: { to, data, value, gasLimit }, + }; +} + +export function evmOperation( + name: string, + weight: number, + to: string, + contract: ethers.Interface, + method: string, + gasLimit: bigint, + args: (worker: LoadWorker, sequence: number) => unknown[], +): LoadOperation { + return { + name, + lane: 'evm', + weight, + async build(worker, sequence) { + return evmCall( + to, + contract.encodeFunctionData(method, args(worker, sequence)), + gasLimit, + ); + }, + }; +} diff --git a/integration_test/load_generator/src/workloads/defi.ts b/integration_test/load_generator/src/workloads/defi.ts new file mode 100644 index 0000000000..d8cb74c9a4 --- /dev/null +++ b/integration_test/load_generator/src/workloads/defi.ts @@ -0,0 +1,148 @@ +import { ethers } from 'ethers'; +import { LoadOperation, WorkloadContext } from './types'; +import { evmCall, evmOperation, requiredContracts } from './common'; + +const ROUTER = new ethers.Interface([ + 'function getAmountsOut(uint256,address[]) view returns(uint256[])', + 'function swapExactTokensForTokens(uint256,uint256,address[],address,uint256) returns(uint256[])', +]); +const LENDING = new ethers.Interface([ + 'function supply(address,uint256,address,uint16)', + 'function withdraw(address,uint256,address) returns(uint256)', + 'function borrow(uint256)', + 'function repayBorrow(uint256) returns(uint256)', +]); +const FARM = new ethers.Interface([ + 'function deposit(uint256,uint256)', + 'function withdraw(uint256,uint256)', +]); +const STAKING = new ethers.Interface([ + 'function stake(uint256) returns(uint256)', + 'function requestWithdrawal(uint256) returns(uint256)', +]); +const VAULT = new ethers.Interface([ + 'function deposit(uint256,address) returns(uint256)', + 'function withdraw(uint256,address,address) returns(uint256)', +]); + +const SWAP_AMOUNT = ethers.parseEther('0.001'); +const POSITION_AMOUNT = ethers.parseEther('0.0001'); +const GAS_LIMIT = 1_500_000n; + +export function defiOperations(context: WorkloadContext): LoadOperation[] { + const contracts = requiredContracts(context, [ + 'router', + 'tokenA', + 'tokenB', + 'lendingPoolProxy', + 'masterChef', + 'liquidStakingProxy', + 'strategyVaultProxy', + ]); + const swap = (name: string, tokenIn: string, tokenOut: string): LoadOperation => ({ + name, + lane: 'evm', + weight: 25, + async build(worker) { + const path = [tokenIn, tokenOut]; + const router = new ethers.Contract(contracts.router, ROUTER, context.provider); + const amounts = (await router.getAmountsOut(SWAP_AMOUNT, path)) as bigint[]; + const minimum = (amounts.at(-1)! * 95n) / 100n; + return evmCall( + contracts.router, + ROUTER.encodeFunctionData('swapExactTokensForTokens', [ + SWAP_AMOUNT, + minimum, + path, + worker.evmAddress, + Math.floor(Date.now() / 1_000) + 120, + ]), + GAS_LIMIT, + ); + }, + }); + return [ + swap('swap_a_to_b', contracts.tokenA, contracts.tokenB), + swap('swap_b_to_a', contracts.tokenB, contracts.tokenA), + evmOperation( + 'lend_supply', + 12, + contracts.lendingPoolProxy, + LENDING, + 'supply', + GAS_LIMIT, + worker => [contracts.tokenA, POSITION_AMOUNT, worker.evmAddress, 0], + ), + evmOperation( + 'lend_borrow', + 8, + contracts.lendingPoolProxy, + LENDING, + 'borrow', + GAS_LIMIT, + () => [POSITION_AMOUNT / 10n], + ), + evmOperation( + 'lend_repay', + 8, + contracts.lendingPoolProxy, + LENDING, + 'repayBorrow', + GAS_LIMIT, + () => [POSITION_AMOUNT / 10n], + ), + evmOperation( + 'lend_withdraw', + 4, + contracts.lendingPoolProxy, + LENDING, + 'withdraw', + GAS_LIMIT, + worker => [contracts.tokenA, POSITION_AMOUNT, worker.evmAddress], + ), + evmOperation('farm_deposit', 6, contracts.masterChef, FARM, 'deposit', GAS_LIMIT, () => [ + 0, + POSITION_AMOUNT, + ]), + evmOperation('farm_withdraw', 3, contracts.masterChef, FARM, 'withdraw', GAS_LIMIT, () => [ + 0, + POSITION_AMOUNT, + ]), + evmOperation( + 'liquid_stake', + 6, + contracts.liquidStakingProxy, + STAKING, + 'stake', + GAS_LIMIT, + () => [POSITION_AMOUNT], + ), + evmOperation( + 'liquid_withdraw', + 2, + contracts.liquidStakingProxy, + STAKING, + 'requestWithdrawal', + GAS_LIMIT, + () => [POSITION_AMOUNT], + ), + evmOperation( + 'vault_deposit', + 6, + contracts.strategyVaultProxy, + VAULT, + 'deposit', + GAS_LIMIT, + worker => [POSITION_AMOUNT, worker.evmAddress], + ), + evmOperation( + 'vault_withdraw', + 2, + contracts.strategyVaultProxy, + VAULT, + 'withdraw', + GAS_LIMIT, + worker => [POSITION_AMOUNT, worker.evmAddress, worker.evmAddress], + ), + ]; +} diff --git a/integration_test/load_generator/src/workloads/nativetransfers.ts b/integration_test/load_generator/src/workloads/nativetransfers.ts new file mode 100644 index 0000000000..e9105df848 --- /dev/null +++ b/integration_test/load_generator/src/workloads/nativetransfers.ts @@ -0,0 +1,70 @@ +import { MsgSend } from 'cosmjs-types/cosmos/bank/v1beta1/tx'; +import { ethers } from 'ethers'; +import { nextWorker } from './common'; +import { LoadOperation, WorkloadContext } from './types'; + +const BANK_PRECOMPILE = '0x0000000000000000000000000000000000001001'; +const BANK = new ethers.Interface(['function send(address,address,string,uint256) returns(bool)']); + +export function nativeTransferOperations(context: WorkloadContext): LoadOperation[] { + return [ + { + name: 'cosmos_bank_send', + lane: 'cosmos', + weight: 40, + async build(worker) { + return { + lane: 'cosmos', + messages: [ + { + typeUrl: '/cosmos.bank.v1beta1.MsgSend', + value: MsgSend.fromPartial({ + fromAddress: worker.seiAddress, + toAddress: nextWorker(context, worker.slot).seiAddress, + amount: [{ denom: 'usei', amount: '1' }], + }), + }, + ], + gas: '150000', + feeUsei: '25000', + memo: 'loadgen bank send', + }; + }, + }, + { + name: 'evm_native_transfer', + lane: 'evm', + weight: 30, + async build(worker) { + return { + lane: 'evm', + transaction: { + to: nextWorker(context, worker.slot).evmAddress, + value: 1n, + gasLimit: 100_000n, + }, + }; + }, + }, + { + name: 'bank_precompile_send', + lane: 'evm', + weight: 30, + async build(worker) { + return { + lane: 'evm', + transaction: { + to: BANK_PRECOMPILE, + data: BANK.encodeFunctionData('send', [ + worker.evmAddress, + nextWorker(context, worker.slot).evmAddress, + 'usei', + 1, + ]), + gasLimit: 500_000n, + }, + }; + }, + }, + ]; +} diff --git a/integration_test/load_generator/src/workloads/scheduler.ts b/integration_test/load_generator/src/workloads/scheduler.ts new file mode 100644 index 0000000000..982816e68f --- /dev/null +++ b/integration_test/load_generator/src/workloads/scheduler.ts @@ -0,0 +1,64 @@ +import { LoadOperation } from './types'; + +export function applyOperationWeights( + operations: LoadOperation[], + overrides: Record, +): LoadOperation[] { + const names = new Set(operations.map(operation => operation.name)); + for (const name of Object.keys(overrides)) { + if (!names.has(name)) throw new Error(`LOAD_MIX references unknown operation ${name}`); + } + if (Object.keys(overrides).length === 0) return operations; + return operations.flatMap(operation => { + const weight = overrides[operation.name]; + return weight === undefined ? [] : [{ ...operation, weight }]; + }); +} + +export function chooseOperation(operations: LoadOperation[], random: number): LoadOperation { + const total = operations.reduce((sum, operation) => sum + operation.weight, 0); + if (operations.length === 0 || total <= 0) + throw new Error('Workload has no enabled operations'); + let cursor = Math.min(Math.max(random, 0), 1 - Number.EPSILON) * total; + for (const operation of operations) { + cursor -= operation.weight; + if (cursor < 0) return operation; + } + return operations[operations.length - 1]; +} + +export function seededRandom(seed: string): () => number { + let state = 2166136261; + for (const character of seed) { + state ^= character.charCodeAt(0); + state = Math.imul(state, 16777619); + } + return () => { + state += 0x6d2b79f5; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +export async function paceUntil(scheduledAt: number, signal: AbortSignal): Promise { + const waitMs = scheduledAt - Date.now(); + if (waitMs <= 0) return !signal.aborted; + return new Promise(resolve => { + const timer = setTimeout(() => { + signal.removeEventListener('abort', stop); + resolve(true); + }, waitMs); + const stop = () => { + clearTimeout(timer); + resolve(false); + }; + signal.addEventListener('abort', stop, { once: true }); + }); +} + +export function nextScheduleAt(scheduledAt: number, tps: number, now = Date.now()): number { + const intervalMs = 1_000 / tps; + return Math.max(scheduledAt + intervalMs, now + intervalMs); +} diff --git a/integration_test/load_generator/src/workloads/tokenops.ts b/integration_test/load_generator/src/workloads/tokenops.ts new file mode 100644 index 0000000000..60df57b2da --- /dev/null +++ b/integration_test/load_generator/src/workloads/tokenops.ts @@ -0,0 +1,181 @@ +import { MsgExecuteContract } from 'cosmjs-types/cosmwasm/wasm/v1/tx'; +import { ethers } from 'ethers'; +import { mapConcurrent } from '../concurrency'; +import { LoadOperation, LoadWorker, WorkloadContext } from './types'; +import { evmOperation, nextWorker, requiredContract } from './common'; + +const ERC20 = new ethers.Interface([ + 'function mint(address,uint256)', + 'function transfer(address,uint256) returns(bool)', + 'function approve(address,uint256) returns(bool)', + 'function burn(uint256)', + 'function balanceOf(address) view returns(uint256)', +]); +const ERC721 = new ethers.Interface([ + 'function safeMint(address,uint256)', + 'function ownerOf(uint256) view returns(address)', + 'function transferFrom(address,address,uint256)', + 'function roundTripTransfer(address,uint256)', +]); +const ERC1155 = new ethers.Interface([ + 'function mint(address,uint256,uint256)', + 'function mintBatch(address,uint256[],uint256[])', + 'function safeTransferFrom(address,address,uint256,uint256,bytes)', + 'function safeBatchTransferFrom(address,address,uint256[],uint256[],bytes)', + 'function balanceOf(uint256,address) view returns(uint256)', +]); +const AMOUNT = ethers.parseEther('0.001'); +const GAS_LIMIT = 800_000n; + +export function tokenOperations(context: WorkloadContext): LoadOperation[] { + const token = requiredContract(context, 'tokenA'); + const nft = requiredContract(context, 'nft'); + const erc1155 = requiredContract(context, 'erc1155'); + const operations: LoadOperation[] = [ + evmOperation('erc20_mint', 18, token, ERC20, 'mint', GAS_LIMIT, worker => [ + worker.evmAddress, + AMOUNT, + ]), + evmOperation('erc20_transfer', 20, token, ERC20, 'transfer', GAS_LIMIT, worker => [ + nextWorker(context, worker.slot).evmAddress, + AMOUNT / 10n, + ]), + evmOperation('erc20_approve', 8, token, ERC20, 'approve', GAS_LIMIT, worker => [ + nextWorker(context, worker.slot).evmAddress, + AMOUNT, + ]), + evmOperation('erc20_burn', 8, token, ERC20, 'burn', GAS_LIMIT, () => [AMOUNT / 20n]), + evmOperation('erc721_mint', 14, nft, ERC721, 'safeMint', GAS_LIMIT, (worker, sequence) => [ + worker.evmAddress, + uniqueTokenId(context.executionId, worker, sequence), + ]), + evmOperation( + 'erc721_round_trip', + 10, + nft, + ERC721, + 'roundTripTransfer', + GAS_LIMIT, + worker => [nextWorker(context, worker.slot).evmAddress, seedTokenId(worker)], + ), + evmOperation('erc1155_mint', 10, erc1155, ERC1155, 'mint', GAS_LIMIT, worker => [ + worker.evmAddress, + worker.index, + 10, + ]), + evmOperation( + 'erc1155_transfer', + 8, + erc1155, + ERC1155, + 'safeTransferFrom', + GAS_LIMIT, + worker => [ + worker.evmAddress, + nextWorker(context, worker.slot).evmAddress, + worker.index, + 1, + '0x', + ], + ), + evmOperation('erc1155_batch_mint', 4, erc1155, ERC1155, 'mintBatch', GAS_LIMIT, worker => [ + worker.evmAddress, + [worker.index * 10, worker.index * 10 + 1], + [2, 2], + ]), + ]; + if (context.cw1155Contract) { + operations.push({ + name: 'cw1155_send', + lane: 'cosmos', + weight: 8, + async build(worker) { + return { + lane: 'cosmos', + messages: [ + { + typeUrl: '/cosmwasm.wasm.v1.MsgExecuteContract', + value: MsgExecuteContract.fromPartial({ + sender: worker.seiAddress, + contract: context.cw1155Contract, + msg: Buffer.from( + JSON.stringify({ + send: { + from: worker.seiAddress, + to: nextWorker(context, worker.slot).seiAddress, + token_id: String(worker.index), + amount: '1', + }, + }), + ), + funds: [], + }), + }, + ], + gas: '600000', + feeUsei: '50000', + memo: 'loadgen cw1155 send', + }; + }, + }); + } + return operations; +} + +export async function prepareTokenFixtures( + workers: LoadWorker[], + context: WorkloadContext, + receiptTimeoutMs: number, +): Promise { + const token = requiredContract(context, 'tokenA'); + const nft = requiredContract(context, 'nft'); + const erc1155 = requiredContract(context, 'erc1155'); + await mapConcurrent(workers, 5, async worker => { + worker.evmNonce = await worker.wallet.provider!.getTransactionCount( + worker.evmAddress, + 'pending', + ); + const erc20 = new ethers.Contract(token, ERC20, worker.wallet); + const erc721 = new ethers.Contract(nft, ERC721, worker.wallet); + const multi = new ethers.Contract(erc1155, ERC1155, worker.wallet); + const sends: Array<() => Promise> = []; + if (((await erc20.balanceOf(worker.evmAddress)) as bigint) < AMOUNT) { + sends.push(() => erc20.mint(worker.evmAddress, AMOUNT, transaction(worker))); + } + const tokenId = seedTokenId(worker); + if (((await erc721.ownerOf(tokenId)) as string) === ethers.ZeroAddress) { + sends.push(() => erc721.safeMint(worker.evmAddress, tokenId, transaction(worker))); + } + if (((await multi.balanceOf(worker.index, worker.evmAddress)) as bigint) < 10n) { + sends.push(() => multi.mint(worker.evmAddress, worker.index, 10, transaction(worker))); + } + for (const send of sends) { + const response = await send(); + await response.wait(1, receiptTimeoutMs); + worker.evmNonce++; + } + }); +} + +function uniqueTokenId(executionId: string, worker: LoadWorker, sequence: number): bigint { + return BigInt( + ethers.keccak256( + ethers.solidityPacked( + ['string', 'address', 'uint256'], + [executionId, worker.evmAddress, sequence + 1], + ), + ), + ); +} + +function seedTokenId(worker: LoadWorker): bigint { + return BigInt( + ethers.keccak256( + ethers.solidityPacked(['string', 'address'], ['sei-loadgen-seed', worker.evmAddress]), + ), + ); +} + +function transaction(worker: LoadWorker): ethers.TransactionRequest { + return { nonce: worker.evmNonce, gasLimit: GAS_LIMIT }; +} diff --git a/integration_test/load_generator/src/workloads/types.ts b/integration_test/load_generator/src/workloads/types.ts new file mode 100644 index 0000000000..fce7bb0759 --- /dev/null +++ b/integration_test/load_generator/src/workloads/types.ts @@ -0,0 +1,44 @@ +import { EncodeObject } from '@cosmjs/proto-signing'; +import { ethers } from 'ethers'; +import { LoadLane } from '../loadMetrics'; +import { ReplayDeploymentManifest } from '../replay/replayTypes'; + +export interface LoadWorker { + slot: number; + index: number; + seiAddress: string; + evmAddress: string; + wallet: ethers.Wallet; + evmNonce: number; +} + +export interface WorkloadContext { + runId: string; + executionId: string; + deployment: ReplayDeploymentManifest; + provider: ethers.JsonRpcProvider; + workers: LoadWorker[]; + cw1155Contract?: string; +} + +export interface EvmLoad { + lane: 'evm'; + transaction: ethers.TransactionRequest; +} + +export interface CosmosLoad { + lane: 'cosmos'; + messages: readonly EncodeObject[]; + memo?: string; + gas?: string; + feeUsei?: string; +} + +export type BuiltLoad = EvmLoad | CosmosLoad; + +export interface LoadOperation { + name: string; + lane: LoadLane; + weight: number; + build(worker: LoadWorker, sequence: number): Promise; +} diff --git a/integration_test/load_generator/test/bucketAudit.spec.ts b/integration_test/load_generator/test/bucketAudit.spec.ts new file mode 100644 index 0000000000..66c234b45e --- /dev/null +++ b/integration_test/load_generator/test/bucketAudit.spec.ts @@ -0,0 +1,46 @@ +import { expect } from 'chai'; +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import { BucketAuditRecord, BucketAuditWriter } from '../src/replay/bucketAudit'; + +const record: BucketAuditRecord = { + recordedAt: new Date(0).toISOString(), + sourceNetwork: 'pacific-1', + targetNetwork: 'arctic-1', + sourceBlock: 1, + lane: 'cosmos', + sequence: 0, + sourceTransactionBytes: 1, + adapter: 'test', + fidelity: 'semantic', + outcome: 'included', +}; + +describe('bucket audit writer', () => { + it('creates parent directories and recovers its queue after a failed write', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'bucket-audit-')); + const auditPath = path.join(root, 'nested', 'audit.jsonl'); + const unmatchedPath = path.join(root, 'other', 'unmatched.jsonl'); + const writer = new BucketAuditWriter(auditPath, unmatchedPath, false); + try { + await writer.initialize(); + await fs.unlink(auditPath); + await fs.mkdir(auditPath); + let failed = false; + try { + await writer.record(record); + } catch { + failed = true; + } + expect(failed).to.equal(true); + await fs.rm(auditPath, { recursive: true }); + await fs.writeFile(auditPath, ''); + await writer.record({ ...record, sequence: 1 }); + await writer.flush(); + expect((await fs.readFile(auditPath, 'utf8')).trim()).to.contain('"sequence":1'); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/integration_test/load_generator/test/core.spec.ts b/integration_test/load_generator/test/core.spec.ts index 933bccc97a..c39526c071 100644 --- a/integration_test/load_generator/test/core.spec.ts +++ b/integration_test/load_generator/test/core.spec.ts @@ -33,6 +33,14 @@ import { summarizePrestateDiff, summarizeStructLogs, } from '../src/replay/traceCapture'; + +const ARCTIC_TARGET = { + TARGET_NETWORK: 'arctic-1', + TARGET_EVM_CHAIN_ID: '713715', + TARGET_COSMOS_CHAIN_ID: 'arctic-1', + TARGET_EVM_RPC: 'https://evm.arctic.example', + TARGET_COSMOS_RPC: 'https://rpc.arctic.example', +}; import { replayEntriesForBlock } from '../src/replay/replayScheduling'; import { SUSHI_V2_PROVENANCE } from '../src/sushiV2'; import { queryEvmAssociation } from '../src/association'; @@ -179,17 +187,55 @@ describe('load generator pure behavior', () => { .fixturePrepareGasLimit, ).to.equal(3_000_000n); expect( - loadTargetConfig({ TARGET_NETWORK: 'arctic-1', USER_COUNT: '5' }).usersPath, + loadTargetConfig({ ...ARCTIC_TARGET, USER_COUNT: '5' }).usersPath, ).to.equal(`${process.cwd()}/runtime/replay-users/arctic-1-5.json`); - expect(loadTargetConfig({ TARGET_NETWORK: 'arctic-1' }).deploymentPath).to.equal( - `${process.cwd()}/runtime/replay-deployments/arctic-1-v4.json`, + expect(loadTargetConfig(ARCTIC_TARGET).deploymentPath).to.equal( + `${process.cwd()}/runtime/replay-deployments/arctic-1-v5.json`, ); + const local = loadTargetConfig({ + TARGET_NETWORK: 'local', + TARGET_EVM_CHAIN_ID: '713714', + TARGET_COSMOS_CHAIN_ID: 'sei', + TARGET_EVM_RPC: 'http://127.0.0.1:8545', + TARGET_COSMOS_RPC: 'http://127.0.0.1:26657', + }); + expect(local.evmChainId).to.equal(713714n); + expect(local.cosmosChainId).to.equal('sei'); + expect(local.evmRpcUrl).to.equal('http://127.0.0.1:8545'); + const custom = loadTargetConfig({ + TARGET_NETWORK: 'devnet-7', + TARGET_EVM_CHAIN_ID: '7007', + TARGET_COSMOS_CHAIN_ID: 'devnet-7', + TARGET_EVM_RPC: 'https://evm.devnet.example', + TARGET_COSMOS_RPC: 'https://rpc.devnet.example', + }); + expect(custom.evmChainId).to.equal(7007n); + expect(custom.cosmosChainId).to.equal('devnet-7'); }); - it('rejects unsafe or unknown target configuration', () => { + it('rejects unsafe or incomplete target configuration', () => { expect(() => loadTargetConfig({ TARGET_NETWORK: 'pacific-1' })).to.throw( - 'TARGET_NETWORK', + 'TARGET_EVM_CHAIN_ID', + ); + expect(() => loadTargetConfig({ TARGET_NETWORK: '../arctic-1' })).to.throw( + 'path-safe', ); + expect(() => + loadTargetConfig({ + TARGET_NETWORK: 'devnet-7', + TARGET_EVM_CHAIN_ID: '0', + TARGET_COSMOS_CHAIN_ID: 'devnet-7', + TARGET_EVM_RPC: 'https://evm.devnet.example', + TARGET_COSMOS_RPC: 'https://rpc.devnet.example', + }), + ).to.throw('TARGET_EVM_CHAIN_ID'); + expect(() => + loadTargetConfig({ + TARGET_NETWORK: 'devnet-7', + TARGET_EVM_CHAIN_ID: '7007', + TARGET_COSMOS_CHAIN_ID: 'devnet-7', + }), + ).to.throw('TARGET_EVM_RPC'); expect(() => loadReplayConfig({ MAX_TPS: '0' })).to.throw('MAX_TPS'); expect(() => loadCaptureConfig({ BLOCKS_PER_BATCH: '21' })).to.throw( 'BLOCKS_PER_BATCH', @@ -211,7 +257,7 @@ describe('load generator pure behavior', () => { }); it('rejects a mismatched Cosmos target chain', async () => { - const target = loadTargetConfig({ TARGET_NETWORK: 'arctic-1' }); + const target = loadTargetConfig(ARCTIC_TARGET); await verifyTargetCosmosRpc(target, { getChainId: async () => 'arctic-1' }); try { await verifyTargetCosmosRpc(target, { getChainId: async () => 'pacific-1' }); diff --git a/integration_test/load_generator/test/loadGenerator.spec.ts b/integration_test/load_generator/test/loadGenerator.spec.ts new file mode 100644 index 0000000000..e951c94688 --- /dev/null +++ b/integration_test/load_generator/test/loadGenerator.spec.ts @@ -0,0 +1,350 @@ +import { expect } from 'chai'; +import { ethers } from 'ethers'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { loadProvisionConfig } from '../src/config'; +import { LoadAuditWriter } from '../src/loadAudit'; +import { loadGeneratorConfig } from '../src/loadConfig'; +import { LoadMetrics } from '../src/loadMetrics'; +import { formatRunSummary, histogramQuantile, summarizeRun } from '../src/loadSummary'; +import { configureProvisioningEnvironment } from '../src/runLoad'; +import { selectWorkerUsers } from '../src/runSynthetic'; +import { + applyOperationWeights, + chooseOperation, + nextScheduleAt, + seededRandom, +} from '../src/workloads/scheduler'; +import { defiOperations } from '../src/workloads/defi'; +import { nativeTransferOperations } from '../src/workloads/nativetransfers'; +import { tokenOperations } from '../src/workloads/tokenops'; +import { LoadOperation, WorkloadContext } from '../src/workloads/types'; +import { + REPLAY_DEPLOYMENT_SCHEMA_VERSION, + ReplayDeploymentManifest, + ReplayUserManifest, +} from '../src/replay/replayTypes'; +import { SUSHI_V2_PROVENANCE } from '../src/sushiV2'; + +describe('multi-mode load generator', () => { + it('parses CLI arguments and enforces run identity for execution', () => { + const config = loadGeneratorConfig( + ['run', '--type', 'defi', '--tps=12.5', '--duration', '30', '--run-id', 'defi-a'], + { + EXECUTE: '1', + WORKER_COUNT: '4', + PARTITION_INDEX: '3', + USERS_PER_PARTITION: '200', + FIXTURE_PREPARE_GAS_LIMIT: '3000000', + }, + ); + expect(config.type).to.equal('defi'); + expect(config.tps).to.equal(12.5); + expect(config.maxTps).to.equal(100); + expect(config.durationSeconds).to.equal(30); + expect(config.runId).to.equal('defi-a'); + expect(config.workerCount).to.equal(4); + expect(config.partitionIndex).to.equal(3); + expect(config.usersPerPartition).to.equal(200); + expect(config.workerIndexOffset).to.equal(600); + expect(config.usersPerTps).to.equal(2); + expect(config.maxWorkerCount).to.equal(200); + expect(config.fixturePrepareGasLimit).to.equal(3_000_000n); + expect(() => loadGeneratorConfig(['--type', 'tokenops'], { EXECUTE: '1' })).to.throw( + 'RUN_ID', + ); + expect(() => loadGeneratorConfig(['--type', 'unknown'], {})).to.throw( + 'type must be one of', + ); + expect(() => loadGeneratorConfig(['--type', 'defi', '--tps', '101'], {})).to.throw( + 'exceeds MAX_SYNTHETIC_TPS 100', + ); + expect( + loadGeneratorConfig(['--type', 'defi', '--tps', '1000'], { + MAX_SYNTHETIC_TPS: '1000', + MAX_WORKER_COUNT: '2000', + }).tps, + ).to.equal(1000); + expect(loadGeneratorConfig(['--type', 'defi', '--tps', '20'], {}).workerCount).to.equal(40); + expect(() => + loadGeneratorConfig(['--type', 'defi', '--tps', '101'], { + MAX_SYNTHETIC_TPS: '101', + }), + ).to.throw('worker count 202 exceeds MAX_WORKER_COUNT 200'); + expect( + loadGeneratorConfig(['--type', 'defi'], { + WORKER_COUNT: '4', + PARTITION_INDEX: '3', + WORKER_INDEX_OFFSET: '20', + }).workerIndexOffset, + ).to.equal(20); + expect(() => + loadGeneratorConfig(['--type', 'defi'], { + WORKER_COUNT: '5', + USERS_PER_PARTITION: '4', + }), + ).to.throw('worker count 5 exceeds USERS_PER_PARTITION 4'); + }); + + it('rejects an unresolved downward-api value instead of defaulting it', () => { + expect(() => loadGeneratorConfig(['--type', 'defi'], { PARTITION_INDEX: '' })).to.throw( + 'PARTITION_INDEX must be a non-negative integer', + ); + expect(() => loadGeneratorConfig(['--type', 'defi'], { METRICS_PORT: ' ' })).to.throw( + 'METRICS_PORT must be a non-negative integer', + ); + expect(loadGeneratorConfig(['--type', 'defi'], { METRICS_PORT: '0' }).metricsPort).to.equal( + 0, + ); + }); + + it('assigns disjoint deterministic user-pool partitions', () => { + const users: ReplayUserManifest['users'] = Array.from({ length: 6 }, (_, offset) => { + const index = offset + 1; + return { + index, + derivationPath: `m/44'/118'/0'/0/${index}`, + seiAddress: `sei${index}`, + evmAddress: `0x${index}`, + }; + }); + expect(selectWorkerUsers(users, 2, 2).map(user => user.index)).to.deep.equal([3, 4]); + expect(() => selectWorkerUsers(users, 4, 3)).to.throw( + 'requires indexes 5-7', + ); + users[2].index = 99; + expect(() => selectWorkerUsers(users, 2, 2)).to.throw( + 'derivation index 99, expected 3', + ); + }); + + it('preserves the provisioned pool size when running a partition', () => { + const runEnvironment: NodeJS.ProcessEnv = { USER_COUNT: '2000' }; + configureProvisioningEnvironment( + { command: 'run', workerCount: 20 }, + runEnvironment, + ); + expect(runEnvironment.USER_COUNT).to.equal('2000'); + + const provisionEnvironment: NodeJS.ProcessEnv = {}; + configureProvisioningEnvironment( + { command: 'provision', workerCount: 40 }, + provisionEnvironment, + ); + expect(provisionEnvironment.USER_COUNT).to.equal('40'); + expect(provisionEnvironment.WORKER_COUNT).to.equal('40'); + }); + + it('reports run totals and latency percentiles from the collected metrics', async () => { + const metrics = new LoadMetrics('defi', 20); + for (let count = 0; count < 100; count++) { + metrics.record('evm', 'swap_a_to_b', 'submitted'); + } + for (let count = 0; count < 90; count++) { + metrics.record('evm', 'swap_a_to_b', 'included'); + metrics.observe('evm', 'swap_a_to_b', 'included', 0.2); + } + for (let count = 0; count < 10; count++) { + metrics.record('evm', 'swap_a_to_b', 'rejected'); + } + metrics.record('cosmos', 'cosmos_bank_send', 'skipped'); + + const summary = summarizeRun( + { + runId: 'summary-test', + executionId: 'execution-a', + loadType: 'defi', + targetTps: 20, + workerCount: 40, + partitionIndex: 1, + }, + { + startedAt: new Date(0).toISOString(), + completedAt: new Date(10_000).toISOString(), + durationSeconds: 10, + }, + 'completed', + await metrics.snapshot(), + ); + + expect(summary.transactions.offered).to.equal(101); + expect(summary.transactions.submitted).to.equal(100); + expect(summary.transactions.included).to.equal(90); + expect(summary.transactions.successRatePercent).to.equal(90); + expect(summary.throughput.includedTps).to.equal(9); + expect(summary.includedLatencySeconds.mean).to.equal(0.2); + expect(summary.includedLatencySeconds.p50).to.equal(0.175); + expect(summary.includedLatencySeconds.p99).to.equal(0.2485); + expect(formatRunSummary(summary)).to.contain('success 90%'); + }); + + it('clamps quantiles above the highest finite latency bucket', () => { + const overflowing = { + buckets: [ + { upperBound: 1, count: 1 }, + { upperBound: Number.POSITIVE_INFINITY, count: 2 }, + ], + count: 2, + sum: 100, + }; + expect(histogramQuantile(overflowing, 0.99)).to.equal(1); + expect(histogramQuantile({ buckets: [], count: 0, sum: 0 }, 0.5)).to.equal(0); + }); + + it('parses large funding targets without number precision loss', () => { + const config = loadProvisionConfig({ FUND_SEI: '1000000000000.123456' }); + expect(config.fundSei).to.equal('1000000000000.123456'); + expect(config.targetUsei).to.equal(1_000_000_000_000_123_456n); + expect(() => loadProvisionConfig({ FUND_SEI: '1.0000001' })).to.throw( + 'at most 6 decimal places', + ); + }); + + it('rotates synthetic audit files at the configured bound', async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'load-audit-')); + try { + const file = path.join(directory, 'transactions.jsonl'); + const writer = new LoadAuditWriter(file, 200, 2); + await writer.initialize(); + for (let sequence = 0; sequence < 6; sequence++) { + await writer.record({ + timestamp: new Date(0).toISOString(), + runId: 'rotation-test', + loadType: 'defi', + sequence, + worker: 1, + operation: 'swap_a_to_b', + lane: 'evm', + outcome: 'included', + }); + } + await writer.flush(); + expect(await fs.readdir(directory)).to.include.members([ + 'transactions.jsonl', + 'transactions.jsonl.1', + 'transactions.jsonl.2', + ]); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it('selects weighted operations deterministically and validates overrides', () => { + const operations = [operation('one', 1), operation('two', 3)]; + expect(chooseOperation(operations, 0).name).to.equal('one'); + expect(chooseOperation(operations, 0.5).name).to.equal('two'); + const first = seededRandom('run-a'); + const second = seededRandom('run-a'); + expect(Array.from({ length: 5 }, first)).to.deep.equal(Array.from({ length: 5 }, second)); + expect(applyOperationWeights(operations, { one: 9 })).to.deep.equal([ + { ...operations[0], weight: 9 }, + ]); + expect(() => applyOperationWeights(operations, { missing: 1 })).to.throw( + 'unknown operation', + ); + }); + + it('paces from the current time instead of bursting after a stall', () => { + expect(nextScheduleAt(1_000, 10, 1_000)).to.equal(1_100); + expect(nextScheduleAt(1_000, 10, 2_000)).to.equal(2_100); + }); + + it('exposes distinct defi, token, and native-transfer operation sets', () => { + const context = workloadContext(); + expect(defiOperations(context).map(item => item.name)).to.include.members([ + 'swap_a_to_b', + 'lend_borrow', + 'vault_deposit', + ]); + expect(tokenOperations(context).map(item => item.name)).to.include.members([ + 'erc20_mint', + 'erc721_round_trip', + 'erc1155_batch_mint', + 'cw1155_send', + ]); + const nativeTransfers = nativeTransferOperations(context); + expect(nativeTransfers.map(item => item.name)).to.have.members([ + 'cosmos_bank_send', + 'evm_native_transfer', + 'bank_precompile_send', + ]); + }); + + it('uses a unique execution id for ERC721 mints when a run restarts', async () => { + const first = workloadContext(); + const second = { ...first, executionId: 'second-execution' }; + const firstMint = tokenOperations(first).find(item => item.name === 'erc721_mint')!; + const secondMint = tokenOperations(second).find(item => item.name === 'erc721_mint')!; + const [firstLoad, secondLoad] = await Promise.all([ + firstMint.build(first.workers[0], 0), + secondMint.build(second.workers[0], 0), + ]); + expect(firstLoad).to.have.property('lane', 'evm'); + expect(secondLoad).to.have.property('lane', 'evm'); + expect( + (firstLoad as { transaction: ethers.TransactionRequest }).transaction.data, + ).not.to.equal((secondLoad as { transaction: ethers.TransactionRequest }).transaction.data); + }); +}); + +function operation(name: string, weight: number): LoadOperation { + return { + name, + weight, + lane: 'evm', + async build() { + return { lane: 'evm', transaction: {} }; + }, + }; +} + +function workloadContext(): WorkloadContext { + const address = (index: number) => + ethers.getAddress(`0x${index.toString(16).padStart(40, '0')}`); + const contracts = { + router: address(1), + tokenA: address(2), + tokenB: address(3), + lendingPoolProxy: address(4), + masterChef: address(5), + liquidStakingProxy: address(6), + strategyVaultProxy: address(7), + nft: address(8), + erc1155: address(9), + }; + const deployment: ReplayDeploymentManifest = { + schemaVersion: REPLAY_DEPLOYMENT_SCHEMA_VERSION, + network: 'arctic-1', + chainId: 713715, + sushiV2: SUSHI_V2_PROVENANCE, + contracts, + }; + const walletA = new ethers.Wallet(ethers.id('load-a')); + const walletB = new ethers.Wallet(ethers.id('load-b')); + return { + runId: 'test-run', + executionId: 'first-execution', + deployment, + provider: {} as ethers.JsonRpcProvider, + workers: [ + { + slot: 0, + index: 1, + seiAddress: 'sei1a', + evmAddress: walletA.address, + wallet: walletA, + evmNonce: 0, + }, + { + slot: 1, + index: 2, + seiAddress: 'sei1b', + evmAddress: walletB.address, + wallet: walletB, + evmNonce: 0, + }, + ], + cw1155Contract: 'sei1cw1155', + }; +} diff --git a/integration_test/load_generator/test/observability.spec.ts b/integration_test/load_generator/test/observability.spec.ts index 699de6c406..b4e1aeaebe 100644 --- a/integration_test/load_generator/test/observability.spec.ts +++ b/integration_test/load_generator/test/observability.spec.ts @@ -17,9 +17,7 @@ interface Dashboard { } describe('replay observability', () => { - const dashboardPath = path.resolve( - 'observability/grafana/dashboards/pacific-replay.json', - ); + const dashboardPath = path.resolve('observability/grafana/dashboards/pacific-replay.json'); it('ships a valid dashboard covering every custom replay metric', () => { const dashboard = JSON.parse(fs.readFileSync(dashboardPath, 'utf8')) as Dashboard; @@ -62,10 +60,7 @@ describe('replay observability', () => { }); it('binds dashboard services to localhost', () => { - const compose = fs.readFileSync( - path.resolve('observability/docker-compose.yml'), - 'utf8', - ); + const compose = fs.readFileSync(path.resolve('observability/docker-compose.yml'), 'utf8'); expect(compose).to.include('127.0.0.1:9090:9090'); expect(compose).to.include('127.0.0.1:3000:3000'); expect(compose).to.include('GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD'); diff --git a/integration_test/load_generator/tsconfig.build.json b/integration_test/load_generator/tsconfig.build.json new file mode 100644 index 0000000000..bfc62e87bc --- /dev/null +++ b/integration_test/load_generator/tsconfig.build.json @@ -0,0 +1,17 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "sourceMap": true + }, + "include": ["src/**/*.ts"], + "exclude": [ + "node_modules", + "artifacts", + "cache", + "typechain-types", + "test", + "hardhat-test" + ] +}