perf: skip_stats backport and correctness/perf improvements #47
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: integration | |
| on: | |
| push: | |
| branches: [main, "rust-v*"] | |
| pull_request: | |
| branches: [main, "rust-v*", next, next/*] | |
| merge_group: | |
| env: | |
| # https://github.com/rust-lang/cargo/issues/10280 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs " | |
| # Override RUSTFLAGS to not treat warnings as errors (setup-rust-toolchain sets -D warnings by default) | |
| # This allows deprecated methods to be warnings instead of errors | |
| RUSTFLAGS: "" | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| jobs: | |
| cloud: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| target: | |
| - name: aws | |
| - name: azure | |
| - name: gcp | |
| env: | |
| AWS_DEFAULT_REGION: "us-east-1" | |
| AWS_ACCESS_KEY_ID: deltalake | |
| AWS_SECRET_ACCESS_KEY: weloverust | |
| AWS_ENDPOINT_URL: http://localhost:4566 | |
| AWS_ALLOW_HTTP: "1" | |
| AZURE_USE_EMULATOR: "1" | |
| AZURE_STORAGE_ALLOW_HTTP: "1" | |
| AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" | |
| AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;" | |
| NEXTEST_PROFILE: 'ci' | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/sccache-action@v0.0.9 | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| cache: true | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - uses: taiki-e/install-action@nextest | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Run tests with rustls (default) | |
| run: | | |
| gmake setup-dat | |
| cargo llvm-cov nextest --cargo-profile=ci \ | |
| --package deltalake-${{ matrix.target.name }} \ | |
| --features integration_test \ | |
| --codecov \ | |
| --output-path codecov.json | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| native_tls: | |
| name: aws-native-tls | |
| runs-on: ubuntu-latest | |
| env: | |
| AWS_DEFAULT_REGION: "us-east-1" | |
| AWS_ACCESS_KEY_ID: deltalake | |
| AWS_SECRET_ACCESS_KEY: weloverust | |
| AWS_ENDPOINT_URL: http://localhost:4566 | |
| AWS_ALLOW_HTTP: "1" | |
| AZURE_USE_EMULATOR: "1" | |
| AZURE_STORAGE_ALLOW_HTTP: "1" | |
| AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" | |
| AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;" | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/sccache-action@v0.0.9 | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| cache: true | |
| - uses: taiki-e/install-action@nextest | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Run tests with native-tls | |
| run: | | |
| gmake setup-dat | |
| cargo nextest r --cargo-profile=ci \ | |
| --package deltalake-aws \ | |
| --no-default-features \ | |
| --features integration_test,native-tls | |
| hdfs: | |
| name: Integration Tests (HDFS) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/sccache-action@v0.0.9 | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| cache: true | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - uses: taiki-e/install-action@nextest | |
| # Install Java and Hadoop for HDFS integration tests | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: "temurin" | |
| java-version: "17" | |
| - name: Download Hadoop | |
| run: | | |
| wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.4.0/hadoop-3.4.0.tar.gz | |
| tar -xf hadoop-3.4.0.tar.gz -C $GITHUB_WORKSPACE | |
| echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH | |
| - name: Run tests with rustls (default) | |
| run: | | |
| gmake setup-dat | |
| cargo llvm-cov nextest --cargo-profile=ci \ | |
| --features integration_test \ | |
| --package deltalake-hdfs \ | |
| --codecov \ | |
| --output-path codecov.json | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| lakefs: | |
| name: Integration Tests (LakeFS v1.48) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/sccache-action@v0.0.9 | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| cache: true | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - uses: taiki-e/install-action@nextest | |
| - name: Download Lakectl | |
| run: | | |
| wget -q https://github.com/treeverse/lakeFS/releases/download/v1.48.1/lakeFS_1.48.1_Linux_x86_64.tar.gz | |
| tar -xf lakeFS_1.48.1_Linux_x86_64.tar.gz -C $GITHUB_WORKSPACE | |
| echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH | |
| - name: Start emulated services | |
| run: docker compose -f docker-compose-lakefs.yml up -d | |
| - name: Run tests | |
| run: | | |
| gmake setup-dat | |
| cargo llvm-cov nextest --cargo-profile=ci \ | |
| --package deltalake-lakefs \ | |
| --features integration_test_lakefs \ | |
| --codecov \ | |
| --output-path codecov.json | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} |