Repository navigation
Pytorch backend: analysis #4716
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Presubmit | |
| on: | |
| push: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| type: boolean | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: false | |
| env: | |
| MAKEFLAGS: "-j$(nproc) -O --shuffle" | |
| CC: "gcc" | |
| TEST_EXCLUDES: "--filter-tags !todo" | |
| jobs: | |
| pre_job: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| should_skip: ${{ steps.skip_check.outputs.should_skip }} | |
| steps: | |
| - id: skip_check | |
| uses: fkirc/skip-duplicate-actions@master | |
| with: | |
| concurrent_skipping: 'same_content' | |
| skip_after_successful_duplicate: 'true' | |
| paths_ignore: '["**/README.md"]' | |
| do_not_skip: '["pull_request"]' | |
| if: ${{ github.event_name != 'workflow_dispatch' }} | |
| efficios_dep: | |
| needs: pre_job | |
| if: ${{ needs.pre_job.outputs.should_skip != 'true' }} | |
| name: Build and Cache Efficios Dependencies | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-thapi | |
| build-and-check: | |
| needs: efficios_dep | |
| runs-on: ubuntu-24.04 | |
| name: Build and Check on ubuntu-24.04 with ${{ matrix.compilers }} | |
| strategy: | |
| matrix: | |
| compilers: ["CC=gcc CXX=g++", "CC=clang CXX=clang++ CFLAGS=\"-Wno-gnu-folding-constant\""] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-thapi | |
| - run: mkdir -p build | |
| - run: ./autogen.sh | |
| - run: ../configure ${{ matrix.compilers }} | |
| working-directory: build | |
| - run: make | |
| working-directory: build | |
| - run: make check | |
| working-directory: build | |
| env: | |
| THAPI_VALGRIND: 1 | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| path: | | |
| build/**/*.log | |
| build/config.log | |
| build/**/tests/*.log | |
| install-with-mpi: | |
| needs: efficios_dep | |
| name: Install with MPI daemon support | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: mpi4py/setup-mpi@v1 | |
| with: | |
| mpi: intelmpi | |
| - uses: ./.github/actions/setup-thapi | |
| - run: mkdir -p build | |
| - run: ./autogen.sh | |
| - run: ../configure --prefix=`pwd`/ici | |
| working-directory: build | |
| - run: make install | |
| working-directory: build | |
| - name: Tar THAPI # https://github.com/actions/download-artifact?tab=readme-ov-file#permission-loss | |
| run: tar -cvf thapi.tar ./build/ici/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: thapi-bin | |
| path: thapi.tar | |
| integration-tests: | |
| needs: install-with-mpi | |
| name: Integration ${{ matrix.bats_file }} ${{matrix.thapi_sync_daemon }} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: mpi4py/setup-mpi@v1 | |
| with: | |
| mpi: intelmpi | |
| - uses: ./.github/actions/setup-thapi | |
| - run: sudo apt-get update | |
| - uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: bats coreutils libpocl2 clinfo | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: thapi-bin | |
| - name: Untar THAPI | |
| run: tar -xvf thapi.tar | |
| # Build ITTAPI from source and expose ITTAPI_ROOT + runtime libs for ITT tests | |
| - name: Build ITTAPI | |
| run: | | |
| set -eux | |
| git clone --depth=1 https://github.com/intel/ittapi.git | |
| cmake -S ittapi -B ittapi/build \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DCMAKE_INSTALL_PREFIX="$GITHUB_WORKSPACE/ittapi/install" | |
| cmake --build ittapi/build --target install -j"$(nproc)" | |
| echo "ITTAPI_ROOT=$GITHUB_WORKSPACE/ittapi/install" >> "$GITHUB_ENV" | |
| # Install Python ittapi for ITT tests | |
| - name: Install Python ittapi | |
| run: | | |
| python3 -m pip install --user -U pip | |
| python3 -m pip install --user -U ittapi | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Install PyTorch (CPU) | |
| run: | | |
| python3 -m pip install --user -U pip | |
| python3 -m pip install --user -U torch==2.14.0 --index-url https://download.pytorch.org/whl/cpu | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| # --- diagnostics: which python3 the iprof probe uses, and whether the tracer can resolve against it --- | |
| echo "== which python3 =="; which -a python3 | |
| echo "== python3 --version =="; python3 --version | |
| echo "== probe as iprof runs it =="; python3 -c "import torch; print(torch.__file__); print(torch.__version__)" | |
| TORCH_LIB="$(python3 -c 'import torch,os; print(os.path.join(os.path.dirname(torch.__file__),"lib"))')" | |
| echo "TORCH_LIB=$TORCH_LIB"; ls -la "$TORCH_LIB"/libtorch_cpu.so | |
| echo "== does libtorch_cpu.so export at::addGlobalCallback? ==" | |
| nm -D "$TORCH_LIB"/libtorch_cpu.so | grep 17addGlobalCallback || echo "NOT EXPORTED / not found in dynsym" | |
| # timeout-minutes converts a hang (e.g. the pytorch tracer deadlocking the | |
| # tracee) into a step failure, so the diagnostic step below actually runs. | |
| - name: Integration test | |
| timeout-minutes: 2 | |
| run: | | |
| export PKG_CONFIG_PATH=./build/ici/lib/pkgconfig/:${PKG_CONFIG_PATH} | |
| bats ${TEST_EXCLUDES} integration_tests/ | |
| # Run the tests twice to make sure we do a proper cleanup | |
| bats ${TEST_EXCLUDES} integration_tests/ | |
| - name: Diagnose pytorch tracer load on failure | |
| if: ${{ failure() }} | |
| timeout-minutes: 2 | |
| run: | | |
| export PKG_CONFIG_PATH=./build/ici/lib/pkgconfig/:${PKG_CONFIG_PATH} | |
| LT="$(python3 -c 'import torch,os; print(os.path.join(os.path.dirname(torch.__file__),"lib","libtorch_cpu.so"))')" | |
| echo "== reproduce failing command (15s cap; 124 = timed out/hung) ==" | |
| timeout 15 iprof -- bash -c "exit 55" && rc=$? || rc=$?; echo "iprof exit=$rc" | |
| echo "== which libtorch_cpu.so binds + any undefined symbol at load ==" | |
| LD_DEBUG=libs,bindings,reloc timeout 15 iprof -- bash -c "exit 55" 2>&1 \ | |
| | grep -iE "libtorch_cpu|addGlobalCallback|error|not found|undefined" | head -60 || true | |
| echo "== do libtorch_cpu.so's own deps resolve? ==" | |
| ldd "$LT" | grep -i "not found" || echo "all deps of libtorch_cpu.so resolve" | |
| echo "== isolated preload (no iprof env-building; 15s cap) ==" | |
| LD_LIBRARY_PATH="$(dirname "$LT"):$LD_LIBRARY_PATH" \ | |
| LD_PRELOAD="$(pkg-config --variable=libdir thapi)/libTracerPytorch.so" \ | |
| timeout 15 bash -c "exit 55" && rc=$? || rc=$?; echo "isolated exit=$rc" | |
| # To get an SSH shell on the runner in the failed state: | |
| # GitHub -> Actions -> Presubmit -> "Run workflow" -> tick "debug_enabled" -> Run. | |
| # On failure this pauses the job and prints an "ssh ...@nyc1.tmate.io" line in the log. | |
| # Connect with that command; access is limited to the actor who launched the run. | |
| - name: Debug over SSH on failure (tmate) | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ failure() && github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} | |
| with: | |
| limit-access-to-actor: true | |
| build-in-tree-and-check: | |
| needs: efficios_dep | |
| name: Build in Tree ubuntu-24.04 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-thapi | |
| - run: ./autogen.sh | |
| - run: ./configure | |
| - run: make | |
| - run: make check | |
| env: | |
| THAPI_VALGRIND: 1 | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: build-in-tree | |
| path: | | |
| ./**/*.log | |
| ./config.log | |
| ./**/tests/*.log | |
| distcheck: | |
| needs: efficios_dep | |
| name: Distcheck ubuntu-24.04 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-thapi | |
| - run: mkdir -p build | |
| - run: ./autogen.sh | |
| - run: ../configure | |
| working-directory: build | |
| - run: make distcheck | |
| working-directory: build | |
| env: | |
| THAPI_VALGRIND: 1 | |
| dist-check: | |
| needs: efficios_dep | |
| name: Dist and Check ubuntu-24.04 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-thapi | |
| - run: mkdir -p build | |
| - run: ./autogen.sh | |
| - run: ../configure | |
| working-directory: build | |
| - run: make dist | |
| working-directory: build | |
| - name: Extract | |
| run: tar -xzvf thapi-* | |
| working-directory: build | |
| - name: Configure | |
| run: | | |
| cd `find . -type d -name "thapi-*"` | |
| mkdir -p build | |
| cd build | |
| ../configure | |
| working-directory: build | |
| - name: Check | |
| run: | | |
| cd `find . -type d -name "thapi-*"` | |
| cd build | |
| make check | |
| working-directory: build | |
| env: | |
| THAPI_VALGRIND: 1 | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: dist-check | |
| path: | | |
| build/**/*.log | |
| build/config.log | |
| build/**/tests/*.log | |
| check-same-generated-files: | |
| needs: efficios_dep | |
| name: Assert generated files are equivalent | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-thapi | |
| - run: pip install pyyaml pytest deepdiff | |
| # Chekout, Configure and build target branch files | |
| - name: Checkout target branch | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.base_ref }} | |
| path: base_branch | |
| - name: Configure target branch | |
| working-directory: base_branch | |
| run: &configure | | |
| ./autogen.sh | |
| mkdir -p build | |
| cd build | |
| ../configure | |
| - name: Build target branch files | |
| working-directory: base_branch/build | |
| run: &build-commands | | |
| make -C utils lttng/tracepoint_gen.h | |
| make -C backends/mpi tracer_mpi.c mpi_library.rb btx_mpi_model.yaml libmpi.la | |
| make -C backends/omp tracer_ompt.c omp_library.rb btx_omp_model.yaml | |
| make -C backends/hip tracer_hip.c hip_library.rb btx_hip_model.yaml libamdhip64.la | |
| make -C backends/cuda tracer_cuda.c cuda_library.rb btx_cuda_model.yaml libcuda.la | |
| make -C backends/cuda tracer_cudart.c libcudart.la | |
| make -C backends/ze tracer_ze.c ze_library.rb btx_ze_model.yaml libze_loader.la | |
| make -C backends/opencl opencl_model.yaml btx_cl_model.yaml libOpenCL.la | |
| make -C backends/cxi cxi_sampling.tp | |
| make -C backends/itt tracer_itt.c itt_library.rb btx_itt_model.yaml libittnotify.la | |
| make -C backends/opencl opencl_profiling.tp | |
| make -C sampling sampling.tp | |
| # Chekout, Configure and build PR branch files | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| path: pr_branch | |
| - name: Configure PR branch | |
| working-directory: pr_branch | |
| run: *configure | |
| - name: Build PR branch files | |
| working-directory: pr_branch/build | |
| run: *build-commands | |
| # Run comparison script | |
| - name: Compare files | |
| run: | | |
| THAPI_REF="base_branch/build" THAPI_NEW="pr_branch/build" pytest -vv pr_branch/utils/ |