WIP: [ci] move CI container images to GHCR, workflows to this repo

.ci/ci-images/manylinux_2_28_aarch64/.dockerignore

@@ -0,0 +1 @@
+*

.ci/ci-images/manylinux_2_28_aarch64/Dockerfile

@@ -0,0 +1,73 @@
+FROM quay.io/pypa/manylinux_2_28_aarch64
+
+# use 'bash' for RUN steps
+SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
+
+# install packages
+RUN <<EOF
+yum update -y
+
+yum install --nodocs -y \
+   epel-release
+
+yum install --nodocs -y \
+   clang-devel \
+   gcc-c++ \
+   hwloc-devel \
+   llvm-devel \
+   llvm-static \
+   ocl-icd-devel \
+   sudo
+
+yum module install --nodocs -y \
+   llvm-toolset
+
+yum clean all
+
+rm -rf /var/cache/yum
+EOF
+
+RUN <<EOF
+# install a newer CMake than what the package manager has
+curl -sL https://cmake.org/files/v3.23/cmake-3.23.1-linux-aarch64.sh -o cmake.sh
+chmod +x cmake.sh
+./cmake.sh --prefix=/usr/local --exclude-subdir
+rm -f ./cmake.sh
+
+# build PoCL
+#
+# NOTE: If this is updated, check if CL_TARGET_OPENCL_VERSION in cmake/IntegratedOpenCL.cmake
+#       needs to be updated (see comments there fore links).
+git clone \
+   --depth 1 \
+   --branch v7.1 \
+   https://github.com/pocl/pocl.git
+
+# explanations for some flags:
+#
+#   * -DCMAKE_{C,CXX}_COMPILER: DEVTOOLSET_ROOTPATH is where manylinux puts the gcc toolset
+#   * -DLLC_HOST_CPU="generic": passed to clang's -march/-mcpu flag.
+#
+cmake \
+   -B pocl/build \
+   -S pocl \
+   -DCMAKE_BUILD_TYPE=release \
+   -DCMAKE_C_COMPILER="${DEVTOOLSET_ROOTPATH}/usr/bin/gcc" \
+   -DCMAKE_CXX_COMPILER="${DEVTOOLSET_ROOTPATH}/usr/bin/g++" \
+   -DENABLE_DOXYGEN=OFF \
+   -DENABLE_EXAMPLES=OFF \
+   -DENABLE_HOST_CPU_DEVICES=ON \
+   -DENABLE_HWLOC=ON \
+   -DENABLE_POCLCC=ON \
+   -DENABLE_SPIRV=ON \
+   -DENABLE_TESTS=OFF \
+   -DENABLE_VALGRIND=OFF \
+   -DINSTALL_OPENCL_HEADERS=OFF \
+   -DLLC_HOST_CPU=generic \
+   -DPOCL_DEBUG_MESSAGES=OFF \
+   -DPOCL_INSTALL_ICD_VENDORDIR=/etc/OpenCL/vendors
+
+cmake --build pocl/build -j4
+
+cmake --install pocl/build
+EOF

.ci/ci-images/manylinux_2_28_x86_64/.dockerignore

@@ -0,0 +1 @@
+*

.ci/ci-images/manylinux_2_28_x86_64/Dockerfile

@@ -0,0 +1,102 @@
+FROM quay.io/pypa/manylinux_2_28_x86_64
+
+# ensure that libraries like libc++ built in this image can be found by the linker
+#ENV LD_LIBRARY_PATH="/usr/local/lib64:${LD_LIBRARY_PATH}:/usr/local/lib"
+
+# use 'bash' for RUN steps
+SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
+
+# install packages
+RUN <<EOF
+yum update -y
+
+yum install --nodocs -y \
+   epel-release
+
+yum install --nodocs -y \
+   clang-devel \
+   gcc-c++ \
+   hwloc-devel \
+   llvm-devel \
+   llvm-static \
+   ocl-icd-devel \
+   sudo
+
+yum module install --nodocs -y \
+   llvm-toolset
+
+yum clean all
+
+rm -rf /var/cache/yum
+EOF
+
+RUN <<EOF
+# install a newer CMake than what the package manager has
+curl -sL https://cmake.org/files/v3.23/cmake-3.23.1-linux-$(arch).sh -o cmake.sh
+chmod +x cmake.sh
+./cmake.sh --prefix=/usr/local --exclude-subdir
+rm -f ./cmake.sh
+
+# build PoCL
+git clone \
+   --depth 1 \
+   --branch v7.1 \
+   https://github.com/pocl/pocl.git
+
+# explanations for some flags:
+#
+#   * -DCMAKE_{C,CXX}_COMPILER: DEVTOOLSET_ROOTPATH is where manylinux puts the gcc toolset
+#   * -DLLC_HOST_CPU="x86_64": passed to clang's -march/-mcpu flag. see https://github.com/chromebrew/chromebrew/pull/9176#issuecomment-1891751465
+#
+cmake \
+   -B pocl/build \
+   -S pocl \
+   -DCMAKE_BUILD_TYPE=release \
+   -DCMAKE_C_COMPILER="${DEVTOOLSET_ROOTPATH}/usr/bin/gcc" \
+   -DCMAKE_CXX_COMPILER="${DEVTOOLSET_ROOTPATH}/usr/bin/g++" \
+   -DENABLE_DOXYGEN=OFF \
+   -DENABLE_EXAMPLES=OFF \
+   -DENABLE_HOST_CPU_DEVICES=ON \
+   -DENABLE_HWLOC=ON \
+   -DENABLE_POCLCC=ON \
+   -DENABLE_SPIRV=ON \
+   -DENABLE_TESTS=OFF \
+   -DENABLE_VALGRIND=OFF \
+   -DINSTALL_OPENCL_HEADERS=OFF \
+   -DLLC_HOST_CPU="x86-64" \
+   -DPOCL_DEBUG_MESSAGES=OFF \
+   -DPOCL_INSTALL_ICD_VENDORDIR=/etc/OpenCL/vendors
+
+cmake --build pocl/build -j4
+
+cmake --install pocl/build
+EOF
+
+# Install Java
+RUN yum install -y \
+      java-1.8.0-openjdk-devel.x86_64 \
+ && yum clean all
+
+ENV JAVA_HOME_8_X64=/usr/lib/jvm/java
+ENV JAVA_HOME=$JAVA_HOME_8_X64
+
+# Install SWIG
+RUN curl -sLk https://sourceforge.net/projects/swig/files/swig/swig-4.0.2/swig-4.0.2.tar.gz/download -o swig.tar.gz \
+ && tar -xzf swig.tar.gz \
+ && cd swig-4.0.2 \
+ && ./configure --prefix=/usr/local --without-pcre \
+ && make \
+ && make install \
+ && cd .. \
+ && rm -f ./swig.tar.gz \
+ && rm -rf ./swig-4.0.2
+
+# Install miniforge
+RUN curl -sL "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh" -o miniforge.sh \
+ && chmod +x miniforge.sh \
+ && ./miniforge.sh -b -p /opt/miniforge \
+ && rm -f ./miniforge.sh \
+ && /opt/miniforge/bin/conda clean -a -y \
+ && chmod -R 777 /opt/miniforge
+
+ENV CONDA=/opt/miniforge/

.ci/setup.sh

@@ -125,13 +125,16 @@ else  # Linux
             sudo apt-get update
             sudo apt-get install --no-install-recommends -y \
                 pocl-opencl-icd
-        elif [[ $(uname -m) == "x86_64" ]]; then
+        else # in manylinux image
             sudo yum update -y
             sudo yum install -y \
+                clinfo \
                 ocl-icd-devel \
                 opencl-headers \
             || exit 1
         fi
+        echo "--- clinfo: ---"
+        clinfo || true
     fi
     if [[ $TASK == "cuda" ]]; then
         echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

.ci/test.sh

@@ -140,11 +140,7 @@ elif [[ $TASK == "bdist" ]]; then
             cp "$(echo "dist/lightgbm-${LGB_VER}-py3-none-macosx"*.whl)" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
         fi
     else
-        if [[ $ARCH == "x86_64" ]]; then
-            PLATFORM="manylinux_2_28_x86_64"
-        else
-            PLATFORM="manylinux2014_$ARCH"
-        fi
+        PLATFORM="manylinux_2_28_$ARCH"
         sh ./build-python.sh bdist_wheel --integrated-opencl || exit 1
         # rename wheel, to fix scikit-build-core choosing the platform 'linux_aarch64' instead of
         # a manylinux tag
@@ -154,7 +150,8 @@ elif [[ $TASK == "bdist" ]]; then
         mv \
             ./dist/tmp.whl \
             "./dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" || exit 1
-        sh .ci/check-python-dists.sh ./dist || exit 1
+        # TODO(jameslamb): re-enable these checks before merging
+        #sh .ci/check-python-dists.sh ./dist || exit 1
         if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
             cp "dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
         fi

.github/workflows/ci-images.yml

@@ -0,0 +1,63 @@
+name: ci-images
+
+on:
+  # TODO(jameslamb): remove 'push' trigger before merging
+  push:
+  workflow_dispatch:
+    inputs:
+      tag-suffix:
+        type: string
+        default: "-dev"
+        description: |
+          Suffix for image tags, including a leading "-" if desired.
+          Set to the empty string to overwrite the main images used by LightGBM's CI.
+
+jobs:
+  build-and-push-images:
+    name: build-ci-images (${{ matrix.tag }})
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    strategy:
+      fail-fast: false
+      # NOTE: Cannot use "{{ github.repository }}" because that'd return "{org}/LightGBM" which results in
+      #       and error like "repository name must be lowercase".
+      matrix:
+        include:
+          - os: ubuntu-24.04-arm
+            dockerfile: .ci/ci-images/manylinux_2_28_aarch64/Dockerfile
+            # TODO(jameslamb): revert hard-coded tag before merging
+            tag: ci-manylinux_2_28_aarch64-dev
+            # tag: ci-manylinux_2_28_aarch64${{ inputs.tag-suffix }}
+          - os: ubuntu-latest
+            dockerfile: .ci/ci-images/manylinux_2_28_x86_64/Dockerfile
+            # TODO(jameslamb): revert hard-coded tag before merging
+            tag: ci-manylinux_2_28_x86_64-dev
+            # tag: ci-manylinux_2_28_x86_64${{ inputs.tag-suffix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+      - name: Log in to the Container registry
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1  # v3.5.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83  # v6.18.0
+        with:
+          context: null
+          file: ${{ matrix.dockerfile }}
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/lightgbm:${{ matrix.tag }}
+      # create an attestation to prove that this image came from a workflow in this repository
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be  # v2.4.0
+        with:
+          subject-name: ghcr.io/${{ github.repository_owner }}/lightgbm
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

.github/workflows/cpp.yml

@@ -74,7 +74,7 @@ jobs:
             task: if-else
             compiler: gcc
             python_version: '3.11'
-            container: 'lightgbm.azurecr.io/vsts-agent:manylinux_2_28_x86_64'
+            container: 'ghcr.io/microsoft/lightgbm:ci-manylinux_2_28_x86_64-dev'
             os-display-name: 'manylinux_2_28'
     steps:
       - name: Install packages used by third-party actions