diff --git a/.github/workflows/amdsmi-build.yml b/.github/workflows/amdsmi-build.yml index 95381641b42..612691c9058 100644 --- a/.github/workflows/amdsmi-build.yml +++ b/.github/workflows/amdsmi-build.yml @@ -22,151 +22,53 @@ env: ROCM_DIR: /opt/rocm jobs: - debian-buildinstall: - name: Build - runs-on: - - self-hosted - - ${{ vars.RUNNER_TYPE }} - continue-on-error: true - strategy: - max-parallel: 10 - matrix: - os: [Ubuntu20, Ubuntu22, Debian10] + manylinux-build: + name: Build (Manylinux Gate) + runs-on: ubuntu-latest container: - image: ${{ vars[format('{0}_DOCKER_IMAGE', matrix.os)] }} - options: --rm --privileged --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size=64G --cap-add=SYS_MODULE -v /lib/modules:/lib/modules -u root + image: quay.io/pypa/manylinux_2_28_x86_64 steps: - uses: actions/checkout@v4 - - name: Set Artifact Metadata - if: github.event_name == 'pull_request' - run: | - # Set PR number and date for artifact naming - echo "PR_NUMBER=PR${{ github.event.pull_request.number }}" >> $GITHUB_ENV - # Set date in MMDDYY-HHMM format (UTC time) - echo "BUILD_DATE=$(date -u +%m%d%y-%H%M)" >> $GITHUB_ENV - - - name: Set Project Directory - run: | - # Find the directory containing the main CMakeLists.txt for AMDSMI - TARGET_DIR=$(find $GITHUB_WORKSPACE -path "*/projects/amdsmi/CMakeLists.txt" -exec dirname {} \;) - - if [ -z "$TARGET_DIR" ]; then - echo "Could not find CMakeLists.txt in projects/amdsmi. Searching root..." - TARGET_DIR=$(find $GITHUB_WORKSPACE -maxdepth 2 -name "CMakeLists.txt" -exec dirname {} \; | head -n 1) - fi - - echo "PROJECT_DIR=$TARGET_DIR" >> $GITHUB_ENV - - - name: Update repositories for Debian10 - if: matrix.os == 'Debian10' - run: | - set -e - echo 'Updating repositories for Debian10 (archived)' - cat > /etc/apt/sources.list << EOF - deb http://archive.debian.org/debian buster main - deb http://archive.debian.org/debian-security buster/updates main - EOF - echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99-disable-check-valid-until - apt update + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Build AMDSMI + shell: bash run: | - set -e - echo 'Building on ${{ matrix.os }}' - BUILD_FOLDER=${{ env.PROJECT_DIR }}/build - RETRIES=3 - - for i in $(seq 1 $RETRIES); do - echo "Build attempt $i for ${{ matrix.os }}..." - rm -rf $BUILD_FOLDER - mkdir -p $BUILD_FOLDER - cd $BUILD_FOLDER - - # Configure, build, and package - if cmake ${{ env.PROJECT_DIR }} -DBUILD_TESTS=ON -DENABLE_ESMI_LIB=ON 2>&1 | tee cmake.log && \ - make -j $(nproc) 2>&1 | tee make.log && \ - make package 2>&1 | tee package.log; then - - # Parse and report warnings as GitHub annotations - echo "::group::Build Warnings" - grep -i "warning" cmake.log make.log package.log | while read -r line; do - echo "::warning::$line" - done - echo "::endgroup::" - - echo "Build successful on attempt $i" - break - else - echo "Build failed on attempt $i" - if [ $i -eq $RETRIES ]; then - echo "All $RETRIES build attempts failed. Exiting." - exit 1 - fi - sleep $((2 * i)) - fi - done - echo "Build completed on ${{ matrix.os }}" - - - name: Upload Debian Package Artifacts - if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v4 - with: - name: amd-smi-lib-deb-${{ matrix.os }}-${{ env.PR_NUMBER }}-${{ env.BUILD_DATE }} - path: ${{ env.PROJECT_DIR }}/build/amd-smi-lib*99999-local_amd64.deb - if-no-files-found: warn - retention-days: 7 - - - name: Install AMDSMI - run: | - cd ${{ env.PROJECT_DIR }}/build - if [ "${{ matrix.os }}" != "Debian10" ]; then - apt update - fi - - RETRIES=3 - for i in $(seq 1 $RETRIES); do - echo "Installation attempt $i for ${{ matrix.os }}..." - if apt install -y ./amd-smi-lib*99999-local_amd64.deb; then - echo "Installation successful on attempt $i" - ln -s /opt/rocm/bin/amd-smi /usr/local/bin - - # Verify Installation - echo 'Verifying installation:' - amd-smi version - python3 -m pip list | grep amd - python3 -m pip list | grep pip - python3 -m pip list | grep setuptools - echo 'Completed installation on ${{ matrix.os }}' - break - else - echo "Installation failed on attempt $i" - if [ $i -eq $RETRIES ]; then - echo "All $RETRIES installation attempts failed. Exiting." - exit 1 - fi - sleep $((2 * i)) - fi - done - echo "Build completed on ${{ matrix.os }}" + set -euo pipefail + PROJECT_DIR=$GITHUB_WORKSPACE/projects/amdsmi + BUILD_DIR=/tmp/amdsmi-build + PY_BUILD=/opt/python/cp38-cp38/bin/python3 + + echo "Installing build prerequisites..." + dnf -y install git make gcc gcc-c++ cmake ninja-build openssl-devel + + echo "Configuring and building AMDSMI..." + rm -rf "$BUILD_DIR" + mkdir -p "$BUILD_DIR" + cd "$BUILD_DIR" + cmake "$PROJECT_DIR" \ + -DBUILD_TESTS=ON \ + -DENABLE_ESMI_LIB=ON \ + -DBUILD_PYTHON_LIB=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython3_EXECUTABLE="$PY_BUILD" + make -j"$(nproc)" + + echo "Building Python wheel..." + $PY_BUILD -m pip install --upgrade pip setuptools wheel + cd "$BUILD_DIR/py-interface/python_package" + rm -rf build dist *.whl *.egg-info + $PY_BUILD -m pip wheel --no-deps --no-build-isolation -w . . + echo "Python wheel built: $(ls *.whl)" + echo "Build gate passed" - - name: Uninstall - if: always() - run: | - set -e - echo 'Uninstalling on ${{ matrix.os }}' - apt remove -y amd-smi-lib || true - rm -f /usr/local/bin/amd-smi - if [ -d /opt/rocm/share/amd_smi ]; then - echo '/opt/rocm/share/amd_smi exists. Removing.' - rm -rf /opt/rocm/share/amd_smi - fi - echo 'Uninstall done on ${{ matrix.os }}' debian-test: name: Tests - needs: debian-buildinstall + needs: manylinux-build runs-on: - self-hosted - ${{ vars.RUNNER_TYPE }} @@ -174,14 +76,16 @@ jobs: strategy: max-parallel: 10 matrix: - os: [Ubuntu20, Ubuntu22, Debian10] + os: [Ubuntu20, Ubuntu22, Ubuntu24, Debian10] container: image: ${{ vars[format('{0}_DOCKER_IMAGE', matrix.os)] }} options: --rm --privileged --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size=64G --cap-add=SYS_MODULE -v /lib/modules:/lib/modules -u root steps: - uses: actions/checkout@v4 - + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Set Project Directory run: | TARGET_DIR=$(find $GITHUB_WORKSPACE -path "*/projects/amdsmi/CMakeLists.txt" -exec dirname {} \;) @@ -202,7 +106,23 @@ jobs: echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99-disable-check-valid-until apt update + - name: Upgrade setuptools for wheel build + if: matrix.os != 'AzureLinux3' + run: | + echo 'Upgrading setuptools for proper wheel metadata' + python3 -m pip install --upgrade pip setuptools wheel + + - name: Clean stale ROCm Python artifacts + run: | + # Remove SWIG-based libamd_smi_python.so that may be baked into the + # Docker image. The package now uses ctypes; the stale extension + # references symbols removed from libamd_smi.so and causes + # "undefined symbol" errors when importing the amdsmi package. + rm -f /opt/rocm/share/amd_smi/amdsmi/libamd_smi_python.so 2>/dev/null || true + ldconfig 2>/dev/null || true + - name: Build and Install for Test + shell: bash run: | set -e echo 'Building for test on ${{ matrix.os }}' @@ -215,9 +135,15 @@ jobs: mkdir -p $BUILD_FOLDER cd $BUILD_FOLDER - if cmake ${{ env.PROJECT_DIR }} -DBUILD_TESTS=ON -DENABLE_ESMI_LIB=ON && \ + if cmake ${{ env.PROJECT_DIR }} -DBUILD_TESTS=ON -DENABLE_ESMI_LIB=ON -DBUILD_PYTHON_LIB=ON && \ make -j $(nproc) && \ + cd $BUILD_FOLDER/py-interface/python_package && \ + rm -rf *.whl *.egg-info build dist && \ + python3 -m pip wheel --no-deps --no-build-isolation -w . . && \ + echo "Python wheel built: $(ls *.whl)" && \ + cd $BUILD_FOLDER && \ make package; then + echo "Build successful on attempt $i" break else @@ -248,6 +174,52 @@ jobs: fi done + + - name: Verify Wheel in Site-Packages + shell: bash + run: | + set -e + echo "=== Wheel site-packages verification on ${{ matrix.os }} ===" + BUILD_FOLDER=${{ env.PROJECT_DIR }}/build + WHEEL=$(ls $BUILD_FOLDER/py-interface/python_package/*.whl 2>/dev/null | head -1) + + if [ -z "$WHEEL" ]; then + echo "ERROR: No .whl file found in $BUILD_FOLDER/py-interface/python_package/" + exit 1 + fi + echo "Found wheel: $WHEEL" + + # Install the wheel into site-packages + echo "Installing wheel via pip..." + python3 -m pip install --force-reinstall "$WHEEL" + + # Verify it landed in site-packages + INSTALL_PATH=$(python3 -c "import amdsmi; print(amdsmi.__file__)") + echo "amdsmi installed at: $INSTALL_PATH" + + if echo "$INSTALL_PATH" | grep -q "site-packages"; then + echo "PASS: Wheel is correctly installed in site-packages" + else + echo "WARNING: amdsmi found at $INSTALL_PATH (not in site-packages)" + fi + + # Verify package metadata + echo "Package info:" + python3 -m pip show amdsmi + + # Smoke-test the import + cd /tmp + python3 -c " + import amdsmi + print('PASS: import amdsmi OK') + amdsmi.amdsmi_init() + print('PASS: amdsmi_init() OK') + devs = amdsmi.amdsmi_get_processor_handles() + print('PASS: Found %d device(s)' % len(devs)) + amdsmi.amdsmi_shut_down() + print('PASS: amdsmi_shut_down() OK') + print('=== Wheel verification passed ===') + " - name: AMDSMI Command Tests shell: bash run: | @@ -354,6 +326,17 @@ jobs: echo "Unit tests passed" fi + echo "Running perf tests..." + if ! ./perf_tests.py -v > /tmp/test-results-${{ matrix.os }}/perf_test_output.txt 2>&1; then + echo "Perf tests failed!" + echo "=============== PERF TEST OUTPUT ===============" + tail -100 /tmp/test-results-${{ matrix.os }}/perf_test_output.txt + echo "=================================================" + exit 1 + else + echo "Perf tests passed" + fi + echo "Python tests done" # Example Tests @@ -385,6 +368,12 @@ jobs: echo "Displaying Unit Test Results for ${{ matrix.os }}" cat /tmp/test-results-${{ matrix.os }}/unit_test_output.txt || echo "No unit test results found for ${{ matrix.os }}" + - name: Perf Test Results + if: always() + run: | + echo "Displaying Perf Test Results for ${{ matrix.os }}" + cat /tmp/test-results-${{ matrix.os }}/perf_test_output.txt || echo "No perf test results found for ${{ matrix.os }}" + - name: Example DRM Test Results if: always() run: | @@ -397,8 +386,9 @@ jobs: echo "Displaying Example NoDRM test results for ${{ matrix.os }}" cat /tmp/test-results-${{ matrix.os }}/amd_smi_nodrm_ex.log || echo "No NoDRM example test results found for ${{ matrix.os }}" - rpm-buildinstall: - name: Build + rpm-test: + name: Tests + needs: [manylinux-build, debian-test] runs-on: - self-hosted - ${{ vars.RUNNER_TYPE }} @@ -420,13 +410,8 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set Artifact Metadata - if: github.event_name == 'pull_request' - run: | - # Set PR number and date for artifact naming - echo "PR_NUMBER=PR${{ github.event.pull_request.number }}" >> $GITHUB_ENV - # Set date in MMDDYY-HHMM format (UTC time) - echo "BUILD_DATE=$(date -u +%m%d%y-%H%M)" >> $GITHUB_ENV + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Set Project Directory run: | @@ -455,236 +440,24 @@ jobs: echo 'Installing more_itertools on ${{ matrix.os }}' python3 -m pip install more_itertools - - name: Build AMDSMI(RHEL10 & AlmaLinux8) - if: matrix.os == 'RHEL10' || matrix.os == 'AlmaLinux8' + - name: Upgrade setuptools for wheel build + if: matrix.os != 'AzureLinux3' run: | - set -e - echo 'Building on ${{ matrix.os }} with retries and QA_RPATHS' - BUILD_FOLDER=${{ env.PROJECT_DIR }}/build - RETRIES=5 - - # Set QA_RPATHS to ignore empty (0x0010) and invalid (0x0002) RPATHs - export QA_RPATHS=$((0x0010 | 0x0002)) + echo 'Upgrading setuptools for proper wheel metadata' + python3 -m pip install --upgrade pip setuptools wheel - for i in $(seq 1 $RETRIES); do - echo "Build attempt $i for ${{ matrix.os }} ..." - rm -rf $BUILD_FOLDER - mkdir -p $BUILD_FOLDER - cd $BUILD_FOLDER - - if cmake ${{ env.PROJECT_DIR }} -DBUILD_TESTS=ON -DENABLE_ESMI_LIB=ON && \ - make -j $(nproc) && \ - make package; then - echo "Build successful on attempt $i" - break - else - echo "Build failed on attempt $i" - if [ $i -eq $RETRIES ]; then - echo "All $RETRIES build attempts failed. Exiting." - exit 1 - fi - sleep $((2 * i)) - fi - done - echo "Build completed on ${{ matrix.os }}" - - - name: Upload RPM Package Artifacts (RHEL10 & AlmaLinux8) - if: github.event_name == 'pull_request' && (matrix.os == 'RHEL10' || matrix.os == 'AlmaLinux8') - uses: actions/upload-artifact@v4 - with: - name: amd-smi-lib-rpm-${{ matrix.os }}-${{ env.PR_NUMBER }}-${{ env.BUILD_DATE }} - path: ${{ env.PROJECT_DIR }}/build/amd-smi-lib-*99999-local*.rpm - if-no-files-found: warn - retention-days: 7 - - - name: Build AMDSMI - if: matrix.os != 'RHEL10' && matrix.os != 'AlmaLinux8' + - name: Clean stale ROCm Python artifacts run: | - set -e - echo 'Building on ${{ matrix.os }}' - BUILD_FOLDER=${{ env.PROJECT_DIR }}/build - RETRIES=3 - - for i in $(seq 1 $RETRIES); do - echo "Build attempt $i for ${{ matrix.os }}..." - rm -rf $BUILD_FOLDER - mkdir -p $BUILD_FOLDER - cd $BUILD_FOLDER - - # Capture build output to parse warnings - if cmake ${{ env.PROJECT_DIR }} -DBUILD_TESTS=ON -DENABLE_ESMI_LIB=ON 2>&1 | tee cmake.log && \ - make -j $(nproc) 2>&1 | tee make.log && \ - make package 2>&1 | tee package.log; then - - # Parse and report warnings as GitHub annotations - echo "::group::Build Warnings" - grep -i "warning" cmake.log make.log package.log | while read -r line; do - echo "::warning::$line" - done - echo "::endgroup::" - - echo "Build successful on attempt $i" - break - else - echo "Build failed on attempt $i" - if [ $i -eq $RETRIES ]; then - echo "All $RETRIES build attempts failed. Exiting." - exit 1 - fi - sleep $((2 * i)) - fi - done - echo "Build completed on ${{ matrix.os }}" - - - name: Upload RPM Package Artifacts - if: github.event_name == 'pull_request' && matrix.os != 'RHEL10' && matrix.os != 'AlmaLinux8' - uses: actions/upload-artifact@v4 - with: - name: amd-smi-lib-rpm-${{ matrix.os }}-${{ env.PR_NUMBER }}-${{ env.BUILD_DATE }} - path: ${{ env.PROJECT_DIR }}/build/amd-smi-lib-*99999-local*.rpm - if-no-files-found: warn - retention-days: 7 - - - name: Install AMDSMI(RHEL10 & AlmaLinux8) - if: matrix.os == 'RHEL10' || matrix.os == 'AlmaLinux8' - run: | - cd ${{ env.PROJECT_DIR }}/build - dnf install --setopt=skip_if_unavailable=True python3-setuptools python3-wheel -y - - RETRIES=3 - for i in $(seq 1 $RETRIES); do - echo "RHEL10: Installation attempt $i..." - if timeout 10m dnf install -y --skip-broken --disablerepo=* ./amd-smi-lib-*99999-local*.rpm; then - echo "Installation successful on attempt $i" - ln -s /opt/rocm/bin/amd-smi /usr/local/bin - - echo 'Verifying installation:' - amd-smi version - python3 -m pip list | grep amd - python3 -m pip list | grep pip - python3 -m pip list | grep setuptools - echo 'Completed installation on RHEL10' - break - else - echo "Installation failed on attempt $i" - if [ $i -eq $RETRIES ]; then - echo "All $RETRIES installation attempts failed. Exiting." - exit 1 - fi - sleep $((2 * i)) - fi - done - - - name: Install AMDSMI - if: matrix.os != 'RHEL10' && matrix.os != 'AlmaLinux8' - run: | - cd ${{ env.PROJECT_DIR }}/build - case ${{ env.PACKAGE_MANAGER }} in - zypper) - timeout 10m zypper --no-refresh --no-gpg-checks install -y ./amd-smi-lib-*99999-local*.rpm - ;; - dnf) - dnf install --setopt=skip_if_unavailable=True python3-setuptools python3-wheel -y - RETRIES=3 - for i in $(seq 1 $RETRIES); do - echo "Attempt $i: Installing AMDSMI package..." - if timeout 10m dnf install -y --skip-broken --disablerepo=* ./amd-smi-lib-*99999-local*.rpm; then - echo "AMDSMI package installed successfully." - break - else - echo "Installation failed on attempt $i. Retrying..." - if [ $i -eq $RETRIES ]; then - echo "All $RETRIES attempts failed. Exiting." - exit 1 - fi - sleep 10 - fi - done - ;; - esac - ln -s /opt/rocm/bin/amd-smi /usr/local/bin - - # Verify Installation - echo 'Verifying installation:' - amd-smi version - python3 -m pip list | grep amd - python3 -m pip list | grep pip - python3 -m pip list | grep setuptools - echo 'Completed installation on ${{ matrix.os }}' - - - name: Uninstall - if: always() - run: | - set -e - echo 'Uninstalling on ${{ matrix.os }}' - case ${{ matrix.os }} in - SLES) - zypper remove -y amd-smi-lib || true - ;; - RHEL8|RHEL9|RHEL10|AlmaLinux8|AzureLinux3) - dnf remove -y amd-smi-lib || true - ;; - esac - rm -f /usr/local/bin/amd-smi - if [ -d /opt/rocm/share/amd_smi ]; then - echo '/opt/rocm/share/amd_smi exists. Removing.' - rm -rf /opt/rocm/share/amd_smi - fi - echo 'Uninstall done on ${{ matrix.os }}' - - rpm-test: - name: Tests - needs: [rpm-buildinstall, debian-test] - runs-on: - - self-hosted - - ${{ vars.RUNNER_TYPE }} - continue-on-error: true - strategy: - max-parallel: 10 - matrix: - os: - - SLES - - RHEL8 - - RHEL9 - - RHEL10 - - AzureLinux3 - - AlmaLinux8 - container: - image: ${{ vars[format('{0}_DOCKER_IMAGE', matrix.os)] }} - options: --rm --privileged --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size=64G --cap-add=SYS_MODULE -v /lib/modules:/lib/modules -u root - - steps: - - uses: actions/checkout@v4 - - - name: Set Project Directory - run: | - TARGET_DIR=$(find $GITHUB_WORKSPACE -path "*/projects/amdsmi/CMakeLists.txt" -exec dirname {} \;) - if [ -z "$TARGET_DIR" ]; then - TARGET_DIR=$(find $GITHUB_WORKSPACE -maxdepth 2 -name "CMakeLists.txt" -exec dirname {} \; | head -n 1) - fi - echo "PROJECT_DIR=$TARGET_DIR" >> $GITHUB_ENV - - - name: Set PkgMgr - run: | - set -e - case "${{ matrix.os }}" in - SLES) - echo "PACKAGE_MANAGER=zypper" >> $GITHUB_ENV - ;; - RHEL8|RHEL9|RHEL10|AlmaLinux8|AzureLinux3) - echo "PACKAGE_MANAGER=dnf" >> $GITHUB_ENV - ;; - esac - - - name: Add more_itertools - if: matrix.os == 'AzureLinux3' - run: | - set -e - echo 'Installing more_itertools on ${{ matrix.os }}' - python3 -m pip install more_itertools + # Remove SWIG-based libamd_smi_python.so that may be baked into the + # Docker image. The package now uses ctypes; the stale extension + # references symbols removed from libamd_smi.so and causes + # "undefined symbol" errors when importing the amdsmi package. + rm -f /opt/rocm/share/amd_smi/amdsmi/libamd_smi_python.so 2>/dev/null || true + ldconfig 2>/dev/null || true - name: Build and Install for Tests (RHEL10 & AlmaLinux8) if: matrix.os == 'RHEL10' || matrix.os == 'AlmaLinux8' + shell: bash run: | set -e echo 'Building for test on RHEL10/AlmaLinux8 with retries and QA_RPATHS' @@ -700,9 +473,15 @@ jobs: mkdir -p $BUILD_FOLDER cd $BUILD_FOLDER - if cmake ${{ env.PROJECT_DIR }} -DBUILD_TESTS=ON -DENABLE_ESMI_LIB=ON && \ + if cmake ${{ env.PROJECT_DIR }} -DBUILD_TESTS=ON -DENABLE_ESMI_LIB=ON -DBUILD_PYTHON_LIB=ON && \ make -j $(nproc) && \ + cd $BUILD_FOLDER/py-interface/python_package && \ + rm -rf *.whl *.egg-info build dist && \ + python3 -m pip wheel --no-deps --no-build-isolation -w . . && \ + echo "Python wheel built: $(ls *.whl)" && \ + cd $BUILD_FOLDER && \ make package; then + echo "Build successful on attempt $i" break else @@ -724,6 +503,11 @@ jobs: echo "Installation successful on attempt $i" ln -s /opt/rocm/bin/amd-smi /usr/local/bin echo 'Install done for test on RHEL10/AlmaLinux8' + + # Verify wheel installation + echo 'Verifying wheel installation (tests)...' + cd /tmp && python3 -c "import amdsmi; print('✓ Import successful'); amdsmi.amdsmi_init(); print('✓ Library loaded'); amdsmi.amdsmi_shut_down(); print('✓ Wheel working!')" + echo 'Python wheel build and install completed for tests on ${{ matrix.os }}' break else echo "Installation failed on attempt $i" @@ -744,8 +528,13 @@ jobs: rm -rf $BUILD_FOLDER mkdir -p $BUILD_FOLDER cd $BUILD_FOLDER - cmake ${{ env.PROJECT_DIR }} -DBUILD_TESTS=ON -DENABLE_ESMI_LIB=ON + cmake ${{ env.PROJECT_DIR }} -DBUILD_TESTS=ON -DENABLE_ESMI_LIB=ON -DBUILD_PYTHON_LIB=ON make -j $(nproc) + cd $BUILD_FOLDER/py-interface/python_package + rm -rf *.whl *.egg-info build dist + python3 -m pip wheel --no-deps --no-build-isolation -w . . + echo "Python wheel built: $(ls *.whl)" + cd $BUILD_FOLDER make package echo 'Installing for test on ${{ matrix.os }}' @@ -775,6 +564,57 @@ jobs: ln -s /opt/rocm/bin/amd-smi /usr/local/bin echo 'Install done for test on ${{ matrix.os }}' + # Verify wheel installation + echo 'Verifying wheel installation (tests)...' + cd /tmp && python3 -c "import amdsmi; print('✓ Import successful'); amdsmi.amdsmi_init(); print('✓ Library loaded'); amdsmi.amdsmi_shut_down(); print('✓ Wheel working!')" + echo 'Python wheel build and install completed for tests on ${{ matrix.os }}' + + + - name: Verify Wheel in Site-Packages + shell: bash + run: | + set -e + echo "=== Wheel site-packages verification on ${{ matrix.os }} ===" + BUILD_FOLDER=${{ env.PROJECT_DIR }}/build + WHEEL=$(ls $BUILD_FOLDER/py-interface/python_package/*.whl 2>/dev/null | head -1) + + if [ -z "$WHEEL" ]; then + echo "ERROR: No .whl file found in $BUILD_FOLDER/py-interface/python_package/" + exit 1 + fi + echo "Found wheel: $WHEEL" + + # Install the wheel into site-packages + echo "Installing wheel via pip..." + python3 -m pip install --force-reinstall "$WHEEL" + + # Verify it landed in site-packages + INSTALL_PATH=$(python3 -c "import amdsmi; print(amdsmi.__file__)") + echo "amdsmi installed at: $INSTALL_PATH" + + if echo "$INSTALL_PATH" | grep -q "site-packages"; then + echo "PASS: Wheel is correctly installed in site-packages" + else + echo "WARNING: amdsmi found at $INSTALL_PATH (not in site-packages)" + fi + + # Verify package metadata + echo "Package info:" + python3 -m pip show amdsmi + + # Smoke-test the import + cd /tmp + python3 -c " + import amdsmi + print('PASS: import amdsmi OK') + amdsmi.amdsmi_init() + print('PASS: amdsmi_init() OK') + devs = amdsmi.amdsmi_get_processor_handles() + print('PASS: Found %d device(s)' % len(devs)) + amdsmi.amdsmi_shut_down() + print('PASS: amdsmi_shut_down() OK') + print('=== Wheel verification passed ===') + " - name: AMDSMI Command Tests shell: bash run: | @@ -881,6 +721,17 @@ jobs: echo "Unit tests passed" fi + echo "Running perf tests..." + if ! ./perf_tests.py -v > /tmp/test-results-${{ matrix.os }}/perf_test_output.txt 2>&1; then + echo "Perf tests failed!" + echo "=============== PERF TEST OUTPUT ===============" + tail -100 /tmp/test-results-${{ matrix.os }}/perf_test_output.txt + echo "=================================================" + exit 1 + else + echo "Perf tests passed" + fi + echo "Python tests done" # Example Tests @@ -912,6 +763,12 @@ jobs: echo "Displaying Unit Test Results for ${{ matrix.os }}" cat /tmp/test-results-${{ matrix.os }}/unit_test_output.txt || echo "No unit test results found for ${{ matrix.os }}" + - name: Perf Test Results + if: always() + run: | + echo "Displaying Perf Test Results for ${{ matrix.os }}" + cat /tmp/test-results-${{ matrix.os }}/perf_test_output.txt || echo "No perf test results found for ${{ matrix.os }}" + - name: Example DRM Test Results if: always() run: | diff --git a/.github/workflows/manylinux-build.yml b/.github/workflows/manylinux-build.yml new file mode 100644 index 00000000000..c9867f789d0 --- /dev/null +++ b/.github/workflows/manylinux-build.yml @@ -0,0 +1,128 @@ +name: AMDSMI ManyLinux Wheels + +on: + pull_request: + branches: [develop] + paths: + - 'projects/amdsmi/**' + - '.github/workflows/manylinux-build.yml' + push: + branches: [develop] + paths: + - 'projects/amdsmi/**' + - '.github/workflows/manylinux-build.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + manylinux-wheels: + name: Build Manylinux Wheels + runs-on: ubuntu-latest + container: + image: quay.io/pypa/manylinux_2_28_x86_64 + + steps: + - uses: actions/checkout@v4 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Build manylinux wheels + run: | + set -euo pipefail + set -x + PROJECT_DIR=$GITHUB_WORKSPACE/projects/amdsmi + BUILD_DIR=/tmp/amdsmi-build + RAW_WHEELS=/tmp/raw-wheels + WHEEL_OUT=$GITHUB_WORKSPACE/wheels + PY_BUILD=/opt/python/cp38-cp38/bin/python3 + + echo "Using manylinux image: quay.io/pypa/manylinux_2_28_x86_64" + + echo "Installing build prerequisites..." + dnf -y install git make gcc gcc-c++ cmake ninja-build openssl-devel + + echo "Configuring and building core..." + rm -rf "$BUILD_DIR" + mkdir -p "$BUILD_DIR" + cd "$BUILD_DIR" + cmake "$PROJECT_DIR" \ + -DBUILD_TESTS=OFF \ + -DENABLE_ESMI_LIB=ON \ + -DBUILD_PYTHON_LIB=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython3_EXECUTABLE="$PY_BUILD" + make -j"$(nproc)" + + echo "Building wheels for all supported Python versions..." + mkdir -p "$RAW_WHEELS" "$WHEEL_OUT" + chmod 777 "$WHEEL_OUT" + $PY_BUILD -m pip install --upgrade pip setuptools wheel auditwheel + + for PY in \ + /opt/python/cp38-cp38/bin/python3 \ + /opt/python/cp39-cp39/bin/python3 \ + /opt/python/cp310-cp310/bin/python3 \ + /opt/python/cp311-cp311/bin/python3 \ + /opt/python/cp312-cp312/bin/python3 \ + /opt/python/cp313-cp313/bin/python3; do + if [ ! -x "$PY" ]; then + echo "Skipping missing interpreter $PY" + continue + fi + echo "Building wheel with $PY" + $PY -m pip install --upgrade pip setuptools wheel + pushd "$BUILD_DIR/py-interface/python_package" + rm -rf build dist *.whl *.egg-info + $PY -m pip wheel --no-deps --no-build-isolation -w "$RAW_WHEELS" . + popd + done + + echo "Raw wheels built:" + ls -al "$RAW_WHEELS" || true + + if ! ls "$RAW_WHEELS"/*.whl >/dev/null 2>&1; then + echo "No wheels were built; aborting." + exit 1 + fi + + echo "Repairing wheels with auditwheel..." + if ! $PY_BUILD -m auditwheel repair "$RAW_WHEELS"/*.whl --wheel-dir "$WHEEL_OUT"; then + echo "auditwheel repair failed; keeping raw wheels only." + cp -v "$RAW_WHEELS"/*.whl "$WHEEL_OUT"/ + fi + sync + + echo "Final wheels:" + ls -al "$WHEEL_OUT" + + - name: Verify wheels exist + run: | + mkdir -p "$GITHUB_WORKSPACE/wheels" + ls -al "$GITHUB_WORKSPACE/wheels" + if ! ls "$GITHUB_WORKSPACE"/wheels/*.whl >/dev/null 2>&1; then + echo "No wheels found; failing." + exit 1 + fi + echo "Wheel summary:" + for whl in "$GITHUB_WORKSPACE"/wheels/*.whl; do + echo " $(basename $whl)" + done + + - name: Upload manylinux wheels (by SHA) + uses: actions/upload-artifact@v4 + with: + name: amdsmi-manylinux-wheels-${{ github.sha }} + path: ${{ github.workspace }}/wheels/*.whl + if-no-files-found: error + retention-days: 30 + + - name: Upload manylinux wheels (downloadable) + uses: actions/upload-artifact@v4 + with: + name: amdsmi-python-wheels + path: ${{ github.workspace }}/wheels/*.whl + if-no-files-found: warn + retention-days: 90 diff --git a/projects/amdsmi/DEBIAN/postinst.in b/projects/amdsmi/DEBIAN/postinst.in index be6b576ab2a..e9d882be18f 100755 --- a/projects/amdsmi/DEBIAN/postinst.in +++ b/projects/amdsmi/DEBIAN/postinst.in @@ -105,92 +105,52 @@ do_ldconfig() { } do_install_amdsmi_python_lib() { - # get python version - local python3_minor_version - python3_minor_version=$(python3 -c 'import sys;print(sys.version_info.minor)') - if [ $? -ne 0 ]; then - echo "[WARNING] Could not determine python version. "\ - "AMD-SMI python library will not be installed." - return - fi + # Drop a .pth file into the system site-packages so that + # "import amdsmi" finds the package installed under + # @CPACK_PACKAGING_INSTALL_PREFIX@/@SHARE_INSTALL_PREFIX@ + # without requiring pip. The .pth approach is version-agnostic: + # it works with any Python 3.x without needing pip or setuptools. - # check if python version is supported - if [ "$python3_minor_version" -lt 6 ]; then - echo "[WARNING] AMD-SMI python library is not "\ - "supported on python version 3.$python3_minor_version. "\ - "AMD-SMI python library will not be installed." - return - fi + local python_lib_path=@CPACK_PACKAGING_INSTALL_PREFIX@/@SHARE_INSTALL_PREFIX@ - local PREVIOUS_PIP_ROOT_USER_ACTION="$PIP_ROOT_USER_ACTION" - export PIP_ROOT_USER_ACTION=ignore - # python3.11 requires --break-system-packages - local PREVIOUS_PIP_BREAK_SYSTEM_PACKAGES="$PIP_BREAK_SYSTEM_PACKAGES" - export PIP_BREAK_SYSTEM_PACKAGES=1 - - # Remove old python library - local amdsmi_pip_list_output - amdsmi_pip_list_output=$(python3 -m pip list --format=columns --disable-pip-version-check) - # check pip list output for amdsmi - if [[ $amdsmi_pip_list_output == *"amdsmi"* ]]; then - echo "Detected old AMD-SMI python library (amdsmi)..." - python3 -m pip uninstall amdsmi --yes --quiet --disable-pip-version-check - echo "Removed old AMD-SMI python library (amdsmi)..." - fi + # Remove stale SWIG-based Python extension (now uses ctypes) + rm -f "${python_lib_path}/amdsmi/libamd_smi_python.so" 2>/dev/null || true - # static builds don't include python lib - if [ "@BUILD_SHARED_LIBS@" != "ON" ]; then - return + # Remove any previously pip-installed amdsmi to avoid conflicts + if python3 -m pip list --format=columns --disable-pip-version-check 2>/dev/null | grep -q amdsmi; then + echo "Removing previously pip-installed amdsmi..." + PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip uninstall amdsmi --yes --quiet --disable-pip-version-check 2>/dev/null || true fi - check_and_install_amdsmi() { - local setuptools_version - setuptools_version=$(python3 -c 'import setuptools; print(setuptools.__version__)') - if [ $? -ne 0 ]; then - echo "[WARNING] Could not determine setuptools version. "\ - "AMD-SMI python library will not be installed." - return + # Determine site-packages directory + local site_packages + site_packages=$(python3 -c 'import site; print(site.getsitepackages()[0])' 2>/dev/null) + if [ -z "$site_packages" ]; then + echo "[WARNING] Could not determine site-packages directory. Trying dist-packages fallback..." + site_packages=$(python3 -c 'import sysconfig; print(sysconfig.get_path("purelib"))' 2>/dev/null) fi - # install python library at @CPACK_PACKAGING_INSTALL_PREFIX@/@SHARE_INSTALL_PREFIX@/amdsmi - local python_lib_path=@CPACK_PACKAGING_INSTALL_PREFIX@/@SHARE_INSTALL_PREFIX@ - local amdsmi_python_lib_path="$python_lib_path" - local amdsmi_setup_py_path="$python_lib_path/setup.py" - - # Decide installation method based on setuptools version - if [[ "$(printf '%s\n' "$setuptools_version" "28.5" | sort -V | head -n1)" == "$setuptools_version" ]]; then - echo "[WARNING] Setuptools version is less than 28.5. AMD-SMI will not be installed." - elif [[ "$(printf '%s\n' "$setuptools_version" "41.0.1" | sort -V | head -n1)" != "41.0.1" ]]; then - echo "Using setup.py for installation due to setuptools version $setuptools_version" - python3 "$amdsmi_setup_py_path" install - else - echo "Using pyproject.toml for installation due to setuptools version $setuptools_version" - python3 -m pip install "$amdsmi_python_lib_path" --quiet --disable-pip-version-check --no-build-isolation --no-index + if [ -z "$site_packages" ] || [ ! -d "$site_packages" ]; then + echo "[WARNING] Could not locate Python site-packages. AMD-SMI .pth file will not be installed." + return fi -} - # Call the function - check_and_install_amdsmi + # Write the .pth file — Python adds this path to sys.path on startup + echo "$python_lib_path" > "$site_packages/amdsmi.pth" + echo "Installed amdsmi.pth -> $site_packages/amdsmi.pth (points to $python_lib_path)" - export PIP_ROOT_USER_ACTION="$PREVIOUS_PIP_ROOT_USER_ACTION" - export PIP_BREAK_SYSTEM_PACKAGES="$PREVIOUS_PIP_BREAK_SYSTEM_PACKAGES" - - # only try to activate argcomplete if such command exists - # python3-argcomplete is recommended but optional, we handle its absence gracefully + # Activate argcomplete if available if command -v activate-global-python-argcomplete &>/dev/null; then activate-global-python-argcomplete 2>/dev/null || { echo "[INFO] Bash completion activation skipped. You can manually enable it with: activate-global-python-argcomplete" } + elif command -v activate-global-python-argcomplete3 &>/dev/null; then + activate-global-python-argcomplete3 2>/dev/null || { + echo "[INFO] Bash completion activation skipped. You can manually enable it with: activate-global-python-argcomplete3" + } else - # try older argcomplete3 version - if command -v activate-global-python-argcomplete3 &>/dev/null; then - activate-global-python-argcomplete3 2>/dev/null || { - echo "[INFO] Bash completion activation skipped. You can manually enable it with: activate-global-python-argcomplete3" - } - else - echo "[WARNING] Could not find argcomplete activation command. "\ - "Argument completion will not work. Install python3-argcomplete package to enable it." - fi + echo "[WARNING] Could not find argcomplete activation command. "\ + "Argument completion will not work. Install python3-argcomplete package to enable it." fi } diff --git a/projects/amdsmi/RPM/post.in b/projects/amdsmi/RPM/post.in index 86cac35a16c..a49d4825e96 100755 --- a/projects/amdsmi/RPM/post.in +++ b/projects/amdsmi/RPM/post.in @@ -104,95 +104,52 @@ do_ldconfig() { } do_install_amdsmi_python_lib() { - # get python version - local python3_minor_version - python3_minor_version=$(python3 -c 'import sys;print(sys.version_info.minor)') - if [ $? -ne 0 ]; then - echo "[WARNING] Could not determine python version. "\ - "AMD-SMI python library will not be installed." - return - fi + # Drop a .pth file into the system site-packages so that + # "import amdsmi" finds the package installed under + # $RPM_INSTALL_PREFIX0/@SHARE_INSTALL_PREFIX@ + # without requiring pip. The .pth approach is version-agnostic: + # it works with any Python 3.x without needing pip or setuptools. - # check if python version is supported - if [ "$python3_minor_version" -lt 6 ]; then - echo "[WARNING] AMD-SMI python library is not "\ - "supported on python version 3.$python3_minor_version. "\ - "AMD-SMI python library will not be installed." - return - fi + local python_lib_path=$RPM_INSTALL_PREFIX0/@SHARE_INSTALL_PREFIX@ - local PREVIOUS_PIP_ROOT_USER_ACTION="$PIP_ROOT_USER_ACTION" - export PIP_ROOT_USER_ACTION=ignore - # python3.11 requires --break-system-packages - local PREVIOUS_PIP_BREAK_SYSTEM_PACKAGES="$PIP_BREAK_SYSTEM_PACKAGES" - export PIP_BREAK_SYSTEM_PACKAGES=1 - - - # Remove old python library - local amdsmi_pip_list_output - amdsmi_pip_list_output=$(python3 -m pip list --format=columns --disable-pip-version-check) - # check pip list output for amdsmi - if [[ $amdsmi_pip_list_output == *"amdsmi"* ]]; then - echo "Detected old AMD-SMI python library (amdsmi)..." - python3 -m pip uninstall amdsmi --yes --quiet --disable-pip-version-check - echo "Removed old AMD-SMI python library (amdsmi)..." - fi + # Remove stale SWIG-based Python extension (now uses ctypes) + rm -f "${python_lib_path}/amdsmi/libamd_smi_python.so" 2>/dev/null || true - # static builds don't include python lib - if [ "@BUILD_SHARED_LIBS@" != "ON" ]; then - return + # Remove any previously pip-installed amdsmi to avoid conflicts + if python3 -m pip list --format=columns --disable-pip-version-check 2>/dev/null | grep -q amdsmi; then + echo "Removing previously pip-installed amdsmi..." + PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip uninstall amdsmi --yes --quiet --disable-pip-version-check 2>/dev/null || true fi - check_and_install_amdsmi() { - local setuptools_version - setuptools_version=$(python3 -c 'import setuptools; print(setuptools.__version__)') - if [ $? -ne 0 ]; then - echo "[WARNING] Could not determine setuptools version. "\ - "AMD-SMI python library will not be installed." - return + # Determine site-packages directory + local site_packages + site_packages=$(python3 -c 'import site; print(site.getsitepackages()[0])' 2>/dev/null) + if [ -z "$site_packages" ]; then + echo "[WARNING] Could not determine site-packages directory. Trying sysconfig fallback..." + site_packages=$(python3 -c 'import sysconfig; print(sysconfig.get_path("purelib"))' 2>/dev/null) fi - # install python library at $RPM_INSTALL_PREFIX0/@SHARE_INSTALL_PREFIX@/amdsmi - local python_lib_path=$RPM_INSTALL_PREFIX0/@SHARE_INSTALL_PREFIX@ - local amdsmi_python_lib_path="$python_lib_path" - local amdsmi_setup_py_path="$python_lib_path/setup.py" - - # Decide installation method based on setuptools version - if [[ "$(printf '%s\n' "$setuptools_version" "28.5" | sort -V | head -n1)" == "$setuptools_version" ]]; then - echo "[WARNING] Setuptools version is less than 28.5. AMD-SMI will not be installed." - elif [[ "$(printf '%s\n' "$setuptools_version" "41.0.1" | sort -V | head -n1)" != "41.0.1" ]]; then - echo "Using setup.py for installation due to setuptools version $setuptools_version" - cd $amdsmi_python_lib_path - python3 setup.py install - cd - - else - echo "Using pyproject.toml for installation due to setuptools version $setuptools_version" - python3 -m pip install "$amdsmi_python_lib_path" --quiet --disable-pip-version-check --no-build-isolation --no-index + if [ -z "$site_packages" ] || [ ! -d "$site_packages" ]; then + echo "[WARNING] Could not locate Python site-packages. AMD-SMI .pth file will not be installed." + return fi -} - # Call the function - check_and_install_amdsmi + # Write the .pth file — Python adds this path to sys.path on startup + echo "$python_lib_path" > "$site_packages/amdsmi.pth" + echo "Installed amdsmi.pth -> $site_packages/amdsmi.pth (points to $python_lib_path)" - export PIP_ROOT_USER_ACTION="$PREVIOUS_PIP_ROOT_USER_ACTION" - export PIP_BREAK_SYSTEM_PACKAGES="$PREVIOUS_PIP_BREAK_SYSTEM_PACKAGES" - - # only try to activate argcomplete if such command exists - # python3-argcomplete is recommended but optional, we handle its absence gracefully + # Activate argcomplete if available if command -v activate-global-python-argcomplete &>/dev/null; then activate-global-python-argcomplete 2>/dev/null || { echo "[INFO] Bash completion activation skipped. You can manually enable it with: activate-global-python-argcomplete" } + elif command -v activate-global-python-argcomplete3 &>/dev/null; then + activate-global-python-argcomplete3 2>/dev/null || { + echo "[INFO] Bash completion activation skipped. You can manually enable it with: activate-global-python-argcomplete3" + } else - # try older argcomplete3 version - if command -v activate-global-python-argcomplete3 &>/dev/null; then - activate-global-python-argcomplete3 2>/dev/null || { - echo "[INFO] Bash completion activation skipped. You can manually enable it with: activate-global-python-argcomplete3" - } - else - echo "[WARNING] Could not find argcomplete activation command. "\ - "Argument completion will not work. Install python3-argcomplete package to enable it." - fi + echo "[WARNING] Could not find argcomplete activation command. "\ + "Argument completion will not work. Install python3-argcomplete package to enable it." fi } diff --git a/projects/amdsmi/amdsmi_cli/CMakeLists.txt b/projects/amdsmi/amdsmi_cli/CMakeLists.txt index 86002caece2..47827dbcb7c 100644 --- a/projects/amdsmi/amdsmi_cli/CMakeLists.txt +++ b/projects/amdsmi/amdsmi_cli/CMakeLists.txt @@ -5,6 +5,7 @@ message("&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&") # Set CLI Build Directory set(PY_PACKAGE_DIR "amdsmi_cli") set(PY_CLI_INSTALL_DIR "${CMAKE_INSTALL_LIBEXECDIR}" CACHE STRING "CLI tool installation directory") +set(PY_LIB_FROM_PKG "${PROJECT_BINARY_DIR}/py-interface/python_package/amdsmi/libamd_smi_python.so") # populate version string configure_file(_version.py.in ${PY_PACKAGE_DIR}/_version.py @ONLY) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_init.py b/projects/amdsmi/amdsmi_cli/amdsmi_init.py index 0eac8e82b75..b60ac6c89e6 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_init.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_init.py @@ -29,12 +29,95 @@ from pathlib import Path +from typing import Optional current_path = os.path.dirname(os.path.abspath(__file__)) python_lib_path = f"{current_path}/../../share/amd_smi" sys.path.insert(0, python_lib_path) -# Prioritize the library from this installation over any pip-installed version +# Only fallback to the python library if its a compatible version +# multiple amdsmi versions installed on the system could cause issues + +# Ideally we want to identify if the installed python library is incompatible and log a solution to the user +# LD library config or reinstall, etc... +# The problem is coming from the switch over between the post install and pypi + + +def _log_version_and_path_diagnostics(): + """Emit best-effort diagnostics about the amdsmi Python package and the shared library location.""" + try: + from amdsmi import _version # type: ignore + pkg_version = getattr(_version, "__version__", "unknown") + except Exception as exc: # pragma: no cover - defensive + pkg_version = f"unavailable ({exc})" + + # Resolve paths from the *wrapper* module, not from this CLI file. + # The wrapper lives in the amdsmi package dir; the CLI lives elsewhere. + try: + import amdsmi.amdsmi_wrapper as _w + wrapper_path = Path(_w.__file__).resolve() + wrapper_dir = wrapper_path.parent + except Exception: + wrapper_dir = Path("") + + print(f"[amdsmi-cli] Python package version: {pkg_version}") + print(f"[amdsmi-cli] CLI dir: {Path(__file__).resolve().parent}") + print(f"[amdsmi-cli] Wrapper dir: {wrapper_dir}") + print(f"[amdsmi-cli] sys.path[0]: {sys.path[0]}") + + +def _check_version_compatibility(expected_version: Optional[str] = None) -> None: + """ + Verify that the Python package version matches the expected CLI/lib version (if provided). + If mismatched, log guidance and abort to avoid loading an incompatible library. + + The actual library loading (pip vs system context) is handled entirely by + amdsmi_wrapper._load_library(). This function only checks version strings + and that the wrapper can resolve *some* loadable library candidate. + """ + try: + from amdsmi import _version # type: ignore + pkg_version = getattr(_version, "__version__", None) + except Exception: + _log_version_and_path_diagnostics() + print("[amdsmi-cli] Failed to read amdsmi Python package version; aborting to avoid incompatibility.") + sys.exit(1) + + if expected_version and pkg_version and pkg_version != expected_version: + _log_version_and_path_diagnostics() + print(f"[amdsmi-cli] Version mismatch: expected {expected_version}, found {pkg_version}.") + print("[amdsmi-cli] Please install a matching amdsmi wheel from PyPI or reinstall the ROCm package,") + print("[amdsmi-cli] and ensure LD_LIBRARY_PATH/ldconfig points to the matching shared library.") + sys.exit(1) + + # Delegate the library-existence check to the wrapper's own detection logic. + # _build_candidate_paths() uses the wrapper's __file__ to correctly resolve + # pip (libamd_smi_python.so next to wrapper) vs system (/opt/rocm/lib/libamd_smi.so). + try: + from amdsmi.amdsmi_wrapper import _build_candidate_paths + candidates = _build_candidate_paths() + for candidate in candidates: + if isinstance(candidate, str): + # bare "libamd_smi.so" — let the dynamic linker resolve it later + return + if candidate.exists(): + return + except Exception: + # If the wrapper isn't importable at all, fall through to the + # ImportError handler in the try/except block below. + return + + _log_version_and_path_diagnostics() + print("[amdsmi-cli] Unable to locate the AMD SMI shared library in expected locations:") + for c in candidates: + print(f" - {c}") + print("[amdsmi-cli] Install the amdsmi wheel that bundles the Python shared library,") + print("[amdsmi-cli] or adjust LD_LIBRARY_PATH/ldconfig to point to a compatible lib.") + sys.exit(1) + try: + # TODO Add version checking & debug to check pathing + # The expected version string can be wired in from packaging if desired. + _check_version_compatibility(expected_version=None) from amdsmi import amdsmi_interface, amdsmi_exception except ImportError as e: print(f"Unhandled import error: {e}") diff --git a/projects/amdsmi/py-interface/CMakeLists.txt b/projects/amdsmi/py-interface/CMakeLists.txt index 525b18d40bb..68444ce7f06 100644 --- a/projects/amdsmi/py-interface/CMakeLists.txt +++ b/projects/amdsmi/py-interface/CMakeLists.txt @@ -12,6 +12,11 @@ set(PY_BUILD_DIR "python_package") # additionally defined in pyproject.toml set(PY_PACKAGE_DIR "${PY_BUILD_DIR}/amdsmi") set(PY_WRAPPER_INSTALL_DIR "${SHARE_INSTALL_PREFIX}" CACHE STRING "Wrapper installation directory") +set(AMDSMI_PYTHON_LIB_NAME "libamd_smi.so") +if(BUILD_PYTHON_LIB) + set(AMDSMI_PYTHON_LIB_NAME "libamd_smi_python.so") +endif() +set(AMDSMI_PYTHON_LIB_PATH "${PROJECT_BINARY_DIR}/src/${AMDSMI_PYTHON_LIB_NAME}") # try to find clang of the right version set(GOOD_CLANG_FOUND FALSE) @@ -35,6 +40,8 @@ if(NOT GOOD_CLANG_FOUND) AUTHOR_WARNING "A wrapper will not be re-generated! Using old wrapper instead.\nSet -DBUILD_WRAPPER=ON to re-build the wrapper" ) + # Still need Python3 for building the wheel package + find_package(Python3 3.6 COMPONENTS Interpreter REQUIRED) add_custom_command( OUTPUT amdsmi_wrapper.py ${PY_PACKAGE_DIR}/amdsmi_wrapper.py DEPENDS ${AMD_SMI} ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py @@ -55,16 +62,13 @@ else() # generate new wrapper configure_file(${PROJECT_SOURCE_DIR}/tools/generator.py generator.py @ONLY COPYONLY) add_custom_command( - OUTPUT - amdsmi.h - ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py - amdsmi_wrapper.py - ${PY_PACKAGE_DIR}/amdsmi_wrapper.py - DEPENDS ${AMD_SMI} python_pre_reqs generator.py ${PROJECT_SOURCE_DIR}/include/amd_smi/amdsmi.h + OUTPUT amdsmi.h ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py amdsmi_wrapper.py + ${PY_PACKAGE_DIR}/amdsmi_wrapper.py + DEPENDS ${AMD_SMI} $<$:${AMD_SMI}_python> python_pre_reqs generator.py ${PROJECT_SOURCE_DIR}/include/amd_smi/amdsmi.h COMMAND cp ${PROJECT_SOURCE_DIR}/include/amd_smi/amdsmi.h ./ - COMMAND - ${Python3_EXECUTABLE} generator.py "$<$:-e -DENABLE_ESMI_LIB>" -i amdsmi.h -l - ${PROJECT_BINARY_DIR}/src/libamd_smi.so -o ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py + COMMAND ${CMAKE_COMMAND} -E env AMDSMI_FIND_LIB_PATH=${CMAKE_CURRENT_BINARY_DIR}/${PY_PACKAGE_DIR}/_find_lib.py + ${Python3_EXECUTABLE} generator.py "$<$:-e -DENABLE_ESMI_LIB>" -i amdsmi.h -l + ${AMDSMI_PYTHON_LIB_PATH} -o ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py COMMAND mkdir -p ${PY_PACKAGE_DIR} COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py ${PY_PACKAGE_DIR}/ ) @@ -72,8 +76,11 @@ endif() # populate version string configure_file(pyproject.toml.in ${PY_BUILD_DIR}/pyproject.toml @ONLY) +configure_file(setup.cfg.in ${PY_BUILD_DIR}/setup.cfg @ONLY) configure_file(_version.py.in ${PY_PACKAGE_DIR}/_version.py @ONLY) -configure_file(setup.py.in ${PY_BUILD_DIR}/setup.py @ONLY) +if(BUILD_PYTHON_LIB) + configure_file(_find_lib.py.in ${PY_PACKAGE_DIR}/_find_lib.py @ONLY) +endif() add_custom_target(python_wrapper DEPENDS amdsmi_wrapper.py) @@ -94,34 +101,73 @@ add_custom_command( ) # copy libamd_smi.so to allow for a self-contained python package -add_custom_command( - OUTPUT ${PY_PACKAGE_DIR}/libamd_smi.so - DEPENDS ${PROJECT_BINARY_DIR}/src/libamd_smi.so - COMMAND cp "${PROJECT_BINARY_DIR}/src/libamd_smi.so" ${PY_PACKAGE_DIR}/ -) +add_custom_command(OUTPUT ${PY_PACKAGE_DIR}/${AMDSMI_PYTHON_LIB_NAME} DEPENDS ${AMDSMI_PYTHON_LIB_PATH} + COMMAND cp "${AMDSMI_PYTHON_LIB_PATH}" ${PY_PACKAGE_DIR}/) add_custom_target( - python_package - ALL - DEPENDS - ${PY_BUILD_DIR}/pyproject.toml - ${PY_BUILD_DIR}/setup.py - ${PY_PACKAGE_DIR}/_version.py - ${PY_PACKAGE_DIR}/__init__.py - ${PY_PACKAGE_DIR}/amdsmi_exception.py - ${PY_PACKAGE_DIR}/amdsmi_interface.py - ${PY_PACKAGE_DIR}/README.md - ${PY_PACKAGE_DIR}/LICENSE - ${PY_PACKAGE_DIR}/libamd_smi.so -) + python_package ALL + DEPENDS ${PY_BUILD_DIR}/pyproject.toml + ${PY_BUILD_DIR}/setup.cfg + ${PY_PACKAGE_DIR}/_version.py + ${PY_PACKAGE_DIR}/__init__.py + ${PY_PACKAGE_DIR}/amdsmi_exception.py + ${PY_PACKAGE_DIR}/amdsmi_interface.py + ${PY_PACKAGE_DIR}/README.md + ${PY_PACKAGE_DIR}/LICENSE + ${PY_PACKAGE_DIR}/${AMDSMI_PYTHON_LIB_NAME} + $<$:${PY_PACKAGE_DIR}/_find_lib.py>) +if(BUILD_PYTHON_LIB) + add_dependencies(python_package ${AMD_SMI}_python) +endif() + +# Build a wheel into the binary dir (no deps, local artifacts only) +add_custom_target( + python_wheel ALL + DEPENDS python_package + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/${PY_BUILD_DIR} + ${Python3_EXECUTABLE} -m pip wheel --no-deps -w ${CMAKE_CURRENT_BINARY_DIR}/${PY_BUILD_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/${PY_BUILD_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${PY_BUILD_DIR} + COMMENT "Building amdsmi wheel without dependencies") install( - FILES - ${CMAKE_CURRENT_BINARY_DIR}/${PY_BUILD_DIR}/pyproject.toml - ${CMAKE_CURRENT_BINARY_DIR}/${PY_BUILD_DIR}/setup.py - ${CMAKE_CURRENT_BINARY_DIR}/${PY_PACKAGE_DIR}/_version.py + FILES ${CMAKE_CURRENT_BINARY_DIR}/${PY_BUILD_DIR}/pyproject.toml + ${CMAKE_CURRENT_BINARY_DIR}/${PY_PACKAGE_DIR}/_version.py DESTINATION ${PY_WRAPPER_INSTALL_DIR} COMPONENT dev ) -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${PY_PACKAGE_DIR} DESTINATION ${PY_WRAPPER_INSTALL_DIR} COMPONENT dev) +install( + DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${PY_PACKAGE_DIR} + DESTINATION ${PY_WRAPPER_INSTALL_DIR} + COMPONENT dev) + +# Install a .pth file into the system site-packages directory so that +# "import amdsmi" works without pip install. The .pth mechanism is Python- +# version agnostic: Python reads *.pth files from site-packages on startup +# and adds the listed paths to sys.path. +install(CODE " + execute_process( + COMMAND ${Python3_EXECUTABLE} -c \"import site; print(site.getsitepackages()[0])\" + OUTPUT_VARIABLE _site_packages + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _site_rc + ) + if(NOT _site_rc EQUAL 0 OR _site_packages STREQUAL \"\") + # Fallback to sysconfig + execute_process( + COMMAND ${Python3_EXECUTABLE} -c \"import sysconfig; print(sysconfig.get_path('purelib'))\" + OUTPUT_VARIABLE _site_packages + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _site_rc + ) + endif() + if(_site_rc EQUAL 0 AND NOT _site_packages STREQUAL \"\") + set(_pth_file \"\${_site_packages}/amdsmi.pth\") + file(WRITE \"\${_pth_file}\" + \"\${CMAKE_INSTALL_PREFIX}/${PY_WRAPPER_INSTALL_DIR}\\n\") + message(STATUS \"Installed \${_pth_file} -> \${CMAKE_INSTALL_PREFIX}/${PY_WRAPPER_INSTALL_DIR}\") + else() + message(WARNING \"Could not determine Python site-packages; amdsmi.pth not installed.\") + endif() +" COMPONENT dev) diff --git a/projects/amdsmi/py-interface/_find_lib.py.in b/projects/amdsmi/py-interface/_find_lib.py.in new file mode 100644 index 00000000000..a7e325c83de --- /dev/null +++ b/projects/amdsmi/py-interface/_find_lib.py.in @@ -0,0 +1,10 @@ +from pathlib import Path + + +def find_smi_library() -> Path: + here = Path(__file__).resolve().parent + candidate = here / "libamd_smi_python.so" + if candidate.exists(): + return candidate + # fall back to the default shared library name for developer builds + return here / "libamd_smi.so" \ No newline at end of file diff --git a/projects/amdsmi/py-interface/amdsmi_wrapper.py b/projects/amdsmi/py-interface/amdsmi_wrapper.py index 00d42529fb5..79618c50384 100644 --- a/projects/amdsmi/py-interface/amdsmi_wrapper.py +++ b/projects/amdsmi/py-interface/amdsmi_wrapper.py @@ -165,54 +165,138 @@ def char_pointer_cast(string, encoding='utf-8'): _libraries = {} from pathlib import Path -# libamd_smi.so can be located in several different places. -# Look for it with below priority: -# 0. Relative to amdsmi_wrapper.py in TheRock: -# `amdsmi_wrapper.py` is located in -# `_rocm_sdk_core/share/amd_smi/amdsmi`, libraries are in -# `_rocm_sdk_core/lib`. -# 1. ROCM_HOME/ROCM_PATH environment variables -# - ROCM_HOME/lib -# - ROCM_PATH/lib (usually set to /opt/rocm/) -# 2. Decided by the linker -# - LD_LIBRARY_PATH env var -# - defined path in /etc/ld.so.conf.d/ -# 3. Relative to amdsmi_wrapper.py -# - parent directory -# - current directory -def find_smi_library(): - err = OSError("Could not load libamd_smi.so") - possible_locations = [] - # 0. - libamd_smi_path = Path(__file__).resolve().parent.parent.parent.parent / "lib/libamd_smi.so.26" - possible_locations.append(libamd_smi_path) - # 1. - rocm_path = os.getenv("ROCM_HOME", os.getenv("ROCM_PATH")) - if rocm_path: - possible_locations.append(os.path.join(rocm_path, "lib/libamd_smi.so")) - # 2. - possible_locations.append("libamd_smi.so") - # 3. - libamd_smi_parent_dir = Path(__file__).resolve().parent / "libamd_smi.so" - libamd_smi_cwd = Path.cwd() / "libamd_smi.so" - possible_locations.append(libamd_smi_parent_dir) - possible_locations.append(libamd_smi_cwd) - - for location in possible_locations: + +# --------------------------------------------------------------------------- +# Dynamic library loading +# --------------------------------------------------------------------------- +# Two installation contexts exist, each shipping its own .so: +# +# 1. Linux system package (RPM / DEB) +# wrapper: /opt/rocm/share/amd_smi/amdsmi/amdsmi_wrapper.py +# library: /opt/rocm/lib/libamd_smi.so +# Note: /opt/rocm may be a symlink (e.g. /opt/rocm -> /opt/rocm-X.Y.Z). +# Path.resolve() handles this transparently. +# +# 2. Python pip package (wheel) +# wrapper: /amdsmi/amdsmi_wrapper.py +# library: /amdsmi/libamd_smi_python.so +# The exact site-packages location varies per distro / venv / conda. +# +# Detection strategy (based on the absolute *resolved* path of THIS file): +# - If libamd_smi_python.so exists next to this wrapper -> pip context +# - Otherwise -> system-package context (derive ROCm root from path or env) +# +# Regardless of which .so is loaded, it is stored under the key +# _libraries['libamd_smi.so'] +# so every downstream ctypes binding in this wrapper works unchanged. +# --------------------------------------------------------------------------- + +_libraries = {} + + +def _detect_install_context(): + """Classify the current install as ``"pip"`` or ``"system"``. + + Returns + ------- + tuple[str, Path] + ``("pip", module_dir)`` - *module_dir* contains the wrapper **and** + ``libamd_smi_python.so``. + ``("system", rocm_root)`` - *rocm_root* is the resolved ROCm prefix + (e.g. ``/opt/rocm``). + + All paths are fully resolved so symlinks like + ``/opt/rocm -> /opt/rocm-X.Y.Z`` are handled automatically. + """ + wrapper_path = Path(__file__).resolve() + module_dir = wrapper_path.parent # .../amdsmi/ + + # pip context + # The wheel ships libamd_smi_python.so right next to the wrapper. + if (module_dir / "libamd_smi_python.so").exists(): + return "pip", module_dir + + # system-package context + # Expected layout: + # /share/amd_smi/amdsmi/amdsmi_wrapper.py + # module_dir = /share/amd_smi/amdsmi + # rocm_root = module_dir / ../../.. (3 dirs up) + potential_rocm_root = module_dir.parent.parent.parent + if (potential_rocm_root / "lib").is_dir(): + return "system", potential_rocm_root + + # Fallback to ROCM_HOME / ROCM_PATH environment variables + for env_var in ("ROCM_HOME", "ROCM_PATH"): + env_val = os.getenv(env_var) + if env_val: + p = Path(env_val).resolve() + if (p / "lib").is_dir(): + return "system", p + + # Last resort - default ROCm location. + return "system", Path("/opt/rocm").resolve() + + +def _build_candidate_paths(): + """Return an ordered list of .so paths to try, best-match first.""" + context, base = _detect_install_context() + candidates = [] + + if context == "pip": + # .so is self-contained inside the wheel / site-packages. + # Do NOT fall back to the system libamd_smi.so: loading both libraries + # in the same process causes segfaults during static initialisation of + # std::variant tables on older toolchains (GCC 8 / glibc 2.28). + candidates.append(base / "libamd_smi_python.so") + return candidates + + # System package - .so lives under /lib/ + candidates.append(base / "lib" / "libamd_smi.so") + + # Fallbacks + for env_var in ("ROCM_HOME", "ROCM_PATH"): + env_val = os.getenv(env_var) + if env_val: + candidates.append(Path(env_val) / "lib" / "libamd_smi.so") + + # Let the dynamic linker try LD_LIBRARY_PATH / ld.so.conf.d + candidates.append("libamd_smi.so") + + return candidates + + +def _load_library(): + """Load the AMD SMI shared library for the detected install context. + + Returns + ------- + tuple[ctypes.CDLL, str] + The loaded library handle and the path that was successfully loaded. + + Raises + ------ + OSError + If none of the candidate paths could be loaded. + """ + candidates = _build_candidate_paths() + last_err = None + mode = getattr(ctypes, "RTLD_GLOBAL", 0) + + for candidate in candidates: try: - lib = ctypes.CDLL(location) - return lib, location - except OSError as e: - err = e - continue - raise err - -try: - _libraries['libamd_smi.so'], location = find_smi_library() - #print(f"found smi lib in [", location, "]") -except OSError as e: - print(e) - print("Unable to find libamd_smi.so library try installing amd-smi-lib from your package manager") + lib = ctypes.CDLL(str(candidate), mode=mode) + return lib, str(candidate) + except OSError as exc: + last_err = exc + + raise last_err or OSError( + "Could not load AMD SMI library. Searched:\n" + + "\n".join(f" - {c}" for c in candidates) + ) + + +# The dict key stays 'libamd_smi.so' regardless of the actual .so loaded +_libraries['libamd_smi.so'], _loaded_lib_path = _load_library() #Add support for amdsmi_free_name_value_pairs amdsmi_free_name_value_pairs = _libraries['libamd_smi.so'].amdsmi_free_name_value_pairs @@ -988,6 +1072,21 @@ class struct_amdsmi_enumeration_info_t(Structure): class struct_amdsmi_pcie_info_t(Structure): pass +class struct_pcie_static_(Structure): + pass + +struct_pcie_static_._pack_ = 1 # source:False +struct_pcie_static_._fields_ = [ + ('max_pcie_width', ctypes.c_uint16), + ('PADDING_0', ctypes.c_ubyte * 2), + ('max_pcie_speed', ctypes.c_uint32), + ('pcie_interface_version', ctypes.c_uint32), + ('slot_type', amdsmi_card_form_factor_t), + ('max_pcie_interface_version', ctypes.c_uint32), + ('PADDING_1', ctypes.c_ubyte * 4), + ('reserved', ctypes.c_uint64 * 9), +] + class struct_pcie_metric_(Structure): pass @@ -1008,21 +1107,6 @@ class struct_pcie_metric_(Structure): ('reserved', ctypes.c_uint64 * 12), ] -class struct_pcie_static_(Structure): - pass - -struct_pcie_static_._pack_ = 1 # source:False -struct_pcie_static_._fields_ = [ - ('max_pcie_width', ctypes.c_uint16), - ('PADDING_0', ctypes.c_ubyte * 2), - ('max_pcie_speed', ctypes.c_uint32), - ('pcie_interface_version', ctypes.c_uint32), - ('slot_type', amdsmi_card_form_factor_t), - ('max_pcie_interface_version', ctypes.c_uint32), - ('PADDING_1', ctypes.c_ubyte * 4), - ('reserved', ctypes.c_uint64 * 9), -] - struct_amdsmi_pcie_info_t._pack_ = 1 # source:False struct_amdsmi_pcie_info_t._fields_ = [ ('pcie_static', struct_pcie_static_), diff --git a/projects/amdsmi/py-interface/setup.cfg.in b/projects/amdsmi/py-interface/setup.cfg.in new file mode 100644 index 00000000000..c3a34cfdac7 --- /dev/null +++ b/projects/amdsmi/py-interface/setup.cfg.in @@ -0,0 +1,21 @@ +[metadata] +name = amdsmi +version = @amd_smi_lib_VERSION_STRING@ +author = AMD +author_email = amd-smi.support@amd.com +description = AMDSMI Python LIB - AMD GPU Monitoring Library +url = https://github.com/ROCm/amdsmi +license_file = amdsmi/LICENSE +long_description = file: amdsmi/README.md +long_description_content_type = text/markdown +classifiers = + Programming Language :: Python :: 3 +python_requires = >=3.6 + +[options] +packages = find: +include_package_data = True +zip_safe = False + +[options.package_data] +* = *.so diff --git a/projects/amdsmi/src/CMakeLists.txt b/projects/amdsmi/src/CMakeLists.txt index 2bf6e0ed329..894b58ef951 100644 --- a/projects/amdsmi/src/CMakeLists.txt +++ b/projects/amdsmi/src/CMakeLists.txt @@ -197,6 +197,53 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL Release) endif() endif() +## Create Python-specific library if BUILD_PYTHON_LIB is enabled +if(BUILD_PYTHON_LIB) + add_library(${AMD_SMI}_python ${SRC_LIST} ${INC_LIST}) + target_link_libraries(${AMD_SMI}_python PRIVATE + rt + Threads::Threads + ${CMAKE_DL_LIBS} + amdsminic + ${FILESYSTEM_LIB} + ) + target_link_directories(${AMD_SMI}_python PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}/nic/ai-nic/amdsmi_unified/build/ + ) + target_include_directories(${AMD_SMI}_python PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/rocm_smi/include + ${PROJECT_SOURCE_DIR}/common/shared_mutex + ${RAS_DECODE_INC_DIR} + ${DRM_INCLUDE_DIRS} + ${DRM_AMDGPU_INCLUDE_DIRS} + ${NIC_INCLUDE_DIRS} + ${PROJECT_SOURCE_DIR}/src/nic/ai-nic/amdsmi_unified/inc + ${PROJECT_SOURCE_DIR}/src/nic/ai-nic/amdsmi_unified/interface + ) + target_include_directories(${AMD_SMI}_python PUBLIC "$" + "$") + set_property(TARGET ${AMD_SMI}_python PROPERTY SOVERSION "${MAJOR}") + set_property(TARGET ${AMD_SMI}_python PROPERTY VERSION "${SO_VERSION_STRING}") + set_target_properties(${AMD_SMI}_python PROPERTIES OUTPUT_NAME "amd_smi_python") + + # Install python-specific shared library into the package + install( + TARGETS ${AMD_SMI}_python + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT dev) + + # Install pre-built Python wheel into share dir (built by CI before make package) + install(CODE " + file(GLOB WHEEL_FILES \"${CMAKE_BINARY_DIR}/py-interface/python_package/*.whl\") + if(WHEEL_FILES) + list(GET WHEEL_FILES 0 WHEEL_FILE) + message(STATUS \"Installing Python wheel into package: \\${WHEEL_FILE}\") + file(INSTALL DESTINATION \"\\${CMAKE_INSTALL_PREFIX}/share/amd_smi\" + TYPE FILE FILES \"\\${WHEEL_FILE}\") + endif() + " COMPONENT dev) +endif() ## Add the install directives for the runtime library. set(_INSTALL_TARGETS ${AMD_SMI} amdsminic) if(BUILD_BOTH_LIBS) diff --git a/projects/amdsmi/tools/build_wheel_debian.py b/projects/amdsmi/tools/build_wheel_debian.py new file mode 100644 index 00000000000..47a13622527 --- /dev/null +++ b/projects/amdsmi/tools/build_wheel_debian.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +""" +build_wheel_debian.py +===================== + +Build AMDSMI Python wheels on Debian / Ubuntu hosts or inside a +manylinux_2_28 container. + +This script automates the full wheel-build pipeline that the CI workflow +(.github/workflows/manylinux-build.yml) performs: + + 1. cmake configure (-DBUILD_PYTHON_LIB=ON) + 2. make -j$(nproc) + 3. pip wheel --no-deps --no-build-isolation (per interpreter) + 4. auditwheel repair (optional, for manylinux tags) + +Requirements +------------ +* cmake, make, gcc/g++, git, python3, pip +* (optional) auditwheel -- installed automatically when --repair is used +* (optional) /opt/python/cpXX-cpXX interpreters -- present in manylinux images + +Examples +-------- +Single-interpreter build on a bare Debian/Ubuntu host:: + + python3 build_wheel_debian.py --project-dir /path/to/amdsmi + +Multi-interpreter manylinux build inside quay.io/pypa/manylinux_2_28_x86_64 docker container:: + + python3 build_wheel_debian.py --project-dir /src/amdsmi --all-pythons + +Compatibility +------------- +* Python >= 3.6 (uses ``universal_newlines`` instead of ``text``, + ``stdout/stderr=PIPE`` instead of ``capture_output``) +* Tested on Ubuntu 20.04 / 22.04 / 24.04 and manylinux_2_28 containers. +""" + +import argparse +import logging +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- +logging.basicConfig( + level=logging.INFO, + format="[%(levelname)s] %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], +) +log = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- +DEFAULT_BUILD_DIR = Path("/tmp/amdsmi-build") +DEFAULT_RAW_WHEELS_DIR = Path("/tmp/raw-wheels") +DEFAULT_OUTPUT_NAME = "wheels" + +# Ordered to match .github/workflows/manylinux-build.yml +MANYLINUX_PYTHONS = [ + "/opt/python/cp38-cp38/bin/python3", + "/opt/python/cp39-cp39/bin/python3", + "/opt/python/cp310-cp310/bin/python3", + "/opt/python/cp311-cp311/bin/python3", + "/opt/python/cp312-cp312/bin/python3", + "/opt/python/cp313-cp313/bin/python3", +] + + +# --------------------------------------------------------------------------- +# Helpers (all Python 3.6-safe) +# --------------------------------------------------------------------------- + +def run(cmd, cwd=None, env=None, check=True): + """Execute *cmd* and stream output to the console.""" + log.info("Running: %s", " ".join(str(c) for c in cmd)) + merged_env = os.environ.copy() + if env: + merged_env.update(env) + return subprocess.run(cmd, check=check, cwd=cwd, env=merged_env) + + +def abort(message, code=1): + log.error(message) + sys.exit(code) + + +def find_project_dir(start): + """Walk up from *start* looking for the first directory with CMakeLists.txt.""" + candidate = start + for _ in range(10): + if (candidate / "CMakeLists.txt").exists(): + return candidate + candidate = candidate.parent + abort("Could not auto-detect project root. Pass --project-dir explicitly.") + + +def best_effort_pip_upgrade(py, packages): + """Try to upgrade *packages* via pip; log a warning on failure.""" + result = subprocess.run( + [py, "-m", "pip", "install", "--upgrade"] + list(packages), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, + ) + if result.returncode != 0: + log.warning( + "pip upgrade failed with %s (packages: %s)\nstdout: %s\nstderr: %s", + py, ", ".join(packages), result.stdout.strip(), result.stderr.strip(), + ) + return result.returncode == 0 + + +def mark_safe_git_dir(path): + """Register *path* as a safe git directory (avoids dubious-ownership errors).""" + git_bin = shutil.which("git") + if not git_bin or not path.exists(): + return True + result = subprocess.run( + [git_bin, "config", "--global", "--add", "safe.directory", str(path)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, + ) + if result.returncode != 0: + log.warning( + "git safe.directory add failed for %s (rc=%s): %s", + path, result.returncode, result.stderr.strip(), + ) + return False + return True + + +def write_temp_git_config(config_path, safe_paths): + """Write a temporary gitconfig that marks *safe_paths* as safe directories.""" + try: + config_path.parent.mkdir(parents=True, exist_ok=True) + lines = [] + for p in safe_paths: + lines.append("[safe]\n\tdirectory = %s\n" % p) + config_path.write_text("\n".join(lines)) + log.info("Temporary git config for safe.directory: %s", config_path) + return {"GIT_CONFIG_GLOBAL": str(config_path)} + except Exception as exc: # noqa: BLE001 + log.warning("Failed to create temporary git config: %s", exc) + return {} + + +def ensure_writable_dir(path, label): + """Return True if *path* is writable, attempting ``sudo chown`` if not.""" + if os.access(path, os.W_OK): + return True + log.warning("%s not writable; attempting sudo chown ...", label) + result = subprocess.run( + ["sudo", "chown", "-R", + "%d:%d" % (os.getuid(), os.getgid()), str(path)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, + ) + if result.returncode == 0 and os.access(path, os.W_OK): + return True + log.error( + "Could not make %s writable (rc=%s): %s", + path, result.returncode, result.stderr.strip(), + ) + return False + + +def collect_interpreters(all_pythons, preferred_python): + """Return a list of Python interpreter paths to build wheels for.""" + if not all_pythons: + return [preferred_python] + found = [] + for candidate in MANYLINUX_PYTHONS: + if Path(candidate).is_file() and os.access(candidate, os.X_OK): + found.append(candidate) + else: + log.debug("Skipping missing interpreter %s", candidate) + if not found: + log.warning("No /opt/python interpreters found; falling back to %s", + preferred_python) + return [preferred_python] + return found + + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- + +def parse_args(): + p = argparse.ArgumentParser( + description="Build AMDSMI Python wheels (Debian / Ubuntu / manylinux).", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--project-dir", type=Path, default=None, + help="AMDSMI project root (auto-detected if omitted).") + p.add_argument("--build-dir", type=Path, default=None, + help="CMake build directory [default: /tmp/amdsmi-build].") + p.add_argument("--raw-wheels-dir", type=Path, default=None, + help="Staging dir for raw wheels [default: /tmp/raw-wheels].") + p.add_argument("--output-dir", type=Path, default=None, + help="Final wheel output dir [default: /wheels].") + p.add_argument("--build-type", default="Release", + choices=["Release", "Debug", "RelWithDebInfo", "MinSizeRel"], + help="CMake build type [default: Release].") + p.add_argument("--enable-esmi", action="store_true", default=True, + help="Enable ESMI library build (default).") + p.add_argument("--no-esmi", action="store_false", dest="enable_esmi", + help="Disable ESMI library build.") + p.add_argument("--python-bin", default="python3", + help="Python interpreter for cmake [default: python3].") + p.add_argument("--all-pythons", action="store_true", + help="Build for all /opt/python interpreters (manylinux).") + p.add_argument("--repair", action="store_true", + help="Run auditwheel repair for manylinux tags.") + p.add_argument("--build-tests", action="store_true", + help="Build C/C++ tests [default: OFF].") + return p.parse_args() + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + args = parse_args() + + # --- project dir & git safety ---------------------------------------- + if args.project_dir is None: + args.project_dir = find_project_dir( + Path(__file__).resolve().parent.parent) + args.project_dir = args.project_dir.resolve() + + git_safe_paths = [] + if not mark_safe_git_dir(args.project_dir): + git_safe_paths.append(args.project_dir) + + esmi_repo = args.project_dir / "esmi_ib_library" + if esmi_repo.exists() and not mark_safe_git_dir(esmi_repo): + git_safe_paths.append(esmi_repo) + + esmi_temp = args.project_dir / "esmi_ib_library_temp" + if esmi_temp.exists(): + log.info("Removing stale esmi_ib_library_temp at %s", esmi_temp) + shutil.rmtree(esmi_temp) + + if esmi_repo.exists() and not ensure_writable_dir(esmi_repo, + "esmi_ib_library"): + abort(str(esmi_repo) + + " is not writable; fix permissions or rerun with sudo.") + + # --- directories ----------------------------------------------------- + if args.build_dir is None: + args.build_dir = DEFAULT_BUILD_DIR + args.build_dir = args.build_dir.resolve() + + raw_wheels_dir = (args.raw_wheels_dir.resolve() + if args.raw_wheels_dir else DEFAULT_RAW_WHEELS_DIR) + output_dir = (args.output_dir.resolve() + if args.output_dir + else args.project_dir / DEFAULT_OUTPUT_NAME) + pkg_dir = args.build_dir / "py-interface" / "python_package" + + # --- interpreters ---------------------------------------------------- + python_bin = shutil.which(args.python_bin) or args.python_bin + python_interpreters = collect_interpreters(args.all_pythons, python_bin) + cmake_python = python_interpreters[0] + + if args.all_pythons and not args.repair: + log.info("Enabling --repair because --all-pythons was requested.") + args.repair = True + + log.info("Project dir : %s", args.project_dir) + log.info("Build dir : %s", args.build_dir) + log.info("Raw wheels : %s", raw_wheels_dir) + log.info("Output dir : %s", output_dir) + log.info("Python (cmake): %s", cmake_python) + log.info("Python targets: %s", ", ".join(python_interpreters)) + + # --- prepare build directory ----------------------------------------- + if args.build_dir == DEFAULT_BUILD_DIR: + if args.build_dir.exists(): + log.info("Removing existing default build dir %s", args.build_dir) + shutil.rmtree(args.build_dir) + args.build_dir.mkdir(parents=True, exist_ok=True) + else: + args.build_dir.mkdir(parents=True, exist_ok=True) + if not os.access(args.build_dir, os.W_OK): + log.warning("Build dir not writable; reclaiming with sudo chown ...") + subprocess.run( + ["sudo", "chown", "-R", + "%d:%d" % (os.getuid(), os.getgid()), + str(args.build_dir)], check=True) + for stale in ("CMakeCache.txt", "CMakeFiles"): + p = args.build_dir / stale + if p.exists(): + log.info("Removing stale %s", p) + shutil.rmtree(p) if p.is_dir() else p.unlink() + + git_env = {} + if git_safe_paths: + git_env = write_temp_git_config( + args.build_dir / "git-safe.config", git_safe_paths) + + # --- prepare wheel directories --------------------------------------- + if raw_wheels_dir.exists(): + log.info("Removing existing raw wheels dir %s", raw_wheels_dir) + shutil.rmtree(raw_wheels_dir) + raw_wheels_dir.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + for existing in output_dir.glob("*.whl"): + log.info("Removing existing wheel %s", existing) + existing.unlink() + + # --- cmake configure & build ----------------------------------------- + run([ + "cmake", str(args.project_dir), + "-DBUILD_TESTS=" + ("ON" if args.build_tests else "OFF"), + "-DENABLE_ESMI_LIB=" + ("ON" if args.enable_esmi else "OFF"), + "-DBUILD_PYTHON_LIB=ON", + "-DCMAKE_BUILD_TYPE=" + args.build_type, + "-DPython3_EXECUTABLE=" + cmake_python, + ], cwd=str(args.build_dir), env=git_env or None) + + run(["make", "-j" + str(os.cpu_count() or 4)], + cwd=str(args.build_dir), env=git_env or None) + + if not pkg_dir.exists(): + abort("Python package directory not found: " + str(pkg_dir)) + + # --- build wheels ---------------------------------------------------- + best_effort_pip_upgrade( + cmake_python, + ["pip", "setuptools", "wheel"] + (["auditwheel"] if args.repair else [])) + + log.info("Building wheels for %d interpreter(s) ...", + len(python_interpreters)) + + for py in python_interpreters: + log.info("--- Building wheel with %s ---", py) + best_effort_pip_upgrade(py, ["pip", "setuptools", "wheel"]) + + for pattern in ("*.whl", "*.egg-info", "build", "dist"): + for path in pkg_dir.glob(pattern): + shutil.rmtree(path) if path.is_dir() else path.unlink() + + run([py, "-m", "pip", "wheel", + "--no-deps", "--no-build-isolation", + "-w", str(raw_wheels_dir), "."], + cwd=str(pkg_dir)) + + raw_wheels = sorted(raw_wheels_dir.glob("*.whl")) + if not raw_wheels: + abort("No wheels found in " + str(raw_wheels_dir)) + + log.info("=== Raw wheel(s) built ===") + for w in raw_wheels: + log.info(" %s (%d KB)", w.name, w.stat().st_size // 1024) + + # --- auditwheel repair (optional) ------------------------------------ + if args.repair: + auditwheel_ok = subprocess.run( + [cmake_python, "-m", "auditwheel", "--version"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ).returncode == 0 + + if auditwheel_ok: + log.info("Repairing wheels with auditwheel ...") + for whl in raw_wheels: + result = run([ + cmake_python, "-m", "auditwheel", "repair", + str(whl), "--wheel-dir", str(output_dir), + ], check=False) + if result.returncode != 0: + log.warning("auditwheel repair failed for %s; " + "copying raw wheel.", whl.name) + shutil.copy2(whl, output_dir) + else: + log.warning("auditwheel not available; copying raw wheels.") + for whl in raw_wheels: + shutil.copy2(whl, output_dir) + else: + for whl in raw_wheels: + shutil.copy2(whl, output_dir) + + # --- summary --------------------------------------------------------- + final_wheels = sorted(output_dir.glob("*.whl")) + log.info("=== Final wheel(s) ===") + for w in final_wheels: + log.info(" %s (%d KB)", w.name, w.stat().st_size // 1024) + log.info("Done. Wheels written to %s", output_dir) + + +if __name__ == "__main__": + main() diff --git a/projects/amdsmi/tools/build_wheel_rpm.py b/projects/amdsmi/tools/build_wheel_rpm.py new file mode 100644 index 00000000000..8d140106beb --- /dev/null +++ b/projects/amdsmi/tools/build_wheel_rpm.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +""" +build_wheel_rpm.py +================== + +Build AMDSMI Python wheels on RPM-based hosts (RHEL, AlmaLinux, SLES, +AzureLinux) or inside a manylinux_2_28 container. + +This script automates the full wheel-build pipeline that the CI workflow +(.github/workflows/manylinux-build.yml) performs: + + 1. (optional) Install build prerequisites via dnf / zypper + 2. cmake configure (-DBUILD_PYTHON_LIB=ON) + 3. make -j$(nproc) + 4. pip wheel --no-deps --no-build-isolation (per interpreter) + 5. auditwheel repair (optional, for manylinux tags) + +Requirements +------------ +* cmake, make, gcc/g++, git, python3, pip +* (optional) auditwheel -- installed automatically when repair is enabled +* (optional) /opt/python/cpXX-cpXX interpreters -- present in manylinux images + +Supported OS Variants +--------------------- +* RHEL 8 / 9 / 10 (dnf) +* AlmaLinux 8 (dnf) +* AzureLinux 3 (dnf) +* SLES (zypper) + +Examples +-------- +Single-interpreter build on a bare RHEL host:: + + python3 build_wheel_rpm.py --project-dir /path/to/amdsmi --skip-install + +Multi-interpreter manylinux build inside quay.io/pypa/manylinux_2_28_x86_64:: + + python3 build_wheel_rpm.py --project-dir /src/amdsmi + +Specify an OS variant explicitly:: + + python3 build_wheel_rpm.py --project-dir /src/amdsmi --os-variant RHEL8 + +Compatibility +------------- +* Python >= 3.6 (f-strings require 3.6; no 3.7+ APIs are used) +* Tested on RHEL 8/9/10, AlmaLinux 8, SLES 15, AzureLinux 3, and + manylinux_2_28 containers. +""" + +import argparse +import glob +import logging +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- +logging.basicConfig( + level=logging.INFO, + format="[%(levelname)s] %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], +) +log = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- +OS_VARIANTS = ("RHEL8", "RHEL9", "RHEL10", "AlmaLinux8", "SLES", "AzureLinux3") +QA_RPATHS_VARIANTS = ("RHEL10", "AlmaLinux8") + +DNF_PREREQS = [ + "git", "make", "gcc", "gcc-c++", "cmake", "ninja-build", "openssl-devel", +] +ZYPPER_PREREQS = [ + "git", "make", "gcc", "gcc-c++", "cmake", "ninja", "libopenssl-devel", +] + + +# --------------------------------------------------------------------------- +# OS variant detection +# --------------------------------------------------------------------------- + +def detect_os_variant(): + """Best-effort detection of the running OS variant from /etc/os-release.""" + os_release = Path("/etc/os-release") + if os_release.exists(): + info = {} + for line in os_release.read_text().splitlines(): + if "=" in line: + k, _, v = line.partition("=") + info[k.strip()] = v.strip().strip('"') + + name = info.get("NAME", "").lower() + version_id = info.get("VERSION_ID", "") + + if "azure linux" in name or "mariner" in name: + return "AzureLinux3" + if "alma" in name: + return "AlmaLinux8" + if "suse" in name or "sles" in name: + return "SLES" + if "rhel" in name or "red hat" in name: + major = version_id.split(".")[0] + return "RHEL%s" % major + + if shutil.which("zypper"): + return "SLES" + return "RHEL9" # generic fallback + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def run(cmd, cwd=None, env=None, check=True): + """Execute *cmd* and stream output to the console.""" + log.info("Running: %s", " ".join(str(c) for c in cmd)) + merged_env = os.environ.copy() + if env: + merged_env.update(env) + return subprocess.run(cmd, check=check, cwd=cwd, env=merged_env) + + +def abort(message, code=1): + log.error(message) + sys.exit(code) + + +def find_project_dir(start): + """Walk up from *start* looking for the first directory with CMakeLists.txt.""" + candidate = start + for _ in range(10): + if (candidate / "CMakeLists.txt").exists(): + return candidate + candidate = candidate.parent + abort("Could not auto-detect project root. Pass --project-dir explicitly.") + + +def best_effort_pip_upgrade(py, packages): + """Try to upgrade *packages* via pip; log a warning on failure.""" + result = subprocess.run( + [py, "-m", "pip", "install", "--upgrade"] + list(packages), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, + ) + if result.returncode != 0: + log.warning( + "pip upgrade failed with %s (packages: %s)\nstdout: %s\nstderr: %s", + py, ", ".join(packages), result.stdout.strip(), result.stderr.strip(), + ) + return result.returncode == 0 + + +def collect_interpreters(python_bins_csv): + """Return a list of Python interpreter paths. + + If *python_bins_csv* is given, split on commas. Otherwise, discover + /opt/python interpreters or fall back to the system ``python3``. + """ + if python_bins_csv: + return [p.strip() for p in python_bins_csv.split(",") if p.strip()] + + found = sorted(glob.glob("/opt/python/cp3*-cp3*/bin/python3")) + if found: + return found + + sys_py = shutil.which("python3") + if sys_py: + return [sys_py] + return [] + + +# --------------------------------------------------------------------------- +# Pipeline steps +# --------------------------------------------------------------------------- + +def install_prerequisites(os_variant, skip): + """Install distro build tools via dnf or zypper.""" + if skip: + log.info("Skipping prerequisite installation (--skip-install).") + return + + if os_variant == "SLES": + log.info("Installing build prerequisites via zypper ...") + run(["zypper", "--non-interactive", "refresh"]) + run(["zypper", "--non-interactive", "install", "-y"] + ZYPPER_PREREQS) + else: + log.info("Installing build prerequisites via dnf ...") + run(["dnf", "-y", "install"] + DNF_PREREQS) + + if os_variant == "AzureLinux3": + log.info("Installing more_itertools for AzureLinux3 ...") + py3 = shutil.which("python3") or "python3" + run([py3, "-m", "pip", "install", "more_itertools"]) + + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- + +def parse_args(): + p = argparse.ArgumentParser( + description="Build AMDSMI Python wheels (RPM-based OS / manylinux).", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--project-dir", type=Path, default=None, + help="AMDSMI project root (auto-detected if omitted).") + p.add_argument("--build-dir", type=Path, + default=Path("/tmp/amdsmi-build"), + help="CMake build directory [default: /tmp/amdsmi-build].") + p.add_argument("--raw-wheels-dir", type=Path, + default=Path("/tmp/raw-wheels"), + help="Staging dir for raw wheels [default: /tmp/raw-wheels].") + p.add_argument("--output-dir", type=Path, + default=Path("/tmp/amdsmi-wheels"), + help="Final wheel output dir [default: /tmp/amdsmi-wheels].") + p.add_argument("--build-type", default="Release", + choices=["Release", "Debug", "RelWithDebInfo", "MinSizeRel"], + help="CMake build type [default: Release].") + p.add_argument("--enable-esmi", action="store_true", default=True, + help="Enable ESMI library build (default).") + p.add_argument("--no-esmi", action="store_false", dest="enable_esmi", + help="Disable ESMI library build.") + p.add_argument("--python-bins", default=None, + help="Comma-separated Python executables (auto-detected).") + p.add_argument("--os-variant", choices=list(OS_VARIANTS), default=None, + help="Target OS variant (auto-detected from /etc/os-release).") + p.add_argument("--skip-install", action="store_true", + help="Skip dnf/zypper prerequisite installation.") + p.add_argument("--repair", action="store_true", default=True, + help="Run auditwheel repair for manylinux tags (default).") + p.add_argument("--skip-repair", action="store_false", dest="repair", + help="Skip the auditwheel repair step.") + p.add_argument("--build-tests", action="store_true", + help="Build C/C++ tests [default: OFF].") + return p.parse_args() + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + args = parse_args() + + # --- OS variant ------------------------------------------------------ + os_variant = args.os_variant or detect_os_variant() + log.info("OS variant : %s", os_variant) + + # --- project dir ----------------------------------------------------- + if args.project_dir is None: + args.project_dir = find_project_dir( + Path(__file__).resolve().parent.parent) + args.project_dir = args.project_dir.resolve() + + # --- interpreters ---------------------------------------------------- + python_interpreters = collect_interpreters(args.python_bins) + if not python_interpreters: + abort("No Python interpreters found. Pass --python-bins explicitly.") + cmake_python = python_interpreters[0] + + log.info("Project dir : %s", args.project_dir) + log.info("Build dir : %s", args.build_dir) + log.info("Raw wheels : %s", args.raw_wheels_dir) + log.info("Output dir : %s", args.output_dir) + log.info("Python (cmake): %s", cmake_python) + log.info("Python targets: %s", ", ".join(python_interpreters)) + + # --- prerequisites --------------------------------------------------- + install_prerequisites(os_variant, args.skip_install) + + # --- prepare build directory ----------------------------------------- + args.build_dir.mkdir(parents=True, exist_ok=True) + for stale in ("CMakeCache.txt", "CMakeFiles"): + p = args.build_dir / stale + if p.exists(): + log.info("Removing stale %s", p) + shutil.rmtree(p) if p.is_dir() else p.unlink() + + # --- cmake configure & build ----------------------------------------- + extra_env = {} + if os_variant in QA_RPATHS_VARIANTS: + qa_rpaths = hex(0x0010 | 0x0002) + extra_env["QA_RPATHS"] = qa_rpaths + log.info("Setting QA_RPATHS=%s for %s", qa_rpaths, os_variant) + + run([ + "cmake", str(args.project_dir), + "-DBUILD_TESTS=" + ("ON" if args.build_tests else "OFF"), + "-DENABLE_ESMI_LIB=" + ("ON" if args.enable_esmi else "OFF"), + "-DBUILD_PYTHON_LIB=ON", + "-DCMAKE_BUILD_TYPE=" + args.build_type, + "-DPython3_EXECUTABLE=" + cmake_python, + ], cwd=str(args.build_dir), env=extra_env or None) + + run(["make", "-j" + str(os.cpu_count() or 4)], + cwd=str(args.build_dir), env=extra_env or None) + + pkg_dir = args.build_dir / "py-interface" / "python_package" + if not pkg_dir.exists(): + abort("Python package directory not found: " + str(pkg_dir)) + + # --- prepare wheel directories --------------------------------------- + raw_wheels_dir = args.raw_wheels_dir + output_dir = args.output_dir + + if raw_wheels_dir.exists(): + log.info("Removing existing raw wheels dir %s", raw_wheels_dir) + shutil.rmtree(raw_wheels_dir) + raw_wheels_dir.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + for existing in output_dir.glob("*.whl"): + log.info("Removing existing wheel %s", existing) + existing.unlink() + + # --- build wheels ---------------------------------------------------- + best_effort_pip_upgrade( + cmake_python, + ["pip", "setuptools", "wheel"] + (["auditwheel"] if args.repair else [])) + + log.info("Building wheels for %d interpreter(s) ...", + len(python_interpreters)) + + for py in python_interpreters: + if not os.path.isfile(py) or not os.access(py, os.X_OK): + log.warning("Skipping missing/non-executable interpreter: %s", py) + continue + + log.info("--- Building wheel with %s ---", py) + best_effort_pip_upgrade(py, ["pip", "setuptools", "wheel"]) + + for pattern in ("*.whl", "*.egg-info", "build", "dist"): + for path in pkg_dir.glob(pattern): + shutil.rmtree(path) if path.is_dir() else path.unlink() + + run([py, "-m", "pip", "wheel", + "--no-deps", "--no-build-isolation", + "-w", str(raw_wheels_dir), "."], + cwd=str(pkg_dir)) + + raw_wheels = sorted(raw_wheels_dir.glob("*.whl")) + if not raw_wheels: + abort("No wheels found in " + str(raw_wheels_dir)) + + log.info("=== Raw wheel(s) built ===") + for w in raw_wheels: + log.info(" %s (%d KB)", w.name, w.stat().st_size // 1024) + + # --- auditwheel repair (optional) ------------------------------------ + if args.repair: + auditwheel_ok = subprocess.run( + [cmake_python, "-m", "auditwheel", "--version"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ).returncode == 0 + + if auditwheel_ok: + log.info("Repairing wheels with auditwheel ...") + for whl in raw_wheels: + result = run([ + cmake_python, "-m", "auditwheel", "repair", + str(whl), "--wheel-dir", str(output_dir), + ], check=False) + if result.returncode != 0: + log.warning("auditwheel repair failed for %s; " + "copying raw wheel.", whl.name) + shutil.copy2(whl, output_dir) + else: + log.warning("auditwheel not available; copying raw wheels.") + for whl in raw_wheels: + shutil.copy2(whl, output_dir) + else: + for whl in raw_wheels: + shutil.copy2(whl, output_dir) + + # --- summary --------------------------------------------------------- + final_wheels = sorted(output_dir.glob("*.whl")) + log.info("=== Final wheel(s) ===") + for w in final_wheels: + log.info(" %s (%d KB)", w.name, w.stat().st_size // 1024) + log.info("Done. Wheels written to %s", output_dir) + + +if __name__ == "__main__": + main() diff --git a/projects/amdsmi/tools/generator.py b/projects/amdsmi/tools/generator.py index 3683ffa0a35..b0d459e17c9 100644 --- a/projects/amdsmi/tools/generator.py +++ b/projects/amdsmi/tools/generator.py @@ -23,6 +23,7 @@ import shutil import platform from subprocess import run, PIPE +from pathlib import Path from ctypeslib.clang2py import main as clangToPy HEADER = """# Copyright (C) Advanced Micro Devices. All rights reserved. @@ -182,54 +183,134 @@ def main(): library_path = os.path.join(os.path.dirname(__file__), library) line_to_replace = "_libraries['{}'] = ctypes.CDLL('{}')".format(library_name, library_path) new_line = f"""from pathlib import Path -# {library_name} can be located in several different places. -# Look for it with below priority: -# 0. Relative to amdsmi_wrapper.py in TheRock: -# `amdsmi_wrapper.py` is located in -# `_rocm_sdk_core/share/amd_smi/amdsmi`, libraries are in -# `_rocm_sdk_core/lib`. -# 1. ROCM_HOME/ROCM_PATH environment variables -# - ROCM_HOME/lib -# - ROCM_PATH/lib (usually set to /opt/rocm/) -# 2. Decided by the linker -# - LD_LIBRARY_PATH env var -# - defined path in /etc/ld.so.conf.d/ -# 3. Relative to amdsmi_wrapper.py -# - parent directory -# - current directory -def find_smi_library(): - err = OSError("Could not load {library_name}") - possible_locations = [] - # 0. - libamd_smi_path = Path(__file__).resolve().parent.parent.parent.parent / "lib/libamd_smi.so.26" - possible_locations.append(libamd_smi_path) - # 1. - rocm_path = os.getenv("ROCM_HOME", os.getenv("ROCM_PATH")) - if rocm_path: - possible_locations.append(os.path.join(rocm_path, "lib/{library_name}")) - # 2. - possible_locations.append("{library_name}") - # 3. - libamd_smi_parent_dir = Path(__file__).resolve().parent / "{library_name}" - libamd_smi_cwd = Path.cwd() / "{library_name}" - possible_locations.append(libamd_smi_parent_dir) - possible_locations.append(libamd_smi_cwd) - - for location in possible_locations: + +# --------------------------------------------------------------------------- +# Dynamic library loading +# --------------------------------------------------------------------------- +# Two installation contexts exist, each shipping its own .so: +# +# 1. Linux system package (RPM / DEB) +# wrapper: /opt/rocm/share/amd_smi/amdsmi/amdsmi_wrapper.py +# library: /opt/rocm/lib/{library_name} +# Note: /opt/rocm may be a symlink (e.g. /opt/rocm -> /opt/rocm-X.Y.Z). +# Path.resolve() handles this transparently. +# +# 2. Python pip package (wheel) +# wrapper: /amdsmi/amdsmi_wrapper.py +# library: /amdsmi/libamd_smi_python.so +# The exact site-packages location varies per distro / venv / conda. +# +# Detection strategy (based on the absolute *resolved* path of THIS file): +# - If libamd_smi_python.so exists next to this wrapper -> pip context +# - Otherwise -> system-package context (derive ROCm root from path or env) +# +# Regardless of which .so is loaded, it is stored under the key +# _libraries['{library_name}'] +# so every downstream ctypes binding in this wrapper works unchanged. +# --------------------------------------------------------------------------- + +_libraries = {{}} + + +def _detect_install_context(): + \"\"\"Classify the current install as ``"pip"`` or ``"system"``. + + Returns + ------- + tuple[str, Path] + ``("pip", module_dir)`` - *module_dir* contains the wrapper **and** + ``libamd_smi_python.so``. + ``("system", rocm_root)`` - *rocm_root* is the resolved ROCm prefix + (e.g. ``/opt/rocm``). + + All paths are fully resolved so symlinks like + ``/opt/rocm -> /opt/rocm-X.Y.Z`` are handled automatically. + \"\"\" + wrapper_path = Path(__file__).resolve() + module_dir = wrapper_path.parent # .../amdsmi/ + + # pip context + # The wheel ships libamd_smi_python.so right next to the wrapper. + if (module_dir / "libamd_smi_python.so").exists(): + return "pip", module_dir + + # system-package context + # Expected layout: + # /share/amd_smi/amdsmi/amdsmi_wrapper.py + # module_dir = /share/amd_smi/amdsmi + # rocm_root = module_dir / ../../.. (3 dirs up) + potential_rocm_root = module_dir.parent.parent.parent + if (potential_rocm_root / "lib").is_dir(): + return "system", potential_rocm_root + + # Fallback to ROCM_HOME / ROCM_PATH environment variables + for env_var in ("ROCM_HOME", "ROCM_PATH"): + env_val = os.getenv(env_var) + if env_val: + p = Path(env_val).resolve() + if (p / "lib").is_dir(): + return "system", p + + # Last resort - default ROCm location. + return "system", Path("/opt/rocm").resolve() + + +def _build_candidate_paths(): + \"\"\"Return an ordered list of .so paths to try, best-match first.\"\"\" + context, base = _detect_install_context() + candidates = [] + + if context == "pip": + # .so lives alongside the wrapper inside the wheel / site-packages + candidates.append(base / "libamd_smi_python.so") + else: + # System package - .so lives under /lib/ + candidates.append(base / "lib" / "{library_name}") + + # Fallbacks + for env_var in ("ROCM_HOME", "ROCM_PATH"): + env_val = os.getenv(env_var) + if env_val: + candidates.append(Path(env_val) / "lib" / "{library_name}") + + # Let the dynamic linker try LD_LIBRARY_PATH / ld.so.conf.d + candidates.append("{library_name}") + + return candidates + + +def _load_library(): + \"\"\"Load the AMD SMI shared library for the detected install context. + + Returns + ------- + tuple[ctypes.CDLL, str] + The loaded library handle and the path that was successfully loaded. + + Raises + ------ + OSError + If none of the candidate paths could be loaded. + \"\"\" + candidates = _build_candidate_paths() + last_err = None + mode = getattr(ctypes, "RTLD_GLOBAL", 0) + + for candidate in candidates: try: - lib = ctypes.CDLL(location) - return lib, location - except OSError as e: - err = e - continue - raise err - -try: - _libraries['{library_name}'], location = find_smi_library() - #print(f"found smi lib in [", location, "]") -except OSError as e: - print(e) - print("Unable to find {library_name} library try installing amd-smi-lib from your package manager") + lib = ctypes.CDLL(str(candidate), mode=mode) + return lib, str(candidate) + except OSError as exc: + last_err = exc + + raise last_err or OSError( + "Could not load AMD SMI library. Searched:\\n" + + "\\n".join(f" - {{c}}" for c in candidates) + ) + + +# The dict key stays '{library_name}' regardless of the actual .so loaded +_libraries['{library_name}'], _loaded_lib_path = _load_library() #Add support for amdsmi_free_name_value_pairs amdsmi_free_name_value_pairs = _libraries['libamd_smi.so'].amdsmi_free_name_value_pairs