From 0e7c30a7e240261be1305d2f49f0faa14fa692d3 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Tue, 10 Feb 2026 22:57:34 -0500 Subject: [PATCH 01/21] Windows: Initial Bring-Up --- .github/workflows/buildRyzenWheels.yml | 257 ++- .github/workflows/mlirAIEDistro.yml | 49 + CMakeLists.txt | 31 +- cmake/modulesXilinx | 2 +- python/CMakeLists.txt | 9 +- python/aie_lit_utils/lit_config_helpers.py | 28 +- python/requirements_dev.txt | 4 +- python/requirements_ml.txt | 2 +- python/utils/compile/cache/utils.py | 52 +- tools/CMakeLists.txt | 4 +- tools/bootgen/CMakeLists.txt | 11 + utils/iron_setup.py | 760 +++++++++ utils/mlir_aie_wheels/pyproject.toml | 2 +- .../python_bindings/CMakeLists.txt | 44 +- .../python_bindings/pyproject.toml | 10 +- .../mlir_aie_wheels/python_bindings/setup.py | 65 +- .../mlir_aie_wheels/scripts/download_mlir.py | 249 +++ utils/mlir_aie_wheels/setup.py | 26 +- utils/run_example.py | 1464 +++++++++++++++++ 19 files changed, 2990 insertions(+), 79 deletions(-) create mode 100644 utils/iron_setup.py create mode 100644 utils/mlir_aie_wheels/scripts/download_mlir.py create mode 100644 utils/run_example.py diff --git a/.github/workflows/buildRyzenWheels.yml b/.github/workflows/buildRyzenWheels.yml index 6da9779b572..3eacf5e28b2 100644 --- a/.github/workflows/buildRyzenWheels.yml +++ b/.github/workflows/buildRyzenWheels.yml @@ -161,8 +161,8 @@ jobs: if [ x"${{ inputs.AIE_COMMIT }}" == x"" ]; then export AIE_PROJECT_COMMIT=$MLIR_VERSION else - MLIR_VERSION = "${{ inputs.AIE_COMMIT }}" - export AIE_PROJECT_COMMIT=${MLIR_VERSION:0:7} + export AIE_PROJECT_COMMIT="${{ inputs.AIE_COMMIT }}" + export AIE_PROJECT_COMMIT="${AIE_PROJECT_COMMIT:0:7}" fi export AIE_VITIS_COMPONENTS='AIE2;AIE2P' export DATETIME=$(date +"%Y%m%d%H") @@ -182,17 +182,252 @@ jobs: path: wheelhouse/repaired_wheel/mlir_aie*whl name: mlir_aie_rtti_${{ matrix.ENABLE_RTTI }}-${{ matrix.python_version }} + + build-windows: + name: Build and upload mlir_aie wheels (Windows) + + runs-on: windows-2022 + + permissions: + id-token: write + contents: write + packages: read + + strategy: + fail-fast: false + matrix: + include: + - python_version: "3.10" + ENABLE_RTTI: ON + + - python_version: "3.10" + ENABLE_RTTI: OFF + + - python_version: "3.11" + ENABLE_RTTI: ON + + - python_version: "3.11" + ENABLE_RTTI: OFF + + - python_version: "3.12" + ENABLE_RTTI: ON + + - python_version: "3.12" + ENABLE_RTTI: OFF + + - python_version: "3.13" + ENABLE_RTTI: ON + + - python_version: "3.13" + ENABLE_RTTI: OFF + + - python_version: "3.14" + ENABLE_RTTI: ON + + - python_version: "3.14" + ENABLE_RTTI: OFF + + steps: + - uses: actions/checkout@v4 + with: + submodules: "true" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + allow-prereleases: true + + - name: Set up MSVC dev environment + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + # We don't get OpenSSL for free on Windows. Manage the pain. + # Try vcpkg's GHA cache to avoid building OpenSSL for every matrix element. + - name: Enable vcpkg binary caching + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL) + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN) + - name: Install OpenSSL + shell: pwsh + run: | + $vcpkgRoot = $env:VCPKG_INSTALLATION_ROOT + if ($vcpkgRoot) { + $vcpkg = Join-Path $vcpkgRoot "vcpkg.exe" + } else { + $vcpkg = (Get-Command vcpkg.exe -ErrorAction Stop).Source + $vcpkgRoot = Split-Path -Parent $vcpkg + } + + $env:VCPKG_BINARY_SOURCES = "clear;x-gha,readwrite" + & $vcpkg install openssl:x64-windows + + "OPENSSL_ROOT_DIR=$($vcpkgRoot -replace '\','/')/installed/x64-windows" | + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Build mlir-aie wheel + shell: bash + env: + # Copy Linux job style: tag-aligned versioning only on version tags. + # Perhaps a literal "" should be here? + AIE_WHEEL_VERSION: ${{ (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) && github.ref_name || '' }} + ENABLE_RTTI: ${{ matrix.ENABLE_RTTI }} + CMAKE_GENERATOR: Ninja + CMAKE_ARGS: -DAIE_BUILD_CHESS_CLANG=OFF + run: | + set -euo pipefail + + # No Vitis on Windows! + unset VITIS XILINXD_LICENSE_FILE || true + + git config --global --add safe.directory "$PWD" + MLIR_VERSION=$(git rev-parse --short HEAD) + echo "Building mlir-aie version $MLIR_VERSION" + + python -m venv aie-venv + source aie-venv/Scripts/activate + + python -m pip install --upgrade pip + pip install -r python/requirements_ml.txt + pip install -r python/requirements_dev.txt + + export ENABLE_RTTI="${ENABLE_RTTI}" + + NO_RTTI="" # Set a default value + NO_RTTI_UNDERSCORE="" # Set a default value + if [ x"$ENABLE_RTTI" == x"OFF" ]; then + NO_RTTI="-no-rtti" + NO_RTTI_UNDERSCORE="_no_rtti" + fi + + VERSION=$(utils/clone-llvm.sh --get-wheel-version) + + # Grab the MLIR distro wheel and extract + pip -q download "mlir${NO_RTTI}==${VERSION}" -f https://github.com/Xilinx/mlir-aie/releases/expanded_assets/mlir-distro + python -m zipfile -e mlir*.whl . + + # Linux-style timestamp magic should work fine on Windows. + find "mlir${NO_RTTI_UNDERSCORE}" -exec touch -a -m -t 201108231405.14 {} \; + + # Match Linux wheel version metadata on non-tag builds. + export DATETIME=$(date +"%Y%m%d%H") + if [ x"${{ inputs.AIE_COMMIT }}" == x"" ]; then + export AIE_PROJECT_COMMIT=$MLIR_VERSION + else + export AIE_PROJECT_COMMIT="${{ inputs.AIE_COMMIT }}" + export AIE_PROJECT_COMMIT="${AIE_PROJECT_COMMIT:0:7}" + fi + export AIE_VITIS_COMPONENTS='AIE2;AIE2P' + + # Try to always forward slash paths. + ROOT_WIN=$(python -c "import os; print(os.getcwd().replace('\\\\', '/'))") + + export MLIR_INSTALL_ABS_PATH="${ROOT_WIN}/mlir${NO_RTTI_UNDERSCORE}" + export MLIR_AIE_SOURCE_DIR="${ROOT_WIN}" + export WHEELHOUSE_DIR="${ROOT_WIN}/wheelhouse" + export CMAKE_MODULE_PATH="${ROOT_WIN}/cmake/modulesXilinx" + + mkdir -p "${WHEELHOUSE_DIR}" + pushd utils/mlir_aie_wheels + + pip install wheel importlib_metadata ninja + CIBW_ARCHS=AMD64 pip wheel . -v -w "${WHEELHOUSE_DIR}" --no-build-isolation + + popd + + # Try to repair the wheel on Windows using delvewheel (closest auditwheel equivalent). + pip install delvewheel + + OPENSSL_BIN=$(python -c "import os, pathlib; print(pathlib.Path(os.environ['OPENSSL_ROOT_DIR']).joinpath('bin').as_posix())") + python -m delvewheel repair --analyze-existing-exes -w "${WHEELHOUSE_DIR}/repaired_wheel" "${WHEELHOUSE_DIR}"/mlir_aie*.whl --add-path "${OPENSSL_BIN}" + + - name: Upload mlir_aie + uses: actions/upload-artifact@v4 + with: + path: wheelhouse/repaired_wheel/mlir_aie*whl + name: mlir_aie_windows_rtti_${{ matrix.ENABLE_RTTI }}-${{ matrix.python_version }} + + publish: + name: Publish wheels + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) + runs-on: ubuntu-latest + needs: [build-repo, build-windows] + permissions: + contents: write + actions: read + steps: + - name: Download wheels (RTTI ON) + uses: actions/download-artifact@v4 + with: + pattern: mlir_aie*_rtti_ON-* + path: wheels_on + merge-multiple: true + + - name: Download wheels (RTTI OFF) + uses: actions/download-artifact@v4 + with: + pattern: mlir_aie*_rtti_OFF-* + path: wheels_off + merge-multiple: true + + - name: Flatten wheels (RTTI ON) + run: | + set -euo pipefail + mkdir -p wheels_on_flat + find wheels_on -name 'mlir_aie*whl' -exec cp -f {} wheels_on_flat/ \; + + - name: Flatten wheels (RTTI OFF) + run: | + set -euo pipefail + mkdir -p wheels_off_flat + find wheels_off -name 'mlir_aie*whl' -exec cp -f {} wheels_off_flat/ \; + + - name: Combine wheels + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + run: | + set -euo pipefail + mkdir -p wheels_all + cp wheels_on_flat/mlir_aie*whl wheels_all/ + cp wheels_off_flat/mlir_aie*whl wheels_all/ + - name: Release - if: | - github.event_name == 'workflow_dispatch' || - github.event_name == 'schedule' || - (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + uses: ncipollo/release-action@v1.12.0 + with: + artifacts: wheels_all/mlir_aie*whl + token: "${{ secrets.GITHUB_TOKEN }}" + tag: ${{ github.ref_name }} + name: ${{ github.ref_name }} + allowUpdates: true + replacesArtifacts: true + makeLatest: true + + - name: Release latest wheels (RTTI ON) + if: github.event_name != 'push' + uses: ncipollo/release-action@v1.12.0 + with: + artifacts: wheels_on_flat/mlir_aie*whl + token: "${{ secrets.GITHUB_TOKEN }}" + tag: latest-wheels-3 + name: latest-wheels-3 + allowUpdates: true + replacesArtifacts: false + makeLatest: false + + - name: Release latest wheels (RTTI OFF) + if: github.event_name != 'push' uses: ncipollo/release-action@v1.12.0 with: - artifacts: wheelhouse/repaired_wheel/mlir_aie*whl + artifacts: wheels_off_flat/mlir_aie*whl token: "${{ secrets.GITHUB_TOKEN }}" - tag: ${{ github.event_name == 'push' && github.ref_name || (matrix.ENABLE_RTTI == 'ON' && 'latest-wheels-3' || 'latest-wheels-no-rtti') }} - name: ${{ github.event_name == 'push' && github.ref_name || (matrix.ENABLE_RTTI == 'ON' && 'latest-wheels-3' || 'latest-wheels-no-rtti') }} + tag: latest-wheels-no-rtti + name: latest-wheels-no-rtti allowUpdates: true - replacesArtifacts: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} - makeLatest: ${{ github.event_name == 'push' }} + replacesArtifacts: false + makeLatest: false diff --git a/.github/workflows/mlirAIEDistro.yml b/.github/workflows/mlirAIEDistro.yml index fbcecd61bf5..4ef8621870e 100644 --- a/.github/workflows/mlirAIEDistro.yml +++ b/.github/workflows/mlirAIEDistro.yml @@ -161,6 +161,35 @@ jobs: MATRIX_OS: ${{ matrix.OS }} MATRIX_ARCH: ${{ matrix.ARCH }} + # We don't get OpenSSL for free on Windows. Manage the pain. + # Try vcpkg's GHA cache to avoid building OpenSSL for every matrix element. + - name: Enable vcpkg binary caching + if: ${{ matrix.OS == 'windows-2022' }} + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL) + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN) + + # Install OpenSSL so bootgen.exe can build against it. + - name: Install OpenSSL + if: ${{ matrix.OS == 'windows-2022' }} + shell: pwsh + run: | + $vcpkgRoot = $env:VCPKG_INSTALLATION_ROOT + if ($vcpkgRoot) { + $vcpkg = Join-Path $vcpkgRoot "vcpkg.exe" + } else { + $vcpkg = (Get-Command vcpkg.exe -ErrorAction Stop).Source + $vcpkgRoot = Split-Path -Parent $vcpkg + } + + $env:VCPKG_BINARY_SOURCES = "clear;x-gha,readwrite" + & $vcpkg install openssl:x64-windows + + "OPENSSL_ROOT_DIR=$($vcpkgRoot -replace '\','/')/installed/x64-windows" | + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - uses: ./.github/actions/setup_ccache id: setup_ccache with: @@ -221,6 +250,26 @@ jobs: PARALLEL_LEVEL=2 \ cibuildwheel --output-dir wheelhouse + - name: Repair wheels (Windows) + if: ${{ matrix.OS == 'windows-2022' }} + shell: pwsh + working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} + run: | + python -m pip install --upgrade pip + pip install delvewheel + + $opensslBin = Join-Path $env:OPENSSL_ROOT_DIR "bin" + New-Item -ItemType Directory -Force -Path wheelhouse\repaired_wheel | Out-Null + + python -m delvewheel repair --analyze-existing-exes ` + -w wheelhouse\repaired_wheel ` + wheelhouse\mlir_aie*.whl ` + --add-path $opensslBin + + Remove-Item wheelhouse\mlir_aie*.whl -Force + Move-Item wheelhouse\repaired_wheel\*.whl wheelhouse\ + Remove-Item wheelhouse\repaired_wheel -Recurse -Force + - name: build aarch ubuntu wheel if: ${{ matrix.OS == 'ubuntu-22.04' && matrix.ARCH == 'aarch64' }} working-directory: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cb8293af41..e4adbd9b577 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,6 +101,16 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) set(AIE_TOOLS_BINARY_DIR ${AIE_BINARY_DIR}/bin) set(AIE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +if(WIN32) + # Prefer in-tree Find*.cmake modules (FindVitis.cmake, FindXRT.cmake, etc.). + list(PREPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modulesXilinx") + # Force xchesscc off on Windows, since the Vitis chess toolchain is not supported. + set(_AIE_BUILD_CHESS_CLANG_DEFAULT OFF) +else() + set(_AIE_BUILD_CHESS_CLANG_DEFAULT ON) +endif() +option(AIE_BUILD_CHESS_CLANG "Build chess-clang (requires Vitis chess toolchain)" ${_AIE_BUILD_CHESS_CLANG_DEFAULT}) + find_package(Vitis 2023.2 COMPONENTS ${AIE_VITIS_COMPONENTS}) configure_file(./utils/vitisVariables.config.in ${CMAKE_BINARY_DIR}/utils/vitisVariables.config @ONLY) @@ -160,12 +170,29 @@ cmake_dependent_option(AIE_ENABLE_PYTHON_PASSES "Enables building of passes that connect to python." OFF "AIE_ENABLE_BINDINGS_PYTHON;LLVM_ENABLE_RTTI" OFF) find_library(XRT_COREUTIL xrt_coreutil HINTS ${XRT_LIB_DIR}) -find_library(UUID uuid) + +# On Windows, the UUID library is linked implicitly. +# Just tell CMake that it's available. +if(WIN32) + set(UUID TRUE) +else() + find_library(UUID uuid) +endif() + cmake_dependent_option(AIE_ENABLE_XRT_PYTHON_BINDINGS "Enables building of python bindings to XRT runtime." ON "XRT_COREUTIL;UUID" OFF) + if (AIE_ENABLE_XRT_PYTHON_BINDINGS) add_library(xrt_coreutil SHARED IMPORTED) - set_property(TARGET xrt_coreutil PROPERTY IMPORTED_LOCATION "${XRT_COREUTIL}") + if (WIN32) + # FindXRT.cmake returns only an import lib. We also need the runtime dll. + set_target_properties(xrt_coreutil PROPERTIES + IMPORTED_IMPLIB "${XRT_COREUTIL}" + IMPORTED_LOCATION "${XRT_BIN_DIR}/xrt_coreutil.dll" + ) + else() + set_property(TARGET xrt_coreutil PROPERTY IMPORTED_LOCATION "${XRT_COREUTIL}") + endif() endif() cmake_dependent_option(AIECC_COMPILE diff --git a/cmake/modulesXilinx b/cmake/modulesXilinx index 6c8f84fe39c..c3ddfc43104 160000 --- a/cmake/modulesXilinx +++ b/cmake/modulesXilinx @@ -1 +1 @@ -Subproject commit 6c8f84fe39c967de07a566af1eef5d1759d5d36f +Subproject commit c3ddfc43104fb0e6f1856f275defd644ba87e477 diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 81e203ad748..58f0acf6dc8 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -211,7 +211,10 @@ if (AIE_ENABLE_PYTHON_PASSES) if (AIE_ENABLE_XRT_PYTHON_BINDINGS) list(APPEND _py_srcs ${CMAKE_CURRENT_SOURCE_DIR}/XRTModule.cpp) - list(APPEND _py_libs xrt_coreutil uuid) + list(APPEND _py_libs xrt_coreutil) + if(NOT WIN32) + list(APPEND _py_libs uuid) + endif() endif() list(APPEND _py_srcs ${CMAKE_CURRENT_SOURCE_DIR}/AIERTModule.cpp) @@ -339,10 +342,12 @@ else () PRIVATE_LINK_LIBS LLVMSupport xrt_coreutil - uuid PYTHON_BINDINGS_LIBRARY nanobind ) + if(NOT WIN32) + target_link_libraries(AIEPythonExtensions.XRT PRIVATE uuid) + endif() target_include_directories(AIEPythonExtensions.XRT INTERFACE ${XRT_INCLUDE_DIR}) target_link_directories(AIEPythonExtensions.XRT INTERFACE ${XRT_LIB_DIR}) endif() diff --git a/python/aie_lit_utils/lit_config_helpers.py b/python/aie_lit_utils/lit_config_helpers.py index 77236bf159c..81f05e2a0df 100644 --- a/python/aie_lit_utils/lit_config_helpers.py +++ b/python/aie_lit_utils/lit_config_helpers.py @@ -216,16 +216,26 @@ def detect_xrt( print(f"xrt found at {os.path.dirname(xrt_lib_dir)}") config.found = True - config.flags = f"-I{xrt_include_dir} -L{xrt_lib_dir} -luuid -lxrt_coreutil" + if os.name == "nt": + config.flags = f"-I{xrt_include_dir} -L{xrt_lib_dir} -lxrt_coreutil" + else: + config.flags = f"-I{xrt_include_dir} -L{xrt_lib_dir} -luuid -lxrt_coreutil" config.substitutions["%xrt_flags"] = config.flags - # Add XRT library directory to LD_LIBRARY_PATH for runtime linking, - # preserving any existing entries from the parent environment. - existing_ld_library_path = os.environ.get("LD_LIBRARY_PATH") - if existing_ld_library_path: - new_ld_library_path = existing_ld_library_path + os.pathsep + xrt_lib_dir + # Runtime library search path. + if os.name == "nt": + # Windows: ensure XRT DLLs can be resolved at runtime. + existing_path = os.environ.get("PATH", "") + config.environment["PATH"] = ( + xrt_bin_dir + os.pathsep + existing_path if existing_path else xrt_bin_dir + ) else: - new_ld_library_path = xrt_lib_dir - config.environment["LD_LIBRARY_PATH"] = new_ld_library_path + # Linux: add XRT library directory to LD_LIBRARY_PATH. + existing_ld_library_path = os.environ.get("LD_LIBRARY_PATH") + if existing_ld_library_path: + new_ld_library_path = existing_ld_library_path + os.pathsep + xrt_lib_dir + else: + new_ld_library_path = xrt_lib_dir + config.environment["LD_LIBRARY_PATH"] = new_ld_library_path # Detect NPU hardware try: @@ -242,7 +252,7 @@ def detect_xrt( # Old: "|[0000:41:00.1] ||RyzenAI-npu1 |" # New: "|[0000:41:00.1] |NPU Phoenix |" pattern = re.compile( - r"[\|]?(\[.+:.+:.+\]).+\|(RyzenAI-(npu\d)|NPU ([\w ]+?))\s*\|" + r"[\|]?(\[.+:.+:.+\]).+\|(RyzenAI-(npu\d)|NPU (\w+))\W*\|" ) for line in output: diff --git a/python/requirements_dev.txt b/python/requirements_dev.txt index 07b59595504..d94984fa5c7 100644 --- a/python/requirements_dev.txt +++ b/python/requirements_dev.txt @@ -1,7 +1,9 @@ -cmake>=3.29, <4.0 +cmake>=3.31, <4.0 pybind11>=2.13.0 setuptools>=61.0 wheel +ninja +cibuildwheel pre-commit nanobind>=2.9 lit diff --git a/python/requirements_ml.txt b/python/requirements_ml.txt index 2c5b613ae7a..188c44858be 100644 --- a/python/requirements_ml.txt +++ b/python/requirements_ml.txt @@ -1,2 +1,2 @@ ---index-url https://download.pytorch.org/whl/cpu +--extra-index-url https://download.pytorch.org/whl/cpu torch diff --git a/python/utils/compile/cache/utils.py b/python/utils/compile/cache/utils.py index e45cb288e89..c4b77b94978 100644 --- a/python/utils/compile/cache/utils.py +++ b/python/utils/compile/cache/utils.py @@ -6,13 +6,53 @@ # # (c) Copyright 2025-2026 Advanced Micro Devices, Inc. import contextlib -import fcntl import os import time +# Cross-platform file locking: +# - POSIX: fcntl.flock +# - Windows: msvcrt.locking +if os.name == "nt": + import msvcrt +else: + import fcntl + from aie.utils.hostruntime.tensor_class import Tensor +def _try_acquire_lock(lock_file) -> bool: + if os.name == "nt": + # msvcrt.locking locks a byte-range starting at the current file position. + # Try to use the first byte to provide a process-wide lock. + # Hacky and brittle, but works well enough to prevent a race. + try: + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1) + return True + except OSError: + return False + else: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except OSError: + return False + + +def _release_lock(lock_file) -> None: + if os.name == "nt": + try: + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + except OSError: + pass + else: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + except OSError: + pass + + def _create_function_cache_key(function, args, kwargs): """ Create a cache key for a function call based on function name and argument types/shapes. @@ -88,18 +128,18 @@ def file_lock(lock_file_path, timeout_seconds=60): # Create lock file if it doesn't exist os.makedirs(os.path.dirname(lock_file_path), exist_ok=True) try: - f = os.open(lock_file_path, os.O_CREAT | os.O_EXCL) + f = os.open(lock_file_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) os.close(f) except FileExistsError: pass # File already exists - lock_file = open(lock_file_path, "a") + lock_file = open(lock_file_path, "a+") # Try to acquire exclusive lock with timeout start_time = time.time() while True: try: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - break + if _try_acquire_lock(lock_file): + break except OSError: # Lock is held by another process if time.time() - start_time > timeout_seconds: @@ -113,7 +153,7 @@ def file_lock(lock_file_path, timeout_seconds=60): finally: if lock_file is not None: try: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + _release_lock(lock_file) except OSError: pass # Ignore errors when releasing lock lock_file.close() diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 83ad092bb87..d448ad98a35 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -13,4 +13,6 @@ add_subdirectory(aie-lsp-server) add_subdirectory(aie-translate) add_subdirectory(aie-visualize) add_subdirectory(bootgen) -add_subdirectory(chess-clang) +if(AIE_BUILD_CHESS_CLANG) + add_subdirectory(chess-clang) +endif() diff --git a/tools/bootgen/CMakeLists.txt b/tools/bootgen/CMakeLists.txt index 4d8095f7ed0..ced5fc39956 100644 --- a/tools/bootgen/CMakeLists.txt +++ b/tools/bootgen/CMakeLists.txt @@ -136,6 +136,10 @@ else() endif() if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") target_compile_definitions(bootgen-lib PRIVATE YY_NO_UNISTD_H) + # Enable correct unwind semantics on MSVC + target_compile_options(bootgen-lib PRIVATE /EHsc) + # Suppress MSVC warning noise + target_compile_options(bootgen-lib PRIVATE /wd4996 /wd4840 /wd4005 /wd5033) endif() target_compile_options(bootgen-lib PRIVATE ${bootgen_warning_ignores}) target_include_directories(bootgen-lib PRIVATE ${BOOTGEN_SOURCE} ${OPENSSL_INCLUDE_DIR}) @@ -148,6 +152,10 @@ add_executable(bootgen ${BOOTGEN_SOURCE}/main.cpp) target_include_directories(bootgen PUBLIC ${BOOTGEN_SOURCE} ${OPENSSL_INCLUDE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/include) target_compile_options(bootgen PRIVATE ${bootgen_warning_ignores}) +if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + target_compile_options(bootgen PRIVATE /EHsc) + target_compile_options(bootgen PRIVATE /wd4996 /wd4840 /wd4005 /wd5033) +endif() target_compile_definitions(bootgen PRIVATE OPENSSL_USE_APPLINK) target_link_libraries(bootgen PRIVATE bootgen-lib OpenSSL::SSL OpenSSL::applink) install(TARGETS bootgen) @@ -160,6 +168,9 @@ if (NOT CMAKE_CXX_COMPILER_ID MATCHES "MSVC") target_compile_options(cdo_driver_mlir_aie PRIVATE -Wno-cast-qual -Wno-sign-compare) endif() +if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + target_compile_options(cdo_driver_mlir_aie PRIVATE /wd4996 /wd4840 /wd4005 /wd5033) +endif() install( TARGETS cdo_driver_mlir_aie DESTINATION lib diff --git a/utils/iron_setup.py b/utils/iron_setup.py new file mode 100644 index 00000000000..6a9432ec967 --- /dev/null +++ b/utils/iron_setup.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python3 +##===------ iron_setup.py - One-stop mlir-aie/Iron setup (Linux/WSL/Windows). -----===## +# +# This file licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +##===------------------------------------------------------------------------------===## +# +# This user-friendly script sets up the environment and installs packages for IRON. +# It is cross-platform and works similarly on native Linux, WSL, and native Windows. +# +# 1) First-time install: +# python3 utils/iron_setup.py (optional: --all) +# +# 2) Update to newest wheels/deps: +# python3 utils/iron_setup.py update (optional: --all) +# +# 3) Set env vars for a new shell/session: +# +# * sh/bash : eval "$(python3 utils/iron_setup.py env)" +# * PowerShell : python utils/iron_setup.py env --shell pwsh | iex +# * cmd.exe : python utils\iron_setup.py env --shell cmd > "%TEMP%\iron_env.bat" && call "%TEMP%\iron_env.bat" +# +##===------------------------------------------------------------------------------===## + + +from __future__ import annotations + +import argparse +import os +import re +import shlex +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +IS_WINDOWS = (os.name == "nt") or sys.platform.startswith("win") +IS_WSL = bool(os.environ.get("WSL_DISTRO_NAME")) + + +# -------------------------------------------------------------------------------------- +# Shell emission helpers +# -------------------------------------------------------------------------------------- + + +def sh_quote(value: str) -> str: + # Safe single quotes in POSIX shells: abc'def --> 'abc'"'"'def' + return "'" + value.replace("'", "'\"'\"'") + "'" + + +def ps_quote(value: str) -> str: + # PowerShell double-quote string; escape embedded ` and ". + escaped = value.replace("`", "``").replace('"', '`"') + return f'"{escaped}"' + + +def cmd_quote(value: str) -> str: + # Simple cmd.exe quoting for paths. + return f'"{value}"' + + +def default_env_shell() -> str: + return "pwsh" if IS_WINDOWS else "sh" + + +def emit_set(shell: str, key: str, value: str) -> list[str]: + shell = shell.lower() + if shell == "pwsh": + return [f"$env:{key} = {ps_quote(value)}"] + if shell == "cmd": + return [f'set "{key}={value}"'] + return [f"export {key}={sh_quote(value)}"] + + +def emit_prepend_path(shell: str, var_name: str, prefix: str) -> list[str]: + shell = shell.lower() + sep = ";" if shell in ("pwsh", "cmd") else ":" + if shell == "pwsh": + p = ps_quote(prefix) + return [ + f"if ($env:{var_name}) {{ $env:{var_name} = {ps_quote(prefix + sep)} + $env:{var_name} }} " + f"else {{ $env:{var_name} = {p} }}" + ] + if shell == "cmd": + return [ + f'if defined {var_name} (set "{var_name}={prefix}{sep}%{var_name}%") else (set "{var_name}={prefix}")' + ] + # 'prefix'${VAR+:$VAR} + return [f"export {var_name}={sh_quote(prefix)}${{{var_name}:+{sep}${var_name}}}"] + + +def emit_append_path_if_exists(shell: str, var_name: str, suffix_dir: str) -> list[str]: + shell = shell.lower() + sep = ";" if shell in ("pwsh", "cmd") else ":" + if shell == "pwsh": + d = ps_quote(suffix_dir) + return [ + f"if (Test-Path {d}) {{ " + f"if ($env:{var_name}) {{ $env:{var_name} = $env:{var_name} + {ps_quote(sep + suffix_dir)} }} " + f"else {{ $env:{var_name} = {d} }} " + f"}}" + ] + if shell == "cmd": + d = suffix_dir + return [ + f'if exist "{d}" (if defined {var_name} (set "{var_name}=%{var_name}%{sep}{d}") else (set "{var_name}={d}"))' + ] + word = f"${var_name}{sep}" + return [ + f"if [ -d {sh_quote(suffix_dir)} ]; then export {var_name}=${{{var_name}:+{word}}}{sh_quote(suffix_dir)}; fi" + ] + + +# -------------------------------------------------------------------------------------- +# Subprocess helpers +# -------------------------------------------------------------------------------------- + + +def _format_cmd(cmd: list[str]) -> str: + return subprocess.list2cmdline(cmd) if IS_WINDOWS else shlex.join(cmd) + + +class CommandError(RuntimeError): + pass + + +def run_checked( + cmd: list[str], + *, + cwd: Optional[Path] = None, + env: Optional[dict[str, str]] = None, + extra_env: Optional[dict[str, str]] = None, + capture: bool = False, +) -> Optional[str]: + run_env: Optional[dict[str, str]] = None + if env is not None: + run_env = dict(env) + elif extra_env: + run_env = os.environ.copy() + + if extra_env: + assert run_env is not None + run_env.update(extra_env) + + proc = subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + env=run_env, + check=False, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None, + text=True if capture else False, + ) + if proc.returncode != 0: + out = "" + if capture and proc.stdout: + out = str(proc.stdout) + msg = f"Command failed ({proc.returncode}): {_format_cmd(cmd)}" + if out: + msg += "\n" + out + raise CommandError(msg) + + return str(proc.stdout) if capture else None + + +def capture_text(cmd: list[str], *, cwd: Optional[Path] = None, env: Optional[dict[str, str]] = None) -> str: + return run_checked(cmd, cwd=cwd, env=env, capture=True) or "" + + +# -------------------------------------------------------------------------------------- +# Venv + pip helpers +# -------------------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class VenvInfo: + venv_dir: Path + python: Path + + +def venv_python_path(venv_dir: Path) -> Path: + # Python layout differs between POSIX and Windows. + return (venv_dir / "Scripts" / "python.exe") if IS_WINDOWS else (venv_dir / "bin" / "python") + + +def ensure_venv(venv_dir: Path, *, python_exe: str) -> VenvInfo: + if not venv_dir.exists(): + run_checked([python_exe, "-m", "venv", str(venv_dir)]) + py = venv_python_path(venv_dir) + if not py.exists(): + raise RuntimeError(f"venv python not found: {py}") + return VenvInfo(venv_dir=venv_dir, python=py) + + +def pip_install(venv: VenvInfo, args: list[str]) -> None: + run_checked([str(venv.python), "-m", "pip"] + args) + + +def pip_install_requirements(venv: VenvInfo, requirements: Path, *, upgrade: bool, force_reinstall: bool) -> None: + cmd = ["install"] + if upgrade: + cmd.append("--upgrade") + if force_reinstall: + cmd.append("--force-reinstall") + cmd += ["-r", str(requirements)] + pip_install(venv, cmd) + + +def pip_install_package( + venv: VenvInfo, + package: str, + *, + upgrade: bool, + force_reinstall: bool, + find_links: Optional[str] = None, + wheelhouse: Optional[Path] = None, + no_index: bool = False, + no_deps: bool = False, +) -> None: + cmd = ["install"] + if upgrade: + cmd.append("--upgrade") + if force_reinstall: + cmd.append("--force-reinstall") + if no_deps: + cmd.append("--no-deps") + if no_index: + cmd.append("--no-index") + if wheelhouse is not None: + cmd += ["--find-links", str(wheelhouse)] + if find_links: + cmd += ["-f", find_links] + cmd.append(package) + pip_install(venv, cmd) + + +def pip_install_prefix( + venv: VenvInfo, + dist_name: str, + candidates: list[str], + *, + require_subdir: Optional[str] = None, +) -> Optional[Path]: + # Resolve a wheel install prefix from site-packages using `pip show`. + try: + out = capture_text([str(venv.python), "-m", "pip", "show", dist_name]).replace("\r", "") + except CommandError: + return None + + location = next( + (line.split(":", 1)[1].strip() for line in out.splitlines() if line.lower().startswith("location:")), + "", + ) + if not location: + return None + + base = Path(location).resolve() + + def _ok(prefix: Path) -> bool: + return prefix.is_dir() and ((require_subdir is None) or (prefix / require_subdir).is_dir()) + + for cand in candidates: + cand = str(cand).strip() + if not cand: + continue + p = (base / cand).resolve() + if _ok(p): + return p + + return None + + +def ensure_mlir_aie_pth(venv: VenvInfo, mlir_aie_install_dir: Path) -> None: + python_dir = (mlir_aie_install_dir / "python").resolve() + if not python_dir.is_dir(): + return + try: + site_pkgs = capture_text( + [str(venv.python), "-c", "import sysconfig; p=sysconfig.get_paths().get('purelib') or sysconfig.get_paths().get('platlib') or ''; print(p)"] + ).strip() + if not site_pkgs: + return + pth_path = Path(site_pkgs) / "mlir-aie.pth" + desired = str(python_dir) + pth_path.parent.mkdir(parents=True, exist_ok=True) + pth_path.write_text(desired + "\n", encoding="utf-8") + except Exception: + return + + +# -------------------------------------------------------------------------------------- +# Windows patches +# -------------------------------------------------------------------------------------- + + +def fixup_llvm_aie_windows(peano_root: Optional[Path]) -> None: + # Windows-only llvm-aie wheel patches: + # - Create libc.a/libm.a aliases for c.lib/m.lib + # - Strip '.deplibs' from crt1.o if present (prevents bogus MSVC runtime deps) + if not IS_WINDOWS or peano_root is None: + return + + libdir = peano_root / "lib" / "aie2p-none-unknown-elf" + if not libdir.is_dir(): + return + + for src_name, dst_name in (("c.lib", "libc.a"), ("m.lib", "libm.a")): + src = libdir / src_name + dst = libdir / dst_name + if src.exists() and not dst.exists(): + try: + shutil.copy2(src, dst) + print(f"[fixup] Created alias: {dst.name} (copy of {src.name})") + except Exception as e: + print(f"[fixup] WARNING: failed to create {dst} from {src}: {e}") + + crt1 = libdir / "crt1.o" + if not crt1.exists(): + return + + objcopy = peano_root / "bin" / "llvm-objcopy.exe" + objcopy_exe = str(objcopy) if objcopy.exists() else (shutil.which("llvm-objcopy") or shutil.which("llvm-objcopy.exe")) + if not objcopy_exe: + print("[fixup] NOTE: llvm-objcopy not found; skipping crt1.o patch.") + return + + bak = crt1.with_suffix(crt1.suffix + ".bak") + if not bak.exists(): + try: + shutil.copy2(crt1, bak) + except Exception: + pass + + proc = subprocess.run( + [objcopy_exe, "--remove-section=.deplibs", str(crt1)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if proc.returncode == 0: + print("[fixup] Patched crt1.o: removed '.deplibs' (if present)") + + +# -------------------------------------------------------------------------------------- +# Install/update +# -------------------------------------------------------------------------------------- + + +# `--all` enables everything, but `--no-` explicitly opts out of sub-features. +def _apply_all_flag(args: argparse.Namespace, argv: list[str]) -> None: + if not getattr(args, "all", False): + return + for flag in ("repo-reqs", "dev", "ml", "notebook", "submodules"): + dest = flag.replace("-", "_") + if (f"--{flag}" in argv) or (f"--no-{flag}" in argv): + continue + setattr(args, dest, True) + + +def update_submodules(repo_root: Path) -> None: + if (repo_root / ".git").exists() and shutil.which("git"): + print("[setup] Updating git submodules...") + run_checked(["git", "submodule", "update", "--init", "--recursive"], cwd=repo_root) + else: + print("[setup] NOTE: Skipping submodules (not a git checkout)") + + +def print_next_steps(repo_root: Path, *, venv_name: str) -> None: + script_path = Path(__file__).resolve() + try: + script_rel = script_path.relative_to(repo_root) + except Exception: + script_rel = script_path + script_str = str(script_rel) + + print("\n[setup] Setup complete. Set environment variables for your current shell session with:") + print(f" # PowerShell: python {script_str} env --venv {venv_name} --shell pwsh | iex") + print(fr' # cmd.exe : python {script_str} env --venv {venv_name} --shell cmd > "%TEMP%\iron_env.bat" && call "%TEMP%\iron_env.bat"') + print(f' # POSIX sh : eval "$(python3 {script_str} env --venv {venv_name} --shell sh)"') + print(" # source /opt/xilinx/xrt/setup.sh") + + print("\n[setup] To update later:") + print(f" python {script_str} update") + + +def install_plan(args: argparse.Namespace, repo_root: Path, *, update_mode: bool) -> None: + _apply_all_flag(args, sys.argv[1:]) + + venv_dir = (repo_root / args.venv).resolve() + venv = ensure_venv(venv_dir, python_exe=args.python) + + print(f"[setup] repo_root : {repo_root}") + print(f"[setup] venv : {venv.venv_dir}") + print(f"[setup] wsl : {IS_WSL}") + print(f"[setup] mode : {'update' if update_mode else 'install'}") + + force_reinstall = bool(getattr(args, "force_reinstall", False)) + pip_reqs = lambda req: pip_install_requirements(venv, req, upgrade=update_mode, force_reinstall=force_reinstall) + pip_pkg = lambda pkg, **kw: pip_install_package(venv, pkg, upgrade=update_mode, force_reinstall=force_reinstall, **kw) + + # Base tooling. + pip_install(venv, ["install", "--upgrade", "pip", "setuptools", "wheel", "packaging"]) + + # Repo requirements + optional extras. + req_specs = [ + ("repo_reqs", "python/requirements.txt", None, None), + ("dev", "python/requirements_dev.txt", ["pre_commit", "install"], repo_root), + ("ml", "python/requirements_ml.txt", None, None), + ("notebook", "python/requirements_notebook.txt", ["ipykernel", "install", "--user", "--name", args.venv], None), + ] + + for flag, req_rel, post_mod_args, post_cwd in req_specs: + if not getattr(args, flag, False): + continue + req = (repo_root / req_rel).resolve() + if not req.exists(): + print(f"[setup] NOTE: Skipping {flag}; missing: {req}") + continue + + pip_reqs(req) + + if post_mod_args: + try: + run_checked([str(venv.python), "-m", *post_mod_args], cwd=post_cwd) + except Exception: + print(f"[setup] WARNING: post-step for {flag} failed (continuing)") + + # mlir_aie wheels (IRON) + # Modes: auto | skip | latest-wheels-3 | wheelhouse[:] + # Auto mode exists until wheels are available for Windows. + mlir_raw = str(getattr(args, "mlir_aie", "auto") or "auto").strip().lower() + mlir_mode, allow_missing_mlir_aie = (("latest-wheels-3", IS_WINDOWS) if mlir_raw == "auto" else (mlir_raw, False)) + if mlir_mode != "skip": + try: + if mlir_mode == "wheelhouse" or mlir_mode.startswith("wheelhouse:"): + wheelhouse_dir = Path(mlir_mode.partition(":")[2] or (repo_root / "utils" / "mlir_aie_wheels" / "wheelhouse")).expanduser() + if not wheelhouse_dir.exists(): raise RuntimeError(f"Wheelhouse directory not found: {wheelhouse_dir}") + print(f"[setup] Installing mlir_aie + aie_python_bindings from wheelhouse: {wheelhouse_dir}") + for pkg in ("mlir_aie", "aie_python_bindings"): + pip_pkg(pkg, wheelhouse=wheelhouse_dir, no_deps=True, no_index=True) + else: + mlir_find_links = "https://github.com/Xilinx/mlir-aie/releases/expanded_assets/latest-wheels-3/" + print(f"[setup] Installing mlir_aie from {mlir_find_links}") + pip_pkg("mlir_aie", find_links=mlir_find_links) + except CommandError: + if allow_missing_mlir_aie: + print("[setup] NOTE: mlir_aie wheels not available for this platform/Python (continuing).") + else: + raise + if (mlir_prefix := pip_install_prefix(venv, "mlir_aie", ["mlir_aie"])): + ensure_mlir_aie_pth(venv, mlir_prefix) + + # llvm-aie wheels (Peano) + llvm_choice = str(getattr(args, "llvm_aie", "nightly") or "nightly").strip() + llvm_find_links = "https://github.com/Xilinx/llvm-aie/releases/expanded_assets/nightly" if llvm_choice == "nightly" else llvm_choice + + print(f"[setup] Installing llvm-aie from {llvm_find_links}") + pip_pkg("llvm-aie", find_links=llvm_find_links) + + if IS_WINDOWS: + peano_prefix = pip_install_prefix(venv, "llvm-aie", ["llvm-aie", "llvm_aie"], require_subdir="bin") + fixup_llvm_aie_windows(peano_prefix) + + if getattr(args, "submodules", False): + update_submodules(repo_root) + + print_next_steps(repo_root, venv_name=args.venv) + + +# -------------------------------------------------------------------------------------- +# NPU detection +# -------------------------------------------------------------------------------------- + + +NPU_REGEX = re.compile(r"NPU Phoenix|NPU Strix|NPU Strix Halo|NPU Krackan|NPU Gorgon|NPU Gorgon Halo|RyzenAI-npu[14567]", re.IGNORECASE,) +NPU2_REGEX = re.compile(r"NPU Strix|NPU Strix Halo|NPU Krackan|NPU Gorgon|NPU Gorgon Halo|RyzenAI-npu[4567]", re.IGNORECASE) +NPU3_REGEX = re.compile(r"NPU Medusa|NPU Medusa Halo|RyzenAI-npu[8]", re.IGNORECASE) + + +# Windows and WSL must use a driver-provided xrt-smi.exe in System32\AMD. +def system32_amd_xrt_smi_dir() -> Optional[Path]: + if IS_WINDOWS: + p = Path(r"C:\Windows\System32\AMD\xrt-smi.exe") + return p.parent if p.exists() else None + if IS_WSL: + p = Path("/mnt/c/Windows/System32/AMD/xrt-smi.exe") + return p.parent if p.exists() else None + return None + + +def xrt_smi_commands() -> list[list[str]]: + out: list[list[str]] = [] + seen: set[tuple[str, ...]] = set() + + def _add(cmd: list[str]) -> None: + key = tuple(cmd) + if key not in seen: + seen.add(key) + out.append(cmd) + + for exe in ("xrt-smi", "xrt-smi.exe"): + if shutil.which(exe): + _add([exe, "examine"]) + + if (sys32 := system32_amd_xrt_smi_dir()): + _add([str(sys32 / "xrt-smi.exe"), "examine"]) + + return out + + +# Detect NPU2 (AIE2P) vs legacy AIE device via xrt-smi output. +def detect_npu2_flag() -> tuple[Optional[bool], str]: + for cmd in xrt_smi_commands(): + try: + out = capture_text(cmd).replace("\r", "") + except Exception: + continue + + # For now, treat newer NPUs as a superset of NPU2. + if NPU3_REGEX.search(out): + return True, f"Detected AIE2PS device via: {_format_cmd(cmd)}" + + if NPU_REGEX.search(out): + if NPU2_REGEX.search(out): + return True, f"Detected AIE2P device via: {_format_cmd(cmd)}" + return False, f"Detected legacy AIE device via: {_format_cmd(cmd)}" + + return None, "WARNING: xrt-smi not available (or no NPU detected)" + + +# -------------------------------------------------------------------------------------- +# Emit env for current shell session +# -------------------------------------------------------------------------------------- + + +def _resolve_windows_xrt_root(args: argparse.Namespace) -> Optional[Path]: + # Prefer --xrt-root / XRT_ROOT. Only export it if the directory exists. + # NOTE: NEVER set XILINX_XRT on Windows. + root = (getattr(args, "xrt_root", "") or os.environ.get("XRT_ROOT") or "").strip() + if root: + p = Path(root).expanduser() + p = p.resolve() if p.is_absolute() else p + return p if p.exists() else None + + default = Path(r"C:/Xilinx/XRT") + return default if default.exists() else None + + +def env_plan(args: argparse.Namespace, repo_root: Path) -> None: + venv_dir = (repo_root / args.venv).resolve() + venv_py = venv_python_path(venv_dir) + + shell = (getattr(args, "shell", "auto") or "auto").lower() + if shell == "auto": + shell = default_env_shell() + + comment = "REM" if shell == "cmd" else "#" + out_lines: list[str] = [] + + # Activate venv. + if venv_py.exists(): + out_lines.append(f"{comment} Activate venv: {venv_dir}") + if shell == "pwsh": + activate = venv_dir / "Scripts" / "Activate.ps1" + out_lines.append(f"if (Test-Path {ps_quote(str(activate))}) {{ . {ps_quote(str(activate))} }}") + elif shell == "cmd": + activate = venv_dir / "Scripts" / "activate.bat" + q = cmd_quote(str(activate)) + out_lines.append(f"if exist {q} call {q}") + else: + activate = (venv_dir / "bin" / "activate") + out_lines.append(f"if [ -f {sh_quote(str(activate))} ]; then source {sh_quote(str(activate))}; fi") + else: + out_lines.append(f"{comment} WARNING: venv not found at: {venv_dir} (run: `python utils/iron_setup.py install`)") + + mlir_prefix: Optional[Path] = None + peano_prefix: Optional[Path] = None + + if venv_py.exists(): + venv = VenvInfo(venv_dir=venv_dir, python=venv_py) + mlir_prefix = pip_install_prefix(venv, "mlir_aie", ["mlir_aie"]) + peano_prefix = pip_install_prefix(venv, "llvm-aie", ["llvm-aie", "llvm_aie"], require_subdir="bin") + + def _override(raw: str, valid, note: str) -> Optional[Path]: + raw = (raw or "").strip() + if not raw: + return None + p = Path(raw) + if not p.is_absolute(): + p = (repo_root / p).resolve() + if p.is_dir() and valid(p): + return p + out_lines.append(f"{comment} NOTE: {note}: {p}") + return None + + # Optional explicit overrides. + override = _override(getattr(args, "mlir_aie_install", ""), lambda p: (p / "bin").is_dir() or (p / "python").is_dir(), "Ignoring --mlir-aie-install (not a valid prefix)") + if override: + mlir_prefix = override + + override = _override(getattr(args, "llvm_aie_install", ""), lambda p: (p / "bin").is_dir(), "Ignoring --llvm-aie-install (missing bin/)") + if override: + peano_prefix = override + + # Driver-provided tools on PATH (kept first). + driver_dir = system32_amd_xrt_smi_dir() + if driver_dir: + out_lines.extend(emit_prepend_path(shell, "PATH", str(driver_dir))) + + # mlir-aie env. + if mlir_prefix: + out_lines.extend(emit_set(shell, "MLIR_AIE_INSTALL_DIR", str(mlir_prefix))) + out_lines.extend(emit_prepend_path(shell, "PATH", str(mlir_prefix / "bin"))) + out_lines.extend(emit_prepend_path(shell, "PYTHONPATH", str(mlir_prefix / "python"))) + lib_var = "PATH" if IS_WINDOWS else "LD_LIBRARY_PATH" + out_lines.extend(emit_prepend_path(shell, lib_var, str(mlir_prefix / "lib"))) + else: + out_lines.append(f"{comment} NOTE: mlir_aie not installed in this venv (or not detected).") + + # llvm-aie env. + if peano_prefix: + out_lines.extend(emit_set(shell, "PEANO_INSTALL_DIR", str(peano_prefix))) + out_lines.append(f"{comment} NOTE: llvm-aie is not added to PATH to avoid conflicts with system clang/clang++.") + out_lines.append(f"{comment} It can be found in: {str(peano_prefix / 'bin')}") + else: + out_lines.append(f"{comment} WARNING: llvm-aie not installed in this venv (run: `python utils/iron_setup.py install`)") + + # XRT env. + if IS_WINDOWS: + # XILINX_XRT must not be set on Windows. + if shell == "pwsh": + out_lines.append(r"Remove-Item Env:\XILINX_XRT -ErrorAction SilentlyContinue") + elif shell == "cmd": + out_lines.append('set "XILINX_XRT="') + else: + out_lines.append("unset XILINX_XRT 2>/dev/null || true") + + xrt_root = _resolve_windows_xrt_root(args) + if xrt_root: + out_lines.extend(emit_set(shell, "XRT_ROOT", str(xrt_root))) + for p in (xrt_root / "ext" / "bin", xrt_root / "lib", xrt_root / "unwrapped", xrt_root): + out_lines.extend(emit_append_path_if_exists(shell, "PATH", str(p))) + out_lines.extend(emit_append_path_if_exists(shell, "PYTHONPATH", str(xrt_root / "python"))) + else: + xrt_root = (getattr(args, "xrt_root", "") or os.environ.get("XRT_ROOT") or os.environ.get("XILINX_XRT") or "").strip() + if xrt_root: + out_lines.extend(emit_set(shell, "XRT_ROOT", xrt_root)) + out_lines.extend(emit_set(shell, "XILINX_XRT", xrt_root)) + if Path("/opt/xilinx/xrt/setup.sh").exists(): + out_lines.append(f"{comment} NOTE: If XRT tools/DLLs are not visible, run:") + out_lines.append(f"{comment} source /opt/xilinx/xrt/setup.sh") + + # NPU detection. + npu2_flag, reason = detect_npu2_flag() + if npu2_flag is not None: + out_lines.extend(emit_set(shell, "NPU2", "1" if npu2_flag else "0")) + out_lines.append(f"{comment} NOTE: {reason}") + + print("\n".join(out_lines)) + + +# -------------------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------------------- + + +def _add_bool_flag_pair( + p: argparse.ArgumentParser, + name: str, + *, + default: bool, + help_on: str, + help_off: str, +) -> None: + dest = name.replace("-", "_") + grp = p.add_mutually_exclusive_group() + grp.add_argument(f"--{name}", dest=dest, action="store_true", help=help_on) + grp.add_argument(f"--no-{name}", dest=dest, action="store_false", help=help_off) + p.set_defaults(**{dest: default}) + + +def _add_install_args(p: argparse.ArgumentParser) -> None: + p.add_argument("--python", default=sys.executable, help="Python executable used to create the venv") + p.add_argument("--venv", default="ironenv", help="Venv directory name relative to repo root") + p.add_argument("--mlir-aie", default="auto", help="mlir_aie wheel source: auto | skip | latest-wheels-3 | wheelhouse[:]") + p.add_argument("--llvm-aie", default="nightly", help="llvm-aie wheels: nightly (default) or a custom -f URL") + p.add_argument("--force-reinstall", dest="force_reinstall", action="store_true", help="Force reinstall packages (pip --force-reinstall). Useful for testers.",) + p.add_argument("--all", action="store_true", help="Enable everything (repo-reqs + dev + ml + notebook + submodules).") + + _add_bool_flag_pair(p, "repo-reqs", default=True, help_on="Install python/requirements.txt (repo Python deps).", help_off="Skip repo Python deps install.") + _add_bool_flag_pair(p, "dev", default=True, help_on="Install python/requirements_dev.txt + pre-commit.", help_off="Skip dev deps.") + _add_bool_flag_pair(p, "ml", default=False, help_on="Install python/requirements_ml.txt.", help_off="Skip ML deps.") + _add_bool_flag_pair(p, "notebook", default=False, help_on="Install python/requirements_notebook.txt + (best-effort) ipykernel registration.", help_off="Skip notebook deps.",) + _add_bool_flag_pair(p, "submodules", default=False, help_on="Update git submodules if this is a git checkout.", help_off="Do not update git submodules.",) + + +def build_arg_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser() + + sub = p.add_subparsers(dest="cmd") + + p_install_common = argparse.ArgumentParser(add_help=False) + _add_install_args(p_install_common) + + p_install = sub.add_parser("install", parents=[p_install_common], help="Install toolchain wheels + deps into the venv") + p_update = sub.add_parser("update", parents=[p_install_common], help="Update to newest wheels + deps in the existing venv") + p_update.set_defaults(submodules=True) + + p_env = sub.add_parser("env", help="Print shell commands to activate venv + export toolchain env vars") + p_env.add_argument("--venv", default="ironenv", help="Venv directory relative to repo root") + p_env.add_argument("--xrt-root", default="", help="Optional XRT install directory (Windows: default C:/Xilinx/XRT). Can also be set via XRT_ROOT.",) + p_env.add_argument("--mlir-aie-install", default="", help="Optional explicit mlir-aie install prefix (relative to repo root unless absolute).") + p_env.add_argument("--llvm-aie-install", default="", help="Optional explicit llvm-aie/peano install prefix (relative to repo root unless absolute).") + p_env.add_argument("--shell", default="auto", choices=["auto", "sh", "pwsh", "cmd"], help="Which shell syntax to emit (default: auto based on platform)") + + return p + + +def main() -> int: + repo_root = Path(__file__).resolve().parent.parent + parser = build_arg_parser() + + # Allow calling without an explicit subcommand (defaults to install). + argv = sys.argv[1:] + subcommands = {"install", "update", "env"} + + if not argv: + argv = ["install"] + elif argv[0] not in subcommands: + # If the user is asking for top-level help, don't force the install help. + if any(a in ("-h", "--help") for a in argv): + parser.print_help() + return 0 + argv = ["install"] + argv + + args = parser.parse_args(argv) + if IS_WINDOWS: + # XILINX_XRT poisons Windows builds. + os.environ.pop("XILINX_XRT", None) + + if args.cmd is None or args.cmd == "install": + install_plan(args, repo_root, update_mode=False) + return 0 + if args.cmd == "update": + install_plan(args, repo_root, update_mode=True) + return 0 + if args.cmd == "env": + env_plan(args, repo_root) + return 0 + + parser.print_help() + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/mlir_aie_wheels/pyproject.toml b/utils/mlir_aie_wheels/pyproject.toml index a9223b90d08..175e10940ce 100644 --- a/utils/mlir_aie_wheels/pyproject.toml +++ b/utils/mlir_aie_wheels/pyproject.toml @@ -64,7 +64,7 @@ environment = { PIP_NO_BUILD_ISOLATION = "false" } before-build = [ "pip install delvewheel", "pip install -r requirements.txt", - "bash {project}\\scripts\\download_mlir.sh", + 'python "{project}/scripts/download_mlir.py"', ] [build-system] diff --git a/utils/mlir_aie_wheels/python_bindings/CMakeLists.txt b/utils/mlir_aie_wheels/python_bindings/CMakeLists.txt index 25736615b21..664fd372db1 100644 --- a/utils/mlir_aie_wheels/python_bindings/CMakeLists.txt +++ b/utils/mlir_aie_wheels/python_bindings/CMakeLists.txt @@ -71,10 +71,21 @@ mlir_configure_python_dev_packages() add_compile_definitions("MLIR_PYTHON_PACKAGE_PREFIX=aie.") if(AIE_ENABLE_XRT_PYTHON_BINDINGS) + if(WIN32) + # Prefer in-tree Find*.cmake modules. + list(PREPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modulesXilinx") + endif() + find_package(XRT) find_library(XRT_COREUTIL xrt_coreutil HINTS ${XRT_LIB_DIR}) add_library(xrt_coreutil SHARED IMPORTED) - set_property(TARGET xrt_coreutil PROPERTY IMPORTED_LOCATION "${XRT_COREUTIL}") + if(WIN32) + set_target_properties(xrt_coreutil PROPERTIES + IMPORTED_IMPLIB "${XRT_COREUTIL}" + ) + else() + set_property(TARGET xrt_coreutil PROPERTY IMPORTED_LOCATION "${XRT_COREUTIL}") + endif() endif() set(_sources @@ -128,20 +139,23 @@ add_mlir_python_modules(AIEMLIRPythonModules # TODO(max): this is not DRY - should be able to fetch it from mlir_aie_install/lib if(AIE_ENABLE_XRT_PYTHON_BINDINGS) - # for build dir (superfluous here because setup.py builds the wheel from the install dir but good for sanity checking) - add_custom_command( - TARGET AIEAggregateCAPI PRE_BUILD - COMMENT "Copying libxrt_coreutil into _mlir_libs during build" - DEPENDS "${XRT_COREUTIL}" - COMMAND "${CMAKE_COMMAND}" -E copy - "${XRT_COREUTIL}" "${CMAKE_CURRENT_BINARY_DIR}/aie/_mlir_libs/libxrt_coreutil.so" - ) + if(NOT WIN32) + # Linux wheel: for now, keep shipping libxrt_coreutil + # Do not ship SIGNED DLLs in Windows wheels. + add_custom_command( + TARGET AIEAggregateCAPI PRE_BUILD + COMMENT "Copying libxrt_coreutil into _mlir_libs during build" + DEPENDS "${XRT_COREUTIL}" + COMMAND "${CMAKE_COMMAND}" -E copy + "${XRT_COREUTIL}" "${CMAKE_CURRENT_BINARY_DIR}/aie/_mlir_libs/libxrt_coreutil.so" + ) # for install - install(FILES - "${XRT_COREUTIL}" - COMPONENT xrt_coreutil - DESTINATION "${CMAKE_INSTALL_PREFIX}/aie/_mlir_libs" + install(FILES + "${XRT_COREUTIL}" + COMPONENT xrt_coreutil + DESTINATION "${CMAKE_INSTALL_PREFIX}/aie/_mlir_libs" # hack - RENAME libxrt_coreutil.so.2 - ) + RENAME libxrt_coreutil.so.2 + ) + endif() endif() diff --git a/utils/mlir_aie_wheels/python_bindings/pyproject.toml b/utils/mlir_aie_wheels/python_bindings/pyproject.toml index 74e6602b028..f37d7f98c96 100644 --- a/utils/mlir_aie_wheels/python_bindings/pyproject.toml +++ b/utils/mlir_aie_wheels/python_bindings/pyproject.toml @@ -1,3 +1,11 @@ +[build-system] +requires = [ + "setuptools>=40.8.0", + "wheel", + "nanobind>=2.9,<3", +] +build-backend = "setuptools.build_meta" + [tool.cibuildwheel] environment = { PIP_NO_BUILD_ISOLATION = "false" } build-verbosity = 3 @@ -40,5 +48,5 @@ repair-wheel-command = [ before-build = [ "pip install delvewheel", "pip install -r requirements.txt", - "bash {project}\\scripts\\download_mlir.sh", + 'python "{project}/scripts/download_mlir.py"', ] diff --git a/utils/mlir_aie_wheels/python_bindings/setup.py b/utils/mlir_aie_wheels/python_bindings/setup.py index ce238b04716..642ab75278e 100644 --- a/utils/mlir_aie_wheels/python_bindings/setup.py +++ b/utils/mlir_aie_wheels/python_bindings/setup.py @@ -8,8 +8,6 @@ from pathlib import Path from pprint import pprint from typing import Union - -from pip._internal.req import parse_requirements from setuptools import Extension, setup from setuptools.command.build_ext import build_ext @@ -18,6 +16,11 @@ def check_env(build, default=0): return os.getenv(build, str(default)) in {"1", "true", "True", "ON", "YES"} +# Always use forward slashes for CMake paths. +def _cmake_path(p: object) -> str: + return os.fspath(p).replace("\\", "/") + + class CMakeExtension(Extension): def __init__(self, name: str, sourcedir: Union[str, Path] = "") -> None: super().__init__(name, sources=[]) @@ -90,16 +93,17 @@ def build_extension(self, ext: CMakeExtension) -> None: MLIR_INSTALL_ABS_PATH = Path("/tmp/m").absolute() cmake_args = [ - f"-G {cmake_generator}", - f"-DMLIR_DIR={MLIR_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'mlir'}", - f"-DAIE_DIR={MLIR_AIE_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'aie'}", - f"-DCMAKE_INSTALL_PREFIX={install_dir}", + "-G", + cmake_generator, + f"-DMLIR_DIR={_cmake_path(MLIR_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'mlir')}", + f"-DAIE_DIR={_cmake_path(MLIR_AIE_INSTALL_ABS_PATH / 'lib' / 'cmake' / 'aie')}", + f"-DCMAKE_INSTALL_PREFIX={_cmake_path(install_dir)}", # get rid of that annoying af git on the end of .17git of libAIEAggregateCAPI.so "-DLLVM_VERSION_SUFFIX=", # Disables generation of "version soname" (i.e. libFoo.so.), which # causes pure duplication of various shlibs for Python wheels. "-DCMAKE_PLATFORM_NO_VERSIONED_SONAME=ON", - f"-DPython3_EXECUTABLE={sys.executable}", + f"-DPython3_EXECUTABLE={_cmake_path(sys.executable)}", "-DMLIR_DETECT_PYTHON_ENV_PRIME_SEARCH=ON", # not used on MSVC, but no harm f"-DCMAKE_BUILD_TYPE={cfg}", @@ -111,9 +115,9 @@ def build_extension(self, ext: CMakeExtension) -> None: if os.getenv("CMAKE_MODULE_PATH"): cmake_module_path = f"{Path(os.getenv('CMAKE_MODULE_PATH')).absolute()}" - cmake_args.append(f"-DCMAKE_MODULE_PATH={cmake_module_path}") + cmake_args.append(f"-DCMAKE_MODULE_PATH={_cmake_path(cmake_module_path)}") if os.getenv("XRT_ROOT"): - xrt_dir = f"{Path(os.getenv('XRT_ROOT')).absolute()}" + xrt_dir = _cmake_path(Path(os.getenv("XRT_ROOT")).absolute()) cmake_args.append(f"-DXRT_ROOT={xrt_dir}") if platform.system() == "Windows": @@ -137,7 +141,6 @@ def build_extension(self, ext: CMakeExtension) -> None: ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" cmake_args += [ - "-GNinja", f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}", ] except ImportError: @@ -180,11 +183,12 @@ def build_extension(self, ext: CMakeExtension) -> None: print("ENV", pprint(os.environ), file=sys.stderr) print("cmake", " ".join(cmake_args), file=sys.stderr) + cmake_src = ext.sourcedir if platform.system() == "Windows": - cmake_args = [c.replace("\\", "\\\\") for c in cmake_args] + cmake_src = _cmake_path(cmake_src) subprocess.run( - ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True + ["cmake", cmake_src, *cmake_args], cwd=build_temp, check=True ) subprocess.run( ["cmake", "--build", ".", "--target", "install", *build_args], @@ -206,6 +210,38 @@ def build_extension(self, ext: CMakeExtension) -> None: if DEBUG: name += "-debug" + +# Read requirements.txt if present (created by the wheel build scripts). +def _read_requirements(req_file: Path) -> list[str]: + try: + lines = req_file.read_text(encoding="utf-8").splitlines() + except OSError: + return [] + + out: list[str] = [] + cont = "" + for raw in lines: + line = raw.strip() + if not line or line.startswith("#"): + continue + + if cont: + line = f"{cont} {line}" + cont = "" + if line.endswith("\\"): + cont = line[:-1].rstrip() + continue + + if " #" in line: + line = line.split(" #", 1)[0].strip() + if not line or line.startswith("-"): + continue + + out.append(line) + + return out + + setup( version=os.getenv("MLIR_AIE_WHEEL_VERSION", version), author="", @@ -221,8 +257,5 @@ def build_extension(self, ext: CMakeExtension) -> None: "aiecc=aie.compiler.aiecc.main:main", ], }, - install_requires=[ - str(ir.requirement) - for ir in parse_requirements("requirements.txt", session="hack") - ], + install_requires=_read_requirements(Path(__file__).parent / "requirements.txt"), ) diff --git a/utils/mlir_aie_wheels/scripts/download_mlir.py b/utils/mlir_aie_wheels/scripts/download_mlir.py new file mode 100644 index 00000000000..c1b20bde0b3 --- /dev/null +++ b/utils/mlir_aie_wheels/scripts/download_mlir.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +##===----- download_mlir.py - Download/unpack host MLIR wheel (cross-platform)-----===## +# +# This file licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +##===------------------------------------------------------------------------------===## +# Python shim for scripts/download_mlir.sh (cross-platform, Windows-friendly). +# +# Env: +# ENABLE_RTTI: ON/OFF (default: ON) +# CIBW_ARCHS: x86_64/aarch64/arm64/AMD64 +# MATRIX_OS: ubuntu-20.04/macos-12/macos-14/windows-* +# +# Optional: +# MLIR_AIE_WHEEL_VERSION: override wheel version (bypass clone-llvm.sh parsing) +# MLIR_AIE_SOURCE_DIR: explicit repo root (for clone-llvm.sh lookup) +# VSWHERE_EXE: explicit vswhere path (for diaguids.lib fixup) +##===------------------------------------------------------------------------------===## + +import os +import re +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + + +def _run(cmd): + cmd = [str(c) for c in cmd] + print("[run]", " ".join(cmd)) + subprocess.check_call(cmd) + + +def _pip(args): + _run([sys.executable, "-m", "pip", *args]) + + +def _try_rmtree(path): + try: + shutil.rmtree(path) + except FileNotFoundError: + return + except Exception as exc: + print(f"[warn] failed to remove '{path}': {exc}") + + +def _find_clone_llvm(script_dir): + src = (os.environ.get("MLIR_AIE_SOURCE_DIR") or "").strip() + if src: + return Path(src) / "utils" / "clone-llvm.sh" + + # Assume we are in utils/mlir_aie_wheels/scripts. + p = script_dir.parent / "clone-llvm.sh" + return p if p.is_file() else script_dir.parent.parent / "clone-llvm.sh" + + +def _sh_var(text, name): + m = re.search(rf"^{re.escape(name)}=(.*)$", text, flags=re.MULTILINE) + if not m: + raise ValueError(f"{name} not found") + return m.group(1).split("#", 1)[0].strip().strip('"').strip("'") + + +def _expand_sh(template, vars): + # Supports the subset clone-llvm.sh uses for WHEEL_VERSION. + pat = re.compile( + r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::0:([0-9]+))?\}|\$([A-Za-z_][A-Za-z0-9_]*)" + ) + + def repl(m): + name = m.group(1) or m.group(3) + val = vars[name] + if m.group(2): + val = val[: int(m.group(2))] + return val + + return pat.sub(repl, template) + + +def _wheel_version(): + direct = (os.environ.get("MLIR_AIE_WHEEL_VERSION") or "").strip() + if direct: + return direct + + clone_llvm = _find_clone_llvm(Path(__file__).resolve().parent) + if not clone_llvm.is_file(): + raise FileNotFoundError(f"clone-llvm.sh not found at: {clone_llvm}") + + text = clone_llvm.read_text(encoding="utf-8", errors="replace") + vars = { + "DATETIME": _sh_var(text, "DATETIME"), + "LLVM_PROJECT_COMMIT": _sh_var(text, "LLVM_PROJECT_COMMIT"), + } + return _expand_sh(_sh_var(text, "WHEEL_VERSION"), vars).strip() + + +# -------------------------------------------------------------------------------------- +# Windows patch +# -------------------------------------------------------------------------------------- + + +def _vswhere_path(): + explicit = (os.environ.get("VSWHERE_EXE") or "").strip() + if explicit and Path(explicit).is_file(): + return Path(explicit) + + pf86 = os.environ.get("ProgramFiles(x86)") + if not pf86: + return None + + p = Path(pf86) / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" + return p if p.is_file() else None + + +def _detect_diaguids_lib(): + if os.name != "nt": + return None + + vsinstalldir = (os.environ.get("VSINSTALLDIR") or "").strip() + if vsinstalldir: + p = Path(vsinstalldir) / "DIA SDK" / "lib" / "amd64" / "diaguids.lib" + if p.is_file(): + return p + + vswhere = _vswhere_path() + if vswhere is None: + return None + + try: + install = subprocess.check_output( + [ + str(vswhere), + "-latest", + "-products", + "*", + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", + "installationPath", + ], + text=True, + stderr=subprocess.STDOUT, + ).strip() + except Exception: + return None + + if not install: + return None + + p = Path(install) / "DIA SDK" / "lib" / "amd64" / "diaguids.lib" + return p if p.is_file() else None + + +def _fixup_llvm_diaguids(mlir_prefix): + # The MLIR wheel's LLVM CMake exports can contain an absolute diaguids.lib path. + if os.name != "nt": + return + + llvm_dir = mlir_prefix / "lib" / "cmake" / "llvm" + if not llvm_dir.is_dir(): + return + + cmake_files = [llvm_dir / "LLVMExports.cmake", llvm_dir / "LLVMTargets.cmake"] + cmake_files = [p for p in cmake_files if p.is_file()] + if not cmake_files: + return + + repl_path = _detect_diaguids_lib() + repl = repl_path.as_posix() if repl_path else "diaguids.lib" + + pat = re.compile(r"([A-Za-z]:[\\/][^\n\"']*diaguids\.lib)", flags=re.IGNORECASE) + patched = 0 + + for cmake_file in cmake_files: + data = cmake_file.read_text(encoding="utf-8", errors="ignore") + + def _repl(m): + nonlocal patched + old = m.group(1) + try: + if Path(old).exists(): + return old + except Exception: + pass + patched += 1 + return repl + + new_data = pat.sub(_repl, data) + if new_data != data: + cmake_file.write_text(new_data, encoding="utf-8") + + if patched: + print(f"[fixup] Patched diaguids.lib path -> {repl_path if repl_path else 'diaguids.lib'}") + + +def main(): + enable_rtti = (os.environ.get("ENABLE_RTTI") or "ON").upper() + no_rtti = enable_rtti == "OFF" + + # Mirror: rm -rf mlir || true (also clear the sibling to avoid stale state). + _try_rmtree(Path("mlir")) + _try_rmtree(Path("mlir_no_rtti")) + + version = _wheel_version() + print(f"Using MLIR version: {version}") + + _pip(["install", "-U", "--force-reinstall", f"mlir-native-tools=={version}"]) + + pkg = f"mlir{'-no-rtti' if no_rtti else ''}=={version}" + cibw_archs = (os.environ.get("CIBW_ARCHS") or "").strip() + matrix_os = (os.environ.get("MATRIX_OS") or "").strip() + + if cibw_archs in {"arm64", "aarch64"}: + if matrix_os in {"macos-12", "macos-14"} and cibw_archs == "arm64": + plat = "macosx_12_0_arm64" + elif matrix_os == "ubuntu-20.04" and cibw_archs == "aarch64": + plat = "linux_aarch64" + else: + raise SystemExit(f"Unsupported CIBW_ARCHS/MATRIX_OS: {cibw_archs}/{matrix_os}") + + _pip(["-q", "download", pkg, "--platform", plat, "--only-binary=:all:"]) + else: + _pip(["-q", "download", pkg]) + + wheels = sorted(Path.cwd().glob("mlir*whl")) + if not wheels: + raise FileNotFoundError("No wheels matched pattern: 'mlir*whl'") + + for wheel in wheels: + print(f"[unzip] {wheel.name}") + with zipfile.ZipFile(wheel, "r") as zf: + zf.extractall(Path.cwd()) + + for d in (Path("mlir"), Path("mlir_no_rtti")): + if d.is_dir(): + _fixup_llvm_diaguids(d) + + print(Path.cwd()) + for p in sorted(Path.cwd().glob("mlir*")): + print(p.name) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/mlir_aie_wheels/setup.py b/utils/mlir_aie_wheels/setup.py index 5d4502cd6ec..41d0a165337 100644 --- a/utils/mlir_aie_wheels/setup.py +++ b/utils/mlir_aie_wheels/setup.py @@ -24,6 +24,11 @@ def check_env(build, default=0): return os.getenv(build, str(default)) in {"1", "true", "True", "ON", "YES"} +# Always use forward slashes for CMake paths. +def _cmake_path(p: object) -> str: + return os.fspath(p).replace("\\", "/") + + class CMakeExtension(Extension): def __init__(self, name: str, sourcedir: Union[str, Path] = "") -> None: super().__init__(name, sources=[]) @@ -127,11 +132,12 @@ def build_extension(self, ext: CMakeExtension) -> None: MLIR_INSTALL_ABS_PATH = Path("/tmp/m").absolute() cmake_args = [ - f"-G {cmake_generator}", - f"-DCMAKE_MODULE_PATH={MLIR_AIE_SOURCE_DIR / 'cmake' / 'modulesXilinx'}", - f"-DCMAKE_PREFIX_PATH={MLIR_INSTALL_ABS_PATH}", - f"-DCMAKE_INSTALL_PREFIX={install_dir}", - f"-DPython3_EXECUTABLE={sys.executable}", + "-G", + cmake_generator, + f"-DCMAKE_MODULE_PATH={_cmake_path(MLIR_AIE_SOURCE_DIR / 'cmake' / 'modulesXilinx')}", + f"-DCMAKE_PREFIX_PATH={_cmake_path(MLIR_INSTALL_ABS_PATH)}", + f"-DCMAKE_INSTALL_PREFIX={_cmake_path(install_dir)}", + f"-DPython3_EXECUTABLE={_cmake_path(sys.executable)}", f"-DCMAKE_BUILD_TYPE={cfg}", # prevent symbol collision that leads to multiple pass registration and such "-DCMAKE_VISIBILITY_INLINES_HIDDEN=ON", @@ -153,7 +159,7 @@ def build_extension(self, ext: CMakeExtension) -> None: ] if os.getenv("XRT_ROOT"): - xrt_dir = f"{Path(os.getenv('XRT_ROOT')).absolute()}" + xrt_dir = _cmake_path(Path(os.getenv("XRT_ROOT")).absolute()) cmake_args.append(f"-DXRT_ROOT={xrt_dir}") if shutil.which("ccache"): @@ -167,8 +173,8 @@ def build_extension(self, ext: CMakeExtension) -> None: "-DCMAKE_C_COMPILER=cl", "-DCMAKE_CXX_COMPILER=cl", "-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded", - "-DCMAKE_C_FLAGS=/MT", - "-DCMAKE_CXX_FLAGS=/MT", + "-DCMAKE_C_FLAGS=/MT /wd4065", + "-DCMAKE_CXX_FLAGS=/MT /wd4065", "-DLLVM_USE_CRT_MINSIZEREL=MT", "-DLLVM_USE_CRT_RELEASE=MT", ] @@ -187,7 +193,6 @@ def build_extension(self, ext: CMakeExtension) -> None: ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" cmake_args += [ - "-GNinja", f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}", ] except ImportError: @@ -226,9 +231,6 @@ def build_extension(self, ext: CMakeExtension) -> None: print("ENV", pprint(os.environ), file=sys.stderr) print("cmake", " ".join(cmake_args), file=sys.stderr) - if platform.system() == "Windows": - cmake_args = [c.replace("\\", "\\\\") for c in cmake_args] - subprocess.run( ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True ) diff --git a/utils/run_example.py b/utils/run_example.py new file mode 100644 index 00000000000..a40772f829d --- /dev/null +++ b/utils/run_example.py @@ -0,0 +1,1464 @@ +#!/usr/bin/env python3 +##===------ run_example.py - Runs IRON examples (Linux/WSL/Windows). -----===## +# +# This file licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +##===------------------------------------------------------------------------------===## +# +# Build + run Makefile-based programming examples (cross platform). +# +# This script is not a Makefile interpreter. It uses the example Makefile as a hint to +# discover the target name, VPATH search roots, and kernel object prerequisites. +# +# It attempts to perform the same basic flow as MOST examples: +# 1) Generate AIE MLIR by running the example's Python generator. +# 2) Compile kernel objects (e.g. scale.cc -> build/scale.o). +# 3) Invoke aiecc to produce: +# - build/final_.xclbin +# - build/insts_.bin +# 4) Optionally build a host executable via CMake and run it. +# +# Usage +# ----- +# From an example directory containing a Makefile: +# +# # PowerShell +# python path\to\run_example.py --example-dir . build +# python path\to\run_example.py --example-dir . run +# +# Or default to the current directory: +# python path\to\run_example.py build +# +##===------------------------------------------------------------------------------===## + + +from __future__ import annotations + +import argparse +import importlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + + +# -------------------------------------------------------------------------------------- +# Platform + process helpers +# -------------------------------------------------------------------------------------- + + +def is_windows() -> bool: + return os.name == "nt" + + +def is_wsl() -> bool: + if is_windows(): + return False + if os.environ.get("WSL_INTEROP") or os.environ.get("WSL_DISTRO_NAME"): + return True + try: + osrelease = Path("/proc/sys/kernel/osrelease").read_text(errors="ignore").lower() + return "microsoft" in osrelease or "wsl" in osrelease + except Exception: + return False + + +def _wsl_to_windows_path(path: Path) -> str: + # wslpath is present inside WSL; avoid depending on it elsewhere. + if not is_wsl(): + return str(path) + wslpath_exe = which("wslpath") + if not wslpath_exe: + raise RuntimeError("wslpath not found; cannot translate WSL paths for Windows tools") + out = subprocess.check_output([wslpath_exe, "-w", str(path)], text=True).strip() + if not out: + raise RuntimeError(f"wslpath returned empty path for: {path}") + return out + + +def which(prog: str) -> Optional[str]: + return shutil.which(prog) + + +def format_cmd(cmd: list[str]) -> str: + # For logging only; quoting does not need to be shell-perfect. + try: + return shlex.join(cmd) + except AttributeError: + return " ".join(shlex.quote(c) for c in cmd) + + +def tool_exe_name(base: str) -> str: + # Tools need `.exe` appended on Windows + return base + (".exe" if is_windows() else "") + + +def host_exe_name(base: str) -> str: + # Host needs `.exe` appended on Windows + return base + (".exe" if (is_windows() or is_wsl()) else "") + + +def _cmake_generator_from_cache(build_dir: Path) -> Optional[str]: + # Read CMAKE_GENERATOR from an existing CMakeCache.txt (if present). + cache = build_dir / "CMakeCache.txt" + try: + for line in cache.read_text(encoding="utf-8", errors="ignore").splitlines(): + if line.startswith("CMAKE_GENERATOR:INTERNAL="): + return line.split("=", 1)[1].strip() or None + except FileNotFoundError: + return None + except Exception: + return None + return None + + +def _is_multi_config_generator(generator: str) -> bool: + g = (generator or "").lower() + return ("visual studio" in g) or ("xcode" in g) or ("ninja multi-config" in g) + + + +def run_checked( + cmd: list[str], + *, + cwd: Optional[Path] = None, + extra_env: Optional[dict[str, str]] = None, +) -> None: + print(f"[run] cwd={cwd or Path.cwd()}") + print(f"[run] $ {format_cmd(cmd)}") + env = None + if extra_env: + env = dict(os.environ) + env.update(extra_env) + subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, check=True) + + +def safe_rmtree(path: Path) -> None: + if path.exists(): + shutil.rmtree(path, ignore_errors=True) + + +def ensure_dir(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + + +def read_text(path: Path) -> str: + return path.read_text(encoding="utf-8", errors="ignore") + + +def _max_mtime(paths: list[Path]) -> float: + mt = 0.0 + for p in paths: + try: + mt = max(mt, p.stat().st_mtime) + except FileNotFoundError: + pass + return mt + + +def _outputs_up_to_date(outputs: list[Path], inputs: list[Path]) -> bool: + if not outputs: + return False + for o in outputs: + if not o.exists(): + return False + in_mt = _max_mtime(inputs) + out_mt = min(o.stat().st_mtime for o in outputs) + return out_mt >= in_mt + + +def _up_to_date_with_stamp( + outputs: list[Path], + inputs: list[Path], + stamp_path: Optional[Path] = None, + stamp_data: Optional[dict] = None, +) -> bool: + # Return True when outputs are newer than inputs and (optionally) the stamp matches. + if not _outputs_up_to_date(outputs, inputs): + return False + if stamp_path is not None and stamp_data is not None: + return _stamp_matches(stamp_path, stamp_data) + return True + + +def _read_json(path: Path) -> Optional[dict]: + try: + return json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return None + except Exception: + return None + + +def _write_json_atomic(path: Path, data: dict) -> None: + # Keep stamps readable for debugging; write atomically to avoid partial files. + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(tmp, path) + + +def _stamp_matches(stamp_path: Path, data: dict) -> bool: + old = _read_json(stamp_path) + return old == data + + +def _looks_like_flag_incompatible(stderr_text: str) -> bool: + # Heuristics for legacy generators that do NOT use argparse flags. + # Keep this broad: if we miss the retry, we break older examples. + t = (stderr_text or "").lower() + if "unrecognized arguments" in t: + return True + if "no such option" in t: + return True + # Some scripts treat "-d" as the device name and reject it as unknown. + if "device name" in t and "unknown" in t and "-d" in t: + return True + if "usage:" in t and ("-d" in t or "-i1s" in t or "-bw" in t): + return True + if "need " in t and " command line argument" in t: + return True + if "indexerror" in t and "list index out of range" in t: + return True + return False + + +def _run_to_file(cmd: list[str], cwd: Path, out_path: Path) -> subprocess.CompletedProcess[str]: + # Write to a temp file and replace on success so failed attempts don't clobber good artifacts. + tmp = out_path.with_suffix(out_path.suffix + ".tmp") + with tmp.open("w", encoding="utf-8") as f: + proc = subprocess.run( + cmd, + cwd=str(cwd), + stdout=f, + stderr=subprocess.PIPE, + text=True, + ) + if proc.returncode == 0: + os.replace(tmp, out_path) + else: + try: + tmp.unlink() + except Exception: + pass + return proc + + +# -------------------------------------------------------------------------------------- +# Makefile discovery (pattern-based) +# -------------------------------------------------------------------------------------- + + +@dataclass +class ExampleMakeInfo: + example_dir: Path + targetname: str + aie_py_src: Optional[str] + vpath_dirs: list[Path] + kernel_object_names: list[str] + has_trace: bool + # Runlist-style examples often generate an ELF (insts.elf) instead of NPU insts (.bin). + uses_elf_insts: bool + xclbin_name_hint: Optional[str] + insts_name_hint: Optional[str] + + # Makefile defaults (best-effort, optional) + default_int_bit_width: Optional[int] + default_in2_size: Optional[int] + default_trace_size: Optional[int] + default_col: Optional[int] + +_TARGET_RE = re.compile(r"^\s*targetname\s*[:?]?=\s*(\S+)\s*$") +_AIE_PY_SRC_RE = re.compile(r"^\s*aie_py_src\s*[:?]?=\s*(\S+)\s*$") +_FINAL_RULE_RE = re.compile(r"^\s*(build/final[^:]*\.xclbin)\s*:\s*(.+?)\s*$") +_INT_BW_RE = re.compile(r"^\s*int_bit_width\s*[:?]?=\s*(\d+)\s*(?:#.*)?$") +_IN2_RE = re.compile(r"^\s*in2_size\s*[:?]?=\s*(\d+)\s*(?:#.*)?$") +_TRACE_RE = re.compile(r"^\s*trace_size\s*[:?]?=\s*(\d+)\s*(?:#.*)?$") +_COL_RE = re.compile(r"^\s*col\s*[:?]?=\s*(\d+)\s*(?:#.*)?$", re.IGNORECASE) +_VAR_REF_RE = re.compile(r"\$\(([^)]+)\)|\${([^}]+)}") +_SIMPLE_ASSIGN_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\?|\+|:)?=\s*(.+?)\s*$") + + +def expand_make_vars(value: str, vars_map: dict[str, str]) -> str: + # ${var}/$(var) expansion. + # This is NOT a full Make interpreter; it only helps with common patterns like: + # targetname=${orig_targetname}_chained + cur = value + for _ in range(6): + def _repl(m: re.Match) -> str: + name = m.group(1) or m.group(2) or "" + return vars_map.get(name, m.group(0)) + + nxt = _VAR_REF_RE.sub(_repl, cur) + if nxt == cur: + break + cur = nxt + return cur + + +def _collapse_makefile_lines(lines: list[str]) -> list[str]: + # Merge line continuations ending in '\\'. + out: list[str] = [] + cur = "" + for raw in lines: + line = raw.rstrip("\n") + if cur: + line = cur + line.lstrip() + if line.rstrip().endswith("\\"): + cur = line.rstrip()[:-1] + " " + continue + cur = "" + out.append(line) + if cur: + out.append(cur) + return out + + +def _parse_vpath(line: str, example_dir: Path, vars_map: dict[str, str]) -> list[Path]: + # Accept: VPATH =, VPATH :=, VPATH ?=, VPATH += + m = re.match(r"^\s*VPATH\s*(?:\?|\+|:)?=\s*(.+?)\s*$", line) + if not m: + return [] + + vpath_raw = m.group(1).strip() + if not vpath_raw: + return [] + + # Make VPATH may be a single path or colon-separated paths. + vpath_raw = expand_make_vars(vpath_raw, vars_map) + raw_parts = [p.strip() for p in vpath_raw.split(":") if p.strip()] + vpath_dirs: list[Path] = [] + for p in raw_parts: + # Expand ${srcdir}/$(srcdir) even if it wasn't in the vars map. + p = p.replace("${srcdir}", str(example_dir)).replace("$(srcdir)", str(example_dir)) + vpath_dirs.append(Path(p).resolve()) + return vpath_dirs + + +def parse_makefile(example_dir: Path) -> ExampleMakeInfo: + mk = example_dir / "Makefile" + if not mk.exists(): + raise FileNotFoundError(f"Makefile not found in: {example_dir}") + + txt = read_text(mk) + lines = _collapse_makefile_lines(txt.splitlines()) + + vars_map: dict[str, str] = { + # Many Makefiles reference ${srcdir}/$(srcdir). We treat it as the + # example directory (matching upstream Makefile conventions). + "srcdir": str(example_dir.resolve()), + } + + targetname: Optional[str] = None + aie_py_src: Optional[str] = None + vpath_dirs: list[Path] = [] + kernel_objs: list[str] = [] + has_trace = False + # Runlist-style examples use --aie-generate-elf and pass an ELF to the host (-i insts.elf). + uses_elf_insts = "--aie-generate-elf" in txt + xclbin_name_hint: Optional[str] = None + insts_name_hint: Optional[str] = None + + + def _sanitize_make_token(tok: str) -> Optional[str]: + # Ignore Make variables like ${@F} / ${ Path: + # Prefer the env var exported by `utils/iron_setup.py env`. + env_root = (os.environ.get("MLIR_AIE_INSTALL_DIR") or "").strip() + if env_root: + p = Path(env_root).resolve() + if (p / "python").exists() or (p / "include").exists(): + return p + + # Equivalent to the Makefile's: + # python3 -c "from aie.utils.config import root_path; print(root_path())" + try: + from aie.utils.config import root_path # type: ignore + except Exception as e: + raise RuntimeError( + "mlir-aie python package 'aie' not importable. Run iron_setup.py env (or install mlir-aie into this Python)." + ) from e + + return Path(root_path()).resolve() + + +def resolve_peano_install_dir() -> Optional[Path]: + # Prefer env vars exported by utils/iron_setup.py env. + v = (os.environ.get("PEANO_INSTALL_DIR") or "").strip() + if v: + p = Path(v).resolve() + if (p / "bin").exists(): + return p + return None + + +def resolve_aie_clangpp(peano_install: Optional[Path]) -> str: + if peano_install is not None: + exe = peano_install / "bin" / tool_exe_name("clang++") + if exe.exists(): + return str(exe) + + exe = which("clang++") or which("clang++.exe") + if exe: + return exe + + raise RuntimeError( + "clang++ not found (llvm-aie/Peano). Run iron_setup.py env or put PEANO_INSTALL_DIR/bin on PATH." + ) + + +def xchesscc_available() -> bool: + return bool(which("xchesscc") or which("xchesscc.exe")) + + +def build_aiecc_command(args_for_aiecc: list[str]) -> list[str]: + return [ + sys.executable, + "-c", + "from aie.compiler.aiecc.main import main; main()", + *args_for_aiecc, + ] + + +# -------------------------------------------------------------------------------------- +# Build steps +# -------------------------------------------------------------------------------------- + + +@dataclass +class BuildConfig: + example_dir: Path + targetname: str + device: str + col: int + int_bit_width: int + in1_size: int + in2_size: int + out_size: int + trace_size: int + placed: bool + chess: bool + config: str + generator: Optional[str] + generator_script: Optional[Path] + extra_generator_args: list[str] = field(default_factory=list) + + +def default_sizes_for_bitwidth(int_bit_width: int) -> tuple[int, int, int]: + # Mirrors common example defaults. + if int_bit_width == 16: + return (8192, 4, 8192) + if int_bit_width == 32: + return (16384, 4, 16384) + # Fallback: keep the 16-bit layout. + return (8192, 4, 8192) + + +def resolve_kernel_sources(info: ExampleMakeInfo) -> list[tuple[str, Path]]: + sources: list[tuple[str, Path]] = [] + for name in info.kernel_object_names: + exts = [".cc", ".cpp", ".cxx", ".c"] + candidates: list[Path] = [] + for ext in exts: + fname = f"{name}{ext}" + candidates.extend([info.example_dir / fname] + [vp / fname for vp in info.vpath_dirs]) + found: Optional[Path] = None + for cand in candidates: + if cand.exists(): + found = cand.resolve() + break + + if not found: + raise FileNotFoundError( + f"Could not locate kernel source for '{name}.[cc/cpp/cxx/c]' in example dir or VPATH.\n" + f"Searched: {candidates}" + ) + + sources.append((name, found)) + + return sources + + +def compile_kernel_objects(cfg: BuildConfig, info: ExampleMakeInfo, build_dir: Path) -> None: + mlir_aie_root = resolve_mlir_aie_root() + peano_install = resolve_peano_install_dir() + clangpp = resolve_aie_clangpp(peano_install) + warning_flags = [ + "-Wno-parentheses", + "-Wno-attributes", + "-Wno-macro-redefined", + "-Wno-empty-body", + "-Wno-missing-template-arg-list-after-template-kw", + ] + if cfg.device == "npu2": + target_flag = "--target=aie2p-none-unknown-elf" + else: + target_flag = "--target=aie2-none-unknown-elf" + + kernel_sources = resolve_kernel_sources(info) + if not kernel_sources: + return + + for obj_name, src_path in kernel_sources: + out_obj = build_dir / f"{obj_name}.o" + stamp = out_obj.with_suffix(out_obj.suffix + ".stamp.json") + ensure_dir(out_obj.parent) + + stamp_data = { + "kind": "kernel_obj", + "src": str(src_path), + "device": cfg.device, + "target": target_flag, + "int_bit_width": cfg.int_bit_width, + "clangpp": str(clangpp), + } + + if _up_to_date_with_stamp([out_obj], [src_path], stamp, stamp_data): + continue + + cmd = [ + clangpp, + "-O2", + "-std=c++20", + target_flag, + *warning_flags, + "-DNDEBUG", + f"-I{mlir_aie_root / 'include'}", + f"-DBIT_WIDTH={cfg.int_bit_width}", + "-c", + str(src_path), + "-o", + str(out_obj), + ] + + run_checked(cmd, cwd=cfg.example_dir) + _write_json_atomic(stamp, stamp_data) + + +def resolve_generator_script(cfg: BuildConfig, info: ExampleMakeInfo) -> Path: + if cfg.generator_script: + p = cfg.generator_script + if not p.is_absolute(): + p = (cfg.example_dir / p).resolve() + if not p.exists(): + raise FileNotFoundError(f"Generator script not found: {p}") + return p + + if info.aie_py_src: + cand = Path(info.aie_py_src) + cand = cand.resolve() if cand.is_absolute() else (cfg.example_dir / cand).resolve() + if cand.exists(): + if cfg.placed: + placed_cand = cand.with_name(f"{cand.stem}_placed{cand.suffix}") + if placed_cand.exists(): + return placed_cand + return cand + + py_name = f"{cfg.targetname}.py" + if cfg.placed: + placed_name = f"{cfg.targetname}_placed.py" + if (cfg.example_dir / placed_name).exists(): + py_name = placed_name + py_path = (cfg.example_dir / py_name).resolve() + if py_path.exists(): + return py_path + + # Fallback: if there's only one plausible generator script, use it. + candidates = [ + p + for p in sorted(cfg.example_dir.glob("*.py")) + if p.name not in {"test.py"} and not p.name.startswith("run_") + ] + if len(candidates) == 1: + return candidates[0].resolve() + if candidates: + raise FileNotFoundError( + "Could not determine generator script. Candidates:\n" + + "\n".join(f" - {p.name}" for p in candidates) + ) + raise FileNotFoundError(f"No python generator script found in: {cfg.example_dir}") + + +def generate_aie_mlir(cfg: BuildConfig, info: ExampleMakeInfo, build_dir: Path, trace: bool) -> Path: + data_size = cfg.in1_size + suffix = "trace_" if trace else "" + out_path = build_dir / f"aie_{suffix}{data_size}.mlir" + stamp = out_path.with_suffix(out_path.suffix + ".stamp.json") + ensure_dir(out_path.parent) + + py_path = resolve_generator_script(cfg, info) + + # The MLIR depends on generator args; using only mtimes can silently reuse stale MLIR. + stamp_data_flags = { + "kind": "generator_mlir", + "mode": "flags", + "script": str(py_path), + "device": cfg.device, + "col": cfg.col, + "in1_size": cfg.in1_size, + "in2_size": cfg.in2_size, + "out_size": cfg.out_size, + "int_bit_width": cfg.int_bit_width, + "trace": bool(trace), + "trace_size": cfg.trace_size, + "extra_args": list(cfg.extra_generator_args), + } + + if _up_to_date_with_stamp([out_path], [py_path], stamp, stamp_data_flags): + print(f"[gen] Up-to-date: {out_path.name}") + return out_path + + base_cmd = [ + sys.executable, + str(py_path), + "-d", + cfg.device, + "-i1s", + str(cfg.in1_size), + "-i2s", + str(cfg.in2_size), + "-os", + str(cfg.out_size), + "-bw", + str(cfg.int_bit_width), + ] + if trace: + base_cmd += ["-t", str(cfg.trace_size)] + if cfg.extra_generator_args: + base_cmd += cfg.extra_generator_args + + print(f"[gen] Writing MLIR: {out_path}") + + proc = _run_to_file(base_cmd, cwd=cfg.example_dir, out_path=out_path) + if proc.returncode == 0: + _write_json_atomic(stamp, stamp_data_flags) + return out_path + + err = proc.stderr or "" + # For older examples, the generator often expects positional args. + # We only retry when it looks like the script rejected argparse flags. + if _looks_like_flag_incompatible(err): + retry_cmds: list[tuple[list[str], dict]] = [] + + # Common legacy conventions: + # a) script.py + # b) script.py + cmd1 = [sys.executable, str(py_path), cfg.device] + stamp1 = dict(stamp_data_flags) + stamp1["mode"] = "positional_device" + retry_cmds.append((cmd1, stamp1)) + + cmd2 = [sys.executable, str(py_path), cfg.device, str(cfg.col)] + stamp2 = dict(stamp_data_flags) + stamp2["mode"] = "positional_device_col" + retry_cmds.append((cmd2, stamp2)) + + for cmd, stamp_data in retry_cmds: + print("[gen] Retrying generator with positional args.") + proc2 = _run_to_file(cmd, cwd=cfg.example_dir, out_path=out_path) + if proc2.returncode == 0: + print("[gen] NOTE: generator does not accept flags; using positional invocation.") + _write_json_atomic(stamp, stamp_data) + return out_path + err = proc2.stderr or err + proc = proc2 + + if err: + print("[gen] stderr:") + print(err.rstrip()) + raise RuntimeError(f"MLIR generator failed (exit={proc.returncode}).") + + +def build_xclbin_and_insts( + cfg: BuildConfig, + info: ExampleMakeInfo, + build_dir: Path, + mlir_path: Path, + trace: bool, +) -> tuple[Path, Path]: + data_size = cfg.in1_size + + # Most examples generate NPU insts (.bin), but "runlist" style examples generate an ELF. + if info.uses_elf_insts: + xclbin_name = info.xclbin_name_hint or "final.xclbin" + insts_name = info.insts_name_hint or "insts.elf" + else: + xclbin_name = f"final_{'trace_' if trace else ''}{data_size}.xclbin" + insts_name = f"insts_{data_size}.bin" + + xclbin_path = build_dir / xclbin_name + insts_path = build_dir / insts_name + stamp = build_dir / f".aiecc_{'trace' if trace else 'run'}_{data_size}.stamp.json" + + ensure_dir(build_dir) + + rel_mlir = os.path.relpath(mlir_path, start=build_dir) + + aiecc_args = [ + "--aie-generate-xclbin", + "--no-compile-host", + f"--xclbin-name={xclbin_name}", + ] + + if info.uses_elf_insts: + aiecc_args += [ + "--aie-generate-elf", + f"--elf-name={insts_name}", + ] + else: + aiecc_args += [ + "--aie-generate-npu-insts", + f"--npu-insts-name={insts_name}", + ] + + # Default to Peano flow unless the user explicitly asked for Chess. + if (not cfg.chess) or (cfg.chess and not xchesscc_available()): + aiecc_args += ["--no-xchesscc", "--no-xbridge"] + + aiecc_args += [rel_mlir] + + cmd = build_aiecc_command(aiecc_args) + + obj_inputs = [build_dir / f"{n}.o" for n in info.kernel_object_names] + inputs = [mlir_path] + [p for p in obj_inputs if p.exists()] + + stamp_data = { + "kind": "aiecc", + "trace": bool(trace), + "uses_elf_insts": bool(info.uses_elf_insts), + "aiecc_args": list(aiecc_args), + } + + if _up_to_date_with_stamp([xclbin_path, insts_path], inputs, stamp, stamp_data): + print(f"[aiecc] Up-to-date: {xclbin_path.name}, {insts_path.name}") + else: + run_checked(cmd, cwd=build_dir) + _write_json_atomic(stamp, stamp_data) + + # Some examples name outputs differently; fall back to the most recent artifacts. + if not xclbin_path.exists(): + candidates = sorted(build_dir.glob("*.xclbin"), key=lambda p: p.stat().st_mtime, reverse=True) + if candidates: + xclbin_path = candidates[0] + else: + raise FileNotFoundError(f"Expected xclbin not produced: {xclbin_path}") + + if not insts_path.exists(): + patterns = ["insts_*.bin", "*.bin"] + if info.uses_elf_insts: + patterns = ["insts*.elf", "*.elf"] + patterns + + inst_candidates: list[Path] = [] + for pat in patterns: + inst_candidates.extend(build_dir.glob(pat)) + + inst_candidates = sorted(set(inst_candidates), key=lambda p: p.stat().st_mtime, reverse=True) + if inst_candidates: + insts_path = inst_candidates[0] + else: + raise FileNotFoundError(f"Expected insts not produced: {insts_path}") + + return (xclbin_path, insts_path) + +def _copy_if_newer(src: Path, dst: Path) -> bool: + # Copy src -> dst if src is newer (or dst missing). Returns True if copied. + if dst.exists(): + try: + if dst.stat().st_mtime >= src.stat().st_mtime: + return False + except FileNotFoundError: + pass + ensure_dir(dst.parent) + shutil.copy2(src, dst) + return True + + +def build_host_exe(cfg: BuildConfig) -> Path: + """Build the host executable for this example via CMake. + + We keep this incremental: + - Configure is skipped if the cached configuration matches the current settings. + - Build is always invoked (CMake will decide if anything is up-to-date). + """ + data_size = cfg.in1_size + target_exe_base = f"{cfg.targetname}_{data_size}" + host_build_dir = cfg.example_dir / "_build" + ensure_dir(host_build_dir) + + use_windows_host = is_windows() or is_wsl() + + # Configure (skip if cached config matches). + configure_stamp = host_build_dir / ".cmake_configure.stamp.json" + configure_data = { + "kind": "cmake_configure", + "schema_version": 2, + "source_dir": str(cfg.example_dir), + "target_exe_base": target_exe_base, + "in1_size": cfg.in1_size, + "in2_size": cfg.in2_size, + "out_size": cfg.out_size, + "int_bit_width": cfg.int_bit_width, + "generator_request": cfg.generator, + "generator_effective": _cmake_generator_from_cache(host_build_dir) or cfg.generator, + "config": cfg.config, + "platform": ("wsl_windows_host" if is_wsl() else ("windows" if is_windows() else "posix")), + } + + need_configure = True + if (host_build_dir / "CMakeCache.txt").exists() and _stamp_matches(configure_stamp, configure_data): + need_configure = False + + if need_configure: + if is_wsl(): + # In WSL, the NPU is exposed via the Windows driver/XRT. Build and run the host via Windows tools, + # using \wsl$ paths so the Windows build can see the sources and artifacts. + win_src = _wsl_to_windows_path(cfg.example_dir) + win_build = _wsl_to_windows_path(host_build_dir) + cmake_configure = [ + "powershell.exe", + "cmake", + "-S", + win_src, + "-B", + win_build, + f"-DTARGET_NAME={target_exe_base}", + f"-DIN1_SIZE={cfg.in1_size}", + f"-DIN2_SIZE={cfg.in2_size}", + f"-DOUT_SIZE={cfg.out_size}", + f"-DINT_BIT_WIDTH={cfg.int_bit_width}", + ] + if cfg.generator: + cmake_configure += ["-G", cfg.generator] + + requested_gen = cfg.generator or "" + if not _is_multi_config_generator(requested_gen): + cmake_configure += [f"-DCMAKE_BUILD_TYPE={cfg.config}"] + else: + cmake_configure = [ + "cmake", + "-S", + str(cfg.example_dir), + "-B", + str(host_build_dir), + f"-DTARGET_NAME={target_exe_base}", + f"-DIN1_SIZE={cfg.in1_size}", + f"-DIN2_SIZE={cfg.in2_size}", + f"-DOUT_SIZE={cfg.out_size}", + f"-DINT_BIT_WIDTH={cfg.int_bit_width}", + ] + if cfg.generator: + cmake_configure += ["-G", cfg.generator] + + # Single-config generators want CMAKE_BUILD_TYPE (including NMake/Ninja on Windows). + # Multi-config generators (VS/Xcode/Ninja Multi-Config) ignore it. + requested_gen = cfg.generator or "" + if not _is_multi_config_generator(requested_gen): + cmake_configure += [f"-DCMAKE_BUILD_TYPE={cfg.config}"] + + run_checked(cmake_configure, cwd=cfg.example_dir) + _write_json_atomic(configure_stamp, configure_data) + + # Build (incremental). + used_gen = _cmake_generator_from_cache(host_build_dir) or cfg.generator or "" + if is_wsl(): + win_build = _wsl_to_windows_path(host_build_dir) + cmake_build = ["powershell.exe", "cmake", "--build", win_build] + else: + cmake_build = ["cmake", "--build", str(host_build_dir)] + if _is_multi_config_generator(used_gen): + cmake_build += ["--config", cfg.config] + run_checked(cmake_build, cwd=cfg.example_dir) + + exe_file = host_exe_name(target_exe_base) + + # Common output locations: + candidates = [ + host_build_dir / cfg.config / exe_file, # multi-config (VS/MSBuild) + host_build_dir / exe_file, # single-config (Ninja/Unix Makefiles) + ] + + src_exe: Optional[Path] = None + for cand in candidates: + if cand.exists(): + src_exe = cand + break + + # Fallback: find by name, pick newest (ignore CMake internals). + if src_exe is None: + hits: list[Path] = [] + for cand in host_build_dir.rglob(exe_file): + if any(part in {"CMakeFiles"} or part.endswith(".dir") for part in cand.parts): + continue + if cand.is_file(): + hits.append(cand) + hits.sort(key=lambda p: p.stat().st_mtime, reverse=True) + if hits: + src_exe = hits[0] + + if src_exe is None: + raise FileNotFoundError( + f"Could not find built host executable '{exe_file}'. Searched: {candidates} (+ recursive fallback)" + ) + + out = cfg.example_dir / exe_file + copied = _copy_if_newer(src_exe, out) + if copied: + print(f"[host] Copied host exe -> {out}") + else: + print(f"[host] Up-to-date: {out}") + return out + +def run_host_exe(exe: Path, xclbin: Path, insts: Path, trace_size: Optional[int]) -> None: + if is_wsl(): + # The host binary is a Windows executable (built via Windows CMake) and must be launched via Windows. + exe_win = _wsl_to_windows_path(exe) + xclbin_win = _wsl_to_windows_path(xclbin) + insts_win = _wsl_to_windows_path(insts) + + def _ps_quote(s: str) -> str: + return "'" + s.replace("'", "''") + "'" + + cmd = ( + f"& {_ps_quote(exe_win)} " + f"-x {_ps_quote(xclbin_win)} " + f"-i {_ps_quote(insts_win)} " + f"-k MLIR_AIE" + ) + if trace_size is not None: + cmd += f" -t {trace_size}" + ps = ["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", cmd] + run_checked(ps, cwd=exe.parent) + return + + cmd = [str(exe), "-x", str(xclbin), "-i", str(insts), "-k", "MLIR_AIE"] + if trace_size is not None: + cmd += ["-t", str(trace_size)] + run_checked(cmd, cwd=exe.parent) + +def run_python_test(example_dir: Path, xclbin: Path, insts: Path, cfg: BuildConfig, trace: bool) -> None: + test_py = example_dir / "test.py" + if not test_py.exists(): + raise FileNotFoundError(f"test.py not found: {test_py}") + + cmd = [ + sys.executable, + str(test_py), + "-x", + str(xclbin), + "-i", + str(insts), + "-k", + "MLIR_AIE", + "-i1s", + str(cfg.in1_size), + "-i2s", + str(cfg.in2_size), + "-os", + str(cfg.out_size), + ] + if trace: + cmd += ["-t", str(cfg.trace_size)] + run_checked(cmd, cwd=example_dir) + + +def _resolve_repo_root_for_trace_scripts(example_dir: Path) -> Optional[Path]: + # Try to find `python/utils/trace` from repo root. + # 1) Prefer the checkout containing this script. + try: + here = Path(__file__).resolve() + for cand in [here.parent, *here.parents]: + if (cand / "python" / "utils" / "trace" / "parse.py").exists(): + return cand + except Exception: + pass + + # 2) Fall back to scanning example_dir parents. + for cand in [example_dir] + list(example_dir.parents)[:10]: + if (cand / "python" / "utils" / "trace" / "parse.py").exists(): + return cand + + return None + + +def _resolve_trace_scripts(example_dir: Path) -> Optional[tuple[Path, Path]]: + # Return the (parse.py, get_trace_summary.py) paths if available. + repo_root = _resolve_repo_root_for_trace_scripts(example_dir) + if repo_root is None: + return None + + parse_py = repo_root / "python" / "utils" / "trace" / "parse.py" + summary_py = repo_root / "python" / "utils" / "trace" / "get_trace_summary.py" + if not parse_py.exists() or not summary_py.exists(): + return None + + return parse_py, summary_py + + +def parse_trace_outputs(example_dir: Path, build_dir: Path, cfg: BuildConfig) -> None: + # Mirrors the Makefile's trace parsing helpers, when available. + scripts = _resolve_trace_scripts(example_dir) + if scripts is None: + print("[trace] NOTE: trace parsing scripts not found; skipping parse/summary.") + return + + parse_py, summary_py = scripts + + data_size = cfg.in1_size + aie_trace_mlir = build_dir / f"aie_trace_{data_size}.mlir" + if not aie_trace_mlir.exists(): + print("[trace] NOTE: aie trace mlir not found; skipping parse/summary.") + return + + out_json = example_dir / f"trace_{cfg.targetname}.json" + + cmd_parse = [ + sys.executable, + str(parse_py), + "--input", + "trace.txt", + "--mlir", + str(aie_trace_mlir), + "--output", + str(out_json), + ] + run_checked(cmd_parse, cwd=example_dir) + + cmd_summary = [sys.executable, str(summary_py), "--input", str(out_json)] + run_checked(cmd_summary, cwd=example_dir) + + +# -------------------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------------------- + + +def build_arg_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description="Run mlir-aie programming examples (python backend).") + + p.add_argument( + "action", + nargs="?", + default="build", + choices=["build", "run", "run_py", "trace", "trace_py", "host", "clean"], + help="What to do (default: build).", + ) + + p.add_argument( + "--example-dir", + default=".", + help="Example directory containing a Makefile (default: cwd).", + ) + p.add_argument( + "--targetname", + default="", + help="Override Makefile targetname (rare). If empty, it is parsed from the Makefile.", + ) + p.add_argument( + "--generator-script", + default="", + help="Override the AIE generator python script (default: from aie_py_src/targetname).", + ) + p.add_argument( + "--gen-arg", + action="append", + default=[], + help="Extra argument passed to the generator script (repeatable).", + ) + p.add_argument( + "--device", + choices=["npu", "npu2"], + default=None, + help="Target device (default: from NPU2 env, else npu2 on Windows / npu on Linux).", + ) + p.add_argument( + "--col", + type=int, + default=None, + help="AIE column index (used by some generator scripts and Makefiles; default: from Makefile if present, else 0).", + ) + p.add_argument( + "--int-bit-width", + type=int, + default=None, + help="Bit width (default: from Makefile, else 16).", + ) + p.add_argument( + "--in1-size", + type=int, + default=None, + help="Input 1 size in bytes (default: inferred from bit width).", + ) + p.add_argument( + "--in2-size", + type=int, + default=None, + help="Input 2 size in bytes (default: Makefile, else inferred from bit width).", + ) + p.add_argument( + "--out-size", + type=int, + default=None, + help="Output size in bytes (default: equals in1-size).", + ) + p.add_argument( + "--trace-size", + type=int, + default=None, + help="Trace buffer size in bytes (default: Makefile, else 8192).", + ) + p.add_argument( + "--placed", + action="store_true", + help="Prefer _placed.py if present.", + ) + p.add_argument( + "--chess", + action="store_true", + help="Use proprietary Chess/xchesscc compiler if available.", + ) + p.add_argument( + "--config", + default="Release", + help="CMake build configuration (Release/Debug).", + ) + p.add_argument( + "--generator", + default=None, + help='CMake generator (e.g. Ninja, "Visual Studio 17 2022"). Default: auto.', + ) + + return p + + +def _default_device_from_env() -> Optional[str]: + # Makefiles commonly use: + # devicename ?= $(if $(filter 1,$(NPU2)),npu2,npu) + v = (os.environ.get("NPU2") or "").strip() + if v == "1": + return "npu2" + if v == "0": + return "npu" + return None + + +def _maybe_warn_trace_target_missing(info: ExampleMakeInfo) -> None: + if not info.has_trace: + print("[trace] NOTE: Makefile has no obvious trace target; attempting trace build anyway.") + + +def _build_config_from_args(args: argparse.Namespace, example_dir: Path, info: ExampleMakeInfo) -> BuildConfig: + # Defaults + device = args.device + if device is None: + device = _default_device_from_env() + if device is None: + device = "npu2" if is_windows() else "npu" + + col = args.col + if col is None: + col = info.default_col if info.default_col is not None else 0 + + bit_width = args.int_bit_width + if bit_width is None: + bit_width = info.default_int_bit_width or 16 + + in1_default, in2_default, _out_default = default_sizes_for_bitwidth(bit_width) + + in1 = args.in1_size if args.in1_size is not None else in1_default + in2 = args.in2_size if args.in2_size is not None else (info.default_in2_size or in2_default) + out = args.out_size if args.out_size is not None else in1 + + trace_size = args.trace_size if args.trace_size is not None else (info.default_trace_size or 8192) + + generator_script = Path(args.generator_script) if args.generator_script else None + + return BuildConfig( + example_dir=example_dir, + targetname=info.targetname, + device=device, + col=col, + int_bit_width=bit_width, + in1_size=in1, + in2_size=in2, + out_size=out, + trace_size=trace_size, + placed=args.placed, + chess=args.chess, + config=args.config, + generator=args.generator, + generator_script=generator_script, + extra_generator_args=args.gen_arg, + ) + + +def main() -> int: + args = build_arg_parser().parse_args() + example_dir = Path(args.example_dir).resolve() + action = args.action + + if action == "clean": + # Clean does not require the toolchain environment; keep it lightweight. + build_dir = example_dir / "build" + host_build_dir = example_dir / "_build" + safe_rmtree(build_dir) + safe_rmtree(host_build_dir) + + targetname = args.targetname + if not targetname: + try: + info_for_clean = parse_makefile(example_dir) + targetname = info_for_clean.targetname + except Exception: + targetname = example_dir.name + + # Host executables are commonly named: _[.exe] + exe_suffix = ".exe" if is_windows() else "" + for exe in example_dir.glob(f"{targetname}_*{exe_suffix}"): + try: + exe.unlink() + except Exception: + pass + + print("[clean] Done.") + return 0 + + preflight_or_die(action) + + info = parse_makefile(example_dir) + if args.targetname: + info.targetname = args.targetname + + cfg = _build_config_from_args(args, example_dir, info) + build_dir = cfg.example_dir / "build" + + is_trace = action in TRACE_ACTIONS + + def _build_aie(trace: bool) -> tuple[Path, Path]: + ensure_dir(build_dir) + compile_kernel_objects(cfg, info, build_dir) + mlir_path = generate_aie_mlir(cfg, info, build_dir, trace=trace) + return build_xclbin_and_insts(cfg, info, build_dir, mlir_path, trace=trace) + + xclbin: Optional[Path] = None + insts: Optional[Path] = None + + if action in AIE_ACTIONS: + if is_trace: + _maybe_warn_trace_target_missing(info) + xclbin, insts = _build_aie(trace=is_trace) + + if action == "build": + print("[done] build") + return 0 + + if action == "host": + exe = build_host_exe(cfg) + print(f"[done] host -> {exe}") + return 0 + + exe: Optional[Path] = None + if action in HOST_BUILD_ACTIONS: + exe = build_host_exe(cfg) + + if action in RUN_HOST_ACTIONS: + assert exe is not None and xclbin is not None and insts is not None + run_host_exe(exe, xclbin, insts, trace_size=cfg.trace_size if is_trace else None) + + if action in RUN_PY_ACTIONS: + assert xclbin is not None and insts is not None + run_python_test(cfg.example_dir, xclbin, insts, cfg, trace=is_trace) + + if is_trace: + parse_trace_outputs(cfg.example_dir, build_dir, cfg) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From ffb65452f3cf619a99c4446b0a3f3eaecdc9a3d7 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Wed, 11 Feb 2026 15:22:53 -0500 Subject: [PATCH 02/21] Revert unintended submodule pointer update --- cmake/modulesXilinx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modulesXilinx b/cmake/modulesXilinx index c3ddfc43104..6c8f84fe39c 160000 --- a/cmake/modulesXilinx +++ b/cmake/modulesXilinx @@ -1 +1 @@ -Subproject commit c3ddfc43104fb0e6f1856f275defd644ba87e477 +Subproject commit 6c8f84fe39c967de07a566af1eef5d1759d5d36f From 056ffb17e73c56bb975f43126876f99f81fa79b1 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Wed, 11 Feb 2026 15:50:00 -0500 Subject: [PATCH 03/21] Bump modulesXilinx submodule to pick up FindXRT Windows fix --- cmake/modulesXilinx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modulesXilinx b/cmake/modulesXilinx index 6c8f84fe39c..d1b317cae9f 160000 --- a/cmake/modulesXilinx +++ b/cmake/modulesXilinx @@ -1 +1 @@ -Subproject commit 6c8f84fe39c967de07a566af1eef5d1759d5d36f +Subproject commit d1b317cae9f96462faafc6583c26a2ea6693ddc9 From 5de58f1d5a9588dee95cc51546aff646c1c5fdd6 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Wed, 11 Feb 2026 19:14:25 -0500 Subject: [PATCH 04/21] fix regex issues and enhance toolchain patching --- python/aie_lit_utils/lit_config_helpers.py | 2 +- utils/iron_setup.py | 84 ++++++++++++---------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/python/aie_lit_utils/lit_config_helpers.py b/python/aie_lit_utils/lit_config_helpers.py index 81f05e2a0df..9f36fe2fee1 100644 --- a/python/aie_lit_utils/lit_config_helpers.py +++ b/python/aie_lit_utils/lit_config_helpers.py @@ -252,7 +252,7 @@ def detect_xrt( # Old: "|[0000:41:00.1] ||RyzenAI-npu1 |" # New: "|[0000:41:00.1] |NPU Phoenix |" pattern = re.compile( - r"[\|]?(\[.+:.+:.+\]).+\|(RyzenAI-(npu\d)|NPU (\w+))\W*\|" + r"[\|]?(\[.+:.+:.+\]).+\|(RyzenAI-(npu\d)|NPU ([\w ]+?))\s*\|" ) for line in output: diff --git a/utils/iron_setup.py b/utils/iron_setup.py index 6a9432ec967..46c99186dd1 100644 --- a/utils/iron_setup.py +++ b/utils/iron_setup.py @@ -304,45 +304,50 @@ def fixup_llvm_aie_windows(peano_root: Optional[Path]) -> None: if not IS_WINDOWS or peano_root is None: return - libdir = peano_root / "lib" / "aie2p-none-unknown-elf" - if not libdir.is_dir(): - return - - for src_name, dst_name in (("c.lib", "libc.a"), ("m.lib", "libm.a")): - src = libdir / src_name - dst = libdir / dst_name - if src.exists() and not dst.exists(): - try: - shutil.copy2(src, dst) - print(f"[fixup] Created alias: {dst.name} (copy of {src.name})") - except Exception as e: - print(f"[fixup] WARNING: failed to create {dst} from {src}: {e}") - - crt1 = libdir / "crt1.o" - if not crt1.exists(): + libroot = peano_root / "lib" + if not libroot.is_dir(): return + # llvm-aie installs are organized by target triple under /lib. + toolchains = [p for p in libroot.iterdir() if p.is_dir() and p.name.endswith("-none-unknown-elf")] + if not toolchains: + print(f"[fixup] NOTE: no *-none-unknown-elf toolchains found under: {libroot} (skipping llvm-aie patches)") objcopy = peano_root / "bin" / "llvm-objcopy.exe" objcopy_exe = str(objcopy) if objcopy.exists() else (shutil.which("llvm-objcopy") or shutil.which("llvm-objcopy.exe")) if not objcopy_exe: - print("[fixup] NOTE: llvm-objcopy not found; skipping crt1.o patch.") - return + print(f"[fixup] NOTE: llvm-objcopy not found; skipping crt1.o patch.") - bak = crt1.with_suffix(crt1.suffix + ".bak") - if not bak.exists(): - try: - shutil.copy2(crt1, bak) - except Exception: - pass + for libdir in toolchains: + if not libdir.is_dir(): + continue + for src_name, dst_name in (("c.lib", "libc.a"), ("m.lib", "libm.a")): + src = libdir / src_name + dst = libdir / dst_name + if src.exists() and not dst.exists(): + try: + shutil.copy2(src, dst) + print(f"[fixup] Created alias in {libdir.name}: {dst.name} (copy of {src.name})") + except Exception as e: + print(f"[fixup] WARNING: failed to create {dst} from {src}: {e}") + + crt1 = libdir / "crt1.o" + if not crt1.exists() or not objcopy_exe: + continue + bak = crt1.with_suffix(crt1.suffix + ".bak") + if not bak.exists(): + try: + shutil.copy2(crt1, bak) + except Exception: + pass - proc = subprocess.run( - [objcopy_exe, "--remove-section=.deplibs", str(crt1)], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - if proc.returncode == 0: - print("[fixup] Patched crt1.o: removed '.deplibs' (if present)") + proc = subprocess.run( + [objcopy_exe, "--remove-section=.deplibs", str(crt1)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if proc.returncode == 0: + print(f"[fixup] Patched {libdir.name}/crt1.o: removed '.deplibs' (if present)") # -------------------------------------------------------------------------------------- @@ -476,9 +481,10 @@ def install_plan(args: argparse.Namespace, repo_root: Path, *, update_mode: bool # -------------------------------------------------------------------------------------- -NPU_REGEX = re.compile(r"NPU Phoenix|NPU Strix|NPU Strix Halo|NPU Krackan|NPU Gorgon|NPU Gorgon Halo|RyzenAI-npu[14567]", re.IGNORECASE,) -NPU2_REGEX = re.compile(r"NPU Strix|NPU Strix Halo|NPU Krackan|NPU Gorgon|NPU Gorgon Halo|RyzenAI-npu[4567]", re.IGNORECASE) -NPU3_REGEX = re.compile(r"NPU Medusa|NPU Medusa Halo|RyzenAI-npu[8]", re.IGNORECASE) +# If we have anything that reports like an NPU, even if unknown type, accept it as at least XDNA. +NPU_REGEX = re.compile(r"(RyzenAI-npu(?![0-3]\b)\d+|\bNPU\b)", re.IGNORECASE) +NPU2_REGEX = re.compile(r"NPU Strix|NPU Strix Halo|NPU Krackan|NPU Gorgon|RyzenAI-npu[4567]", re.IGNORECASE) +NPU3_REGEX = re.compile(r"NPU Medusa|RyzenAI-npu[8]", re.IGNORECASE) # Windows and WSL must use a driver-provided xrt-smi.exe in System32\AMD. @@ -512,7 +518,7 @@ def _add(cmd: list[str]) -> None: return out -# Detect NPU2 (AIE2P) vs legacy AIE device via xrt-smi output. +# Detect NPU2 (or greater) capability via xrt-smi output. def detect_npu2_flag() -> tuple[Optional[bool], str]: for cmd in xrt_smi_commands(): try: @@ -522,12 +528,12 @@ def detect_npu2_flag() -> tuple[Optional[bool], str]: # For now, treat newer NPUs as a superset of NPU2. if NPU3_REGEX.search(out): - return True, f"Detected AIE2PS device via: {_format_cmd(cmd)}" + return True, f"Detected NPU2-capable device via: {_format_cmd(cmd)}" if NPU_REGEX.search(out): if NPU2_REGEX.search(out): - return True, f"Detected AIE2P device via: {_format_cmd(cmd)}" - return False, f"Detected legacy AIE device via: {_format_cmd(cmd)}" + return True, f"Detected NPU2 device via: {_format_cmd(cmd)}" + return False, f"Detected NPU device via: {_format_cmd(cmd)}" return None, "WARNING: xrt-smi not available (or no NPU detected)" From 8f186b5b4467817b31ea895bd0aeedf0011c339d Mon Sep 17 00:00:00 2001 From: thomthehound Date: Thu, 12 Feb 2026 15:49:59 -0500 Subject: [PATCH 05/21] NPU3 is detected but not supported --- utils/iron_setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/iron_setup.py b/utils/iron_setup.py index 46c99186dd1..1acc2cb9f69 100644 --- a/utils/iron_setup.py +++ b/utils/iron_setup.py @@ -526,9 +526,9 @@ def detect_npu2_flag() -> tuple[Optional[bool], str]: except Exception: continue - # For now, treat newer NPUs as a superset of NPU2. + # For now, we detect but do not support NPU3. if NPU3_REGEX.search(out): - return True, f"Detected NPU2-capable device via: {_format_cmd(cmd)}" + return False, f"Detected unsupported device via: {_format_cmd(cmd)}" if NPU_REGEX.search(out): if NPU2_REGEX.search(out): From 7e069328bca5b71bc9b4f1a803517459f8d5a081 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Thu, 12 Feb 2026 17:51:46 -0500 Subject: [PATCH 06/21] Workflows fix attempt --- .github/workflows/buildRyzenWheels.yml | 4 +++- .github/workflows/mlirAIEDistro.yml | 4 +++- python/CMakeLists.txt | 2 +- utils/iron_setup.py | 14 ++++---------- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/buildRyzenWheels.yml b/.github/workflows/buildRyzenWheels.yml index 3eacf5e28b2..0d3c913fc17 100644 --- a/.github/workflows/buildRyzenWheels.yml +++ b/.github/workflows/buildRyzenWheels.yml @@ -262,10 +262,12 @@ jobs: $vcpkgRoot = Split-Path -Parent $vcpkg } + Remove-Item Env:VCPKG_ROOT -ErrorAction SilentlyContinue + $env:VCPKG_BINARY_SOURCES = "clear;x-gha,readwrite" & $vcpkg install openssl:x64-windows - "OPENSSL_ROOT_DIR=$($vcpkgRoot -replace '\','/')/installed/x64-windows" | + "OPENSSL_ROOT_DIR=$($vcpkgRoot.Replace('\','/'))/installed/x64-windows" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Build mlir-aie wheel diff --git a/.github/workflows/mlirAIEDistro.yml b/.github/workflows/mlirAIEDistro.yml index 4ef8621870e..0ea5495208f 100644 --- a/.github/workflows/mlirAIEDistro.yml +++ b/.github/workflows/mlirAIEDistro.yml @@ -184,10 +184,12 @@ jobs: $vcpkgRoot = Split-Path -Parent $vcpkg } + Remove-Item Env:VCPKG_ROOT -ErrorAction SilentlyContinue + $env:VCPKG_BINARY_SOURCES = "clear;x-gha,readwrite" & $vcpkg install openssl:x64-windows - "OPENSSL_ROOT_DIR=$($vcpkgRoot -replace '\','/')/installed/x64-windows" | + "OPENSSL_ROOT_DIR=$($vcpkgRoot.Replace('\','/'))/installed/x64-windows" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - uses: ./.github/actions/setup_ccache diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 58f0acf6dc8..4b0f08b9fe9 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -346,7 +346,7 @@ else () nanobind ) if(NOT WIN32) - target_link_libraries(AIEPythonExtensions.XRT PRIVATE uuid) + target_link_libraries(AIEPythonExtensions.XRT INTERFACE uuid) endif() target_include_directories(AIEPythonExtensions.XRT INTERFACE ${XRT_INCLUDE_DIR}) target_link_directories(AIEPythonExtensions.XRT INTERFACE ${XRT_LIB_DIR}) diff --git a/utils/iron_setup.py b/utils/iron_setup.py index 1acc2cb9f69..39113bd94b0 100644 --- a/utils/iron_setup.py +++ b/utils/iron_setup.py @@ -481,9 +481,11 @@ def install_plan(args: argparse.Namespace, repo_root: Path, *, update_mode: bool # -------------------------------------------------------------------------------------- -# If we have anything that reports like an NPU, even if unknown type, accept it as at least XDNA. -NPU_REGEX = re.compile(r"(RyzenAI-npu(?![0-3]\b)\d+|\bNPU\b)", re.IGNORECASE) +# If anything reports like an NPU, even if unknown type, accept it as an NPU. +NPU_REGEX = re.compile(r"(RyzenAI-npu[(?![2-3]\b)\d+]|\bNPU\b)", re.IGNORECASE) +# Only the following devices are NPU2 NPU2_REGEX = re.compile(r"NPU Strix|NPU Strix Halo|NPU Krackan|NPU Gorgon|RyzenAI-npu[4567]", re.IGNORECASE) +# Reserved for future use. NPU3_REGEX = re.compile(r"NPU Medusa|RyzenAI-npu[8]", re.IGNORECASE) @@ -514,27 +516,19 @@ def _add(cmd: list[str]) -> None: if (sys32 := system32_amd_xrt_smi_dir()): _add([str(sys32 / "xrt-smi.exe"), "examine"]) - return out -# Detect NPU2 (or greater) capability via xrt-smi output. def detect_npu2_flag() -> tuple[Optional[bool], str]: for cmd in xrt_smi_commands(): try: out = capture_text(cmd).replace("\r", "") except Exception: continue - - # For now, we detect but do not support NPU3. - if NPU3_REGEX.search(out): - return False, f"Detected unsupported device via: {_format_cmd(cmd)}" - if NPU_REGEX.search(out): if NPU2_REGEX.search(out): return True, f"Detected NPU2 device via: {_format_cmd(cmd)}" return False, f"Detected NPU device via: {_format_cmd(cmd)}" - return None, "WARNING: xrt-smi not available (or no NPU detected)" From 8140809cbd33d68faaa3bd955887e56ca71bdfcb Mon Sep 17 00:00:00 2001 From: thomthehound Date: Thu, 12 Feb 2026 18:32:23 -0500 Subject: [PATCH 07/21] fix NPU regex and add documentation --- docs/buildHostWinNative.md | 568 +++++++++++++++++++++++++++++++++++++ utils/iron_setup.py | 14 +- 2 files changed, 574 insertions(+), 8 deletions(-) create mode 100644 docs/buildHostWinNative.md diff --git a/docs/buildHostWinNative.md b/docs/buildHostWinNative.md new file mode 100644 index 00000000000..c47a53b64a0 --- /dev/null +++ b/docs/buildHostWinNative.md @@ -0,0 +1,568 @@ +# Native Windows Setup and Build Instructions + +This guide covers the **native Windows** (not WSL) setup for **IRON/mlir-aie**. + +These instructions will guide you through installing and configuring everything required to both build and execute programs on the Ryzen™ AI NPU, entirely within Windows and without the need for a POSIX environment. The instructions were tested on a GMKtec Evo X-2 running Windows 11 Pro and using Visual Studio 2026 with Ryzen™ AI NPU driver version 1.6.1. + +During the course of this guide, you will: +1) Prepare the Windows toolchain to collect dependencies and compile C++ projects. +2) Manually compile the Windows Xilinx™ RunTime (XRT) directly from source. +3) Set up your native Windows mlir-aie/IRON environment. +4) Learn how to compile your own mlir-aie wheels. +5) Compile and run an IRON example test project directly on Windows. + +> NOTE: IRON on native Windows is still **experimental** and requires significantly more complicated/manual steps than the WSL2 build (`docs/buildHostWin.md`). Additionally, this workflow may be subject to change as it is further developed. Should everything work flawlessly the first time, it will likely take you approximately **two hours** to complete this guide. If you run into problems, this can easily become a weekend project. + +## Shell choice (read this first) + +Unlike most distributions of Linux, Windows is packaged with *two* shell environments: the legacy DOS `cmd.exe` (Command Prompt), which is barely more than a 'dumb terminal', and PowerShell, which is comparable to Bash on POSIX systems in terms of flexiblity and scripting. While it presents limitations, this guide assumes familiarity with `cmd.exe` and, given that some steps are simpler because of it's terminal-like nature, is written with that in mind. However, PowerShell is strongly recommended for the adventurous as it may make *using* mlir-aie/IRON easier in the long run, especially as all modern version of Windows now use PowerShell as the default prompt. + +**Recommended (path of least resistance):** +- Use **"x64 Native Tools Command Prompt for VS"** (**cmd.exe**) for the XRT build and most/all of this guide. This should "just work". + +**PowerShell (advanced users):** +- PowerShell is supported for most steps, but the setup is more complex and not completely documented here. +- Because Visual Studio does not currently ship a "Native Tools" prompt version for PowerShell, you must place the build tools on PATH manually. The two easiest ways to do this are to: + + Either open the **Native Tools cmd prompt first**, then run `pwsh` (assumes PowerShell 7, else `powershell` for 5.1) from inside it (PATH and env vars should be inherited) + + **-OR-** + + You can manually set up the env vars by invoking the Visual Studio `vcvarsall.bat` script from within an already-open PowerShell session: +```powershell +& 'C:/Program Files/Microsoft Visual Studio/18/Community/VC/Auxiliary/Build/vcvarsall.bat' x64 +``` +- Further hints for the "PowerShell-first" path are available in the Addendum. + +--- + +## 1) Install base tooling + +You will need the following tools installed and accessible on your Windows system: + +> NOTE: This guide assumes you are installing everything into default locations, as a single-user (not system-wide/"all users"), and that the option to place binaries on PATH is taken whenever given (notably for Python and CMake). If you install to custom paths, please adjust the commands below accordingly. + +- **Visual Studio** (Community or Build Tools) +- **Python >= 3.12** (CPython or Miniforge/Conda: see Addendum) +- **OpenSSL** (used to compile the mlir-aie wheels as well as the Boost dependency for XRT) +- **CMake** (highly recommended, but also installed by VS) +- **Git** (optional unless not selected in VS installer) + +Additionally, we will be updating your NPU driver and installing the dependency package manager **vcpkg**. + +### 1.1 Visual Studio / build tools components (required) + +In the Visual Studio installer, select: + +- Workload: **Desktop development with C++** +- Individual components to explicitly check: + - **C++ x64/x86 Spectre-mitigated libraries** (XRT forces `/Qspectre`) + - **C++ Clang Compiler for Windows** (Large. Can be skipped if you already have LLVM installed and on PATH) + - **MSBuild support for LLVM (clang-cl) toolset** + - **Git for Windows** (Optional, but must be installed separately if not selected here) + +### 1.2 Quick install using `winget` + +> NOTE: Choose **either** VS Studio Community (recommended) or Build Tools. + +```bat +REM Visual Studio Community (recommended) OR Build Tools +winget install -e --id Microsoft.VisualStudio.Community +REM winget install -e --id Microsoft.VisualStudio.BuildTools + +REM Python 3.12 (CPython) +winget install -e --id Python.Python.3.12 + +REM OpenSSL (optional) +REM OpenSSL will be compiled by vcpkg if you don't have it installed. +winget install -e --id OpenSSL.OpenSSL + +REM CMake +winget install -e --id Kitware.CMake + +REM Git (optional unless not selected in VS installer) +winget install -e --id Git.Git +``` + +### 1.3 Manual download links + +If you don't have/want/like `winget`, download from the following links: + +```text +Visual Studio: + https://visualstudio.microsoft.com/downloads/ + +Python: + https://www.python.org/downloads/windows/ + +OpenSSL: + https://slproweb.com/products/Win32OpenSSL.html + +CMake: + https://cmake.org/download/ + +Git: + https://git-scm.com/download/win +``` + +### 1.4 Install the Ryzen™ AI / NPU driver + +Install **the latest NPU driver** for your machine. + +AMD™'s Ryzen™ AI docs provide driver version guidance and installation steps: + +```text +Ryzen(TM) AI Software installation instructions: + https://ryzenai.docs.amd.com/en/1.6/inst.html +``` + +After install + reboot, verify the driver can talk to the NPU: + +```bat +"%WINDIR%/System32/AMD/xrt-smi.exe" examine +``` + +If `xrt-smi.exe` is missing, or `examine` fails, **stop here** and fix the driver install first. + + +### 1.5 Clone the repos + +Pick a working directory root (these examples use `C:/dev`). + +```bat +mkdir C:/dev +cd C:/dev + +REM XRT +git clone --recurse-submodules https://github.com/Xilinx/XRT.git + +REM mlir-aie +git clone --recurse-submodules https://github.com/Xilinx/mlir-aie.git + +REM vcpkg (needed to install XRT dependencies) +git clone https://github.com/microsoft/vcpkg.git +cd C:/dev/vcpkg +bootstrap-vcpkg.bat +``` + +### 1.6 Configure vcpkg + +NOTE: Visual Studio bundles its own minimal vcpkg instance. If you see any errors containing: +```text +Could not locate a manifest (vcpkg.json) ... This vcpkg distribution does not have a classic mode instance. +``` +then the above standalone vcpkg clone is not being used. Make sure your `VCPKG_ROOT` env var points to the git clone and that it is on PATH: + +```bat +set "VCPKG_ROOT=C:/dev/vcpkg" +set "PATH=%VCPKG_ROOT%;%PATH%" +``` + +You will need to do this **every time** you open a new Native Tools cmd prompt if you want to use the packages installed via the standalone vcpkg. + +--- + +## 2) Build + install XRT + +### 2.1 Install XRT deps via vcpkg and pip + +Open: + +- **Search bar --> native tools --> x64 Native Tools Command Prompt for VS** + +From the Native Tools cmd prompt: + +```bat +set "VCPKG_ROOT=C:/dev/vcpkg" +cd "%VCPKG_ROOT%" + +REM Force 64-bit libraries (vcpkg default is x86) +REM Omit `openssl` if you installed it from binaries (saves several minutes) +vcpkg.exe install --triplet x64-windows boost opencl openssl protobuf +``` +This will take a considerable amount of time (20-30 minutes) as vcpkg has to compile boost, openssl, and protobuf from source. + +You must also install pybind11 into your Python environment +```bat +pip install --upgrade pip +pip install pybind11 +``` + +### 2.2 Configure / build / install XRT + +Set your paths: + +```bat +set "XRT_REPO=C:/dev/XRT" +set "XRT_SRC=%XRT_REPO%/src" +set "XRT_BUILD=%XRT_REPO%/build/WRelease" +set "XRT_ROOT=C:/Xilinx/XRT" +set "VCPKG_ROOT=C:/dev/vcpkg" +``` + +Configure: + +```bat +cmake -S "%XRT_SRC%" -B "%XRT_BUILD%" -A x64 ^ + -DXRT_NPU=1 ^ + -DBOOST_ROOT="%VCPKG_ROOT%/installed/x64-windows" ^ + -DKHRONOS="%VCPKG_ROOT%/installed/x64-windows" ^ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +``` + +Build: + +```bat +cmake --build "%XRT_BUILD%" --config Release --parallel +``` +Building/compiling XRT will also take some time, usually 10-20 minutes depending on the machine. + +Install: + +```bat +cmake --install "%XRT_BUILD%" --config Release --prefix "%XRT_ROOT%" +``` + +### 2.3 Patch the XRT install (runtime DLLs + signed `xrt_coreutil`) + +#### 2.3.1 Copy required runtime DLLs into `ext/bin` + +XRT tools will fail to run if these minimal DLLs aren't available in the `ext/bin` directory. + +```bat +set "EXTBIN=%XRT_ROOT%/ext/bin" +if not exist "%EXTBIN%" mkdir "%EXTBIN%" + +copy "%VCPKG_ROOT%\installed\x64-windows\bin\boost_filesystem*.dll" "%EXTBIN%\" >NUL +copy "%VCPKG_ROOT%\installed\x64-windows\bin\boost_program_options*.dll" "%EXTBIN%\" >NUL +copy "%VCPKG_ROOT%\installed\x64-windows\bin\libprotobuf.dll" "%EXTBIN%\" >NUL +``` + +#### 2.3.2 Replace `xrt_coreutil.dll` with the driver-signed version + +On Windows, the NPU driver installs a **signed** `xrt_coreutil.dll`. In practice you want that one, not the XRT-built version. +Here we copy, with overwrite, from the driver location into your XRT install. + +```bat +set "SIGNED_DLL=%WINDIR%\System32\AMD\xrt_coreutil.dll" +if not exist "%SIGNED_DLL%" set "SIGNED_DLL=%WINDIR%\System32\xrt_coreutil.dll" + +if not exist "%SIGNED_DLL%" ( + echo ERROR: Signed xrt_coreutil.dll not found in System32. Check your driver install. + exit /b 1 +) + +copy "%SIGNED_DLL%" "%XRT_ROOT%\xrt_coreutil.dll" /y +``` + +#### 2.3.3 Rebuild the import lib for `xrt_coreutil` + +We need to be able to link against the signed library. It would be silly at this point to introduce a POSIX environment just to get access to `gendef`, so we will do it manually. + +A short PowerShell script, runable from cmd, is provided below to create and place `xrt_coreutil.def` and `xrt_coreutil.lib` for you: + +```bat +powershell -NoProfile -ExecutionPolicy Bypass -Command ^ + "$dll=Join-Path $env:XRT_ROOT 'xrt_coreutil.dll';" ^ + "$libDir=Join-Path $env:XRT_ROOT 'lib';" ^ + "Push-Location $libDir;" ^ + "@('LIBRARY xrt_coreutil','EXPORTS')|Set-Content xrt_coreutil.def -Encoding ascii;" ^ + "dumpbin /exports $dll|Where-Object{$_ -match '^\s+(\d+)\s+[0-9A-F]+\s+[0-9A-F]+\s+(\S+)'}|ForEach-Object{""$($matches[2]) @$($matches[1])""}|Select-Object -Unique|Add-Content xrt_coreutil.def -Encoding ascii;" ^ + "lib /nologo /def:xrt_coreutil.def /machine:x64 /out:xrt_coreutil.lib;" ^ + "Pop-Location" +``` + +### 2.4 Validate XRT + +```bat +REM XRT tool (from your install) +"%XRT_ROOT%/unwrapped/loader.bat" -exec xclbinutil --help +``` +If you see the help message, XRT is installed correctly. + +--- + +## 3) Set up IRON (create the venv + baseline deps) + +From the **mlir-aie repo root** (still in the same Native Tools cmd prompt): + +```bat +cd C:/dev/mlir-aie + +REM Create/refresh the IRON venv and install deps. +python utils/iron_setup.py +``` + +Activate the venv and apply the IRON toolchain env vars to your current **cmd.exe** session: + +```bat +python utils/iron_setup.py env --shell cmd > "%TEMP%\iron_env.bat" && call "%TEMP%\iron_env.bat" +``` + +> If you prefer PowerShell, use `python utils/iron_setup.py env --shell pwsh | iex`. + +--- + +## 4) Build + install mlir-aie wheels locally + +### 4.1 Build the wheels + +>NOTE: The build process will create a `/tmp/` directory at the root of the drive containing your mlir-aie repo (e.g. `C:/tmp/`). It can safely be deleted after the build completes, and it **must** be deleted if you want to re-run the build. + +Make sure you are in your venv (after `activate.bat`) and have applied the env vars from section 3.1. + +Choose what CPython tags to build (example shows CPython 3.12 only): + +```bat +set "CIBW_BUILD=cp312-*" +``` + +And point to your OpenSSL vcpkg install (needed for the wheels): +>NOTE: if you installed pre-built OpenSSL binaries (i.e. not through vcpkg), `OPENSSL_ROOT_DIR` should already be on PATH and this may be skipped. + +```bat +set "OPENSSL_ROOT_DIR=%VCPKG_ROOT%/installed/x64-windows" +``` + +Build: + +```bat +python utils/mlir_aie_wheels/scripts/build_local.py +``` +Compliation will take some time (10-15 minutes). + +In the output directory: + +- `utils/mlir_aie_wheels/wheelhouse/` + +You should see **two wheels**: + +- `mlir_aie-...whl` +- `mlir_aie_python_bindings-...whl` + +### 4.2 Install your local wheels into IRON + +Install from the wheelhouse: + +```bat +python utils/iron_setup.py install --mlir-aie wheelhouse +``` + +### 4.3 Re-apply mlir-aie env vars + +After installing, re-run `iron_setup.py env` so it exports `MLIR_AIE_INSTALL_DIR` and updates PATH/PYTHONPATH: + +```bat +python utils/iron_setup.py env --shell cmd > "%TEMP%\iron_env.bat" && call "%TEMP%\iron_env.bat" +``` + +> Again, if you prefer PowerShell, use `python utils/iron_setup.py env --shell pwsh | iex`. + +--- + +### 4.4 Validate the mlir-aie install + +```bat +python -c "import aie; import importlib.util as u; print('aie.xrt:', bool(u.find_spec('aie.xrt')))" +``` + +If this prints `aie.xrt: True`, you're good to go. + +## 5) Build and run an example + +From inside an example directory (recommended): + +```bat +cd /d programming_examples/basic/vector_scalar_mul +python ../../../utils/run_example.py build +python ../../../utils/run_example.py run +``` + +Or from the repo root: + +```bat +python utils/run_example.py run --example-dir programming_examples/basic/vector_scalar_mul +``` +If this prints `PASS!` at the end, your IRON system is fully configured and functional. + +--- + +## Troubleshooting + +### XRT build fails with "could not find any instance of Visual Studio." + +With newer versions of VS, this error is usually because older CMake installs occur earlier on PATH. Strawbery Perl, for instance, tends to do this, as can having ad older CMake installed into your Python/Conda environmet. + +Check which CMake is being used: + +```bat +where cmake +cmake --version +``` +Alter your PATH ordering in Windows "Environment Variables" or uninstall/upgrade conflicting CMake installs. Then delete your XRT `WRelease` build dir and reconfigure. +The standalone CMake installation in this guide is intended to mitigate this problem, but it must be first on PATH to be effective. + +### XRT build fails with Spectre-related errors (MSB8042, /Qspectre) + +Install **C++ Spectre-mitigated libs** in the Visual Studio installer, then delete your XRT build dir and reconfigure. + +### CMake can't find Python (or finds the wrong one) + +- Make sure the intended `python.exe` is first on PATH or activate your conda env before configuring XRT. +- Confirm `%PYLIB%` exists (`dir "%PYLIB%"`). +- Wipe the build dir and reconfigure (CMake caches the first Python it sees). + +### CMake can't find Boost / Protobuf + +- Confirm you have set `%VCPKG_ROOT%` correctly. +- Confirm you installed the packages for the same triplet (`x64-windows`) you're building with. +- If you changed vcpkg settings, wipe the XRT build dir and reconfigure. + +### `xrt-smi` missing or fails + +- Reinstall/upgrade the NPU driver (run installer as admin; reboot). +- Confirm it landed at `%WINDIR%/System32/AMD/xrt-smi.exe`. + +### XRT tools fail to start due to missing DLLs + +- You may need the VC++ runtime: install "Visual C++ 2015-2022 Redistributable (x64)". +- Confirm you copied Boost/Protobuf DLLs into `%XRT_ROOT%/ext/bin`. + +### You installed XRT somewhere else + +Set `XRT_ROOT` accordingly (cmd): + +```bat +set "XRT_ROOT=D:/path/to/XRT" +``` + +and re-run the "cmd-friendly" env snippet in section 3.1. + +### Wheels fail to build due to `Ninja` missing: "no such file or directory" + +This error message is misleading: it is asctually due to a stale build cache. +Ninja is istalled for you, but you must delete your `/tmp/` directory (e.g. `C:/tmp/`), and the `build` directories under `utils/mlir_aie_wheels/`, **every time** before re-running the build. + +--- + +## Addendum + +### Dev Shell for PowerShell 7 + +If you prefer to use PowerShell, it may be convenient to always launch it with the VS build tools already enabled. +To do so, simply navigate to `C:/Users/%USERNAME%/Documents/PowerShell` and paste the following script in `profile.ps1` (or create it if it does not exist). +Be sure to include the region tags and respect any existing such tags in your profile. + +```powershell +#region enter dev tools mode (amd64) +<# VS Developer Shell (amd64) on every pwsh start #> + +# Set this in a shell/session to skip DevShell init: +# $env:DISABLE_VSDEVSHELL = "1" +if (-not $env:DISABLE_VSDEVSHELL) { + # Avoid re-entering DevShell if we're already in it. + if (-not $env:VSCMD_VER) { + $startLocation = Get-Location + # The install directory should be in the same place, with the same name, for all distributions of VS. + # Query `vswhere` from that location to find the real/latest VS and launch the DevShell script. + try { + $vswhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" + if (Test-Path $vswhere) { + $vsInstall = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + $devShell = Join-Path $vsInstall 'Common7/Tools/Launch-VsDevShell.ps1' + if (Test-Path $devShell) { + & $devShell -Arch amd64 -HostArch amd64 + } + } + } catch { + Write-Verbose "VS Dev Shell init skipped: $($_.Exception.Message)" + } finally { + # Keep your original working directory even if DevShell changes it + Set-Location $startLocation + } + } +} + +# Prefer standalone/"classic mode" vcpkg. +# Comment out the next line to disable this whole standalone-vcpkg block (env vars + PATH). +$enableStandaloneVcpkg = $true + +if ((Get-Variable enableStandaloneVcpkg -ValueOnly -ErrorAction SilentlyContinue)) { + # This overrides the DevShell env var pointing to the stripped-down vcpkg that ships with VS. + $preferredVcpkgRoot = "C:/dev/vcpkg" + if (Test-Path (Join-Path $preferredVcpkgRoot "vcpkg.exe")) { + $env:VCPKG_ROOT = $preferredVcpkgRoot + # Match the DevShell architecture (amd64) to prevent implicit x86 builds. + $env:VCPKG_DEFAULT_TRIPLET = "x64-windows" + # Add VCPKG_ROOT to PATH (once). + $vcpkgPath = (Resolve-Path -LiteralPath $env:VCPKG_ROOT).Path + $pathParts = @($env:Path -split ';' | Where-Object { $_ }) + $vcpkgNorm = $vcpkgPath.TrimEnd('\','/') + $hasVcpkg = $pathParts | ForEach-Object { $_.TrimEnd('\','/') } | Where-Object { $_ -ieq $vcpkgNorm } | Select-Object -First 1 + if (-not $hasVcpkg) { + $env:Path = "$vcpkgPath;$env:Path" + } + } +} + +# Print output confirming the running version and vcpkg directory. +if ($env:VSCMD_ARG_TGT_ARCH) { + Write-Host "VS DevShell: Target = $($env:VSCMD_ARG_TGT_ARCH) Host = $($env:VSCMD_ARG_HOST_ARCH) Version = $env:VSCMD_VER" +} else { + Write-Warning "VS DevShell not active (no VSCMD_* vars found)." +} +Write-Host "VCPKG_ROOT = $env:VCPKG_ROOT" +if ($env:VCPKG_DEFAULT_TRIPLET) { Write-Host "VCPKG_TRIPLET = $env:VCPKG_DEFAULT_TRIPLET" } +#endregion +``` + +### Notes on PowerShell usage +- Install the latest PowerShell 7+ from either: +winget +```powershell +winget install -e --id Microsoft.Powershell +``` +or manually from: +```text +https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell +``` +#### Some commands/scripts may need minor syntax changes to run in PowerShell. +- You can run cmd commands from within PowerShell by prefixing them with `cmd /c`, e.g.: + ```powershell + cmd /c "set XRT_ROOT=C:/Xilinx/XRT" + ``` +- Alternatively, you can set env vars in PowerShell using `$env:VAR_NAME = "value"`, e.g.: + ```powershell + $env:XRT_ROOT = "C:/Xilinx/XRT" + ``` +- Likewise, commands with variables like `%VCPKG_ROOT%` will need to be changed to `$($env:VCPKG_ROOT)` in PowerShell. So: + ```powershell + set "PATH=%VCPKG_ROOT%;%PATH%" + ``` + becomes: + ```powershell + $env:PATH = "$($env:VCPKG_ROOT);$($env:PATH)" + ``` + But for CMake: + ```powershell + cmake -S "$env:XRT_SRC" -B "$env:XRT_BUILD" -A x64 ` + -DXRT_NPU=1 ` + -DBOOST_ROOT="$env:VCPKG_INSTALLED/installed/x64-windows" ` + -DKHRONOS="$env:VCPKG_INSTALLED/installed/x64-windows" ` + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + ``` +- When copying multi-line commands from this doc, ensure that line continuation characters (carrot `^` for cmd, backtick `` ` `` for PowerShell) are adjusted accordingly. +- Some command names slightly differ. For instance, the `cmd.exe` `where` is `Get-Command` (or aliased directly as `where.exe`) in PowerShell, and `copy` is `Copy-Item` (or `cp`), etc. + +### Using Conda/Miniforge Python +- If you have Ryzen™ AI Software installed, and you followed the driver install instructions, you very likely have Conda/Miniforge installed as well. +- You can use the Conda Python for building/running XRT and IRON, but it is recommended to create a separate Conda env for IRON development to avoid conflicts with the driver-installed env. + ```powershell + conda create -n py312 python=3.12 + ``` +- Ensure you activate your conda env every time you enter a new shell before running Python commands: + ```powershell + conda activate py312 + ``` +- In this case, you can skip installing Python via `winget` or the official installer. + diff --git a/utils/iron_setup.py b/utils/iron_setup.py index 39113bd94b0..e44dbebc1a3 100644 --- a/utils/iron_setup.py +++ b/utils/iron_setup.py @@ -481,12 +481,11 @@ def install_plan(args: argparse.Namespace, repo_root: Path, *, update_mode: bool # -------------------------------------------------------------------------------------- -# If anything reports like an NPU, even if unknown type, accept it as an NPU. -NPU_REGEX = re.compile(r"(RyzenAI-npu[(?![2-3]\b)\d+]|\bNPU\b)", re.IGNORECASE) +NPU_REGEX = re.compile(r"NPU Phoenix|RyzenAI-npu[1]", re.IGNORECASE) # Only the following devices are NPU2 -NPU2_REGEX = re.compile(r"NPU Strix|NPU Strix Halo|NPU Krackan|NPU Gorgon|RyzenAI-npu[4567]", re.IGNORECASE) +NPU2_REGEX = re.compile(r"NPU Strix|NPU Strix Halo|NPU Krackan|RyzenAI-npu[456]", re.IGNORECASE) # Reserved for future use. -NPU3_REGEX = re.compile(r"NPU Medusa|RyzenAI-npu[8]", re.IGNORECASE) +NPU3_REGEX = re.compile(r"NPU Medusa|RyzenAI-npu[3]", re.IGNORECASE) # Windows and WSL must use a driver-provided xrt-smi.exe in System32\AMD. @@ -525,10 +524,9 @@ def detect_npu2_flag() -> tuple[Optional[bool], str]: out = capture_text(cmd).replace("\r", "") except Exception: continue - if NPU_REGEX.search(out): - if NPU2_REGEX.search(out): - return True, f"Detected NPU2 device via: {_format_cmd(cmd)}" - return False, f"Detected NPU device via: {_format_cmd(cmd)}" + if NPU2_REGEX.search(out): + return True, f"Detected NPU2 device via: {_format_cmd(cmd)}" + return False, f"Detected NPU device via: {_format_cmd(cmd)}" return None, "WARNING: xrt-smi not available (or no NPU detected)" From 417e1ea851a8e22ae48f16936db4f4fa739a8bcf Mon Sep 17 00:00:00 2001 From: thomthehound Date: Thu, 12 Feb 2026 20:10:40 -0500 Subject: [PATCH 08/21] Workflows fix attempt No2 --- .github/workflows/buildRyzenWheels.yml | 2 +- .github/workflows/mlirAIEDistro.yml | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/buildRyzenWheels.yml b/.github/workflows/buildRyzenWheels.yml index 0d3c913fc17..556ce1735af 100644 --- a/.github/workflows/buildRyzenWheels.yml +++ b/.github/workflows/buildRyzenWheels.yml @@ -344,7 +344,7 @@ jobs: pip install delvewheel OPENSSL_BIN=$(python -c "import os, pathlib; print(pathlib.Path(os.environ['OPENSSL_ROOT_DIR']).joinpath('bin').as_posix())") - python -m delvewheel repair --analyze-existing-exes -w "${WHEELHOUSE_DIR}/repaired_wheel" "${WHEELHOUSE_DIR}"/mlir_aie*.whl --add-path "${OPENSSL_BIN}" + python -m delvewheel repair --ignore-existing --analyze-existing-exes --add-path "${OPENSSL_BIN}" -w "${WHEELHOUSE_DIR}/repaired_wheel" "${WHEELHOUSE_DIR}"/mlir_aie*.whl - name: Upload mlir_aie uses: actions/upload-artifact@v4 diff --git a/.github/workflows/mlirAIEDistro.yml b/.github/workflows/mlirAIEDistro.yml index 0ea5495208f..21fd8231f37 100644 --- a/.github/workflows/mlirAIEDistro.yml +++ b/.github/workflows/mlirAIEDistro.yml @@ -263,10 +263,7 @@ jobs: $opensslBin = Join-Path $env:OPENSSL_ROOT_DIR "bin" New-Item -ItemType Directory -Force -Path wheelhouse\repaired_wheel | Out-Null - python -m delvewheel repair --analyze-existing-exes ` - -w wheelhouse\repaired_wheel ` - wheelhouse\mlir_aie*.whl ` - --add-path $opensslBin + python -m delvewheel repair --ignore-existing --analyze-existing-exes -add-path $opensslBin -w wheelhouse\repaired_wheel wheelhouse\mlir_aie*.whl Remove-Item wheelhouse\mlir_aie*.whl -Force Move-Item wheelhouse\repaired_wheel\*.whl wheelhouse\ From a7e47d1e17c1837422c602c99d51fd01c00cc339 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Thu, 12 Feb 2026 22:25:37 -0500 Subject: [PATCH 09/21] Black formatting and Workflows fix attempt No3 --- .github/workflows/buildRyzenWheels.yml | 1 + python/aie_lit_utils/lit_config_helpers.py | 8 +- python/requirements.txt | 12 +- utils/iron_setup.py | 335 ++++++++++++++---- .../mlir_aie_wheels/python_bindings/setup.py | 4 +- .../mlir_aie_wheels/scripts/download_mlir.py | 8 +- utils/run_example.py | 126 +++++-- 7 files changed, 384 insertions(+), 110 deletions(-) diff --git a/.github/workflows/buildRyzenWheels.yml b/.github/workflows/buildRyzenWheels.yml index 556ce1735af..c2cba47d971 100644 --- a/.github/workflows/buildRyzenWheels.yml +++ b/.github/workflows/buildRyzenWheels.yml @@ -293,6 +293,7 @@ jobs: source aie-venv/Scripts/activate python -m pip install --upgrade pip + pip install -r python/requirements.txt pip install -r python/requirements_ml.txt pip install -r python/requirements_dev.txt diff --git a/python/aie_lit_utils/lit_config_helpers.py b/python/aie_lit_utils/lit_config_helpers.py index 9f36fe2fee1..49d61ec9abc 100644 --- a/python/aie_lit_utils/lit_config_helpers.py +++ b/python/aie_lit_utils/lit_config_helpers.py @@ -226,13 +226,17 @@ def detect_xrt( # Windows: ensure XRT DLLs can be resolved at runtime. existing_path = os.environ.get("PATH", "") config.environment["PATH"] = ( - xrt_bin_dir + os.pathsep + existing_path if existing_path else xrt_bin_dir + xrt_bin_dir + os.pathsep + existing_path + if existing_path + else xrt_bin_dir ) else: # Linux: add XRT library directory to LD_LIBRARY_PATH. existing_ld_library_path = os.environ.get("LD_LIBRARY_PATH") if existing_ld_library_path: - new_ld_library_path = existing_ld_library_path + os.pathsep + xrt_lib_dir + new_ld_library_path = ( + existing_ld_library_path + os.pathsep + xrt_lib_dir + ) else: new_ld_library_path = xrt_lib_dir config.environment["LD_LIBRARY_PATH"] = new_ld_library_path diff --git a/python/requirements.txt b/python/requirements.txt index 9caee14e430..b44d2a8aeb0 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,9 +1,13 @@ aiofiles -dataclasses>=0.6, <=0.8 -numpy>=1.19.5, <2.0 # 2.1 would be nice for typing improvements, but it doesn't seem to work well with pyxrt +# dataclasses is deprecated above Python 3.6 +dataclasses>=0.6,<=0.8; python_version < "3.7" +# NumPy: 1.26.x tops out at Python 3.12; NumPy 2.1+ for Python 3.13+ +# pyxrt may or may not function properly with cp313/cp314 +numpy>=1.19.5,<2.0; python_version < "3.13" +numpy>=2.1.0,<3.0; python_version >= "3.13" rich ml_dtypes cloudpickle # required by eudsl when it is vendored instead of installed -eudsl-python-extras==0.1.0.20251215.1715+3c7ac1b \ - --config-settings="EUDSL_PYTHON_EXTRAS_HOST_PACKAGE_PREFIX=aie" -f https://llvm.github.io/eudsl +eudsl-python-extras==0.1.0.20251215.1715+3c7ac1b \ + --config-settings=EUDSL_PYTHON_EXTRAS_HOST_PACKAGE_PREFIX=aie diff --git a/utils/iron_setup.py b/utils/iron_setup.py index e44dbebc1a3..6021938d110 100644 --- a/utils/iron_setup.py +++ b/utils/iron_setup.py @@ -167,7 +167,9 @@ def run_checked( return str(proc.stdout) if capture else None -def capture_text(cmd: list[str], *, cwd: Optional[Path] = None, env: Optional[dict[str, str]] = None) -> str: +def capture_text( + cmd: list[str], *, cwd: Optional[Path] = None, env: Optional[dict[str, str]] = None +) -> str: return run_checked(cmd, cwd=cwd, env=env, capture=True) or "" @@ -184,7 +186,11 @@ class VenvInfo: def venv_python_path(venv_dir: Path) -> Path: # Python layout differs between POSIX and Windows. - return (venv_dir / "Scripts" / "python.exe") if IS_WINDOWS else (venv_dir / "bin" / "python") + return ( + (venv_dir / "Scripts" / "python.exe") + if IS_WINDOWS + else (venv_dir / "bin" / "python") + ) def ensure_venv(venv_dir: Path, *, python_exe: str) -> VenvInfo: @@ -200,7 +206,9 @@ def pip_install(venv: VenvInfo, args: list[str]) -> None: run_checked([str(venv.python), "-m", "pip"] + args) -def pip_install_requirements(venv: VenvInfo, requirements: Path, *, upgrade: bool, force_reinstall: bool) -> None: +def pip_install_requirements( + venv: VenvInfo, requirements: Path, *, upgrade: bool, force_reinstall: bool +) -> None: cmd = ["install"] if upgrade: cmd.append("--upgrade") @@ -247,12 +255,18 @@ def pip_install_prefix( ) -> Optional[Path]: # Resolve a wheel install prefix from site-packages using `pip show`. try: - out = capture_text([str(venv.python), "-m", "pip", "show", dist_name]).replace("\r", "") + out = capture_text([str(venv.python), "-m", "pip", "show", dist_name]).replace( + "\r", "" + ) except CommandError: return None location = next( - (line.split(":", 1)[1].strip() for line in out.splitlines() if line.lower().startswith("location:")), + ( + line.split(":", 1)[1].strip() + for line in out.splitlines() + if line.lower().startswith("location:") + ), "", ) if not location: @@ -261,7 +275,9 @@ def pip_install_prefix( base = Path(location).resolve() def _ok(prefix: Path) -> bool: - return prefix.is_dir() and ((require_subdir is None) or (prefix / require_subdir).is_dir()) + return prefix.is_dir() and ( + (require_subdir is None) or (prefix / require_subdir).is_dir() + ) for cand in candidates: cand = str(cand).strip() @@ -280,7 +296,11 @@ def ensure_mlir_aie_pth(venv: VenvInfo, mlir_aie_install_dir: Path) -> None: return try: site_pkgs = capture_text( - [str(venv.python), "-c", "import sysconfig; p=sysconfig.get_paths().get('purelib') or sysconfig.get_paths().get('platlib') or ''; print(p)"] + [ + str(venv.python), + "-c", + "import sysconfig; p=sysconfig.get_paths().get('purelib') or sysconfig.get_paths().get('platlib') or ''; print(p)", + ] ).strip() if not site_pkgs: return @@ -308,12 +328,22 @@ def fixup_llvm_aie_windows(peano_root: Optional[Path]) -> None: if not libroot.is_dir(): return # llvm-aie installs are organized by target triple under /lib. - toolchains = [p for p in libroot.iterdir() if p.is_dir() and p.name.endswith("-none-unknown-elf")] + toolchains = [ + p + for p in libroot.iterdir() + if p.is_dir() and p.name.endswith("-none-unknown-elf") + ] if not toolchains: - print(f"[fixup] NOTE: no *-none-unknown-elf toolchains found under: {libroot} (skipping llvm-aie patches)") + print( + f"[fixup] NOTE: no *-none-unknown-elf toolchains found under: {libroot} (skipping llvm-aie patches)" + ) objcopy = peano_root / "bin" / "llvm-objcopy.exe" - objcopy_exe = str(objcopy) if objcopy.exists() else (shutil.which("llvm-objcopy") or shutil.which("llvm-objcopy.exe")) + objcopy_exe = ( + str(objcopy) + if objcopy.exists() + else (shutil.which("llvm-objcopy") or shutil.which("llvm-objcopy.exe")) + ) if not objcopy_exe: print(f"[fixup] NOTE: llvm-objcopy not found; skipping crt1.o patch.") @@ -326,7 +356,9 @@ def fixup_llvm_aie_windows(peano_root: Optional[Path]) -> None: if src.exists() and not dst.exists(): try: shutil.copy2(src, dst) - print(f"[fixup] Created alias in {libdir.name}: {dst.name} (copy of {src.name})") + print( + f"[fixup] Created alias in {libdir.name}: {dst.name} (copy of {src.name})" + ) except Exception as e: print(f"[fixup] WARNING: failed to create {dst} from {src}: {e}") @@ -347,7 +379,9 @@ def fixup_llvm_aie_windows(peano_root: Optional[Path]) -> None: check=False, ) if proc.returncode == 0: - print(f"[fixup] Patched {libdir.name}/crt1.o: removed '.deplibs' (if present)") + print( + f"[fixup] Patched {libdir.name}/crt1.o: removed '.deplibs' (if present)" + ) # -------------------------------------------------------------------------------------- @@ -369,7 +403,9 @@ def _apply_all_flag(args: argparse.Namespace, argv: list[str]) -> None: def update_submodules(repo_root: Path) -> None: if (repo_root / ".git").exists() and shutil.which("git"): print("[setup] Updating git submodules...") - run_checked(["git", "submodule", "update", "--init", "--recursive"], cwd=repo_root) + run_checked( + ["git", "submodule", "update", "--init", "--recursive"], cwd=repo_root + ) else: print("[setup] NOTE: Skipping submodules (not a git checkout)") @@ -382,17 +418,27 @@ def print_next_steps(repo_root: Path, *, venv_name: str) -> None: script_rel = script_path script_str = str(script_rel) - print("\n[setup] Setup complete. Set environment variables for your current shell session with:") - print(f" # PowerShell: python {script_str} env --venv {venv_name} --shell pwsh | iex") - print(fr' # cmd.exe : python {script_str} env --venv {venv_name} --shell cmd > "%TEMP%\iron_env.bat" && call "%TEMP%\iron_env.bat"') - print(f' # POSIX sh : eval "$(python3 {script_str} env --venv {venv_name} --shell sh)"') + print( + "\n[setup] Setup complete. Set environment variables for your current shell session with:" + ) + print( + f" # PowerShell: python {script_str} env --venv {venv_name} --shell pwsh | iex" + ) + print( + rf' # cmd.exe : python {script_str} env --venv {venv_name} --shell cmd > "%TEMP%\iron_env.bat" && call "%TEMP%\iron_env.bat"' + ) + print( + f' # POSIX sh : eval "$(python3 {script_str} env --venv {venv_name} --shell sh)"' + ) print(" # source /opt/xilinx/xrt/setup.sh") print("\n[setup] To update later:") print(f" python {script_str} update") -def install_plan(args: argparse.Namespace, repo_root: Path, *, update_mode: bool) -> None: +def install_plan( + args: argparse.Namespace, repo_root: Path, *, update_mode: bool +) -> None: _apply_all_flag(args, sys.argv[1:]) venv_dir = (repo_root / args.venv).resolve() @@ -404,18 +450,29 @@ def install_plan(args: argparse.Namespace, repo_root: Path, *, update_mode: bool print(f"[setup] mode : {'update' if update_mode else 'install'}") force_reinstall = bool(getattr(args, "force_reinstall", False)) - pip_reqs = lambda req: pip_install_requirements(venv, req, upgrade=update_mode, force_reinstall=force_reinstall) - pip_pkg = lambda pkg, **kw: pip_install_package(venv, pkg, upgrade=update_mode, force_reinstall=force_reinstall, **kw) + pip_reqs = lambda req: pip_install_requirements( + venv, req, upgrade=update_mode, force_reinstall=force_reinstall + ) + pip_pkg = lambda pkg, **kw: pip_install_package( + venv, pkg, upgrade=update_mode, force_reinstall=force_reinstall, **kw + ) # Base tooling. - pip_install(venv, ["install", "--upgrade", "pip", "setuptools", "wheel", "packaging"]) + pip_install( + venv, ["install", "--upgrade", "pip", "setuptools", "wheel", "packaging"] + ) # Repo requirements + optional extras. req_specs = [ ("repo_reqs", "python/requirements.txt", None, None), ("dev", "python/requirements_dev.txt", ["pre_commit", "install"], repo_root), ("ml", "python/requirements_ml.txt", None, None), - ("notebook", "python/requirements_notebook.txt", ["ipykernel", "install", "--user", "--name", args.venv], None), + ( + "notebook", + "python/requirements_notebook.txt", + ["ipykernel", "install", "--user", "--name", args.venv], + None, + ), ] for flag, req_rel, post_mod_args, post_cwd in req_specs: @@ -438,13 +495,23 @@ def install_plan(args: argparse.Namespace, repo_root: Path, *, update_mode: bool # Modes: auto | skip | latest-wheels-3 | wheelhouse[:] # Auto mode exists until wheels are available for Windows. mlir_raw = str(getattr(args, "mlir_aie", "auto") or "auto").strip().lower() - mlir_mode, allow_missing_mlir_aie = (("latest-wheels-3", IS_WINDOWS) if mlir_raw == "auto" else (mlir_raw, False)) + mlir_mode, allow_missing_mlir_aie = ( + ("latest-wheels-3", IS_WINDOWS) if mlir_raw == "auto" else (mlir_raw, False) + ) if mlir_mode != "skip": try: if mlir_mode == "wheelhouse" or mlir_mode.startswith("wheelhouse:"): - wheelhouse_dir = Path(mlir_mode.partition(":")[2] or (repo_root / "utils" / "mlir_aie_wheels" / "wheelhouse")).expanduser() - if not wheelhouse_dir.exists(): raise RuntimeError(f"Wheelhouse directory not found: {wheelhouse_dir}") - print(f"[setup] Installing mlir_aie + aie_python_bindings from wheelhouse: {wheelhouse_dir}") + wheelhouse_dir = Path( + mlir_mode.partition(":")[2] + or (repo_root / "utils" / "mlir_aie_wheels" / "wheelhouse") + ).expanduser() + if not wheelhouse_dir.exists(): + raise RuntimeError( + f"Wheelhouse directory not found: {wheelhouse_dir}" + ) + print( + f"[setup] Installing mlir_aie + aie_python_bindings from wheelhouse: {wheelhouse_dir}" + ) for pkg in ("mlir_aie", "aie_python_bindings"): pip_pkg(pkg, wheelhouse=wheelhouse_dir, no_deps=True, no_index=True) else: @@ -453,21 +520,29 @@ def install_plan(args: argparse.Namespace, repo_root: Path, *, update_mode: bool pip_pkg("mlir_aie", find_links=mlir_find_links) except CommandError: if allow_missing_mlir_aie: - print("[setup] NOTE: mlir_aie wheels not available for this platform/Python (continuing).") + print( + "[setup] NOTE: mlir_aie wheels not available for this platform/Python (continuing)." + ) else: raise - if (mlir_prefix := pip_install_prefix(venv, "mlir_aie", ["mlir_aie"])): + if mlir_prefix := pip_install_prefix(venv, "mlir_aie", ["mlir_aie"]): ensure_mlir_aie_pth(venv, mlir_prefix) # llvm-aie wheels (Peano) llvm_choice = str(getattr(args, "llvm_aie", "nightly") or "nightly").strip() - llvm_find_links = "https://github.com/Xilinx/llvm-aie/releases/expanded_assets/nightly" if llvm_choice == "nightly" else llvm_choice + llvm_find_links = ( + "https://github.com/Xilinx/llvm-aie/releases/expanded_assets/nightly" + if llvm_choice == "nightly" + else llvm_choice + ) print(f"[setup] Installing llvm-aie from {llvm_find_links}") pip_pkg("llvm-aie", find_links=llvm_find_links) if IS_WINDOWS: - peano_prefix = pip_install_prefix(venv, "llvm-aie", ["llvm-aie", "llvm_aie"], require_subdir="bin") + peano_prefix = pip_install_prefix( + venv, "llvm-aie", ["llvm-aie", "llvm_aie"], require_subdir="bin" + ) fixup_llvm_aie_windows(peano_prefix) if getattr(args, "submodules", False): @@ -483,7 +558,9 @@ def install_plan(args: argparse.Namespace, repo_root: Path, *, update_mode: bool NPU_REGEX = re.compile(r"NPU Phoenix|RyzenAI-npu[1]", re.IGNORECASE) # Only the following devices are NPU2 -NPU2_REGEX = re.compile(r"NPU Strix|NPU Strix Halo|NPU Krackan|RyzenAI-npu[456]", re.IGNORECASE) +NPU2_REGEX = re.compile( + r"NPU Strix|NPU Strix Halo|NPU Krackan|RyzenAI-npu[456]", re.IGNORECASE +) # Reserved for future use. NPU3_REGEX = re.compile(r"NPU Medusa|RyzenAI-npu[3]", re.IGNORECASE) @@ -513,7 +590,7 @@ def _add(cmd: list[str]) -> None: if shutil.which(exe): _add([exe, "examine"]) - if (sys32 := system32_amd_xrt_smi_dir()): + if sys32 := system32_amd_xrt_smi_dir(): _add([str(sys32 / "xrt-smi.exe"), "examine"]) return out @@ -564,16 +641,22 @@ def env_plan(args: argparse.Namespace, repo_root: Path) -> None: out_lines.append(f"{comment} Activate venv: {venv_dir}") if shell == "pwsh": activate = venv_dir / "Scripts" / "Activate.ps1" - out_lines.append(f"if (Test-Path {ps_quote(str(activate))}) {{ . {ps_quote(str(activate))} }}") + out_lines.append( + f"if (Test-Path {ps_quote(str(activate))}) {{ . {ps_quote(str(activate))} }}" + ) elif shell == "cmd": activate = venv_dir / "Scripts" / "activate.bat" q = cmd_quote(str(activate)) out_lines.append(f"if exist {q} call {q}") else: - activate = (venv_dir / "bin" / "activate") - out_lines.append(f"if [ -f {sh_quote(str(activate))} ]; then source {sh_quote(str(activate))}; fi") + activate = venv_dir / "bin" / "activate" + out_lines.append( + f"if [ -f {sh_quote(str(activate))} ]; then source {sh_quote(str(activate))}; fi" + ) else: - out_lines.append(f"{comment} WARNING: venv not found at: {venv_dir} (run: `python utils/iron_setup.py install`)") + out_lines.append( + f"{comment} WARNING: venv not found at: {venv_dir} (run: `python utils/iron_setup.py install`)" + ) mlir_prefix: Optional[Path] = None peano_prefix: Optional[Path] = None @@ -581,7 +664,9 @@ def env_plan(args: argparse.Namespace, repo_root: Path) -> None: if venv_py.exists(): venv = VenvInfo(venv_dir=venv_dir, python=venv_py) mlir_prefix = pip_install_prefix(venv, "mlir_aie", ["mlir_aie"]) - peano_prefix = pip_install_prefix(venv, "llvm-aie", ["llvm-aie", "llvm_aie"], require_subdir="bin") + peano_prefix = pip_install_prefix( + venv, "llvm-aie", ["llvm-aie", "llvm_aie"], require_subdir="bin" + ) def _override(raw: str, valid, note: str) -> Optional[Path]: raw = (raw or "").strip() @@ -596,11 +681,19 @@ def _override(raw: str, valid, note: str) -> Optional[Path]: return None # Optional explicit overrides. - override = _override(getattr(args, "mlir_aie_install", ""), lambda p: (p / "bin").is_dir() or (p / "python").is_dir(), "Ignoring --mlir-aie-install (not a valid prefix)") + override = _override( + getattr(args, "mlir_aie_install", ""), + lambda p: (p / "bin").is_dir() or (p / "python").is_dir(), + "Ignoring --mlir-aie-install (not a valid prefix)", + ) if override: mlir_prefix = override - override = _override(getattr(args, "llvm_aie_install", ""), lambda p: (p / "bin").is_dir(), "Ignoring --llvm-aie-install (missing bin/)") + override = _override( + getattr(args, "llvm_aie_install", ""), + lambda p: (p / "bin").is_dir(), + "Ignoring --llvm-aie-install (missing bin/)", + ) if override: peano_prefix = override @@ -613,25 +706,37 @@ def _override(raw: str, valid, note: str) -> Optional[Path]: if mlir_prefix: out_lines.extend(emit_set(shell, "MLIR_AIE_INSTALL_DIR", str(mlir_prefix))) out_lines.extend(emit_prepend_path(shell, "PATH", str(mlir_prefix / "bin"))) - out_lines.extend(emit_prepend_path(shell, "PYTHONPATH", str(mlir_prefix / "python"))) + out_lines.extend( + emit_prepend_path(shell, "PYTHONPATH", str(mlir_prefix / "python")) + ) lib_var = "PATH" if IS_WINDOWS else "LD_LIBRARY_PATH" out_lines.extend(emit_prepend_path(shell, lib_var, str(mlir_prefix / "lib"))) else: - out_lines.append(f"{comment} NOTE: mlir_aie not installed in this venv (or not detected).") + out_lines.append( + f"{comment} NOTE: mlir_aie not installed in this venv (or not detected)." + ) # llvm-aie env. if peano_prefix: out_lines.extend(emit_set(shell, "PEANO_INSTALL_DIR", str(peano_prefix))) - out_lines.append(f"{comment} NOTE: llvm-aie is not added to PATH to avoid conflicts with system clang/clang++.") - out_lines.append(f"{comment} It can be found in: {str(peano_prefix / 'bin')}") + out_lines.append( + f"{comment} NOTE: llvm-aie is not added to PATH to avoid conflicts with system clang/clang++." + ) + out_lines.append( + f"{comment} It can be found in: {str(peano_prefix / 'bin')}" + ) else: - out_lines.append(f"{comment} WARNING: llvm-aie not installed in this venv (run: `python utils/iron_setup.py install`)") + out_lines.append( + f"{comment} WARNING: llvm-aie not installed in this venv (run: `python utils/iron_setup.py install`)" + ) # XRT env. if IS_WINDOWS: # XILINX_XRT must not be set on Windows. if shell == "pwsh": - out_lines.append(r"Remove-Item Env:\XILINX_XRT -ErrorAction SilentlyContinue") + out_lines.append( + r"Remove-Item Env:\XILINX_XRT -ErrorAction SilentlyContinue" + ) elif shell == "cmd": out_lines.append('set "XILINX_XRT="') else: @@ -640,11 +745,25 @@ def _override(raw: str, valid, note: str) -> Optional[Path]: xrt_root = _resolve_windows_xrt_root(args) if xrt_root: out_lines.extend(emit_set(shell, "XRT_ROOT", str(xrt_root))) - for p in (xrt_root / "ext" / "bin", xrt_root / "lib", xrt_root / "unwrapped", xrt_root): + for p in ( + xrt_root / "ext" / "bin", + xrt_root / "lib", + xrt_root / "unwrapped", + xrt_root, + ): out_lines.extend(emit_append_path_if_exists(shell, "PATH", str(p))) - out_lines.extend(emit_append_path_if_exists(shell, "PYTHONPATH", str(xrt_root / "python"))) + out_lines.extend( + emit_append_path_if_exists( + shell, "PYTHONPATH", str(xrt_root / "python") + ) + ) else: - xrt_root = (getattr(args, "xrt_root", "") or os.environ.get("XRT_ROOT") or os.environ.get("XILINX_XRT") or "").strip() + xrt_root = ( + getattr(args, "xrt_root", "") + or os.environ.get("XRT_ROOT") + or os.environ.get("XILINX_XRT") + or "" + ).strip() if xrt_root: out_lines.extend(emit_set(shell, "XRT_ROOT", xrt_root)) out_lines.extend(emit_set(shell, "XILINX_XRT", xrt_root)) @@ -682,18 +801,71 @@ def _add_bool_flag_pair( def _add_install_args(p: argparse.ArgumentParser) -> None: - p.add_argument("--python", default=sys.executable, help="Python executable used to create the venv") - p.add_argument("--venv", default="ironenv", help="Venv directory name relative to repo root") - p.add_argument("--mlir-aie", default="auto", help="mlir_aie wheel source: auto | skip | latest-wheels-3 | wheelhouse[:]") - p.add_argument("--llvm-aie", default="nightly", help="llvm-aie wheels: nightly (default) or a custom -f URL") - p.add_argument("--force-reinstall", dest="force_reinstall", action="store_true", help="Force reinstall packages (pip --force-reinstall). Useful for testers.",) - p.add_argument("--all", action="store_true", help="Enable everything (repo-reqs + dev + ml + notebook + submodules).") + p.add_argument( + "--python", + default=sys.executable, + help="Python executable used to create the venv", + ) + p.add_argument( + "--venv", default="ironenv", help="Venv directory name relative to repo root" + ) + p.add_argument( + "--mlir-aie", + default="auto", + help="mlir_aie wheel source: auto | skip | latest-wheels-3 | wheelhouse[:]", + ) + p.add_argument( + "--llvm-aie", + default="nightly", + help="llvm-aie wheels: nightly (default) or a custom -f URL", + ) + p.add_argument( + "--force-reinstall", + dest="force_reinstall", + action="store_true", + help="Force reinstall packages (pip --force-reinstall). Useful for testers.", + ) + p.add_argument( + "--all", + action="store_true", + help="Enable everything (repo-reqs + dev + ml + notebook + submodules).", + ) - _add_bool_flag_pair(p, "repo-reqs", default=True, help_on="Install python/requirements.txt (repo Python deps).", help_off="Skip repo Python deps install.") - _add_bool_flag_pair(p, "dev", default=True, help_on="Install python/requirements_dev.txt + pre-commit.", help_off="Skip dev deps.") - _add_bool_flag_pair(p, "ml", default=False, help_on="Install python/requirements_ml.txt.", help_off="Skip ML deps.") - _add_bool_flag_pair(p, "notebook", default=False, help_on="Install python/requirements_notebook.txt + (best-effort) ipykernel registration.", help_off="Skip notebook deps.",) - _add_bool_flag_pair(p, "submodules", default=False, help_on="Update git submodules if this is a git checkout.", help_off="Do not update git submodules.",) + _add_bool_flag_pair( + p, + "repo-reqs", + default=True, + help_on="Install python/requirements.txt (repo Python deps).", + help_off="Skip repo Python deps install.", + ) + _add_bool_flag_pair( + p, + "dev", + default=True, + help_on="Install python/requirements_dev.txt + pre-commit.", + help_off="Skip dev deps.", + ) + _add_bool_flag_pair( + p, + "ml", + default=False, + help_on="Install python/requirements_ml.txt.", + help_off="Skip ML deps.", + ) + _add_bool_flag_pair( + p, + "notebook", + default=False, + help_on="Install python/requirements_notebook.txt + (best-effort) ipykernel registration.", + help_off="Skip notebook deps.", + ) + _add_bool_flag_pair( + p, + "submodules", + default=False, + help_on="Update git submodules if this is a git checkout.", + help_off="Do not update git submodules.", + ) def build_arg_parser() -> argparse.ArgumentParser: @@ -704,16 +876,45 @@ def build_arg_parser() -> argparse.ArgumentParser: p_install_common = argparse.ArgumentParser(add_help=False) _add_install_args(p_install_common) - p_install = sub.add_parser("install", parents=[p_install_common], help="Install toolchain wheels + deps into the venv") - p_update = sub.add_parser("update", parents=[p_install_common], help="Update to newest wheels + deps in the existing venv") + p_install = sub.add_parser( + "install", + parents=[p_install_common], + help="Install toolchain wheels + deps into the venv", + ) + p_update = sub.add_parser( + "update", + parents=[p_install_common], + help="Update to newest wheels + deps in the existing venv", + ) p_update.set_defaults(submodules=True) - p_env = sub.add_parser("env", help="Print shell commands to activate venv + export toolchain env vars") - p_env.add_argument("--venv", default="ironenv", help="Venv directory relative to repo root") - p_env.add_argument("--xrt-root", default="", help="Optional XRT install directory (Windows: default C:/Xilinx/XRT). Can also be set via XRT_ROOT.",) - p_env.add_argument("--mlir-aie-install", default="", help="Optional explicit mlir-aie install prefix (relative to repo root unless absolute).") - p_env.add_argument("--llvm-aie-install", default="", help="Optional explicit llvm-aie/peano install prefix (relative to repo root unless absolute).") - p_env.add_argument("--shell", default="auto", choices=["auto", "sh", "pwsh", "cmd"], help="Which shell syntax to emit (default: auto based on platform)") + p_env = sub.add_parser( + "env", help="Print shell commands to activate venv + export toolchain env vars" + ) + p_env.add_argument( + "--venv", default="ironenv", help="Venv directory relative to repo root" + ) + p_env.add_argument( + "--xrt-root", + default="", + help="Optional XRT install directory (Windows: default C:/Xilinx/XRT). Can also be set via XRT_ROOT.", + ) + p_env.add_argument( + "--mlir-aie-install", + default="", + help="Optional explicit mlir-aie install prefix (relative to repo root unless absolute).", + ) + p_env.add_argument( + "--llvm-aie-install", + default="", + help="Optional explicit llvm-aie/peano install prefix (relative to repo root unless absolute).", + ) + p_env.add_argument( + "--shell", + default="auto", + choices=["auto", "sh", "pwsh", "cmd"], + help="Which shell syntax to emit (default: auto based on platform)", + ) return p diff --git a/utils/mlir_aie_wheels/python_bindings/setup.py b/utils/mlir_aie_wheels/python_bindings/setup.py index 642ab75278e..f79d878c170 100644 --- a/utils/mlir_aie_wheels/python_bindings/setup.py +++ b/utils/mlir_aie_wheels/python_bindings/setup.py @@ -187,9 +187,7 @@ def build_extension(self, ext: CMakeExtension) -> None: if platform.system() == "Windows": cmake_src = _cmake_path(cmake_src) - subprocess.run( - ["cmake", cmake_src, *cmake_args], cwd=build_temp, check=True - ) + subprocess.run(["cmake", cmake_src, *cmake_args], cwd=build_temp, check=True) subprocess.run( ["cmake", "--build", ".", "--target", "install", *build_args], cwd=build_temp, diff --git a/utils/mlir_aie_wheels/scripts/download_mlir.py b/utils/mlir_aie_wheels/scripts/download_mlir.py index c1b20bde0b3..e40cfe862aa 100644 --- a/utils/mlir_aie_wheels/scripts/download_mlir.py +++ b/utils/mlir_aie_wheels/scripts/download_mlir.py @@ -193,7 +193,9 @@ def _repl(m): cmake_file.write_text(new_data, encoding="utf-8") if patched: - print(f"[fixup] Patched diaguids.lib path -> {repl_path if repl_path else 'diaguids.lib'}") + print( + f"[fixup] Patched diaguids.lib path -> {repl_path if repl_path else 'diaguids.lib'}" + ) def main(): @@ -219,7 +221,9 @@ def main(): elif matrix_os == "ubuntu-20.04" and cibw_archs == "aarch64": plat = "linux_aarch64" else: - raise SystemExit(f"Unsupported CIBW_ARCHS/MATRIX_OS: {cibw_archs}/{matrix_os}") + raise SystemExit( + f"Unsupported CIBW_ARCHS/MATRIX_OS: {cibw_archs}/{matrix_os}" + ) _pip(["-q", "download", pkg, "--platform", plat, "--only-binary=:all:"]) else: diff --git a/utils/run_example.py b/utils/run_example.py index a40772f829d..1fe73162c8a 100644 --- a/utils/run_example.py +++ b/utils/run_example.py @@ -49,7 +49,6 @@ from pathlib import Path from typing import Optional - # -------------------------------------------------------------------------------------- # Platform + process helpers # -------------------------------------------------------------------------------------- @@ -65,7 +64,9 @@ def is_wsl() -> bool: if os.environ.get("WSL_INTEROP") or os.environ.get("WSL_DISTRO_NAME"): return True try: - osrelease = Path("/proc/sys/kernel/osrelease").read_text(errors="ignore").lower() + osrelease = ( + Path("/proc/sys/kernel/osrelease").read_text(errors="ignore").lower() + ) return "microsoft" in osrelease or "wsl" in osrelease except Exception: return False @@ -77,7 +78,9 @@ def _wsl_to_windows_path(path: Path) -> str: return str(path) wslpath_exe = which("wslpath") if not wslpath_exe: - raise RuntimeError("wslpath not found; cannot translate WSL paths for Windows tools") + raise RuntimeError( + "wslpath not found; cannot translate WSL paths for Windows tools" + ) out = subprocess.check_output([wslpath_exe, "-w", str(path)], text=True).strip() if not out: raise RuntimeError(f"wslpath returned empty path for: {path}") @@ -125,7 +128,6 @@ def _is_multi_config_generator(generator: str) -> bool: return ("visual studio" in g) or ("xcode" in g) or ("ninja multi-config" in g) - def run_checked( cmd: list[str], *, @@ -230,7 +232,9 @@ def _looks_like_flag_incompatible(stderr_text: str) -> bool: return False -def _run_to_file(cmd: list[str], cwd: Path, out_path: Path) -> subprocess.CompletedProcess[str]: +def _run_to_file( + cmd: list[str], cwd: Path, out_path: Path +) -> subprocess.CompletedProcess[str]: # Write to a temp file and replace on success so failed attempts don't clobber good artifacts. tmp = out_path.with_suffix(out_path.suffix + ".tmp") with tmp.open("w", encoding="utf-8") as f: @@ -275,6 +279,7 @@ class ExampleMakeInfo: default_trace_size: Optional[int] default_col: Optional[int] + _TARGET_RE = re.compile(r"^\s*targetname\s*[:?]?=\s*(\S+)\s*$") _AIE_PY_SRC_RE = re.compile(r"^\s*aie_py_src\s*[:?]?=\s*(\S+)\s*$") _FINAL_RULE_RE = re.compile(r"^\s*(build/final[^:]*\.xclbin)\s*:\s*(.+?)\s*$") @@ -283,7 +288,9 @@ class ExampleMakeInfo: _TRACE_RE = re.compile(r"^\s*trace_size\s*[:?]?=\s*(\d+)\s*(?:#.*)?$") _COL_RE = re.compile(r"^\s*col\s*[:?]?=\s*(\d+)\s*(?:#.*)?$", re.IGNORECASE) _VAR_REF_RE = re.compile(r"\$\(([^)]+)\)|\${([^}]+)}") -_SIMPLE_ASSIGN_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\?|\+|:)?=\s*(.+?)\s*$") +_SIMPLE_ASSIGN_RE = re.compile( + r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\?|\+|:)?=\s*(.+?)\s*$" +) def expand_make_vars(value: str, vars_map: dict[str, str]) -> str: @@ -292,6 +299,7 @@ def expand_make_vars(value: str, vars_map: dict[str, str]) -> str: # targetname=${orig_targetname}_chained cur = value for _ in range(6): + def _repl(m: re.Match) -> str: name = m.group(1) or m.group(2) or "" return vars_map.get(name, m.group(0)) @@ -337,7 +345,9 @@ def _parse_vpath(line: str, example_dir: Path, vars_map: dict[str, str]) -> list vpath_dirs: list[Path] = [] for p in raw_parts: # Expand ${srcdir}/$(srcdir) even if it wasn't in the vars map. - p = p.replace("${srcdir}", str(example_dir)).replace("$(srcdir)", str(example_dir)) + p = p.replace("${srcdir}", str(example_dir)).replace( + "$(srcdir)", str(example_dir) + ) vpath_dirs.append(Path(p).resolve()) return vpath_dirs @@ -366,7 +376,6 @@ def parse_makefile(example_dir: Path) -> ExampleMakeInfo: xclbin_name_hint: Optional[str] = None insts_name_hint: Optional[str] = None - def _sanitize_make_token(tok: str) -> Optional[str]: # Ignore Make variables like ${@F} / ${ str: ) return ( "Activate the mlir-aie environment first (sets MLIR_AIE_INSTALL_DIR, PEANO_INSTALL_DIR, XRT_ROOT, PATH):\n" - " eval \"$(python3 utils/iron_setup.py env --shell bash)\"" + ' eval "$(python3 utils/iron_setup.py env --shell bash)"' ) @@ -529,7 +538,9 @@ def preflight_or_die(action: str) -> None: if not which("bootgen"): missing.append("bootgen (required by aiecc to package .xclbin)") if not which("xclbinutil"): - missing.append("xclbinutil (required by aiecc; usually from XRT/Vitis/Ryzen AI)") + missing.append( + "xclbinutil (required by aiecc; usually from XRT/Vitis/Ryzen AI)" + ) if need_kernel_compile: peano = resolve_peano_install_dir() @@ -647,7 +658,9 @@ def resolve_kernel_sources(info: ExampleMakeInfo) -> list[tuple[str, Path]]: candidates: list[Path] = [] for ext in exts: fname = f"{name}{ext}" - candidates.extend([info.example_dir / fname] + [vp / fname for vp in info.vpath_dirs]) + candidates.extend( + [info.example_dir / fname] + [vp / fname for vp in info.vpath_dirs] + ) found: Optional[Path] = None for cand in candidates: if cand.exists(): @@ -665,7 +678,9 @@ def resolve_kernel_sources(info: ExampleMakeInfo) -> list[tuple[str, Path]]: return sources -def compile_kernel_objects(cfg: BuildConfig, info: ExampleMakeInfo, build_dir: Path) -> None: +def compile_kernel_objects( + cfg: BuildConfig, info: ExampleMakeInfo, build_dir: Path +) -> None: mlir_aie_root = resolve_mlir_aie_root() peano_install = resolve_peano_install_dir() clangpp = resolve_aie_clangpp(peano_install) @@ -732,7 +747,9 @@ def resolve_generator_script(cfg: BuildConfig, info: ExampleMakeInfo) -> Path: if info.aie_py_src: cand = Path(info.aie_py_src) - cand = cand.resolve() if cand.is_absolute() else (cfg.example_dir / cand).resolve() + cand = ( + cand.resolve() if cand.is_absolute() else (cfg.example_dir / cand).resolve() + ) if cand.exists(): if cfg.placed: placed_cand = cand.with_name(f"{cand.stem}_placed{cand.suffix}") @@ -765,7 +782,9 @@ def resolve_generator_script(cfg: BuildConfig, info: ExampleMakeInfo) -> Path: raise FileNotFoundError(f"No python generator script found in: {cfg.example_dir}") -def generate_aie_mlir(cfg: BuildConfig, info: ExampleMakeInfo, build_dir: Path, trace: bool) -> Path: +def generate_aie_mlir( + cfg: BuildConfig, info: ExampleMakeInfo, build_dir: Path, trace: bool +) -> Path: data_size = cfg.in1_size suffix = "trace_" if trace else "" out_path = build_dir / f"aie_{suffix}{data_size}.mlir" @@ -843,7 +862,9 @@ def generate_aie_mlir(cfg: BuildConfig, info: ExampleMakeInfo, build_dir: Path, print("[gen] Retrying generator with positional args.") proc2 = _run_to_file(cmd, cwd=cfg.example_dir, out_path=out_path) if proc2.returncode == 0: - print("[gen] NOTE: generator does not accept flags; using positional invocation.") + print( + "[gen] NOTE: generator does not accept flags; using positional invocation." + ) _write_json_atomic(stamp, stamp_data) return out_path err = proc2.stderr or err @@ -923,7 +944,9 @@ def build_xclbin_and_insts( # Some examples name outputs differently; fall back to the most recent artifacts. if not xclbin_path.exists(): - candidates = sorted(build_dir.glob("*.xclbin"), key=lambda p: p.stat().st_mtime, reverse=True) + candidates = sorted( + build_dir.glob("*.xclbin"), key=lambda p: p.stat().st_mtime, reverse=True + ) if candidates: xclbin_path = candidates[0] else: @@ -938,7 +961,9 @@ def build_xclbin_and_insts( for pat in patterns: inst_candidates.extend(build_dir.glob(pat)) - inst_candidates = sorted(set(inst_candidates), key=lambda p: p.stat().st_mtime, reverse=True) + inst_candidates = sorted( + set(inst_candidates), key=lambda p: p.stat().st_mtime, reverse=True + ) if inst_candidates: insts_path = inst_candidates[0] else: @@ -946,6 +971,7 @@ def build_xclbin_and_insts( return (xclbin_path, insts_path) + def _copy_if_newer(src: Path, dst: Path) -> bool: # Copy src -> dst if src is newer (or dst missing). Returns True if copied. if dst.exists(): @@ -985,13 +1011,18 @@ def build_host_exe(cfg: BuildConfig) -> Path: "out_size": cfg.out_size, "int_bit_width": cfg.int_bit_width, "generator_request": cfg.generator, - "generator_effective": _cmake_generator_from_cache(host_build_dir) or cfg.generator, + "generator_effective": _cmake_generator_from_cache(host_build_dir) + or cfg.generator, "config": cfg.config, - "platform": ("wsl_windows_host" if is_wsl() else ("windows" if is_windows() else "posix")), + "platform": ( + "wsl_windows_host" if is_wsl() else ("windows" if is_windows() else "posix") + ), } need_configure = True - if (host_build_dir / "CMakeCache.txt").exists() and _stamp_matches(configure_stamp, configure_data): + if (host_build_dir / "CMakeCache.txt").exists() and _stamp_matches( + configure_stamp, configure_data + ): need_configure = False if need_configure: @@ -1059,8 +1090,8 @@ def build_host_exe(cfg: BuildConfig) -> Path: # Common output locations: candidates = [ - host_build_dir / cfg.config / exe_file, # multi-config (VS/MSBuild) - host_build_dir / exe_file, # single-config (Ninja/Unix Makefiles) + host_build_dir / cfg.config / exe_file, # multi-config (VS/MSBuild) + host_build_dir / exe_file, # single-config (Ninja/Unix Makefiles) ] src_exe: Optional[Path] = None @@ -1073,7 +1104,9 @@ def build_host_exe(cfg: BuildConfig) -> Path: if src_exe is None: hits: list[Path] = [] for cand in host_build_dir.rglob(exe_file): - if any(part in {"CMakeFiles"} or part.endswith(".dir") for part in cand.parts): + if any( + part in {"CMakeFiles"} or part.endswith(".dir") for part in cand.parts + ): continue if cand.is_file(): hits.append(cand) @@ -1094,7 +1127,10 @@ def build_host_exe(cfg: BuildConfig) -> Path: print(f"[host] Up-to-date: {out}") return out -def run_host_exe(exe: Path, xclbin: Path, insts: Path, trace_size: Optional[int]) -> None: + +def run_host_exe( + exe: Path, xclbin: Path, insts: Path, trace_size: Optional[int] +) -> None: if is_wsl(): # The host binary is a Windows executable (built via Windows CMake) and must be launched via Windows. exe_win = _wsl_to_windows_path(exe) @@ -1112,7 +1148,14 @@ def _ps_quote(s: str) -> str: ) if trace_size is not None: cmd += f" -t {trace_size}" - ps = ["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", cmd] + ps = [ + "powershell.exe", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-Command", + cmd, + ] run_checked(ps, cwd=exe.parent) return @@ -1121,7 +1164,10 @@ def _ps_quote(s: str) -> str: cmd += ["-t", str(trace_size)] run_checked(cmd, cwd=exe.parent) -def run_python_test(example_dir: Path, xclbin: Path, insts: Path, cfg: BuildConfig, trace: bool) -> None: + +def run_python_test( + example_dir: Path, xclbin: Path, insts: Path, cfg: BuildConfig, trace: bool +) -> None: test_py = example_dir / "test.py" if not test_py.exists(): raise FileNotFoundError(f"test.py not found: {test_py}") @@ -1219,7 +1265,9 @@ def parse_trace_outputs(example_dir: Path, build_dir: Path, cfg: BuildConfig) -> def build_arg_parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser(description="Run mlir-aie programming examples (python backend).") + p = argparse.ArgumentParser( + description="Run mlir-aie programming examples (python backend)." + ) p.add_argument( "action", @@ -1329,10 +1377,14 @@ def _default_device_from_env() -> Optional[str]: def _maybe_warn_trace_target_missing(info: ExampleMakeInfo) -> None: if not info.has_trace: - print("[trace] NOTE: Makefile has no obvious trace target; attempting trace build anyway.") + print( + "[trace] NOTE: Makefile has no obvious trace target; attempting trace build anyway." + ) -def _build_config_from_args(args: argparse.Namespace, example_dir: Path, info: ExampleMakeInfo) -> BuildConfig: +def _build_config_from_args( + args: argparse.Namespace, example_dir: Path, info: ExampleMakeInfo +) -> BuildConfig: # Defaults device = args.device if device is None: @@ -1351,10 +1403,18 @@ def _build_config_from_args(args: argparse.Namespace, example_dir: Path, info: E in1_default, in2_default, _out_default = default_sizes_for_bitwidth(bit_width) in1 = args.in1_size if args.in1_size is not None else in1_default - in2 = args.in2_size if args.in2_size is not None else (info.default_in2_size or in2_default) + in2 = ( + args.in2_size + if args.in2_size is not None + else (info.default_in2_size or in2_default) + ) out = args.out_size if args.out_size is not None else in1 - trace_size = args.trace_size if args.trace_size is not None else (info.default_trace_size or 8192) + trace_size = ( + args.trace_size + if args.trace_size is not None + else (info.default_trace_size or 8192) + ) generator_script = Path(args.generator_script) if args.generator_script else None @@ -1448,7 +1508,9 @@ def _build_aie(trace: bool) -> tuple[Path, Path]: if action in RUN_HOST_ACTIONS: assert exe is not None and xclbin is not None and insts is not None - run_host_exe(exe, xclbin, insts, trace_size=cfg.trace_size if is_trace else None) + run_host_exe( + exe, xclbin, insts, trace_size=cfg.trace_size if is_trace else None + ) if action in RUN_PY_ACTIONS: assert xclbin is not None and insts is not None From 1faf84793acc14793993c8b80f49be940c9ce1c5 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Fri, 13 Feb 2026 20:26:47 -0500 Subject: [PATCH 10/21] Fix minor documentation error --- docs/buildHostWinNative.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/buildHostWinNative.md b/docs/buildHostWinNative.md index c47a53b64a0..52554530851 100644 --- a/docs/buildHostWinNative.md +++ b/docs/buildHostWinNative.md @@ -267,7 +267,7 @@ powershell -NoProfile -ExecutionPolicy Bypass -Command ^ "$libDir=Join-Path $env:XRT_ROOT 'lib';" ^ "Push-Location $libDir;" ^ "@('LIBRARY xrt_coreutil','EXPORTS')|Set-Content xrt_coreutil.def -Encoding ascii;" ^ - "dumpbin /exports $dll|Where-Object{$_ -match '^\s+(\d+)\s+[0-9A-F]+\s+[0-9A-F]+\s+(\S+)'}|ForEach-Object{""$($matches[2]) @$($matches[1])""}|Select-Object -Unique|Add-Content xrt_coreutil.def -Encoding ascii;" ^ + "dumpbin /exports $dll|Where-Object{$_ -match '^^\s+(\d+)\s+[0-9A-F]+\s+[0-9A-F]+\s+(\S+)'}|ForEach-Object{('{0} @{1}' -f $matches[2],$matches[1])}|Select-Object -Unique|Add-Content xrt_coreutil.def -Encoding ascii;" ^ "lib /nologo /def:xrt_coreutil.def /machine:x64 /out:xrt_coreutil.lib;" ^ "Pop-Location" ``` From 42bd97e083fa19db0245f9705275b439db11bc82 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Mon, 16 Feb 2026 21:03:42 -0500 Subject: [PATCH 11/21] PR fork-branch targetting patch --- .github/actions/setup_base/action.yml | 2 +- .github/workflows/mlirAIEDistro.yml | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup_base/action.yml b/.github/actions/setup_base/action.yml index 476a4eecba7..b1ca9dba29d 100644 --- a/.github/actions/setup_base/action.yml +++ b/.github/actions/setup_base/action.yml @@ -127,7 +127,7 @@ runs: rm -rf "$WORKSPACE_ROOT" fi - git clone --recursive -b ${{ github.head_ref || github.ref_name }} https://github.com/${{ github.repository }}.git "$WORKSPACE_ROOT" + git clone --recursive -b ${{ github.event.pull_request.head.ref || github.ref_name }} https://github.com/${{ github.event.pull_request.head.repo.full_name || github.repository }}.git "$WORKSPACE_ROOT" ls "$WORKSPACE_ROOT" if [ x"${{ inputs.MATRIX_OS }}" == x"windows-2022" ]; then diff --git a/.github/workflows/mlirAIEDistro.yml b/.github/workflows/mlirAIEDistro.yml index 21fd8231f37..d96e27581df 100644 --- a/.github/workflows/mlirAIEDistro.yml +++ b/.github/workflows/mlirAIEDistro.yml @@ -51,7 +51,12 @@ jobs: run: | if [ x"${{ inputs.AIE_COMMIT }}" == x"" ]; then sudo apt install jq - AIE_PROJECT_COMMIT=$(curl -s https://api.github.com/repos/Xilinx/mlir-aie/commits/${{ github.head_ref || github.ref_name }} | jq -r '.sha[:8]') + if [ x"${{ github.event_name }}" == x"pull_request" ]; then + AIE_PROJECT_COMMIT="${{ github.event.pull_request.head.sha }}" + AIE_PROJECT_COMMIT="${AIE_PROJECT_COMMIT:0:8}" + else + AIE_PROJECT_COMMIT=$(curl -s https://api.github.com/repos/Xilinx/mlir-aie/commits/${{ github.ref_name }} | jq -r '.sha[:8]') + fi else AIE_PROJECT_COMMIT="${{ inputs.AIE_COMMIT }}" fi @@ -221,7 +226,8 @@ jobs: shell: bash run: | - git clone --recursive https://github.com/${{ github.repository }}.git + AIE_REPO="${{ github.event.pull_request.head.repo.full_name || github.repository }}" + git clone --recursive https://github.com/${AIE_REPO}.git pushd mlir-aie git reset --hard ${{ needs.get_aie_project_commit.outputs.AIE_PROJECT_COMMIT }} git submodule update From f25c41377c7697ac64c18bb736f10239de51912f Mon Sep 17 00:00:00 2001 From: thomthehound Date: Mon, 16 Feb 2026 22:44:07 -0500 Subject: [PATCH 12/21] Fix typo and add correct whl expansion in mlirAIEDistro.yml --- .github/workflows/mlirAIEDistro.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mlirAIEDistro.yml b/.github/workflows/mlirAIEDistro.yml index d96e27581df..790d51fca47 100644 --- a/.github/workflows/mlirAIEDistro.yml +++ b/.github/workflows/mlirAIEDistro.yml @@ -269,7 +269,8 @@ jobs: $opensslBin = Join-Path $env:OPENSSL_ROOT_DIR "bin" New-Item -ItemType Directory -Force -Path wheelhouse\repaired_wheel | Out-Null - python -m delvewheel repair --ignore-existing --analyze-existing-exes -add-path $opensslBin -w wheelhouse\repaired_wheel wheelhouse\mlir_aie*.whl + $wheels = Get-ChildItem wheelhouse\mlir_aie*.whl | ForEach-Object { $_.FullName } + python -m delvewheel repair --ignore-existing --analyze-existing-exes --add-path "$opensslBin" -w wheelhouse\repaired_wheel $wheels Remove-Item wheelhouse\mlir_aie*.whl -Force Move-Item wheelhouse\repaired_wheel\*.whl wheelhouse\ From 62ebef61a91ce9cd4bcf3bafbb2c6dd63a2a6515 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Tue, 17 Feb 2026 00:30:01 -0500 Subject: [PATCH 13/21] Skip MLIR AIE Distro releases from forked PRs (no write access) --- .github/workflows/mlirAIEDistro.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mlirAIEDistro.yml b/.github/workflows/mlirAIEDistro.yml index 790d51fca47..abdb193a9e0 100644 --- a/.github/workflows/mlirAIEDistro.yml +++ b/.github/workflows/mlirAIEDistro.yml @@ -486,6 +486,7 @@ jobs: path: dist - name: Release current commit + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} uses: ncipollo/release-action@v1.12.0 with: artifacts: "dist/*.whl,dist/*.tar.xz" From c1e21c14906ef67558b08677d4d535e58c286f16 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Tue, 24 Feb 2026 16:42:50 -0500 Subject: [PATCH 14/21] add build_local.py --- utils/mlir_aie_wheels/scripts/build_local.py | 237 +++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 utils/mlir_aie_wheels/scripts/build_local.py diff --git a/utils/mlir_aie_wheels/scripts/build_local.py b/utils/mlir_aie_wheels/scripts/build_local.py new file mode 100644 index 00000000000..7c4b89cb072 --- /dev/null +++ b/utils/mlir_aie_wheels/scripts/build_local.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +##===------ build_local.py - Local wheel build orchestration (cross-platform)------===## +# +# This file licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +##===------------------------------------------------------------------------------===## +# Python shim for scripts/build_local.sh (cross-platform, Windows-friendly). +# +# Workflow: +# 1) Detect platform (linux/macos/windows) +# 2) Export the same env vars as build_local.sh +# 3) Run cibuildwheel for the main wheel project +# 4) Rename the main wheel's python tag (cpXYZ-cpXYZ -> py3-none) +# 5) Stage inputs into python_bindings/, unzip the mlir_aie wheel into it +# 6) Run cibuildwheel for python_bindings, outputting into ../wheelhouse +##===------------------------------------------------------------------------------===## + + +from __future__ import annotations + +import os +import platform +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Iterable, Optional + + +DEFAULT_PIP_FIND_LINKS = "https://github.com/Xilinx/mlir-aie/releases/expanded_assets/mlir-distro" + + +# -------------------------------------------------------------------------------------- +# Helpers +# -------------------------------------------------------------------------------------- + + +def _run(cmd: list[str], *, cwd: Optional[Path] = None, env: Optional[dict[str, str]] = None) -> None: + pretty = " ".join(cmd) + print(f"[run] $ {pretty}") + subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, check=True) + + +def _which(exe: str) -> Optional[str]: + return shutil.which(exe) + + +def _cibuildwheel_cmd() -> list[str]: + return [sys.executable, "-m", "cibuildwheel"] + + +def _copy_tree_merge(src: Path, dst: Path) -> None: + dst.mkdir(parents=True, exist_ok=True) + shutil.copytree(src, dst, dirs_exist_ok=True) + + +def _unzip_overwrite(zip_path: Path, dst_dir: Path) -> None: + import zipfile + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(dst_dir) + + +# -------------------------------------------------------------------------------------- +# Platform detection + env +# -------------------------------------------------------------------------------------- + + +def _detect_machine() -> str: + sysname = platform.system().lower() + if sysname.startswith("linux"): + return "linux" + if sysname.startswith("darwin"): + return "macos" + if sysname.startswith("windows"): + return "windows" + # Fall back to the uname-style string for debugging. + return f"UNKNOWN:{platform.system()}" + + +def _apply_env_for_machine(machine: str) -> None: + os.environ.setdefault("APPLY_PATCHES", "true") + os.environ.setdefault("CIBW_BUILD", "cp312-* cp313-* cp314-*") + + if machine == "linux": + os.environ.setdefault("MATRIX_OS", "ubuntu-22.04") + os.environ.setdefault("CIBW_ARCHS", "x86_64") + os.environ.setdefault("ARCH", "x86_64") + os.environ.setdefault("PARALLEL_LEVEL", "15") + elif machine == "macos": + os.environ.setdefault("MATRIX_OS", "macos-14") + os.environ.setdefault("CIBW_ARCHS", "arm64") + os.environ.setdefault("ARCH", "arm64") + os.environ.setdefault("PARALLEL_LEVEL", "32") + else: + # Treat everything else as Windows, mirroring the bash script. + os.environ.setdefault("MATRIX_OS", "windows-2022") + os.environ.setdefault("CIBW_ARCHS", "AMD64") + os.environ.setdefault("ARCH", "AMD64") + + +# -------------------------------------------------------------------------------------- +# Wheelhouse mutation +# -------------------------------------------------------------------------------------- + + +# Rename cpXYZ-cpXYZ -> py3-none in mlir*.whl. +def _rename_main_wheel_python_tag(wheelhouse: Path) -> None: + tag_re = re.compile(r"cp\d{2,3}-cp\d{2,3}") + + for whl in wheelhouse.glob("mlir*whl"): + new_name = tag_re.sub("py3-none", whl.name, count=1) + if new_name == whl.name: + continue + dst = whl.with_name(new_name) + print(f"[rename] {whl.name} -> {dst.name}") + whl.rename(dst) + + +def _copy_ccache_from_wheelhouse(wheelhouse: Path, host_ccache_dir: Path) -> None: + src = wheelhouse / ".ccache" + if not src.is_dir(): + return + host_ccache_dir.mkdir(parents=True, exist_ok=True) + + for item in src.iterdir(): + dst = host_ccache_dir / item.name + if item.is_dir(): + shutil.copytree(item, dst, dirs_exist_ok=True) + else: + shutil.copy2(item, dst) + + +# -------------------------------------------------------------------------------------- +# Main +# -------------------------------------------------------------------------------------- + + +def main() -> int: + here = Path(__file__).resolve().parent + project_root = (here / "..").resolve() # utils/mlir_aie_wheels + wheelhouse = project_root / "wheelhouse" + python_bindings = project_root / "python_bindings" + + # If a nested checkout doesn't exist and the user didn't set MLIR_AIE_SOURCE_DIR, + # point to the repo root so we can find python/requirements_dev.txt and CMake source. + src_env = os.environ.get("MLIR_AIE_SOURCE_DIR", "").strip() + if src_env: + repo_root = Path(src_env).resolve() + else: + nested_checkout = project_root / "mlir-aie" + if (nested_checkout / "utils" / "iron_setup.py").exists(): + repo_root = nested_checkout.resolve() + else: + repo_root = project_root.parent.parent.resolve() + os.environ["MLIR_AIE_SOURCE_DIR"] = str(repo_root) + print(f"[info] MLIR_AIE_SOURCE_DIR not set; defaulting to {repo_root}") + + machine = _detect_machine() + print(machine) + + _apply_env_for_machine(machine) + + # For local builds, default to the official mlir-distro unless the user overrides. + if not os.environ.get("PIP_FIND_LINKS", "").strip(): + os.environ["PIP_FIND_LINKS"] = DEFAULT_PIP_FIND_LINKS + print(f"[info] PIP_FIND_LINKS not set; defaulting to {DEFAULT_PIP_FIND_LINKS}") + + # Ccache stat/config dumps when ccache is available. + host_ccache_dir: Optional[Path] = None + if _which("ccache"): + _run(["ccache", "--show-stats"]) + _run(["ccache", "--print-stats"]) + _run(["ccache", "--show-config"]) + + proc = subprocess.run( + ["ccache", "--get-config", "cache_dir"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + cache_dir_str = proc.stdout.strip() + if cache_dir_str: + host_ccache_dir = Path(cache_dir_str) + os.environ["HOST_CCACHE_DIR"] = cache_dir_str + + # cibuildwheel "$HERE"/.. --platform "$machine" + _run( + _cibuildwheel_cmd() + + [ + str(project_root), + "--platform", + machine, + "--output-dir", + str(wheelhouse), + ], + cwd=project_root, + ) + + _rename_main_wheel_python_tag(wheelhouse) + + # if [ -d wheelhouse/.ccache ], copy back to HOST_CCACHE_DIR + if host_ccache_dir is not None: + _copy_ccache_from_wheelhouse(wheelhouse, host_ccache_dir) + + # Stage inputs into python_bindings + shutil.copy2(project_root / "requirements.txt", python_bindings / "requirements.txt") + + # Prefer a wheel-local requirements_dev.txt if present, fallback to repo_root/python/. + dev_req_dst = python_bindings / "requirements_dev.txt" + for src in [project_root / "requirements_dev.txt", repo_root / "python" / "requirements_dev.txt"]: + if src.is_file(): + shutil.copy2(src, dev_req_dst) + break + + _copy_tree_merge(project_root / "scripts", python_bindings / "scripts") + + for whl in wheelhouse.glob("mlir_aie*.whl"): + shutil.copy2(whl, python_bindings / whl.name) + + # unzip -o -q mlir_aie\*.whl ; rm -rf mlir_aie*.whl + for whl in python_bindings.glob("mlir_aie*.whl"): + _unzip_overwrite(whl, python_bindings) + + for whl in python_bindings.glob("mlir_aie*.whl"): + whl.unlink() + + # cibuildwheel --platform "$machine" --output-dir ../wheelhouse + _run(_cibuildwheel_cmd() + ["--platform", machine, "--output-dir", "../wheelhouse"], cwd=python_bindings) + + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) From 27bad7c756a813d46ae19515f6da02b4a894d377 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Mon, 9 Mar 2026 14:33:06 -0400 Subject: [PATCH 15/21] resolve conflicts Signed-off-by: thomthehound --- python/CMakeLists.txt | 9 +-------- python/requirements.txt | 4 ++-- tools/CMakeLists.txt | 5 ++--- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 4b0f08b9fe9..9e6113e9647 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -228,9 +228,6 @@ if (AIE_ENABLE_PYTHON_PASSES) PRIVATE_LINK_LIBS ${_py_libs} - - PYTHON_BINDINGS_LIBRARY - nanobind ) target_include_directories( AIEPythonExtensions.MLIR @@ -325,8 +322,6 @@ else () AIECAPI PRIVATE_LINK_LIBS LLVMSupport - PYTHON_BINDINGS_LIBRARY - nanobind ) if(AIE_ENABLE_XRT_PYTHON_BINDINGS) @@ -342,8 +337,6 @@ else () PRIVATE_LINK_LIBS LLVMSupport xrt_coreutil - PYTHON_BINDINGS_LIBRARY - nanobind ) if(NOT WIN32) target_link_libraries(AIEPythonExtensions.XRT INTERFACE uuid) @@ -551,4 +544,4 @@ add_custom_command( ${CMAKE_BINARY_DIR}/bin ) # during install -install(PROGRAMS compiler/aiecc.py compiler/txn2mlir.py DESTINATION bin) +install(PROGRAMS compiler/aiecc.py compiler/txn2mlir.py DESTINATION bin) \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt index b44d2a8aeb0..516c2529c01 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -9,5 +9,5 @@ rich ml_dtypes cloudpickle # required by eudsl when it is vendored instead of installed -f https://llvm.github.io/eudsl -eudsl-python-extras==0.1.0.20251215.1715+3c7ac1b \ - --config-settings=EUDSL_PYTHON_EXTRAS_HOST_PACKAGE_PREFIX=aie +eudsl-python-extras==0.1.0.20260308.1929+09d24cd \ + --config-settings=EUDSL_PYTHON_EXTRAS_HOST_PACKAGE_PREFIX=aie \ No newline at end of file diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 446016b41c9..f1aa3502c5c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -13,9 +13,8 @@ endif() add_subdirectory(aie-lsp-server) add_subdirectory(aie-translate) add_subdirectory(aie-visualize) +add_subdirectory(aiecc) add_subdirectory(bootgen) if(AIE_BUILD_CHESS_CLANG) add_subdirectory(chess-clang) -endif() -add_subdirectory(aiecc) - +endif() \ No newline at end of file From ed6168e5d2bce9daa3110a8ea41e74f2eaf31422 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Mon, 9 Mar 2026 16:59:38 -0400 Subject: [PATCH 16/21] windows local build enhancements Signed-off-by: thomthehound --- .../mlir_aie_wheels/python_bindings/setup.py | 124 ++++++++++++---- utils/mlir_aie_wheels/requirements.txt | 3 +- utils/mlir_aie_wheels/scripts/build_local.py | 30 +++- utils/mlir_aie_wheels/scripts/build_local.sh | 25 +++- .../mlir_aie_wheels/scripts/download_mlir.py | 87 ++++++------ utils/mlir_aie_wheels/setup.py | 132 +++++++++++++----- 6 files changed, 283 insertions(+), 118 deletions(-) diff --git a/utils/mlir_aie_wheels/python_bindings/setup.py b/utils/mlir_aie_wheels/python_bindings/setup.py index f79d878c170..9ff80ee00b5 100644 --- a/utils/mlir_aie_wheels/python_bindings/setup.py +++ b/utils/mlir_aie_wheels/python_bindings/setup.py @@ -21,6 +21,57 @@ def _cmake_path(p: object) -> str: return os.fspath(p).replace("\\", "/") +def _windows_short_root() -> Path: + return Path(os.getenv("AIE_WHEEL_BUILD_ROOT", "C:/tmp/aiewhls")).absolute() + + +def _windows_tree_alias_name(tree_name: str) -> str: + aliases = { + "mlir": "m", + "mlir_no_rtti": "mnr", + "mlir_aie": "a", + "mlir_aie_no_rtti": "anr", + } + return aliases.get(tree_name, tree_name) + + +def _remove_tree(path: Path) -> None: + shutil.rmtree(path, ignore_errors=True) + + +def _windows_short_dir(name: str, *, clean: bool = False) -> Path: + path = _windows_short_root() / name + if clean and path.exists(): + _remove_tree(path) + path.mkdir(parents=True, exist_ok=True) + return path + + +def _windows_short_alias(name: str, target: Path) -> Path: + link = _windows_short_root() / name + target = target.absolute() + if link.exists(): + try: + if link.resolve() == target.resolve(): + return link + except OSError: + pass + subprocess.run( + ["cmd", "/c", "rmdir", str(link)], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + link.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["cmd", "/c", "mklink", "/J", str(link), str(target)], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return link + + class CMakeExtension(Extension): def __init__(self, name: str, sourcedir: Union[str, Path] = "") -> None: super().__init__(name, sources=[]) @@ -82,15 +133,17 @@ def build_extension(self, ext: CMakeExtension) -> None: / ("mlir" if check_env("ENABLE_RTTI", 1) else "mlir_no_rtti"), ) ).absolute() + cmake_src = Path(ext.sourcedir) if platform.system() == "Windows": - # fatal error LNK1170: line in command file contains 131071 or more characters - if not Path("/tmp/a").exists(): - shutil.move(MLIR_AIE_INSTALL_ABS_PATH, "/tmp/a") - MLIR_AIE_INSTALL_ABS_PATH = Path("/tmp/a").absolute() - if not Path("/tmp/m").exists(): - shutil.move(MLIR_INSTALL_ABS_PATH, "/tmp/m") - MLIR_INSTALL_ABS_PATH = Path("/tmp/m").absolute() + # Keep source and dependency paths short without moving the installed trees. + cmake_src = _windows_short_alias("pb", cmake_src).absolute() + MLIR_AIE_INSTALL_ABS_PATH = _windows_short_alias( + _windows_tree_alias_name(MLIR_AIE_INSTALL_ABS_PATH.name), MLIR_AIE_INSTALL_ABS_PATH + ).absolute() + MLIR_INSTALL_ABS_PATH = _windows_short_alias( + _windows_tree_alias_name(MLIR_INSTALL_ABS_PATH.name), MLIR_INSTALL_ABS_PATH + ).absolute() cmake_args = [ "-G", @@ -134,19 +187,18 @@ def build_extension(self, ext: CMakeExtension) -> None: cmake_args += [item for item in os.getenv("CMAKE_ARGS").split(" ") if item] build_args = [] - if self.compiler.compiler_type != "msvc": - if not cmake_generator or cmake_generator == "Ninja": - try: - import ninja - - ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" - cmake_args += [ - f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}", - ] - except ImportError: - pass + if not cmake_generator or cmake_generator == "Ninja": + try: + import ninja - else: + ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" + cmake_args += [ + f"-DCMAKE_MAKE_PROGRAM:FILEPATH={_cmake_path(ninja_executable_path)}", + ] + except ImportError: + pass + + if self.compiler.compiler_type == "msvc": single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) if not single_config and not contains_arch: @@ -172,27 +224,37 @@ def build_extension(self, ext: CMakeExtension) -> None: cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] if "PARALLEL_LEVEL" not in os.environ: - build_args += [f"-j{str(2 * os.cpu_count())}"] + build_args += [f"-j{str(os.cpu_count() or 2)}"] else: build_args += [f"-j{os.getenv('PARALLEL_LEVEL')}"] + cleanup_build_temp = False build_temp = Path(self.build_temp) / ext.name - if not build_temp.exists(): + if platform.system() == "Windows": + build_temp = _windows_short_dir("bind", clean=True) + cleanup_build_temp = True + elif not build_temp.exists(): build_temp.mkdir(parents=True) print("ENV", pprint(os.environ), file=sys.stderr) print("cmake", " ".join(cmake_args), file=sys.stderr) - cmake_src = ext.sourcedir - if platform.system() == "Windows": - cmake_src = _cmake_path(cmake_src) - - subprocess.run(["cmake", cmake_src, *cmake_args], cwd=build_temp, check=True) - subprocess.run( - ["cmake", "--build", ".", "--target", "install", *build_args], - cwd=build_temp, - check=True, - ) + build_succeeded = False + try: + subprocess.run( + ["cmake", _cmake_path(cmake_src), *cmake_args], + cwd=build_temp, + check=True, + ) + subprocess.run( + ["cmake", "--build", ".", "--target", "install", *build_args], + cwd=build_temp, + check=True, + ) + build_succeeded = True + finally: + if cleanup_build_temp and build_succeeded: + _remove_tree(build_temp) commit_hash = os.environ.get("AIE_PROJECT_COMMIT", "deadbeef") diff --git a/utils/mlir_aie_wheels/requirements.txt b/utils/mlir_aie_wheels/requirements.txt index 487a72814dc..e6df7765f16 100644 --- a/utils/mlir_aie_wheels/requirements.txt +++ b/utils/mlir_aie_wheels/requirements.txt @@ -1,7 +1,8 @@ cmake>=3.30 dataclasses importlib-metadata -ninja +ninja<1.13; platform_system=="Windows" +ninja; platform_system!="Windows" numpy pip pybind11[global]>=2.10.4 diff --git a/utils/mlir_aie_wheels/scripts/build_local.py b/utils/mlir_aie_wheels/scripts/build_local.py index 7c4b89cb072..ca73a40f3e3 100644 --- a/utils/mlir_aie_wheels/scripts/build_local.py +++ b/utils/mlir_aie_wheels/scripts/build_local.py @@ -20,6 +20,7 @@ from __future__ import annotations +import argparse import os import platform import re @@ -33,6 +34,29 @@ DEFAULT_PIP_FIND_LINKS = "https://github.com/Xilinx/mlir-aie/releases/expanded_assets/mlir-distro" +# -------------------------------------------------------------------------------------- +# Command line +# -------------------------------------------------------------------------------------- + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + python_group = parser.add_mutually_exclusive_group() + python_group.add_argument("--cp312", action="store_true", help="Build only cp312 wheels.") + python_group.add_argument("--cp313", action="store_true", help="Build only cp313 wheels.") + python_group.add_argument("--cp314", action="store_true", help="Build only cp314 wheels.") + return parser.parse_args(argv) + + +def _apply_requested_python(args: argparse.Namespace) -> None: + if args.cp312: + os.environ["CIBW_BUILD"] = "cp312-*" + elif args.cp313: + os.environ["CIBW_BUILD"] = "cp313-*" + elif args.cp314: + os.environ["CIBW_BUILD"] = "cp314-*" + + # -------------------------------------------------------------------------------------- # Helpers # -------------------------------------------------------------------------------------- @@ -99,6 +123,8 @@ def _apply_env_for_machine(machine: str) -> None: os.environ.setdefault("MATRIX_OS", "windows-2022") os.environ.setdefault("CIBW_ARCHS", "AMD64") os.environ.setdefault("ARCH", "AMD64") + os.environ.setdefault("PARALLEL_LEVEL", "15") + os.environ.setdefault("AIE_WHEEL_BUILD_ROOT", "C:/tmp/aiewhls") # -------------------------------------------------------------------------------------- @@ -138,7 +164,8 @@ def _copy_ccache_from_wheelhouse(wheelhouse: Path, host_ccache_dir: Path) -> Non # -------------------------------------------------------------------------------------- -def main() -> int: +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv or sys.argv[1:]) here = Path(__file__).resolve().parent project_root = (here / "..").resolve() # utils/mlir_aie_wheels wheelhouse = project_root / "wheelhouse" @@ -161,6 +188,7 @@ def main() -> int: machine = _detect_machine() print(machine) + _apply_requested_python(args) _apply_env_for_machine(machine) # For local builds, default to the official mlir-distro unless the user overrides. diff --git a/utils/mlir_aie_wheels/scripts/build_local.sh b/utils/mlir_aie_wheels/scripts/build_local.sh index df1c41d808c..c6850179668 100755 --- a/utils/mlir_aie_wheels/scripts/build_local.sh +++ b/utils/mlir_aie_wheels/scripts/build_local.sh @@ -13,6 +13,17 @@ case "${unameOut}" in esac echo "${machine}" +case "${1:-}" in + "") ;; + --cp312) export CIBW_BUILD=cp312-* ;; + --cp313) export CIBW_BUILD=cp313-* ;; + --cp314) export CIBW_BUILD=cp314-* ;; + *) + echo "usage: $0 [--cp312|--cp313|--cp314]" >&2 + exit 1 + ;; +esac + # rsync -avpP --exclude .git --exclude cmake-build-debug --exclude cmake-build-release ../../llvm/* llvm-project/ export APPLY_PATCHES=true @@ -20,20 +31,22 @@ export APPLY_PATCHES=true if [ "$machine" == "linux" ]; then export MATRIX_OS=ubuntu-20.04 export CIBW_ARCHS=x86_64 - export CIBW_BUILD=cp311-manylinux_x86_64 + export CIBW_BUILD=${CIBW_BUILD:-cp311-manylinux_x86_64} export ARCH=x86_64 export PARALLEL_LEVEL=15 elif [ "$machine" == "macos" ]; then export MATRIX_OS=macos-11 export CIBW_ARCHS=arm64 - export CIBW_BUILD=cp311-macosx_arm64 + export CIBW_BUILD=${CIBW_BUILD:-cp311-macosx_arm64} export ARCH=arm64 export PARALLEL_LEVEL=32 else - export MATRIX_OS=windows-2019 + export MATRIX_OS=windows-2022 export CIBW_ARCHS=AMD64 - export CIBW_BUILD=cp311-win_amd64 + export CIBW_BUILD=${CIBW_BUILD:-cp311-win_amd64} export ARCH=AMD64 + export PARALLEL_LEVEL=${PARALLEL_LEVEL:-15} + export AIE_WHEEL_BUILD_ROOT=${AIE_WHEEL_BUILD_ROOT:-C:/tmp/aiewhls} fi ccache --show-stats @@ -43,7 +56,7 @@ ccache --show-config export HOST_CCACHE_DIR="$(ccache --get-config cache_dir)" cibuildwheel "$HERE"/.. --platform "$machine" -rename 's/cp311-cp311/py3-none/' "$HERE/../wheelhouse/"mlir*whl +rename 's/cp[0-9]+-cp[0-9]+/py3-none/' "$HERE/../wheelhouse/"mlir*whl if [ -d "$HERE/../wheelhouse/.ccache" ]; then cp -R "$HERE/../wheelhouse/.ccache/"* "$HOST_CCACHE_DIR/" @@ -55,7 +68,7 @@ cp -R "$HERE/../scripts" "$HERE/../python_bindings" cp -R "$HERE/../wheelhouse/"mlir_aie*.whl "$HERE/../python_bindings" pushd "$HERE/../python_bindings" -# escape to prevent 'Filename not matched' when both the py3-none whl and the cp311 wheel +# escape to prevent 'Filename not matched' when both the py3-none whl and a cpXYZ wheel exist unzip -o -q mlir_aie\*.whl rm -rf mlir_aie*.whl diff --git a/utils/mlir_aie_wheels/scripts/download_mlir.py b/utils/mlir_aie_wheels/scripts/download_mlir.py index e40cfe862aa..989d1a8e30a 100644 --- a/utils/mlir_aie_wheels/scripts/download_mlir.py +++ b/utils/mlir_aie_wheels/scripts/download_mlir.py @@ -16,7 +16,6 @@ # Optional: # MLIR_AIE_WHEEL_VERSION: override wheel version (bypass clone-llvm.sh parsing) # MLIR_AIE_SOURCE_DIR: explicit repo root (for clone-llvm.sh lookup) -# VSWHERE_EXE: explicit vswhere path (for diaguids.lib fixup) ##===------------------------------------------------------------------------------===## import os @@ -102,31 +101,17 @@ def _wheel_version(): # -------------------------------------------------------------------------------------- -def _vswhere_path(): - explicit = (os.environ.get("VSWHERE_EXE") or "").strip() - if explicit and Path(explicit).is_file(): - return Path(explicit) +def _find_vs_install_dir(): + vsinstalldir = (os.environ.get("VSINSTALLDIR") or "").strip() + if vsinstalldir: + return Path(vsinstalldir) pf86 = os.environ.get("ProgramFiles(x86)") if not pf86: return None - p = Path(pf86) / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" - return p if p.is_file() else None - - -def _detect_diaguids_lib(): - if os.name != "nt": - return None - - vsinstalldir = (os.environ.get("VSINSTALLDIR") or "").strip() - if vsinstalldir: - p = Path(vsinstalldir) / "DIA SDK" / "lib" / "amd64" / "diaguids.lib" - if p.is_file(): - return p - - vswhere = _vswhere_path() - if vswhere is None: + vswhere = Path(pf86) / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" + if not vswhere.is_file(): return None try: @@ -147,15 +132,25 @@ def _detect_diaguids_lib(): except Exception: return None - if not install: + return Path(install) if install else None + + +def _find_diaguids_lib(): + if os.name != "nt": + return None + + install_dir = _find_vs_install_dir() + if install_dir is None: return None - p = Path(install) / "DIA SDK" / "lib" / "amd64" / "diaguids.lib" - return p if p.is_file() else None + diaguids = install_dir / "DIA SDK" / "lib" / "amd64" / "diaguids.lib" + return diaguids if diaguids.is_file() else None def _fixup_llvm_diaguids(mlir_prefix): - # The MLIR wheel's LLVM CMake exports can contain an absolute diaguids.lib path. + # The MLIR wheel's LLVM CMake exports can contain an absolute diaguids.lib path + # from the machine that built the wheel. Rewrite it to the current machine's + # DIA SDK path. if os.name != "nt": return @@ -164,38 +159,38 @@ def _fixup_llvm_diaguids(mlir_prefix): return cmake_files = [llvm_dir / "LLVMExports.cmake", llvm_dir / "LLVMTargets.cmake"] - cmake_files = [p for p in cmake_files if p.is_file()] + cmake_files = [cmake_file for cmake_file in cmake_files if cmake_file.is_file()] if not cmake_files: return - repl_path = _detect_diaguids_lib() - repl = repl_path.as_posix() if repl_path else "diaguids.lib" - pat = re.compile(r"([A-Za-z]:[\\/][^\n\"']*diaguids\.lib)", flags=re.IGNORECASE) - patched = 0 + file_data = {} + found_reference = False for cmake_file in cmake_files: data = cmake_file.read_text(encoding="utf-8", errors="ignore") + file_data[cmake_file] = data + if pat.search(data): + found_reference = True + + if not found_reference: + return + + diaguids = _find_diaguids_lib() + if diaguids is None: + raise FileNotFoundError("Could not locate diaguids.lib for LLVM CMake export fixup") + + replacement = diaguids.as_posix() + patched = 0 - def _repl(m): - nonlocal patched - old = m.group(1) - try: - if Path(old).exists(): - return old - except Exception: - pass - patched += 1 - return repl - - new_data = pat.sub(_repl, data) - if new_data != data: + for cmake_file, data in file_data.items(): + new_data, count = pat.subn(replacement, data) + if count: cmake_file.write_text(new_data, encoding="utf-8") + patched += count if patched: - print( - f"[fixup] Patched diaguids.lib path -> {repl_path if repl_path else 'diaguids.lib'}" - ) + print(f"[fixup] Patched diaguids.lib path -> {replacement}") def main(): diff --git a/utils/mlir_aie_wheels/setup.py b/utils/mlir_aie_wheels/setup.py index 41d0a165337..08f6b04f6ab 100644 --- a/utils/mlir_aie_wheels/setup.py +++ b/utils/mlir_aie_wheels/setup.py @@ -29,6 +29,57 @@ def _cmake_path(p: object) -> str: return os.fspath(p).replace("\\", "/") +def _windows_short_root() -> Path: + return Path(os.getenv("AIE_WHEEL_BUILD_ROOT", "C:/tmp/aiewhls")).absolute() + + +def _windows_tree_alias_name(tree_name: str) -> str: + aliases = { + "mlir": "m", + "mlir_no_rtti": "mnr", + "mlir_aie": "a", + "mlir_aie_no_rtti": "anr", + } + return aliases.get(tree_name, tree_name) + + +def _remove_tree(path: Path) -> None: + shutil.rmtree(path, ignore_errors=True) + + +def _windows_short_dir(name: str, *, clean: bool = False) -> Path: + path = _windows_short_root() / name + if clean and path.exists(): + _remove_tree(path) + path.mkdir(parents=True, exist_ok=True) + return path + + +def _windows_short_alias(name: str, target: Path) -> Path: + link = _windows_short_root() / name + target = target.absolute() + if link.exists(): + try: + if link.resolve() == target.resolve(): + return link + except OSError: + pass + subprocess.run( + ["cmd", "/c", "rmdir", str(link)], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + link.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["cmd", "/c", "mklink", "/J", str(link), str(target)], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return link + + class CMakeExtension(Extension): def __init__(self, name: str, sourcedir: Union[str, Path] = "") -> None: super().__init__(name, sources=[]) @@ -124,17 +175,21 @@ def build_extension(self, ext: CMakeExtension) -> None: / ("mlir" if check_env("ENABLE_RTTI", 1) else "mlir_no_rtti"), ) ).absolute() + cmake_source_dir = Path(ext.sourcedir) + cmake_module_root = MLIR_AIE_SOURCE_DIR if platform.system() == "Windows": - # fatal error LNK1170: line in command file contains 131071 or more characters - if not Path("/tmp/m").exists(): - shutil.move(MLIR_INSTALL_ABS_PATH, "/tmp/m") - MLIR_INSTALL_ABS_PATH = Path("/tmp/m").absolute() + # Keep source and dependency paths short without moving the installed trees. + cmake_source_dir = _windows_short_alias("src", cmake_source_dir).absolute() + cmake_module_root = cmake_source_dir + MLIR_INSTALL_ABS_PATH = _windows_short_alias( + _windows_tree_alias_name(MLIR_INSTALL_ABS_PATH.name), MLIR_INSTALL_ABS_PATH + ).absolute() cmake_args = [ "-G", cmake_generator, - f"-DCMAKE_MODULE_PATH={_cmake_path(MLIR_AIE_SOURCE_DIR / 'cmake' / 'modulesXilinx')}", + f"-DCMAKE_MODULE_PATH={_cmake_path(cmake_module_root / 'cmake' / 'modulesXilinx')}", f"-DCMAKE_PREFIX_PATH={_cmake_path(MLIR_INSTALL_ABS_PATH)}", f"-DCMAKE_INSTALL_PREFIX={_cmake_path(install_dir)}", f"-DPython3_EXECUTABLE={_cmake_path(sys.executable)}", @@ -186,19 +241,18 @@ def build_extension(self, ext: CMakeExtension) -> None: cmake_args += [item for item in os.getenv("CMAKE_ARGS").split(" ") if item] build_args = [] - if self.compiler.compiler_type != "msvc": - if not cmake_generator or cmake_generator == "Ninja": - try: - import ninja - - ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" - cmake_args += [ - f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}", - ] - except ImportError: - pass - - else: + if not cmake_generator or cmake_generator == "Ninja": + try: + import ninja + + ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" + cmake_args += [ + f"-DCMAKE_MAKE_PROGRAM:FILEPATH={_cmake_path(ninja_executable_path)}", + ] + except ImportError: + pass + + if self.compiler.compiler_type == "msvc": single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) if not single_config and not contains_arch: @@ -223,28 +277,40 @@ def build_extension(self, ext: CMakeExtension) -> None: if archs: cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] - build_args += [f"-j{os.getenv('PARALLEL_LEVEL', 2 * os.cpu_count())}"] + build_args += [f"-j{os.getenv('PARALLEL_LEVEL', os.cpu_count() or 2)}"] + cleanup_build_temp = False build_temp = Path(self.build_temp) / ext.name - if not build_temp.exists(): + if platform.system() == "Windows": + build_temp = _windows_short_dir("main", clean=True) + cleanup_build_temp = True + elif not build_temp.exists(): build_temp.mkdir(parents=True) print("ENV", pprint(os.environ), file=sys.stderr) print("cmake", " ".join(cmake_args), file=sys.stderr) - subprocess.run( - ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True - ) - subprocess.run( - ["cmake", "--build", ".", "--target", "install", *build_args], - cwd=build_temp, - check=True, - ) + build_succeeded = False + try: + subprocess.run( + ["cmake", _cmake_path(cmake_source_dir), *cmake_args], + cwd=build_temp, + check=True, + ) + subprocess.run( + ["cmake", "--build", ".", "--target", "install", *build_args], + cwd=build_temp, + check=True, + ) - # Vendor eudsl-python-extras - # Install eudsl to install_dir/python so it merges with mlir-aie's package structure (aie/extras). - target_dir = Path(install_dir) / "python" - req_file = Path(MLIR_AIE_SOURCE_DIR) / "python" / "requirements.txt" - install_eudsl(req_file, target_dir) + # Vendor eudsl-python-extras + # Install eudsl to install_dir/python so it merges with mlir-aie's package structure (aie/extras). + target_dir = Path(install_dir) / "python" + req_file = Path(MLIR_AIE_SOURCE_DIR) / "python" / "requirements.txt" + install_eudsl(req_file, target_dir) + build_succeeded = True + finally: + if cleanup_build_temp and build_succeeded: + _remove_tree(build_temp) class DevelopWithPth(develop): From 4c95c6e50edc018991bdf6e7d5cfbc79e5bbb57d Mon Sep 17 00:00:00 2001 From: thomthehound Date: Mon, 9 Mar 2026 20:10:17 -0400 Subject: [PATCH 17/21] fix Peano-side legacy LLVM IR drift Signed-off-by: thomthehound --- tools/aiecc/aiecc.cpp | 65 +++++++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/tools/aiecc/aiecc.cpp b/tools/aiecc/aiecc.cpp index 7a61ef2a48e..a983b17bf0b 100644 --- a/tools/aiecc/aiecc.cpp +++ b/tools/aiecc/aiecc.cpp @@ -824,11 +824,11 @@ static StringRef getPeanoInstallDir() { return *cachedPeanoDir; } -// Downgrade LLVM IR for compatibility with Chess toolchain's older LLVM. -// The Chess LLVM is based on an older version that doesn't support modern -// memory/capture attributes. This function performs string replacements +// Downgrade LLVM IR for compatibility with legacy LLVM-based toolchains. +// Older Chess and Peano toolchains do not support some newer memory/capture +// attributes and GEP spellings. This function performs string replacements // matching the Python downgrade_ir_for_chess() function. -static std::string downgradeIRForChess(StringRef llvmIR) { +static std::string downgradeIRForLegacyLLVM(StringRef llvmIR) { std::string result = llvmIR.str(); // Replace memory attributes @@ -1921,14 +1921,13 @@ static LogicalResult compileCore(MLIRContext &context, ModuleOp moduleOp, << bufOrErr.getError().message() << "\n"; return failure(); } - std::string downgradedIR = downgradeIRForChess((*bufOrErr)->getBuffer()); + std::string downgradedIR = downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); // Write downgraded IR to .chesshack.ll SmallString<128> chessHackPath(tmpDirName); sys::path::append(chessHackPath, deviceName.str() + "_core_" + std::to_string(core.col) + - "_" + std::to_string(core.col) + "_" + - std::to_string(core.row) + ".chesshack.ll"); + "_" + std::to_string(core.row) + ".chesshack.ll"); { std::error_code ec; raw_fd_ostream chessHackFile(chessHackPath, ec); @@ -2014,6 +2013,38 @@ static LogicalResult compileCore(MLIRContext &context, ModuleOp moduleOp, return failure(); } + auto bufOrErr = MemoryBuffer::getFile(llvmIRPath); + if (!bufOrErr) { + std::lock_guard lock(outputMutex); + llvm::errs() << "Error reading LLVM IR file: " + << bufOrErr.getError().message() << "\n"; + return failure(); + } + std::string downgradedIR = + downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); + + SmallString<128> peanoHackPath(tmpDirName); + sys::path::append(peanoHackPath, + deviceName.str() + "_core_" + std::to_string(core.col) + + "_" + std::to_string(core.row) + ".peanohack.ll"); + { + std::error_code ec; + raw_fd_ostream peanoHackFile(peanoHackPath, ec); + if (ec) { + std::lock_guard lock(outputMutex); + llvm::errs() << "Error writing peanohack file: " << ec.message() + << "\n"; + return failure(); + } + peanoHackFile << downgradedIR; + } + + if (verbose) { + std::lock_guard lock(outputMutex); + llvm::outs() << "Applied IR downgrade for Peano: " << peanoHackPath + << "\n"; + } + // Run opt SmallString<128> optPath(tmpDirName); sys::path::append(optPath, deviceName.str() + "_core_" + @@ -2026,7 +2057,7 @@ static LogicalResult compileCore(MLIRContext &context, ModuleOp moduleOp, ">,strip", "-inline-threshold=10", "-S", - std::string(llvmIRPath), + std::string(peanoHackPath), "-o", std::string(optPath)}; @@ -2668,7 +2699,7 @@ compileCoresUnified(MLIRContext &context, ModuleOp moduleOp, << bufOrErr.getError().message() << "\n"; return failure(); } - std::string downgradedIR = downgradeIRForChess((*bufOrErr)->getBuffer()); + std::string downgradedIR = downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); SmallString<128> chessHackPath(tmpDirName); sys::path::append(chessHackPath, deviceName.str() + "_input.chesshack.ll"); @@ -2735,7 +2766,7 @@ compileCoresUnified(MLIRContext &context, ModuleOp moduleOp, return failure(); } - // Apply peanohack (strip debug info for compatibility) + // Apply legacy LLVM IR downgrade for Peano compatibility. auto bufOrErr = MemoryBuffer::getFile(llvmIRPath); if (!bufOrErr) { llvm::errs() << "Error reading unified LLVM IR: " @@ -2743,10 +2774,12 @@ compileCoresUnified(MLIRContext &context, ModuleOp moduleOp, return failure(); } - // Write peanohacked version + std::string downgradedIR = + downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); + SmallString<128> peanohackPath(tmpDirName); sys::path::append(peanohackPath, - deviceName.str() + "_input.llpeanohack.ll"); + deviceName.str() + "_input.peanohack.ll"); { std::error_code ec; raw_fd_ostream peanohackFile(peanohackPath, ec); @@ -2755,8 +2788,12 @@ compileCoresUnified(MLIRContext &context, ModuleOp moduleOp, << "\n"; return failure(); } - // Simple peanohack: just copy for now (could add attribute stripping) - peanohackFile << (*bufOrErr)->getBuffer(); + peanohackFile << downgradedIR; + } + + if (verbose) { + llvm::outs() << "Applied IR downgrade for unified Peano input: " + << peanohackPath << "\n"; } // Run opt From 83de3e12a027f3c82cfa548e916e867e469be0a6 Mon Sep 17 00:00:00 2001 From: thomthehound Date: Tue, 10 Mar 2026 14:32:36 -0400 Subject: [PATCH 18/21] Ninja 1.13.0 bug workaround Signed-off-by: thomthehound --- tools/aiecc/aiecc.cpp | 9 +++++---- utils/mlir_aie_wheels/pyproject.toml | 2 +- utils/mlir_aie_wheels/python_bindings/pyproject.toml | 1 + utils/mlir_aie_wheels/python_bindings/setup.py | 6 ++++-- utils/mlir_aie_wheels/requirements.txt | 7 +++---- utils/mlir_aie_wheels/scripts/download_mlir.py | 4 +++- utils/mlir_aie_wheels/setup.py | 3 ++- 7 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tools/aiecc/aiecc.cpp b/tools/aiecc/aiecc.cpp index a983b17bf0b..0e761ca74da 100644 --- a/tools/aiecc/aiecc.cpp +++ b/tools/aiecc/aiecc.cpp @@ -1921,7 +1921,8 @@ static LogicalResult compileCore(MLIRContext &context, ModuleOp moduleOp, << bufOrErr.getError().message() << "\n"; return failure(); } - std::string downgradedIR = downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); + std::string downgradedIR = + downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); // Write downgraded IR to .chesshack.ll SmallString<128> chessHackPath(tmpDirName); @@ -2699,7 +2700,8 @@ compileCoresUnified(MLIRContext &context, ModuleOp moduleOp, << bufOrErr.getError().message() << "\n"; return failure(); } - std::string downgradedIR = downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); + std::string downgradedIR = + downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); SmallString<128> chessHackPath(tmpDirName); sys::path::append(chessHackPath, deviceName.str() + "_input.chesshack.ll"); @@ -2778,8 +2780,7 @@ compileCoresUnified(MLIRContext &context, ModuleOp moduleOp, downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); SmallString<128> peanohackPath(tmpDirName); - sys::path::append(peanohackPath, - deviceName.str() + "_input.peanohack.ll"); + sys::path::append(peanohackPath, deviceName.str() + "_input.peanohack.ll"); { std::error_code ec; raw_fd_ostream peanohackFile(peanohackPath, ec); diff --git a/utils/mlir_aie_wheels/pyproject.toml b/utils/mlir_aie_wheels/pyproject.toml index 8249b5acd13..0f4727c938d 100644 --- a/utils/mlir_aie_wheels/pyproject.toml +++ b/utils/mlir_aie_wheels/pyproject.toml @@ -68,7 +68,7 @@ before-build = [ ] [build-system] -requires = ["setuptools>=61.0", "nanobind", "pip>=22.1"] +requires = ["setuptools>=61.0", "nanobind", "pip>=22.1", 'ninja!=1.13.0; platform_system=="Windows"'] build-backend = "setuptools.build_meta" [project.scripts] diff --git a/utils/mlir_aie_wheels/python_bindings/pyproject.toml b/utils/mlir_aie_wheels/python_bindings/pyproject.toml index f37d7f98c96..9590be4a8a9 100644 --- a/utils/mlir_aie_wheels/python_bindings/pyproject.toml +++ b/utils/mlir_aie_wheels/python_bindings/pyproject.toml @@ -3,6 +3,7 @@ requires = [ "setuptools>=40.8.0", "wheel", "nanobind>=2.9,<3", + 'ninja!=1.13.0; platform_system=="Windows"', ] build-backend = "setuptools.build_meta" diff --git a/utils/mlir_aie_wheels/python_bindings/setup.py b/utils/mlir_aie_wheels/python_bindings/setup.py index 9ff80ee00b5..3b19ddea4ef 100644 --- a/utils/mlir_aie_wheels/python_bindings/setup.py +++ b/utils/mlir_aie_wheels/python_bindings/setup.py @@ -139,10 +139,12 @@ def build_extension(self, ext: CMakeExtension) -> None: # Keep source and dependency paths short without moving the installed trees. cmake_src = _windows_short_alias("pb", cmake_src).absolute() MLIR_AIE_INSTALL_ABS_PATH = _windows_short_alias( - _windows_tree_alias_name(MLIR_AIE_INSTALL_ABS_PATH.name), MLIR_AIE_INSTALL_ABS_PATH + _windows_tree_alias_name(MLIR_AIE_INSTALL_ABS_PATH.name), + MLIR_AIE_INSTALL_ABS_PATH, ).absolute() MLIR_INSTALL_ABS_PATH = _windows_short_alias( - _windows_tree_alias_name(MLIR_INSTALL_ABS_PATH.name), MLIR_INSTALL_ABS_PATH + _windows_tree_alias_name(MLIR_INSTALL_ABS_PATH.name), + MLIR_INSTALL_ABS_PATH, ).absolute() cmake_args = [ diff --git a/utils/mlir_aie_wheels/requirements.txt b/utils/mlir_aie_wheels/requirements.txt index e6df7765f16..9e453c11670 100644 --- a/utils/mlir_aie_wheels/requirements.txt +++ b/utils/mlir_aie_wheels/requirements.txt @@ -1,12 +1,11 @@ -cmake>=3.30 +cmake>=3.31 dataclasses importlib-metadata -ninja<1.13; platform_system=="Windows" -ninja; platform_system!="Windows" +ninja!=1.13.0; platform_system=="Windows" numpy pip pybind11[global]>=2.10.4 rich setuptools>=61.0 wheel -nanobind>=2.5 +nanobind>=2.5 \ No newline at end of file diff --git a/utils/mlir_aie_wheels/scripts/download_mlir.py b/utils/mlir_aie_wheels/scripts/download_mlir.py index 989d1a8e30a..ec1f6138b7a 100644 --- a/utils/mlir_aie_wheels/scripts/download_mlir.py +++ b/utils/mlir_aie_wheels/scripts/download_mlir.py @@ -178,7 +178,9 @@ def _fixup_llvm_diaguids(mlir_prefix): diaguids = _find_diaguids_lib() if diaguids is None: - raise FileNotFoundError("Could not locate diaguids.lib for LLVM CMake export fixup") + raise FileNotFoundError( + "Could not locate diaguids.lib for LLVM CMake export fixup" + ) replacement = diaguids.as_posix() patched = 0 diff --git a/utils/mlir_aie_wheels/setup.py b/utils/mlir_aie_wheels/setup.py index 08f6b04f6ab..61d6bc699ea 100644 --- a/utils/mlir_aie_wheels/setup.py +++ b/utils/mlir_aie_wheels/setup.py @@ -183,7 +183,8 @@ def build_extension(self, ext: CMakeExtension) -> None: cmake_source_dir = _windows_short_alias("src", cmake_source_dir).absolute() cmake_module_root = cmake_source_dir MLIR_INSTALL_ABS_PATH = _windows_short_alias( - _windows_tree_alias_name(MLIR_INSTALL_ABS_PATH.name), MLIR_INSTALL_ABS_PATH + _windows_tree_alias_name(MLIR_INSTALL_ABS_PATH.name), + MLIR_INSTALL_ABS_PATH, ).absolute() cmake_args = [ From f13b0979b140e3d4ecccafdcd067c9c61fceb38d Mon Sep 17 00:00:00 2001 From: thomthehound Date: Wed, 11 Mar 2026 13:57:04 -0400 Subject: [PATCH 19/21] Ninja 1.13.0 bug workaround No2 Signed-off-by: thomthehound --- .github/workflows/buildRyzenWheels.yml | 2 +- python/requirements_dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/buildRyzenWheels.yml b/.github/workflows/buildRyzenWheels.yml index ab565b0b625..677166a5ee7 100644 --- a/.github/workflows/buildRyzenWheels.yml +++ b/.github/workflows/buildRyzenWheels.yml @@ -335,7 +335,7 @@ jobs: mkdir -p "${WHEELHOUSE_DIR}" pushd utils/mlir_aie_wheels - pip install wheel importlib_metadata ninja + pip install wheel importlib_metadata "ninja!=1.13.0" CIBW_ARCHS=AMD64 pip wheel . -v -w "${WHEELHOUSE_DIR}" --no-build-isolation popd diff --git a/python/requirements_dev.txt b/python/requirements_dev.txt index d94984fa5c7..670bc774e8f 100644 --- a/python/requirements_dev.txt +++ b/python/requirements_dev.txt @@ -2,7 +2,7 @@ cmake>=3.31, <4.0 pybind11>=2.13.0 setuptools>=61.0 wheel -ninja +ninja!=1.13.0 cibuildwheel pre-commit nanobind>=2.9 From a54355ffffa46d5bb343ccbf03ce41a795ffa6cc Mon Sep 17 00:00:00 2001 From: thomthehound Date: Thu, 12 Mar 2026 13:28:47 -0400 Subject: [PATCH 20/21] MLIR AIE Distro: only unpack the tested whl Signed-off-by: thomthehound --- .github/workflows/mlirAIEDistro.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mlirAIEDistro.yml b/.github/workflows/mlirAIEDistro.yml index abdb193a9e0..f6954429114 100644 --- a/.github/workflows/mlirAIEDistro.yml +++ b/.github/workflows/mlirAIEDistro.yml @@ -431,7 +431,7 @@ jobs: python -m pip install --upgrade pip pip install -r python/requirements.txt pip install -r python/requirements_dev.txt - unzip -o -q dist/mlir_aie\*.whl + unzip -o -q dist/mlir_aie*py3-none*.whl export PYTHONPATH=mlir_aie/python From 6dae5cff4071ba39a334d1b0d4421d23a59a854e Mon Sep 17 00:00:00 2001 From: thomthehound Date: Fri, 13 Mar 2026 20:38:18 -0400 Subject: [PATCH 21/21] roll back aiecc changes Signed-off-by: thomthehound --- tools/aiecc/aiecc.cpp | 134 +++++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/tools/aiecc/aiecc.cpp b/tools/aiecc/aiecc.cpp index 3ba1a8a21cf..4983a088bf3 100644 --- a/tools/aiecc/aiecc.cpp +++ b/tools/aiecc/aiecc.cpp @@ -828,11 +828,11 @@ static StringRef getPeanoInstallDir() { return *cachedPeanoDir; } -// Downgrade LLVM IR for compatibility with legacy LLVM-based toolchains. -// Older Chess and Peano toolchains do not support some newer memory/capture -// attributes and GEP spellings. This function performs string replacements +// Downgrade LLVM IR for compatibility with Chess toolchain's older LLVM. +// The Chess LLVM is based on an older version that doesn't support modern +// memory/capture attributes. This function performs string replacements // matching the Python downgrade_ir_for_chess() function. -static std::string downgradeIRForLegacyLLVM(StringRef llvmIR) { +static std::string downgradeIRForChess(StringRef llvmIR) { std::string result = llvmIR.str(); // Replace memory attributes @@ -1886,6 +1886,53 @@ struct CoreCompilationResult { bool success; }; +/// Downgrade LLVM IR for Peano compatibility. +/// Strips LLVM 23+ features that Peano 19's opt/llc can't parse: +/// - 'nuw' flag on getelementptr (LLVM 23 feature) +/// - 'nocreateundeforpoison' attribute (with any trailing whitespace) +static std::string downgradeIRForPeano(StringRef ir) { + std::string result = ir.str(); + // Strip 'nuw' from 'getelementptr inbounds nuw' -> 'getelementptr inbounds' + const std::string nuwFrom = "getelementptr inbounds nuw"; + const std::string nuwTo = "getelementptr inbounds"; + size_t pos = 0; + while ((pos = result.find(nuwFrom, pos)) != std::string::npos) { + result.erase(pos + nuwTo.size(), nuwFrom.size() - nuwTo.size()); + pos += nuwTo.size(); + } + // Strip 'nocreateundeforpoison' and any trailing whitespace + const std::string nocreate = "nocreateundeforpoison"; + pos = 0; + while ((pos = result.find(nocreate, pos)) != std::string::npos) { + size_t end = pos + nocreate.size(); + while (end < result.size() && (result[end] == ' ' || result[end] == '\t')) + ++end; + result.erase(pos, end - pos); + } + return result; +} + +/// Apply peanohack: read LLVM IR, downgrade for Peano, write to output path. +static LogicalResult applyPeanoHack(StringRef inputPath, StringRef outputPath) { + auto bufOrErr = llvm::MemoryBuffer::getFile(inputPath); + if (!bufOrErr) { + std::lock_guard lock(outputMutex); + llvm::errs() << "Error reading LLVM IR for peanohack: " + << bufOrErr.getError().message() << "\n"; + return failure(); + } + std::string hacked = downgradeIRForPeano((*bufOrErr)->getBuffer()); + std::error_code ec; + llvm::raw_fd_ostream out(outputPath, ec); + if (ec) { + std::lock_guard lock(outputMutex); + llvm::errs() << "Error writing peanohack file: " << ec.message() << "\n"; + return failure(); + } + out << hacked; + return success(); +} + static LogicalResult compileCore(MLIRContext &context, ModuleOp moduleOp, StringRef deviceName, const CoreInfo &core, StringRef tmpDirName, StringRef aieTarget, @@ -2040,14 +2087,14 @@ static LogicalResult compileCore(MLIRContext &context, ModuleOp moduleOp, << bufOrErr.getError().message() << "\n"; return failure(); } - std::string downgradedIR = - downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); + std::string downgradedIR = downgradeIRForChess((*bufOrErr)->getBuffer()); // Write downgraded IR to .chesshack.ll SmallString<128> chessHackPath(tmpDirName); sys::path::append(chessHackPath, deviceName.str() + "_core_" + std::to_string(core.col) + - "_" + std::to_string(core.row) + ".chesshack.ll"); + "_" + std::to_string(core.col) + "_" + + std::to_string(core.row) + ".chesshack.ll"); { std::error_code ec; raw_fd_ostream chessHackFile(chessHackPath, ec); @@ -2133,39 +2180,15 @@ static LogicalResult compileCore(MLIRContext &context, ModuleOp moduleOp, return failure(); } - auto bufOrErr = MemoryBuffer::getFile(llvmIRPath); - if (!bufOrErr) { - std::lock_guard lock(outputMutex); - llvm::errs() << "Error reading LLVM IR file: " - << bufOrErr.getError().message() << "\n"; - return failure(); - } - std::string downgradedIR = - downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); - - SmallString<128> peanoHackPath(tmpDirName); - sys::path::append(peanoHackPath, + // Apply peanohack to downgrade IR for Peano compatibility + SmallString<128> peanohackPath(tmpDirName); + sys::path::append(peanohackPath, deviceName.str() + "_core_" + std::to_string(core.col) + "_" + std::to_string(core.row) + ".peanohack.ll"); - { - std::error_code ec; - raw_fd_ostream peanoHackFile(peanoHackPath, ec); - if (ec) { - std::lock_guard lock(outputMutex); - llvm::errs() << "Error writing peanohack file: " << ec.message() - << "\n"; - return failure(); - } - peanoHackFile << downgradedIR; - } - - if (verbose) { - std::lock_guard lock(outputMutex); - llvm::outs() << "Applied IR downgrade for Peano: " << peanoHackPath - << "\n"; - } + if (failed(applyPeanoHack(llvmIRPath, peanohackPath))) + return failure(); - // Run opt + // Run opt on peanohacked IR SmallString<128> optPath(tmpDirName); sys::path::append(optPath, deviceName.str() + "_core_" + std::to_string(core.col) + "_" + @@ -2177,7 +2200,7 @@ static LogicalResult compileCore(MLIRContext &context, ModuleOp moduleOp, ">,strip", "-inline-threshold=10", "-S", - std::string(peanoHackPath), + std::string(peanohackPath), "-o", std::string(optPath)}; @@ -2786,8 +2809,7 @@ compileCoresUnified(MLIRContext &context, ModuleOp moduleOp, << bufOrErr.getError().message() << "\n"; return failure(); } - std::string downgradedIR = - downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); + std::string downgradedIR = downgradeIRForChess((*bufOrErr)->getBuffer()); SmallString<128> chessHackPath(tmpDirName); sys::path::append(chessHackPath, deviceName.str() + "_input.chesshack.ll"); @@ -2854,34 +2876,12 @@ compileCoresUnified(MLIRContext &context, ModuleOp moduleOp, return failure(); } - // Apply legacy LLVM IR downgrade for Peano compatibility. - auto bufOrErr = MemoryBuffer::getFile(llvmIRPath); - if (!bufOrErr) { - llvm::errs() << "Error reading unified LLVM IR: " - << bufOrErr.getError().message() << "\n"; - return failure(); - } - - std::string downgradedIR = - downgradeIRForLegacyLLVM((*bufOrErr)->getBuffer()); - + // Apply peanohack to downgrade IR for Peano compatibility SmallString<128> peanohackPath(tmpDirName); - sys::path::append(peanohackPath, deviceName.str() + "_input.peanohack.ll"); - { - std::error_code ec; - raw_fd_ostream peanohackFile(peanohackPath, ec); - if (ec) { - llvm::errs() << "Error writing peanohack file: " << ec.message() - << "\n"; - return failure(); - } - peanohackFile << downgradedIR; - } - - if (verbose) { - llvm::outs() << "Applied IR downgrade for unified Peano input: " - << peanohackPath << "\n"; - } + sys::path::append(peanohackPath, + deviceName.str() + "_input.llpeanohack.ll"); + if (failed(applyPeanoHack(llvmIRPath, peanohackPath))) + return failure(); // Run opt SmallString<128> optPath(tmpDirName);