diff --git a/.gitignore b/.gitignore index 6cae97504c..040b9e09cd 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,5 @@ jj-workflow.mdc third_party baml_repl_history.txt + +result diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 5acd2c6c96..775e761c45 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -188,6 +188,9 @@ authors = ["Boundary "] description = "BAML Toolchain" license-file = "LICENSE" +[workspace.metadata.crane] +name = "baml-workspace" + [workspace.metadata.workspaces] allow_branch = "canary" diff --git a/engine/language_client_typescript/build.rs b/engine/language_client_typescript/build.rs index 9fc2367889..569dfa1779 100644 --- a/engine/language_client_typescript/build.rs +++ b/engine/language_client_typescript/build.rs @@ -1,5 +1,51 @@ extern crate napi_build; +use std::{ + env, + path::{Path, PathBuf}, +}; + fn main() { + // These env vars are occasionally set on systems with particular + // requirements for napi inputs and outputs. Most builds can ignore + // this and will simply run the napi_build::setup() in the `false` + // branch. + if let (Ok(tmp_path), Ok(pkg_name), Ok(out_dir)) = ( + env::var("TYPE_DEF_TMP_PATH"), + env::var("CARGO_PKG_NAME"), + env::var("OUT_DIR"), + ) { + if Path::new(&tmp_path).parent().is_some() { + let bridge_dir = PathBuf::from(out_dir).join("napi-type-def"); + let _ = std::fs::create_dir_all(&bridge_dir); + let symlink_path = bridge_dir.join(pkg_name); + + // Remove any stale symlink/file from previous builds. + let _ = std::fs::remove_file(&symlink_path); + + #[cfg(unix)] + { + let _ = std::os::unix::fs::symlink(&tmp_path, &symlink_path); + } + + #[cfg(windows)] + { + let target = if Path::new(&tmp_path).is_dir() { + std::os::windows::fs::symlink_dir(&tmp_path, &symlink_path) + } else { + std::os::windows::fs::symlink_file(&tmp_path, &symlink_path) + }; + let _ = target; + } + + // Point napi-derive at the directory that now contains the symlink so it writes + // through to the legacy TYPE_DEF_TMP_PATH provided by the older CLI. + println!( + "cargo:rustc-env=NAPI_TYPE_DEF_TMP_FOLDER={}", + bridge_dir.display() + ); + } + } + napi_build::setup(); } diff --git a/flake.lock b/flake.lock index 8ce2fd7b5f..4c6e1f1a90 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,20 @@ { "nodes": { + "crane": { + "locked": { + "lastModified": 1759511609, + "narHash": "sha256-uU6Dq5OUgI9pEiMDwZZhvsoxYD+36xKIkYyFXDr//6A=", + "owner": "ipetkov", + "repo": "crane", + "rev": "c5b48a59ccd5179ea626f47b05d2828c37fb31b7", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "fenix": { "inputs": { "nixpkgs": [ @@ -8,11 +23,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1741070164, - "narHash": "sha256-zgHp8rxIbJFeF2DuEMAhKqfdUnclcjaVfdhLNgX5nUM=", + "lastModified": 1759560021, + "narHash": "sha256-J/rtMKVUAEqOFj0ogvcHKK8HbaKhw+tiNrDOpEM+ZDY=", "owner": "nix-community", "repo": "fenix", - "rev": "c36306dbcc4ad8128e659ea072ad35e02936b03e", + "rev": "6ffcbf59c119b0c6384c7d98f18cea06a9af7e9c", "type": "github" }, "original": { @@ -24,11 +39,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1733328505, - "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", + "lastModified": 1747046372, + "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=", "owner": "edolstra", "repo": "flake-compat", - "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", + "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", "type": "github" }, "original": { @@ -57,27 +72,27 @@ }, "nixpkgs": { "locked": { - "lastModified": 1735563628, - "narHash": "sha256-OnSAY7XDSx7CtDoqNh8jwVwh4xNL/2HaJxGjryLWzX8=", + "lastModified": 1759381078, + "narHash": "sha256-gTrEEp5gEspIcCOx9PD8kMaF1iEmfBcTbO0Jag2QhQs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b134951a4c9f3c995fd7be05f3243f8ecd65d798", + "rev": "7df7ff7d8e00218376575f0acdcc5d66741351ee", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-24.05", "repo": "nixpkgs", + "rev": "7df7ff7d8e00218376575f0acdcc5d66741351ee", "type": "github" } }, "nixpkgs-unstable": { "locked": { - "lastModified": 1745391562, - "narHash": "sha256-sPwcCYuiEopaafePqlG826tBhctuJsLx/mhKKM5Fmjo=", + "lastModified": 1759381078, + "narHash": "sha256-gTrEEp5gEspIcCOx9PD8kMaF1iEmfBcTbO0Jag2QhQs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8a2f738d9d1f1d986b5a4cd2fd2061a7127237d7", + "rev": "7df7ff7d8e00218376575f0acdcc5d66741351ee", "type": "github" }, "original": { @@ -89,6 +104,7 @@ }, "root": { "inputs": { + "crane": "crane", "fenix": "fenix", "flake-compat": "flake-compat", "flake-utils": "flake-utils", @@ -99,11 +115,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1741011961, - "narHash": "sha256-bssSxw3Z9CUNB9+f3EHAX/2urT15e12Jy6YU8tHyWkk=", + "lastModified": 1759301569, + "narHash": "sha256-7StxDed3v2fAWLkl+Hse9FlpjT7Dk7Cn/4vxTFyEhIg=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "02862f5d52c30b476a5dca909a17aa4386d1fdc5", + "rev": "472037b789cf593172d6adf3b8d9f7a429f6cd9b", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index af2e468e5b..eae94d68a7 100644 --- a/flake.nix +++ b/flake.nix @@ -1,19 +1,22 @@ { inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + nixpkgs.url = "github:NixOS/nixpkgs/7df7ff7d8e00218376575f0acdcc5d66741351ee"; nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixos-unstable"; flake-utils.url = "github:numtide/flake-utils"; fenix = { url = "github:nix-community/fenix"; inputs.nixpkgs.follows = "nixpkgs"; }; + crane = { + url = "github:ipetkov/crane"; + }; flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; }; - outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils, fenix, ... }: + outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils, fenix, crane, ... }: flake-utils.lib.eachDefaultSystem (system: @@ -21,8 +24,8 @@ let pkgs = nixpkgs.legacyPackages.${system}; pkgs-unstable = nixpkgs-unstable.legacyPackages.${system}; - clang = pkgs.llvmPackages_17.clang; - pythonEnv = pkgs.python39.withPackages (ps: []); + clang = pkgs.llvmPackages.clang; + pythonEnv = pkgs.python3.withPackages (ps: []); toolchain = fenix.packages.${system}.fromToolchainFile { file = ./rust-toolchain.toml; @@ -31,30 +34,70 @@ version = (builtins.fromTOML (builtins.readFile ./engine/Cargo.toml)).workspace.package.version; - appleDeps = with pkgs.darwin.apple_sdk.frameworks; [ - CoreServices - System - SystemConfiguration - pkgs.libiconv-darwin - ]; + appleDeps = pkgs.lib.optionals pkgs.stdenv.isDarwin (with pkgs.darwin; [ + libiconv + ]); rustPlatform = pkgs.makeRustPlatform { inherit (fenix.packages.${system}.minimal) cargo rustc; inherit (fenix.packages.${system}.latest) rust-std; }; + craneLib = (crane.mkLib pkgs).overrideToolchain toolchain; + + # Common source filtering for crane + src = pkgs.lib.cleanSourceWith { + src = ./engine; + filter = path: type: + let baseName = baseNameOf path; in + !pkgs.lib.hasInfix "target" path && + !pkgs.lib.hasInfix ".git" path && + !pkgs.lib.hasInfix ".jj" path && + !pkgs.lib.hasInfix ".so" path && + !pkgs.lib.hasInfix ".node" path && + !pkgs.lib.hasInfix "node_modules" path && + baseName != "result"; + }; + + # Common arguments for all crane builds + commonArgs = { + inherit src version buildInputs nativeBuildInputs; + strictDeps = true; + + LIBCLANG_PATH = pkgs.libclang.lib + "/lib/"; + BINDGEN_EXTRA_CLANG_ARGS = if pkgs.stdenv.isDarwin then + "-I${pkgs.llvmPackages.libclang.lib}/lib/clang/${pkgs.llvmPackages.libclang.version}/headers " + else + "-isystem ${pkgs.llvmPackages.libclang.lib}/lib/clang/${pkgs.llvmPackages.libclang.version}/include -isystem ${pkgs.llvmPackages.libclang.lib}/include -isystem ${pkgs.glibc.dev}/include"; + RUSTFLAGS = if pkgs.stdenv.isDarwin + then "--cfg tracing_unstable" + else "--cfg tracing_unstable -C target-feature=+crt-static"; + OPENSSL_STATIC = "1"; + OPENSSL_DIR = "${pkgs.openssl.dev}"; + OPENSSL_LIB_DIR = "${pkgs.openssl.out}/lib"; + OPENSSL_INCLUDE_DIR = "${pkgs.openssl.dev}/include"; + PROTOC_GEN_GO_PATH = "${pkgs.protoc-gen-go}/bin/protoc-gen-go"; + SKIP_BAML_VALIDATION = "1"; + }; + + # Build dependencies only (this will be cached separately) + cargoArtifacts = craneLib.buildDepsOnly commonArgs; + + devEnvInputs = (with pkgs; [ + ]); + buildInputs = (with pkgs; [ cmake git go gotools + ruby + ruby.devEnv mise openssl pkg-config - lld_17 + lld pythonEnv - ruby - ruby.devEnv maturin pnpm protoc-gen-go @@ -76,21 +119,20 @@ libjpeg giflib librsvg - ]) ++ (if pkgs.stdenv.isDarwin then appleDeps else []); + ]) ++ appleDeps; nativeBuildInputs = [ pkgs.cmake pkgs.openssl pkgs.pkg-config - pkgs.ruby pythonEnv pkgs.maturin pkgs.perl - pkgs.lld_17 + pkgs.ruby + ] ++ pkgs.lib.optionals (!pkgs.stdenv.isDarwin) [ + pkgs.lld pkgs.gcc ]; - wheelName = "baml_py-${version}-cp38-abi3-linux_x86_64.whl"; - bamlRustPackage = { pname, buildPhase ? null, @@ -99,37 +141,39 @@ buildType, extraAttrs ? {}, }: - rustPlatform.buildRustPackage ({ - inherit pname version; - src = ./engine; - filter = path: type: - let baseName = baseNameOf path; in - !pkgs.lib.hasInfix "target" path && - !pkgs.lib.hasInfix ".git" path && - !pkgs.lib.hasInfix ".jj" path && - !pkgs.lib.hasInfix ".so" path && - !pkgs.lib.hasInfix ".node" path && - !pkgs.lib.hasInfix "node_modules" path && - baseName != "result"; - - LIBCLANG_PATH = pkgs.libclang.lib + "/lib/"; - BINDGEN_EXTRA_CLANG_ARGS = if pkgs.stdenv.isDarwin then - "-I${pkgs.llvmPackages_17.libclang.lib}/lib/clang/17/headers " - else - "-isystem ${pkgs.llvmPackages_17.libclang.lib}/lib/clang/17/include -isystem ${pkgs.llvmPackages_17.libclang.lib}/include -isystem ${pkgs.glibc.dev}/include"; - RUSTFLAGS = if pkgs.stdenv.isDarwin - then "--cfg tracing_unstable -C linker=lld" - else "--cfg tracing_unstable -Zlinker-features=+lld -C linker=gcc"; - OPENSSL_STATIC = "1"; - OPENSSL_DIR = "${pkgs.openssl.dev}"; - OPENSSL_LIB_DIR = "${pkgs.openssl.out}/lib"; - OPENSSL_INCLUDE_DIR = "${pkgs.openssl.dev}/include"; - inherit buildInputs; + let + cargoProfileDir = if buildType == "release" then "release" else "debug"; + releaseFlag = if buildType == "release" then "--release" else ""; + + # Crane build function based on build type + buildFn = if buildType == "release" + then craneLib.buildPackage + else craneLib.buildPackage; + + # Unset DEVELOPER_DIR_FOR_TARGET on macOS to avoid SDK conflicts + preBuildWrapper = pkgs.lib.optionalString pkgs.stdenv.isDarwin '' + unset DEVELOPER_DIR_FOR_TARGET + ''; + in + buildFn ({ + inherit pname cargoArtifacts; + inherit (commonArgs) src version buildInputs; + inherit (commonArgs) LIBCLANG_PATH BINDGEN_EXTRA_CLANG_ARGS RUSTFLAGS; + inherit (commonArgs) OPENSSL_STATIC OPENSSL_DIR OPENSSL_LIB_DIR OPENSSL_INCLUDE_DIR; + inherit (commonArgs) PROTOC_GEN_GO_PATH SKIP_BAML_VALIDATION; + + CARGO_PROFILE_DIR = cargoProfileDir; + CARGO_RELEASE_FLAG = releaseFlag; + CARGO_BUILD_RUSTFLAGS = commonArgs.RUSTFLAGS; + nativeBuildInputs = nativeBuildInputs ++ nativeBuildInputsExtra; doCheck = false; - inherit buildType; - cargoLock = { lockFile = ./engine/Cargo.lock; outputHashes = {}; }; - SKIP_BAML_VALIDATION = "1"; + + # Set CARGO_PROFILE to control release vs debug builds + CARGO_PROFILE = if buildType == "release" then "release" else "dev"; + + # Prevent SDK conflicts on macOS + preBuild = preBuildWrapper + (extraAttrs.preBuild or ""); } // (if buildPhase != null then { inherit buildPhase; } else {}) // (if installPhase != null then { inherit installPhase; } else {}) @@ -140,15 +184,22 @@ packages.default = bamlRustPackage { pname = "baml-cli"; - buildType = "debug"; + buildType = "release"; installPhase = '' runHook preInstall - echo "Listing baml binaries in target/debug:" - find target/debug -type f -name "baml*" + build_root=''${CARGO_TARGET_DIR:-target} + profile_dir="$build_root/$CARGO_PROFILE_DIR" + echo "Listing baml binaries under $profile_dir:" + find "$profile_dir" -maxdepth 1 -type f -name "baml*" || true mkdir -p $out/bin - BINARY_NAME=$(find target/debug -type f -executable -name "baml*" | head -n1) + BINARY_NAME="$profile_dir/baml-cli" + if [ ! -x "$BINARY_NAME" ]; then + echo "Unable to locate the compiled CLI binary at $BINARY_NAME" >&2 + exit 1 + fi echo "Found binary: $BINARY_NAME" cp "$BINARY_NAME" $out/bin/baml-cli + strip $out/bin/baml-cli 2>/dev/null || true runHook postInstall ''; extraAttrs = { @@ -159,33 +210,145 @@ }; }; + packages."baml-cli-musl" = if pkgs.stdenv.isDarwin + then throw "musl builds are not supported on macOS - use the default package instead" + else let + muslPkgs = pkgs.pkgsStatic; + + muslCommonArgs = commonArgs // { + buildInputs = (with muslPkgs; [ + cmake + git + openssl + pkg-config + pythonEnv + gcc + ]); + nativeBuildInputs = [ + pkgs.cmake + muslPkgs.openssl + pkgs.pkg-config + pythonEnv + pkgs.perl + ]; + CARGO_BUILD_TARGET = "x86_64-unknown-linux-musl"; + CARGO_BUILD_RUSTFLAGS = "--cfg tracing_unstable -C target-feature=+crt-static"; + OPENSSL_STATIC = "1"; + OPENSSL_DIR = "${muslPkgs.openssl.dev}"; + OPENSSL_LIB_DIR = "${muslPkgs.openssl.out}/lib"; + OPENSSL_INCLUDE_DIR = "${muslPkgs.openssl.dev}/include"; + }; + in craneLib.buildPackage (muslCommonArgs // { + pname = "baml-cli"; + cargoExtraArgs = "--target x86_64-unknown-linux-musl"; + + installPhase = '' + runHook preInstall + build_root=''${CARGO_TARGET_DIR:-target} + mkdir -p $out/bin + BINARY_NAME="$build_root/x86_64-unknown-linux-musl/release/baml-cli" + if [ ! -x "$BINARY_NAME" ]; then + echo "Unable to locate the compiled musl CLI binary at $BINARY_NAME" >&2 + exit 1 + fi + cp "$BINARY_NAME" $out/bin/baml-cli + strip $out/bin/baml-cli 2>/dev/null || true + runHook postInstall + ''; + }); + + packages."baml-cli-debug" = bamlRustPackage { + pname = "baml-cli"; + buildType = "debug"; + installPhase = '' + runHook preInstall + build_root=''${CARGO_TARGET_DIR:-target} + profile_dir="$build_root/$CARGO_PROFILE_DIR" + echo "Listing baml binaries under $profile_dir:" + find "$profile_dir" -maxdepth 1 -type f -name "baml*" || true + mkdir -p $out/bin + BINARY_NAME="$profile_dir/baml-cli" + if [ ! -x "$BINARY_NAME" ]; then + echo "Unable to locate the compiled CLI binary at $BINARY_NAME" >&2 + exit 1 + fi + echo "Found binary: $BINARY_NAME" + cp "$BINARY_NAME" $out/bin/baml-cli + runHook postInstall + ''; + extraAttrs = { + PYTHON_SYS_EXECUTABLE = "${pythonEnv}/bin/python3"; + LD_LIBRARY_PATH = "${pythonEnv}/lib"; + PYTHONPATH = "${pythonEnv}/${pythonEnv.sitePackages}"; + }; + }; + packages.pyLib = bamlRustPackage { + pname = "baml-cli"; + buildType = "release"; + nativeBuildInputsExtra = [ pkgs.maturin pythonEnv ]; + buildPhase = '' + # Unset conflicting environment variable for macOS SDK + echo "Unsetting DEVELOPER_DIR_FOR_TARGET" + unset DEVELOPER_DIR_FOR_TARGET + + cargo build $CARGO_RELEASE_FLAG + cd language_client_python + maturin build --offline $CARGO_RELEASE_FLAG --target-dir ../target --interpreter ${pythonEnv}/bin/python3 + ''; + installPhase = '' + mkdir -p $out/lib + ls ../target/wheels + wheel_path=$(find ../target/wheels -maxdepth 1 -type f -name 'baml_py-*.whl' | head -n1) + if [ -z "$wheel_path" ]; then + echo "No wheel produced by maturin build" >&2 + exit 1 + fi + # Preserve the actual wheel filename with platform tags + cp "$wheel_path" "$out/lib/" + echo "$wheel_path" > $out/wheel-name.txt + ''; + }; + + packages."baml-py-debug" = bamlRustPackage { pname = "baml-cli"; buildType = "debug"; nativeBuildInputsExtra = [ pkgs.maturin pythonEnv ]; buildPhase = '' + # Unset conflicting environment variable for macOS SDK + echo "Unsetting DEVELOPER_DIR_FOR_TARGET" + unset DEVELOPER_DIR_FOR_TARGET + cargo build cd language_client_python - maturin build --offline --target-dir ../target + maturin build --offline --target-dir ../target --interpreter ${pythonEnv}/bin/python3 ''; installPhase = '' mkdir -p $out/lib ls ../target/wheels - cp ../target/wheels/${wheelName} $out/lib/ - touch $out/results.txt - ls -la $out >> $out/results.txt - ls -la $out/lib >> $out/results.txt - ls -la $out/lib/${wheelName} >> $out/results.txt + wheel_path=$(find ../target/wheels -maxdepth 1 -type f -name 'baml_py-*.whl' | head -n1) + if [ -z "$wheel_path" ]; then + echo "No wheel produced by maturin build" >&2 + exit 1 + fi + # Preserve the actual wheel filename with platform tags + cp "$wheel_path" "$out/lib/" + echo "$wheel_path" > $out/wheel-name.txt ''; }; - packages.baml-py = pkgs.python39Packages.buildPythonPackage { + packages.baml-py = pkgs.python3Packages.buildPythonPackage { pname = "baml-py"; inherit version; format = "wheel"; - src = "${packages.pyLib}/lib/${wheelName}"; - propagatedBuildInputs = with pkgs.python39.pkgs; [ + # Find the actual wheel file with platform tags + src = let + wheelDir = "${packages.pyLib}/lib"; + wheelFile = builtins.head (builtins.attrNames (builtins.readDir wheelDir)); + in "${wheelDir}/${wheelFile}"; + + propagatedBuildInputs = with pkgs.python3.pkgs; [ pydantic typing-extensions ]; @@ -197,22 +360,32 @@ description = "Python bindings for BAML"; homepage = "https://github.com/boundaryml/baml"; license = licenses.mit; - platforms = platforms.linux; + platforms = platforms.unix; }; }; packages.tsLib = bamlRustPackage { pname = "baml-ts"; - buildType = "debug"; + buildType = "release"; nativeBuildInputsExtra = [ pkgs-unstable.nodejs_20 pkgs.napi-rs-cli pkgs.pnpm ]; buildPhase = '' + # Unset conflicting environment variable for macOS SDK + echo "Unsetting DEVELOPER_DIR_FOR_TARGET" + unset DEVELOPER_DIR_FOR_TARGET + # Build the CLI echo "Building the CLI" - cargo build -p baml-cli + cargo build $CARGO_RELEASE_FLAG -p baml-cli # Build specifically the typescript FFI crate echo "Building the typescript FFI crate" - cargo build -p baml-typescript-ffi + cargo build $CARGO_RELEASE_FLAG -p baml-typescript-ffi + + # The build artifacts are in the crane-managed target directory + echo "CARGO_TARGET_DIR is: ''${CARGO_TARGET_DIR:-target}" + echo "Looking for build outputs..." + find . -name "libbaml.*" -type f 2>/dev/null || true + cd language_client_typescript echo "Listing current directory contents:" @@ -220,12 +393,44 @@ # Copy the built library to where napi expects it echo "Copying the built library to where napi expects it" - mkdir -p target/debug - find ../target -name "*.so" -o -name "*.dylib" -o -name "*.dll" - cp ../target/debug/libbaml.so target/debug/libbaml_typescript_ffi.so + build_root=''${CARGO_TARGET_DIR:-../target} + cargo_lib_dir="$build_root" + mkdir -p "$build_root/$CARGO_PROFILE_DIR" + echo "Searching for shared libraries in $cargo_lib_dir:" + find "$cargo_lib_dir" -name "*.so" -o -name "*.dylib" -o -name "*.dll" 2>/dev/null || true + shared_lib=$(find "$cargo_lib_dir" -type f \( -name "libbaml.so" -o -name "libbaml.dylib" -o -name "libbaml.dll" \) 2>/dev/null | head -n1) + if [ -z "$shared_lib" ]; then + echo "Unable to locate built shared library" >&2 + echo "Trying absolute search from root of build..." + find .. -name "libbaml.*" -type f 2>/dev/null || true + exit 1 + fi + lib_basename=$(basename "$shared_lib") + case "$lib_basename" in + *.so) + cp "$shared_lib" "$build_root/$CARGO_PROFILE_DIR/libbaml_typescript_ffi.so" + ;; + *.dylib) + cp "$shared_lib" "$build_root/$CARGO_PROFILE_DIR/libbaml_typescript_ffi.dylib" + cp "$shared_lib" "$build_root/$CARGO_PROFILE_DIR/libbaml_typescript_ffi.so" + ;; + *.dll) + cp "$shared_lib" "$build_root/$CARGO_PROFILE_DIR/libbaml_typescript_ffi.dll" + ;; + esac + + mkdir -p dist + cp "$shared_lib" "dist/$lib_basename" + + # Only create symlink if it doesn't already exist with the right name + ffi_lib=$(find "$cargo_lib_dir/$CARGO_PROFILE_DIR" -type f -name 'libbaml_typescript_ffi*.dylib' | head -n1) + if [ -n "$ffi_lib" ] && [ "$(basename "$ffi_lib")" != "libbaml_typescript_ffi.dylib" ]; then + mkdir -p "$build_root/$CARGO_PROFILE_DIR" + ln -sf "$ffi_lib" "$build_root/$CARGO_PROFILE_DIR/libbaml_typescript_ffi.dylib" + fi # Build the native module directly with release flag - napi build --platform --js ./native.js --dts ./native.d.ts + env -u DEVELOPER_DIR_FOR_TARGET napi build --platform $CARGO_RELEASE_FLAG --js ./native.js --dts ./native.d.ts # Compile TypeScript files using the Nix-provided TypeScript ${pkgs.nodePackages.typescript}/bin/tsc ./typescript_src/*.ts --outDir ./dist --module commonjs --allowJs --declaration true || true @@ -239,6 +444,68 @@ # Copy the native modules cp *.node dist/ + if [ "$(uname)" = "Darwin" ]; then + echo "Fixing macOS Mach-O install names for bundled native modules" + + pending=1 + while [ "$pending" -eq 1 ]; do + pending=0 + for bundle in dist/*.node dist/*.dylib dist/*.so; do + [ -e "$bundle" ] || continue + chmod +w "$bundle" 2>/dev/null || true + + case "$(basename "$bundle")" in + *.dylib|*.so|*.node) + install_name_tool -id "@loader_path/$(basename "$bundle")" "$bundle" + ;; + esac + + for dep in $(otool -L "$bundle" | tail -n +2 | awk '{print $1}'); do + [ -z "$dep" ] && continue + case "$dep" in + /System/*|@loader_path/*|@rpath/*) + ;; + /usr/lib/libiconv.2.dylib|/usr/lib/libcharset.1.dylib|/usr/lib/libSystem.B.dylib) + install_name_tool -change "$dep" "$dep" "$bundle" + ;; + *) + dep_name=$(basename "$dep") + + case "$dep_name" in + libiconv.2.dylib) + install_name_tool -change "$dep" "/usr/lib/libiconv.2.dylib" "$bundle" + continue + ;; + libcharset.1.dylib) + install_name_tool -change "$dep" "/usr/lib/libcharset.1.dylib" "$bundle" + continue + ;; + libSystem.B.dylib) + install_name_tool -change "$dep" "/usr/lib/libSystem.B.dylib" "$bundle" + continue + ;; + esac + + dest="dist/$dep_name" + if [ ! -e "$dest" ]; then + echo " bundling $dep_name" + cp "$dep" "$dest" + chmod +w "$dest" 2>/dev/null || true + pending=1 + fi + echo " rewriting $(basename "$bundle") dependency $dep" + install_name_tool -change "$dep" "@loader_path/$dep_name" "$bundle" + ;; + esac + done + done + done + strip -x dist/*.dylib 2>/dev/null || true + strip -x dist/*.node 2>/dev/null || true + else + strip dist/*.so 2>/dev/null || true + fi + # Create minimal package.json and package-lock.json cat > dist/package.json << EOF { @@ -251,11 +518,14 @@ "*.js", "*.ts", "*.node", + "*.dylib", + "*.so", + "*.dll", "bin/baml-cli" ], "dependencies": {}, - "os": ["linux"], - "cpu": ["x64"] + "os": ["linux", "darwin"], + "cpu": ["x64", "arm64"] } EOF @@ -280,7 +550,146 @@ # Copy the CLI binary mkdir -p dist/bin - cp ../target/debug/baml-cli dist/bin/baml-cli + cp "$cargo_lib_dir/$CARGO_PROFILE_DIR/baml-cli" dist/bin/baml-cli + strip dist/bin/baml-cli 2>/dev/null || true + ''; + installPhase = '' + mkdir -p $out/lib + cp -r dist/* $out/lib/ + ''; + extraAttrs = { + SKIP_BAML_VALIDATION = "1"; + }; + }; + + packages."tsLib-debug" = bamlRustPackage { + pname = "baml-ts"; + buildType = "debug"; + nativeBuildInputsExtra = [ pkgs-unstable.nodejs_20 pkgs.napi-rs-cli pkgs.pnpm ]; + buildPhase = '' + echo "Unsetting DEVELOPER_DIR_FOR_TARGET" + unset DEVELOPER_DIR_FOR_TARGET + + echo "Building the CLI" + cargo build -p baml-cli + + echo "Building the typescript FFI crate" + cargo build -p baml-typescript-ffi + + echo "CARGO_TARGET_DIR is: ''${CARGO_TARGET_DIR:-target}" + echo "Looking for build outputs..." + find . -name "libbaml.*" -type f 2>/dev/null || true + + cd language_client_typescript + + echo "Listing current directory contents:" + ls -la + + echo "Copying the built library to where napi expects it" + build_root=''${CARGO_TARGET_DIR:-../target} + cargo_lib_dir="$build_root" + mkdir -p "$build_root/debug" + echo "Searching for shared libraries in $cargo_lib_dir:" + find "$cargo_lib_dir" -name "*.so" -o -name "*.dylib" -o -name "*.dll" 2>/dev/null || true + shared_lib=$(find "$cargo_lib_dir" -type f \( -name "libbaml.so" -o -name "libbaml.dylib" -o -name "libbaml.dll" \) 2>/dev/null | head -n1) + if [ -z "$shared_lib" ]; then + echo "Unable to locate built shared library" >&2 + echo "Trying absolute search from root of build..." + find .. -name "libbaml.*" -type f 2>/dev/null || true + exit 1 + fi + lib_basename=$(basename "$shared_lib") + case "$lib_basename" in + *.so) + cp "$shared_lib" "$build_root/debug/libbaml_typescript_ffi.so" + ;; + *.dylib) + cp "$shared_lib" "$build_root/debug/libbaml_typescript_ffi.dylib" + cp "$shared_lib" "$build_root/debug/libbaml_typescript_ffi.so" + ;; + *.dll) + cp "$shared_lib" "$build_root/debug/libbaml_typescript_ffi.dll" + ;; + esac + + mkdir -p dist + cp "$shared_lib" "dist/$lib_basename" + + # Only create symlink if it doesn't already exist with the right name + ffi_lib=$(find "$cargo_lib_dir/debug" -type f -name 'libbaml_typescript_ffi*.dylib' | head -n1) + if [ -n "$ffi_lib" ] && [ "$(basename "$ffi_lib")" != "libbaml_typescript_ffi.dylib" ]; then + mkdir -p "$build_root/debug" + ln -sf "$ffi_lib" "$build_root/debug/libbaml_typescript_ffi.dylib" + fi + + env -u DEVELOPER_DIR_FOR_TARGET napi build --platform --js ./native.js --dts ./native.d.ts + + ${pkgs.nodePackages.typescript}/bin/tsc ./typescript_src/*.ts --outDir ./dist --module commonjs --allowJs --declaration true || true + + cp *.js dist/ || true + cp *.d.ts dist/ || true + cp *.node dist/ + + if [ "$(uname)" = "Darwin" ]; then + echo "Fixing macOS Mach-O install names for bundled native modules" + + pending=1 + while [ "$pending" -eq 1 ]; do + pending=0 + for bundle in dist/*.node dist/*.dylib dist/*.so; do + [ -e "$bundle" ] || continue + chmod +w "$bundle" 2>/dev/null || true + + case "$(basename "$bundle")" in + *.dylib|*.so|*.node) + install_name_tool -id "@loader_path/$(basename "$bundle")" "$bundle" + ;; + esac + + for dep in $(otool -L "$bundle" | tail -n +2 | awk '{print $1}'); do + [ -z "$dep" ] && continue + case "$dep" in + /System/*|/usr/lib/*|@loader_path/*|@rpath/*) + ;; + *) + dep_name=$(basename "$dep") + dest="dist/$dep_name" + + case "$dep_name" in + libiconv.2.dylib) + echo " remapping $dep_name to /usr/lib/libiconv.2.dylib" + install_name_tool -change "$dep" "/usr/lib/libiconv.2.dylib" "$bundle" + continue + ;; + libcharset.1.dylib) + echo " remapping $dep_name to /usr/lib/libcharset.1.dylib" + install_name_tool -change "$dep" "/usr/lib/libcharset.1.dylib" "$bundle" + continue + ;; + libintl.8.dylib) + echo " remapping $dep_name to /usr/local/lib/libintl.8.dylib" + install_name_tool -change "$dep" "/usr/local/lib/libintl.8.dylib" "$bundle" + continue + ;; + esac + + if [ ! -e "$dest" ]; then + echo " bundling $dep_name" + cp "$dep" "$dest" + chmod +w "$dest" 2>/dev/null || true + pending=1 + fi + echo " rewriting $(basename "$bundle") dependency $dep" + install_name_tool -change "$dep" "@loader_path/$dep_name" "$bundle" + ;; + esac + done + done + done + fi + + mkdir -p dist/bin + cp "$cargo_lib_dir/debug/baml-cli" dist/bin/baml-cli ''; installPhase = '' mkdir -p $out/lib @@ -288,7 +697,6 @@ ''; extraAttrs = { SKIP_BAML_VALIDATION = "1"; - cargoLock = { lockFile = ./engine/Cargo.lock; }; }; }; @@ -304,7 +712,7 @@ src = npmSource; - npmDepsHash = "sha256-p7AxgJSqngcwHwKsjF6u+fS0E27KY6/ulGIIRlZLsFU="; + npmDepsHash = "sha256-6l5OwLGhW+c2mUhVUDwxH5rs5pzxd0+uTOrx14q04KY="; forceEmptyCache = true; buildInputs = [ pkgs-unstable.nodejs_20 ]; @@ -329,6 +737,37 @@ ''; }; + packages."baml-ts-debug" = let + npmSource = pkgs.runCommand "baml-ts-${version}-debug-source" {} '' + mkdir -p $out + cp -r ${packages."tsLib-debug"}/lib/* $out/ + ''; + in pkgs.buildNpmPackage { + pname = "baml"; + inherit version; + + src = npmSource; + + npmDepsHash = "sha256-6l5OwLGhW+c2mUhVUDwxH5rs5pzxd0+uTOrx14q04KY="; + forceEmptyCache = true; + + buildInputs = [ pkgs-unstable.nodejs_20 ]; + + NPM_CONFIG_CACHE = "./tmp/npm"; + NPM_CONFIG_TMP = "./tmp/npm"; + NPM_CONFIG_PREFIX = "./tmp/npm"; + + buildPhase = '' + mkdir -p tmp/npm + npm pack + ''; + + installPhase = '' + mkdir -p $out/lib + cp boundaryml-baml-${version}.tgz $out/lib/ + ''; + }; + devShell = pkgs.mkShell rec { inherit buildInputs; PATH="${clang}/bin:$PATH"; @@ -339,6 +778,11 @@ "" # Rely on default includes provided by stdenv.cc + libclang else "-isystem ${pkgs.llvmPackages_17.libclang.lib}/lib/clang/17/include -isystem ${pkgs.llvmPackages_17.libclang.lib}/include -isystem ${pkgs.glibc.dev}/include"; + + # Prevent SDK conflicts on macOS + shellHook = pkgs.lib.optionalString pkgs.stdenv.isDarwin '' + unset DEVELOPER_DIR_FOR_TARGET + ''; }; } );