Update build instructions for Mac, fix scripts

Author: rstz

.bazelrc

@@ -60,7 +60,6 @@ build --strategy=Genrule=standalone
 build --action_env TF_NEED_CUDA="0"
 
 # Host specifics options.
-build:macos --config=release_base
 build:macos --features=-supports_dynamic_linker
 # On Intel machines, AVX2 is available
 build:macos_intel --config=avx_linux

configure/setup.py

@@ -68,7 +68,7 @@ def get_tag(self):
       if platform.system() == "Darwin":
         # Uncomment on of the lines below to adapt the platform string when
         # cross-compiling.
-        # plat = "macosx_12_0_arm64"
+        plat = "macosx_12_0_arm64"
         # plat = "macosx_10_15_x86_64"
         pass
       return python, abi, plat

documentation/installation.md

@@ -73,7 +73,7 @@ cd decision-forests
 
 **Optional:** TensorFlow Decision Forests depends on
 [Yggdrasil Decision Forests](https://github.com/google/yggdrasil-decision-forests)
-. If you want to edit the Yggdrasil code, you can clone the Yggdrasil github and
+. If you want to edit the Yggdrasil code, you can clone the Yggdrasil repo and
 change the path accordingly in
 `third_party/yggdrasil_decision_forests/workspace.bzl`.
 
@@ -145,59 +145,38 @@ of python using pyenv by running the following command. See the header of
 
 -   XCode command line tools
 -   Bazel (recommended [Bazelisk](https://github.com/bazelbuild/bazelisk))
--   Python >= 3.9
--   Git
+-   Homebrew packages: GNU coreutils, GNU sed, GNU grep
 -   Pyenv (for building the Pip packages with multiple Python versions)
 
-#### Building  / Packaging (Apple CPU)
+#### Arm64 CPU
 
-If you have a MacOS machine with Apple CPU, you can build with the following
-instructions.
+For MacOS systems with ARM64 CPU, follow these steps:
 
-1.  Clone the three repositories and adjust paths.
+1.  Prepare your environment
 
     ```
     git clone https://github.com/tensorflow/decision-forests.git
-    git clone https://github.com/google/yggdrasil-decision-forests.git
-    git clone --branch boost-1.75.0 https://github.com/boostorg/boost.git
-    (cd boost && git submodule update --init --checkout --force)
-    # Adjust path TF-DF --> YDF
-    perl -0777 -i.original -pe 's/    http_archive\(\n        name = "ydf",\n        urls = \["https:\/\/github.com\/google\/yggdrasil-decision-forests\/archive\/refs\/heads\/main.zip"\],\n        strip_prefix = "yggdrasil-decision-forests-main",\n    \)/    native.local_repository\(\n        name = "ydf",\n        path = "..\/yggdrasil-decision-forests",\n    \)/igs' decision-forests/third_party/yggdrasil_decision_forests/workspace.bzl
-    # Adjust path YDF --> Boost
-    perl -0777 -i.original -pe 's/    new_git_repository\(\n        name = "org_boost",\n        branch = branch,\n        build_file_content = build_file_content,\n        init_submodules = True,\n        recursive_init_submodules = True,\n        remote = "https:\/\/github.com\/boostorg\/boost",\n    \)/    native.new_local_repository\(\n        name = "org_boost",\n        path = "..\/boost",\n        build_file_content = build_file_content,\n    \)/igs' yggdrasil-decision-forests/third_party/boost/workspace.bzl
-    ```
-
-    You may need to adjust the test_bazel.sh script manually to fix the
-    Tensorflow commit hash, since it is sometimes broken for MacOS builds.
-
-1.  (Optional) Create a fresh virtual environment and activate it
-
-    ```
     python3 -m venv venv
     source venv/source/activate
     ```
 
-1.  Adjust the TensorFlow dependency for Apple CPUs
-
-    ```
-    perl -0777 -i.original -pe 's/tensorflow~=/tensorflow-macos~=/igs' decision-forests/configure/setup.py
-    ```
-
-1.  Decide which Python version you want to use and run
+1.  Decide which Python version and TensorFlow version you want to use and run
 
     ```
     cd decision-forests
-    # This will compile with the latest Tensorflow version in the tensorflow-macos repository.
-    RUN_TESTS=1 PY_VERSION=3.9 TF_VERSION=mac-arm64 ./tools/test_bazel.sh
+    export TF_VERSION=2.15.0  # Change to the TensorFlow Version you need.
+    export PY_VERSION=3.9     # Change to the Python you need.
+    export RUN_TESTS=1        # Change to 0 if you want to skip tests.
+    ./tools/test_bazel.sh     # Takes ~15 minutes on a modern Mac.
     ```
 
-1.  Build the Pip Packages
+1.  Package the code.
 
     ```
-    # First, we deactivate our virtualenv, since the Pip script uses a different one.
+    # Building the packages uses different virtualenvs through Pyenv.
     deactivate
     # Build the packages.
-    ./tools/build_pip_package.sh ALL_VERSIONS_MAC_ARM64
+    ./tools/build_pip_package.sh ALL_VERSIONS
     ```
 
 1.  The packages can be found in `decision-forests/dist/`.
@@ -207,22 +186,29 @@ instructions.
 If you have a MacOS machine with Apple CPU, cross-compile TF-DF for MacOS
 machines with Intel CPUs as follows.
 
-1.  Follow Steps 1-3 and 5 of the guide for Apple CPUs, **skip Step 4**.
-    You may need to run `bazel --bazelrc=tensorflow_bazelrc clean --expunge` to
-    clean your build directory.
+1.  Prepare your environment
+
+    ```
+    git clone https://github.com/tensorflow/decision-forests.git
+    python3 -m venv venv
+    source venv/source/activate
+    ```
 
 1.  Decide which Python version you want to use and run
 
     ```
     cd decision-forests
-    # This will compile with the latest Tensorflow version in the tensorflow-macos repository.
-    RUN_TESTS=0 PY_VERSION=3.9 TF_VERSION=mac-intel-crosscompile ./tools/test_bazel.sh
+    export TF_VERSION=2.15.0        # Change to the TensorFlow Version you need.
+    export PY_VERSION=3.9           # Change to the Python you need.
+    export RUN_TESTS=0              # Cross-compiled packages cannot be tested.
+    export MAC_INTEL_CROSSCOMPILE=1
+    ./tools/test_bazel.sh           # Takes ~15 minutes on a modern Mac.
     ```
 
-1.  Build the Pip Packages
+1.  Package the code.
 
     ```
-    # First, we deactivate our virtualenv, since the Pip script uses a different one.
+    # Building the packages uses different virtualenvs through Pyenv.
     deactivate
     # Build the packages.
     ./tools/build_pip_package.sh ALL_VERSIONS_MAC_INTEL_CROSSCOMPILE

third_party/yggdrasil_decision_forests/workspace.bzl

@@ -13,6 +13,7 @@ def deps(from_git_repo = True):
         )
     else:
         # You can also clone the YDF repository manually.
+        # Note that you need to manually apply the patch for Tensorflow >= 2.16 or nightly.
         native.local_repository(
             name = "ydf",
             # When downloading from Github, you might need - instead of _ as folder name

tools/build_pip_package.sh

@@ -102,6 +102,7 @@ function assemble_files() {
 
   # When cross-compiling, adapt the platform string.
   if [ ${ARG} == "ALL_VERSIONS_MAC_INTEL_CROSSCOMPILE" ]; then
+    sed -i'.bak' -e "s/plat = \"macosx_12_0_arm64\"/# plat = \"macosx_12_0_arm64\"/" ${SRCPK}/setup.py
     sed -i'.bak' -e "s/# plat = \"macosx_10_15_x86_64\"/plat = \"macosx_10_15_x86_64\"/" ${SRCPK}/setup.py
   fi
 

tools/test_bazel.sh

@@ -13,18 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-
 # Build and test TF-DF.
 # Options
 #  RUN_TESTS: Run the unit tests e.g. 0 or 1.
 #  PY_VERSION: Version of Python to be used, must be at least 3.9
 #  STARTUP_FLAGS: Any flags given to bazel on startup
 #  TF_VERSION: Tensorflow version to use or "nightly".
-#              For cross-compiling with Apple Silicon for Mac Intel, use 
-#              mac-intel-crosscompile.
-#              Tests will not work when cross-compiling (obviously).
-# FULL_COMPILATION: If 1, compile all parts of TF-DF. This may take a long time.
+#  MAC_INTEL_CROSSCOMPILE: Cross-compile for Intel Macs
+#  FULL_COMPILATION: If 1, compile all parts of TF-DF. This may take a long time.
 #
 # Usage example
 #
@@ -48,58 +44,72 @@ function is_macos() {
   [[ "${PLATFORM}" == "darwin" ]]
 }
 
+if is_macos; then
+  # 1. Check if the current shell is Bash
+  if [[ $SHELL != "/bin/bash" ]]; then
+      echo "Error: This script requires Bash. Please run it in a Bash shell."
+      exit 1  # Exit with an error code
+  fi
+  if ! command -v gsort &> /dev/null; then  
+      echo "Error: GNU coreutils is not installed. Please install it with 'brew install coreutils'"
+      exit 1
+  fi
+  if ! command -v ggrep &> /dev/null; then
+      echo "Error: GNU grep is not installed. Please install it with 'brew install grep'"
+      exit 1
+  fi
+  if ! command -v gsed &> /dev/null; then
+      echo "Error: GNU sed is not installed. Please install it with 'brew install gnu-sed'"
+      exit 1
+  fi
+  # Tensorflow requires the use of GNU realpath instead of MacOS realpath.
+  # See https://github.com/tensorflow/tensorflow/issues/60088#issuecomment-1499766349
+  export PATH="/opt/homebrew/opt/coreutils/libexec/gnubin:$PATH"
+  export PATH="/opt/homebrew/opt/grep/libexec/gnubin:$PATH"
+  export PATH="/opt/homebrew/opt/gnu-sed/libexec/gnubin:$PATH"
+fi
+
 # Install Pip dependencies
 ${PYTHON} -m ensurepip --upgrade || true
-${PYTHON} -m pip install pip setuptools --upgrade
-${PYTHON} -m pip install numpy pandas scikit-learn
+${PYTHON} -m pip install -q pip setuptools --upgrade
+${PYTHON} -m pip install -q numpy pandas scikit-learn
 
 # Install Tensorflow at the chosen version.
 if [ ${TF_VERSION} == "nightly" ]; then
-  ${PYTHON} -m pip install tf-nightly tf-keras-nightly --force-reinstall
+  ${PYTHON} -m pip install -q tf-nightly tf-keras-nightly --force-reinstall
   TF_MINOR="nightly"
 else
-  ${PYTHON} -m pip install tensorflow==${TF_VERSION} --force-reinstall
+  ${PYTHON} -m pip install -q tensorflow==${TF_VERSION} --force-reinstall
+  TF_MINOR=$(echo $TF_VERSION | grep -oP '[0-9]+\.[0-9]+')
   if [[ $TF_VERSION == *"rc"* ]]; then
-    ${PYTHON} -m pip install tf-keras --pre --upgrade
+    # Unfortunately, the TF-Keras RC may not match the TensorFlow RC (e.g. for 2.16).
+    # Just install the latest one that's available and hope for the best.
+    ${PYTHON} -m pip install -q tf-keras --pre --upgrade
   else
-    TF_MINOR=$(echo $TF_VERSION | grep -oP '[0-9]+\.[0-9]+')
-    ${PYTHON} -m pip install tf-keras==${TF_MINOR}
+    ${PYTHON} -m pip install -q tf-keras==${TF_MINOR}
   fi
 fi
-ext=""
 
-pip list
-
-if is_macos; then
-  ext='""'
-  # Tensorflow requires the use of GNU realpath instead of MacOS realpath.
-  # See https://github.com/tensorflow/tensorflow/issues/60088#issuecomment-1499766349
-  # If missing, install coreutils via homebrew: `brew install coreutils`
-  export PATH="/opt/homebrew/opt/coreutils/libexec/gnubin:$PATH"
-fi
+${PYTHON} -m pip list
 
 # For Tensorflow versions > 2.15, apply compatibility patches.
 
 if [[ ${TF_MINOR} != "2.15" ]]; then
-  sed -i $ext "s/tensorflow:tf.patch/tensorflow:tf-216.patch/" WORKSPACE
-  sed -i $ext "s/# patch_args = \[\"-p1\"\],/patch_args = \[\"-p1\"\],/" third_party/yggdrasil_decision_forests/workspace.bzl
-  sed -i $ext "s/# patches = \[\"\/\/third_party\/yggdrasil_decision_forests:ydf.patch\"\],/patches = \[\"\/\/third_party\/yggdrasil_decision_forests:ydf.patch\"\],/" third_party/yggdrasil_decision_forests/workspace.bzl
+  sed -i "s/tensorflow:tf.patch/tensorflow:tf-216.patch/" WORKSPACE
+  sed -i "s/# patch_args = \[\"-p1\"\],/patch_args = \[\"-p1\"\],/" third_party/yggdrasil_decision_forests/workspace.bzl
+  sed -i 's/# patches = \["@ydf\/\/yggdrasil_decision_forests:ydf.patch"\],/patches = \["\/\/third_party\/yggdrasil_decision_forests:ydf.patch"\],/' third_party/yggdrasil_decision_forests/workspace.bzl
 fi
 
 # Get the commit SHA
 short_commit_sha=$(${PYTHON} -c 'import tensorflow as tf; print(tf.__git_version__)' | tail -1)
-if is_macos; then
-  short_commit_sha=$(echo $short_commit_sha | perl -nle 'print $& while m{(?<=-g)[0-9a-f]*$}g')
-else
-  short_commit_sha=$(echo $short_commit_sha | grep -oP '(?<=-g)[0-9a-f]*$')
-fi
+short_commit_sha=$(echo $short_commit_sha | grep -oP '(?<=-g)[0-9a-f]*$')
 echo "Found tensorflow commit sha: $short_commit_sha"
 commit_slug=$(curl -s "https://api.github.com/repos/tensorflow/tensorflow/commits/$short_commit_sha" | grep "sha" | head -n 1 | cut -d '"' -f 4)
 # Update TF dependency to the chosen version
-sed -E -i $ext "s/strip_prefix = \"tensorflow-2\.[0-9]+\.[0-9]+(-rc[0-9]+)?\",/strip_prefix = \"tensorflow-${commit_slug}\",/" WORKSPACE
-sed -E -i $ext "s|\"https://github.com/tensorflow/tensorflow/archive/v.+\.zip\"|\"https://github.com/tensorflow/tensorflow/archive/${commit_slug}.zip\"|" WORKSPACE
+sed -E -i "s/strip_prefix = \"tensorflow-2\.[0-9]+\.[0-9]+(-rc[0-9]+)?\",/strip_prefix = \"tensorflow-${commit_slug}\",/" WORKSPACE
+sed -E -i "s|\"https://github.com/tensorflow/tensorflow/archive/v.+\.zip\"|\"https://github.com/tensorflow/tensorflow/archive/${commit_slug}.zip\"|" WORKSPACE
 prev_shasum=$(grep -A 1 -e "strip_prefix.*tensorflow-" WORKSPACE | tail -1 | awk -F '"' '{print $2}')
-sed -i $ext "s/sha256 = \"${prev_shasum}\",//" WORKSPACE
+sed -i "s/sha256 = \"${prev_shasum}\",//" WORKSPACE
 
 # Get build configuration for chosen version.
 TENSORFLOW_BAZELRC="tensorflow_bazelrc"
@@ -121,14 +131,18 @@ else
   FLAGS="${FLAGS} --config=linux"
 fi
 
-if [ ${TF_VERSION} == "mac-intel-crosscompile" ]; then
+if [ ${MAC_INTEL_CROSSCOMPILE} == 1 ]; then
   TFDF_TMPDIR="${TMPDIR}tf_dep"
   rm -rf ${TFDF_TMPDIR}
   mkdir -p ${TFDF_TMPDIR}
   # Download the Intel CPU Tensorflow package
-  pip download --no-deps --platform=macosx_10_15_x86_64 --dest=$TFDF_TMPDIR tensorflow
-  unzip -q $TFDF_TMPDIR/tensorflow* -d $TFDF_TMPDIR
-
+  if [ ${TF_VERSION} == "nightly" ]; then
+    ${PYTHON} -m pip download --no-deps --platform=macosx_10_15_x86_64 --dest=$TFDF_TMPDIR tf-nightly
+    unzip -q $TFDF_TMPDIR/tf_nightly* -d $TFDF_TMPDIR
+  else
+    ${PYTHON} -m pip download --no-deps --platform=macosx_10_15_x86_64 --dest=$TFDF_TMPDIR tensorflow
+    unzip -q $TFDF_TMPDIR/tensorflow* -d $TFDF_TMPDIR
+  fi
   # Find the path to the pre-compiled version of TensorFlow installed in the
   # "tensorflow" pip package.
   SHARED_LIBRARY_DIR=$(readlink -f $TFDF_TMPDIR/tensorflow)
@@ -171,7 +185,7 @@ STARTUP_FLAGS="${STARTUP_FLAGS} --bazelrc=${TENSORFLOW_BAZELRC}"
 #
 # FLAGS="$FLAGS --config=rbe_cpu_linux --config=tensorflow_testing_rbe_linux --config=rbe_linux_py3"
 
-if [ ${TF_VERSION} == "mac-intel-crosscompile" ]; then
+if [ ${MAC_INTEL_CROSSCOMPILE} == 1 ]; then
   # Using darwin_x86_64 fails here, tensorflow expects "darwin".
   FLAGS="${FLAGS} --cpu=darwin --apple_platform_type=macos"
 fi