Docker build for robotic surgery to enable webrtc streaming

tools/check_missing_license.sh

@@ -28,6 +28,7 @@ INCLUDE_PATTERNS=(
   "*.hpp"
   "*.cu"
   "*.cpp"
+  "Dockerfile*"
 )
 
 # Directories to explicitly exclude

tools/env_setup/install_isaac.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Get the parent directory of the current script
+PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)"
+
+# Check if running in a conda environment
+if [ -z "$CONDA_DEFAULT_ENV" ]; then
+    echo "Error: No active conda environment detected"
+    echo "Please activate a conda environment before running this script"
+    exit 1
+fi
+echo "Using conda environment: $CONDA_DEFAULT_ENV"
+
+# Check if NVIDIA GPU is available
+if ! nvidia-smi &> /dev/null; then
+    echo "Error: NVIDIA GPU not found or driver not installed"
+    exit 1
+fi
+
+# Check if the third_party directory exists, if yes, then exit
+if [ -d "$PROJECT_ROOT/third_party" ]; then
+    echo "Error: third_party directory already exists"
+    echo "Please remove the third_party directory before running this script"
+    exit 1
+fi
+
+# ---- Clone IsaacLab ----
+echo "Cloning IsaacLab..."
+# CLONING REPOSITORIES INTO PROJECT_ROOT/third_party
+echo "Cloning repositories into $PROJECT_ROOT/third_party..."
+mkdir $PROJECT_ROOT/third_party
+git clone -b v2.0.2 [email protected]:isaac-sim/IsaacLab.git $PROJECT_ROOT/third_party/IsaacLab
+pushd $PROJECT_ROOT/third_party/IsaacLab
+
+# ---- Install IsaacSim and necessary dependencies ----
+echo "Installing IsaacSim..."
+pip install 'isaacsim[all,extscache]==4.5.0' \
+    git+ssh://[email protected]/isaac-for-healthcare/[email protected] \
+    --extra-index-url https://pypi.nvidia.com
+
+echo "Installing IsaacLab ..."
+yes Yes | ./isaaclab.sh --install
+popd
+
+# ---- Install libstdcxx-ng for mesa ----
+echo "Installing libstdcxx-ng..."
+conda install -c conda-forge libstdcxx-ng=13.2.0 -y
+
+echo "IsaacSim and dependencies installed successfully!"

tools/env_setup/install_robotic_surgery_extensions.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Get the parent directory of the current script
+PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)"
+
+# Can take EXTS_DIR from input, default to $PROJECT_ROOT/workflows/robotic_surgery/scripts/simulation/exts
+EXTS_DIR=${1:-$PROJECT_ROOT/workflows/robotic_surgery/scripts/simulation/exts}
+
+# Allow setting the python in PYTHON_EXECUTABLE
+PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python}
+
+# ---- Install robotic.surgery.assets and robotic.surgery.tasks ----
+echo "Installing robotic.surgery.assets and robotic.surgery.tasks..."
+$PYTHON_EXECUTABLE -m pip install --no-build-isolation -e $EXTS_DIR/robotic.surgery.assets
+$PYTHON_EXECUTABLE -m pip install --no-build-isolation -e $EXTS_DIR/robotic.surgery.tasks
+
+echo "Extensions installed successfully!"

tools/env_setup_robot_surgery.sh

@@ -18,7 +18,7 @@
 set -e
 
 # Get the parent directory of the current script
-PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. && pwd)"
+PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 # Check if running in a conda environment
 if [ -z "$CONDA_DEFAULT_ENV" ]; then
@@ -41,36 +41,11 @@ if [ -d "$PROJECT_ROOT/third_party" ]; then
     exit 1
 fi
 
+# Run the installation scripts
+echo "Installing IsaacSim and dependencies..."
+bash $PROJECT_ROOT/env_setup/install_isaac.sh
 
-# ---- Clone IsaacLab ----
-echo "Cloning IsaacLab..."
-# CLONING REPOSITORIES INTO PROJECT_ROOT/third_party
-echo "Cloning repositories into $PROJECT_ROOT/third_party..."
-mkdir $PROJECT_ROOT/third_party
-git clone -b v2.0.2 [email protected]:isaac-sim/IsaacLab.git $PROJECT_ROOT/third_party/IsaacLab
-pushd $PROJECT_ROOT/third_party/IsaacLab
+echo "Installing extensions..."
+bash $PROJECT_ROOT/env_setup/install_robotic_surgery_extensions.sh
 
-
-# ---- Install IsaacSim and necessary dependencies ----
-echo "Installing IsaacSim..."
-pip install 'isaacsim[all,extscache]==4.5.0' \
-    git+ssh://[email protected]/isaac-for-healthcare/[email protected] \
-    --extra-index-url https://pypi.nvidia.com
-
-
-echo "Installing IsaacLab ..."
-yes Yes | ./isaaclab.sh --install
-popd
-
-
-# ---- Install robotic.surgery.assets and robotic.surgery.tasks ----
-echo "Installing robotic.surgery.assets and robotic.surgery.tasks..."
-pip install -e $PROJECT_ROOT/workflows/robotic_surgery/scripts/simulation/exts/robotic.surgery.assets
-pip install -e $PROJECT_ROOT/workflows/robotic_surgery/scripts/simulation/exts/robotic.surgery.tasks
-
-
-# ---- Install libstdcxx-ng for mesa ----
-echo "Installing libstdcxx-ng..."
-conda install -c conda-forge libstdcxx-ng=13.2.0 -y
-
-echo "Dependencies installed successfully!"
+echo "All dependencies installed successfully!"

workflows/robotic_surgery/docker/Dockerfile

@@ -0,0 +1,46 @@
+# syntax=docker/dockerfile:1.4
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM nvcr.io/nvidia/isaac-lab:2.0.2
+
+# This line enables SSH access for the following RUN command
+RUN apt-get update && apt-get install -y git openssh-client
+
+# Add GitHub to known hosts to avoid "host key verification failed"
+RUN mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
+
+WORKDIR /workspace/robotic_surgery/scripts
+
+# Fix livestream public endpoint address issue in 2.0.2/2.1.0
+RUN sed -i '/--\/app\/livestream\/publicEndpointAddress=/d' /workspace/isaaclab/source/isaaclab/isaaclab/app/app_launcher.py
+
+COPY workflows/robotic_surgery/scripts .
+
+COPY tools/env_setup/install_robotic_surgery_extensions.sh /tmp/install_robotic_surgery_extensions.sh
+
+ENV PYTHONPATH=/workspace/robotic_surgery/scripts
+
+ENV PYTHON_EXECUTABLE=/workspace/isaaclab/_isaac_sim/python.sh
+
+RUN --mount=type=ssh $PYTHON_EXECUTABLE -m pip install --no-deps git+ssh://[email protected]/isaac-for-healthcare/[email protected]
+
+# Install the robotic surgery extensions
+# Extension installation also needs to know which python to use via the PYTHON_EXECUTABLE environment variable
+RUN /tmp/install_robotic_surgery_extensions.sh simulation/exts
+
+# Clean up the install script
+RUN rm /tmp/install_robotic_surgery_extensions.sh

workflows/robotic_surgery/docker/README.md

@@ -0,0 +1,56 @@
+# Simulation LiveStream from Remote Docker Container
+
+This document describes how to run the simulation in a remote docker container and stream the simulation to a local machine.
+
+## Prerequisites
+
+Please refer to [Livestream Clients Guide in Isaac Sim](https://docs.isaacsim.omniverse.nvidia.com/latest/installation/manual_livestream_clients.html#isaac-sim-short-webrtc-streaming-client) and download [Isaac Sim WebRTC Streamiing Client](https://docs.isaacsim.omniverse.nvidia.com/latest/installation/download.html#isaac-sim-latest-release)
+
+## Build Docker Image
+
+To build the docker image, you will need to set up the SSH agent and add your SSH key to the agent, so that the docker build process can access the private repository.
+
+```bash
+export DOCKER_BUILDKIT=1
+eval "$(ssh-agent -s)"
+ssh-add ~/.ssh/id_ed25519  # Replace with your SSH key
+docker build --ssh default -f workflows/robotic_surgery/docker/Dockerfile -t robotic_surgery:latest .
+```
+
+## Prepare the I4H Asset Locally
+
+Please refer to the [Environment Setup](../README.md#environment-setup) for instructions to prepare the I4H assets locally.
+
+This will create a directory `~/.cache/i4h-assets/<sha256>` containing the I4H assets, which will be mounted to the docker container.
+
+## Run the Container
+
+```bash
+docker run --name isaac-sim --entrypoint bash -it --runtime=nvidia --gpus all -e "ACCEPT_EULA=Y" --rm --network=host \
+    -e "PRIVACY_CONSENT=Y" \
+    -v ~/docker/isaac-sim/cache/kit:/isaac-sim/kit/cache:rw \
+    -v ~/docker/isaac-sim/cache/ov:/root/.cache/ov:rw \
+    -v ~/docker/isaac-sim/cache/pip:/root/.cache/pip:rw \
+    -v ~/docker/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw \
+    -v ~/docker/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw \
+    -v ~/docker/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw \
+    -v ~/docker/isaac-sim/data:/root/.local/share/ov/data:rw \
+    -v ~/docker/isaac-sim/documents:/root/Documents:rw \
+    -v ~/.cache/i4h-assets:/root/.cache/i4h-assets:rw \
+    robotic_surgery:latest
+```
+
+### Run the Simulation
+
+In the container, run the simulation with `--livestream 2` to stream the simulation to the local machine. For example, to run the `reach_psm_sm.py` script, run the following command:
+
+```bash
+python simulation/scripts/environments/state_machine/reach_psm_sm.py --livestream 2
+```
+
+Please refer to the [simulation README](../scripts/simulation/README.md) for other examples.
+
+
+### Open the WebRTC Client
+
+Wait for the simulation to start (you should see the message "Resetting the state machine" when the simulation is ready), and open the WebRTC client to view the simulation.

workflows/robotic_surgery/scripts/simulation/scripts/environments/state_machine/lift_block_sm.py

@@ -33,7 +33,7 @@
 args_cli = parser.parse_args()
 
 # launch omniverse app
-app_launcher = AppLauncher(headless=args_cli.headless)
+app_launcher = AppLauncher(headless=args_cli.headless, livestream=args_cli.livestream)
 simulation_app = app_launcher.app
 
 """Rest everything else."""

workflows/robotic_surgery/scripts/simulation/scripts/environments/state_machine/lift_needle_organs_sm.py

@@ -33,7 +33,7 @@
 args_cli = parser.parse_args()
 
 # launch omniverse app
-app_launcher = AppLauncher(headless=args_cli.headless)
+app_launcher = AppLauncher(headless=args_cli.headless, livestream=args_cli.livestream)
 simulation_app = app_launcher.app
 
 """Rest everything else."""

workflows/robotic_surgery/scripts/simulation/scripts/environments/state_machine/lift_needle_sm.py

@@ -33,7 +33,7 @@
 args_cli = parser.parse_args()
 
 # launch omniverse app
-app_launcher = AppLauncher(headless=args_cli.headless)
+app_launcher = AppLauncher(headless=args_cli.headless, livestream=args_cli.livestream)
 simulation_app = app_launcher.app
 
 """Rest everything else."""

workflows/robotic_surgery/scripts/simulation/scripts/environments/state_machine/reach_dual_psm_sm.py

@@ -33,7 +33,7 @@
 args_cli = parser.parse_args()
 
 # launch omniverse app
-app_launcher = AppLauncher(headless=args_cli.headless)
+app_launcher = AppLauncher(headless=args_cli.headless, livestream=args_cli.livestream)
 simulation_app = app_launcher.app
 
 """Rest everything else."""

workflows/robotic_surgery/scripts/simulation/scripts/environments/state_machine/reach_psm_sm.py

@@ -33,7 +33,7 @@
 args_cli = parser.parse_args()
 
 # launch omniverse app
-app_launcher = AppLauncher(headless=args_cli.headless)
+app_launcher = AppLauncher(headless=args_cli.headless, livestream=args_cli.livestream)
 simulation_app = app_launcher.app
 
 """Rest everything else."""

workflows/robotic_surgery/scripts/simulation/scripts/environments/state_machine/reach_star_sm.py

@@ -33,7 +33,7 @@
 args_cli = parser.parse_args()
 
 # launch omniverse app
-app_launcher = AppLauncher(headless=args_cli.headless)
+app_launcher = AppLauncher(headless=args_cli.headless, livestream=args_cli.livestream)
 simulation_app = app_launcher.app
 
 """Rest everything else."""

workflows/robotic_ultrasound/tests/test_simulation/test_integration_pi0_eval.py

@@ -21,7 +21,7 @@
 SM_CASES = [
     (
         "python -u -m simulation.imitation_learning.pi0_policy.eval --enable_camera --headless",
-        300,
+        360,
         "Resetting the environment.",
     ),
 ]