isaac-for-healthcare · mingxin-zheng · Jul 8, 2025 · Jul 7, 2025 · Jul 8, 2025 · Jul 8, 2025
diff --git a/tools/env_setup/install_isaac.sh b/tools/env_setup/install_isaac.sh
@@ -35,7 +35,7 @@ mkdir -p "$PROJECT_ROOT/third_party"
 # ---- Install IsaacSim ----
 echo "Installing IsaacSim..."
 pip install 'isaacsim[all,extscache]==4.5.0' \
-    git+ssh://[email protected]/isaac-for-healthcare/[email protected].0rc1 \
+    git+ssh://[email protected]/isaac-for-healthcare/[email protected].0rc2 \
     --extra-index-url https://pypi.nvidia.com
 
 ISAACLAB_DIR="$PROJECT_ROOT/third_party/IsaacLab"

diff --git a/workflows/robotic_surgery/README.md b/workflows/robotic_surgery/README.md
@@ -89,7 +89,7 @@ Download the required assets using:
 i4h-asset-retrieve
 ```
 
-This will download assets to `~/.cache/i4h-assets/<sha256>`. For more details, refer to the [Asset Container Helper](https://github.com/isaac-for-healthcare/i4h-asset-catalog/blob/v0.2.0rc1/docs/catalog_helper.md).
+This will download assets to `~/.cache/i4h-assets/<sha256>`. For more details, refer to the [Asset Container Helper](https://github.com/isaac-for-healthcare/i4h-asset-catalog/blob/v0.2.0rc2/docs/catalog_helper.md).
 
 ### Environment Variables
 

diff --git a/workflows/robotic_surgery/docker/Dockerfile b/workflows/robotic_surgery/docker/Dockerfile
@@ -15,42 +15,69 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM nvcr.io/nvidia/isaac-lab:2.1.0
-
-# Apply patches to IsaacLab
-COPY tools/env_setup/patches/events_random_texture.patch /tmp/
-COPY tools/env_setup/patches/from_files_semantic_tags.patch /tmp/
-
-RUN cd /workspace/isaaclab && \
-    patch -p1 < /tmp/events_random_texture.patch && \
-    patch -p1 < /tmp/from_files_semantic_tags.patch && \
-    rm /tmp/events_random_texture.patch /tmp/from_files_semantic_tags.patch
-
-
-# This line enables SSH access for the following RUN command
-RUN apt-get update && apt-get install -y git openssh-client
-
-# Add GitHub to known hosts to avoid "host key verification failed"
-RUN mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
-
-WORKDIR /workspace/robotic_surgery/scripts
-
-# Fix livestream public endpoint address issue in 2.0.2/2.1.0
-RUN sed -i '/--\/app\/livestream\/publicEndpointAddress=/d' /workspace/isaaclab/source/isaaclab/isaaclab/app/app_launcher.py
-
-COPY workflows/robotic_surgery/scripts .
-
-COPY tools/env_setup/install_robotic_surgery_extensions.sh /tmp/install_robotic_surgery_extensions.sh
-
-ENV PYTHONPATH=/workspace/robotic_surgery/scripts
-
-ENV PYTHON_EXECUTABLE=/workspace/isaaclab/_isaac_sim/python.sh
-
-RUN --mount=type=ssh $PYTHON_EXECUTABLE -m pip install --no-deps git+ssh://[email protected]/isaac-for-healthcare/[email protected]
-
-# Install the robotic surgery extensions
-# Extension installation also needs to know which python to use via the PYTHON_EXECUTABLE environment variable
-RUN /tmp/install_robotic_surgery_extensions.sh simulation/exts
-
-# Clean up the install script
-RUN rm /tmp/install_robotic_surgery_extensions.sh
+FROM nvidia/cuda:12.8.1-devel-ubuntu24.04
+
+SHELL ["/bin/bash", "-c"]
+
+# Install all packages in a single layer to avoid caching issues
+RUN apt-get update && \
+    apt-get install -y \
+        wget \
+        curl \
+        jq \
+        vim \
+        git \
+        xvfb \
+        build-essential \
+        cmake \
+        vulkan-tools \
+        unzip \
+        lsb-release \
+        libglib2.0-0 \
+        libdbus-1-3 \
+        libopengl0 \
+        libxcb-keysyms1 \
+        libglu1-mesa && \
+    rm -rf /var/lib/apt/lists/*
+
+# Add github.com to list of known hosts for git clone with ssh
+# use COPY command instead
+RUN mkdir -p ~/.ssh && \
+    curl --silent https://api.github.com/meta \
+        | jq --raw-output '"github.com "+.ssh_keys[]' >> ~/.ssh/known_hosts
+#RUN mkdir -p ~/.ssh
+#COPY ./github /root/.ssh/known_hosts
+
+# all devices should be visible
+ENV NVIDIA_VISIBLE_DEVICES=all
+# set 'compute' driver cap to use Cuda
+# set 'video' driver cap to use the video encoder
+# set 'graphics' driver cap to use OpenGL/EGL
+# set 'display' to allow use of virtual display
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,video,compute,utility,display
+
+# Install Everything in conda environment
+
+WORKDIR /workspace/i4h-workflows
+
+COPY tools /workspace/i4h-workflows/tools
+COPY workflows /workspace/i4h-workflows/workflows
+
+# Install miniconda3 and create robotic_surgery
+RUN --mount=type=ssh \
+    mkdir -p ~/miniconda3 && \
+    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh && \
+    bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3 && \
+    rm ~/miniconda3/miniconda.sh && \
+    source ~/miniconda3/bin/activate && \
+    conda init --all && \
+    conda create -n robotic_surgery python=3.10.14 -y
+
+RUN --mount=type=ssh \
+    source ~/miniconda3/bin/activate && \
+    conda activate robotic_surgery && \
+    cd /workspace/i4h-workflows && \
+    BUILD_DOCKER_IMAGE=true bash tools/env_setup_robot_surgery.sh
+
+ENV PYTHONPATH=/workspace/i4h-workflows/workflows/robotic_surgery/scripts
+ENV RTI_LICENSE_FILE=/root/rti/rti_license.dat
diff --git a/workflows/robotic_surgery/docker/README.md b/workflows/robotic_surgery/docker/README.md
@@ -1,27 +1,48 @@
-# Simulation LiveStream from Remote Docker Container
+# Robotic Surgery Docker Container
 
-This document describes how to run the simulation in a remote docker container and stream the simulation to a local machine.
+This guide provides instructions for running robotic surgery simulations using Docker containers with Isaac Sim.
 
 ## Prerequisites
 
-Please refer to [Livestream Clients Guide in Isaac Sim](https://docs.isaacsim.omniverse.nvidia.com/latest/installation/manual_livestream_clients.html#isaac-sim-short-webrtc-streaming-client) and download [Isaac Sim WebRTC Streaming Client](https://docs.isaacsim.omniverse.nvidia.com/latest/installation/download.html#isaac-sim-latest-release)
+- **Docker Engine**
+- **NVIDIA Docker Runtime**
+- **Git** with SSH key access to private repositories
+- **X11 forwarding** support (for GUI mode)
 
-## Build Docker Image
+## Build the Docker Image
 
-To build the docker image, you will need to set up the SSH agent and add your SSH key to the agent, so that the docker build process can access the private repository.
+```sh
+# Clone the repository
+git clone https://github.com/isaac-for-healthcare/i4h-workflows.git
+cd i4h-workflows
 
-```bash
+# Enable BuildKit
 export DOCKER_BUILDKIT=1
+
+# Set up SSH agent for private repository access
 eval "$(ssh-agent -s)"
-ssh-add ~/.ssh/id_ed25519  # Replace with your SSH key
+ssh-add ~/.ssh/id_ed25519  # Replace with your SSH key path
+
+# Build the Docker image with SSH forwarding
 docker build --ssh default -f workflows/robotic_surgery/docker/Dockerfile -t robotic_surgery:latest .
 ```
 
-## Run the Container
+## Running the Container
 
-```bash
-docker run --name isaac-sim --entrypoint bash -it --runtime=nvidia --gpus all -e "ACCEPT_EULA=Y" --rm --network=host \
+```sh
+# Allow Docker to access X11 display
+xhost +local:docker
+
+# Run container with GUI support
+docker run --name isaac-sim -it --gpus all --rm \
+    --network=host \
+    --runtime=nvidia \
+    --entrypoint=bash \
+    -e DISPLAY=$DISPLAY \
+    -e "OMNI_KIT_ACCEPT_EULA=Y" \
+    -e "ACCEPT_EULA=Y" \
     -e "PRIVACY_CONSENT=Y" \
+    -v /tmp/.X11-unix:/tmp/.X11-unix \
     -v ~/docker/isaac-sim/cache/kit:/isaac-sim/kit/cache:rw \
     -v ~/docker/isaac-sim/cache/ov:/root/.cache/ov:rw \
     -v ~/docker/isaac-sim/cache/pip:/root/.cache/pip:rw \
@@ -34,19 +55,61 @@ docker run --name isaac-sim --entrypoint bash -it --runtime=nvidia --gpus all -e
     robotic_surgery:latest
 ```
 
-### Run the Simulation
+## Running the Simulation
+
+### 1. Interactive GUI Mode with X11 Forwarding
+
+For interactive development and debugging:
+
+```bash
+# Inside the container
+conda activate robotic_surgery
+
+# Run simulation with GUI
+python workflows/robotic_surgery/scripts/simulation/scripts/environments/state_machine/reach_psm_sm.py
+```
+
+Please refer to the [Simulation README](../scripts/simulation/README.md) for more details.
+
+### 2. Headless Streaming Mode
 
-In the container, run the simulation with `--livestream 2` to stream the simulation to the local machine. For example, to run the `reach_psm_sm.py` script, run the following command:
+For remote access:
 
 ```bash
-docker exec -it isaac-sim bash
-# Inside the container, run the simulation
-python simulation/scripts/environments/state_machine/reach_psm_sm.py --livestream 2
+# Inside the container
+conda activate robotic_surgery
+
+# Run simulation with WebRTC streaming
+python workflows/robotic_surgery/scripts/simulation/scripts/environments/state_machine/reach_psm_sm.py --livestream 2
 ```
 
-Please refer to the [simulation README](../scripts/simulation/README.md) for other examples.
+#### WebRTC Client Setup
+
+1. **Download the Isaac Sim WebRTC Client**:
+   - Visit the [Isaac Sim Download Page](https://docs.isaacsim.omniverse.nvidia.com/4.5.0/installation/download.html)
+   - Download the **Isaac Sim WebRTC Streaming Client**
 
+2. **Configure Connection**:
+   - Open the WebRTC client
+   - Enter the server IP address (container host IP)
+   - Wait for the simulation to initialize (look for "Resetting the state machine" message)
+   - Click "Connect"
 
-### Open the WebRTC Client
+3. **Network Requirements**:
+   - Ensure ports are accessible: `TCP/UDP 47995-48012`, `TCP/UDP 49000-49007`, `TCP 49100`
+   - For remote access, configure firewall rules accordingly
 
-Wait for the simulation to start (you should see the message "Resetting the state machine" when the simulation is ready), and open the WebRTC client to view the simulation.
+## Troubleshooting
+
+### Common Issues
+
+**Display Issues**:
+```bash
+# Reset X11 permissions
+xhost -local:docker
+xhost +local:docker
+
+# Check DISPLAY variable
+echo $DISPLAY
+```
+This value should be set for interactive mode.
diff --git a/workflows/robotic_ultrasound/README.md b/workflows/robotic_ultrasound/README.md
@@ -111,8 +111,8 @@ conda activate robotic_ultrasound
 
 ##### Install Raytracing Ultrasound Simulator
 Choose one of the following options:
-- **(Experimental)** Download the pre-release version from [here](https://github.com/isaac-for-healthcare/i4h-sensor-simulation/releases/tag/v0.2.0rc1) and extract to `workflows/robotic_ultrasound/scripts/raysim`
-- **(Recommended)** Install and build following instructions in [Raytracing Ultrasound Simulator](https://github.com/isaac-for-healthcare/i4h-sensor-simulation/tree/main/ultrasound-raytracing#installation)
+- **(Use pre-built binary)** Download the pre-release version from [here](https://github.com/isaac-for-healthcare/i4h-sensor-simulation/releases/tag/v0.2.0rc2) and extract to `workflows/robotic_ultrasound/scripts/raysim`
+- **(Compiling from source)** Install and build following instructions in [Raytracing Ultrasound Simulator](https://github.com/isaac-for-healthcare/i4h-sensor-simulation/tree/main/ultrasound-raytracing#installation)
 
 ##### Install All Dependencies
 The main script `tools/env_setup_robot_us.sh` installs all necessary dependencies. It first installs common base components and then policy-specific packages based on an argument.
@@ -154,7 +154,7 @@ Download the required assets using:
 i4h-asset-retrieve
 ```
 
-This will download assets to `~/.cache/i4h-assets/<sha256>`. For more details, refer to the [Asset Container Helper](https://github.com/isaac-for-healthcare/i4h-asset-catalog/blob/v0.2.0rc1/docs/catalog_helper.md).
+This will download assets to `~/.cache/i4h-assets/<sha256>`. For more details, refer to the [Asset Container Helper](https://github.com/isaac-for-healthcare/i4h-asset-catalog/blob/v0.2.0rc2/docs/catalog_helper.md).
 
 **Note**: During asset download, you may see warnings about blocking functions. This is expected behavior and the download will complete successfully despite these warnings.