Merge branch 'main' into chhwang/local-channel

Author: chhwang

.azure-pipelines/templates/nccl-test.yaml

@@ -79,24 +79,24 @@ steps:
       parallel-scp -t 0 -r -h ${HOSTFILE} -x "-i ${KeyFilePath}" -O $SSH_OPTION ${ROOT_DIR} ${DST_DIR}
     workingDirectory: '$(System.DefaultWorkingDirectory)'
 
-- task: Bash@3
-  name: GenerateExecutionFile
-  displayName: Generate execution file
-  inputs:
-    targetType: 'inline'
-    script: |
-      set -e
-      HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
-      ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
-      SSH_OPTION="StrictHostKeyChecking=no"
-      KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-      parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
-        -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
-        cd /root/mscclpp/msccl-users;  \
-        mkdir -p execution-files;      \
-        cd /root/mscclpp/msccl-users;  \
-        bash algos/mscclpp_a100/generate_execution_plan.sh"'
-    workingDirectory: '$(System.DefaultWorkingDirectory)'
+# - task: Bash@3
+#   name: GenerateExecutionFile
+#   displayName: Generate execution file
+#   inputs:
+#     targetType: 'inline'
+#     script: |
+#       set -e
+#       HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
+#       ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
+#       SSH_OPTION="StrictHostKeyChecking=no"
+#       KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
+#       parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
+#         -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
+#         cd /root/mscclpp/msccl-users;  \
+#         mkdir -p execution-files;      \
+#         cd /root/mscclpp/msccl-users;  \
+#         bash algos/mscclpp_a100/generate_execution_plan.sh"'
+#     workingDirectory: '$(System.DefaultWorkingDirectory)'
 
 - task: Bash@3
   name: InstallNcclTests
@@ -116,56 +116,56 @@ steps:
         MPI=1 MPI_HOME=/usr/local/mpi make -j"'
     workingDirectory: '$(System.DefaultWorkingDirectory)'
 
-- task: Bash@3
-  name: RunNcclAllReduceTest
-  displayName: Run NCCL AllReduce Test
-  inputs:
-    targetType: inline
-    script: |
-      set -e
-      HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
-      ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
-      SSH_OPTION="StrictHostKeyChecking=no"
-      KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-      parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
-        -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
-        cd /root/mscclpp;                                         \
-        mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN  -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/all_reduce_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20"'
-    workingDirectory: '$(System.DefaultWorkingDirectory)'
+# - task: Bash@3
+#   name: RunNcclAllReduceTest
+#   displayName: Run NCCL AllReduce Test
+#   inputs:
+#     targetType: inline
+#     script: |
+#       set -e
+#       HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
+#       ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
+#       SSH_OPTION="StrictHostKeyChecking=no"
+#       KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
+#       parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
+#         -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
+#         cd /root/mscclpp;                                         \
+#         mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN  -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/all_reduce_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20"'
+#     workingDirectory: '$(System.DefaultWorkingDirectory)'
 
-- task: Bash@3
-  name: RunNcclAllGatherTest
-  displayName: Run NCCL AllGather Test
-  inputs:
-    targetType: inline
-    script: |
-      set -e
-      HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
-      ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
-      SSH_OPTION="StrictHostKeyChecking=no"
-      KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-      parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
-        -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
-        cd /root/mscclpp;                                         \
-        mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN  -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/all_gather_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20"'
-    workingDirectory: '$(System.DefaultWorkingDirectory)'
+# - task: Bash@3
+#   name: RunNcclAllGatherTest
+#   displayName: Run NCCL AllGather Test
+#   inputs:
+#     targetType: inline
+#     script: |
+#       set -e
+#       HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
+#       ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
+#       SSH_OPTION="StrictHostKeyChecking=no"
+#       KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
+#       parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
+#         -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
+#         cd /root/mscclpp;                                         \
+#         mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN  -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/all_gather_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20"'
+#     workingDirectory: '$(System.DefaultWorkingDirectory)'
 
-- task: Bash@3
-  name: RunNcclReduceScatterTest
-  displayName: Run NCCL Reduce Scatter Test
-  inputs:
-    targetType: inline
-    script: |
-      set -e
-      HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
-      ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
-      SSH_OPTION="StrictHostKeyChecking=no"
-      KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-      parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
-        -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
-        cd /root/mscclpp;                                         \
-        mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN  -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20"'
-    workingDirectory: '$(System.DefaultWorkingDirectory)'
+# - task: Bash@3
+#   name: RunNcclReduceScatterTest
+#   displayName: Run NCCL Reduce Scatter Test
+#   inputs:
+#     targetType: inline
+#     script: |
+#       set -e
+#       HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
+#       ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
+#       SSH_OPTION="StrictHostKeyChecking=no"
+#       KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
+#       parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
+#         -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
+#         cd /root/mscclpp;                                         \
+#         mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN  -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20"'
+#     workingDirectory: '$(System.DefaultWorkingDirectory)'
 
 - task: Bash@3
   name: InstallNccl
@@ -245,25 +245,25 @@ steps:
         mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN -x MSCCLPP_ENABLE_NCCL_FALLBACK=TRUE -x MSCCLPP_NCCL_LIB_PATH=/root/nccl/build/lib/libnccl.so -x MSCCLPP_FORCE_NCCL_FALLBACK_OPERATION="allreduce" /root/nccl-tests/build/broadcast_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20"'
     workingDirectory: '$(System.DefaultWorkingDirectory)'
 
-- task: Bash@3
-  name: RunNcclReduceScatterFallbaclkToNcclTest
-  displayName: Run NCCL ReduceScatter Test with or without Fallback to NCCL operation
-  inputs:
-    targetType: 'inline'
-    script: |
-      set -e
-      HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
-      ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
-      SSH_OPTION="StrictHostKeyChecking=no"
-      KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
-      parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
-        -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
-        cd /root/mscclpp;                                         \
-        echo \"mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN -x MSCCLPP_ENABLE_NCCL_FALLBACK=TRUE -x MSCCLPP_NCCL_LIB_PATH=/root/nccl/build/lib/libnccl.so -x MSCCLPP_FORCE_NCCL_FALLBACK_OPERATION=\"reducescatter\" /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20\";                                                                 \
-        mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN -x MSCCLPP_ENABLE_NCCL_FALLBACK=TRUE -x MSCCLPP_NCCL_LIB_PATH=/root/nccl/build/lib/libnccl.so -x MSCCLPP_FORCE_NCCL_FALLBACK_OPERATION="reducescatter" /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20;                                                                            \
-        echo \"mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN -x MSCCLPP_ENABLE_NCCL_FALLBACK=TRUE -x MSCCLPP_NCCL_LIB_PATH=/root/nccl/build/lib/libnccl.so -x MSCCLPP_FORCE_NCCL_FALLBACK_OPERATION=\"broadcast\" -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20\"; \
-        mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN -x MSCCLPP_ENABLE_NCCL_FALLBACK=TRUE -x MSCCLPP_NCCL_LIB_PATH=/root/nccl/build/lib/libnccl.so -x MSCCLPP_FORCE_NCCL_FALLBACK_OPERATION="broadcast" -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20"'
-    workingDirectory: '$(System.DefaultWorkingDirectory)'
+# - task: Bash@3
+#   name: RunNcclReduceScatterFallbaclkToNcclTest
+#   displayName: Run NCCL ReduceScatter Test with or without Fallback to NCCL operation
+#   inputs:
+#     targetType: 'inline'
+#     script: |
+#       set -e
+#       HOSTFILE=$(System.DefaultWorkingDirectory)/mscclpp/test/deploy/hostfile_ci
+#       ROOT_DIR=$(System.DefaultWorkingDirectory)/mscclpp
+#       SSH_OPTION="StrictHostKeyChecking=no"
+#       KeyFilePath=${SSHKEYFILE_SECUREFILEPATH}
+#       parallel-ssh -i -t 0 -h ${HOSTFILE} -x "-i ${KeyFilePath}"  \
+#         -O $SSH_OPTION 'sudo docker exec -t mscclpp-test bash -c "\
+#         cd /root/mscclpp;                                         \
+#         echo \"mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN -x MSCCLPP_ENABLE_NCCL_FALLBACK=TRUE -x MSCCLPP_NCCL_LIB_PATH=/root/nccl/build/lib/libnccl.so -x MSCCLPP_FORCE_NCCL_FALLBACK_OPERATION=\"reducescatter\" /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20\";                                                                 \
+#         mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN -x MSCCLPP_ENABLE_NCCL_FALLBACK=TRUE -x MSCCLPP_NCCL_LIB_PATH=/root/nccl/build/lib/libnccl.so -x MSCCLPP_FORCE_NCCL_FALLBACK_OPERATION="reducescatter" /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20;                                                                            \
+#         echo \"mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN -x MSCCLPP_ENABLE_NCCL_FALLBACK=TRUE -x MSCCLPP_NCCL_LIB_PATH=/root/nccl/build/lib/libnccl.so -x MSCCLPP_FORCE_NCCL_FALLBACK_OPERATION=\"broadcast\" -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20\"; \
+#         mpirun -np 8 --bind-to numa --allow-run-as-root -x LD_PRELOAD=/root/mscclpp/build/apps/nccl/libmscclpp_nccl.so -x NCCL_DEBUG=WARN -x MSCCLPP_ENABLE_NCCL_FALLBACK=TRUE -x MSCCLPP_NCCL_LIB_PATH=/root/nccl/build/lib/libnccl.so -x MSCCLPP_FORCE_NCCL_FALLBACK_OPERATION="broadcast" -x MSCCLPP_EXECUTION_PLAN_DIR=/root/mscclpp/msccl-users/execution-files /root/nccl-tests/build/reduce_scatter_perf -b 1K -e 1G -f 2 -d half -G 20 -w 10 -n 20"'
+#     workingDirectory: '$(System.DefaultWorkingDirectory)'
 
 - task: AzureCLI@2
   name: StopVMSS

.devcontainer/Dockerfile

@@ -1,13 +1,55 @@
 ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
-ARG USERNAME=mscclpp
+ARG USERNAME=devuser
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
+ARG SSH_PORT=22345
 
-# Create the user
-RUN groupadd --gid $USER_GID $USERNAME && \
-    useradd --uid $USER_UID --gid $USER_GID -m $USERNAME && \
-    echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
-    chmod 0440 /etc/sudoers.d/$USERNAME
+# Create or modify the user
+RUN if getent group $USER_GID > /dev/null; then \
+        EXISTING_GROUP=$(getent group $USER_GID | cut -d: -f1); \
+        if [ "$EXISTING_GROUP" != "$USERNAME" ]; then \
+            groupmod -n $USERNAME $EXISTING_GROUP; \
+        fi; \
+    else \
+        groupadd --gid $USER_GID $USERNAME; \
+    fi && \
+    if id -u $USER_UID > /dev/null 2>&1; then \
+        EXISTING_USER=$(getent passwd $USER_UID | cut -d: -f1); \
+        if [ "$EXISTING_USER" != "$USERNAME" ]; then \
+            usermod -l $USERNAME -d /home/$USERNAME -m $EXISTING_USER; \
+        fi; \
+    else \
+        useradd --uid $USER_UID --gid $USER_GID -m $USERNAME; \
+    fi && \
+    usermod -g $USERNAME $USERNAME && \
+    echo "$USERNAME ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+RUN rm -rf /etc/apt/sources.list.d/cuda-* && \
+    apt-get update && \
+    apt install -y --no-install-recommends \
+        clang-format \
+        openssh-server \
+        gdb \
+        doxygen \
+        graphviz \
+        && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+RUN python3 -m pip install --no-cache-dir \
+    black \
+    pytest \
+    breathe \
+    sphinx_rtd_theme \
+    myst_parser \
+    sphinxcontrib.mermaid
+
+RUN sed -i "s/^Port 22/Port ${SSH_PORT}/" /etc/ssh/sshd_config && \
+    mkdir -p /home/$USERNAME/.ssh && \
+    ssh-keygen -t rsa -f /home/$USERNAME/.ssh/id_rsa -N "" -q && \
+    cat /home/$USERNAME/.ssh/id_rsa.pub >> /home/$USERNAME/.ssh/authorized_keys && \
+    chown -R $USERNAME:$USERNAME /home/$USERNAME/.ssh
 
 USER $USERNAME

.devcontainer/devcontainer.json

@@ -3,10 +3,17 @@
     "build": {
         "dockerfile": "Dockerfile",
         "args": {
-            "BASE_IMAGE": "ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda12.8"
+            "BASE_IMAGE": "ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda12.9",
+            "USERNAME": "devuser",
+            "SSH_PORT": "22345"
         }
     },
-    "remoteUser": "mscclpp",
+    "remoteUser": "devuser",
+    "containerEnv": {
+        "LC_ALL": "C",
+        "LANG": "C",
+        "LANGUAGE": "C"
+    },
     "customizations": {
         "vscode": {
             "extensions": [
@@ -15,18 +22,30 @@
                 "ms-python.vscode-pylance",
                 // C++
                 "ms-vscode.cpptools",
-                "ms-vscode.cpptools-extension-pack",
                 "ms-vscode.cmake-tools"
-            ]
+            ],
+            "settings": {
+                "terminal.integrated.defaultProfile.linux": "bash",
+                "C_Cpp.default.includePath": [
+                    "${workspaceFolder}/**",
+                    "/usr/local/cuda/include",
+                    "/usr/include"
+                ],
+                "C_Cpp.default.cStandard": "c17",
+                "C_Cpp.default.cppStandard": "c++17"
+            }
         }
     },
     "privileged": true,
     "runArgs": [
+        "--cap-add=SYS_PTRACE",
         "--net=host",
         "--ipc=host",
-        "--gpus=all",
-        "--ulimit=memlock=-1:-1"
+        "--ulimit=memlock=-1:-1",
+        "--gpus=all"
     ],
-    "workspaceFolder": "/home/mscclpp/mscclpp",
-    "workspaceMount": "source=${localWorkspaceFolder},target=/home/mscclpp/mscclpp,type=bind,consistency=cached"
+    "workspaceFolder": "/home/devuser/mscclpp",
+    "workspaceMount": "source=${localWorkspaceFolder},target=/home/devuser/mscclpp,type=bind,consistency=cached",
+    "postStartCommand": "sudo service ssh start",
+    "postCreateCommand": "bash /home/devuser/mscclpp/tools/install.sh nvidia /usr"
 }

.devcontainer/devcontainer_amd.json

@@ -0,0 +1,54 @@
+{
+    "name": "MSCCL++ Dev Container",
+    "build": {
+        "dockerfile": "Dockerfile",
+        "args": {
+            "BASE_IMAGE": "ghcr.io/microsoft/mscclpp/mscclpp:base-dev-rocm6.2",
+            "USERNAME": "devuser",
+            "SSH_PORT": "22345"
+        }
+    },
+    "remoteUser": "devuser",
+    "containerEnv": {
+        "LC_ALL": "C",
+        "LANG": "C",
+        "LANGUAGE": "C"
+    },
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                // Python
+                "ms-python.python",
+                "ms-python.vscode-pylance",
+                // C++
+                "ms-vscode.cpptools",
+                "ms-vscode.cmake-tools"
+            ],
+            "settings": {
+                "terminal.integrated.defaultProfile.linux": "bash",
+                "C_Cpp.default.includePath": [
+                    "${workspaceFolder}/**",
+                    "/opt/rocm/include",
+                    "/usr/include"
+                ],
+                "C_Cpp.default.cStandard": "c17",
+                "C_Cpp.default.cppStandard": "c++17"
+            }
+        }
+    },
+    "privileged": true,
+    "runArgs": [
+        "--cap-add=SYS_PTRACE",
+        "--net=host",
+        "--ipc=host",
+        "--ulimit=memlock=-1:-1",
+        "--security-opt=seccomp=unconfined",
+        "--group-add=video",
+        "--device=/dev/kfd",
+        "--device=/dev/dri"
+    ],
+    "workspaceFolder": "/home/devuser/mscclpp",
+    "workspaceMount": "source=${localWorkspaceFolder},target=/home/devuser/mscclpp,type=bind,consistency=cached",
+    "postStartCommand": "sudo service ssh start",
+    "postCreateCommand": "bash /home/devuser/mscclpp/tools/install.sh amd /usr"
+}

.github/instructions/dsl-api-document.instructions.md

@@ -0,0 +1,23 @@
+---
+applyTo: 'python/mscclpp/language/*.py'
+---
+
+# Instructions for DSL API Documentation
+
+## Overview
+The MSCCL++ DSL (Domain Specific Language) provides a Python API for defining distributed GPU communication patterns. All API functions should have comprehensive Google-style docstrings.
+
+## Documentation Requirements
+- Add google-style docstrings to the DSL API functions in the `mscclpp.language` package.
+- Ensure that each function's docstring includes:
+  - A brief description of what the function does.
+  - Parameters with their types and descriptions.
+  - Return type and description.
+  - Any exceptions raised by the function, if applicable.
+  - Usage examples where appropriate.
+
+## Implementation Steps
+1. Open each Python file in the `python.mscclpp.language` folder, exclude `__init__.py` and internal folders.
+2. For each function in the file, add a Google-style docstring that follows the documentation requirements outlined above.
+3. Ensure that the docstrings are clear, concise, and accurately describe the function's behavior.
+4. Review the docstrings for consistency in style and formatting.