Adding files to deploy DocSum application on ROCm vLLM

DocSum/Dockerfile-vllm-rocm

@@ -0,0 +1,18 @@
+# Copyright (c) 2024 Advanced Micro Devices, Inc.
+
+FROM rocm/vllm:rocm6.3.1_mi300_ubuntu22.04_py3.12_vllm_0.6.6
+
+# Set the working directory
+WORKDIR /workspace
+
+# Expose the port used by the API server
+EXPOSE 8011
+
+# Set environment variables
+ENV HUGGINGFACE_HUB_CACHE=/workspace
+ENV VLLM_USE_TRITON_FLASH_ATTENTION=0
+ENV PYTORCH_JIT=0
+
+# Set the entrypoint to the api_server.py script
+RUN cp /usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py /workspace/api_server.py
+ENTRYPOINT ["python3", "/workspace/api_server.py"]

DocSum/docker_compose/amd/gpu/rocm/README.md

@@ -1,3 +1,5 @@
+Copyright (C) 2024 Advanced Micro Devices, Inc.
+
 # Build and deploy DocSum Application on AMD GPU (ROCm)
 
 ## Build images

DocSum/docker_compose/amd/gpu/rocm/README_vllm.md

@@ -0,0 +1,128 @@
+Copyright (C) 2024 Advanced Micro Devices, Inc.
+
+# Build and deploy DocSum Application on AMD GPU (ROCm)
+
+## Build images
+
+## 🚀 Build Docker Images
+
+First of all, you need to build Docker Images locally and install the python package of it.
+
+### 1. Build LLM Image
+
+```bash
+git clone https://github.com/opea-project/GenAIComps.git
+cd GenAIComps
+docker build -t opea/llm-docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile .
+```
+
+### 2. Build Whisper Image
+
+```bash
+cd GenAIComps
+docker build -t opea/whisper:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/src/integrations/dependency/whisper/Dockerfile .
+```
+
+### 3. Build MegaService Docker Image
+
+To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command:
+
+```bash
+git clone https://github.com/opea-project/GenAIExamples
+cd GenAIExamples/DocSum/
+docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+```
+
+### 4. Build Gradio UI Docker Image
+
+Build the frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/DocSum/ui
+docker build -t opea/docsum-gradio-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile.gradio .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `opea/llm-docsum:latest`
+2. `opea/docsum:latest`
+3. `opea/docsum-gradio-ui:latest`
+4. `opea/llm-vllm-rocm:latest`
+5. `opea/whisper:latest`
+
+## 🚀 Start Microservices and MegaService
+
+### Required Models
+
+Default model is "Intel/neural-chat-7b-v3-3". Change "DOCSUM_LLM_MODEL_ID" in environment variables below if you want to use another model.
+For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable.
+
+### Setup Environment Variables
+
+```bash
+export HUGGINGFACEHUB_API_TOKEN='your huggingfacehub token'
+```
+
+Edit the file set_env_vllm.sh - set the desired values of the variables in it
+Note: Please replace value HOST_IP with your server IP address, do not use localhost.
+
+Set values:
+
+```bash
+cd GenAiExamples/DocSum/docker_compose/amd/gpu/rocm
+. set_env_vllm.sh
+```
+
+#### Set GPU settings in compose_vllm.yaml:
+
+Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered<node>, where <node> is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+Example for set isolation for 1 GPU
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+```
+
+Example for set isolation for 2 GPUs
+
+```
+      - /dev/dri/card0:/dev/dri/card0
+      - /dev/dri/renderD128:/dev/dri/renderD128
+      - /dev/dri/card1:/dev/dri/card1
+      - /dev/dri/renderD129:/dev/dri/renderD129
+```
+
+Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus)
+
+### Start Microservice Docker Containers
+
+```bash
+cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm
+docker compose -f compose_vllm.yaml up -d
+```
+
+### Validate Microservices
+
+1. vLLM Service
+
+   ```bash
+   curl http://${host_ip}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions \
+     -X POST \
+     -d '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' \
+     -H 'Content-Type: application/json'
+   ```
+
+2. LLM Microservice
+
+   ```bash
+   curl http://${host_ip}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum \
+     -X POST \
+     -d '{"messages":"What is Deep Learning?"}' \
+     -H 'Content-Type: application/json'
+   ```
+
+3. MegaService
+
+   ```bash
+   curl http://${host_ip}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum -H "Content-Type: application/json" -d '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}'
+   ```

DocSum/docker_compose/amd/gpu/rocm/compose.yaml

@@ -6,7 +6,7 @@ services:
     image: ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
     container_name: docsum-tgi-service
     ports:
-      - "${DOCSUM_TGI_SERVICE_PORT}:80"
+      - "${DOCSUM_TGI_SERVICE_PORT:-8008}:80"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
@@ -15,64 +15,52 @@ services:
       HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
       host_ip: ${host_ip}
       DOCSUM_TGI_SERVICE_PORT: ${DOCSUM_TGI_SERVICE_PORT}
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f http://localhost:80/health || exit 1" ]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     volumes:
       - "/var/opea/docsum-service/data:/data"
     shm_size: 1g
     devices:
       - /dev/kfd:/dev/kfd
-      - /dev/dri/${DOCSUM_CARD_ID}:/dev/dri/${DOCSUM_CARD_ID}
-      - /dev/dri/${DOCSUM_RENDER_ID}:/dev/dri/${DOCSUM_RENDER_ID}
+      - /dev/dri/:/dev/dri/
     cap_add:
       - SYS_PTRACE
     group_add:
       - video
     security_opt:
       - seccomp:unconfined
     ipc: host
-    healthcheck:
-      test: ["CMD-SHELL", "curl -f http://${host_ip}:${DOCSUM_TGI_SERVICE_PORT}/health || exit 1"]
-      interval: 10s
-      timeout: 10s
-      retries: 100
-    command: --model-id ${DOCSUM_LLM_MODEL_ID} --max-input-length ${MAX_INPUT_TOKENS} --max-total-tokens ${MAX_TOTAL_TOKENS}
+    command: --model-id ${DOCSUM_LLM_MODEL_ID} --max-input-length ${DOCSUM_MAX_INPUT_TOKENS} --max-total-tokens ${DOCSUM_MAX_TOTAL_TOKENS}
 
   docsum-llm-server:
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
     container_name: docsum-llm-server
     depends_on:
-      docsum-tgi-service:
-        condition: service_healthy
+      - docsum-tgi-service
     ports:
       - "${DOCSUM_LLM_SERVER_PORT}:9000"
     ipc: host
-    group_add:
-      - video
-    security_opt:
-      - seccomp:unconfined
-    cap_add:
-      - SYS_PTRACE
-    devices:
-      - /dev/kfd:/dev/kfd
-      - /dev/dri/${DOCSUM_CARD_ID}:/dev/dri/${DOCSUM_CARD_ID}
-      - /dev/dri/${DOCSUM_RENDER_ID}:/dev/dri/${DOCSUM_RENDER_ID}
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
+      LLM_ENDPOINT: ${DOCSUM_TGI_LLM_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      MAX_INPUT_TOKENS: ${MAX_INPUT_TOKENS}
-      MAX_TOTAL_TOKENS: ${MAX_TOTAL_TOKENS}
+      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
+      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
       LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
-      DocSum_COMPONENT_NAME: ${DocSum_COMPONENT_NAME}
+      DocSum_COMPONENT_NAME: "OpeaDocSumTgi"
       LOGFLAG: ${LOGFLAG:-False}
     restart: unless-stopped
 
   whisper:
     image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
     container_name: whisper-service
     ports:
-      - "7066:7066"
+      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
     ipc: host
     environment:
       no_proxy: ${no_proxy}
@@ -89,13 +77,12 @@ services:
     ports:
       - "${DOCSUM_BACKEND_SERVER_PORT}:8888"
     environment:
-      - no_proxy=${no_proxy}
-      - https_proxy=${https_proxy}
-      - http_proxy=${http_proxy}
-      - MEGA_SERVICE_HOST_IP=${HOST_IP}
-      - LLM_SERVICE_HOST_IP=${HOST_IP}
-      - ASR_SERVICE_HOST_IP=${ASR_SERVICE_HOST_IP}
-
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      MEGA_SERVICE_HOST_IP: ${HOST_IP}
+      LLM_SERVICE_HOST_IP: ${HOST_IP}
+      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
     ipc: host
     restart: always
 
@@ -105,13 +92,13 @@ services:
     depends_on:
       - docsum-backend-server
     ports:
-      - "5173:5173"
+      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
     environment:
-      - no_proxy=${no_proxy}
-      - https_proxy=${https_proxy}
-      - http_proxy=${http_proxy}
-      - BACKEND_SERVICE_ENDPOINT=${BACKEND_SERVICE_ENDPOINT}
-      - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      BACKEND_SERVICE_ENDPOINT: ${BACKEND_SERVICE_ENDPOINT}
+      DOC_BASE_URL: ${BACKEND_SERVICE_ENDPOINT}
     ipc: host
     restart: always
 

DocSum/docker_compose/amd/gpu/rocm/compose_vllm.yaml

@@ -0,0 +1,105 @@
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  docsum-vllm-service:
+    image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest}
+    container_name: docsum-vllm-service
+    ports:
+      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_HUB_DISABLE_PROGRESS_BARS: 1
+      HF_HUB_ENABLE_HF_TRANSFER: 0
+      VLLM_USE_TRITON_FLASH_ATTENTION: 0
+      PYTORCH_JIT: 0
+    volumes:
+      - "./data:/data"
+    shm_size: 20G
+    devices:
+      - /dev/kfd:/dev/kfd
+      - /dev/dri/:/dev/dri/
+    cap_add:
+      - SYS_PTRACE
+    group_add:
+      - video
+    security_opt:
+      - seccomp:unconfined
+      - apparmor=unconfined
+    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
+    ipc: host
+
+  docsum-llm-server:
+    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+    container_name: docsum-llm-server
+    depends_on:
+      - docsum-vllm-service
+    ports:
+      - "${DOCSUM_LLM_SERVER_PORT}:9000"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      LLM_ENDPOINT: ${DOCSUM_LLM_ENDPOINT}
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
+      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
+      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
+      DocSum_COMPONENT_NAME: "OpeaDocSumvLLM"
+      LOGFLAG: ${LOGFLAG:-False}
+    restart: unless-stopped
+
+  whisper:
+    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
+    container_name: whisper-service
+    ports:
+      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+    restart: unless-stopped
+
+  docsum-backend-server:
+    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
+    container_name: docsum-backend-server
+    depends_on:
+      - docsum-vllm-service
+      - docsum-llm-server
+    ports:
+      - "${DOCSUM_BACKEND_SERVER_PORT}:8888"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      MEGA_SERVICE_HOST_IP: ${HOST_IP}
+      LLM_SERVICE_HOST_IP: ${HOST_IP}
+      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
+    ipc: host
+    restart: always
+
+  docsum-gradio-ui:
+    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
+    container_name: docsum-ui-server
+    depends_on:
+      - docsum-backend-server
+    ports:
+      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      BACKEND_SERVICE_ENDPOINT: ${BACKEND_SERVICE_ENDPOINT}
+      DOC_BASE_URL: ${BACKEND_SERVICE_ENDPOINT}
+    ipc: host
+    restart: always
+
+networks:
+  default:
+    driver: bridge