Adding files to deploy DocSum application on ROCm vLLM

DocSum/docker_compose/amd/gpu/rocm/compose.yaml

@@ -6,7 +6,7 @@ services:
     image: ghcr.io/huggingface/text-generation-inference:2.4.1-rocm
     container_name: docsum-tgi-service
     ports:
-      - "${DOCSUM_TGI_SERVICE_PORT}:80"
+      - "${DOCSUM_TGI_SERVICE_PORT:-8008}:80"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
@@ -16,12 +16,11 @@ services:
       host_ip: ${host_ip}
       DOCSUM_TGI_SERVICE_PORT: ${DOCSUM_TGI_SERVICE_PORT}
     volumes:
-      - "/var/opea/docsum-service/data:/data"
-    shm_size: 1g
+      - "${MODEL_CACHE:-./data}:/data"
+    shm_size: 20g
     devices:
       - /dev/kfd:/dev/kfd
-      - /dev/dri/${DOCSUM_CARD_ID}:/dev/dri/${DOCSUM_CARD_ID}
-      - /dev/dri/${DOCSUM_RENDER_ID}:/dev/dri/${DOCSUM_RENDER_ID}
+      - /dev/dri/:/dev/dri/
     cap_add:
       - SYS_PTRACE
     group_add:
@@ -34,7 +33,7 @@ services:
       interval: 10s
       timeout: 10s
       retries: 100
-    command: --model-id ${DOCSUM_LLM_MODEL_ID} --max-input-length ${MAX_INPUT_TOKENS} --max-total-tokens ${MAX_TOTAL_TOKENS}
+    command: --model-id ${DOCSUM_LLM_MODEL_ID} --max-input-length ${DOCSUM_MAX_INPUT_TOKENS} --max-total-tokens ${DOCSUM_MAX_TOTAL_TOKENS}
 
   docsum-llm-server:
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
@@ -45,34 +44,24 @@ services:
     ports:
       - "${DOCSUM_LLM_SERVER_PORT}:9000"
     ipc: host
-    group_add:
-      - video
-    security_opt:
-      - seccomp:unconfined
-    cap_add:
-      - SYS_PTRACE
-    devices:
-      - /dev/kfd:/dev/kfd
-      - /dev/dri/${DOCSUM_CARD_ID}:/dev/dri/${DOCSUM_CARD_ID}
-      - /dev/dri/${DOCSUM_RENDER_ID}:/dev/dri/${DOCSUM_RENDER_ID}
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
+      LLM_ENDPOINT: ${DOCSUM_TGI_LLM_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
-      MAX_INPUT_TOKENS: ${MAX_INPUT_TOKENS}
-      MAX_TOTAL_TOKENS: ${MAX_TOTAL_TOKENS}
+      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
+      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
       LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
-      DocSum_COMPONENT_NAME: ${DocSum_COMPONENT_NAME}
+      DocSum_COMPONENT_NAME: "OpeaDocSumTgi"
       LOGFLAG: ${LOGFLAG:-False}
     restart: unless-stopped
 
   whisper:
     image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
     container_name: whisper-service
     ports:
-      - "7066:7066"
+      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
     ipc: host
     environment:
       no_proxy: ${no_proxy}
@@ -89,13 +78,14 @@ services:
     ports:
       - "${DOCSUM_BACKEND_SERVER_PORT}:8888"
     environment:
-      - no_proxy=${no_proxy}
-      - https_proxy=${https_proxy}
-      - http_proxy=${http_proxy}
-      - MEGA_SERVICE_HOST_IP=${HOST_IP}
-      - LLM_SERVICE_HOST_IP=${HOST_IP}
-      - ASR_SERVICE_HOST_IP=${ASR_SERVICE_HOST_IP}
-
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      MEGA_SERVICE_HOST_IP: ${HOST_IP}
+      LLM_SERVICE_HOST_IP: ${HOST_IP}
+      LLM_SERVICE_PORT: ${DOCSUM_LLM_SERVER_PORT}
+      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
+      ASR_SERVICE_PORT: ${DOCSUM_WHISPER_PORT}
     ipc: host
     restart: always
 
@@ -107,11 +97,11 @@ services:
     ports:
       - "5173:5173"
     environment:
-      - no_proxy=${no_proxy}
-      - https_proxy=${https_proxy}
-      - http_proxy=${http_proxy}
-      - BACKEND_SERVICE_ENDPOINT=${BACKEND_SERVICE_ENDPOINT}
-      - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      BACKEND_SERVICE_ENDPOINT: ${BACKEND_SERVICE_ENDPOINT}
+      DOC_BASE_URL: ${BACKEND_SERVICE_ENDPOINT}
     ipc: host
     restart: always
 

DocSum/docker_compose/amd/gpu/rocm/compose_vllm.yaml

@@ -0,0 +1,111 @@
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  docsum-vllm-service:
+    image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}
+    container_name: docsum-vllm-service
+    ports:
+      - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      HF_HUB_DISABLE_PROGRESS_BARS: 1
+      HF_HUB_ENABLE_HF_TRANSFER: 0
+      VLLM_USE_TRITON_FLASH_ATTENTION: 0
+      PYTORCH_JIT: 0
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT:-8081}/health || exit 1" ]
+      interval: 10s
+      timeout: 10s
+      retries: 100
+    volumes:
+      - "${MODEL_CACHE:-./data}:/data"
+    shm_size: 20G
+    devices:
+      - /dev/kfd:/dev/kfd
+      - /dev/dri/:/dev/dri/
+    cap_add:
+      - SYS_PTRACE
+    group_add:
+      - video
+    security_opt:
+      - seccomp:unconfined
+      - apparmor=unconfined
+    command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
+    ipc: host
+
+  docsum-llm-server:
+    image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+    container_name: docsum-llm-server
+    depends_on:
+      docsum-vllm-service:
+        condition: service_healthy
+    ports:
+      - "${DOCSUM_LLM_SERVER_PORT}:9000"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      LLM_ENDPOINT: ${DOCSUM_LLM_ENDPOINT}
+      HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN}
+      MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS}
+      MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS}
+      LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID}
+      DocSum_COMPONENT_NAME: "OpeaDocSumvLLM"
+      LOGFLAG: ${LOGFLAG:-False}
+    restart: unless-stopped
+
+  whisper:
+    image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
+    container_name: whisper-service
+    ports:
+      - "${DOCSUM_WHISPER_PORT:-7066}:7066"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+    restart: unless-stopped
+
+  docsum-backend-server:
+    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
+    container_name: docsum-backend-server
+    depends_on:
+      - docsum-vllm-service
+      - docsum-llm-server
+    ports:
+      - "${DOCSUM_BACKEND_SERVER_PORT}:8888"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      MEGA_SERVICE_HOST_IP: ${HOST_IP}
+      LLM_SERVICE_HOST_IP: ${HOST_IP}
+      ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP}
+    ipc: host
+    restart: always
+
+  docsum-gradio-ui:
+    image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest}
+    container_name: docsum-ui-server
+    depends_on:
+      - docsum-backend-server
+    ports:
+      - "${DOCSUM_FRONTEND_PORT:-5173}:5173"
+    environment:
+      no_proxy: ${no_proxy}
+      https_proxy: ${https_proxy}
+      http_proxy: ${http_proxy}
+      BACKEND_SERVICE_ENDPOINT: ${BACKEND_SERVICE_ENDPOINT}
+      DOC_BASE_URL: ${BACKEND_SERVICE_ENDPOINT}
+    ipc: host
+    restart: always
+
+networks:
+  default:
+    driver: bridge

DocSum/docker_compose/amd/gpu/rocm/set_env.sh

@@ -3,15 +3,16 @@
 # Copyright (C) 2024 Advanced Micro Devices, Inc.
 # SPDX-License-Identifier: Apache-2.0
 
-export MAX_INPUT_TOKENS=2048
-export MAX_TOTAL_TOKENS=4096
-export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.4.1-rocm"
+export HOST_IP=''
+export DOCSUM_MAX_INPUT_TOKENS="2048"
+export DOCSUM_MAX_TOTAL_TOKENS="4096"
 export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-export HOST_IP=${host_ip}
 export DOCSUM_TGI_SERVICE_PORT="8008"
 export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
-export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export DOCSUM_WHISPER_PORT="7066"
+export ASR_SERVICE_HOST_IP="${HOST_IP}"
 export DOCSUM_LLM_SERVER_PORT="9000"
-export DOCSUM_BACKEND_SERVER_PORT="8888"
-export DOCSUM_FRONTEND_PORT="5173"
+export DOCSUM_BACKEND_SERVER_PORT="18072"
+export DOCSUM_FRONTEND_PORT="18073"
 export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"

DocSum/docker_compose/amd/gpu/rocm/set_env_vllm.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+export HOST_IP=''
+export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export DOCSUM_MAX_INPUT_TOKENS=2048
+export DOCSUM_MAX_TOTAL_TOKENS=4096
+export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export DOCSUM_VLLM_SERVICE_PORT="8008"
+export DOCSUM_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}"
+export DOCSUM_WHISPER_PORT="7066"
+export ASR_SERVICE_HOST_IP="${HOST_IP}"
+export DOCSUM_LLM_SERVER_PORT="9000"
+export DOCSUM_BACKEND_SERVER_PORT="18072"
+export DOCSUM_FRONTEND_PORT="18073"
+export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"

DocSum/docker_image_build/build.yaml

@@ -49,6 +49,11 @@ services:
       dockerfile: comps/llms/src/doc-summarization/Dockerfile
     extends: docsum
     image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest}
+  vllm-rocm:
+    build:
+      context: GenAIComps
+      dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu
+    image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}
   vllm:
     build:
       context: vllm