@@ -69,20 +69,6 @@ Build microservice docker.
6969docker build --no-cache -t opea/llm-vllm-ray:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/vllm-ray/docker/Dockerfile.microservice .
7070```
7171
72- #### 5.4 Use Ray Serve
73-
74- Build Ray Serve docker.
75-
76- ``` bash
77- docker build --no-cache -t ray_serve:habana --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/ray_serve/docker/Dockerfile.rayserve .
78- ```
79-
80- Build microservice docker.
81-
82- ``` bash
83- docker build --no-cache -t opea/llm-ray:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/ray_serve/docker/Dockerfile.microservice .
84- ```
85-
8672### 6. Build Dataprep Image
8773
8874``` bash
@@ -159,7 +145,7 @@ Then run the command `docker images`, you will have the following 8 Docker Image
1591451 . ` opea/embedding-tei:latest `
1601462 . ` opea/retriever-redis:latest `
1611473 . ` opea/reranking-tei:latest `
162- 4 . ` opea/llm-tgi:latest ` or ` opea/llm-vllm:latest ` or ` opea/llm-vllm-ray:latest ` or ` opea/llm-ray:latest `
148+ 4 . ` opea/llm-tgi:latest ` or ` opea/llm-vllm:latest ` or ` opea/llm-vllm-ray:latest `
1631495 . ` opea/tei-gaudi:latest `
1641506 . ` opea/dataprep-redis:latest `
1651517 . ` opea/chatqna:latest ` or ` opea/chatqna-guardrails:latest `
@@ -192,7 +178,6 @@ export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
192178export TGI_LLM_ENDPOINT=" http://${host_ip} :8008"
193179export vLLM_LLM_ENDPOINT=" http://${host_ip} :8008"
194180export vLLM_RAY_LLM_ENDPOINT=" http://${host_ip} :8008"
195- export RAY_Serve_LLM_ENDPOINT=" http://${host_ip} :8008"
196181export LLM_SERVICE_PORT=9000
197182export REDIS_URL=" redis://${host_ip} :6379"
198183export INDEX_NAME=" rag-redis"
@@ -243,12 +228,6 @@ If use vllm-on-ray for llm backend.
243228docker compose -f compose_vllm_ray.yaml up -d
244229```
245230
246- If use ray serve for llm backend.
247-
248- ``` bash
249- docker compose -f compose_ray_serve.yaml up -d
250- ```
251-
252231If you want to enable guardrails microservice in the pipeline, please follow the below command instead:
253232
254233``` bash
@@ -342,13 +321,6 @@ curl http://${your_ip}:8008/v1/chat/completions \
342321 -d ' {"model": "${LLM_MODEL_ID}", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}'
343322```
344323
345- ``` bash
346- # Ray Serve Service
347- curl http://${your_ip} :8008/v1/chat/completions \
348- -H " Content-Type: application/json" \
349- -d ' {"model": "${LLM_MODEL_ID_NAME}", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 32 }'
350- ```
351-
3523247 . LLM Microservice
353325
354326``` bash
0 commit comments