FAQGen Megaservice

.github/workflows/scripts/build_push.sh

@@ -46,7 +46,7 @@ function docker_build() {
 # $1 is like "apple orange pear"
 for MEGA_SVC in $1; do
     case $MEGA_SVC in
-        "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum"|"Translation"|"AudioQnA"|"SearchQnA")
+        "ChatQnA"|"CodeGen"|"CodeTrans"|"DocSum"|"Translation"|"AudioQnA"|"SearchQnA"|"FaqGen")
             cd $MEGA_SVC/docker
             IMAGE_NAME="$(getImagenameFromMega $MEGA_SVC)"
             docker_build ${IMAGE_NAME}

FaqGen/README.md

@@ -0,0 +1,17 @@
+# FAQ Generation Application
+
+In today's data-driven world, organizations across various industries face the challenge of managing and understanding vast amounts of information. Legal documents, contracts, regulations, and customer inquiries often contain critical insights buried within dense text. Extracting and presenting these insights in a concise and accessible format is crucial for decision-making, compliance, and customer satisfaction.
+
+Our FAQ Generation Application leverages the power of large language models (LLMs) to revolutionize the way you interact with and comprehend complex textual data. By harnessing cutting-edge natural language processing techniques, our application can automatically generate comprehensive and natural-sounding frequently asked questions (FAQs) from your documents, legal texts, customer queries, and other sources. In this example use case, we utilize LangChain to implement FAQ Generation and facilitate LLM inference using Text Generation Inference on Intel Xeon and Gaudi2 processors.
+
+# Deploy FAQ Generation Service
+
+The FAQ Generation service can be effortlessly deployed on either Intel Gaudi2 or Intel XEON Scalable Processors.
+
+## Deploy FAQ Generation on Gaudi
+
+Refer to the [Gaudi Guide](./docker/gaudi/README.md) for instructions on deploying FAQ Generation on Gaudi.
+
+## Deploy FAQ Generation on Xeon
+
+Refer to the [Xeon Guide](./docker/xeon/README.md) for instructions on deploying FAQ Generation on Xeon.

FaqGen/docker/Dockerfile

@@ -0,0 +1,32 @@
+
+
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+FROM langchain/langchain:latest
+
+
+RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
+    libgl1-mesa-glx \
+    libjemalloc-dev \
+    vim
+
+RUN useradd -m -s /bin/bash user && \
+    mkdir -p /home/user && \
+    chown -R user /home/user/
+
+RUN cd /home/user/ && \
+    git clone https://github.com/opea-project/GenAIComps.git
+
+RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
+    pip install -r /home/user/GenAIComps/requirements.txt
+
+COPY ./faqgen.py /home/user/faqgen.py
+
+ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
+
+USER user
+
+WORKDIR /home/user
+
+ENTRYPOINT ["python", "faqgen.py"]

FaqGen/docker/faqgen.py

@@ -0,0 +1,36 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import os
+
+from comps import FaqGenGateway, MicroService, ServiceOrchestrator, ServiceType
+
+MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_PORT = int(os.getenv("MEGA_SERVICE_PORT", 8888))
+LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0")
+LLM_SERVICE_PORT = int(os.getenv("LLM_SERVICE_PORT", 9000))
+
+
+class FaqGenService:
+    def __init__(self, host="0.0.0.0", port=8000):
+        self.host = host
+        self.port = port
+        self.megaservice = ServiceOrchestrator()
+
+    def add_remote_service(self):
+        llm = MicroService(
+            name="llm",
+            host=LLM_SERVICE_HOST_IP,
+            port=LLM_SERVICE_PORT,
+            endpoint="/v1/faqgen",
+            use_remote_service=True,
+            service_type=ServiceType.LLM,
+        )
+        self.megaservice.add(llm)
+        self.gateway = FaqGenGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
+
+
+if __name__ == "__main__":
+    faqgen = FaqGenService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT)
+    faqgen.add_remote_service()

FaqGen/docker/gaudi/README.md

@@ -0,0 +1,171 @@
+# Build MegaService of FAQ Generation on Gaudi
+
+This document outlines the deployment process for a FAQ Generation application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Gaudi server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as llm. We will publish the Docker images to Docker Hub, which will simplify the deployment process for this service.
+
+## 🚀 Build Docker Images
+
+First of all, you need to build Docker Images locally. This step can be ignored once the Docker images are published to Docker hub.
+
+```bash
+git clone https://github.com/opea-project/GenAIComps.git
+cd GenAIComps
+```
+
+### 1. Pull TGI Gaudi Image
+
+As TGI Gaudi has been officially published as a Docker image, we simply need to pull it:
+
+```bash
+docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1
+```
+
+### 2. Build LLM Image
+
+```bash
+docker build -t opea/llm-faqgen-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/faq-generation/tgi/Dockerfile .
+```
+
+### 3. Build MegaService Docker Image
+
+To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `faqgen.py` Python script. Build the MegaService Docker image using the command below:
+
+```bash
+git clone https://github.com/opea-project/GenAIExamples
+cd GenAIExamples/FaqGen/docker/
+docker build --no-cache -t opea/faqgen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+```
+
+### 4. Build UI Docker Image
+
+Construct the frontend Docker image using the command below:
+
+```bash
+cd GenAIExamples/FaqGen/docker/ui/
+docker build -t opea/faqgen-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile .
+```
+
+### 5. Build react UI Docker Image (Optional)
+
+Build the frontend Docker image based on react framework via below command:
+
+```bash
+cd GenAIExamples/FaqGen/docker/ui
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/faqgen"
+docker build -t opea/faqgen-react-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
+```
+
+Then run the command `docker images`, you will have the following Docker Images:
+
+1. `ghcr.io/huggingface/tgi-gaudi:1.2.1`
+2. `opea/llm-faqgen-tgi:latest`
+3. `opea/faqgen:latest`
+4. `opea/faqgen-ui:latest`
+5. `opea/faqgen-react-ui:latest`
+
+## 🚀 Start Microservices and MegaService
+
+### Setup Environment Variables
+
+Since the `docker_compose.yaml` will consume some environment variables, you need to setup them in advance as below.
+
+```bash
+export no_proxy=${your_no_proxy}
+export http_proxy=${your_http_proxy}
+export https_proxy=${your_http_proxy}
+export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export TGI_LLM_ENDPOINT="http://${your_ip}:8008"
+export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
+export MEGA_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/faqgen"
+```
+
+Note: Please replace with `host_ip` with your external IP address, do not use localhost.
+
+### Start Microservice Docker Containers
+
+```bash
+cd GenAIExamples/FaqGen/docker/gaudi
+docker compose -f docker_compose.yaml up -d
+```
+
+### Validate Microservices
+
+1. TGI Service
+
+```bash
+curl http://${your_ip}:8008/generate \
+  -X POST \
+  -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
+  -H 'Content-Type: application/json'
+```
+
+2. LLM Microservice
+
+```bash
+curl http://${host_ip}:9000/v1/faqgen \
+  -X POST \
+  -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
+  -H 'Content-Type: application/json'
+```
+
+3. MegaService
+
+```bash
+curl http://${host_ip}:8888/v1/faqgen -H "Content-Type: application/json" -d '{
+     "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
+     }'
+```
+
+## Enable LangSmith to Monitor an Application (Optional)
+
+LangSmith offers a suite of tools to debug, evaluate, and monitor language models and intelligent agents. It can be used to assess benchmark data for each microservice. Before launching your services with `docker compose -f docker_compose.yaml up -d`, you need to enable LangSmith tracing by setting the `LANGCHAIN_TRACING_V2` environment variable to true and configuring your LangChain API key.
+
+Here's how you can do it:
+
+1. Install the latest version of LangSmith:
+
+```bash
+pip install -U langsmith
+```
+
+2. Set the necessary environment variables:
+
+```bash
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=ls_...
+```
+
+## 🚀 Launch the UI
+
+Open this URL `http://{host_ip}:5173` in your browser to access the frontend.
+
+![project-screenshot](../../assets/img/faqgen_ui_text.png)
+
+## 🚀 Launch the React UI (Optional)
+
+To access the FAQGen (react based) frontend, modify the UI service in the `docker_compose.yaml` file. Replace `faqgen-xeon-ui-server` service with the `faqgen-xeon-react-ui-server` service as per the config below:
+
+```bash
+  faqgen-xeon-react-ui-server:
+    image: opea/faqgen-react-ui:latest
+    container_name: faqgen-xeon-react-ui-server
+    environment:
+      - no_proxy=${no_proxy}
+      - https_proxy=${https_proxy}
+      - http_proxy=${http_proxy}
+    ports:
+      - 5174:80
+    depends_on:
+      - faqgen-xeon-backend-server
+    ipc: host
+    restart: always
+```
+
+Open this URL `http://{host_ip}:5174` in your browser to access the react based frontend.
+
+- Create FAQs from Text input
+  ![project-screenshot](../../assets/img/faqgen_react_ui_text.png)
+
+- Create FAQs from Text Files
+  ![project-screenshot](../../assets/img/faqgen_react_ui_text_files.png)

FaqGen/docker/gaudi/docker_compose.yaml

@@ -0,0 +1,77 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+version: "3.8"
+
+services:
+  tgi_service:
+    image: ghcr.io/huggingface/tgi-gaudi:2.0.1
+    container_name: tgi-gaudi-server
+    ports:
+      - "8008:80"
+    volumes:
+      - "./data:/data"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HABANA_VISIBLE_DEVICES: all
+      OMPI_MCA_btl_vader_single_copy_mechanism: none
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+    runtime: habana
+    cap_add:
+      - SYS_NICE
+    ipc: host
+    command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048
+  llm_faqgen:
+    image: opea/llm-faqgen-tgi:latest
+    container_name: llm-faqgen-server
+    depends_on:
+      - tgi_service
+    ports:
+      - "9000:9000"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
+      LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
+      LANGCHAIN_PROJECT: "opea-llm-service"
+    restart: unless-stopped
+  faqgen-gaudi-backend-server:
+    image: opea/faqgen:latest
+    container_name: faqgen-gaudi-backend-server
+    depends_on:
+      - tgi_service
+      - llm_faqgen
+    ports:
+      - "8888:8888"
+    environment:
+      - no_proxy=${no_proxy}
+      - https_proxy=${https_proxy}
+      - http_proxy=${http_proxy}
+      - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
+    ipc: host
+    restart: always
+  faqgen-gaudi-ui-server:
+    image: opea/faqgen-ui:latest
+    container_name: faqgen-gaudi-ui-server
+    depends_on:
+      - faqgen-gaudi-backend-server
+    ports:
+      - "5173:5173"
+    environment:
+      - no_proxy=${no_proxy}
+      - https_proxy=${https_proxy}
+      - http_proxy=${http_proxy}
+      - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+    ipc: host
+    restart: always
+
+networks:
+  default:
+    driver: bridge

FaqGen/docker/ui/docker/Dockerfile

@@ -0,0 +1,26 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Use node 20.11.1 as the base image
+FROM node:20.11.1
+
+# Update package manager and install Git
+RUN apt-get update -y && apt-get install -y git
+
+# Copy the front-end code repository
+COPY svelte /home/user/svelte
+
+# Set the working directory
+WORKDIR /home/user/svelte
+
+# Install front-end dependencies
+RUN npm install
+
+# Build the front-end application
+RUN npm run build
+
+# Expose the port of the front-end application
+EXPOSE 5173
+
+# Run the front-end application in preview mode
+CMD ["npm", "run", "preview", "--", "--port", "5173", "--host", "0.0.0.0"]

FaqGen/docker/ui/docker/Dockerfile.react

@@ -0,0 +1,20 @@
+FROM node as vite-app
+
+COPY . /usr/app
+WORKDIR /usr/app/react
+
+ARG BACKEND_SERVICE_ENDPOINT
+ENV VITE_FAQ_GEN_URL=$BACKEND_SERVICE_ENDPOINT
+
+RUN ["npm", "install"]
+RUN ["npm", "run", "build"]
+
+
+FROM nginx:alpine
+EXPOSE 80
+
+
+COPY --from=vite-app /usr/app/react/nginx.conf /etc/nginx/conf.d/default.conf
+COPY --from=vite-app /usr/app/react/dist /usr/share/nginx/html
+
+ENTRYPOINT ["nginx", "-g", "daemon off;"]