diff --git a/example/transform/huggingface_model_gemma_QA.ipynb.ipynb b/example/transform/huggingface_model_gemma_QA.ipynb.ipynb new file mode 100644 index 00000000..28b10274 --- /dev/null +++ b/example/transform/huggingface_model_gemma_QA.ipynb.ipynb @@ -0,0 +1,486 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "78c52d4f-e076-471a-b711-076e5d176fee", + "metadata": {}, + "source": [ + "# Using Open-source HuggingFace Models (Based on Gemma-7b-it) to Generate QAs from Raw Data in JSON format\n", + "\n", + "In this example, we will show you how to generate question-answers (QAs) from given text strings using open-source Huggingface models **using [Gemma-7b-it](https://huggingface.co/google/gemma-7b-it)** via uniflow's [HuggingFaceModelFlow](https://github.com/CambioML/uniflow/blob/main/uniflow/flow/model_flow.py#L86).\n", + "\n", + "### Before running the code\n", + "\n", + "You will need to `uniflow` conda environment to run this notebook. You can set up the environment following the instruction: https://github.com/CambioML/uniflow/tree/main#installation.\n", + "\n", + "Next, you will need a valid Huggingface Token to access the Gemma model. Once you have the token, set it as the environment variable `HF_TOKEN` within a `.env` file in the root directory of this repository. Check [this](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables) for more details.\n", + "\n", + "### Update system path" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "aa1dcf25-e694-4e4d-b449-3851ad03b2f3", + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "\n", + "import sys\n", + "\n", + "sys.path.append(\".\")\n", + "sys.path.append(\"..\")\n", + "sys.path.append(\"../..\")" + ] + }, + { + "cell_type": "markdown", + "id": "868a7956-af63-4f18-9299-c43293ba1fa1", + "metadata": {}, + "source": [ + "### Install libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ed29a309-bf91-4582-996a-29374df894b2", + "metadata": {}, + "outputs": [], + "source": [ + "!{sys.executable} -m pip install -q transformers accelerate bitsandbytes scipy tiktoken" + ] + }, + { + "cell_type": "markdown", + "id": "69523656-4791-40d3-84d7-4818e3669e42", + "metadata": {}, + "source": [ + "### Import dependency" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c24454fc-615b-4a2e-8950-1d0d7497784c", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/envs/uniflow/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pprint import pprint\n", + "from dotenv import load_dotenv\n", + "from IPython.display import Markdown, display\n", + "\n", + "from uniflow.flow.client import TransformClient\n", + "from uniflow.flow.config import GemmaTransformConfig\n", + "from uniflow.op.prompt import Context, PromptTemplate\n", + "\n", + "load_dotenv()" + ] + }, + { + "cell_type": "markdown", + "id": "4bd10644-3089-45c3-8f36-a6b5dc37a3f4", + "metadata": {}, + "source": [ + "### Prepare sample prompts\n", + "\n", + "First, we need to demonstrate sample prompts for LLM, those include instruction and sample json format. We do this by giving a sample instruction and list of `Context` examples to the `PromptTemplate` class. Let's create raw text for context." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0fc570b3-88a9-477c-8623-dfd70aa1f930", + "metadata": {}, + "outputs": [], + "source": [ + "raw_context_input = [\n", + " \"\"\"We believe our success depends upon our capabilities in areas such as design, research and development, \\\n", + "production and marketing and is supported and protected by our intellectual property rights, such as \\\n", + "trademarks, utility and design patents, copyrights, and trade secrets, among others. We have followed a policy \\\n", + "of applying for and registering intellectual property rights in the United States and select foreign countries \\\n", + "on trademarks, inventions, innovations and designs that we deem valuable. W e also continue to vigorously \\\n", + "protect our intellectual property, including trademarks, patents and trade secrets against third-party \\\n", + "infringement and misappropriation.\"\"\",\n", + " \"\"\"Snoopy can be selfish, gluttonous, and lazy at times, and occasionally mocks his owner, Charlie Brown. \n", + "But on the whole, he shows great love, care, and loyalty for his owner (even though he cannot even remember \n", + "his name and always refers to him as \"the round-headed kid\"). In the 1990s comic strips, he is obsessed \n", + "with cookies, particularly the chocolate-chip variety. This, and other instances in which he indulges in large \n", + "chocolate-based meals and snacks, indicate that chocolate is not poisonous to Snoopy, the way it is for real dogs.\"\"\",\n", + " \"\"\"Since the 2000s and especially the late 2010s, Intel has faced increasing competition from AMD, resulting in a \n", + "significant decline of its dominance and market share in the PC market. Nevertheless, with a 68.4% market share as of \n", + "2023, Intel still leads the x86 market by a wide margin.\"\"\"\n", + "]\n", + "\n", + "raw_context_input = raw_context_input" + ] + }, + { + "cell_type": "markdown", + "id": "3606645d-1dfa-4449-8fb6-3ca17c337241", + "metadata": {}, + "source": [ + "Next, for the given raw text strings `raw_context_input` above, we convert them to the `Context` class to be processed by `uniflow`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "049a36b8-cf88-4953-901f-5b00e8c1db08", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sample size of processed input data: 3\n", + "\n", + "Example uniflow context data:\n", + "[Context(context='We believe our success depends upon our capabilities in areas such as design, research and development, production and marketing and is supported and protected by our intellectual property rights, such as trademarks, utility and design patents, copyrights, and trade secrets, among others. We have followed a policy of applying for and registering intellectual property rights in the United States and select foreign countries on trademarks, inventions, innovations and designs that we deem valuable. W e also continue to vigorously protect our intellectual property, including trademarks, patents and trade secrets against third-party infringement and misappropriation.'),\n", + " Context(context='Snoopy can be selfish, gluttonous, and lazy at times, and occasionally mocks his owner, Charlie Brown. \\nBut on the whole, he shows great love, care, and loyalty for his owner (even though he cannot even remember \\nhis name and always refers to him as \"the round-headed kid\"). In the 1990s comic strips, he is obsessed \\nwith cookies, particularly the chocolate-chip variety. This, and other instances in which he indulges in large \\nchocolate-based meals and snacks, indicate that chocolate is not poisonous to Snoopy, the way it is for real dogs.'),\n", + " Context(context='Since the 2000s and especially the late 2010s, Intel has faced increasing competition from AMD, resulting in a \\nsignificant decline of its dominance and market share in the PC market. Nevertheless, with a 68.4% market share as of \\n2023, Intel still leads the x86 market by a wide margin.')]\n" + ] + } + ], + "source": [ + "\n", + "input_data = [\n", + " Context(context=data)\n", + " for data in raw_context_input\n", + "]\n", + "\n", + "print(\"sample size of processed input data: \", len(input_data))\n", + "\n", + "print(\"\\nExample uniflow context data:\")\n", + "pprint(input_data[:3])\n" + ] + }, + { + "cell_type": "markdown", + "id": "5cf1951f-b5e5-4fae-9f03-8f981cee8c2d", + "metadata": {}, + "source": [ + "### Use LLM to generate data\n", + "\n", + "In this example, we will use the [GemmaTransformConfig](https://github.com/CambioML/uniflow/blob/main/uniflow/flow/config.py)'s default LLM to generate questions and answers. Let's import the config and client of this model.\n", + "\n", + "Here in this example, we use define our own `PromptTemplate` to the `GemmaTransformConfig`, but you can use your default instructions and examples instead if you want.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0be2ca63-1708-440b-9709-6798d9076b8a", + "metadata": {}, + "outputs": [], + "source": [ + "guided_prompt = PromptTemplate(\n", + " instruction=\"Generate one question and its corresponding answer based on context. Following the format and structure of the examples below to include the same context, question, and answer in the response.\",\n", + " few_shot_prompt=[\n", + " Context(\n", + " context=\"In 1948, Claude E. Shannon published A Mathematical Theory of\\nCommunication (Shannon, 1948) establishing the theory of\\ninformation. In his article, Shannon introduced the concept of\\ninformation entropy for the first time. We will begin our journey here.\",\n", + " question=\"Who published A Mathematical Theory of Communication in 1948?\",\n", + " answer=\"Claude E. Shannon.\",\n", + " )\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2a228817-3905-4453-a4f8-51b5784eb5ec", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "tokenizer_config.json: 100%|██████████| 2.16k/2.16k [00:00<00:00, 15.6MB/s]\n", + "tokenizer.model: 100%|██████████| 4.24M/4.24M [00:00<00:00, 73.3MB/s]\n", + "tokenizer.json: 100%|██████████| 17.5M/17.5M [00:00<00:00, 219MB/s]\n", + "special_tokens_map.json: 100%|██████████| 888/888 [00:00<00:00, 6.63MB/s]\n", + "config.json: 100%|██████████| 694/694 [00:00<00:00, 5.47MB/s]\n", + "The `load_in_4bit` and `load_in_8bit` arguments are deprecated and will be removed in the future versions. Please, pass a `BitsAndBytesConfig` object in `quantization_config` argument instead.\n", + "model.safetensors.index.json: 100%|██████████| 20.9k/20.9k [00:00<00:00, 109MB/s]\n", + "model-00001-of-00004.safetensors: 100%|██████████| 5.00G/5.00G [00:21<00:00, 233MB/s]\n", + "model-00002-of-00004.safetensors: 100%|██████████| 4.98G/4.98G [00:28<00:00, 174MB/s]\n", + "model-00003-of-00004.safetensors: 100%|██████████| 4.98G/4.98G [00:31<00:00, 159MB/s]\n", + "model-00004-of-00004.safetensors: 100%|██████████| 2.11G/2.11G [00:15<00:00, 133MB/s]\n", + "Downloading shards: 100%|██████████| 4/4 [01:38<00:00, 24.51s/it]\n", + "Loading checkpoint shards: 100%|██████████| 4/4 [00:05<00:00, 1.29s/it]\n", + "generation_config.json: 100%|██████████| 137/137 [00:00<00:00, 987kB/s]\n" + ] + } + ], + "source": [ + "config = GemmaTransformConfig(\n", + " prompt_template=guided_prompt,\n", + ")\n", + "\n", + "client = TransformClient(config)" + ] + }, + { + "cell_type": "markdown", + "id": "598ebcf1-c4a1-4c41-8ee1-df6cb8c9e5ba", + "metadata": {}, + "source": [ + "Now we call the `run` method on the `client` object to execute the question-answer generation operation on the data shown above." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "cdc5d5bf-17c6-4d07-b90b-5e4525b1f219", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/1 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "**current context**: We believe our success depends upon our capabilities in areas such as design, research and development, production and marketing and is supported and protected by our intellectual property rights, such as trademarks, utility and design patents, copyrights, and trade secrets, among others. We have followed a policy of applying for and registering intellectual property rights in the United States and select foreign countries on trademarks, inventions, innovations and designs that we deem valuable. W e also continue to vigorously protect our intellectual property, including trademarks, patents and trade secrets against third-party infringement and misappropriation.\n", + "\n", + "**Question:** What does current text suggest about protection IP Rights related with company's achievements\n", + "\n", + " **Answer**: The provided texts suggests that Company has implemented policies designed specifically aimed at protecting their Intellectual Property(IP )Rights which includes trademark,utility & Design Patents,,copyrights&trade Secrets.The organization actively applies foe registration Of these right US And selected Foreign Countries while ensuring vigorous enforcement Against Infringement or Mis appropriation By Third Parties.. This strategy safeguards Their Achievements From Unauthorized Use Or Exploitation,. Therefore" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "---" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "**current context**: Snoopy can be selfish, gluttonous, and lazy at times, and occasionally mocks his owner, Charlie Brown. \n", + "But on the whole, he shows great love, care, and loyalty for his owner (even though he cannot even remember \n", + "his name and always refers to him as \"the round-headed kid\"). In the 1990s comic strips, he is obsessed \n", + "with cookies, particularly the chocolate-chip variety. This, and other instances in which he indulges in large \n", + "chocolate-based meals and snacks, indicate that chocolate is not poisonous to Snoopy, the way it is for real dogs.\n", + "\n", + "**Question:** What does a passage describe about snoop'S relationship with food specifically chocolates\n", + "\n", + " **Answer**: The text describes Snoop’ s obsession over Chocolate Cookies and indicates this treat isn t harmful like actual dog poison" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "---" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "**current context**: Since the 2000s and especially the late 2010s, Intel has faced increasing competition from AMD, resulting in a \n", + "significant decline of its dominance and market share in the PC market. Nevertheless, with a 68.4% market share as of \n", + "2023, Intel still leads the x86 market by a wide margin.\n", + "\n", + "**Question:** What company is leading the X-series processor marketplace currently despite facing increased competitive pressure since around year end two thousand or later specifically during Late twenty eleven'S era?\n", + "\n", + " **Answer**: Despite fierce rivalry against Advanced Micro Devices Inc.(AMD), intel maintains leadership position within this industry sector holding approximately sixty eight point four percent(or roughly seventy percentage )market coverage worldwide according data available through recent reports. Therefore,Intel remains dominant force even amidst growing challenges presented via competitors such As AmD" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import re\n", + "\n", + "def extract_current_context(text):\n", + " \"\"\"Extracts the text after 'current context' and adds '**' to 'current context'.\"\"\"\n", + " match = re.search(r'(current context:.+)', text, re.DOTALL)\n", + " if match:\n", + " extracted_text = match.group(1)\n", + " extracted_text = extracted_text.replace(\"current context\", \"**current context**\")\n", + " return extracted_text\n", + " else:\n", + " return \"No match found.\"\n", + "\n", + "for response in outputs[0]['output'][0]['response']:\n", + " processed_response = extract_current_context(response)\n", + " display(Markdown(\"---\"))\n", + " display(Markdown(processed_response))" + ] + }, + { + "cell_type": "markdown", + "id": "bae7a846-8b7f-49c9-8b68-14863b43a068", + "metadata": {}, + "source": [ + "## End of the notebook\n", + "\n", + "Check more Uniflow use cases in the [example folder](https://github.com/CambioML/uniflow/tree/main/example/model#examples)!\n", + "\n", + "\n", + " \n", + "" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/uniflow/flow/config.py b/uniflow/flow/config.py index 9125ea95..bdd4b50e 100644 --- a/uniflow/flow/config.py +++ b/uniflow/flow/config.py @@ -419,6 +419,25 @@ class TransformForClusteringOpenAIGPT4Config: ) +@dataclass +class GemmaTransformConfig(TransformConfig): + model_config: HuggingfaceModelConfig = field( + default_factory=lambda: HuggingfaceModelConfig( + model_name="google/gemma-7b-it", + batch_size=4, + max_new_tokens=100, + # token is needed for accessing Gemma model + ) + ) + flow_name: str = "TransformHuggingFaceFlow" + prompt_template: PromptTemplate = field( + default_factory=lambda: PromptTemplate( + instruction="You are Gemma, a large language model trained by Google. Write out your reasoning step-by-step to be sure you get the right answers!", + few_shot_prompt=[], + ) + ) + + ########################################################### # All AutoRater Config # ########################################################### diff --git a/uniflow/op/model/lm/model_server.py b/uniflow/op/model/lm/model_server.py index 4239c7e4..8e91c852 100644 --- a/uniflow/op/model/lm/model_server.py +++ b/uniflow/op/model/lm/model_server.py @@ -260,7 +260,11 @@ def __call__(self, data: List[str]) -> List[str]: class HuggingfaceModelServer(AbsModelServer): """Huggingface Model Server Class.""" - PATTERN = r"\[\/?INST\]||<>|\[ASST\]|\[\/ASST\]" + MODEL_PATTERNS = { + "default": r"\[\/?INST\]||<>|\[ASST\]|\[\/ASST\]", + "google/gemma-2b-it": r"user||model", + "google/gemma-7b-it": r"user||model", + } def __init__( self, prompt_template: PromptTemplate, model_config: Dict[str, Any] @@ -268,6 +272,9 @@ def __init__( # import in class level to avoid installing transformers package super().__init__(prompt_template, model_config) self._model_config = HuggingfaceModelConfig(**self._model_config) + self.pattern = self.MODEL_PATTERNS.get( + self._model_config.model_name, self.MODEL_PATTERNS["default"] + ) if self._model_config.neuron is False: try: from transformers import ( # pylint: disable=import-outside-toplevel @@ -349,7 +356,27 @@ def _preprocess(self, data: List[str]) -> List[str]: """ # add role and content key to data for apply_chat_template # as argument - data = [[{"role": "user", "content": d}] for d in data] + + if self._model_config.model_name in ( + "google/gemma-2b-it", + "google/gemma-7b-it", + ): + data = [ + [ + { + "role": "user", + "content": re.sub( + r"(.*)\ncontext:", + r"\1\ncurrent context:", + d, + flags=re.DOTALL, + ), + } + ] + for d in data + ] + else: + data = [[{"role": "user", "content": d}] for d in data] # if response_start_key is provided (few shot mode), add it with colon after # the end of instruction token for better instruction following performance. # Below is an example, if you have a QA prompt template like this for 1 shot mode: @@ -360,7 +387,12 @@ def _preprocess(self, data: List[str]) -> List[str]: # answer: ... <-- few shot answer # context: ... [/INST] <-- input context with [/INST] # question: <-- response_start_key is added here !!! - if self._model_config.response_start_key: + # with or without response start key, Gemma has no difference to answer the question + if ( + self._model_config.response_start_key + and self._model_config.model_name + not in ("google/gemma-2b-it", "google/gemma-7b-it") + ): data = [ self._tokenizer.apply_chat_template(d, tokenize=False) + f"\n{self._model_config.response_start_key}: " # noqa: W503 @@ -368,10 +400,22 @@ def _preprocess(self, data: List[str]) -> List[str]: ] # if response_start_key is not provided, simply add the instruction token # using apply_chat_template + # Gemma heavily need add_generation_prompt to generate good response + elif self._model_config.model_name in ( + "google/gemma-2b-it", + "google/gemma-7b-it", + ): + data = [ + self._tokenizer.apply_chat_template( + d, tokenize=False, add_generation_prompt=True + ) + for d in data + ] else: data = [ self._tokenizer.apply_chat_template(d, tokenize=False) for d in data ] + return data def _postprocess(self, data: List[str]) -> List[str]: @@ -387,7 +431,7 @@ def _postprocess(self, data: List[str]) -> List[str]: # clean up instruction token. for output_list in data: for d in output_list: - response = re.sub(self.PATTERN, "", d["generated_text"]).strip() + response = re.sub(self.pattern, "", d["generated_text"]).strip() response_list.append(response) # if response_format is json_object, parse the response into json_object. @@ -398,7 +442,10 @@ def _postprocess(self, data: List[str]) -> List[str]: ): # if example_keys (through few shot prompt) are provided, # parse the response into json_object. - if self._example_keys: + if self._example_keys and self._model_config.model_name not in ( + "google/gemma-2b-it", + "google/gemma-7b-it", + ): keywords = [f"{example_key}:" for example_key in self._example_keys] pattern = "|".join(map(re.escape, keywords)) json_response_list = []