From e575e785eb2dcc8f5f581890d04c346e55782bf0 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Wed, 22 Jan 2025 11:08:11 +0200 Subject: [PATCH 1/8] initial commit Signed-off-by: zhouyu5 --- .../guided_decoding/outlines_logits_processors.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/guided_decoding/outlines_logits_processors.py b/vllm/model_executor/guided_decoding/outlines_logits_processors.py index e4eb3f16e56c..59299341091a 100644 --- a/vllm/model_executor/guided_decoding/outlines_logits_processors.py +++ b/vllm/model_executor/guided_decoding/outlines_logits_processors.py @@ -29,6 +29,7 @@ from outlines_core.fsm.json_schema import build_regex_from_schema from pydantic import BaseModel from transformers import PreTrainedTokenizerBase +from vllm.platforms import current_platform class BaseLogitsProcessor: @@ -89,7 +90,10 @@ def __call__(self, input_ids: List[int], allowed_tokens = allowed_tokens.masked_select( allowed_tokens < scores.shape[-1]) mask.index_fill_(0, allowed_tokens, 0) - scores.add_(mask) + if current_platform.is_hpu(): + scores = scores.add(mask) + else: + scores.add_(mask) return scores From 663618c95deaa0362714eba5c7c2aa1afe4ebee9 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Thu, 23 Jan 2025 10:11:43 +0200 Subject: [PATCH 2/8] add comment Signed-off-by: zhouyu5 --- .../guided_decoding/outlines_logits_processors.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vllm/model_executor/guided_decoding/outlines_logits_processors.py b/vllm/model_executor/guided_decoding/outlines_logits_processors.py index 59299341091a..26659b741d87 100644 --- a/vllm/model_executor/guided_decoding/outlines_logits_processors.py +++ b/vllm/model_executor/guided_decoding/outlines_logits_processors.py @@ -91,6 +91,10 @@ def __call__(self, input_ids: List[int], allowed_tokens < scores.shape[-1]) mask.index_fill_(0, allowed_tokens, 0) if current_platform.is_hpu(): + # Workaround for HPU bug where add_() raise RuntimeError: + # synNodeCreateWithId failed for node: strided_insert + # with synStatus 1 [Invalid argument], hopefully it will + # be fixed in the future releases of the HPU runtime. scores = scores.add(mask) else: scores.add_(mask) From 6038128d779a7012fcc186c3b532475cd744f941 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Thu, 23 Jan 2025 11:04:02 +0200 Subject: [PATCH 3/8] fix format Signed-off-by: zhouyu5 --- .../guided_decoding/outlines_logits_processors.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/guided_decoding/outlines_logits_processors.py b/vllm/model_executor/guided_decoding/outlines_logits_processors.py index 26659b741d87..6accfac33b00 100644 --- a/vllm/model_executor/guided_decoding/outlines_logits_processors.py +++ b/vllm/model_executor/guided_decoding/outlines_logits_processors.py @@ -29,6 +29,7 @@ from outlines_core.fsm.json_schema import build_regex_from_schema from pydantic import BaseModel from transformers import PreTrainedTokenizerBase + from vllm.platforms import current_platform @@ -91,8 +92,8 @@ def __call__(self, input_ids: List[int], allowed_tokens < scores.shape[-1]) mask.index_fill_(0, allowed_tokens, 0) if current_platform.is_hpu(): - # Workaround for HPU bug where add_() raise RuntimeError: - # synNodeCreateWithId failed for node: strided_insert + # Workaround for HPU bug where add_() raise RuntimeError: + # synNodeCreateWithId failed for node: strided_insert # with synStatus 1 [Invalid argument], hopefully it will # be fixed in the future releases of the HPU runtime. scores = scores.add(mask) From 19757851141febeb3b0579f25d3f80cff9bcb433 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Fri, 24 Jan 2025 05:15:54 +0200 Subject: [PATCH 4/8] add ci test Signed-off-by: zhouyu5 --- .buildkite/run-hpu-test.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.buildkite/run-hpu-test.sh b/.buildkite/run-hpu-test.sh index 1edcb1d2669e..39afee0412c9 100644 --- a/.buildkite/run-hpu-test.sh +++ b/.buildkite/run-hpu-test.sh @@ -20,5 +20,10 @@ trap remove_docker_container_and_exit EXIT remove_docker_container # Run the image and launch offline inference -docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference/basic.py +docker run --runtime=habana --name=hpu-test --network=host \ + -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true \ + --entrypoint="" hpu-test-env \ + /bin/bash -c "python3 examples/offline_inference/basic.py \ + && python3 -m pip install pytest \ + && pytest -v -s /workspace/vllm/tests/entrypoints/llm/test_guided_generate.py" EXITCODE=$? From f5caa088b6bee8b1f15b9c9b1a35fd3b7278d3ac Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Sun, 26 Jan 2025 08:41:56 +0200 Subject: [PATCH 5/8] Revert "add ci test" This reverts commit 38d47131699889d7b4f666a35c36b39bec18aceb. Signed-off-by: zhouyu5 --- .buildkite/run-hpu-test.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.buildkite/run-hpu-test.sh b/.buildkite/run-hpu-test.sh index 39afee0412c9..1edcb1d2669e 100644 --- a/.buildkite/run-hpu-test.sh +++ b/.buildkite/run-hpu-test.sh @@ -20,10 +20,5 @@ trap remove_docker_container_and_exit EXIT remove_docker_container # Run the image and launch offline inference -docker run --runtime=habana --name=hpu-test --network=host \ - -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true \ - --entrypoint="" hpu-test-env \ - /bin/bash -c "python3 examples/offline_inference/basic.py \ - && python3 -m pip install pytest \ - && pytest -v -s /workspace/vllm/tests/entrypoints/llm/test_guided_generate.py" +docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference/basic.py EXITCODE=$? From 1904e69d7f17ec2b7c63374877cdf10b42b5ddf9 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Mon, 10 Feb 2025 03:21:05 +0200 Subject: [PATCH 6/8] Dummy commit for triggering CI Signed-off-by: zhouyu5 From 10003bbc6694675ed9172b388547fb534eeb130e Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Mon, 10 Feb 2025 10:18:27 +0200 Subject: [PATCH 7/8] Dummy commit for triggering CI Signed-off-by: zhouyu5 From 56b6644f62070c743c8fc4a47939dbad1dd55a60 Mon Sep 17 00:00:00 2001 From: zhouyu5 Date: Tue, 11 Feb 2025 03:22:16 +0200 Subject: [PATCH 8/8] Dummy commit for triggering CI Signed-off-by: zhouyu5