From 14161b1c9bf36780d9c09b72a5b60745924eee10 Mon Sep 17 00:00:00 2001 From: Jerzy Zagorski Date: Fri, 27 Sep 2024 12:48:09 +0200 Subject: [PATCH 1/3] Parse literals out of override-neuron-config For now it was only possible to add string parameters, if a bool or number was passed, the neuron initialization failed. The parsing was also inconsisten with how other composite configs were passed (see --mm-processor-kwargs) --- tests/engine/test_arg_utils.py | 22 ++++++++++++++-------- vllm/engine/arg_utils.py | 9 +++------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 360ac1bfbad9..66c2d5d256dd 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -42,22 +42,28 @@ def test_bad_nullable_kvs(arg): nullable_kvs(arg) -@pytest.mark.parametrize(("arg", "expected"), [ - (None, None), - ("{}", {}), +@pytest.mark.parametrize(("arg", "expected", "option"), [ + (None, None, "mm-processor-kwargs"), + ("{}", {}, "mm-processor-kwargs"), ('{"num_crops": 4}', { "num_crops": 4 - }), + }, "mm-processor-kwargs"), ('{"foo": {"bar": "baz"}}', { "foo": { "bar": "baz" } - }), + }, "mm-processor-kwargs"), + ('{"cast_logits_dtype":"bfloat16","sequence_parallel_norm":true,"sequence_parallel_norm_threshold":2048}', + { + "cast_logits_dtype": "bfloat16", + "sequence_parallel_norm": True, + "sequence_parallel_norm_threshold": 2048, + }, "override-neuron-config"), ]) -def test_mm_processor_kwargs_prompt_parser(arg, expected): +def test_mm_processor_kwargs_prompt_parser(arg, expected, option): parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) if arg is None: args = parser.parse_args([]) else: - args = parser.parse_args(["--mm-processor-kwargs", arg]) - assert args.mm_processor_kwargs == expected + args = parser.parse_args([f"--{option}", arg]) + assert getattr(args, option.replace("-", "_")) == expected diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 0d4559e37742..152470430e60 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -789,13 +789,10 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: "lower performance.") parser.add_argument( '--override-neuron-config', - type=lambda configs: { - str(key): value - for key, value in - (config.split(':') for config in configs.split(',')) - }, + type=json.loads, default=None, - help="override or set neuron device configuration.") + help="Override or set neuron device configuration. " + "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'") return parser From f93ec216e6430062b43ea99df1e90fba1ffacf4f Mon Sep 17 00:00:00 2001 From: Jerzy Zagorski Date: Mon, 30 Sep 2024 11:26:51 +0200 Subject: [PATCH 2/3] Fix test name --- tests/engine/test_arg_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 66c2d5d256dd..28ad3456a3a5 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -60,7 +60,7 @@ def test_bad_nullable_kvs(arg): "sequence_parallel_norm_threshold": 2048, }, "override-neuron-config"), ]) -def test_mm_processor_kwargs_prompt_parser(arg, expected, option): +def test_composite_arg_parser(arg, expected, option): parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) if arg is None: args = parser.parse_args([]) From 63dcb6c886dba16f21507dbac70e093dc6a5cde6 Mon Sep 17 00:00:00 2001 From: Jerzy Zagorski Date: Thu, 3 Oct 2024 15:50:55 +0200 Subject: [PATCH 3/3] Prevent autoformating of test parameters --- tests/engine/test_arg_utils.py | 42 ++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 28ad3456a3a5..f7dc167fea6e 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -42,24 +42,38 @@ def test_bad_nullable_kvs(arg): nullable_kvs(arg) +# yapf: disable @pytest.mark.parametrize(("arg", "expected", "option"), [ (None, None, "mm-processor-kwargs"), ("{}", {}, "mm-processor-kwargs"), - ('{"num_crops": 4}', { - "num_crops": 4 - }, "mm-processor-kwargs"), - ('{"foo": {"bar": "baz"}}', { - "foo": { - "bar": "baz" - } - }, "mm-processor-kwargs"), - ('{"cast_logits_dtype":"bfloat16","sequence_parallel_norm":true,"sequence_parallel_norm_threshold":2048}', - { - "cast_logits_dtype": "bfloat16", - "sequence_parallel_norm": True, - "sequence_parallel_norm_threshold": 2048, - }, "override-neuron-config"), + ( + '{"num_crops": 4}', + { + "num_crops": 4 + }, + "mm-processor-kwargs" + ), + ( + '{"foo": {"bar": "baz"}}', + { + "foo": + { + "bar": "baz" + } + }, + "mm-processor-kwargs" + ), + ( + '{"cast_logits_dtype":"bfloat16","sequence_parallel_norm":true,"sequence_parallel_norm_threshold":2048}', + { + "cast_logits_dtype": "bfloat16", + "sequence_parallel_norm": True, + "sequence_parallel_norm_threshold": 2048, + }, + "override-neuron-config" + ), ]) +# yapf: enable def test_composite_arg_parser(arg, expected, option): parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) if arg is None: