From d60ec21dc669493e7524646214cf873e7fb107b3 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Mon, 23 May 2022 11:06:40 +0200 Subject: [PATCH 1/3] Fix metadata validation --- src/datasets/utils/metadata.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/datasets/utils/metadata.py b/src/datasets/utils/metadata.py index eac998ab189..135cbce8cc2 100644 --- a/src/datasets/utils/metadata.py +++ b/src/datasets/utils/metadata.py @@ -5,6 +5,10 @@ from dataclasses import dataclass, fields from pathlib import Path from typing import Any, Callable, ClassVar, Dict, List, Optional, Tuple, Type, Union +try: # Python >= 3.8 + from typing import get_args +except ImportError: + get_args = lambda x: x.__args__ # loading package files: https://stackoverflow.com/a/20885799 @@ -132,7 +136,7 @@ def validate_type(value: Any, expected_type: Type): # Add more `elif` statements if primitive type checking is needed else: expected_type_name = str(expected_type).split(".", 1)[-1].split("[")[0] # typing.List[str] -> List - expected_type_args = expected_type.__args__ + expected_type_args = get_args(expected_type) if expected_type_name == "Union": for type_arg in expected_type_args: @@ -161,7 +165,7 @@ def validate_type(value: Any, expected_type: Type): if expected_type_name == "Dict": if not isinstance(value, dict): return f"Expected `{expected_type}` with length > 0. Found value of type: `{type(value)}`, with length: {len(value)}.\n" - if expected_type_args != Dict.__args__: # if we specified types for keys and values + if expected_type_args != get_args(Dict): # if we specified types for keys and values key_type, value_type = expected_type_args key_error_string = "" value_error_string = "" @@ -174,7 +178,7 @@ def validate_type(value: Any, expected_type: Type): else: # `List`/`Tuple` if not isinstance(value, (list, tuple)): return f"Expected `{expected_type}` with length > 0. Found value of type: `{type(value)}`, with length: {len(value)}.\n" - if expected_type_args != List.__args__: # if we specified types for the items in the list + if expected_type_args != get_args(List): # if we specified types for the items in the list value_type = expected_type_args[0] value_error_string = "" for v in value: From 5bfc03a869172d7976dfbc50623a2094c9f3f667 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Mon, 23 May 2022 11:14:25 +0200 Subject: [PATCH 2/3] Fix style --- src/datasets/utils/metadata.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/datasets/utils/metadata.py b/src/datasets/utils/metadata.py index 135cbce8cc2..fe2e8112c3a 100644 --- a/src/datasets/utils/metadata.py +++ b/src/datasets/utils/metadata.py @@ -5,6 +5,8 @@ from dataclasses import dataclass, fields from pathlib import Path from typing import Any, Callable, ClassVar, Dict, List, Optional, Tuple, Type, Union + + try: # Python >= 3.8 from typing import get_args except ImportError: From b03cf47183787876dcc14fd8f9422f49a2444359 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Mon, 23 May 2022 11:20:12 +0200 Subject: [PATCH 3/3] Fix style --- src/datasets/utils/metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/datasets/utils/metadata.py b/src/datasets/utils/metadata.py index fe2e8112c3a..7fe362714ad 100644 --- a/src/datasets/utils/metadata.py +++ b/src/datasets/utils/metadata.py @@ -10,7 +10,9 @@ try: # Python >= 3.8 from typing import get_args except ImportError: - get_args = lambda x: x.__args__ + + def get_args(tp): + return tp.__args__ # loading package files: https://stackoverflow.com/a/20885799