From 7b937be1eb494942bebbd7ded0304507797cab33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Tue, 3 Jan 2023 11:47:18 +0100 Subject: [PATCH 1/8] [issue-406] rename json parser package to jsonlikedict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/parser/{json => jsonlikedict}/__init__.py | 0 .../{json => jsonlikedict}/actor_parser.py | 2 +- .../annotation_parser.py | 4 ++-- .../{json => jsonlikedict}/checksum_parser.py | 2 +- .../creation_info_parser.py | 6 +++--- .../dict_parsing_functions.py | 0 .../extracted_licensing_info_parser.py | 2 +- .../{json => jsonlikedict}/file_parser.py | 6 +++--- .../{json => jsonlikedict}/json_parser.py | 18 +++++++++--------- .../license_expression_parser.py | 2 +- .../{json => jsonlikedict}/package_parser.py | 8 ++++---- .../relationship_parser.py | 2 +- .../{json => jsonlikedict}/snippet_parser.py | 4 ++-- tests/parser/test_actor_parser.py | 2 +- tests/parser/test_annotation_parser.py | 2 +- tests/parser/test_checksum_parser.py | 2 +- tests/parser/test_creation_info_parser.py | 2 +- tests/parser/test_dict_parsing_functions.py | 2 +- .../test_extracted_licensing_info_parser.py | 2 +- tests/parser/test_file_parser.py | 4 ++-- tests/parser/test_json_parser.py | 10 +++++----- tests/parser/test_license_expression_parser.py | 2 +- tests/parser/test_package_parser.py | 4 ++-- tests/parser/test_relationship_parser.py | 2 +- tests/parser/test_snippet_parser.py | 2 +- 25 files changed, 46 insertions(+), 46 deletions(-) rename src/parser/{json => jsonlikedict}/__init__.py (100%) rename src/parser/{json => jsonlikedict}/actor_parser.py (96%) rename src/parser/{json => jsonlikedict}/annotation_parser.py (97%) rename src/parser/{json => jsonlikedict}/checksum_parser.py (92%) rename src/parser/{json => jsonlikedict}/creation_info_parser.py (96%) rename src/parser/{json => jsonlikedict}/dict_parsing_functions.py (100%) rename src/parser/{json => jsonlikedict}/extracted_licensing_info_parser.py (94%) rename src/parser/{json => jsonlikedict}/file_parser.py (94%) rename src/parser/{json => jsonlikedict}/json_parser.py (86%) rename src/parser/{json => jsonlikedict}/license_expression_parser.py (94%) rename src/parser/{json => jsonlikedict}/package_parser.py (97%) rename src/parser/{json => jsonlikedict}/relationship_parser.py (98%) rename src/parser/{json => jsonlikedict}/snippet_parser.py (96%) diff --git a/src/parser/json/__init__.py b/src/parser/jsonlikedict/__init__.py similarity index 100% rename from src/parser/json/__init__.py rename to src/parser/jsonlikedict/__init__.py diff --git a/src/parser/json/actor_parser.py b/src/parser/jsonlikedict/actor_parser.py similarity index 96% rename from src/parser/json/actor_parser.py rename to src/parser/jsonlikedict/actor_parser.py index 31082e0fc..907233aa7 100644 --- a/src/parser/json/actor_parser.py +++ b/src/parser/jsonlikedict/actor_parser.py @@ -13,7 +13,7 @@ from src.model.actor import Actor, ActorType from src.parser.error import SPDXParsingError -from src.parser.json.dict_parsing_functions import construct_or_raise_parsing_error +from src.parser.jsonlikedict.dict_parsing_functions import construct_or_raise_parsing_error class ActorParser: diff --git a/src/parser/json/annotation_parser.py b/src/parser/jsonlikedict/annotation_parser.py similarity index 97% rename from src/parser/json/annotation_parser.py rename to src/parser/jsonlikedict/annotation_parser.py index 85bf4115d..8fa928228 100644 --- a/src/parser/json/annotation_parser.py +++ b/src/parser/jsonlikedict/annotation_parser.py @@ -14,8 +14,8 @@ from src.model.actor import Actor from src.model.annotation import Annotation, AnnotationType from src.parser.error import SPDXParsingError -from src.parser.json.actor_parser import ActorParser -from src.parser.json.dict_parsing_functions import construct_or_raise_parsing_error, \ +from src.parser.jsonlikedict.actor_parser import ActorParser +from src.parser.jsonlikedict.dict_parsing_functions import construct_or_raise_parsing_error, \ parse_field_or_log_error, append_parsed_field_or_log_error, raise_parsing_error_if_logger_has_messages from src.datetime_conversions import datetime_from_str from src.parser.logger import Logger diff --git a/src/parser/json/checksum_parser.py b/src/parser/jsonlikedict/checksum_parser.py similarity index 92% rename from src/parser/json/checksum_parser.py rename to src/parser/jsonlikedict/checksum_parser.py index d6f82c412..a73a9d443 100644 --- a/src/parser/json/checksum_parser.py +++ b/src/parser/jsonlikedict/checksum_parser.py @@ -11,7 +11,7 @@ from typing import Dict, Optional from src.model.checksum import Checksum, ChecksumAlgorithm -from src.parser.json.dict_parsing_functions import raise_parsing_error_if_logger_has_messages, json_str_to_enum_name, \ +from src.parser.jsonlikedict.dict_parsing_functions import raise_parsing_error_if_logger_has_messages, json_str_to_enum_name, \ construct_or_raise_parsing_error from src.parser.logger import Logger diff --git a/src/parser/json/creation_info_parser.py b/src/parser/jsonlikedict/creation_info_parser.py similarity index 96% rename from src/parser/json/creation_info_parser.py rename to src/parser/jsonlikedict/creation_info_parser.py index 51c115d10..4b51056ad 100644 --- a/src/parser/json/creation_info_parser.py +++ b/src/parser/jsonlikedict/creation_info_parser.py @@ -17,9 +17,9 @@ from src.model.external_document_ref import ExternalDocumentRef from src.model.version import Version from src.parser.error import SPDXParsingError -from src.parser.json.actor_parser import ActorParser -from src.parser.json.checksum_parser import ChecksumParser -from src.parser.json.dict_parsing_functions import append_parsed_field_or_log_error, \ +from src.parser.jsonlikedict.actor_parser import ActorParser +from src.parser.jsonlikedict.checksum_parser import ChecksumParser +from src.parser.jsonlikedict.dict_parsing_functions import append_parsed_field_or_log_error, \ raise_parsing_error_if_logger_has_messages, construct_or_raise_parsing_error, parse_field_or_log_error, \ parse_field_or_no_assertion from src.datetime_conversions import datetime_from_str diff --git a/src/parser/json/dict_parsing_functions.py b/src/parser/jsonlikedict/dict_parsing_functions.py similarity index 100% rename from src/parser/json/dict_parsing_functions.py rename to src/parser/jsonlikedict/dict_parsing_functions.py diff --git a/src/parser/json/extracted_licensing_info_parser.py b/src/parser/jsonlikedict/extracted_licensing_info_parser.py similarity index 94% rename from src/parser/json/extracted_licensing_info_parser.py rename to src/parser/jsonlikedict/extracted_licensing_info_parser.py index dcb4360bb..1c6b51e85 100644 --- a/src/parser/json/extracted_licensing_info_parser.py +++ b/src/parser/jsonlikedict/extracted_licensing_info_parser.py @@ -12,7 +12,7 @@ from src.model.extracted_licensing_info import ExtractedLicensingInfo from src.model.spdx_no_assertion import SpdxNoAssertion -from src.parser.json.dict_parsing_functions import construct_or_raise_parsing_error, parse_field_or_no_assertion +from src.parser.jsonlikedict.dict_parsing_functions import construct_or_raise_parsing_error, parse_field_or_no_assertion from src.parser.logger import Logger diff --git a/src/parser/json/file_parser.py b/src/parser/jsonlikedict/file_parser.py similarity index 94% rename from src/parser/json/file_parser.py rename to src/parser/jsonlikedict/file_parser.py index 8b92b0290..0c63c7f24 100644 --- a/src/parser/json/file_parser.py +++ b/src/parser/jsonlikedict/file_parser.py @@ -15,11 +15,11 @@ from src.model.license_expression import LicenseExpression from src.model.spdx_no_assertion import SpdxNoAssertion from src.model.spdx_none import SpdxNone -from src.parser.json.checksum_parser import ChecksumParser -from src.parser.json.dict_parsing_functions import raise_parsing_error_if_logger_has_messages, \ +from src.parser.jsonlikedict.checksum_parser import ChecksumParser +from src.parser.jsonlikedict.dict_parsing_functions import raise_parsing_error_if_logger_has_messages, \ construct_or_raise_parsing_error, parse_field_or_log_error, \ parse_field_or_no_assertion_or_none -from src.parser.json.license_expression_parser import LicenseExpressionParser +from src.parser.jsonlikedict.license_expression_parser import LicenseExpressionParser from src.parser.logger import Logger diff --git a/src/parser/json/json_parser.py b/src/parser/jsonlikedict/json_parser.py similarity index 86% rename from src/parser/json/json_parser.py rename to src/parser/jsonlikedict/json_parser.py index e81e71636..8440433ff 100644 --- a/src/parser/json/json_parser.py +++ b/src/parser/jsonlikedict/json_parser.py @@ -12,19 +12,19 @@ from src.model.document import Document from src.parser.error import SPDXParsingError -from src.parser.json.annotation_parser import AnnotationParser -from src.parser.json.creation_info_parser import CreationInfoParser -from src.parser.json.dict_parsing_functions import raise_parsing_error_if_logger_has_messages, \ +from src.parser.jsonlikedict.annotation_parser import AnnotationParser +from src.parser.jsonlikedict.creation_info_parser import CreationInfoParser +from src.parser.jsonlikedict.dict_parsing_functions import raise_parsing_error_if_logger_has_messages, \ construct_or_raise_parsing_error, parse_list_of_elements -from src.parser.json.extracted_licensing_info_parser import ExtractedLicensingInfoParser -from src.parser.json.file_parser import FileParser +from src.parser.jsonlikedict.extracted_licensing_info_parser import ExtractedLicensingInfoParser +from src.parser.jsonlikedict.file_parser import FileParser from src.parser.logger import Logger -from src.parser.json.package_parser import PackageParser -from src.parser.json.relationship_parser import RelationshipParser -from src.parser.json.snippet_parser import SnippetParser +from src.parser.jsonlikedict.package_parser import PackageParser +from src.parser.jsonlikedict.relationship_parser import RelationshipParser +from src.parser.jsonlikedict.snippet_parser import SnippetParser -class JsonParser: +class JsonLikeDictParser: logger: Logger creation_info_parser: CreationInfoParser package_parser: PackageParser diff --git a/src/parser/json/license_expression_parser.py b/src/parser/jsonlikedict/license_expression_parser.py similarity index 94% rename from src/parser/json/license_expression_parser.py rename to src/parser/jsonlikedict/license_expression_parser.py index 5fc0df4e1..b1ce2468a 100644 --- a/src/parser/json/license_expression_parser.py +++ b/src/parser/jsonlikedict/license_expression_parser.py @@ -12,7 +12,7 @@ from src.model.license_expression import LicenseExpression from src.parser.error import SPDXParsingError -from src.parser.json.dict_parsing_functions import construct_or_raise_parsing_error, append_parsed_field_or_log_error, \ +from src.parser.jsonlikedict.dict_parsing_functions import construct_or_raise_parsing_error, append_parsed_field_or_log_error, \ raise_parsing_error_if_logger_has_messages from src.parser.logger import Logger diff --git a/src/parser/json/package_parser.py b/src/parser/jsonlikedict/package_parser.py similarity index 97% rename from src/parser/json/package_parser.py rename to src/parser/jsonlikedict/package_parser.py index 82c1e4d95..ed58d8096 100644 --- a/src/parser/json/package_parser.py +++ b/src/parser/jsonlikedict/package_parser.py @@ -18,13 +18,13 @@ from src.model.spdx_no_assertion import SpdxNoAssertion from src.model.spdx_none import SpdxNone from src.parser.error import SPDXParsingError -from src.parser.json.actor_parser import ActorParser -from src.parser.json.checksum_parser import ChecksumParser -from src.parser.json.dict_parsing_functions import append_parsed_field_or_log_error, \ +from src.parser.jsonlikedict.actor_parser import ActorParser +from src.parser.jsonlikedict.checksum_parser import ChecksumParser +from src.parser.jsonlikedict.dict_parsing_functions import append_parsed_field_or_log_error, \ raise_parsing_error_if_logger_has_messages, json_str_to_enum_name, construct_or_raise_parsing_error, \ parse_field_or_log_error, parse_field_or_no_assertion_or_none, parse_field_or_no_assertion from src.datetime_conversions import datetime_from_str -from src.parser.json.license_expression_parser import LicenseExpressionParser +from src.parser.jsonlikedict.license_expression_parser import LicenseExpressionParser from src.parser.logger import Logger diff --git a/src/parser/json/relationship_parser.py b/src/parser/jsonlikedict/relationship_parser.py similarity index 98% rename from src/parser/json/relationship_parser.py rename to src/parser/jsonlikedict/relationship_parser.py index e2c910e51..5221f1c15 100644 --- a/src/parser/json/relationship_parser.py +++ b/src/parser/jsonlikedict/relationship_parser.py @@ -13,7 +13,7 @@ from src.model.relationship import Relationship, RelationshipType from src.model.typing.constructor_type_errors import ConstructorTypeErrors from src.parser.error import SPDXParsingError -from src.parser.json.dict_parsing_functions import raise_parsing_error_if_logger_has_messages, json_str_to_enum_name, \ +from src.parser.jsonlikedict.dict_parsing_functions import raise_parsing_error_if_logger_has_messages, json_str_to_enum_name, \ construct_or_raise_parsing_error, \ parse_field_or_log_error, parse_field_or_no_assertion_or_none from src.parser.logger import Logger diff --git a/src/parser/json/snippet_parser.py b/src/parser/jsonlikedict/snippet_parser.py similarity index 96% rename from src/parser/json/snippet_parser.py rename to src/parser/jsonlikedict/snippet_parser.py index 97ab6632c..c91f90826 100644 --- a/src/parser/json/snippet_parser.py +++ b/src/parser/jsonlikedict/snippet_parser.py @@ -16,10 +16,10 @@ from src.model.spdx_no_assertion import SpdxNoAssertion from src.model.spdx_none import SpdxNone from src.parser.error import SPDXParsingError -from src.parser.json.dict_parsing_functions import construct_or_raise_parsing_error, parse_field_or_log_error, \ +from src.parser.jsonlikedict.dict_parsing_functions import construct_or_raise_parsing_error, parse_field_or_log_error, \ parse_field_or_no_assertion_or_none -from src.parser.json.license_expression_parser import LicenseExpressionParser +from src.parser.jsonlikedict.license_expression_parser import LicenseExpressionParser from src.parser.logger import Logger diff --git a/tests/parser/test_actor_parser.py b/tests/parser/test_actor_parser.py index d73376c9d..93429e42c 100644 --- a/tests/parser/test_actor_parser.py +++ b/tests/parser/test_actor_parser.py @@ -13,7 +13,7 @@ from src.model.actor import ActorType from src.parser.error import SPDXParsingError -from src.parser.json.actor_parser import ActorParser +from src.parser.jsonlikedict.actor_parser import ActorParser @pytest.mark.parametrize("actor_string,expected_type,expected_name,expected_mail", [ diff --git a/tests/parser/test_annotation_parser.py b/tests/parser/test_annotation_parser.py index 0330b5b0b..1ffd819f8 100644 --- a/tests/parser/test_annotation_parser.py +++ b/tests/parser/test_annotation_parser.py @@ -16,7 +16,7 @@ from src.model.actor import Actor, ActorType from src.model.annotation import AnnotationType, Annotation from src.parser.error import SPDXParsingError -from src.parser.json.annotation_parser import AnnotationParser +from src.parser.jsonlikedict.annotation_parser import AnnotationParser def test_parse_annotation(): diff --git a/tests/parser/test_checksum_parser.py b/tests/parser/test_checksum_parser.py index 11ad1610c..86d4b6d5d 100644 --- a/tests/parser/test_checksum_parser.py +++ b/tests/parser/test_checksum_parser.py @@ -14,7 +14,7 @@ from src.model.checksum import ChecksumAlgorithm from src.parser.error import SPDXParsingError -from src.parser.json.checksum_parser import ChecksumParser +from src.parser.jsonlikedict.checksum_parser import ChecksumParser def test_parse_checksum(): diff --git a/tests/parser/test_creation_info_parser.py b/tests/parser/test_creation_info_parser.py index 8126b7fae..0193c1938 100644 --- a/tests/parser/test_creation_info_parser.py +++ b/tests/parser/test_creation_info_parser.py @@ -18,7 +18,7 @@ from src.model.external_document_ref import ExternalDocumentRef from src.model.version import Version from src.parser.error import SPDXParsingError -from src.parser.json.creation_info_parser import CreationInfoParser +from src.parser.jsonlikedict.creation_info_parser import CreationInfoParser def test_parse_creation_info(): diff --git a/tests/parser/test_dict_parsing_functions.py b/tests/parser/test_dict_parsing_functions.py index c77458f94..8d9f80d32 100644 --- a/tests/parser/test_dict_parsing_functions.py +++ b/tests/parser/test_dict_parsing_functions.py @@ -16,7 +16,7 @@ from src.model.spdx_no_assertion import SpdxNoAssertion from src.model.spdx_none import SpdxNone from src.parser.error import SPDXParsingError -from src.parser.json.dict_parsing_functions import json_str_to_enum_name, \ +from src.parser.jsonlikedict.dict_parsing_functions import json_str_to_enum_name, \ parse_field_or_no_assertion, parse_field_or_no_assertion_or_none from src.datetime_conversions import datetime_from_str diff --git a/tests/parser/test_extracted_licensing_info_parser.py b/tests/parser/test_extracted_licensing_info_parser.py index 8b529aace..848576d37 100644 --- a/tests/parser/test_extracted_licensing_info_parser.py +++ b/tests/parser/test_extracted_licensing_info_parser.py @@ -13,7 +13,7 @@ import pytest from src.parser.error import SPDXParsingError -from src.parser.json.extracted_licensing_info_parser import ExtractedLicensingInfoParser +from src.parser.jsonlikedict.extracted_licensing_info_parser import ExtractedLicensingInfoParser def test_parse_extracted_licensing_info(): diff --git a/tests/parser/test_file_parser.py b/tests/parser/test_file_parser.py index e3ae52ea6..bf26e29e8 100644 --- a/tests/parser/test_file_parser.py +++ b/tests/parser/test_file_parser.py @@ -16,8 +16,8 @@ from src.model.file import FileType from src.model.license_expression import LicenseExpression from src.parser.error import SPDXParsingError -from src.parser.json.dict_parsing_functions import parse_list_of_elements -from src.parser.json.file_parser import FileParser +from src.parser.jsonlikedict.dict_parsing_functions import parse_list_of_elements +from src.parser.jsonlikedict.file_parser import FileParser def test_parse_file(): diff --git a/tests/parser/test_json_parser.py b/tests/parser/test_json_parser.py index 65a24cc47..78c2733f7 100644 --- a/tests/parser/test_json_parser.py +++ b/tests/parser/test_json_parser.py @@ -13,18 +13,18 @@ import pytest from src.model.document import Document -from src.parser.json.json_parser import JsonParser +from src.parser.jsonlikedict.json_parser import JsonLikeDictParser def test_parse_json_file_not_found(): with pytest.raises(FileNotFoundError) as err: wrong_file_path = os.path.join(os.path.dirname(__file__), 'hnjfkjsedhnflsiafg.json') - JsonParser().parse(wrong_file_path) + JsonLikeDictParser().parse(wrong_file_path) assert err.value.args[1] == "No such file or directory" def test_parse_json_with_2_3_example(): - doc = JsonParser().parse(os.path.join(os.path.dirname(__file__),"../data/formats/SPDXJSONExample-v2.3.spdx.json")) + doc = JsonLikeDictParser().parse(os.path.join(os.path.dirname(__file__), "../data/formats/SPDXJSONExample-v2.3.spdx.json")) assert type(doc) == Document assert len(doc.annotations) == 5 assert len(doc.files) == 5 @@ -34,7 +34,7 @@ def test_parse_json_with_2_3_example(): assert len(doc.extracted_licensing_info) == 5 def test_parse_json_with_2_2_example(): - doc = JsonParser().parse(os.path.join(os.path.dirname(__file__),"../data/formats/SPDXJSONExample-v2.2.spdx.json")) + doc = JsonLikeDictParser().parse(os.path.join(os.path.dirname(__file__), "../data/formats/SPDXJSONExample-v2.2.spdx.json")) assert type(doc) == Document assert len(doc.annotations) == 5 assert len(doc.files) == 4 @@ -44,7 +44,7 @@ def test_parse_json_with_2_2_example(): assert len(doc.extracted_licensing_info) == 5 def test_parse_json_with_2_1_example(): - doc = JsonParser().parse(os.path.join(os.path.dirname(__file__),"../data/formats/SPDXJsonExample.json")) + doc = JsonLikeDictParser().parse(os.path.join(os.path.dirname(__file__), "../data/formats/SPDXJsonExample.json")) assert type(doc) == Document assert len(doc.annotations) == 1 assert len(doc.files) == 2 diff --git a/tests/parser/test_license_expression_parser.py b/tests/parser/test_license_expression_parser.py index 89e23a59a..7e30e2ab2 100644 --- a/tests/parser/test_license_expression_parser.py +++ b/tests/parser/test_license_expression_parser.py @@ -14,7 +14,7 @@ from src.model.license_expression import LicenseExpression from src.parser.error import SPDXParsingError -from src.parser.json.license_expression_parser import LicenseExpressionParser +from src.parser.jsonlikedict.license_expression_parser import LicenseExpressionParser @pytest.mark.parametrize("invalid_license_expression,expected_message", diff --git a/tests/parser/test_package_parser.py b/tests/parser/test_package_parser.py index d5b9d9e7e..5d04331e3 100644 --- a/tests/parser/test_package_parser.py +++ b/tests/parser/test_package_parser.py @@ -18,8 +18,8 @@ from src.model.license_expression import LicenseExpression from src.model.package import PackageVerificationCode, ExternalPackageRef, ExternalPackageRefCategory, PackagePurpose from src.parser.error import SPDXParsingError -from src.parser.json.dict_parsing_functions import parse_list_of_elements -from src.parser.json.package_parser import PackageParser +from src.parser.jsonlikedict.dict_parsing_functions import parse_list_of_elements +from src.parser.jsonlikedict.package_parser import PackageParser def test_parse_package(): diff --git a/tests/parser/test_relationship_parser.py b/tests/parser/test_relationship_parser.py index 27f2c4f09..83bdd212e 100644 --- a/tests/parser/test_relationship_parser.py +++ b/tests/parser/test_relationship_parser.py @@ -15,7 +15,7 @@ from src.model.relationship import RelationshipType, Relationship from src.model.spdx_no_assertion import SpdxNoAssertion from src.parser.error import SPDXParsingError -from src.parser.json.relationship_parser import RelationshipParser +from src.parser.jsonlikedict.relationship_parser import RelationshipParser def test_parse_relationship(): diff --git a/tests/parser/test_snippet_parser.py b/tests/parser/test_snippet_parser.py index 96e1baee9..c068ffd97 100644 --- a/tests/parser/test_snippet_parser.py +++ b/tests/parser/test_snippet_parser.py @@ -14,7 +14,7 @@ from src.model.license_expression import LicenseExpression from src.parser.error import SPDXParsingError -from src.parser.json.snippet_parser import SnippetParser +from src.parser.jsonlikedict.snippet_parser import SnippetParser def test_parse_snippet(): From 466b8dc0f5ee35f42361a77df858162d5f197586 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Tue, 3 Jan 2023 12:02:34 +0100 Subject: [PATCH 2/8] [issue-406] extract json parser and restructure test order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/parser/json/__init__.py | 0 src/parser/json/json_parser.py | 22 +++++++ ...son_parser.py => json_like_dict_parser.py} | 32 +++++----- tests/parser/json/__init__.py | 0 tests/parser/json/test_json_parser.py | 58 +++++++++++++++++++ tests/parser/jsonlikedict/__init__.py | 0 .../{ => jsonlikedict}/test_actor_parser.py | 0 .../test_annotation_parser.py | 0 .../test_checksum_parser.py | 0 .../test_creation_info_parser.py | 0 .../test_dict_parsing_functions.py | 0 .../test_extracted_licensing_info_parser.py | 0 .../{ => jsonlikedict}/test_file_parser.py | 0 .../test_license_expression_parser.py | 0 .../{ => jsonlikedict}/test_package_parser.py | 0 .../test_relationship_parser.py | 0 .../{ => jsonlikedict}/test_snippet_parser.py | 0 tests/parser/test_json_parser.py | 54 ----------------- 18 files changed, 95 insertions(+), 71 deletions(-) create mode 100644 src/parser/json/__init__.py create mode 100644 src/parser/json/json_parser.py rename src/parser/jsonlikedict/{json_parser.py => json_like_dict_parser.py} (70%) create mode 100644 tests/parser/json/__init__.py create mode 100644 tests/parser/json/test_json_parser.py create mode 100644 tests/parser/jsonlikedict/__init__.py rename tests/parser/{ => jsonlikedict}/test_actor_parser.py (100%) rename tests/parser/{ => jsonlikedict}/test_annotation_parser.py (100%) rename tests/parser/{ => jsonlikedict}/test_checksum_parser.py (100%) rename tests/parser/{ => jsonlikedict}/test_creation_info_parser.py (100%) rename tests/parser/{ => jsonlikedict}/test_dict_parsing_functions.py (100%) rename tests/parser/{ => jsonlikedict}/test_extracted_licensing_info_parser.py (100%) rename tests/parser/{ => jsonlikedict}/test_file_parser.py (100%) rename tests/parser/{ => jsonlikedict}/test_license_expression_parser.py (100%) rename tests/parser/{ => jsonlikedict}/test_package_parser.py (100%) rename tests/parser/{ => jsonlikedict}/test_relationship_parser.py (100%) rename tests/parser/{ => jsonlikedict}/test_snippet_parser.py (100%) delete mode 100644 tests/parser/test_json_parser.py diff --git a/src/parser/json/__init__.py b/src/parser/json/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/parser/json/json_parser.py b/src/parser/json/json_parser.py new file mode 100644 index 000000000..e02da72c7 --- /dev/null +++ b/src/parser/json/json_parser.py @@ -0,0 +1,22 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from typing import Dict + +from src.model.document import Document +from src.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser + + +def parse_from_file(file_name: str) -> Document: + with open(file_name) as file: + input_doc_as_dict: Dict = json.load(file) + + return JsonLikeDictParser().parse(input_doc_as_dict) diff --git a/src/parser/jsonlikedict/json_parser.py b/src/parser/jsonlikedict/json_like_dict_parser.py similarity index 70% rename from src/parser/jsonlikedict/json_parser.py rename to src/parser/jsonlikedict/json_like_dict_parser.py index 8440433ff..db78629bb 100644 --- a/src/parser/jsonlikedict/json_parser.py +++ b/src/parser/jsonlikedict/json_like_dict_parser.py @@ -9,6 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +from typing import Dict from src.model.document import Document from src.parser.error import SPDXParsingError @@ -44,24 +45,21 @@ def __init__(self): self.relationship_parser = RelationshipParser() self.annotation_parser = AnnotationParser() - def parse(self, filename: str) -> Document: + def parse(self, json_like_dict: Dict) -> Document: - with open(filename) as file: - input_doc_as_dict = json.load(file) - - fields_to_parse = [("creation_info", input_doc_as_dict, self.creation_info_parser.parse_creation_info, False), - ("packages", input_doc_as_dict.get("packages"), lambda x: parse_list_of_elements(x, - self.package_parser.parse_package, - self.package_parser.logger), True), - ("files", input_doc_as_dict.get("files"), lambda x: parse_list_of_elements(x, - self.file_parser.parse_file, - self.file_parser.logger), True), - ("annotations", input_doc_as_dict, self.annotation_parser.parse_all_annotations, True), - ("snippets", input_doc_as_dict.get("snippets"), lambda x: parse_list_of_elements(x, - self.snippet_parser.parse_snippet, - self.snippet_parser.logger), True), - ("relationships", input_doc_as_dict, self.relationship_parser.parse_all_relationships, True), - ("extracted_licensing_info", input_doc_as_dict.get("hasExtractedLicensingInfos"), + fields_to_parse = [("creation_info", json_like_dict, self.creation_info_parser.parse_creation_info, False), + ("packages", json_like_dict.get("packages"), lambda x: parse_list_of_elements(x, + self.package_parser.parse_package, + self.package_parser.logger), True), + ("files", json_like_dict.get("files"), lambda x: parse_list_of_elements(x, + self.file_parser.parse_file, + self.file_parser.logger), True), + ("annotations", json_like_dict, self.annotation_parser.parse_all_annotations, True), + ("snippets", json_like_dict.get("snippets"), lambda x: parse_list_of_elements(x, + self.snippet_parser.parse_snippet, + self.snippet_parser.logger), True), + ("relationships", json_like_dict, self.relationship_parser.parse_all_relationships, True), + ("extracted_licensing_info", json_like_dict.get("hasExtractedLicensingInfos"), lambda x: parse_list_of_elements(x, self.extracted_licensing_info_parser.parse_extracted_licensing_info, self.extracted_licensing_info_parser.logger), True)] diff --git a/tests/parser/json/__init__.py b/tests/parser/json/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parser/json/test_json_parser.py b/tests/parser/json/test_json_parser.py new file mode 100644 index 000000000..b21d4b4cb --- /dev/null +++ b/tests/parser/json/test_json_parser.py @@ -0,0 +1,58 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest + +from src.model.document import Document +from src.parser.json import json_parser +from src.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser + +def test_parse_json_file_not_found(): + with pytest.raises(FileNotFoundError) as err: + wrong_file_path = os.path.join(os.path.dirname(__file__), 'hnjfkjsedhnflsiafg.json') + json_parser.parse_from_file(wrong_file_path) + + assert err.value.args[1] == "No such file or directory" + + +def test_parse_json_with_2_3_example(): + doc = json_parser.parse_from_file(os.path.join(os.path.dirname(__file__), + "../../data/formats/SPDXJSONExample-v2.3.spdx.json")) + assert type(doc) == Document + assert len(doc.annotations) == 5 + assert len(doc.files) == 5 + assert len(doc.packages) == 4 + assert len(doc.snippets) == 1 + assert len(doc.relationships) == 23 + assert len(doc.extracted_licensing_info) == 5 + +def test_parse_json_with_2_2_example(): + doc = json_parser.parse_from_file(os.path.join(os.path.dirname(__file__), + "../../data/formats/SPDXJSONExample-v2.2.spdx.json")) + assert type(doc) == Document + assert len(doc.annotations) == 5 + assert len(doc.files) == 4 + assert len(doc.packages) == 4 + assert len(doc.snippets) == 1 + assert len(doc.relationships) == 11 + assert len(doc.extracted_licensing_info) == 5 + +def test_parse_json_with_2_1_example(): + doc = json_parser.parse_from_file(os.path.join(os.path.dirname(__file__), + "../../data/formats/SPDXJsonExample.json")) + assert type(doc) == Document + assert len(doc.annotations) == 1 + assert len(doc.files) == 2 + assert len(doc.packages) == 1 + assert len(doc.snippets) == 1 + assert len(doc.relationships) == 3 + assert len(doc.extracted_licensing_info) == 4 diff --git a/tests/parser/jsonlikedict/__init__.py b/tests/parser/jsonlikedict/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parser/test_actor_parser.py b/tests/parser/jsonlikedict/test_actor_parser.py similarity index 100% rename from tests/parser/test_actor_parser.py rename to tests/parser/jsonlikedict/test_actor_parser.py diff --git a/tests/parser/test_annotation_parser.py b/tests/parser/jsonlikedict/test_annotation_parser.py similarity index 100% rename from tests/parser/test_annotation_parser.py rename to tests/parser/jsonlikedict/test_annotation_parser.py diff --git a/tests/parser/test_checksum_parser.py b/tests/parser/jsonlikedict/test_checksum_parser.py similarity index 100% rename from tests/parser/test_checksum_parser.py rename to tests/parser/jsonlikedict/test_checksum_parser.py diff --git a/tests/parser/test_creation_info_parser.py b/tests/parser/jsonlikedict/test_creation_info_parser.py similarity index 100% rename from tests/parser/test_creation_info_parser.py rename to tests/parser/jsonlikedict/test_creation_info_parser.py diff --git a/tests/parser/test_dict_parsing_functions.py b/tests/parser/jsonlikedict/test_dict_parsing_functions.py similarity index 100% rename from tests/parser/test_dict_parsing_functions.py rename to tests/parser/jsonlikedict/test_dict_parsing_functions.py diff --git a/tests/parser/test_extracted_licensing_info_parser.py b/tests/parser/jsonlikedict/test_extracted_licensing_info_parser.py similarity index 100% rename from tests/parser/test_extracted_licensing_info_parser.py rename to tests/parser/jsonlikedict/test_extracted_licensing_info_parser.py diff --git a/tests/parser/test_file_parser.py b/tests/parser/jsonlikedict/test_file_parser.py similarity index 100% rename from tests/parser/test_file_parser.py rename to tests/parser/jsonlikedict/test_file_parser.py diff --git a/tests/parser/test_license_expression_parser.py b/tests/parser/jsonlikedict/test_license_expression_parser.py similarity index 100% rename from tests/parser/test_license_expression_parser.py rename to tests/parser/jsonlikedict/test_license_expression_parser.py diff --git a/tests/parser/test_package_parser.py b/tests/parser/jsonlikedict/test_package_parser.py similarity index 100% rename from tests/parser/test_package_parser.py rename to tests/parser/jsonlikedict/test_package_parser.py diff --git a/tests/parser/test_relationship_parser.py b/tests/parser/jsonlikedict/test_relationship_parser.py similarity index 100% rename from tests/parser/test_relationship_parser.py rename to tests/parser/jsonlikedict/test_relationship_parser.py diff --git a/tests/parser/test_snippet_parser.py b/tests/parser/jsonlikedict/test_snippet_parser.py similarity index 100% rename from tests/parser/test_snippet_parser.py rename to tests/parser/jsonlikedict/test_snippet_parser.py diff --git a/tests/parser/test_json_parser.py b/tests/parser/test_json_parser.py deleted file mode 100644 index 78c2733f7..000000000 --- a/tests/parser/test_json_parser.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2022 spdx contributors -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pytest - -from src.model.document import Document -from src.parser.jsonlikedict.json_parser import JsonLikeDictParser - -def test_parse_json_file_not_found(): - with pytest.raises(FileNotFoundError) as err: - wrong_file_path = os.path.join(os.path.dirname(__file__), 'hnjfkjsedhnflsiafg.json') - JsonLikeDictParser().parse(wrong_file_path) - - assert err.value.args[1] == "No such file or directory" - - -def test_parse_json_with_2_3_example(): - doc = JsonLikeDictParser().parse(os.path.join(os.path.dirname(__file__), "../data/formats/SPDXJSONExample-v2.3.spdx.json")) - assert type(doc) == Document - assert len(doc.annotations) == 5 - assert len(doc.files) == 5 - assert len(doc.packages) == 4 - assert len(doc.snippets) == 1 - assert len(doc.relationships) == 23 - assert len(doc.extracted_licensing_info) == 5 - -def test_parse_json_with_2_2_example(): - doc = JsonLikeDictParser().parse(os.path.join(os.path.dirname(__file__), "../data/formats/SPDXJSONExample-v2.2.spdx.json")) - assert type(doc) == Document - assert len(doc.annotations) == 5 - assert len(doc.files) == 4 - assert len(doc.packages) == 4 - assert len(doc.snippets) == 1 - assert len(doc.relationships) == 11 - assert len(doc.extracted_licensing_info) == 5 - -def test_parse_json_with_2_1_example(): - doc = JsonLikeDictParser().parse(os.path.join(os.path.dirname(__file__), "../data/formats/SPDXJsonExample.json")) - assert type(doc) == Document - assert len(doc.annotations) == 1 - assert len(doc.files) == 2 - assert len(doc.packages) == 1 - assert len(doc.snippets) == 1 - assert len(doc.relationships) == 3 - assert len(doc.extracted_licensing_info) == 4 From 243c1d7170ac24ce19c6c3cdc6b40b0aba189b6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Tue, 3 Jan 2023 11:34:45 +0100 Subject: [PATCH 3/8] [issue-406] add xml writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/writer/xml/__init__.py | 10 ++++++++++ src/writer/xml/xml_writer.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 src/writer/xml/__init__.py create mode 100644 src/writer/xml/xml_writer.py diff --git a/src/writer/xml/__init__.py b/src/writer/xml/__init__.py new file mode 100644 index 000000000..cbc5c4070 --- /dev/null +++ b/src/writer/xml/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/writer/xml/xml_writer.py b/src/writer/xml/xml_writer.py new file mode 100644 index 000000000..9afa2cef2 --- /dev/null +++ b/src/writer/xml/xml_writer.py @@ -0,0 +1,36 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List + +import xmltodict + +from src.jsonschema.document_converter import DocumentConverter +from src.model.document import Document +from src.validation.document_validator import validate_full_spdx_document +from src.validation.validation_message import ValidationMessage + + +def write_document(document: Document, file_name: str, validate: bool = True, converter: DocumentConverter = None): + """ + Serializes the provided document to XML and writes it to a file with the provided name. Unless validate is set + to False, validates the document before serialization. Unless a DocumentConverter instance is provided, + a new one is created. + """ + if validate: + validation_messages: List[ValidationMessage] = validate_full_spdx_document(document, + document.creation_info.spdx_version) + if validation_messages: + raise ValueError(f"Document is not valid. The following errors were detected: {validation_messages}") + if converter is None: + converter = DocumentConverter() + document_dict = converter.convert(document) + with open(file_name, "w") as out: + xmltodict.unparse(document_dict, out, encoding="utf-8", pretty=True) From 0bd3708cafaa792d8e37c06c6ff582b4f57a3fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Tue, 3 Jan 2023 12:57:02 +0100 Subject: [PATCH 4/8] [issue-406] add xml parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/parser/xml/__init__.py | 0 src/parser/xml/xml_parser.py | 75 ++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 src/parser/xml/__init__.py create mode 100644 src/parser/xml/xml_parser.py diff --git a/src/parser/xml/__init__.py b/src/parser/xml/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/parser/xml/xml_parser.py b/src/parser/xml/xml_parser.py new file mode 100644 index 000000000..3f1237e28 --- /dev/null +++ b/src/parser/xml/xml_parser.py @@ -0,0 +1,75 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Any + +import xmltodict + +from src.model.document import Document +from src.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser + + +LIST_LIKE_FIELDS = [ + "creators", + "externalDocumentRefs", + "extractedLicenseInfos", + "seeAlsos", + "annotations", + "relationships", + "snippets", + "reviewers", + "fileTypes", + "licenseInfoFromFiles", + "licenseInfoInFiles", + "artifactOf", + "fileContributors", + "fileDependencies", + "files", + "documentDescribes", + "packages", + "checksums", + "hasFiles", + "externalRefs", + "ranges", + "licenseInfoInSnippets", + "packageVerificationCodeExcludedFiles", + ] + + +def parse_from_file(file_name: str) -> Document: + with open(file_name) as file: + parsed_xml: Dict = xmltodict.parse(file.read(), encoding="utf-8") + + input_doc_as_dict: Dict = _fix_list_like_fields(parsed_xml) + + return JsonLikeDictParser().parse(input_doc_as_dict) + + +def _fix_list_like_fields(data: Any) -> Any: + """ + XML files do not contain lists. Thus, single fields that should be a list in SPDX have to be manually cast. + This method takes a parsed dictionary and converts all values with key from LIST_LIKE_FIELDS to lists. + """ + if isinstance(data, dict): + new_data = {} + for key, value in data.items(): + if key in LIST_LIKE_FIELDS and not isinstance(value, list): + new_data[key] = [_fix_list_like_fields(value)] + else: + new_data[key] = _fix_list_like_fields(value) + return new_data + + if isinstance(data, list): + new_data = [] + for element in data: + new_data.append(_fix_list_like_fields(element)) + return new_data + + return data From 0f7060cca25b2d4899a82791fba90b674ec115e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Tue, 3 Jan 2023 11:17:48 +0100 Subject: [PATCH 5/8] [issue-406] add yaml writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/writer/yaml/__init__.py | 10 ++++++++++ src/writer/yaml/yaml_writer.py | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 src/writer/yaml/__init__.py create mode 100644 src/writer/yaml/yaml_writer.py diff --git a/src/writer/yaml/__init__.py b/src/writer/yaml/__init__.py new file mode 100644 index 000000000..cbc5c4070 --- /dev/null +++ b/src/writer/yaml/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/writer/yaml/yaml_writer.py b/src/writer/yaml/yaml_writer.py new file mode 100644 index 000000000..304c09631 --- /dev/null +++ b/src/writer/yaml/yaml_writer.py @@ -0,0 +1,36 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List + +import yaml + +from src.jsonschema.document_converter import DocumentConverter +from src.model.document import Document +from src.validation.document_validator import validate_full_spdx_document +from src.validation.validation_message import ValidationMessage + + +def write_document_to_file(document: Document, file_name: str, validate: bool = True, converter: DocumentConverter = None): + """ + Serializes the provided document to yaml and writes it to a file with the provided name. Unless validate is set + to False, validates the document before serialization. Unless a DocumentConverter instance is provided, + a new one is created. + """ + if validate: + validation_messages: List[ValidationMessage] = validate_full_spdx_document(document, + document.creation_info.spdx_version) + if validation_messages: + raise ValueError(f"Document is not valid. The following errors were detected: {validation_messages}") + if converter is None: + converter = DocumentConverter() + document_dict = converter.convert(document) + with open(file_name, "w") as out: + yaml.safe_dump(document_dict, out, indent=2) From 1ff8097fa65dc75233d95ce1abcf7bfa0f70623b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Tue, 3 Jan 2023 13:00:20 +0100 Subject: [PATCH 6/8] [issue-406] add yaml parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/parser/yaml/__init__.py | 0 src/parser/yaml/yaml_parser.py | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 src/parser/yaml/__init__.py create mode 100644 src/parser/yaml/yaml_parser.py diff --git a/src/parser/yaml/__init__.py b/src/parser/yaml/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/parser/yaml/yaml_parser.py b/src/parser/yaml/yaml_parser.py new file mode 100644 index 000000000..90c6abf0c --- /dev/null +++ b/src/parser/yaml/yaml_parser.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict + +import yaml + +from src.model.document import Document +from src.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser + + +def parse_from_file(file_name: str) -> Document: + with open(file_name) as file: + input_doc_as_dict: Dict = yaml.safe_load(file) + + return JsonLikeDictParser().parse(input_doc_as_dict) From 785ec4d5e8e486eac64a7c7e0e718f41369efd8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Tue, 3 Jan 2023 17:24:21 +0100 Subject: [PATCH 7/8] [issue-406] fix xml specific problems, update parse/write_anything MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/parser/jsonlikedict/package_parser.py | 9 ++++++++- src/parser/jsonlikedict/snippet_parser.py | 20 +++++++++++++++++-- src/parser/parse_anything.py | 10 ++++++---- src/parser/xml/xml_parser.py | 7 ++++++- src/writer/write_anything.py | 8 +++++--- src/writer/xml/xml_writer.py | 4 ++-- .../jsonlikedict/test_snippet_parser.py | 4 ++-- 7 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/parser/jsonlikedict/package_parser.py b/src/parser/jsonlikedict/package_parser.py index ed58d8096..a5a2f46e6 100644 --- a/src/parser/jsonlikedict/package_parser.py +++ b/src/parser/jsonlikedict/package_parser.py @@ -60,8 +60,15 @@ def parse_package(self, package_dict: Dict) -> Package: external_refs: List[ExternalPackageRef] = parse_field_or_log_error(logger, package_dict.get("externalRefs"), self.parse_external_refs) - files_analyzed: Optional[bool] = parse_field_or_log_error(logger, package_dict.get("filesAnalyzed"), + files_analyzed: Optional[Union[bool, str]] = parse_field_or_log_error(logger, package_dict.get("filesAnalyzed"), lambda x: x, True) + + if isinstance(files_analyzed, str): # XML does not support boolean typed values + if files_analyzed.lower() == "true": + files_analyzed = True + elif files_analyzed.lower() == "false": + files_analyzed = False + homepage: Optional[str] = package_dict.get("homepage") license_comments: Optional[str] = package_dict.get("licenseComments") license_concluded = parse_field_or_log_error( diff --git a/src/parser/jsonlikedict/snippet_parser.py b/src/parser/jsonlikedict/snippet_parser.py index c91f90826..778defc9f 100644 --- a/src/parser/jsonlikedict/snippet_parser.py +++ b/src/parser/jsonlikedict/snippet_parser.py @@ -41,9 +41,13 @@ def parse_snippet(self, snippet_dict: Dict) -> Snippet: spdx_id: Optional[str] = snippet_dict.get("SPDXID") file_spdx_id: Optional[str] = snippet_dict.get("snippetFromFile") name: Optional[str] = snippet_dict.get("name") + ranges: Dict = parse_field_or_log_error(logger, snippet_dict.get("ranges", []), self.parse_ranges, default={}) - byte_range: Tuple[int, int] = ranges.get(RangeType.BYTE) - line_range: Optional[Tuple[int, int]] = ranges.get(RangeType.LINE) + byte_range: Optional[Tuple[Union[int, str], Union[int, str]]] = ranges.get(RangeType.BYTE) + line_range: Optional[Tuple[Union[int, str], Union[int, str]]] = ranges.get(RangeType.LINE) + byte_range = self.convert_range_from_str(byte_range) + line_range = self.convert_range_from_str(line_range) + attribution_texts: List[str] = snippet_dict.get("attributionTexts", []) comment: Optional[str] = snippet_dict.get("comment") copyright_text: Optional[str] = snippet_dict.get("copyrightText") @@ -114,3 +118,15 @@ def validate_pointer_and_get_type(pointer: Dict) -> RangeType: if "offset" not in pointer and "lineNumber" not in pointer: raise ValueError('Couldn\'t determine type of pointer: neither "offset" nor "lineNumber" provided as key.') return RangeType.BYTE if "offset" in pointer else RangeType.LINE + + @staticmethod + def convert_range_from_str(_range: Tuple[Union[int, str], Union[int, str]]) -> Tuple[Union[int, str], Union[int, str]]: + # XML does not support integers, so we have to convert from string (if possible) + if not _range: + return _range + + if isinstance(_range[0], str) and _range[0].isdigit(): + _range = int(_range[0]), _range[1] + if isinstance(_range[1], str) and _range[1].isdigit(): + _range = _range[0], int(_range[1]) + return _range diff --git a/src/parser/parse_anything.py b/src/parser/parse_anything.py index 541188a59..eae23db85 100644 --- a/src/parser/parse_anything.py +++ b/src/parser/parse_anything.py @@ -9,7 +9,9 @@ # See the License for the specific language governing permissions and # limitations under the License. from src.formats import file_name_to_format, FileFormat -from src.parser.json.json_parser import JsonParser +from src.parser.json import json_parser +from src.parser.xml import xml_parser +from src.parser.yaml import yaml_parser def parse_file(file_name: str): @@ -19,8 +21,8 @@ def parse_file(file_name: str): elif input_format == FileFormat.TAG_VALUE: raise NotImplementedError("Currently, the tag-value parser is not implemented") elif input_format == FileFormat.JSON: - return JsonParser().parse(file_name) + return json_parser.parse_from_file(file_name) elif input_format == FileFormat.XML: - raise NotImplementedError("Currently, the xml parser is not implemented") + return xml_parser.parse_from_file(file_name) elif input_format == FileFormat.YAML: - raise NotImplementedError("Currently, the yaml parser is not implemented") + return yaml_parser.parse_from_file(file_name) diff --git a/src/parser/xml/xml_parser.py b/src/parser/xml/xml_parser.py index 3f1237e28..1c7d36f2f 100644 --- a/src/parser/xml/xml_parser.py +++ b/src/parser/xml/xml_parser.py @@ -13,6 +13,7 @@ import xmltodict from src.model.document import Document +from src.parser.error import SPDXParsingError from src.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser @@ -40,6 +41,7 @@ "ranges", "licenseInfoInSnippets", "packageVerificationCodeExcludedFiles", + "attributionTexts" ] @@ -47,7 +49,10 @@ def parse_from_file(file_name: str) -> Document: with open(file_name) as file: parsed_xml: Dict = xmltodict.parse(file.read(), encoding="utf-8") - input_doc_as_dict: Dict = _fix_list_like_fields(parsed_xml) + input_doc_as_dict: Dict = _fix_list_like_fields(parsed_xml).get("Document") + + if not input_doc_as_dict: + raise SPDXParsingError(['Did not find the XML top level tag "Document".']) return JsonLikeDictParser().parse(input_doc_as_dict) diff --git a/src/writer/write_anything.py b/src/writer/write_anything.py index ca01ba86e..f0a2d7176 100644 --- a/src/writer/write_anything.py +++ b/src/writer/write_anything.py @@ -12,16 +12,18 @@ from src.model.document import Document from src.writer.json import json_writer from src.writer.tagvalue import tagvalue_writer +from src.writer.xml import xml_writer +from src.writer.yaml import yaml_writer def write_file(document: Document, file_name: str, validate: bool = True): output_format = file_name_to_format(file_name) if output_format == FileFormat.JSON: - json_writer.write_document(document, file_name, validate) + json_writer.write_document(document, file_name, validate=False) elif output_format == FileFormat.YAML: - raise NotImplementedError("Currently, the yaml writer is not implemented") + yaml_writer.write_document_to_file(document, file_name, validate=False) elif output_format == FileFormat.XML: - raise NotImplementedError("Currently, the xml writer is not implemented") + xml_writer.write_document_to_file(document, file_name, validate=False) elif output_format == FileFormat.TAG_VALUE: tagvalue_writer.write_document_to_file(document, file_name) elif output_format == FileFormat.RDF_XML: diff --git a/src/writer/xml/xml_writer.py b/src/writer/xml/xml_writer.py index 9afa2cef2..23d8448d6 100644 --- a/src/writer/xml/xml_writer.py +++ b/src/writer/xml/xml_writer.py @@ -18,7 +18,7 @@ from src.validation.validation_message import ValidationMessage -def write_document(document: Document, file_name: str, validate: bool = True, converter: DocumentConverter = None): +def write_document_to_file(document: Document, file_name: str, validate: bool = True, converter: DocumentConverter = None): """ Serializes the provided document to XML and writes it to a file with the provided name. Unless validate is set to False, validates the document before serialization. Unless a DocumentConverter instance is provided, @@ -31,6 +31,6 @@ def write_document(document: Document, file_name: str, validate: bool = True, co raise ValueError(f"Document is not valid. The following errors were detected: {validation_messages}") if converter is None: converter = DocumentConverter() - document_dict = converter.convert(document) + document_dict = {"Document": converter.convert(document)} with open(file_name, "w") as out: xmltodict.unparse(document_dict, out, encoding="utf-8", pretty=True) diff --git a/tests/parser/jsonlikedict/test_snippet_parser.py b/tests/parser/jsonlikedict/test_snippet_parser.py index c068ffd97..91d1d446c 100644 --- a/tests/parser/jsonlikedict/test_snippet_parser.py +++ b/tests/parser/jsonlikedict/test_snippet_parser.py @@ -92,7 +92,7 @@ def test_parse_snippet_with_invalid_snippet_range(): "reference": "SPDXRef-DoapSource" }, "startPointer": { - "offset": "310", + "offset": "310s", "reference": "SPDXRef-DoapSource" } }] @@ -105,7 +105,7 @@ def test_parse_snippet_with_invalid_snippet_range(): ["Error while constructing Snippet: ['SetterError Snippet: type of argument " '"file_spdx_id" must be str; got NoneType instead: None\', \'SetterError ' 'Snippet: type of argument "byte_range"[0] must be int; got str instead: ' - "(\\'310\\', 23)']"]) + "(\\'310s\\', 23)']"]) def test_parse_invalid_snippet_range(): From 1108639147bae3182c09a212eaf25a89f3f2b2ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armin=20T=C3=A4nzer?= Date: Thu, 5 Jan 2023 12:11:13 +0100 Subject: [PATCH 8/8] [issue-406] fix copyright texts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Armin Tänzer --- src/parser/xml/xml_parser.py | 2 +- src/parser/yaml/yaml_parser.py | 2 +- src/writer/xml/__init__.py | 10 ---------- src/writer/xml/xml_writer.py | 2 +- src/writer/yaml/__init__.py | 10 ---------- src/writer/yaml/yaml_writer.py | 2 +- 6 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/parser/xml/xml_parser.py b/src/parser/xml/xml_parser.py index 1c7d36f2f..fa4dccfce 100644 --- a/src/parser/xml/xml_parser.py +++ b/src/parser/xml/xml_parser.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 spdx contributors +# Copyright (c) 2023 spdx contributors # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/src/parser/yaml/yaml_parser.py b/src/parser/yaml/yaml_parser.py index 90c6abf0c..b607a153d 100644 --- a/src/parser/yaml/yaml_parser.py +++ b/src/parser/yaml/yaml_parser.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 spdx contributors +# Copyright (c) 2023 spdx contributors # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/src/writer/xml/__init__.py b/src/writer/xml/__init__.py index cbc5c4070..e69de29bb 100644 --- a/src/writer/xml/__init__.py +++ b/src/writer/xml/__init__.py @@ -1,10 +0,0 @@ -# Copyright (c) 2022 spdx contributors -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/src/writer/xml/xml_writer.py b/src/writer/xml/xml_writer.py index 23d8448d6..06f7a83de 100644 --- a/src/writer/xml/xml_writer.py +++ b/src/writer/xml/xml_writer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 spdx contributors +# Copyright (c) 2023 spdx contributors # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/src/writer/yaml/__init__.py b/src/writer/yaml/__init__.py index cbc5c4070..e69de29bb 100644 --- a/src/writer/yaml/__init__.py +++ b/src/writer/yaml/__init__.py @@ -1,10 +0,0 @@ -# Copyright (c) 2022 spdx contributors -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/src/writer/yaml/yaml_writer.py b/src/writer/yaml/yaml_writer.py index 304c09631..434314921 100644 --- a/src/writer/yaml/yaml_writer.py +++ b/src/writer/yaml/yaml_writer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 spdx contributors +# Copyright (c) 2023 spdx contributors # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at