diff --git a/src/datetime_conversions.py b/src/datetime_conversions.py new file mode 100644 index 000000000..3c49d9391 --- /dev/null +++ b/src/datetime_conversions.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from datetime import datetime + +from src.parser.error import SPDXParsingError + + +def datetime_from_str(date_str: str) -> datetime: + if not isinstance(date_str, str): + raise SPDXParsingError([f"Could not convert str to datetime, invalid type: {type(date_str).__name__}"]) + try: + date = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + raise SPDXParsingError( + [f'Could not convert str to datetime, format of {date_str} does not match "%Y-%m-%dT%H:%M:%SZ"']) + return date + +def datetime_to_iso_string(date: datetime) -> str: + """ + Return an ISO-8601 representation of a datetime object. + """ + return date.isoformat() + "Z" + diff --git a/src/model/actor.py b/src/model/actor.py index fff81c811..a44da8939 100644 --- a/src/model/actor.py +++ b/src/model/actor.py @@ -29,3 +29,10 @@ class Actor: def __init__(self, actor_type: ActorType, name: str, email: Optional[str] = None): check_types_and_set_values(self, locals()) + + def to_serialized_string(self) -> str: + """ + All serialization formats use the same representation of an actor, so this method is included in the data model + """ + optional_email = f" ({self.email})" if self.email else "" + return "".join([f"{self.actor_type.name.title()}:", f" {self.name}", optional_email]) diff --git a/src/parser/json/annotation_parser.py b/src/parser/json/annotation_parser.py index 44bbe6242..85bf4115d 100644 --- a/src/parser/json/annotation_parser.py +++ b/src/parser/json/annotation_parser.py @@ -15,8 +15,9 @@ from src.model.annotation import Annotation, AnnotationType from src.parser.error import SPDXParsingError from src.parser.json.actor_parser import ActorParser -from src.parser.json.dict_parsing_functions import datetime_from_str, construct_or_raise_parsing_error, \ +from src.parser.json.dict_parsing_functions import construct_or_raise_parsing_error, \ parse_field_or_log_error, append_parsed_field_or_log_error, raise_parsing_error_if_logger_has_messages +from src.datetime_conversions import datetime_from_str from src.parser.logger import Logger diff --git a/src/parser/json/creation_info_parser.py b/src/parser/json/creation_info_parser.py index 5f8eaab40..51c115d10 100644 --- a/src/parser/json/creation_info_parser.py +++ b/src/parser/json/creation_info_parser.py @@ -19,9 +19,10 @@ from src.parser.error import SPDXParsingError from src.parser.json.actor_parser import ActorParser from src.parser.json.checksum_parser import ChecksumParser -from src.parser.json.dict_parsing_functions import append_parsed_field_or_log_error, datetime_from_str, \ +from src.parser.json.dict_parsing_functions import append_parsed_field_or_log_error, \ raise_parsing_error_if_logger_has_messages, construct_or_raise_parsing_error, parse_field_or_log_error, \ parse_field_or_no_assertion +from src.datetime_conversions import datetime_from_str from src.parser.logger import Logger diff --git a/src/parser/json/dict_parsing_functions.py b/src/parser/json/dict_parsing_functions.py index e7b0dd707..e413802f9 100644 --- a/src/parser/json/dict_parsing_functions.py +++ b/src/parser/json/dict_parsing_functions.py @@ -8,7 +8,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime from typing import Any, Callable, Dict, List, Optional from src.model.spdx_no_assertion import SpdxNoAssertion @@ -18,17 +17,6 @@ from src.parser.logger import Logger -def datetime_from_str(date_str: str) -> datetime: - if not isinstance(date_str, str): - raise SPDXParsingError([f"Could not convert str to datetime, invalid type: {type(date_str).__name__}"]) - try: - date = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ") - except ValueError: - raise SPDXParsingError( - [f'Could not convert str to datetime, format of {date_str} does not match "%Y-%m-%dT%H:%M:%SZ"']) - return date - - def json_str_to_enum_name(json_str: str) -> str: if not isinstance(json_str, str): raise SPDXParsingError([f"Type for enum must be str not {type(json_str).__name__}"]) diff --git a/src/parser/json/package_parser.py b/src/parser/json/package_parser.py index 5c2c53494..82c1e4d95 100644 --- a/src/parser/json/package_parser.py +++ b/src/parser/json/package_parser.py @@ -20,9 +20,10 @@ from src.parser.error import SPDXParsingError from src.parser.json.actor_parser import ActorParser from src.parser.json.checksum_parser import ChecksumParser -from src.parser.json.dict_parsing_functions import append_parsed_field_or_log_error, datetime_from_str, \ +from src.parser.json.dict_parsing_functions import append_parsed_field_or_log_error, \ raise_parsing_error_if_logger_has_messages, json_str_to_enum_name, construct_or_raise_parsing_error, \ parse_field_or_log_error, parse_field_or_no_assertion_or_none, parse_field_or_no_assertion +from src.datetime_conversions import datetime_from_str from src.parser.json.license_expression_parser import LicenseExpressionParser from src.parser.logger import Logger diff --git a/src/writer/tagvalue/__init__.py b/src/writer/tagvalue/__init__.py new file mode 100644 index 000000000..b0981f064 --- /dev/null +++ b/src/writer/tagvalue/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/writer/tagvalue/annotation_writer.py b/src/writer/tagvalue/annotation_writer.py new file mode 100644 index 000000000..cb7c29ef4 --- /dev/null +++ b/src/writer/tagvalue/annotation_writer.py @@ -0,0 +1,26 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TextIO + +from src.datetime_conversions import datetime_to_iso_string +from src.model.annotation import Annotation +from src.writer.tagvalue.tagvalue_writer_helper_functions import write_value, write_text_value + + +def write_annotation(annotation: Annotation, text_output: TextIO): + """ + Write the fields of a single annotation to text_output. + """ + write_value("Annotator", annotation.annotator.to_serialized_string(), text_output) + write_value("AnnotationDate", datetime_to_iso_string(annotation.annotation_date), text_output) + write_text_value("AnnotationComment", annotation.annotation_comment, text_output, True) + write_value("AnnotationType", annotation.annotation_type.name, text_output) + write_value("SPDXREF", annotation.spdx_id, text_output, True) diff --git a/src/writer/tagvalue/checksum_writer.py b/src/writer/tagvalue/checksum_writer.py new file mode 100644 index 000000000..c5803181e --- /dev/null +++ b/src/writer/tagvalue/checksum_writer.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from src.model.checksum import Checksum, ChecksumAlgorithm + + +def write_checksum_to_tag_value(checksum: Checksum) -> str: + algorithm_name: str = checksum.algorithm.name + # Convert underscores to dashes, and other Blake2b-specific casing rules + if "_" in algorithm_name: + algorithm_name = CHECKSUM_ALGORITHM_TO_TV.get(algorithm_name) + if algorithm_name is None: + raise ValueError(f"Missing conversion rule for converting {checksum.algorithm.name} to tag-value string") + return "{0}: {1}".format(algorithm_name, checksum.value) + + +CHECKSUM_ALGORITHM_TO_TV = { + ChecksumAlgorithm.BLAKE2B_256.name: "BLAKE2b-256", + ChecksumAlgorithm.BLAKE2B_384.name: "BLAKE2b-384", + ChecksumAlgorithm.BLAKE2B_512.name: "BLAKE2b-512", + ChecksumAlgorithm.SHA3_256.name: "SHA3-256", + ChecksumAlgorithm.SHA3_384.name: "SHA3-384", + ChecksumAlgorithm.SHA3_512.name: "SHA3-512" +} diff --git a/src/writer/tagvalue/creation_info_writer.py b/src/writer/tagvalue/creation_info_writer.py new file mode 100644 index 000000000..3e7cc20d3 --- /dev/null +++ b/src/writer/tagvalue/creation_info_writer.py @@ -0,0 +1,46 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TextIO + +from src.datetime_conversions import datetime_to_iso_string +from src.model.document import CreationInfo +from src.writer.tagvalue.tagvalue_writer_helper_functions import write_value, write_text_value, write_optional_heading, \ + write_separator + + +def write_creation_info(creation_info: CreationInfo, text_output: TextIO): + """ + Write the creation info to text_output. + """ + write_value("SPDXVersion", creation_info.spdx_version, text_output) + write_value("DataLicense", creation_info.data_license, text_output) + write_value("DocumentNamespace", creation_info.document_namespace, text_output, True) + write_value("DocumentName", creation_info.name, text_output, True) + write_value("LicenseListVersion", str(creation_info.spdx_version), text_output, True) + write_value("SPDXID", creation_info.spdx_id, text_output) + write_text_value("DocumentComment", creation_info.document_comment, text_output, True) + + write_optional_heading(creation_info.external_document_refs, "\n## External Document References\n", text_output) + for external_document_ref in creation_info.external_document_refs: + external_document_ref_str = " ".join([external_document_ref.document_ref_id, external_document_ref.document_uri, + external_document_ref.checksum.algorithm.name + ": " + external_document_ref.checksum.value]) + write_value("ExternalDocumentRef", external_document_ref_str, text_output) + write_separator(text_output) + + text_output.write("## Creation Information\n") + # Write sorted creators + for creator in creation_info.creators: + write_value("Creator", creator.to_serialized_string(), text_output) + + # write created + write_value("Created", datetime_to_iso_string(creation_info.created), text_output) + # possible comment + write_text_value("CreatorComment", creation_info.creator_comment, text_output, True) diff --git a/src/writer/tagvalue/extracted_licensing_info_writer.py b/src/writer/tagvalue/extracted_licensing_info_writer.py new file mode 100644 index 000000000..e77587379 --- /dev/null +++ b/src/writer/tagvalue/extracted_licensing_info_writer.py @@ -0,0 +1,28 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TextIO + +from src.model.extracted_licensing_info import ExtractedLicensingInfo +from src.writer.tagvalue.tagvalue_writer_helper_functions import write_value, write_text_value + + +def write_extracted_licensing_info(extracted_licensing_info: ExtractedLicensingInfo, text_output: TextIO): + """ + Write extracted licenses fields to out. + """ + write_value("LicenseID", extracted_licensing_info.license_id, text_output) + write_value("LicenseName", extracted_licensing_info.license_name, text_output, True) + write_text_value("LicenseComment", extracted_licensing_info.comment, text_output, True) + + for cross_reference in sorted(extracted_licensing_info.cross_references): + write_value("LicenseCrossReference", cross_reference, text_output) + + write_text_value("ExtractedText", extracted_licensing_info.extracted_text, text_output) diff --git a/src/writer/tagvalue/file_writer.py b/src/writer/tagvalue/file_writer.py new file mode 100644 index 000000000..635f01815 --- /dev/null +++ b/src/writer/tagvalue/file_writer.py @@ -0,0 +1,42 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TextIO + +from src.model.file import File +from src.writer.tagvalue.tagvalue_writer_helper_functions import write_value, write_text_value, \ + write_field_or_none_or_no_assertion +from src.writer.tagvalue.checksum_writer import write_checksum_to_tag_value + + +def write_file(file: File, text_output: TextIO): + """ + Write all file information to output_text. + """ + text_output.write("## File Information\n") + write_value("FileName", file.name, text_output) + write_value("SPDXID", file.spdx_id, text_output, True) + for file_type in file.file_type: + write_value("FileType", file_type.name, text_output) + for file_checksum in file.checksums: + write_value("FileChecksum", write_checksum_to_tag_value(file_checksum), text_output) + write_field_or_none_or_no_assertion("LicenseConcluded", file.concluded_license, text_output, True) + write_field_or_none_or_no_assertion("LicenseInfoInFile", file.license_info_in_file, text_output, True) + write_field_or_none_or_no_assertion("FileCopyrightText", file.copyright_text, text_output, True) + write_text_value("LicenseComments", file.license_comment, text_output, True) + + for attribution_text in file.attribution_texts: + write_text_value("FileAttributionText", attribution_text, text_output) + + write_text_value("FileComment", file.comment, text_output, True) + write_text_value("FileNotice", file.notice, text_output, True) + + for contributor in sorted(file.contributors): + write_value("FileContributor", contributor, text_output, True) diff --git a/src/writer/tagvalue/package_writer.py b/src/writer/tagvalue/package_writer.py new file mode 100644 index 000000000..dde9de6ce --- /dev/null +++ b/src/writer/tagvalue/package_writer.py @@ -0,0 +1,86 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TextIO + +from src.datetime_conversions import datetime_to_iso_string +from src.model.package import Package, PackageVerificationCode +from src.writer.tagvalue.tagvalue_writer_helper_functions import write_value, write_text_value, \ + write_field_or_none_or_no_assertion, transform_enum_name_to_tv, write_actor_or_no_assertion +from src.writer.tagvalue.checksum_writer import write_checksum_to_tag_value + + +def write_package(package: Package, text_output: TextIO): + """ + Write all package information to text_output. + """ + text_output.write("## Package Information\n") + + write_value("PackageName", package.name, text_output, True) + write_value("SPDXID", package.spdx_id, text_output, True) + write_value("PackageVersion", package.version, text_output, True) + write_value("PackageDownloadLocation", package.download_location, text_output) + write_value("FilesAnalyzed", package.files_analyzed, text_output, True) + write_text_value("PackageSummary", package.summary, text_output, True) + for attribution_text in package.attribution_texts: + write_text_value("PackageAttributionText", attribution_text, text_output) + + write_text_value("PackageSourceInfo", package.source_info, text_output, True) + write_value("PackageFileName", package.file_name, text_output, True) + write_actor_or_no_assertion("PackageSupplier", package.supplier, text_output, True) + write_actor_or_no_assertion("PackageOriginator", package.originator, text_output, True) + + for package_checksum in package.checksums: + write_value("PackageChecksum", write_checksum_to_tag_value(package_checksum), text_output, True) + + if package.verification_code: + package_verification_code = write_package_verification_code(package.verification_code) + write_value("PackageVerificationCode", package_verification_code, text_output, True) + + write_text_value("PackageDescription", package.description, text_output, True) + write_text_value("PackageComment", package.comment, text_output, True) + + write_field_or_none_or_no_assertion("PackageLicenseDeclared", package.license_declared, text_output, True) + write_field_or_none_or_no_assertion("PackageLicenseConcluded", package.license_concluded, text_output, True) + write_field_or_none_or_no_assertion("PackageLicenseInfoFromFiles", package.license_info_from_files, text_output, + True) + + write_text_value("PackageLicenseComments", package.license_comment, text_output, True) + write_field_or_none_or_no_assertion("PackageCopyrightText", package.copyright_text, text_output, True) + + write_value("PackageHomePage", package.homepage, text_output, True) + + for external_reference in package.external_references: + external_reference_str = " ".join( + [transform_enum_name_to_tv(external_reference.category.name), external_reference.reference_type, + external_reference.locator] + ) + write_value("ExternalRef", external_reference_str, text_output, True) + if external_reference.comment: + write_text_value("ExternalRefComment", external_reference.comment, text_output) + + if package.primary_package_purpose: + write_value("PrimaryPackagePurpose", transform_enum_name_to_tv(package.primary_package_purpose.name), + text_output) + + if package.built_date: + write_value("BuiltDate", datetime_to_iso_string(package.built_date), text_output) + if package.release_date: + write_value("ReleaseDate", datetime_to_iso_string(package.release_date), text_output) + if package.valid_until_date: + write_value("ValidUntilDate", datetime_to_iso_string(package.valid_until_date), text_output) + + +def write_package_verification_code(verification_code: PackageVerificationCode): + if not verification_code.excluded_files: + return verification_code.value + + excluded_files_str = " (excludes: " + " ".join(verification_code.excluded_files) + ")" + return verification_code.value + excluded_files_str diff --git a/src/writer/tagvalue/relationship_writer.py b/src/writer/tagvalue/relationship_writer.py new file mode 100644 index 000000000..4740b4503 --- /dev/null +++ b/src/writer/tagvalue/relationship_writer.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TextIO + +from src.model.relationship import Relationship +from src.writer.tagvalue.tagvalue_writer_helper_functions import write_value, write_text_value + + +def write_relationship(relationship: Relationship, text_output: TextIO): + """ + Write a relationship to text_output. + """ + write_value("Relationship", " ".join( + [relationship.spdx_element_id, relationship.relationship_type.name, relationship.related_spdx_element_id]), + text_output) + write_text_value("RelationshipComment", relationship.comment, text_output, True) diff --git a/src/writer/tagvalue/snippet_writer.py b/src/writer/tagvalue/snippet_writer.py new file mode 100644 index 000000000..d59e233f0 --- /dev/null +++ b/src/writer/tagvalue/snippet_writer.py @@ -0,0 +1,34 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TextIO + +from src.model.snippet import Snippet +from src.writer.tagvalue.tagvalue_writer_helper_functions import write_value, write_text_value, write_range, \ + write_field_or_none_or_no_assertion + + +def write_snippet(snippet: Snippet, output_text: TextIO): + """ + Write snippet fields to out. + """ + output_text.write("## Snippet Information\n") + write_value("SnippetSPDXID", snippet.spdx_id, output_text) + write_value("SnippetFromFileSPDXID", snippet.file_spdx_id, output_text) + write_text_value("SnippetCopyrightText", snippet.copyright_text, output_text, True) + write_range("SnippetByteRange", snippet.byte_range, output_text) + write_range("SnippetLineRange", snippet.line_range, output_text, True) + write_value("SnippetName", snippet.name, output_text, True) + write_text_value("SnippetComment", snippet.comment, output_text, True) + write_text_value("SnippetLicenseComments", snippet.license_comment, output_text, True) + for attribution_text in snippet.attribution_texts: + write_text_value("SnippetAttributionText", attribution_text, output_text) + write_field_or_none_or_no_assertion("SnippetLicenseConcluded", snippet.concluded_license, output_text, True) + write_field_or_none_or_no_assertion("LicenseInfoInSnippet", snippet.license_info_in_snippet, output_text, True) diff --git a/src/writer/tagvalue/tagvalue_writer.py b/src/writer/tagvalue/tagvalue_writer.py new file mode 100644 index 000000000..216bfeb49 --- /dev/null +++ b/src/writer/tagvalue/tagvalue_writer.py @@ -0,0 +1,87 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TextIO + +from src.model.document import Document +from src.writer.tagvalue.annotation_writer import write_annotation +from src.writer.tagvalue.creation_info_writer import write_creation_info +from src.writer.tagvalue.extracted_licensing_info_writer import write_extracted_licensing_info +from src.writer.tagvalue.file_writer import write_file +from src.writer.tagvalue.package_writer import write_package +from src.writer.tagvalue.relationship_writer import write_relationship +from src.writer.tagvalue.snippet_writer import write_snippet +from src.writer.tagvalue.tagvalue_writer_helper_functions import write_separator, scan_relationships, \ + determine_files_containing_snippets, write_optional_heading, write_list_of_elements + + +def write_document_to_file(document: Document, file_name: str): + with open(file_name, "w") as out: + write_document(document, out) + + +def write_document(document: Document, text_output: TextIO): + """ + Write a SPDX tag value document. + - document - src.document instance. + - text_output - file like object that will be written to. + """ + + text_output.write("## Document Information\n") + # Write out creation info + write_creation_info(document.creation_info, text_output) + write_separator(text_output) + + # Write sorted annotations + write_optional_heading(document.annotations, "## Annotations\n", text_output) + write_list_of_elements(document.annotations, write_annotation, text_output, True) + + relationships_to_write, contained_files_by_package_id = scan_relationships(document.relationships, + document.packages, document.files) + contained_snippets_by_file_id = determine_files_containing_snippets(document.snippets, document.files) + packaged_file_ids = [file.spdx_id for files_list in contained_files_by_package_id.values() + for file in files_list] + filed_snippet_ids = [snippet.spdx_id for snippets_list in contained_snippets_by_file_id.values() + for snippet in snippets_list] + + # Write Relationships + write_optional_heading(relationships_to_write, "## Relationships\n", text_output) + write_list_of_elements(relationships_to_write, write_relationship, text_output) + write_separator(text_output) + + # Write snippet info + for snippet in document.snippets: + if snippet.spdx_id not in filed_snippet_ids: + write_snippet(snippet, text_output) + write_separator(text_output) + + # Write file info + for file in document.files: + if file.spdx_id not in packaged_file_ids: + write_file(file, text_output) + write_separator(text_output) + if file.spdx_id in contained_snippets_by_file_id: + write_list_of_elements(contained_snippets_by_file_id[file.spdx_id], write_snippet, text_output, True) + + # Write package info + for package in document.packages: + write_package(package, text_output) + write_separator(text_output) + if package.spdx_id in contained_files_by_package_id: + for file in contained_files_by_package_id[package.spdx_id]: + write_file(file, text_output) + write_separator(text_output) + if file.spdx_id in contained_snippets_by_file_id: + write_list_of_elements(contained_snippets_by_file_id[file.spdx_id], write_snippet, text_output, True) + break + + write_optional_heading(document.extracted_licensing_info, "## License Information\n", text_output) + write_list_of_elements(document.extracted_licensing_info, write_extracted_licensing_info, text_output, True) diff --git a/src/writer/tagvalue/tagvalue_writer_helper_functions.py b/src/writer/tagvalue/tagvalue_writer_helper_functions.py new file mode 100644 index 000000000..95ca5654e --- /dev/null +++ b/src/writer/tagvalue/tagvalue_writer_helper_functions.py @@ -0,0 +1,131 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TextIO, Tuple, List, Dict, Any, Union, Callable + +from src.model.actor import Actor +from src.model.file import File +from src.model.license_expression import LicenseExpression +from src.model.package import Package +from src.model.relationship import Relationship +from src.model.snippet import Snippet +from src.model.spdx_no_assertion import SpdxNoAssertion +from src.model.spdx_none import SpdxNone + + +def write_separator(out: TextIO): + out.write("\n") + + +def write_value(tag: str, value: Union[bool, str, SpdxNone, SpdxNoAssertion], out: TextIO, optional: bool = False): + if optional and not value: + return + out.write(f"{tag}: {value}\n") + + +def write_range(tag: str, value: Tuple[int, int], out: TextIO, optional: bool = False): + if optional and not value: + return + out.write(f"{tag}: {value[0]}:{value[1]}\n") + + +def write_text_value(tag: str, value: str, out: TextIO, optional: bool = False): + if optional and not value: + return + if "\n" in value: + out.write(f"{tag}: {value}\n") + else: + write_value(tag, value, out, True) + + +def transform_enum_name_to_tv(enum_str: str) -> str: + return enum_str.replace("_", "-") + + +def write_optional_heading(optional_field: Any, heading: str, text_output: TextIO): + if not optional_field: + return + text_output.write(heading) + + +def write_list_of_elements(list_of_elements: List[Any], write_method: Callable[[Any, TextIO], None], + text_output: TextIO, with_separator: bool = False): + for element in list_of_elements: + write_method(element, text_output) + if with_separator: + write_separator(text_output) + + +def write_actor_or_no_assertion(tag: str, element_to_write: Any, text_output: TextIO, optional: bool): + if optional and not element_to_write: + return + if isinstance(element_to_write, SpdxNoAssertion): + write_value(tag, element_to_write, text_output) + return + if isinstance(element_to_write, Actor): + write_value(tag, element_to_write.to_serialized_string(), text_output) + return + else: + write_value(tag, element_to_write, text_output) + + +def write_field_or_none_or_no_assertion(tag: str, element_to_write: Union[ + List[LicenseExpression], LicenseExpression, SpdxNoAssertion, SpdxNone], text_output: TextIO, + optional: bool = False): + if optional and not element_to_write: + return + if isinstance(element_to_write, (SpdxNone, SpdxNoAssertion)): + write_value(tag, element_to_write, text_output) + return + if isinstance(element_to_write, LicenseExpression): + write_value(tag, element_to_write.expression_string, text_output) + return + if isinstance(element_to_write, str): + write_value(tag, element_to_write, text_output) + return + if isinstance(element_to_write, list): + for element in element_to_write: + write_value(tag, element.expression_string, text_output) + + +def scan_relationships(relationships: List[Relationship], packages: List[Package], files: List[File]) \ + -> Tuple[List, Dict]: + contained_files_by_package_id = dict() + relationships_to_write = [] + files_by_spdx_id = {file.spdx_id: file for file in files} + packages_spdx_ids = [package.spdx_id for package in packages] + for relationship in relationships: + if relationship.relationship_type == "CONTAINS" and \ + relationship.spdx_element_id in packages_spdx_ids and \ + relationship.related_spdx_element in files_by_spdx_id.keys(): + contained_files_by_package_id.setdefault(relationship.spdx_element_id, []).append( + files_by_spdx_id[relationship.related_spdx_element]) + if relationship.has_comment: + relationships_to_write.append(relationship) + elif relationship.relationship_type == "CONTAINED_BY" and \ + relationship.related_spdx_element in packages_spdx_ids and \ + relationship.spdx_element_id in files_by_spdx_id: + contained_files_by_package_id.setdefault(relationship.related_spdx_element, []).append( + files_by_spdx_id[relationship.spdx_element_id]) + if relationship.has_comment: + relationships_to_write.append(relationship) + else: + relationships_to_write.append(relationship) + + return relationships_to_write, contained_files_by_package_id + + +def determine_files_containing_snippets(snippets: List[Snippet], files: List[File]) -> Dict: + contained_snippets_by_file_id = dict() + for snippet in snippets: + if snippet.file_spdx_id in [file.spdx_id for file in files]: + contained_snippets_by_file_id.setdefault(snippet.file_spdx_id, []).append(snippet) + + return contained_snippets_by_file_id diff --git a/tests/parser/test_dict_parsing_functions.py b/tests/parser/test_dict_parsing_functions.py index 688f44df9..3c8b872b9 100644 --- a/tests/parser/test_dict_parsing_functions.py +++ b/tests/parser/test_dict_parsing_functions.py @@ -16,8 +16,9 @@ from src.model.spdx_no_assertion import SpdxNoAssertion from src.model.spdx_none import SpdxNone from src.parser.error import SPDXParsingError -from src.parser.json.dict_parsing_functions import datetime_from_str, json_str_to_enum_name, \ +from src.parser.json.dict_parsing_functions import json_str_to_enum_name, \ parse_field_or_no_assertion, parse_field_or_no_assertion_or_none +from src.datetime_conversions import datetime_from_str def test_datetime_from_str(): diff --git a/tests/writer/tagvalue/__init__.py b/tests/writer/tagvalue/__init__.py new file mode 100644 index 000000000..c30b311b3 --- /dev/null +++ b/tests/writer/tagvalue/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/writer/tagvalue/expected_results/expected_tag_value.spdx b/tests/writer/tagvalue/expected_results/expected_tag_value.spdx new file mode 100644 index 000000000..799178b40 --- /dev/null +++ b/tests/writer/tagvalue/expected_results/expected_tag_value.spdx @@ -0,0 +1,54 @@ +## Document Information +SPDXVersion: spdxVersion +DataLicense: dataLicense +DocumentNamespace: documentNamespace +DocumentName: documentName +SPDXID: documentId +DocumentComment: comment + +## External Document References +ExternalDocumentRef: docRefId externalDocumentUri SHA1: externalRefSha1 + +## Creation Information +Creator: Tool: tools-python (tools-python@github.com) +Created: 2022-12-01T00:00:00Z + +## Annotations +Annotator: Person: reviewerName +AnnotationDate: 2022-12-02T00:00:00Z +AnnotationComment: reviewComment +AnnotationType: REVIEW +SPDXREF: documentId + +Annotator: Tool: toolName +AnnotationDate: 2022-12-03T00:00:00Z +AnnotationComment: otherComment +AnnotationType: OTHER +SPDXREF: fileId + +## Relationships +Relationship: documentId DESCRIBES packageId +Relationship: documentId DESCRIBES fileId +RelationshipComment: relationshipComment +Relationship: relationshipOriginId AMENDS relationShipTargetId + +## Snippet Information +SnippetSPDXID: snippetId +SnippetFromFileSPDXID: snippetFileId +SnippetByteRange: 1:2 + +## File Information +FileName: fileName +SPDXID: fileId +FileChecksum: SHA1: fileSha1 + +## Package Information +PackageName: packageName +SPDXID: packageId +PackageDownloadLocation: NONE +FilesAnalyzed: True + + +## License Information +LicenseID: licenseId +ExtractedText: licenseText diff --git a/tests/writer/tagvalue/test_package_writer.py b/tests/writer/tagvalue/test_package_writer.py new file mode 100644 index 000000000..1fbd469ea --- /dev/null +++ b/tests/writer/tagvalue/test_package_writer.py @@ -0,0 +1,54 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from datetime import datetime +from unittest.mock import patch, mock_open, call + +from src.model.license_expression import LicenseExpression +from src.model.package import PackagePurpose +from src.model.spdx_no_assertion import SpdxNoAssertion +from src.model.spdx_none import SpdxNone +from src.writer.tagvalue.package_writer import write_package +from tests.valid_defaults import get_package, get_package_verification_code, get_actor, get_checksum, \ + get_external_package_ref + + +def test_package_writer(): + package = get_package("SPDXRef-Package", "package name", "www.download.com", "version", "file_name", SpdxNoAssertion(), + get_actor(), True, + get_package_verification_code(), [get_checksum()], "https://homepage.com", "source_info", None, + [LicenseExpression("expression")], + SpdxNone(), "comment on license", "copyright", "summary", "description", "comment", + [get_external_package_ref()], ["text"], PackagePurpose.OTHER, datetime(2022, 1, 1), None, None) + + m = mock_open() + with patch('{}.open'.format(__name__), m, create=True): + with open('foo', 'w') as h: + write_package(package, h) + + m.assert_called_once_with('foo', 'w') + handle = m() + handle.write.assert_has_calls( + [call('## Package Information\n'), call('PackageName: package name\n'), call('SPDXID: SPDXRef-Package\n'), + call('PackageVersion: version\n'), call('PackageDownloadLocation: www.download.com\n'), + call('FilesAnalyzed: True\n'), call('PackageSummary: summary\n'), call('PackageAttributionText: text\n'), + call('PackageSourceInfo: source_info\n'), call('PackageFileName: file_name\n'), + call('PackageSupplier: NOASSERTION\n'), + call("PackageOriginator: Person: person name\n"), + call('PackageChecksum: SHA1: 85ed0817af83a24ad8da68c2b5094de69833983c\n'), + call('PackageVerificationCode: 85ed0817af83a24ad8da68c2b5094de69833983c\n'), + call('PackageDescription: description\n'), call('PackageComment: comment\n'), + call('PackageLicenseDeclared: NONE\n'), + call("PackageLicenseInfoFromFiles: expression\n"), + call('PackageLicenseComments: comment on license\n'), call('PackageCopyrightText: copyright\n'), + call('PackageHomePage: https://homepage.com\n'), + call('ExternalRef: SECURITY cpe22Type cpe:/o:canonical:ubuntu_linux:10.04:-:lts\n'), + call('ExternalRefComment: external package ref comment\n'), call('PrimaryPackagePurpose: OTHER\n'), + call('ReleaseDate: 2022-01-01T00:00:00Z\n')]) diff --git a/tests/writer/tagvalue/test_tagvalue_writer.py b/tests/writer/tagvalue/test_tagvalue_writer.py new file mode 100644 index 000000000..98b786b7e --- /dev/null +++ b/tests/writer/tagvalue/test_tagvalue_writer.py @@ -0,0 +1,63 @@ +# Copyright (c) 2022 spdx contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +import pytest + +from src.model.actor import Actor, ActorType +from src.model.annotation import Annotation, AnnotationType +from src.model.checksum import ChecksumAlgorithm, Checksum +from src.model.document import CreationInfo, Document +from src.model.external_document_ref import ExternalDocumentRef +from src.model.extracted_licensing_info import ExtractedLicensingInfo +from src.model.file import File +from src.model.package import Package +from src.model.relationship import RelationshipType, Relationship +from src.model.snippet import Snippet +from src.model.spdx_none import SpdxNone +from src.writer.tagvalue.tagvalue_writer import write_document_to_file + + +@pytest.fixture +def temporary_file_path() -> str: + temporary_file_path = "temp_test_tag_value_writer_output.spdx" + yield temporary_file_path + os.remove(temporary_file_path) + + +def test_write_tag_value(temporary_file_path: str): + creation_info = CreationInfo("spdxVersion", "documentId", "documentName", "documentNamespace", + [Actor(ActorType.TOOL, "tools-python", "tools-python@github.com")], + datetime(2022, 12, 1), document_comment="comment", data_license="dataLicense", + external_document_refs=[ExternalDocumentRef("docRefId", "externalDocumentUri", + Checksum(ChecksumAlgorithm.SHA1, + "externalRefSha1"))]) + package = Package("packageId", "packageName", SpdxNone()) + file = File("fileName", "fileId", [Checksum(ChecksumAlgorithm.SHA1, "fileSha1")]) + snippet = Snippet("snippetId", "snippetFileId", (1, 2)) + annotations = [ + Annotation("documentId", AnnotationType.REVIEW, Actor(ActorType.PERSON, "reviewerName"), datetime(2022, 12, 2), + "reviewComment"), + Annotation("fileId", AnnotationType.OTHER, Actor(ActorType.TOOL, "toolName"), datetime(2022, 12, 3), + "otherComment")] + extracted_licensing_info = [ExtractedLicensingInfo("licenseId", "licenseText")] + relationships = [Relationship(creation_info.spdx_id, RelationshipType.DESCRIBES, "packageId"), + Relationship(creation_info.spdx_id, RelationshipType.DESCRIBES, "fileId", "relationshipComment"), + Relationship("relationshipOriginId", RelationshipType.AMENDS, "relationShipTargetId")] + document = Document(creation_info, annotations=annotations, extracted_licensing_info=extracted_licensing_info, + relationships=relationships, packages=[package], files=[file], snippets=[snippet]) + + write_document_to_file(document, temporary_file_path) + + # without a tag-value parser we can only test that no errors occur while writing + # as soon as the tag-value parser is implemented (https://github.com/spdx/tools-python/issues/382) we can test for equality between the temporary file and the expected file in ./expected_results