diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fc247b2c47..d0b7157655 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -54,6 +54,10 @@ v33.0.0 (2024-01-16) project pipeline. https://github.com/nexB/scancode.io/issues/997 +- In "map_deploy_to_develop" pipeline, add support for path patterns + in About file attributes documenting resource paths. + https://github.com/nexB/scancode.io/issues/1004 + - Fix an issue where the pipeline details cannot be fetched when using URLs that include credentials such as "user:pass@domain". https://github.com/nexB/scancode.io/issues/998 diff --git a/scanpipe/models.py b/scanpipe/models.py index e8461cdbac..565bb86299 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1793,12 +1793,12 @@ def profile(self, print_results=False): print(output_str) -def posix_regex_to_django_regex_lookup(regex_pattern): +def convert_glob_to_django_regex(glob_pattern): """ - Convert a POSIX-style regex pattern to an equivalent pattern compatible with the - Django regex lookup. + Convert a glob pattern to an equivalent django regex pattern + compatible with the Django regex lookup. """ - escaped_pattern = re.escape(regex_pattern) + escaped_pattern = re.escape(glob_pattern) escaped_pattern = escaped_pattern.replace(r"\*", ".*") # Replace \* with .* escaped_pattern = escaped_pattern.replace(r"\?", ".") # Replace \? with . escaped_pattern = f"^{escaped_pattern}$" # Add start and end anchors @@ -1906,8 +1906,8 @@ def has_value(self, field_name): return self.filter(~Q((f"{field_name}__in", EMPTY_VALUES))) def path_pattern(self, pattern): - """Resources with a path that match the provided ``pattern``.""" - return self.filter(path__regex=posix_regex_to_django_regex_lookup(pattern)) + """Resources with a path that match the provided glob ``pattern``.""" + return self.filter(path__regex=convert_glob_to_django_regex(pattern)) def has_directory_content_fingerprint(self): """ diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index 2f5956426a..39cc90a21f 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -23,7 +23,9 @@ from collections import Counter from collections import defaultdict from contextlib import suppress +from dataclasses import dataclass from pathlib import Path +from re import match as regex_match from django.contrib.postgres.aggregates.general import ArrayAgg from django.core.exceptions import MultipleObjectsReturned @@ -43,6 +45,7 @@ from scanpipe import pipes from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource +from scanpipe.models import convert_glob_to_django_regex from scanpipe.pipes import LoopProgress from scanpipe.pipes import flag from scanpipe.pipes import get_resource_diff_ratio @@ -774,78 +777,232 @@ def _map_javascript_resource( resource.update(status=flag.MAPPED) -def _map_about_file_resource(project, about_file_resource, to_resources): - about_file_location = str(about_file_resource.location_path) - package_data = resolve.resolve_about_package(about_file_location) +@dataclass +class AboutFileIndexes: + """ + About file indexes are used to create packages from + About files and map the resources described in them + to the respective packages created, using regex path + patterns and other About file data. + """ + + # Mapping of About file paths and the regex pattern + # string for the files documented + regex_by_about_path: dict + # Mapping of About file paths and a list of path pattern + # strings, for the files to be ignored + ignore_regex_by_about_path: dict + # Resource objects for About files present in the codebase, + # by their path + about_resources_by_path: dict + # mapping of package data present in the About file, by path + about_pkgdata_by_path: dict + # List of mapped resources for each About file, by path + mapped_resources_by_aboutpath: dict + + @classmethod + def create_indexes(cls, project, from_about_files, logger=None): + """ + Return an ABOUT file index, containing path pattern mappings, + package data, and resources, created from `from_about_files`, + the About file resources. + """ + about_pkgdata_by_path = {} + regex_by_about_path = {} + ignore_regex_by_about_path = {} + about_resources_by_path = {} + mapped_resources_by_aboutpath = {} + + count_indexed_about_files = 0 + + for about_file_resource in from_about_files: + package_data = resolve.resolve_about_package( + input_location=str(about_file_resource.location_path) + ) + error_message_details = { + "path": about_file_resource.path, + "package_data": package_data, + } + if not package_data: + project.add_error( + description="Cannot create package from ABOUT file", + model="map_about_files", + details=error_message_details, + ) + continue - error_message_details = { - "path": about_file_resource.path, - "package_data": package_data, - } - if not package_data: - project.add_error( - description="Cannot create package from ABOUT file", - model="map_about_files", - details=error_message_details, + about_pkgdata_by_path[about_file_resource.path] = package_data + files_pattern = package_data.get("filename") + if not files_pattern: + # Cannot map anything without the about_resource value. + project.add_error( + description="ABOUT file does not have about_resource", + model="map_about_files", + details=error_message_details, + ) + continue + else: + count_indexed_about_files += 1 + regex = convert_glob_to_django_regex(files_pattern) + regex_by_about_path[about_file_resource.path] = regex + + if extra_data := package_data.get("extra_data"): + ignore_regex = [] + for pattern in extra_data.get("ignored_resources", []): + ignore_regex.append(convert_glob_to_django_regex(pattern)) + if ignore_regex: + ignore_regex_by_about_path[about_file_resource.path] = ignore_regex + + about_resources_by_path[about_file_resource.path] = about_file_resource + mapped_resources_by_aboutpath[about_file_resource.path] = [] + + if logger: + logger( + f"Created mapping index from {count_indexed_about_files:,d} .ABOUT " + f"files in the from/ codebase." + ) + + return cls( + about_pkgdata_by_path=about_pkgdata_by_path, + regex_by_about_path=regex_by_about_path, + ignore_regex_by_about_path=ignore_regex_by_about_path, + about_resources_by_path=about_resources_by_path, + mapped_resources_by_aboutpath=mapped_resources_by_aboutpath, ) - return - filename = package_data.get("filename") - if not filename: - # Cannot map anything without the about_resource value. - project.add_error( - description="ABOUT file does not have about_resource", - model="map_about_files", - details=error_message_details, + def get_matched_about_path(self, to_resource): + """ + Map `to_resource` using the about file index, and if + mapped, return the path string to the About file it + was mapped to, and if not mapped or ignored, return + None. + """ + resource_mapped = False + for about_path, regex_pattern in self.regex_by_about_path.items(): + if regex_match(pattern=regex_pattern, string=to_resource.path): + resource_mapped = True + break + + if not resource_mapped: + return + + ignore_regex_patterns = self.ignore_regex_by_about_path.get(about_path, []) + ignore_resource = False + for ignore_regex_pattern in ignore_regex_patterns: + if regex_match(pattern=ignore_regex_pattern, string=to_resource.path): + ignore_resource = True + break + + if ignore_resource: + return + + return about_path + + def map_deployed_to_devel_using_about(self, to_resources): + """ + Return mapped resources which are mapped using the + path patterns in About file indexes. Resources are + mapped for each About file in the index, and + their status is updated accordingly. + """ + mapped_to_resources = [] + + for to_resource in to_resources: + about_path = self.get_matched_about_path(to_resource) + if not about_path: + continue + + mapped_resources_about = self.mapped_resources_by_aboutpath.get(about_path) + if mapped_resources_about: + mapped_resources_about.append(to_resource) + else: + self.mapped_resources_by_aboutpath[about_path] = [to_resource] + mapped_to_resources.append(to_resource) + to_resource.update(status=flag.ABOUT_MAPPED) + + return mapped_to_resources + + def get_about_file_companions(self, about_path): + """ + Given an ``about_path`` path string to an About file, + get CodebaseResource objects for the companion license + and notice files. + """ + about_file_resource = self.about_resources_by_path.get(about_path) + about_file_extra_data = self.about_pkgdata_by_path.get(about_path).get( + "extra_data" ) - return - ignored_resources = [] - if extra_data := package_data.get("extra_data"): - ignored_resources = extra_data.get("ignored_resources") - - # Fetch all resources that are covered by the .ABOUT file. - codebase_resources = to_resources.filter(path__contains=f"/{filename.lstrip('/')}") - if not codebase_resources: - # If there's nothing to map on the ``to/`` do not create the package. - project.add_warning( - description=( - "Resource paths listed at about_resource is not found" - " in the to/ codebase" - ), - model="map_about_files", - details=error_message_details, + about_file_companion_names = [ + about_file_extra_data.get("license_file"), + about_file_extra_data.get("notice_file"), + ] + about_file_companions = about_file_resource.siblings().filter( + name__in=about_file_companion_names ) - return + return about_file_companions + + def create_about_packages_relations(self, project): + """ + Create packages using About file package data, if the About file + has mapped resources on the to/ codebase and creates the mappings + for the package created and mapped resources. + """ + about_purls = set() + mapped_about_resources = [] + + for about_path, mapped_resources in self.mapped_resources_by_aboutpath.items(): + about_file_resource = self.about_resources_by_path[about_path] + package_data = self.about_pkgdata_by_path[about_path] + + if not mapped_resources: + error_message_details = { + "path": about_path, + "package_data": package_data, + } + project.add_warning( + description=( + "Resource paths listed at about_resource is not found" + " in the to/ codebase" + ), + model="map_about_files", + details=error_message_details, + ) + continue - # Ignore resources for paths in `ignored_resources` attribute - if ignored_resources: - lookups = Q() - for resource_path in ignored_resources: - lookups |= Q(**{"path__contains": resource_path}) - codebase_resources = codebase_resources.filter(~lookups) + # Create the Package using .ABOUT data and assign related codebase_resources + about_package = pipes.update_or_create_package( + project=project, + package_data=package_data, + codebase_resources=mapped_resources, + ) + about_purls.add(about_package.purl) + mapped_about_resources.append(about_file_resource) - # Create the Package using .ABOUT data and assigned related codebase_resources - pipes.update_or_create_package(project, package_data, codebase_resources) + # Map the .ABOUT file resource to all related resources in the ``to/`` side. + for mapped_resource in mapped_resources: + pipes.make_relation( + from_resource=about_file_resource, + to_resource=mapped_resource, + map_type="about_file", + ) - # Map the .ABOUT file resource to all related resources in the ``to/`` side. - for to_resource in codebase_resources: - pipes.make_relation( - from_resource=about_file_resource, - to_resource=to_resource, - map_type="about_file", - ) + about_file_resource.update(status=flag.ABOUT_MAPPED) + + about_file_companions = self.get_about_file_companions(about_path) + about_file_companions.update(status=flag.ABOUT_MAPPED) - codebase_resources.update(status=flag.ABOUT_MAPPED) - about_file_resource.update(status=flag.ABOUT_MAPPED) + return about_purls, mapped_about_resources def map_about_files(project, logger=None): """Map ``from/`` .ABOUT files to their related ``to/`` resources.""" project_resources = project.codebaseresources - from_files = project_resources.files().from_codebase() - from_about_files = from_files.filter(extension=".ABOUT") - to_resources = project_resources.to_codebase() + from_about_files = ( + project_resources.files().from_codebase().filter(extension=".ABOUT") + ) + if not from_about_files.exists(): + return if logger: logger( @@ -853,15 +1010,30 @@ def map_about_files(project, logger=None): f"codebase." ) - for about_file_resource in from_about_files: - _map_about_file_resource(project, about_file_resource, to_resources) + indexes = AboutFileIndexes.create_indexes( + project=project, from_about_files=from_about_files + ) - about_file_companions = ( - about_file_resource.siblings() - .filter(name__startswith=about_file_resource.name_without_extension) - .filter(extension__in=[".LICENSE", ".NOTICE"]) + # Ignoring empty or ignored files as they are not relevant anyway + to_resources = project_resources.to_codebase().no_status() + mapped_to_resources = indexes.map_deployed_to_devel_using_about( + to_resources=to_resources, + ) + if logger: + logger( + f"Mapped {len(mapped_to_resources):,d} resources from the " + f"to/ codebase to the About files in the from. codebase." + ) + + about_purls, mapped_about_resources = indexes.create_about_packages_relations( + project=project, + ) + if logger: + logger( + f"Created {len(about_purls):,d} new packages from " + f"{len(mapped_about_resources):,d} About files which " + f"were mapped to resources in the to/ side." ) - about_file_companions.update(status=flag.ABOUT_MAPPED) def map_javascript_post_purldb_match(project, logger=None): diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 25b343d9b8..7f99afadb5 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -86,18 +86,15 @@ def resolve_about_package(input_location): if value: package_data[field_name] = value + package_data["extra_data"] = {} + if about_resource := about_data.get("about_resource"): package_data["filename"] = list(about_resource.keys())[0] if ignored_resources := about_data.get("ignored_resources"): - extra_data = {"ignored_resources": list(ignored_resources.keys())} - package_data["extra_data"] = extra_data - - if license_expression := about_data.get("license_expression"): - package_data["declared_license_expression"] = license_expression + package_data["extra_data"]["ignored_resources"] = list(ignored_resources.keys()) - if notice_dict := about_data.get("notice_file"): - package_data["notice_text"] = list(notice_dict.values())[0] + populate_license_notice_fields_about(package_data, about_data) for field_name, value in about_data.items(): if field_name.startswith("checksum_"): @@ -107,6 +104,23 @@ def resolve_about_package(input_location): return package_data +def populate_license_notice_fields_about(package_data, about_data): + """ + Populate ``package_data`` with license and notice attributes + from ``about_data``. + """ + if license_expression := about_data.get("license_expression"): + package_data["declared_license_expression"] = license_expression + + if notice_dict := about_data.get("notice_file"): + package_data["notice_text"] = list(notice_dict.values())[0] + package_data["extra_data"]["notice_file"] = list(notice_dict.keys())[0] + + if license_dict := about_data.get("license_file"): + package_data["extra_data"]["license_file"] = list(license_dict.keys())[0] + package_data["extracted_license_statement"] = list(license_dict.values())[0] + + def resolve_about_packages(input_location): """ Wrap ``resolve_about_package`` to return a list as expected by the diff --git a/scanpipe/tests/data/d2d/about_files/expected.json b/scanpipe/tests/data/d2d/about_files/expected.json index 8fcccf5773..52fa3342fb 100644 --- a/scanpipe/tests/data/d2d/about_files/expected.json +++ b/scanpipe/tests/data/d2d/about_files/expected.json @@ -111,12 +111,14 @@ "other_license_expression": "", "other_license_expression_spdx": "", "other_license_detections": [], - "extracted_license_statement": "", + "extracted_license_statement": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n ", "notice_text": "notice", "source_packages": [], "extra_data": { + "notice_file": "flume-ng-node-1.9.0-sources.NOTICE", + "license_file": "flume-ng-node-1.9.0-sources.LICENSE", "ignored_resources": [ - "flume-ng-node-1.9.0.jar-extract/org/apache/flume/node/ConfigurationProvider.class" + "*flume-ng-node-*.jar-extract/org/apache/flume/node/ConfigurationProvider.class" ] }, "package_uid": "", @@ -167,9 +169,9 @@ "status": "not-deployed", "tag": "from", "extension": ".ABOUT", - "md5": "04c25308d59068db649ebfd5d8103338", - "sha1": "9625fc925a01cfa22b6e1ce083a1b1802c2ce78c", - "sha256": "46142b274cda38e9c183001b58a72c8fc442b2162ae85e9a453a2c7a1a86d427", + "md5": "4f8ac19bc3661bbaac91fb8652b6c4cb", + "sha1": "ac27839cc3010c96b34796be83bc2e6b6bc1882b", + "sha256": "055133491484d680991ed98ba1f791e88f4552955e15d9c5b3b14c5d46b4ee16", "sha512": "", "programming_language": "", "is_binary": false, @@ -198,9 +200,9 @@ "status": "about-mapped", "tag": "from", "extension": ".ABOUT", - "md5": "b1d5c62c364d4470557bfba7d0338758", - "sha1": "828f79d9fc0619a5b869c46a54b10ee3573a00bb", - "sha256": "de514210e135dddffb6ace69aa5fe27e1873146e05eeb5b05c6de1f8c00b0010", + "md5": "c7fab493a90ebf247954e1e30582ba8f", + "sha1": "3090f7d036bf68c5421364ac03a094715348f9de", + "sha256": "71f10662c0806de172a04a07fffbb3d22bd72ddacef4188849a480afd4e46849", "sha512": "", "programming_language": "", "is_binary": false, @@ -229,9 +231,9 @@ "status": "about-mapped", "tag": "from", "extension": ".LICENSE", - "md5": "2b42edef8fa55315f34f2370b4715ca9", - "sha1": "58853eb8199b5afe72a73a25fd8cf8c94285174b", - "sha256": "43070e2d4e532684de521b885f385d0841030efa2b1a20bafb76133a5e1379c1", + "md5": "94c82ae800466538d15278d6be4feedc", + "sha1": "3837fdbc9d942bcd1c5f2d419148e944f7ce996a", + "sha256": "a4da19948e6906fa8af95a258b9a354f641adc6215956f0ec63f429a10f0f603", "sha512": "", "programming_language": "", "is_binary": false, diff --git a/scanpipe/tests/data/d2d/about_files/from-with-about-file.zip b/scanpipe/tests/data/d2d/about_files/from-with-about-file.zip index ccce7aedae..7dbed4ca63 100644 Binary files a/scanpipe/tests/data/d2d/about_files/from-with-about-file.zip and b/scanpipe/tests/data/d2d/about_files/from-with-about-file.zip differ diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 3f3a24b610..417fceae80 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -33,6 +33,7 @@ from scanpipe.models import Project from scanpipe.pipes import d2d from scanpipe.pipes import flag +from scanpipe.pipes import scancode from scanpipe.pipes.input import copy_input from scanpipe.pipes.input import copy_inputs from scanpipe.tests import make_resource_directory @@ -1258,6 +1259,126 @@ def test_scanpipe_pipes_flag_whitespace_files(self): flag.IGNORED_WHITESPACE_FILE, non_whitespace_resource.status ) + def test_scanpipe_pipes_create_about_file_indexes(self): + input_dir = self.project1.input_path + input_resources = [ + self.data_location / "d2d/about_files/to-with-jar.zip", + self.data_location / "d2d/about_files/from-with-about-file.zip", + ] + copy_inputs(input_resources, input_dir) + self.from_files, self.to_files = d2d.get_inputs(self.project1) + + inputs_with_codebase_path_destination = [ + (self.from_files, self.project1.codebase_path / d2d.FROM), + (self.to_files, self.project1.codebase_path / d2d.TO), + ] + + for input_files, codebase_path in inputs_with_codebase_path_destination: + for input_file_path in input_files: + scancode.extract_archive(input_file_path, codebase_path) + + scancode.extract_archives( + self.project1.codebase_path, + recurse=True, + ) + + pipes.collect_and_create_codebase_resources(self.project1) + + from_about_files = ( + self.project1.codebaseresources.files() + .from_codebase() + .filter(extension=".ABOUT") + ) + about_file_indexes = d2d.AboutFileIndexes.create_indexes( + project=self.project1, + from_about_files=from_about_files, + ) + + about_path = "from/flume-ng-node-1.9.0-sources.ABOUT" + about_notice_path = "from/flume-ng-node-1.9.0-sources.NOTICE" + + about_notice_file = self.project1.codebaseresources.get(path=about_notice_path) + + self.assertIn( + about_path, list(about_file_indexes.about_resources_by_path.keys()) + ) + about_regex = d2d.convert_glob_to_django_regex( + glob_pattern="*flume-ng-node-*.jar*" + ) + self.assertEqual( + about_file_indexes.regex_by_about_path.get(about_path), about_regex + ) + self.assertEqual( + about_file_indexes.about_pkgdata_by_path.get(about_path).get("name"), + "log4j", + ) + self.assertIn( + about_notice_file, about_file_indexes.get_about_file_companions(about_path) + ) + to_resource = self.project1.codebaseresources.get( + path=( + "to/flume-ng-node-1.9.0.jar-extract/org/apache/" + "flume/node/AbstractZooKeeperConfigurationProvider.class" + ) + ) + self.assertEqual( + about_file_indexes.get_matched_about_path(to_resource), about_path + ) + + def test_scanpipe_pipes_map_d2d_using_about(self): + input_dir = self.project1.input_path + input_resources = [ + self.data_location / "d2d/about_files/to-with-jar.zip", + self.data_location / "d2d/about_files/from-with-about-file.zip", + ] + copy_inputs(input_resources, input_dir) + self.from_files, self.to_files = d2d.get_inputs(self.project1) + + inputs_with_codebase_path_destination = [ + (self.from_files, self.project1.codebase_path / d2d.FROM), + (self.to_files, self.project1.codebase_path / d2d.TO), + ] + + for input_files, codebase_path in inputs_with_codebase_path_destination: + for input_file_path in input_files: + scancode.extract_archive(input_file_path, codebase_path) + + scancode.extract_archives( + self.project1.codebase_path, + recurse=True, + ) + + pipes.collect_and_create_codebase_resources(self.project1) + + from_about_files = ( + self.project1.codebaseresources.files() + .from_codebase() + .filter(extension=".ABOUT") + ) + about_file_indexes = d2d.AboutFileIndexes.create_indexes( + project=self.project1, + from_about_files=from_about_files, + ) + + to_resources = self.project1.codebaseresources.to_codebase() + about_file_indexes.map_deployed_to_devel_using_about( + to_resources=to_resources, + ) + + about_path = "from/flume-ng-node-1.9.0-sources.ABOUT" + to_resource = self.project1.codebaseresources.get( + path=( + "to/flume-ng-node-1.9.0.jar-extract/org/apache/" + "flume/node/AbstractZooKeeperConfigurationProvider.class" + ) + ) + self.assertIn( + to_resource, + about_file_indexes.mapped_resources_by_aboutpath.get(about_path), + ) + + about_file_indexes.create_about_packages_relations(self.project1) + def test_scanpipe_pipes_d2d_match_purldb_resources_post_process(self): to_map = self.data_location / "d2d-javascript" / "to" / "main.js.map" to_mini = self.data_location / "d2d-javascript" / "to" / "main.js" diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index afe29caab6..f8802cd4d1 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -92,6 +92,8 @@ def test_scanpipe_pipes_resolve_resolve_packages(self): "filename": "Django-4.0.8-py3-none-any.whl", "download_url": "https://python.org/Django-4.0.8-py3-none-any.whl", "declared_license_expression": "bsd-new", + "extra_data": {"license_file": "bsd-new.LICENSE"}, + "extracted_license_statement": None, "md5": "386349753c386e574dceca5067e2788a", "name": "django", "sha1": "4cc6f7abda928a0b12cd1f1cd8ad3677519ca04e", @@ -114,6 +116,8 @@ def test_scanpipe_pipes_resolve_resolve_about_packages(self): "filename": "Django-4.0.8-py3-none-any.whl", "download_url": "https://python.org/Django-4.0.8-py3-none-any.whl", "declared_license_expression": "bsd-new", + "extra_data": {"license_file": "bsd-new.LICENSE"}, + "extracted_license_statement": None, "md5": "386349753c386e574dceca5067e2788a", "name": "django", "sha1": "4cc6f7abda928a0b12cd1f1cd8ad3677519ca04e", @@ -124,7 +128,7 @@ def test_scanpipe_pipes_resolve_resolve_about_packages(self): input_location = self.manifest_location / "poor_values.ABOUT" package = resolve.resolve_about_packages(str(input_location)) - expected = {"name": "project"} + expected = {"extra_data": {}, "name": "project"} self.assertEqual([expected], package) def test_scanpipe_pipes_resolve_spdx_package_to_discovered_package_data(self): diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 1b0197b255..1e98f3d284 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -63,8 +63,8 @@ from scanpipe.models import RunInProgressError from scanpipe.models import RunNotAllowedToStart from scanpipe.models import UUIDTaggedItem +from scanpipe.models import convert_glob_to_django_regex from scanpipe.models import get_project_work_directory -from scanpipe.models import posix_regex_to_django_regex_lookup from scanpipe.pipes.fetch import Download from scanpipe.pipes.input import copy_input from scanpipe.tests import dependency_data1 @@ -686,7 +686,7 @@ def test_scanpipe_model_update_mixin(self): package.refresh_from_db() self.assertEqual("pkg:deb/debian/adduser@3.118?arch=all", package.package_url) - def test_scanpipe_model_posix_regex_to_django_regex_lookup(self): + def test_scanpipe_model_convert_glob_to_django_regex(self): test_data = [ ("", r"^$"), # Single segment @@ -718,7 +718,7 @@ def test_scanpipe_model_posix_regex_to_django_regex_lookup(self): ] for pattern, expected in test_data: - self.assertEqual(expected, posix_regex_to_django_regex_lookup(pattern)) + self.assertEqual(expected, convert_glob_to_django_regex(pattern)) def test_scanpipe_run_model_set_scancodeio_version(self): run1 = Run.objects.create(project=self.project1) @@ -1568,6 +1568,7 @@ def test_scanpipe_codebase_resource_queryset_path_pattern(self): make_resource_file(self.project1, path="dir/.example") make_resource_file(self.project1, path="dir/subdir/readme.html") make_resource_file(self.project1, path="foo$.class") + make_resource_file(self.project1, path="example-1.0.jar") patterns = [ "example", @@ -1584,6 +1585,7 @@ def test_scanpipe_codebase_resource_queryset_path_pattern(self): "dir/*/readme.*", r"*$.class", "*readme.htm?", + "example-*.jar", ] for pattern in patterns: