Add new license rule model attributes

To the license Rule class:

- Adds is_license_clue attribute
- Adds is_deprecated attribute

Also implements related processing.

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

etc/scripts/licenses/buildrules-template.txt

@@ -222,6 +222,8 @@ def cli(licenses_file):
             base_name = "false-positive"
         elif rule.is_license_intro:
             base_name = "license-intro"
+        elif rule.is_license_clue:
+            base_name = "license-clue"
         else:
             base_name = rule.license_expression
 

etc/scripts/licenses/buildrules.py

@@ -61,6 +61,8 @@
     "is_license_tag",
     "is_license_reference",
     "is_license_intro",
+    "is_license_clue",
+    "is_deprecated",
     "has_unknown",
     "only_known_words",
     "notes",

etc/scripts/licenses/report_license_rules.py

@@ -1189,7 +1189,12 @@ def get_license_tokens():
     yield 'licensed'
 
 
-def load_rules(rules_data_dir=rules_data_dir, with_checks=True, is_builtin=True):
+def load_rules(
+    rules_data_dir=rules_data_dir,
+    with_checks=True,
+    is_builtin=True,
+    ignore_deprecated=True,
+):
     """
     Return an iterable of rules loaded from rule files in ``rules_data_dir``.
     Optionally check for consistency if ``with_checks`` is True.
@@ -1211,7 +1216,12 @@ def load_rules(rules_data_dir=rules_data_dir, with_checks=True, is_builtin=True)
                 space_problems.append(rule_file)
 
             try:
-                yield Rule.from_file(rule_file=rule_file)
+                rule = Rule.from_file(rule_file=rule_file)
+                if rule.is_deprecated and ignore_deprecated:
+                    continue 
+                else:
+                    yield rule
+
             except Exception as re:
                 if with_checks:
                     model_errors.append(str(re))
@@ -1387,6 +1397,22 @@ class BasicRule:
             'after. Mutually exclusive from any other is_license_* flag')
     )
 
+    is_license_clue = attr.ib(
+        default=False,
+        repr=False,
+        metadata=dict(
+            help='True if this is rule text is a clue to a license '
+            'but cannot be considered in a proper license detection '
+            'as a license text/notice/reference/tag/intro as it is'
+            'merely a clue and does not actually point to or refer to '
+            'the actual license directly. This is still valuable information '
+            'useful in determining the license/origin of a file, but this '
+            'should not be summarized/present in the license expression for '
+            'a package or a file, nor its list of license detections. '
+            'considered in the context of the detection that it precedes. '
+            'Mutually exclusive from any other is_license_* flag')
+    )
+
     is_false_positive = attr.ib(
         default=False,
         repr=False,
@@ -1505,6 +1531,19 @@ class BasicRule:
             'built at runtime, such as an SPDX license rule.')
     )
 
+    is_deprecated = attr.ib(
+        default=False,
+        repr=False,
+        metadata=dict(
+            help='Flag set to True if this rule is deleted, '
+            'and not to be used anymore in license detection. '
+            'This happens usually when a rule is renamed/assigned '
+            'to a seperate license-expression, promoted to being a '
+            'license text or just plain retired. This is used to '
+            'preserve the link to the rule, and therefore make links '
+            'to rules as permanent.')
+    )
+
     ###########################################################################
     # lists of clues that can be ignored when detected in this license as they
     # are part of the license or rule text itself
@@ -1769,6 +1808,7 @@ def validate(self, licensing=None, thorough=False):
             self.is_license_reference,
             self.is_license_tag,
             self.is_license_intro,
+            self.is_license_clue,
         )
 
         has_license_flags = any(license_flags)
@@ -1924,6 +1964,7 @@ def to_reference(self):
         data['is_license_reference'] = self.is_license_reference
         data['is_license_tag'] = self.is_license_tag
         data['is_license_intro'] = self.is_license_intro
+        data['is_license_clue'] = self.is_license_clue
         data['is_continuous'] = self.is_continuous
         data['is_builtin'] = self.is_builtin
         data['is_from_license'] = self.is_from_license
@@ -1961,6 +2002,7 @@ def to_dict(self, include_text=False):
             'is_license_reference',
             'is_license_tag',
             'is_license_intro',
+            'is_license_clue',
             'is_continuous',
         )
 
@@ -2253,7 +2295,9 @@ def load(self, rule_file, with_checks=True):
         self.is_license_tag = data.get('is_license_tag', False)
         self.is_license_reference = data.get('is_license_reference', False)
         self.is_license_intro = data.get('is_license_intro', False)
+        self.is_license_clue = data.get('is_license_clue', False)
         self.is_continuous = data.get('is_continuous', False)
+        self.is_deprecated = data.get('is_deprecated', False)
 
         self.referenced_filenames = data.get('referenced_filenames', []) or []
 

src/licensedcode/models.py

@@ -216,7 +216,7 @@ def dget(s):
             name = i_title or am_nm or ext_nm or nm
             descriptions = [s_title, i_title, nm]
 
-    datasource_id =get_datasource_id(package_type=package_type)
+    datasource_id = get_datasource_id(package_type=package_type)
 
     descriptions = unique(descriptions)
     descriptions = [d for d in descriptions if d and d.strip() and d != name]

src/packagedcode/jar_manifest.py

@@ -250,6 +250,7 @@ class LicenseFilter(object):
     is_license_tag=LicenseFilter(min_coverage=100),
     is_license_reference=LicenseFilter(min_score=50, min_coverage=100),
     is_license_intro=LicenseFilter(min_score=100, min_coverage=100),
+    is_license_clue=LicenseFilter(min_score=100, min_coverage=100),
 )
 
 
@@ -268,6 +269,7 @@ def is_good_license(license_match_object):
             ('is_license_reference', license_match_object.rule.is_license_reference),
             ('is_license_tag', license_match_object.rule.is_license_tag),
             ('is_license_intro', license_match_object.rule.is_license_intro),
+            ('is_license_clue', license_match_object.rule.is_license_clue),
         ]
     )
     matched = False

src/summarycode/score.py

@@ -643,6 +643,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -669,6 +670,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -695,6 +697,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -721,6 +724,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": true,
@@ -747,6 +751,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -773,6 +778,7 @@
       "is_license_reference": true,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -799,6 +805,7 @@
       "is_license_reference": true,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -825,6 +832,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": true,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -851,6 +859,7 @@
       "is_license_reference": true,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -877,6 +886,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -905,6 +915,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -941,6 +952,7 @@
       "is_license_reference": true,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -967,6 +979,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": true,
@@ -993,6 +1006,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -1030,6 +1044,7 @@
       "is_license_reference": true,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -1058,6 +1073,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -1084,6 +1100,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": true,
@@ -1120,6 +1137,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": true,
@@ -1150,6 +1168,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,

tests/licensedcode/data/licenses_reference_reporting/license-reference-works-with-clues.expected.json

@@ -287,6 +287,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -317,6 +318,7 @@
       "is_license_reference": false,
       "is_license_tag": true,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": true,
       "is_builtin": true,
       "is_from_license": false,
@@ -343,6 +345,7 @@
       "is_license_reference": false,
       "is_license_tag": true,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -369,6 +372,7 @@
       "is_license_reference": false,
       "is_license_tag": true,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,

tests/licensedcode/data/licenses_reference_reporting/scan-matched-text-with-reference.expected.json

@@ -284,6 +284,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -314,6 +315,7 @@
       "is_license_reference": false,
       "is_license_tag": true,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": true,
       "is_builtin": true,
       "is_from_license": false,
@@ -340,6 +342,7 @@
       "is_license_reference": false,
       "is_license_tag": true,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -366,6 +369,7 @@
       "is_license_reference": false,
       "is_license_tag": true,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,