fix(python): impove package name normalization

pkg/dependency/parser/python/poetry/parse.go

@@ -105,7 +105,7 @@ func (p *Parser) parseDependencies(deps map[string]any, pkgVersions map[string][
 }
 
 func (p *Parser) parseDependency(name string, versRange any, pkgVersions map[string][]string) (string, error) {
-	name = python.NormalizePkgName(name)
+	name = python.NormalizePkgName(name, true)
 	vers, ok := pkgVersions[name]
 	if !ok {
 		return "", xerrors.Errorf("no version found for %q", name)

pkg/dependency/parser/python/pyproject/pyproject.go

@@ -55,7 +55,7 @@ func (d *Dependencies) UnmarshalTOML(data any) error {
 	switch deps := data.(type) {
 	case map[string]any: // For Poetry v1
 		d.Set = set.New[string](lo.MapToSlice(deps, func(pkgName string, _ any) string {
-			return python.NormalizePkgName(pkgName)
+			return python.NormalizePkgName(pkgName, true)
 		})...)
 	case []any: // For Poetry v2
 		d.Set = set.New[string]()

pkg/dependency/parser/python/python.go

@@ -1,14 +1,22 @@
 package python
 
-import "strings"
-
-// NormalizePkgName normalizes the package name based on pep-0426
-func NormalizePkgName(name string) string {
-	// The package names don't use `_`, `.` or upper case, but dependency names can contain them.
-	// We need to normalize those names.
-	// cf. https://peps.python.org/pep-0426/#name
-	name = strings.ToLower(name)              // e.g. https://github.com/python-poetry/poetry/blob/c8945eb110aeda611cc6721565d7ad0c657d453a/poetry.lock#L819
-	name = strings.ReplaceAll(name, "_", "-") // e.g. https://github.com/python-poetry/poetry/blob/c8945eb110aeda611cc6721565d7ad0c657d453a/poetry.lock#L50
-	name = strings.ReplaceAll(name, ".", "-") // e.g. https://github.com/python-poetry/poetry/blob/c8945eb110aeda611cc6721565d7ad0c657d453a/poetry.lock#L816
+import (
+	"regexp"
+	"strings"
+)
+
+var normalizePkgNameRegexp = regexp.MustCompile(`[-_.]+`)
+
+// NormalizePkgName normalizes the package name based on pep-0503 (with the option to disable conversion to lowercase).
+// cf. https://peps.python.org/pep-0503/#normalized-names:
+// The name should be lowercased with all runs of the characters ., -, or _ replaced with a single - character.
+func NormalizePkgName(name string, inLowerCase bool) string {
+	name = normalizePkgNameRegexp.ReplaceAllString(name, "-")
+
+	// pep-0503 requires that all packages names MUST be lowercase.
+	// But there are cases where the original case should be preserved (e.g. dist-info dir names).
+	if inLowerCase {
+		name = strings.ToLower(name)
+	}
 	return name
 }

pkg/dependency/parser/python/python_test.go

@@ -10,30 +10,45 @@ import (
 
 func Test_NormalizePkgName(t *testing.T) {
 	tests := []struct {
-		pkgName  string
-		expected string
+		pkgName   string
+		lowerCase bool
+		expected  string
 	}{
 		{
-			pkgName:  "SecretStorage",
-			expected: "secretstorage",
+			pkgName:   "SecretStorage",
+			lowerCase: true,
+			expected:  "secretstorage",
 		},
 		{
-			pkgName:  "pywin32-ctypes",
-			expected: "pywin32-ctypes",
+			pkgName:   "SecretStorage",
+			lowerCase: false,
+			expected:  "SecretStorage",
 		},
 		{
-			pkgName:  "jaraco.classes",
-			expected: "jaraco-classes",
+			pkgName:   "pywin32-ctypes",
+			lowerCase: true,
+			expected:  "pywin32-ctypes",
 		},
 		{
-			pkgName:  "green_gdk",
-			expected: "green-gdk",
+			pkgName:   "jaraco.classes",
+			lowerCase: true,
+			expected:  "jaraco-classes",
+		},
+		{
+			pkgName:   "green_gdk",
+			lowerCase: true,
+			expected:  "green-gdk",
+		},
+		{
+			pkgName:   "foo--bar__baz",
+			lowerCase: true,
+			expected:  "foo-bar-baz",
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.pkgName, func(t *testing.T) {
-			assert.Equal(t, tt.expected, python.NormalizePkgName(tt.pkgName))
+			assert.Equal(t, tt.expected, python.NormalizePkgName(tt.pkgName, tt.lowerCase))
 		})
 	}
 }

pkg/fanal/analyzer/language/python/pip/pip.go

@@ -2,7 +2,6 @@ package pip
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"io/fs"
 	"os"
@@ -15,6 +14,7 @@ import (
 	"golang.org/x/xerrors"
 
 	goversion "github.com/aquasecurity/go-version/pkg/version"
+	"github.com/aquasecurity/trivy/pkg/dependency/parser/python"
 	"github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging"
 	"github.com/aquasecurity/trivy/pkg/dependency/parser/python/pip"
 	"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
@@ -109,19 +109,15 @@ func (a pipLibraryAnalyzer) Version() int {
 
 // pkgLicense parses `METADATA` pkg file to look for licenses
 func (a pipLibraryAnalyzer) pkgLicense(pkgName, pkgVer, spDir string) []string {
-	// METADATA path is `**/site-packages/<pkg_name>-<pkg_version>.dist-info/METADATA`
-	pkgDir := fmt.Sprintf("%s-%s.dist-info", pkgName, pkgVer)
-	metadataPath := filepath.Join(spDir, pkgDir, "METADATA")
-	metadataFile, err := os.Open(metadataPath)
-	if os.IsNotExist(err) {
-		a.logger.Debug("No package metadata found", log.String("site-packages", pkgDir),
-			log.String("name", pkgName), log.String("version", pkgVer))
+	metadataFile := a.metadataFile(pkgName, pkgVer, spDir)
+	if metadataFile == nil {
 		return nil
 	}
+	defer metadataFile.Close()
 
 	metadataPkg, _, err := a.metadataParser.Parse(metadataFile)
 	if err != nil {
-		a.logger.Warn("Unable to parse METADATA file", log.FilePath(metadataPath), log.Err(err))
+		a.logger.Warn("Unable to parse METADATA file", log.FilePath(metadataFile.Name()), log.Err(err))
 		return nil
 	}
 
@@ -231,3 +227,38 @@ func (a pipLibraryAnalyzer) sortPythonDirs(entries []os.DirEntry) []string {
 		return "python" + v.String()
 	})
 }
+
+// metadataFile returns METADATA file for package (if exists)
+func (a pipLibraryAnalyzer) metadataFile(pkgName, pkgVer, spDir string) *os.File {
+	pkgDirs := distInfoDirs(pkgName, pkgVer)
+	for _, pkgDir := range distInfoDirs(pkgName, pkgVer) {
+		metadataPath := filepath.Join(spDir, pkgDir, "METADATA")
+		metadataFile, err := os.Open(metadataPath)
+		if err == nil {
+			return metadataFile
+		}
+	}
+
+	a.logger.Debug("No package metadata found", log.String("site-packages", spDir),
+		log.String("dist-info", strings.Join(pkgDirs, ", ")), log.String("name", pkgName), log.String("version", pkgVer))
+	return nil
+}
+
+// distInfoDir returns normalized dist-info dir name for package
+// cf. https://packaging.python.org/en/latest/specifications/recording-installed-packages/#the-dist-info-directory
+// e.g. `foo-1.0.dist-info` or `foo_bar-1.0.dist-info`
+func distInfoDirs(name, version string) []string {
+	dirs := []string{
+		// Any packages don't use lower case.
+		// e.g. Flask uses `Flask-2.0.1.dist-info`
+		python.NormalizePkgName(name, false),
+		python.NormalizePkgName(name, true),
+	}
+
+	for i := range dirs {
+		dirs[i] = strings.ReplaceAll(dirs[i], "-", "_")
+		dirs[i] = dirs[i] + "-" + version + ".dist-info"
+	}
+
+	return dirs
+}

pkg/fanal/analyzer/language/python/pip/pip_test.go

@@ -23,21 +23,21 @@ func Test_pipAnalyzer_Analyze(t *testing.T) {
 				FilePath: "requirements.txt",
 				Packages: types.Packages{
 					{
-						Name:    "click",
-						Version: "8.0.0",
+						Name:    "annotated-types",
+						Version: "0.7.0",
 						Locations: []types.Location{
 							{
 								StartLine: 1,
 								EndLine:   1,
 							},
 						},
 						Licenses: []string{
-							"BSD License",
+							"MIT License",
 						},
 					},
 					{
-						Name:    "Flask",
-						Version: "2.0.0",
+						Name:    "click",
+						Version: "8.0.0",
 						Locations: []types.Location{
 							{
 								StartLine: 2,
@@ -49,14 +49,27 @@ func Test_pipAnalyzer_Analyze(t *testing.T) {
 						},
 					},
 					{
-						Name:    "itsdangerous",
+						Name:    "Flask",
 						Version: "2.0.0",
 						Locations: []types.Location{
 							{
 								StartLine: 3,
 								EndLine:   3,
 							},
 						},
+						Licenses: []string{
+							"BSD License",
+						},
+					},
+					{
+						Name:    "itsdangerous",
+						Version: "2.0.0",
+						Locations: []types.Location{
+							{
+								StartLine: 4,
+								EndLine:   4,
+							},
+						},
 					},
 				},
 			},
@@ -100,8 +113,8 @@ func Test_pipAnalyzer_Analyze(t *testing.T) {
 						FilePath: "requirements.txt",
 						Packages: types.Packages{
 							{
-								Name:    "click",
-								Version: "8.0.0",
+								Name:    "annotated-types",
+								Version: "0.7.0",
 								Locations: []types.Location{
 									{
 										StartLine: 1,
@@ -110,8 +123,8 @@ func Test_pipAnalyzer_Analyze(t *testing.T) {
 								},
 							},
 							{
-								Name:    "Flask",
-								Version: "2.0.0",
+								Name:    "click",
+								Version: "8.0.0",
 								Locations: []types.Location{
 									{
 										StartLine: 2,
@@ -120,7 +133,7 @@ func Test_pipAnalyzer_Analyze(t *testing.T) {
 								},
 							},
 							{
-								Name:    "itsdangerous",
+								Name:    "Flask",
 								Version: "2.0.0",
 								Locations: []types.Location{
 									{
@@ -129,6 +142,16 @@ func Test_pipAnalyzer_Analyze(t *testing.T) {
 									},
 								},
 							},
+							{
+								Name:    "itsdangerous",
+								Version: "2.0.0",
+								Locations: []types.Location{
+									{
+										StartLine: 4,
+										EndLine:   4,
+									},
+								},
+							},
 						},
 					},
 				},

pkg/fanal/analyzer/language/python/pip/testdata/happy/requirements.txt

@@ -1,3 +1,4 @@
+annotated-types==0.7.0
 click==8.0.0
 Flask==2.0.0
 itsdangerous==2.0.0

pkg/fanal/analyzer/language/python/pip/testdata/libs/common-dir/lib/site-packages/annotated_types-0.7.0.dist-info/METADATA

@@ -0,0 +1,28 @@
+Metadata-Version: 2.3
+Name: annotated-types
+Version: 0.7.0
+Summary: Reusable constraint types to use with typing.Annotated
+Project-URL: Homepage, https://github.com/annotated-types/annotated-types
+Project-URL: Source, https://github.com/annotated-types/annotated-types
+Project-URL: Changelog, https://github.com/annotated-types/annotated-types/releases
+Author-email: Adrian Garcia Badaracco <[email protected]>, Samuel Colvin <[email protected]>, Zac Hatfield-Dodds <[email protected]>
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Environment :: MacOS X
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.8
+Requires-Dist: typing-extensions>=4.0.0; python_version < '3.9'
+Description-Content-Type: text/markdown

pkg/fanal/analyzer/language/python/pip/testdata/libs/python-dir/lib/python3.10/site-packages/annotated_types-0.7.0.dist-info/METADATA

@@ -0,0 +1,28 @@
+Metadata-Version: 2.3
+Name: annotated-types
+Version: 0.7.0
+Summary: Reusable constraint types to use with typing.Annotated
+Project-URL: Homepage, https://github.com/annotated-types/annotated-types
+Project-URL: Source, https://github.com/annotated-types/annotated-types
+Project-URL: Changelog, https://github.com/annotated-types/annotated-types/releases
+Author-email: Adrian Garcia Badaracco <[email protected]>, Samuel Colvin <[email protected]>, Zac Hatfield-Dodds <[email protected]>
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Environment :: MacOS X
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.8
+Requires-Dist: typing-extensions>=4.0.0; python_version < '3.9'
+Description-Content-Type: text/markdown