From 9907aa81e1206f3e010c903b6200c8e40ee113df Mon Sep 17 00:00:00 2001 From: osv-robot Date: Fri, 31 Oct 2025 00:28:35 +0000 Subject: [PATCH 01/22] test: update snapshots --- cmd/osv-scanner/update/__snapshots__/command_test.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/osv-scanner/update/__snapshots__/command_test.snap b/cmd/osv-scanner/update/__snapshots__/command_test.snap index 7bfc2b3da1d..268a5ec403a 100755 --- a/cmd/osv-scanner/update/__snapshots__/command_test.snap +++ b/cmd/osv-scanner/update/__snapshots__/command_test.snap @@ -256,7 +256,7 @@ file not found: ./testdata/does_not_exist.xml com.fasterxml.jackson.core jackson-core - 2.20.0 + 2.20.1 junit From 5905b95e7cc9482544a5b68efafa38aaa880ac04 Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 31 Oct 2025 11:41:39 +1100 Subject: [PATCH 02/22] First good attempt --- .../__snapshots__/resolve_test.snap | 2 + internal/scalibrplugin/presets.go | 7 + internal/utility/types/cast.go | 10 ++ pkg/osvscanner/internal/scanners/lockfile.go | 41 +++++- pkg/osvscanner/scan.go | 123 ++++++++++++------ 5 files changed, 140 insertions(+), 43 deletions(-) create mode 100644 internal/utility/types/cast.go diff --git a/internal/scalibrplugin/__snapshots__/resolve_test.snap b/internal/scalibrplugin/__snapshots__/resolve_test.snap index 29ae8cfda50..8fafde6c2b0 100755 --- a/internal/scalibrplugin/__snapshots__/resolve_test.snap +++ b/internal/scalibrplugin/__snapshots__/resolve_test.snap @@ -56,6 +56,8 @@ javascript/bunlock javascript/packagelockjson javascript/pnpmlock javascript/yarnlock +os/apk +os/dpkg osv/osvscannerjson php/composerlock python/pdmlock diff --git a/internal/scalibrplugin/presets.go b/internal/scalibrplugin/presets.go index e786bd0c0f1..41d06bc25f9 100644 --- a/internal/scalibrplugin/presets.go +++ b/internal/scalibrplugin/presets.go @@ -113,6 +113,13 @@ var ExtractorPresets = map[string]extractors.InitMap{ stacklock.Name: {stacklock.NewDefault}, osvscannerjson.Name: {osvscannerjson.New}, + + // --- OS "lockfiles" --- + // These have very strict FileRequired paths, so we can safely enable them for source scanning as well. + // Alpine + apk.Name: {apk.NewDefault}, + // Debian + dpkg.Name: {dpkg.NewDefault}, }, "directory": { gitrepo.Name: {gitrepo.New}, diff --git a/internal/utility/types/cast.go b/internal/utility/types/cast.go new file mode 100644 index 00000000000..1960c23041d --- /dev/null +++ b/internal/utility/types/cast.go @@ -0,0 +1,10 @@ +package types + +func MustCastSlice[OUT, IN any](a []IN) []OUT { + out := make([]OUT, len(a)) + for i := range a { + out[i] = any(a[i]).(OUT) + } + + return out +} diff --git a/pkg/osvscanner/internal/scanners/lockfile.go b/pkg/osvscanner/internal/scanners/lockfile.go index 927d4e16170..aec7aa9ed65 100644 --- a/pkg/osvscanner/internal/scanners/lockfile.go +++ b/pkg/osvscanner/internal/scanners/lockfile.go @@ -45,7 +45,10 @@ import ( "github.com/google/osv-scanner/v2/internal/scalibrextract/language/python/requirementsenhancable" ) -var lockfileExtractorMapping = map[string][]string{ +// OSV-Scanner and OSV-Scalibr has different plugin/override naming conventions. +var osvscannerScalibrExtractionMapping = map[string][]string{ + "apk-installed": {apk.Name}, + "dpkg-status": {dpkg.Name}, "pubspec.lock": {pubspec.Name}, "pnpm-lock.yaml": {pnpmlock.Name}, "yarn.lock": {yarnlock.Name}, @@ -102,7 +105,7 @@ func ScanSingleFileWithMapping(scanPath string, pluginsToUse []plugin.Plugin) ([ var err error var inventories []*extractor.Package - parseAs, path := parseLockfilePath(scanPath) + parseAs, path := ParseLockfilePath(scanPath) path, err = filepath.Abs(path) if err != nil { @@ -124,7 +127,7 @@ func ScanSingleFileWithMapping(scanPath string, pluginsToUse []plugin.Plugin) ([ inventories, err = scalibrextract.ExtractWithExtractors(context.Background(), path, pluginsToUse) default: // A specific parseAs without a special case is selected // Find and extract with the extractor of parseAs - if names, ok := lockfileExtractorMapping[parseAs]; ok && len(names) > 0 { + if names, ok := osvscannerScalibrExtractionMapping[parseAs]; ok && len(names) > 0 { i := slices.IndexFunc(pluginsToUse, func(plug plugin.Plugin) bool { _, ok = plug.(filesystem.Extractor) @@ -162,7 +165,7 @@ func ScanSingleFileWithMapping(scanPath string, pluginsToUse []plugin.Plugin) ([ return inventories, nil } -func parseLockfilePath(scanArg string) (string, string) { +func ParseLockfilePath(scanArg string) (string, string) { if (runtime.GOOS == "windows" && filepath.IsAbs(scanArg)) || !strings.Contains(scanArg, ":") { scanArg = ":" + scanArg } @@ -171,3 +174,33 @@ func parseLockfilePath(scanArg string) (string, string) { return splits[0], splits[1] } + +func ParseAsToPlugin(parseAs string, pluginsToUse []plugin.Plugin) (filesystem.Extractor, error) { + switch parseAs { + case "": // No specific parseAs specified + return nil, nil + case "osv-scanner": + return osvscannerjson.Extractor{}, nil + default: + // Find and extract with the extractor of parseAs + if names, ok := osvscannerScalibrExtractionMapping[parseAs]; ok && len(names) > 0 { + i := slices.IndexFunc(pluginsToUse, func(plug plugin.Plugin) bool { + _, ok = plug.(filesystem.Extractor) + + return ok && slices.Contains(names, plug.Name()) + }) + if i < 0 { + return nil, fmt.Errorf("could not determine extractor, requested %s", parseAs) + } + + fsysExtractor, ok := pluginsToUse[i].(filesystem.Extractor) + if !ok { + return nil, fmt.Errorf("invalid extractor name %s", parseAs) + } + + return fsysExtractor, nil + } else { + return nil, fmt.Errorf("could not determine extractor, requested %s", parseAs) + } + } +} diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index bdd133239b2..23c23910130 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -12,16 +12,17 @@ import ( scalibr "github.com/google/osv-scalibr" "github.com/google/osv-scalibr/enricher/reachability/java" "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxmlnet" "github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements" "github.com/google/osv-scalibr/extractor/filesystem/language/python/requirementsnet" + "github.com/google/osv-scalibr/extractor/filesystem/simplefileapi" "github.com/google/osv-scalibr/fs" "github.com/google/osv-scalibr/inventory" "github.com/google/osv-scalibr/plugin" "github.com/google/osv-scalibr/stats" "github.com/google/osv-scanner/v2/internal/cmdlogger" "github.com/google/osv-scanner/v2/internal/imodels" - "github.com/google/osv-scanner/v2/internal/scalibrextract" "github.com/google/osv-scanner/v2/internal/scalibrextract/filesystem/vendored" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/java/pomxmlenhanceable" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/python/requirementsenhancable" @@ -111,62 +112,69 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes } // --- Lockfiles --- - lockfilePlugins := omitDirExtractors(plugins) + //lockfilePlugins := omitDirExtractors(plugins) + + // --- Directories --- + + scanner := scalibr.New() + + // Build list of paths for each root + // On linux this would return a map with just one entry of / + rootMap := map[string][]string{} + + // Also build a map of specific plugin overrides that the user specify + // map[path]parseAs + overrideMap := map[string]filesystem.Extractor{} + var specificPaths []string for _, lockfileElem := range actions.LockfilePaths { - invs, err := scanners.ScanSingleFileWithMapping(lockfileElem, lockfilePlugins) - if err != nil { + parseAs, path := scanners.ParseLockfilePath(lockfileElem) + + //cmdlogger.Infof("Scanning lockfiles %s", path) + if absPath, err := pathToRootMap(rootMap, path); err != nil { return nil, err - } + } else if parseAs != "" { + specificPaths = append(specificPaths, absPath) - scannedInventories = append(scannedInventories, invs...) + plug, err := scanners.ParseAsToPlugin(parseAs, plugins) + if err != nil { + return nil, err + } + overrideMap[absPath] = plug + } } - // --- SBOMs --- + // --- SBOMs (Deprecated) --- // none of the SBOM extractors need configuring sbomExtractors := scalibrplugin.Resolve([]string{"sbom"}, []string{}) + +SBOMLoop: for _, sbomPath := range actions.SBOMPaths { - path, err := filepath.Abs(sbomPath) - if err != nil { - cmdlogger.Errorf("Failed to resolved path %q with error: %s", path, err) + if absPath, err := pathToRootMap(rootMap, sbomPath); err != nil { return nil, err - } - - invs, err := scanners.ScanSingleFile(path, sbomExtractors) - if err != nil { - cmdlogger.Infof("Failed to parse SBOM %q with error: %s", path, err) - - if errors.Is(err, scalibrextract.ErrExtractorNotFound) { - cmdlogger.Infof("If you believe this is a valid SBOM, make sure the filename follows format per your SBOMs specification.") + } else { + specificPaths = append(specificPaths, absPath) + + for _, se := range sbomExtractors { + // All sbom extractors are filesystem extractors + sbomExtractor := se.(filesystem.Extractor) + if sbomExtractor.FileRequired(simplefileapi.New(absPath, nil)) { + overrideMap[absPath] = sbomExtractor + continue SBOMLoop + } } + cmdlogger.Errorf("Failed to parse SBOM %q: Invalid SBOM filename.", sbomPath) + cmdlogger.Errorf("If you believe this is a valid SBOM, make sure the filename follows format per your SBOMs specification.") - return nil, err + return nil, fmt.Errorf("invalid SBOM filename: %s", sbomPath) } - - scannedInventories = append(scannedInventories, invs...) } - // --- Directories --- - - scanner := scalibr.New() - - // Build list of paths for each root - // On linux this would return a map with just one entry of / - rootMap := map[string][]string{} for _, path := range actions.DirectoryPaths { cmdlogger.Infof("Scanning dir %s", path) - absPath, err := filepath.Abs(path) - if err != nil { + if _, err := pathToRootMap(rootMap, path); err != nil { return nil, err } - - _, err = os.Stat(absPath) - if err != nil { - return nil, fmt.Errorf("failed to scan dir: %w", err) - } - - root := getRootDir(absPath) - rootMap[root] = append(rootMap[root], absPath) } testlogger.BeginDirScanMarker() @@ -208,6 +216,14 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes StoreAbsolutePath: true, PrintDurationAnalysis: false, ErrorOnFSErrors: false, + ExtractorOverride: func(api filesystem.FileAPI) []filesystem.Extractor { + ext, ok := overrideMap[filepath.Join(root, api.Path())] + if ok { + return []filesystem.Extractor{ext} + } else { + return []filesystem.Extractor{} + } + }, }) if sr.Status.Status == plugin.ScanStatusFailed { return nil, errors.New(sr.Status.FailureReason) @@ -215,15 +231,26 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes for _, status := range sr.PluginStatus { if status.Status.Status != plugin.ScanStatusSucceeded { builder := strings.Builder{} - + criticalError := false for _, fileError := range status.Status.FileErrors { if len(status.Status.FileErrors) > 1 { // If there is more than 1 file error, write them on new lines builder.WriteString("\n\t") } builder.WriteString(fmt.Sprintf("%s: %s", fileError.FilePath, fileError.ErrorMessage)) + + // Check if the erroring file was a path specifically passed in (not a result of a file walk) + for _, path := range specificPaths { + if strings.Contains(filepath.ToSlash(path), filepath.ToSlash(fileError.FilePath)) { + criticalError = true + break + } + } } cmdlogger.Errorf("Error during extraction: (extracting as %s) %s", status.Name, builder.String()) + if criticalError { + return nil, errors.New("extraction failed on specified lockfile") + } } } genericFindings = append(genericFindings, sr.Inventory.GenericFindings...) @@ -260,6 +287,24 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes return &scanResult, nil } +// pathToRootMap saves the absolute path into the root map, and returns the absolute path +func pathToRootMap(rootMap map[string][]string, path string) (string, error) { + absPath, err := filepath.Abs(path) + if err != nil { + return "", err + } + + _, err = os.Stat(absPath) + if err != nil { + return "", fmt.Errorf("failed to resolve path: %w", err) + } + + root := getRootDir(absPath) + rootMap[root] = append(rootMap[root], absPath) + + return absPath, nil +} + // getRootDir returns the root directory on each system. // On Unix systems, it'll be / // On Windows, it will most likely be the drive (e.g. C:\) From 434d8b84373d564878db67ebb45e2a29dc33756d Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 31 Oct 2025 14:56:19 +1100 Subject: [PATCH 03/22] Fixed scanning --- pkg/osvscanner/invsort.go | 43 +++++++++++++ pkg/osvscanner/scan.go | 118 ++++++++++++++++++++++++++++-------- pkg/osvscanner/scan_test.go | 98 ++++++++++++++++++++++++++++++ pkg/osvscanner/stats.go | 2 + 4 files changed, 235 insertions(+), 26 deletions(-) create mode 100644 pkg/osvscanner/invsort.go create mode 100644 pkg/osvscanner/scan_test.go diff --git a/pkg/osvscanner/invsort.go b/pkg/osvscanner/invsort.go new file mode 100644 index 00000000000..03a3bb7a848 --- /dev/null +++ b/pkg/osvscanner/invsort.go @@ -0,0 +1,43 @@ +package osvscanner + +import ( + "cmp" + "fmt" + + "github.com/google/osv-scalibr/converter" + "github.com/google/osv-scalibr/extractor" +) + +// InventorySort is a comparator function for Inventories, to be used in +// tests with cmp.Diff to disregard the order in which the Inventories +// are reported. +func inventorySort(a, b *extractor.Package) int { + aLoc := fmt.Sprintf("%v", a.Locations) + bLoc := fmt.Sprintf("%v", b.Locations) + + var aExtr, bExtr string + var aPURL, bPURL string + + aPURLStruct := converter.ToPURL(a) + bPURLStruct := converter.ToPURL(b) + + if aPURLStruct != nil { + aPURL = aPURLStruct.String() + } + + if bPURLStruct != nil { + bPURL = bPURLStruct.String() + } + + aSourceCode := fmt.Sprintf("%v", a.SourceCode) + bSourceCode := fmt.Sprintf("%v", b.SourceCode) + + return cmp.Or( + cmp.Compare(aLoc, bLoc), + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Version, b.Version), + cmp.Compare(aSourceCode, bSourceCode), + cmp.Compare(aExtr, bExtr), + cmp.Compare(aPURL, bPURL), + ) +} diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index 23c23910130..04d87348b5f 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "strings" scalibr "github.com/google/osv-scalibr" @@ -20,9 +21,9 @@ import ( "github.com/google/osv-scalibr/fs" "github.com/google/osv-scalibr/inventory" "github.com/google/osv-scalibr/plugin" - "github.com/google/osv-scalibr/stats" "github.com/google/osv-scanner/v2/internal/cmdlogger" "github.com/google/osv-scanner/v2/internal/imodels" + "github.com/google/osv-scanner/v2/internal/scalibrextract" "github.com/google/osv-scanner/v2/internal/scalibrextract/filesystem/vendored" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/java/pomxmlenhanceable" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/python/requirementsenhancable" @@ -114,8 +115,6 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes // --- Lockfiles --- //lockfilePlugins := omitDirExtractors(plugins) - // --- Directories --- - scanner := scalibr.New() // Build list of paths for each root @@ -126,21 +125,35 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes // map[path]parseAs overrideMap := map[string]filesystem.Extractor{} var specificPaths []string + statsCollector := fileOpenedPrinter{ + filesExtracted: make(map[string]struct{}), + } + // --- Directories --- + for _, path := range actions.DirectoryPaths { + cmdlogger.Infof("Scanning dir %s", path) + if _, err := pathToRootMap(rootMap, path, actions.Recursive); err != nil { + return nil, err + } + } + + // --- Lockfiles --- for _, lockfileElem := range actions.LockfilePaths { parseAs, path := scanners.ParseLockfilePath(lockfileElem) //cmdlogger.Infof("Scanning lockfiles %s", path) - if absPath, err := pathToRootMap(rootMap, path); err != nil { + if absPath, err := pathToRootMap(rootMap, path, actions.Recursive); err != nil { return nil, err - } else if parseAs != "" { + } else { specificPaths = append(specificPaths, absPath) - plug, err := scanners.ParseAsToPlugin(parseAs, plugins) - if err != nil { - return nil, err + if parseAs != "" { + plug, err := scanners.ParseAsToPlugin(parseAs, plugins) + if err != nil { + return nil, err + } + overrideMap[absPath] = plug } - overrideMap[absPath] = plug } } @@ -150,7 +163,7 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes SBOMLoop: for _, sbomPath := range actions.SBOMPaths { - if absPath, err := pathToRootMap(rootMap, sbomPath); err != nil { + if absPath, err := pathToRootMap(rootMap, sbomPath, actions.Recursive); err != nil { return nil, err } else { specificPaths = append(specificPaths, absPath) @@ -170,22 +183,15 @@ SBOMLoop: } } - for _, path := range actions.DirectoryPaths { - cmdlogger.Infof("Scanning dir %s", path) - if _, err := pathToRootMap(rootMap, path); err != nil { - return nil, err - } - } - testlogger.BeginDirScanMarker() osCapability := determineOS() - var statsCollector stats.Collector - if actions.StatsCollector != nil { - statsCollector = actions.StatsCollector - } else { - statsCollector = fileOpenedPrinter{} - } + //var statsCollector stats.Collector + //if actions.StatsCollector != nil { + // statsCollector = actions.StatsCollector + //} else { + // statsCollector = fileOpenedPrinter{} + //} // For each root, run scalibr's scan() once. for root, paths := range rootMap { @@ -225,6 +231,7 @@ SBOMLoop: } }, }) + if sr.Status.Status == plugin.ScanStatusFailed { return nil, errors.New(sr.Status.FailureReason) } @@ -253,6 +260,21 @@ SBOMLoop: } } } + + // Check if specific paths have been extracted + for _, path := range specificPaths { + key, _ := filepath.Rel(root, path) + if _, ok := statsCollector.filesExtracted[key]; !ok { + return nil, fmt.Errorf("%w: %q", scalibrextract.ErrExtractorNotFound, path) + } + } + + slices.SortFunc(sr.Inventory.Packages, inventorySort) + invsCompact := slices.CompactFunc(sr.Inventory.Packages, func(a, b *extractor.Package) bool { + return inventorySort(a, b) == 0 + }) + sr.Inventory.Packages = invsCompact + genericFindings = append(genericFindings, sr.Inventory.GenericFindings...) scannedInventories = append(scannedInventories, sr.Inventory.Packages...) } @@ -287,24 +309,68 @@ SBOMLoop: return &scanResult, nil } -// pathToRootMap saves the absolute path into the root map, and returns the absolute path -func pathToRootMap(rootMap map[string][]string, path string) (string, error) { +// pathToRootMap saves the absolute path into the root map, and returns the absolute path. +// path is only saved if it does not fall under an existing path. +// IMPORTANT: it does not remove existing paths already added to the rootMap, so add directories before specific files. +func pathToRootMap(rootMap map[string][]string, path string, recursive bool) (string, error) { absPath, err := filepath.Abs(path) if err != nil { return "", err } - _, err = os.Stat(absPath) + fi, err := os.Stat(absPath) if err != nil { return "", fmt.Errorf("failed to resolve path: %w", err) } root := getRootDir(absPath) + // If path is a directory and we are not recursively scanning, then always add it as a target. + if fi.IsDir() && !recursive { + rootMap[root] = append(rootMap[root], absPath) + return absPath, nil + } + + // Otherwise, only add if it's not a descendent of an existing path + for _, existing := range rootMap[root] { + if isDescendent(existing, absPath, recursive) { + return absPath, nil + } + } rootMap[root] = append(rootMap[root], absPath) return absPath, nil } +func isDescendent(potentialParent, path string, recursive bool) bool { + rel, err := filepath.Rel(potentialParent, path) + if err != nil { + // This should never happen + return false + } + + if rel == "." { + // Same as an existing path, skip + return true + } + + if strings.HasPrefix(rel, "..") { + return false + } + + depths := len(strings.Split(rel, string(filepath.Separator))) + if recursive { + // Descendant of existing dir, and we are recursively scanning, so skip + return true + } + + if depths == 1 { + // Direct child of existing dir, skip + return true + } + + return false +} + // getRootDir returns the root directory on each system. // On Unix systems, it'll be / // On Windows, it will most likely be the drive (e.g. C:\) diff --git a/pkg/osvscanner/scan_test.go b/pkg/osvscanner/scan_test.go new file mode 100644 index 00000000000..ac55d390a6a --- /dev/null +++ b/pkg/osvscanner/scan_test.go @@ -0,0 +1,98 @@ +package osvscanner + +import ( + "path/filepath" + "testing" +) + + +func Test_isDescendent(t *testing.T) { + tests := []struct { + name string + potentialParent string + path string + recursive bool + want bool + }{ + { + name: "same path", + potentialParent: "/a/b", + path: "/a/b", + recursive: true, + want: true, + }, + { + name: "direct child, recursive", + potentialParent: "/a/b", + path: "/a/b/c", + recursive: true, + want: true, + }, + { + name: "direct child, non-recursive", + potentialParent: "/a/b", + path: "/a/b/c", + recursive: false, + want: true, + }, + { + name: "grandchild, recursive", + potentialParent: "/a/b", + path: "/a/b/c/d", + recursive: true, + want: true, + }, + { + name: "grandchild, non-recursive", + potentialParent: "/a/b", + path: "/a/b/c/d", + recursive: false, + want: false, + }, + { + name: "not a descendent", + potentialParent: "/a/b", + path: "/a/c", + recursive: true, + want: false, + }, + { + name: "different root", + potentialParent: "/a/b", + path: "/x/y", + recursive: true, + want: false, + }, + { + name: "relative path, direct child, recursive", + potentialParent: "a/b", + path: "a/b/c", + recursive: true, + want: true, + }, + { + name: "relative path, grandchild, non-recursive", + potentialParent: "a/b", + path: "a/b/c/d", + recursive: false, + want: false, + }, + { + name: "relative path, not a descendent", + potentialParent: "a/b", + path: "a/c", + recursive: true, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Normalize paths for the current OS + potentialParent := filepath.FromSlash(tt.potentialParent) + path := filepath.FromSlash(tt.path) + if got := isDescendent(potentialParent, path, tt.recursive); got != tt.want { + t.Errorf("isDescendent(%q, %q, %v) = %v, want %v", tt.potentialParent, tt.path, tt.recursive, got, tt.want) + } + }) + } +} diff --git a/pkg/osvscanner/stats.go b/pkg/osvscanner/stats.go index 0ec3a7755d5..1d986d2d1d7 100644 --- a/pkg/osvscanner/stats.go +++ b/pkg/osvscanner/stats.go @@ -10,11 +10,13 @@ import ( type fileOpenedPrinter struct { stats.NoopCollector + filesExtracted map[string]struct{} } var _ stats.Collector = &fileOpenedPrinter{} func (c fileOpenedPrinter) AfterExtractorRun(_ string, extractorstats *stats.AfterExtractorStats) { + c.filesExtracted[extractorstats.Path] = struct{}{} if extractorstats.Error != nil { // Don't log scanned if error occurred return } From f0915aa83511e512912448b947afa8016d97574d Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 31 Oct 2025 15:04:13 +1100 Subject: [PATCH 04/22] Update snaps --- .../source/__snapshots__/command_test.snap | 189 +++++++++--------- 1 file changed, 96 insertions(+), 93 deletions(-) diff --git a/cmd/osv-scanner/scan/source/__snapshots__/command_test.snap b/cmd/osv-scanner/scan/source/__snapshots__/command_test.snap index 81fb6b861b7..562da5cbc6f 100755 --- a/cmd/osv-scanner/scan/source/__snapshots__/command_test.snap +++ b/cmd/osv-scanner/scan/source/__snapshots__/command_test.snap @@ -415,13 +415,13 @@ Error during extraction: (extracting as php/composerlock) /testdata/loc --- [TestCommand/config_file_can_be_broad - 1] -Scanned /testdata/locks-insecure/osv-scanner-flutter-deps.json file as a osv-scanner and found 3 packages Scanning dir ./testdata/locks-many-with-insecure Scanning dir ./testdata/locks-insecure Scanning dir ./testdata/maven-transitive Scanned /testdata/locks-insecure/bun.lock file and found 2 packages Scanned /testdata/locks-insecure/composer.lock file and found 1 package Scanned /testdata/locks-insecure/osv-scanner-custom.json file and found 2 packages +Scanned /testdata/locks-insecure/osv-scanner-flutter-deps.json file and found 3 packages Scanned /testdata/locks-many-with-insecure/Gemfile.lock file and found 1 package Scanned /testdata/locks-many-with-insecure/alpine.cdx.xml file and found 15 packages Scanned /testdata/locks-many-with-insecure/composer.lock file and found 1 package @@ -429,30 +429,30 @@ Scanned /testdata/locks-many-with-insecure/package-lock.json file and f Scanned /testdata/locks-many-with-insecure/yarn.lock file and found 1 package Scanned /testdata/maven-transitive/pom.xml file and found 3 packages Filtered 1 local/unscannable package/s from the scan. +Package npm/has-flag/4.0.0 has been filtered out because: (no reason given) +Package npm/wrappy/1.0.2 has been filtered out because: (no reason given) Package npm/ansi-html/0.0.1 has been filtered out because: (no reason given) Package npm/balanced-match/1.0.2 has been filtered out because: (no reason given) -Package npm/has-flag/4.0.0 has been filtered out because: (no reason given) Package Maven/org.apache.logging.log4j:log4j-api/2.14.1 has been filtered out because: it makes the table output really really long Package Maven/org.apache.logging.log4j:log4j-core/2.14.1 has been filtered out because: it makes the table output really really long Package Maven/org.apache.logging.log4j:log4j-web/2.14.1 has been filtered out because: it makes the table output really really long -Package npm/wrappy/1.0.2 has been filtered out because: (no reason given) Filtered 7 ignored package/s from the scan. +overriding license for package Packagist/league/flysystem/1.0.8 with 0BSD overriding license for package Alpine/alpine-baselayout/3.4.0-r0 with MIT overriding license for package Alpine/alpine-baselayout-data/3.4.0-r0 with MIT overriding license for package Alpine/alpine-keys/2.4-r1 with MIT overriding license for package Alpine/apk-tools/2.12.10-r1 with MIT overriding license for package Alpine/busybox-binsh/1.36.1-r27 with MIT overriding license for package Alpine/ca-certificates-bundle/20220614-r4 with MIT -overriding license for package Packagist/league/flysystem/1.0.8 with 0BSD overriding license for package Alpine/libc-utils/0.7.2-r3 with MIT overriding license for package Alpine/libcrypto3/3.0.8-r0 with MIT overriding license for package Alpine/libssl3/3.0.8-r0 with MIT overriding license for package Alpine/musl/1.2.3-r4 with MIT overriding license for package Alpine/musl-utils/1.2.3-r4 with MIT overriding license for package Alpine/scanelf/1.3.5-r1 with MIT -overriding license for package Packagist/sentry/sdk/2.0.4 with 0BSD overriding license for package Alpine/ssl_client/1.36.1-r27 with MIT overriding license for package Alpine/zlib/1.2.13-r0 with MIT +overriding license for package Packagist/sentry/sdk/2.0.4 with 0BSD Total 2 packages affected by 2 known vulnerabilities (1 Critical, 1 High, 0 Medium, 0 Low, 0 Unknown) from 5 ecosystems. 1 vulnerability can be fixed. @@ -979,7 +979,7 @@ Scanned /testdata/locks-many-with-insecure/package-lock.json file and f --- [TestCommand/go_packages_in_osv-scanner.json_format - 1] -Scanned /testdata/locks-insecure/osv-scanner.json file as a osv-scanner and found 2 packages +Scanned /testdata/locks-insecure/osv-scanner.json file and found 2 packages Total 2 packages affected by 13 known vulnerabilities (0 Critical, 0 High, 0 Medium, 0 Low, 13 Unknown) from 1 ecosystem. 13 vulnerabilities can be fixed. @@ -1092,7 +1092,7 @@ Scanning dir ./testdata/locks-none-does-not-exist --- [TestCommand/no_lockfiles_with_allow_flag_but_another_error_happens_is_not_fine - 2] -failed to scan dir: stat /testdata/locks-none-does-not-exist: no such file or directory +failed to resolve path: stat /testdata/locks-none-does-not-exist: no such file or directory --- @@ -1141,13 +1141,13 @@ No package sources found, --help for usage information. [TestCommand/one_file_that_does_not_match_the_supported_sbom_file_names - 1] Warning: --sbom has been deprecated in favor of -L -Failed to parse SBOM "/testdata/locks-many/composer.lock" with error: could not determine extractor suitable to this file -If you believe this is a valid SBOM, make sure the filename follows format per your SBOMs specification. --- [TestCommand/one_file_that_does_not_match_the_supported_sbom_file_names - 2] -could not determine extractor suitable to this file +Failed to parse SBOM "./testdata/locks-many/composer.lock": Invalid SBOM filename. +If you believe this is a valid SBOM, make sure the filename follows format per your SBOMs specification. +invalid SBOM filename: ./testdata/locks-many/composer.lock --- @@ -1202,7 +1202,7 @@ No issues found [TestCommand/one_specific_supported_sbom_with_duplicate_PURLs - 1] Warning: --sbom has been deprecated in favor of -L -Scanned /testdata/sbom-insecure/with-duplicates.cdx.xml file and found 15 packages +Scanned /testdata/sbom-insecure/with-duplicates.cdx.xml file and found 17 packages Filtered 1 local/unscannable package/s from the scan. Total 2 packages affected by 3 known vulnerabilities (1 Critical, 2 High, 0 Medium, 0 Low, 0 Unknown) from 1 ecosystem. 0 vulnerabilities can be fixed. @@ -1223,7 +1223,7 @@ Total 2 packages affected by 3 known vulnerabilities (1 Critical, 2 High, 0 Medi --- [TestCommand/one_specific_supported_sbom_with_duplicate_PURLs_using_-L_flag - 1] -Scanned /testdata/sbom-insecure/with-duplicates.cdx.xml file and found 15 packages +Scanned /testdata/sbom-insecure/with-duplicates.cdx.xml file and found 17 packages Filtered 1 local/unscannable package/s from the scan. Total 2 packages affected by 3 known vulnerabilities (1 Critical, 2 High, 0 Medium, 0 Low, 0 Unknown) from 1 ecosystem. 0 vulnerabilities can be fixed. @@ -1739,7 +1739,7 @@ Total 0 packages affected by 0 known vulnerabilities (0 Critical, 0 High, 0 Medi --- [TestCommand_CommitSupport/offline_uses_git_tags - 1] -Scanned /testdata/locks-git/osv-scanner.json file as a osv-scanner and found 11 packages +Scanned /testdata/locks-git/osv-scanner.json file and found 11 packages Loaded GIT local db from /osv-scanner/GIT/all.zip Skipping commit scanning for: 45fda76bc1b9fd74d10e85e0ce9b65a12dcc58b0 Total 8 packages affected by 17 known vulnerabilities (4 Critical, 4 High, 3 Medium, 0 Low, 6 Unknown) from 1 ecosystem. @@ -1775,7 +1775,7 @@ Total 8 packages affected by 17 known vulnerabilities (4 Critical, 4 High, 3 Med --- [TestCommand_CommitSupport/online_uses_git_commits - 1] -Scanned /testdata/locks-git/osv-scanner.json file as a osv-scanner and found 11 packages +Scanned /testdata/locks-git/osv-scanner.json file and found 11 packages Total 10 packages affected by 37 known vulnerabilities (6 Critical, 7 High, 14 Medium, 4 Low, 6 Unknown) from 1 ecosystem. 0 vulnerabilities can be fixed. @@ -1873,8 +1873,7 @@ CVE-2022-1304 and 2 aliases have been filtered out because: (no reason given) GO-2022-0274 and 2 aliases have been filtered out because: (no reason given) CVE-2025-26519 and 1 alias have been filtered out because: (no reason given) CVE-2018-25032 and 1 alias have been filtered out because: (no reason given) -CVE-2018-25032 and 1 alias have been filtered out because: (no reason given) -Filtered 9 vulnerabilities from output +Filtered 8 vulnerabilities from output testdata/osv-scanner-partial-ignores-config.toml has unused ignores: - CVE-2025-26519 - CVE-2018-25032 @@ -2571,7 +2570,7 @@ could not determine extractor, requested package-lock.json --- [TestCommand_GithubActions/scanning_osv-scanner_custom_format - 1] -Scanned /testdata/locks-insecure/osv-scanner-flutter-deps.json file as a osv-scanner and found 3 packages +Scanned /testdata/locks-insecure/osv-scanner-flutter-deps.json file and found 3 packages Total 1 package affected by 2 known vulnerabilities (0 Critical, 2 High, 0 Medium, 0 Low, 0 Unknown) from 1 ecosystem. 0 vulnerabilities can be fixed. @@ -2768,7 +2767,7 @@ Total 1 package affected by 2 known vulnerabilities (0 Critical, 2 High, 0 Mediu --- [TestCommand_GithubActions/scanning_osv-scanner_custom_format_output_json - 2] -Scanned /testdata/locks-insecure/osv-scanner-flutter-deps.json file as a osv-scanner and found 3 packages +Scanned /testdata/locks-insecure/osv-scanner-flutter-deps.json file and found 3 packages --- @@ -3293,10 +3292,10 @@ Scanned /testdata/locks-many/Gemfile.lock file and found 1 package Scanned /testdata/locks-many/composer.lock file and found 1 package Scanned /testdata/locks-many/package-lock.json file and found 1 package Scanned /testdata/locks-many/yarn.lock file and found 1 package -Package npm/ansi-html/0.0.8 has been filtered out because: (no reason given) -Package npm/balanced-match/1.0.2 has been filtered out because: (no reason given) Package npm/has-flag/4.0.0 has been filtered out because: (no reason given) Package npm/wrappy/1.0.2 has been filtered out because: (no reason given) +Package npm/ansi-html/0.0.8 has been filtered out because: (no reason given) +Package npm/balanced-match/1.0.2 has been filtered out because: (no reason given) Filtered 4 ignored package/s from the scan. overriding license for package Packagist/league/flysystem/1.0.8 with 0BSD overriding license for package Packagist/sentry/sdk/2.0.4 with 0BSD @@ -3615,10 +3614,10 @@ Scanned /testdata/locks-many-with-insecure/composer.lock file and found Scanned /testdata/locks-many-with-insecure/package-lock.json file and found 1 package Scanned /testdata/locks-many-with-insecure/yarn.lock file and found 1 package Filtered 1 local/unscannable package/s from the scan. -Loaded Alpine local db from /osv-scanner/Alpine/all.zip -Loaded npm local db from /osv-scanner/npm/all.zip Loaded RubyGems local db from /osv-scanner/RubyGems/all.zip +Loaded Alpine local db from /osv-scanner/Alpine/all.zip Loaded Packagist local db from /osv-scanner/Packagist/all.zip +Loaded npm local db from /osv-scanner/npm/all.zip Total 2 packages affected by 2 known vulnerabilities (0 Critical, 2 High, 0 Medium, 0 Low, 0 Unknown) from 2 ecosystems. 1 vulnerability can be fixed. @@ -3644,10 +3643,10 @@ Scanned /testdata/locks-many-with-insecure/composer.lock file and found Scanned /testdata/locks-many-with-insecure/package-lock.json file and found 1 package Scanned /testdata/locks-many-with-insecure/yarn.lock file and found 1 package Filtered 1 local/unscannable package/s from the scan. -Loaded Alpine local db from /osv-scanner/Alpine/all.zip -Loaded npm local db from /osv-scanner/npm/all.zip Loaded RubyGems local db from /osv-scanner/RubyGems/all.zip +Loaded Alpine local db from /osv-scanner/Alpine/all.zip Loaded Packagist local db from /osv-scanner/Packagist/all.zip +Loaded npm local db from /osv-scanner/npm/all.zip Total 2 packages affected by 2 known vulnerabilities (0 Critical, 2 High, 0 Medium, 0 Low, 0 Unknown) from 2 ecosystems. 1 vulnerability can be fixed. @@ -3730,8 +3729,8 @@ Scanned /testdata/locks-gitignore/subdir/composer.lock file and found 1 Scanned /testdata/locks-gitignore/subdir/yarn.lock file and found 1 package Scanned /testdata/locks-gitignore/yarn.lock file and found 1 package Loaded RubyGems local db from /osv-scanner/RubyGems/all.zip -Loaded npm local db from /osv-scanner/npm/all.zip Loaded Packagist local db from /osv-scanner/Packagist/all.zip +Loaded npm local db from /osv-scanner/npm/all.zip No issues found --- @@ -3751,8 +3750,8 @@ Scanned /testdata/locks-gitignore/subdir/composer.lock file and found 1 Scanned /testdata/locks-gitignore/subdir/yarn.lock file and found 1 package Scanned /testdata/locks-gitignore/yarn.lock file and found 1 package Loaded RubyGems local db from /osv-scanner/RubyGems/all.zip -Loaded npm local db from /osv-scanner/npm/all.zip Loaded Packagist local db from /osv-scanner/Packagist/all.zip +Loaded npm local db from /osv-scanner/npm/all.zip No issues found --- @@ -3765,8 +3764,8 @@ No issues found Scanning dir ./testdata/locks-one-with-nested Scanned /testdata/locks-one-with-nested/nested/composer.lock file and found 1 package Scanned /testdata/locks-one-with-nested/yarn.lock file and found 1 package -Loaded npm local db from /osv-scanner/npm/all.zip Loaded Packagist local db from /osv-scanner/Packagist/all.zip +Loaded npm local db from /osv-scanner/npm/all.zip No issues found --- @@ -3779,8 +3778,8 @@ No issues found Scanning dir ./testdata/locks-one-with-nested Scanned /testdata/locks-one-with-nested/nested/composer.lock file and found 1 package Scanned /testdata/locks-one-with-nested/yarn.lock file and found 1 package -Loaded npm local db from /osv-scanner/npm/all.zip Loaded Packagist local db from /osv-scanner/Packagist/all.zip +Loaded npm local db from /osv-scanner/npm/all.zip No issues found --- @@ -4353,11 +4352,11 @@ Total 0 packages affected by 0 known vulnerabilities (0 Critical, 0 High, 0 Medi --- [TestCommand_LocalDatabases_AlwaysOffline/a_bunch_of_different_lockfiles_and_ecosystem - 2] +could not load db for RubyGems ecosystem: unable to fetch OSV database: no offline version of the OSV database is available could not load db for Alpine ecosystem: unable to fetch OSV database: no offline version of the OSV database is available +could not load db for Packagist ecosystem: unable to fetch OSV database: no offline version of the OSV database is available could not load db for npm ecosystem: unable to fetch OSV database: no offline version of the OSV database is available -could not load db for RubyGems ecosystem: unable to fetch OSV database: no offline version of the OSV database is available could not load db for PyPI ecosystem: unable to fetch OSV database: no offline version of the OSV database is available -could not load db for Packagist ecosystem: unable to fetch OSV database: no offline version of the OSV database is available --- @@ -4385,16 +4384,16 @@ Total 0 packages affected by 0 known vulnerabilities (0 Critical, 0 High, 0 Medi --- [TestCommand_LocalDatabases_AlwaysOffline/a_bunch_of_different_lockfiles_and_ecosystem - 4] +could not load db for RubyGems ecosystem: unable to fetch OSV database: no offline version of the OSV database is available could not load db for Alpine ecosystem: unable to fetch OSV database: no offline version of the OSV database is available +could not load db for Packagist ecosystem: unable to fetch OSV database: no offline version of the OSV database is available could not load db for npm ecosystem: unable to fetch OSV database: no offline version of the OSV database is available -could not load db for RubyGems ecosystem: unable to fetch OSV database: no offline version of the OSV database is available could not load db for PyPI ecosystem: unable to fetch OSV database: no offline version of the OSV database is available -could not load db for Packagist ecosystem: unable to fetch OSV database: no offline version of the OSV database is available --- [TestCommand_LockfileWithExplicitParseAs/"apk-installed"_is_supported - 1] -Scanned /testdata/locks-many/installed file as a apk-installed and found 1 package +Scanned /testdata/locks-many/installed file and found 1 package Loaded filter from: /testdata/locks-many/osv-scanner-test.toml No issues found @@ -4405,7 +4404,7 @@ No issues found --- [TestCommand_LockfileWithExplicitParseAs/"dpkg-status"_is_supported - 1] -Scanned /testdata/locks-many/status file as a dpkg-status and found 1 package +Scanned /testdata/locks-many/status file and found 1 package Loaded filter from: /testdata/locks-many/osv-scanner-test.toml No issues found @@ -4431,7 +4430,8 @@ No issues found --- [TestCommand_LockfileWithExplicitParseAs/absolute_paths_can_have_explicit_parse_as - 2] -(extracting as javascript/packagelockjson) could not extract: invalid character '#' looking for beginning of value +Error during extraction: (extracting as javascript/packagelockjson) /testdata/locks-many/yarn.lock: could not extract: invalid character '#' looking for beginning of value +extraction failed on specified lockfile --- @@ -4462,7 +4462,7 @@ No issues found --- [TestCommand_LockfileWithExplicitParseAs/empty_works_as_an_escape_(no_fixture_because_it's_not_valid_on_Windows) - 2] -stat /path/to/my:file: no such file or directory +failed to resolve path: stat /path/to/my:file: no such file or directory --- @@ -4471,25 +4471,27 @@ stat /path/to/my:file: no such file or directory --- [TestCommand_LockfileWithExplicitParseAs/empty_works_as_an_escape_(no_fixture_because_it's_not_valid_on_Windows)#01 - 2] -stat /path/to/my:project/package-lock.json: no such file or directory +failed to resolve path: stat /path/to/my:project/package-lock.json: no such file or directory --- [TestCommand_LockfileWithExplicitParseAs/files_that_error_on_parsing_stop_parsable_files_from_being_checked - 1] - +Scanning dir ./testdata/locks-insecure +Scanning dir ./testdata/locks-many --- [TestCommand_LockfileWithExplicitParseAs/files_that_error_on_parsing_stop_parsable_files_from_being_checked - 2] -(extracting as rust/cargolock) could not extract: toml: line 1: expected '.' or '=', but got '{' instead +Error during extraction: (extracting as rust/cargolock) /testdata/locks-insecure/my-package-lock.json: could not extract: toml: line 1: expected '.' or '=', but got '{' instead +extraction failed on specified lockfile --- [TestCommand_LockfileWithExplicitParseAs/multiple,_+_output_order_is_deterministic - 1] -Scanned /testdata/locks-insecure/my-package-lock.json file as a package-lock.json and found 1 package -Scanned /testdata/locks-insecure/my-yarn.lock file as a yarn.lock and found 1 package Scanning dir ./testdata/locks-insecure Scanned /testdata/locks-insecure/bun.lock file and found 2 packages Scanned /testdata/locks-insecure/composer.lock file and found 1 package +Scanned /testdata/locks-insecure/my-package-lock.json file and found 1 package +Scanned /testdata/locks-insecure/my-yarn.lock file and found 1 package Scanned /testdata/locks-insecure/osv-scanner-custom.json file and found 2 packages Total 3 packages affected by 3 known vulnerabilities (1 Critical, 2 High, 0 Medium, 0 Low, 0 Unknown) from 2 ecosystems. 3 vulnerabilities can be fixed. @@ -4510,11 +4512,11 @@ Total 3 packages affected by 3 known vulnerabilities (1 Critical, 2 High, 0 Medi --- [TestCommand_LockfileWithExplicitParseAs/multiple,_+_output_order_is_deterministic_2 - 1] -Scanned /testdata/locks-insecure/my-yarn.lock file as a yarn.lock and found 1 package -Scanned /testdata/locks-insecure/my-package-lock.json file as a package-lock.json and found 1 package Scanning dir ./testdata/locks-insecure Scanned /testdata/locks-insecure/bun.lock file and found 2 packages Scanned /testdata/locks-insecure/composer.lock file and found 1 package +Scanned /testdata/locks-insecure/my-package-lock.json file and found 1 package +Scanned /testdata/locks-insecure/my-yarn.lock file and found 1 package Scanned /testdata/locks-insecure/osv-scanner-custom.json file and found 2 packages Total 3 packages affected by 3 known vulnerabilities (1 Critical, 2 High, 0 Medium, 0 Low, 0 Unknown) from 2 ecosystems. 3 vulnerabilities can be fixed. @@ -4535,7 +4537,7 @@ Total 3 packages affected by 3 known vulnerabilities (1 Critical, 2 High, 0 Medi --- [TestCommand_LockfileWithExplicitParseAs/one_lockfile_with_local_path - 1] -Scanned /testdata/locks-many/replace-local.mod file as a go.mod and found 1 package +Scanned /testdata/locks-many/replace-local.mod file and found 1 package Filtered 1 local/unscannable package/s from the scan. No issues found @@ -4550,7 +4552,8 @@ No issues found --- [TestCommand_LockfileWithExplicitParseAs/parse-as_takes_priority,_even_if_it's_wrong - 2] -(extracting as javascript/packagelockjson) could not extract: invalid character '#' looking for beginning of value +Error during extraction: (extracting as javascript/packagelockjson) /testdata/locks-many/yarn.lock: could not extract: invalid character '#' looking for beginning of value +extraction failed on specified lockfile --- @@ -4564,10 +4567,10 @@ could not determine extractor, requested my-file --- [TestCommand_LockfileWithExplicitParseAs/when_an_explicit_parse-as_is_given,_it's_applied_to_that_file - 1] -Scanned /testdata/locks-insecure/my-package-lock.json file as a package-lock.json and found 1 package Scanning dir ./testdata/locks-insecure Scanned /testdata/locks-insecure/bun.lock file and found 2 packages Scanned /testdata/locks-insecure/composer.lock file and found 1 package +Scanned /testdata/locks-insecure/my-package-lock.json file and found 1 package Scanned /testdata/locks-insecure/osv-scanner-custom.json file and found 2 packages Total 2 packages affected by 2 known vulnerabilities (1 Critical, 1 High, 0 Medium, 0 Low, 0 Unknown) from 2 ecosystems. 2 vulnerabilities can be fixed. @@ -4591,7 +4594,7 @@ Total 2 packages affected by 2 known vulnerabilities (1 Critical, 1 High, 0 Medi --- [TestCommand_MoreLockfiles/Package.resolved_-_Unsupported_ecosystem,_should_not_be_scanned - 2] -could not determine extractor suitable to this file +could not determine extractor suitable to this file: "/testdata/locks-scalibr/Package.resolved" --- @@ -4600,7 +4603,7 @@ could not determine extractor suitable to this file --- [TestCommand_MoreLockfiles/Podfile.lock_-_Unsupported_ecosystem,_should_not_be_scanned - 2] -could not determine extractor suitable to this file +could not determine extractor suitable to this file: "/testdata/locks-scalibr/Podfile.lock" --- @@ -4623,7 +4626,7 @@ Total 1 package affected by 1 known vulnerability (0 Critical, 0 High, 0 Medium, --- [TestCommand_MoreLockfiles/depsjson - 1] -Scanned /testdata/locks-scalibr/depsjson file as a deps.json and found 4 packages +Scanned /testdata/locks-scalibr/depsjson file and found 4 packages Total 1 package affected by 1 known vulnerability (0 Critical, 0 High, 0 Medium, 0 Low, 1 Unknown) from 1 ecosystem. 1 vulnerability can be fixed. @@ -4641,7 +4644,7 @@ Total 1 package affected by 1 known vulnerability (0 Critical, 0 High, 0 Medium, --- [TestCommand_MoreLockfiles/gems.locked - 1] -Scanned /testdata/locks-scalibr/gems.locked file and found 25 packages +Scanned /testdata/locks-scalibr/gems.locked file and found 26 packages Total 2 packages affected by 5 known vulnerabilities (0 Critical, 2 High, 0 Medium, 0 Low, 3 Unknown) from 1 ecosystem. 5 vulnerabilities can be fixed. @@ -4834,75 +4837,75 @@ Total 3 packages affected by 8 known vulnerabilities (0 Critical, 3 High, 4 Medi --- [TestCommand_Transitive/resolves_transitive_dependencies_with_native_data_source - 1] -Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/log4j/log4j-web/2.14.1/log4j-web-2.14.1.pom -Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/log4j/log4j-web/2.14.1/log4j-web-2.14.1.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services/10.0.0/play-services-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/log4j/log4j/2.14.1/log4j-2.14.1.pom -Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/log4j/log4j/2.14.1/log4j-2.14.1.pom -Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/logging-parent/3/logging-parent-3.pom -Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/logging-parent/3/logging-parent-3.pom -Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/apache/23/apache-23.pom -Fetching response from: https://repo.maven.apache.org/maven2/org/apache/apache/23/apache-23.pom -Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/log4j/log4j-api/2.14.1/log4j-api-2.14.1.pom -Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/log4j/log4j-api/2.14.1/log4j-api-2.14.1.pom -Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/log4j/log4j-core/2.14.1/log4j-core-2.14.1.pom -Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/log4j/log4j-core/2.14.1/log4j-core-2.14.1.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-ads/10.0.0/play-services-ads-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/animated-vector-drawable/24.0.0/animated-vector-drawable-24.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/appcompat-v7/24.0.0/appcompat-v7-24.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/mediarouter-v7/24.0.0/mediarouter-v7-24.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/palette-v7/24.0.0/palette-v7-24.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/support-annotations/24.0.0/support-annotations-24.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/support-v4/24.0.0/support-v4-24.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/support-vector-drawable/24.0.0/support-vector-drawable-24.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-ads-lite/10.0.0/play-services-ads-lite-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-analytics/10.0.0/play-services-analytics-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-ads/10.0.0/play-services-ads-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-analytics-impl/10.0.0/play-services-analytics-impl-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-analytics/10.0.0/play-services-analytics-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-appinvite/10.0.0/play-services-appinvite-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-auth/10.0.0/play-services-auth-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-auth-base/10.0.0/play-services-auth-base-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-auth/10.0.0/play-services-auth-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-awareness/10.0.0/play-services-awareness-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-base/10.0.0/play-services-base-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-basement/10.0.0/play-services-basement-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-cast-framework/10.0.0/play-services-cast-framework-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-cast/10.0.0/play-services-cast-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-clearcut/10.0.0/play-services-clearcut-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-basement/10.0.0/play-services-basement-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-config/10.0.0/firebase-config-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-awareness/10.0.0/play-services-awareness-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-crash/10.0.0/firebase-crash-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-drive/10.0.0/play-services-drive-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-auth/10.0.0/firebase-auth-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-database-connection/10.0.0/firebase-database-connection-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-database/10.0.0/firebase-database-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-iid/10.0.0/firebase-iid-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-messaging/10.0.0/firebase-messaging-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-storage/10.0.0/firebase-storage-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-storage-common/10.0.0/firebase-storage-common-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-common/10.0.0/firebase-common-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-fitness/10.0.0/play-services-fitness-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-games/10.0.0/play-services-games-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-gass/10.0.0/play-services-gass-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-gcm/10.0.0/play-services-gcm-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-appindexing/10.0.0/firebase-appindexing-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-identity/10.0.0/play-services-identity-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-iid/10.0.0/play-services-iid-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-instantapps/10.0.0/play-services-instantapps-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-location/10.0.0/play-services-location-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-maps/10.0.0/play-services-maps-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-analytics/10.0.0/firebase-analytics-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-analytics-impl/10.0.0/firebase-analytics-impl-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-nearby/10.0.0/play-services-nearby-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-panorama/10.0.0/play-services-panorama-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-places/10.0.0/play-services-places-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-plus/10.0.0/play-services-plus-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-safetynet/10.0.0/play-services-safetynet-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-tagmanager-api/10.0.0/play-services-tagmanager-api-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-tagmanager/10.0.0/play-services-tagmanager-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-tagmanager-v4-impl/10.0.0/play-services-tagmanager-v4-impl-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-tagmanager/10.0.0/play-services-tagmanager-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-tasks/10.0.0/play-services-tasks-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-vision/10.0.0/play-services-vision-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-wallet/10.0.0/play-services-wallet-10.0.0.pom Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services-wearable/10.0.0/play-services-wearable-10.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/mediarouter-v7/24.0.0/mediarouter-v7-24.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/support-v4/24.0.0/support-v4-24.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/palette-v7/24.0.0/palette-v7-24.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/appcompat-v7/24.0.0/appcompat-v7-24.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/support-annotations/24.0.0/support-annotations-24.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/support-vector-drawable/24.0.0/support-vector-drawable-24.0.0.pom -Fetching response from: https://dl.google.com/dl/android/maven2/com/android/support/animated-vector-drawable/24.0.0/animated-vector-drawable-24.0.0.pom -Scanned /testdata/maven-transitive/registry.xml file as a pom.xml and found 59 packages +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/android/gms/play-services/10.0.0/play-services-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-analytics-impl/10.0.0/firebase-analytics-impl-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-analytics/10.0.0/firebase-analytics-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-appindexing/10.0.0/firebase-appindexing-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-auth/10.0.0/firebase-auth-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-common/10.0.0/firebase-common-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-config/10.0.0/firebase-config-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-crash/10.0.0/firebase-crash-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-database-connection/10.0.0/firebase-database-connection-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-database/10.0.0/firebase-database-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-iid/10.0.0/firebase-iid-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-messaging/10.0.0/firebase-messaging-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-storage-common/10.0.0/firebase-storage-common-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/com/google/firebase/firebase-storage/10.0.0/firebase-storage-10.0.0.pom +Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/apache/23/apache-23.pom +Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/log4j/log4j-api/2.14.1/log4j-api-2.14.1.pom +Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/log4j/log4j-core/2.14.1/log4j-core-2.14.1.pom +Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/log4j/log4j-web/2.14.1/log4j-web-2.14.1.pom +Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/log4j/log4j/2.14.1/log4j-2.14.1.pom +Fetching response from: https://dl.google.com/dl/android/maven2/org/apache/logging/logging-parent/3/logging-parent-3.pom +Fetching response from: https://repo.maven.apache.org/maven2/org/apache/apache/23/apache-23.pom +Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/log4j/log4j-api/2.14.1/log4j-api-2.14.1.pom +Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/log4j/log4j-core/2.14.1/log4j-core-2.14.1.pom +Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/log4j/log4j-web/2.14.1/log4j-web-2.14.1.pom +Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/log4j/log4j/2.14.1/log4j-2.14.1.pom +Fetching response from: https://repo.maven.apache.org/maven2/org/apache/logging/logging-parent/3/logging-parent-3.pom +Scanned /testdata/maven-transitive/registry.xml file and found 59 packages Total 2 packages affected by 5 known vulnerabilities (2 Critical, 1 High, 2 Medium, 0 Low, 0 Unknown) from 1 ecosystem. 5 vulnerabilities can be fixed. @@ -4954,7 +4957,7 @@ Total 3 packages affected by 9 known vulnerabilities (0 Critical, 3 High, 6 Medi --- [TestCommand_Transitive/scans_dependencies_from_multiple_registries - 1] -Scanned /testdata/maven-transitive/registry.xml file as a pom.xml and found 59 packages +Scanned /testdata/maven-transitive/registry.xml file and found 59 packages Total 2 packages affected by 5 known vulnerabilities (2 Critical, 1 High, 2 Medium, 0 Low, 0 Unknown) from 1 ecosystem. 5 vulnerabilities can be fixed. @@ -4976,7 +4979,7 @@ Total 2 packages affected by 5 known vulnerabilities (2 Critical, 1 High, 2 Medi --- [TestCommand_Transitive/scans_pom.xml_with_non_UTF-8_encoding - 1] -Scanned /testdata/maven-transitive/encoding.xml file as a pom.xml and found 2 packages +Scanned /testdata/maven-transitive/encoding.xml file and found 2 packages Total 1 package affected by 1 known vulnerability (0 Critical, 0 High, 1 Medium, 0 Low, 0 Unknown) from 1 ecosystem. 1 vulnerability can be fixed. @@ -4994,7 +4997,7 @@ Total 1 package affected by 1 known vulnerability (0 Critical, 0 High, 1 Medium, --- [TestCommand_Transitive/scans_transitive_dependencies_by_specifying_pom.xml - 1] -Scanned /testdata/maven-transitive/abc.xml file as a pom.xml and found 3 packages +Scanned /testdata/maven-transitive/abc.xml file and found 3 packages Total 1 package affected by 4 known vulnerabilities (2 Critical, 1 High, 1 Medium, 0 Low, 0 Unknown) from 1 ecosystem. 4 vulnerabilities can be fixed. From 7c1f964710457475de25c981e62ad056f4a874fc Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 31 Oct 2025 16:12:21 +1100 Subject: [PATCH 05/22] Add some more comments --- pkg/osvscanner/internal/scanners/lockfile.go | 91 -------------------- pkg/osvscanner/osvscanner.go | 1 + pkg/osvscanner/scan.go | 17 ++-- pkg/osvscanner/stats.go | 3 +- 4 files changed, 10 insertions(+), 102 deletions(-) diff --git a/pkg/osvscanner/internal/scanners/lockfile.go b/pkg/osvscanner/internal/scanners/lockfile.go index aec7aa9ed65..119e0cf1ced 100644 --- a/pkg/osvscanner/internal/scanners/lockfile.go +++ b/pkg/osvscanner/internal/scanners/lockfile.go @@ -2,14 +2,12 @@ package scanners import ( - "context" "fmt" "path/filepath" "runtime" "slices" "strings" - "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scalibr/extractor/filesystem" "github.com/google/osv-scalibr/extractor/filesystem/language/cpp/conanlock" "github.com/google/osv-scalibr/extractor/filesystem/language/dart/pubspec" @@ -37,9 +35,6 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/os/apk" "github.com/google/osv-scalibr/extractor/filesystem/os/dpkg" "github.com/google/osv-scalibr/plugin" - "github.com/google/osv-scanner/v2/internal/cmdlogger" - "github.com/google/osv-scanner/v2/internal/output" - "github.com/google/osv-scanner/v2/internal/scalibrextract" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/java/pomxmlenhanceable" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/osv/osvscannerjson" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/python/requirementsenhancable" @@ -79,92 +74,6 @@ var osvscannerScalibrExtractionMapping = map[string][]string{ // "Package.resolved": {packageresolved.Name}, } -// ScanSingleFile is similar to ScanSingleFileWithMapping, just without supporting the :/path/to/lockfile prefix identifier -func ScanSingleFile(path string, extractorsToUse []plugin.Plugin) ([]*extractor.Package, error) { - invs, err := scalibrextract.ExtractWithExtractors(context.Background(), path, extractorsToUse) - if err != nil { - return nil, err - } - - pkgCount := len(invs) - if pkgCount > 0 { - cmdlogger.Infof( - "Scanned %s file and found %d %s", - path, - pkgCount, - output.Form(pkgCount, "package", "packages"), - ) - } - - return invs, nil -} - -// ScanSingleFileWithMapping will load, identify, and parse the lockfile path passed in, and add the dependencies specified -// within to `query` -func ScanSingleFileWithMapping(scanPath string, pluginsToUse []plugin.Plugin) ([]*extractor.Package, error) { - var err error - var inventories []*extractor.Package - - parseAs, path := ParseLockfilePath(scanPath) - - path, err = filepath.Abs(path) - if err != nil { - cmdlogger.Errorf("Failed to resolved path %q with error: %s", path, err) - return nil, err - } - - // special case for the APK and DPKG parsers because they have a very generic name while - // living at a specific location, so they are not included in the map of parsers - // used by lockfile.Parse to avoid false-positives when scanning projects - switch parseAs { - case "apk-installed": - inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, apk.New(apk.DefaultConfig())) - case "dpkg-status": - inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, dpkg.New(dpkg.DefaultConfig())) - case "osv-scanner": - inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, osvscannerjson.Extractor{}) - case "": // No specific parseAs specified - inventories, err = scalibrextract.ExtractWithExtractors(context.Background(), path, pluginsToUse) - default: // A specific parseAs without a special case is selected - // Find and extract with the extractor of parseAs - if names, ok := osvscannerScalibrExtractionMapping[parseAs]; ok && len(names) > 0 { - i := slices.IndexFunc(pluginsToUse, func(plug plugin.Plugin) bool { - _, ok = plug.(filesystem.Extractor) - - return ok && slices.Contains(names, plug.Name()) - }) - if i < 0 { - return nil, fmt.Errorf("could not determine extractor, requested %s", parseAs) - } - inventories, err = scalibrextract.ExtractWithExtractor(context.Background(), path, pluginsToUse[i].(filesystem.Extractor)) - } else { - return nil, fmt.Errorf("could not determine extractor, requested %s", parseAs) - } - } - - if err != nil { - return nil, err - } - - parsedAsComment := "" - - if parseAs != "" { - parsedAsComment = fmt.Sprintf("as a %s ", parseAs) - } - - pkgCount := len(inventories) - - cmdlogger.Infof( - "Scanned %s file %sand found %d %s", - path, - parsedAsComment, - pkgCount, - output.Form(pkgCount, "package", "packages"), - ) - - return inventories, nil -} - func ParseLockfilePath(scanArg string) (string, string) { if (runtime.GOOS == "windows" && filepath.IsAbs(scanArg)) || !strings.Contains(scanArg, ":") { scanArg = ":" + scanArg diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index be884f7924f..41df5fb5131 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -77,6 +77,7 @@ type ExperimentalScannerActions struct { PluginsDisabled []string PluginsNoDefaults bool + // Currently unused. StatsCollector stats.Collector } diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index 04d87348b5f..75603c4ac33 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -124,7 +124,9 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes // Also build a map of specific plugin overrides that the user specify // map[path]parseAs overrideMap := map[string]filesystem.Extractor{} + // List of specific paths the user passes in so that we can check that they all get processed. var specificPaths []string + statsCollector := fileOpenedPrinter{ filesExtracted: make(map[string]struct{}), } @@ -141,7 +143,6 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes for _, lockfileElem := range actions.LockfilePaths { parseAs, path := scanners.ParseLockfilePath(lockfileElem) - //cmdlogger.Infof("Scanning lockfiles %s", path) if absPath, err := pathToRootMap(rootMap, path, actions.Recursive); err != nil { return nil, err } else { @@ -186,13 +187,6 @@ SBOMLoop: testlogger.BeginDirScanMarker() osCapability := determineOS() - //var statsCollector stats.Collector - //if actions.StatsCollector != nil { - // statsCollector = actions.StatsCollector - //} else { - // statsCollector = fileOpenedPrinter{} - //} - // For each root, run scalibr's scan() once. for root, paths := range rootMap { capabilities := plugin.Capabilities{ @@ -216,7 +210,7 @@ SBOMLoop: SkipDirRegex: nil, SkipDirGlob: nil, UseGitignore: !actions.NoIgnore, - Stats: statsCollector, + Stats: &statsCollector, ReadSymlinks: false, MaxInodes: 0, StoreAbsolutePath: true, @@ -232,9 +226,11 @@ SBOMLoop: }, }) + // --- Check status of the run --- if sr.Status.Status == plugin.ScanStatusFailed { return nil, errors.New(sr.Status.FailureReason) } + for _, status := range sr.PluginStatus { if status.Status.Status != plugin.ScanStatusSucceeded { builder := strings.Builder{} @@ -261,7 +257,8 @@ SBOMLoop: } } - // Check if specific paths have been extracted + // Check if specific paths have been extracted. + // This allows us to error if a specific file provided by the user failed to extract, and return an error for them. for _, path := range specificPaths { key, _ := filepath.Rel(root, path) if _, ok := statsCollector.filesExtracted[key]; !ok { diff --git a/pkg/osvscanner/stats.go b/pkg/osvscanner/stats.go index 1d986d2d1d7..edf7d9ca16d 100644 --- a/pkg/osvscanner/stats.go +++ b/pkg/osvscanner/stats.go @@ -10,12 +10,13 @@ import ( type fileOpenedPrinter struct { stats.NoopCollector + filesExtracted map[string]struct{} } var _ stats.Collector = &fileOpenedPrinter{} -func (c fileOpenedPrinter) AfterExtractorRun(_ string, extractorstats *stats.AfterExtractorStats) { +func (c *fileOpenedPrinter) AfterExtractorRun(_ string, extractorstats *stats.AfterExtractorStats) { c.filesExtracted[extractorstats.Path] = struct{}{} if extractorstats.Error != nil { // Don't log scanned if error occurred return From c071760c0397042b98a8ee3ec73a7aee7cebcd57 Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 31 Oct 2025 16:18:00 +1100 Subject: [PATCH 06/22] Remove more unused functions --- internal/scalibrextract/errors.go | 3 - internal/scalibrextract/extract.go | 161 ----------------------------- internal/scalibrextract/invsort.go | 43 -------- pkg/osvscanner/scan.go | 3 - 4 files changed, 210 deletions(-) delete mode 100644 internal/scalibrextract/extract.go delete mode 100644 internal/scalibrextract/invsort.go diff --git a/internal/scalibrextract/errors.go b/internal/scalibrextract/errors.go index 0a410500cf3..39f51e7053f 100644 --- a/internal/scalibrextract/errors.go +++ b/internal/scalibrextract/errors.go @@ -2,7 +2,4 @@ package scalibrextract import "errors" -var ErrIncompatibleFileFormat = errors.New("file format is incompatible, but this is expected") -var ErrNotImplemented = errors.New("not implemented") -var ErrWrongExtractor = errors.New("this extractor did not create this inventory") var ErrExtractorNotFound = errors.New("could not determine extractor suitable to this file") diff --git a/internal/scalibrextract/extract.go b/internal/scalibrextract/extract.go deleted file mode 100644 index 57d64839ec2..00000000000 --- a/internal/scalibrextract/extract.go +++ /dev/null @@ -1,161 +0,0 @@ -// Package scalibrextract provides functions to easily use scalibr extractors within osv-scanner. -package scalibrextract - -import ( - "context" - "fmt" - "io/fs" - "os" - "path/filepath" - "runtime" - "slices" - "strings" - - "github.com/google/osv-scalibr/extractor" - "github.com/google/osv-scalibr/extractor/filesystem" - "github.com/google/osv-scalibr/extractor/filesystem/simplefileapi" - "github.com/google/osv-scalibr/plugin" - - scalibrfs "github.com/google/osv-scalibr/fs" -) - -// ExtractWithExtractor attempts to extract the file at the given path with the extractor passed in -// -// # Extract attempts to extract the file at the given path -// -// Args: -// - ctx: the context to use for extraction -// - localPath: the path to the lockfile -// - ext: the extractor to use -// -// Returns: -// - []*extractor.Package: the extracted lockfile data -// - error: any errors encountered during extraction -func ExtractWithExtractor(ctx context.Context, localPath string, ext filesystem.Extractor) ([]*extractor.Package, error) { - info, err := os.Stat(localPath) - if err != nil { - return nil, err - } - - return extractWithExtractor(ctx, localPath, info, ext) -} - -// ExtractWithExtractors attempts to extract the file at the given path -// by choosing the extractor which passes the FileRequired test -// TODO: Optimise to pass in FileInfo here. -// TODO: Remove reporter -// -// Args: -// - ctx: the context to use for extraction -// - localPath: the path to the lockfile -// - extractors: a slice of extractors to try -// - r: reporter to output logs to -// -// Returns: -// - []*extractor.Package: the extracted lockfile data -// - error: any errors encountered during extraction -// -// If no extractors are found, then ErrExtractorNotFound is returned. -func ExtractWithExtractors(ctx context.Context, localPath string, plugins []plugin.Plugin) ([]*extractor.Package, error) { - info, err := os.Stat(localPath) - if err != nil { - return nil, err - } - - result := []*extractor.Package{} - extractorFound := false - for _, plug := range plugins { - ext, ok := plug.(filesystem.Extractor) - - if !ok || !ext.FileRequired(simplefileapi.New(localPath, info)) { - continue - } - extractorFound = true - - invs, err := extractWithExtractor(ctx, localPath, info, ext) - if err != nil { - return nil, err - } - - result = append(result, invs...) - } - - if !extractorFound { - return nil, ErrExtractorNotFound - } - - return result, nil -} - -func extractWithExtractor(ctx context.Context, localPath string, info fs.FileInfo, ext filesystem.Extractor) ([]*extractor.Package, error) { - // Create a scan input centered at the system root directory, - // to give access to the full filesystem for each extractor. - rootDir := getRootDir(localPath) - si, err := createScanInput(localPath, rootDir, info) - if err != nil { - return nil, err - } - - invs, err := ext.Extract(ctx, si) - if err != nil { - return nil, fmt.Errorf("(extracting as %s) %w", ext.Name(), err) - } - - for i := range invs.Packages { - // Set parent extractor - invs.Packages[i].Plugins = append(invs.Packages[i].Plugins, ext.Name()) - - // Make Location relative to the scan root as we are performing local scanning - for i2 := range invs.Packages[i].Locations { - invs.Packages[i].Locations[i2] = filepath.Join(rootDir, invs.Packages[i].Locations[i2]) - } - } - - slices.SortFunc(invs.Packages, inventorySort) - invsCompact := slices.CompactFunc(invs.Packages, func(a, b *extractor.Package) bool { - return inventorySort(a, b) == 0 - }) - - return invsCompact, nil -} - -func createScanInput(path string, root string, fileInfo fs.FileInfo) (*filesystem.ScanInput, error) { - reader, err := os.Open(path) - if err != nil { - return nil, err - } - - // Rel will strip root from the input path. - path, err = filepath.Rel(root, path) - if err != nil { - return nil, err - } - - // Convert path to slashes, as go FS expects unix like paths - path = filepath.ToSlash(path) - - si := filesystem.ScanInput{ - FS: os.DirFS(root).(scalibrfs.FS), - Path: path, - Root: root, - Reader: reader, - Info: fileInfo, - } - - return &si, nil -} - -// getRootDir returns the root directory on each system. -// On Unix systems, it'll be / -// On Windows, it will most likely be the drive (e.g. C:\) -func getRootDir(path string) string { - if runtime.GOOS == "windows" { - return filepath.VolumeName(path) + "\\" - } - - if strings.HasPrefix(path, "/") { - return "/" - } - - return "" -} diff --git a/internal/scalibrextract/invsort.go b/internal/scalibrextract/invsort.go deleted file mode 100644 index 8b9eea03ffd..00000000000 --- a/internal/scalibrextract/invsort.go +++ /dev/null @@ -1,43 +0,0 @@ -package scalibrextract - -import ( - "cmp" - "fmt" - - "github.com/google/osv-scalibr/converter" - "github.com/google/osv-scalibr/extractor" -) - -// InventorySort is a comparator function for Inventories, to be used in -// tests with cmp.Diff to disregard the order in which the Inventories -// are reported. -func inventorySort(a, b *extractor.Package) int { - aLoc := fmt.Sprintf("%v", a.Locations) - bLoc := fmt.Sprintf("%v", b.Locations) - - var aExtr, bExtr string - var aPURL, bPURL string - - aPURLStruct := converter.ToPURL(a) - bPURLStruct := converter.ToPURL(b) - - if aPURLStruct != nil { - aPURL = aPURLStruct.String() - } - - if bPURLStruct != nil { - bPURL = bPURLStruct.String() - } - - aSourceCode := fmt.Sprintf("%v", a.SourceCode) - bSourceCode := fmt.Sprintf("%v", b.SourceCode) - - return cmp.Or( - cmp.Compare(aLoc, bLoc), - cmp.Compare(a.Name, b.Name), - cmp.Compare(a.Version, b.Version), - cmp.Compare(aSourceCode, bSourceCode), - cmp.Compare(aExtr, bExtr), - cmp.Compare(aPURL, bPURL), - ) -} diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index 75603c4ac33..e221e555a8c 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -112,9 +112,6 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes plugins = append(plugins, java.NewDefault()) } - // --- Lockfiles --- - //lockfilePlugins := omitDirExtractors(plugins) - scanner := scalibr.New() // Build list of paths for each root From 01699921efc5675f9d45bf7a7b930d89725a006d Mon Sep 17 00:00:00 2001 From: Rex P Date: Fri, 31 Oct 2025 16:26:37 +1100 Subject: [PATCH 07/22] Fix lints --- internal/scalibrextract/errors.go | 5 -- internal/utility/types/cast.go | 1 + pkg/osvscanner/internal/scanners/lockfile.go | 8 ++- pkg/osvscanner/scan.go | 75 ++++++++------------ pkg/osvscanner/scan_test.go | 4 +- 5 files changed, 40 insertions(+), 53 deletions(-) delete mode 100644 internal/scalibrextract/errors.go diff --git a/internal/scalibrextract/errors.go b/internal/scalibrextract/errors.go deleted file mode 100644 index 39f51e7053f..00000000000 --- a/internal/scalibrextract/errors.go +++ /dev/null @@ -1,5 +0,0 @@ -package scalibrextract - -import "errors" - -var ErrExtractorNotFound = errors.New("could not determine extractor suitable to this file") diff --git a/internal/utility/types/cast.go b/internal/utility/types/cast.go index 1960c23041d..7fa8a974792 100644 --- a/internal/utility/types/cast.go +++ b/internal/utility/types/cast.go @@ -1,3 +1,4 @@ +// Package types provides type conversion utility functions package types func MustCastSlice[OUT, IN any](a []IN) []OUT { diff --git a/pkg/osvscanner/internal/scanners/lockfile.go b/pkg/osvscanner/internal/scanners/lockfile.go index 119e0cf1ced..6ebd997edd1 100644 --- a/pkg/osvscanner/internal/scanners/lockfile.go +++ b/pkg/osvscanner/internal/scanners/lockfile.go @@ -2,6 +2,7 @@ package scanners import ( + "errors" "fmt" "path/filepath" "runtime" @@ -84,10 +85,11 @@ func ParseLockfilePath(scanArg string) (string, string) { return splits[0], splits[1] } +// ParseAsToPlugin finds the parseAs extractor in the list of pluginsToUse func ParseAsToPlugin(parseAs string, pluginsToUse []plugin.Plugin) (filesystem.Extractor, error) { switch parseAs { case "": // No specific parseAs specified - return nil, nil + return nil, errors.New("no parseAs specified") case "osv-scanner": return osvscannerjson.Extractor{}, nil default: @@ -108,8 +110,8 @@ func ParseAsToPlugin(parseAs string, pluginsToUse []plugin.Plugin) (filesystem.E } return fsysExtractor, nil - } else { - return nil, fmt.Errorf("could not determine extractor, requested %s", parseAs) } + + return nil, fmt.Errorf("could not determine extractor, requested %s", parseAs) } } diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index e221e555a8c..b49d82f63d7 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -23,7 +23,6 @@ import ( "github.com/google/osv-scalibr/plugin" "github.com/google/osv-scanner/v2/internal/cmdlogger" "github.com/google/osv-scanner/v2/internal/imodels" - "github.com/google/osv-scanner/v2/internal/scalibrextract" "github.com/google/osv-scanner/v2/internal/scalibrextract/filesystem/vendored" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/java/pomxmlenhanceable" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/python/requirementsenhancable" @@ -34,6 +33,8 @@ import ( "github.com/ossf/osv-schema/bindings/go/osvschema" ) +var ErrExtractorNotFound = errors.New("could not determine extractor suitable to this file") + func configurePlugins(plugins []plugin.Plugin, accessors ExternalAccessors, actions ScannerActions) { for _, plug := range plugins { if accessors.DependencyClients[osvschema.EcosystemMaven] != nil && accessors.MavenRegistryAPIClient != nil { @@ -77,21 +78,6 @@ func getPlugins(defaultPlugins []string, accessors ExternalAccessors, actions Sc return plugins } -// omitDirExtractors removes any plugins that require extracting from a directory -func omitDirExtractors(plugins []plugin.Plugin) []plugin.Plugin { - filtered := make([]plugin.Plugin, 0, len(plugins)) - - for _, plug := range plugins { - if plug.Requirements().ExtractFromDirs { - continue - } - - filtered = append(filtered, plug) - } - - return filtered -} - // scan essentially converts ScannerActions into imodels.ScanResult by performing the extractions func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanResult, error) { //nolint:prealloc // We don't know how many inventories we will retrieve @@ -122,6 +108,7 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes // map[path]parseAs overrideMap := map[string]filesystem.Extractor{} // List of specific paths the user passes in so that we can check that they all get processed. + //nolint:prealloc // Does not matter in this case var specificPaths []string statsCollector := fileOpenedPrinter{ @@ -139,19 +126,19 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes // --- Lockfiles --- for _, lockfileElem := range actions.LockfilePaths { parseAs, path := scanners.ParseLockfilePath(lockfileElem) - - if absPath, err := pathToRootMap(rootMap, path, actions.Recursive); err != nil { + absPath, err := pathToRootMap(rootMap, path, actions.Recursive) + if err != nil { return nil, err - } else { - specificPaths = append(specificPaths, absPath) + } - if parseAs != "" { - plug, err := scanners.ParseAsToPlugin(parseAs, plugins) - if err != nil { - return nil, err - } - overrideMap[absPath] = plug + specificPaths = append(specificPaths, absPath) + + if parseAs != "" { + plug, err := scanners.ParseAsToPlugin(parseAs, plugins) + if err != nil { + return nil, err } + overrideMap[absPath] = plug } } @@ -161,24 +148,24 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes SBOMLoop: for _, sbomPath := range actions.SBOMPaths { - if absPath, err := pathToRootMap(rootMap, sbomPath, actions.Recursive); err != nil { + absPath, err := pathToRootMap(rootMap, sbomPath, actions.Recursive) + if err != nil { return nil, err - } else { - specificPaths = append(specificPaths, absPath) - - for _, se := range sbomExtractors { - // All sbom extractors are filesystem extractors - sbomExtractor := se.(filesystem.Extractor) - if sbomExtractor.FileRequired(simplefileapi.New(absPath, nil)) { - overrideMap[absPath] = sbomExtractor - continue SBOMLoop - } + } + specificPaths = append(specificPaths, absPath) + + for _, se := range sbomExtractors { + // All sbom extractors are filesystem extractors + sbomExtractor := se.(filesystem.Extractor) + if sbomExtractor.FileRequired(simplefileapi.New(absPath, nil)) { + overrideMap[absPath] = sbomExtractor + continue SBOMLoop } - cmdlogger.Errorf("Failed to parse SBOM %q: Invalid SBOM filename.", sbomPath) - cmdlogger.Errorf("If you believe this is a valid SBOM, make sure the filename follows format per your SBOMs specification.") - - return nil, fmt.Errorf("invalid SBOM filename: %s", sbomPath) } + cmdlogger.Errorf("Failed to parse SBOM %q: Invalid SBOM filename.", sbomPath) + cmdlogger.Errorf("If you believe this is a valid SBOM, make sure the filename follows format per your SBOMs specification.") + + return nil, fmt.Errorf("invalid SBOM filename: %s", sbomPath) } testlogger.BeginDirScanMarker() @@ -217,9 +204,9 @@ SBOMLoop: ext, ok := overrideMap[filepath.Join(root, api.Path())] if ok { return []filesystem.Extractor{ext} - } else { - return []filesystem.Extractor{} } + + return []filesystem.Extractor{} }, }) @@ -259,7 +246,7 @@ SBOMLoop: for _, path := range specificPaths { key, _ := filepath.Rel(root, path) if _, ok := statsCollector.filesExtracted[key]; !ok { - return nil, fmt.Errorf("%w: %q", scalibrextract.ErrExtractorNotFound, path) + return nil, fmt.Errorf("%w: %q", ErrExtractorNotFound, path) } } diff --git a/pkg/osvscanner/scan_test.go b/pkg/osvscanner/scan_test.go index ac55d390a6a..3a97b3696ee 100644 --- a/pkg/osvscanner/scan_test.go +++ b/pkg/osvscanner/scan_test.go @@ -5,8 +5,9 @@ import ( "testing" ) - func Test_isDescendent(t *testing.T) { + t.Parallel() + tests := []struct { name string potentialParent string @@ -87,6 +88,7 @@ func Test_isDescendent(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() // Normalize paths for the current OS potentialParent := filepath.FromSlash(tt.potentialParent) path := filepath.FromSlash(tt.path) From 6a02a5a62083f38a46162109e69b90d37166a39f Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 14:03:23 +1100 Subject: [PATCH 08/22] Make scanning git commits directly also work --- .../vcs/gitcommitdirect/extractor.go | 55 +++++++++++++++++++ pkg/osvscanner/osvscanner_test.go | 1 + pkg/osvscanner/scan.go | 15 ++--- 3 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 internal/scalibrextract/vcs/gitcommitdirect/extractor.go diff --git a/internal/scalibrextract/vcs/gitcommitdirect/extractor.go b/internal/scalibrextract/vcs/gitcommitdirect/extractor.go new file mode 100644 index 00000000000..b817122d707 --- /dev/null +++ b/internal/scalibrextract/vcs/gitcommitdirect/extractor.go @@ -0,0 +1,55 @@ +// Package gitcommitdirect provides an dummy extractor that returns a preset list of commits +package gitcommitdirect + +import ( + "context" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/standalone" + "github.com/google/osv-scalibr/inventory" + "github.com/google/osv-scalibr/plugin" +) + +const ( + // Name is the unique name of this extractor. + Name = "vcs/gitcommitdirect" +) + +// Extractor extracts git repository hashes including submodule hashes. +// This extractor will not return an error, and will just return no results if we fail to extract +type Extractor struct { + commits []string +} + +// New returns a new instance of the extractor. +func New(commits []string) standalone.Extractor { + return &Extractor{ + commits: commits, + } +} + +// Name of the extractor. +func (e *Extractor) Name() string { return Name } + +// Version of the extractor. +func (e *Extractor) Version() int { return 0 } + +// Requirements of the extractor. +func (e *Extractor) Requirements() *plugin.Capabilities { + return &plugin.Capabilities{} +} + +func (e *Extractor) Extract(ctx context.Context, input *standalone.ScanInput) (inventory.Inventory, error) { + pkgs := make([]*extractor.Package, 0, len(e.commits)) + for _, commit := range e.commits { + pkgs = append(pkgs, &extractor.Package{ + SourceCode: &extractor.SourceCodeIdentifier{Commit: commit}, + }) + } + + return inventory.Inventory{ + Packages: pkgs, + }, nil +} + +var _ standalone.Extractor = &Extractor{} diff --git a/pkg/osvscanner/osvscanner_test.go b/pkg/osvscanner/osvscanner_test.go index 80786125921..de270c5c2c0 100644 --- a/pkg/osvscanner/osvscanner_test.go +++ b/pkg/osvscanner/osvscanner_test.go @@ -52,3 +52,4 @@ func TestDoScan_LogHandlerOverride(t *testing.T) { t.Errorf("altOutput.Len() = %d, want %d", altOutput.Len(), 0) } } + diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index b49d82f63d7..2be05c4f9e2 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -26,6 +26,7 @@ import ( "github.com/google/osv-scanner/v2/internal/scalibrextract/filesystem/vendored" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/java/pomxmlenhanceable" "github.com/google/osv-scanner/v2/internal/scalibrextract/language/python/requirementsenhancable" + "github.com/google/osv-scanner/v2/internal/scalibrextract/vcs/gitcommitdirect" "github.com/google/osv-scanner/v2/internal/scalibrextract/vcs/gitrepo" "github.com/google/osv-scanner/v2/internal/scalibrplugin" "github.com/google/osv-scanner/v2/internal/testlogger" @@ -168,6 +169,9 @@ SBOMLoop: return nil, fmt.Errorf("invalid SBOM filename: %s", sbomPath) } + // --- Add git commits directly --- + gitDirectPlugin := gitcommitdirect.New(actions.GitCommits) + testlogger.BeginDirScanMarker() osCapability := determineOS() @@ -185,7 +189,7 @@ SBOMLoop: } sr := scanner.Scan(context.Background(), &scalibr.ScanConfig{ - Plugins: plugin.FilterByCapabilities(plugins, &capabilities), + Plugins: append(plugin.FilterByCapabilities(plugins, &capabilities), gitDirectPlugin), Capabilities: &capabilities, ScanRoots: fs.RealFSScanRoots(root), PathsToExtract: paths, @@ -262,15 +266,6 @@ SBOMLoop: testlogger.EndDirScanMarker() - // Add on additional direct dependencies passed straight from ScannerActions: - for _, commit := range actions.GitCommits { - inv := &extractor.Package{ - SourceCode: &extractor.SourceCodeIdentifier{Commit: commit}, - } - - scannedInventories = append(scannedInventories, inv) - } - if len(scannedInventories) == 0 { return nil, ErrNoPackagesFound } From ce25d12450e98676dc5e84499b02031dc6fbff36 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 14:16:03 +1100 Subject: [PATCH 09/22] Test git commit scan --- .../__snapshots__/osvscanner_test.snap | 101 ++++++++++++++++++ pkg/osvscanner/osvscanner_test.go | 38 +++++++ pkg/osvscanner/scan.go | 7 ++ 3 files changed, 146 insertions(+) create mode 100755 pkg/osvscanner/__snapshots__/osvscanner_test.snap diff --git a/pkg/osvscanner/__snapshots__/osvscanner_test.snap b/pkg/osvscanner/__snapshots__/osvscanner_test.snap new file mode 100755 index 00000000000..366797802aa --- /dev/null +++ b/pkg/osvscanner/__snapshots__/osvscanner_test.snap @@ -0,0 +1,101 @@ + +[TestDoScan/Test_curl_git_scanning - 1] +{ + "results": [ + { + "source": { + "path": "", + "type": "unknown" + }, + "packages": [ + { + "package": { + "name": "", + "version": "", + "ecosystem": "", + "commit": "33dffa3909a67e1b5d22647128ab7eb6e53fd0c7" + }, + "vulnerabilities": [ + { + "modified": "2025-11-02T06:08:01Z", + "published": "2025-10-31T07:15:38Z", + "schema_version": "1.7.3", + "id": "CVE-2025-63675", + "aliases": [ + "GHSA-97w9-v595-3h5q" + ], + "details": "cryptidy through 1.2.4 allows code execution via untrusted data because pickle.loads is used. This occurs in aes_decrypt_message in symmetric_encryption.py.", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:L/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:L" + } + ], + "affected": [ + { + "ranges": [ + { + "type": "GIT", + "events": [ + { + "introduced": "0" + }, + { + "last_affected": "33dffa3909a67e1b5d22647128ab7eb6e53fd0c7" + } + ], + "repo": "https://github.com/netinvent/cryptidy" + } + ], + "versions": [ + "v1.0.4", + "v1.0.5", + "v1.0.7", + "v1.1.0", + "v1.2.0", + "v1.2.1", + "v1.2.2", + "v1.2.3", + "v1.2.4" + ], + "database_specific": { + "source": "https://storage.googleapis.com/cve-osv-conversion/osv-output/CVE-2025-63675.json" + } + } + ], + "references": [ + { + "type": "PACKAGE", + "url": "https://github.com/javiermorales36/cryptidy-analysis" + }, + { + "type": "WEB", + "url": "https://github.com/netinvent/cryptidy/blob/cebc9ffd54cc20679d15a1a43ca9a5da645b0c58/cryptidy/symmetric_encryption.py#L220-L238" + } + ] + } + ], + "groups": [ + { + "ids": [ + "CVE-2025-63675" + ], + "aliases": [ + "CVE-2025-63675", + "GHSA-97w9-v595-3h5q" + ], + "max_severity": "6.9" + } + ] + } + ] + } + ], + "experimental_config": { + "licenses": { + "summary": false, + "allowlist": null + } + } +} +--- diff --git a/pkg/osvscanner/osvscanner_test.go b/pkg/osvscanner/osvscanner_test.go index de270c5c2c0..6c199fabf37 100644 --- a/pkg/osvscanner/osvscanner_test.go +++ b/pkg/osvscanner/osvscanner_test.go @@ -2,9 +2,12 @@ package osvscanner_test import ( "bytes" + "errors" "log/slog" "testing" + "github.com/google/osv-scanner/v2/internal/testutility" + "github.com/google/osv-scanner/v2/pkg/models" "github.com/google/osv-scanner/v2/pkg/osvscanner" ) @@ -53,3 +56,38 @@ func TestDoScan_LogHandlerOverride(t *testing.T) { } } +func TestDoScan(t *testing.T) { + type args struct { + actions osvscanner.ScannerActions + } + tests := []struct { + name string + args args + want models.VulnerabilityResults + wantErr error + }{ + { + name: "Test curl git scanning", + args: args{ + actions: osvscanner.ScannerActions{ + GitCommits: []string{"33dffa3909a67e1b5d22647128ab7eb6e53fd0c7"}, + }, + }, + want: models.VulnerabilityResults{}, + wantErr: osvscanner.ErrVulnerabilitiesFound, + }, + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := osvscanner.DoScan(tt.args.actions) + if !errors.Is(err, tt.wantErr) { + t.Errorf("DoScan() error = %v, wantErr %v", err, tt.wantErr) + return + } + + snap := testutility.NewSnapshot() + snap.MatchJSON(t, got) + }) + } +} diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index 2be05c4f9e2..c3bfad9327c 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -172,6 +172,13 @@ SBOMLoop: // --- Add git commits directly --- gitDirectPlugin := gitcommitdirect.New(actions.GitCommits) + if len(rootMap) == 0 && len(actions.GitCommits) > 0 { + // Even if there's no actual paths, if we have git commits, still do the scan + rootMap = map[string][]string{ + "/": {}, + } + } + testlogger.BeginDirScanMarker() osCapability := determineOS() From 54f941ea1c5e55c517c914bc4c5c61bd258bb6a3 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 14:17:09 +1100 Subject: [PATCH 10/22] Address PR comments --- pkg/osvscanner/osvscanner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index 41df5fb5131..f0694212046 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -78,6 +78,7 @@ type ExperimentalScannerActions struct { PluginsNoDefaults bool // Currently unused. + // TODO(another-rex): Use or wrap this StatsCollector stats.Collector } From 6dd52760026465117aec7d8289832acdbc040f78 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 14:17:53 +1100 Subject: [PATCH 11/22] Fix lints --- internal/scalibrextract/vcs/gitcommitdirect/extractor.go | 2 +- pkg/osvscanner/osvscanner_test.go | 3 +++ pkg/osvscanner/scan.go | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/scalibrextract/vcs/gitcommitdirect/extractor.go b/internal/scalibrextract/vcs/gitcommitdirect/extractor.go index b817122d707..d311150149c 100644 --- a/internal/scalibrextract/vcs/gitcommitdirect/extractor.go +++ b/internal/scalibrextract/vcs/gitcommitdirect/extractor.go @@ -39,7 +39,7 @@ func (e *Extractor) Requirements() *plugin.Capabilities { return &plugin.Capabilities{} } -func (e *Extractor) Extract(ctx context.Context, input *standalone.ScanInput) (inventory.Inventory, error) { +func (e *Extractor) Extract(_ context.Context, _ *standalone.ScanInput) (inventory.Inventory, error) { pkgs := make([]*extractor.Package, 0, len(e.commits)) for _, commit := range e.commits { pkgs = append(pkgs, &extractor.Package{ diff --git a/pkg/osvscanner/osvscanner_test.go b/pkg/osvscanner/osvscanner_test.go index 6c199fabf37..269925ccc5a 100644 --- a/pkg/osvscanner/osvscanner_test.go +++ b/pkg/osvscanner/osvscanner_test.go @@ -57,6 +57,8 @@ func TestDoScan_LogHandlerOverride(t *testing.T) { } func TestDoScan(t *testing.T) { + t.Parallel() + type args struct { actions osvscanner.ScannerActions } @@ -80,6 +82,7 @@ func TestDoScan(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() got, err := osvscanner.DoScan(tt.args.actions) if !errors.Is(err, tt.wantErr) { t.Errorf("DoScan() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index c3bfad9327c..962efc5ff18 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -81,7 +81,6 @@ func getPlugins(defaultPlugins []string, accessors ExternalAccessors, actions Sc // scan essentially converts ScannerActions into imodels.ScanResult by performing the extractions func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanResult, error) { - //nolint:prealloc // We don't know how many inventories we will retrieve var scannedInventories []*extractor.Package var genericFindings []*inventory.GenericFinding From c89d94373a0c78aed75997f079986b5fc83fb514 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 14:19:36 +1100 Subject: [PATCH 12/22] Add function comment --- pkg/osvscanner/scan.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index 962efc5ff18..05e04665490 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -323,6 +323,9 @@ func pathToRootMap(rootMap map[string][]string, path string, recursive bool) (st return absPath, nil } +// isDescendent returns whether `path` is either a descendent or a direct child of `potentialParent` +// recursive = true: checks for descendents +// recursive = false: checks for direct children func isDescendent(potentialParent, path string, recursive bool) bool { rel, err := filepath.Rel(potentialParent, path) if err != nil { From cce5024e9289d8247782d2a1a73e025abaaca00b Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 14:24:42 +1100 Subject: [PATCH 13/22] Fix windows path override issue? --- pkg/osvscanner/scan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index 05e04665490..245efbfcffc 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -211,7 +211,7 @@ SBOMLoop: PrintDurationAnalysis: false, ErrorOnFSErrors: false, ExtractorOverride: func(api filesystem.FileAPI) []filesystem.Extractor { - ext, ok := overrideMap[filepath.Join(root, api.Path())] + ext, ok := overrideMap[filepath.Join(root, filepath.FromSlash(api.Path()))] if ok { return []filesystem.Extractor{ext} } From dffb3f70cc86e037be588f1475f8021143e2b489 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 14:39:43 +1100 Subject: [PATCH 14/22] Address PR comments --- pkg/osvscanner/internal/scanners/lockfile.go | 13 +++++++++---- pkg/osvscanner/scan.go | 3 +-- pkg/osvscanner/stats.go | 4 ++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/osvscanner/internal/scanners/lockfile.go b/pkg/osvscanner/internal/scanners/lockfile.go index 6ebd997edd1..a3bff8bde5e 100644 --- a/pkg/osvscanner/internal/scanners/lockfile.go +++ b/pkg/osvscanner/internal/scanners/lockfile.go @@ -75,14 +75,19 @@ var osvscannerScalibrExtractionMapping = map[string][]string{ // "Package.resolved": {packageresolved.Name}, } +// ParseLockfilePath returns (parseAs, path) func ParseLockfilePath(scanArg string) (string, string) { - if (runtime.GOOS == "windows" && filepath.IsAbs(scanArg)) || !strings.Contains(scanArg, ":") { - scanArg = ":" + scanArg + if runtime.GOOS == "windows" && filepath.IsAbs(scanArg) { + return "", scanArg } - splits := strings.SplitN(scanArg, ":", 2) + parseAs, path, found := strings.Cut(scanArg, ":") +if !found { + path = parseAs + parseAs = "" + } - return splits[0], splits[1] + return parseAs, path } // ParseAsToPlugin finds the parseAs extractor in the list of pluginsToUse diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index 245efbfcffc..bd89e8a412a 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -108,8 +108,7 @@ func scan(accessors ExternalAccessors, actions ScannerActions) (*imodels.ScanRes // map[path]parseAs overrideMap := map[string]filesystem.Extractor{} // List of specific paths the user passes in so that we can check that they all get processed. - //nolint:prealloc // Does not matter in this case - var specificPaths []string + specificPaths := make([]string, 0, len(actions.LockfilePaths)+len(actions.SBOMPaths)) statsCollector := fileOpenedPrinter{ filesExtracted: make(map[string]struct{}), diff --git a/pkg/osvscanner/stats.go b/pkg/osvscanner/stats.go index edf7d9ca16d..d22171ae863 100644 --- a/pkg/osvscanner/stats.go +++ b/pkg/osvscanner/stats.go @@ -17,6 +17,10 @@ type fileOpenedPrinter struct { var _ stats.Collector = &fileOpenedPrinter{} func (c *fileOpenedPrinter) AfterExtractorRun(_ string, extractorstats *stats.AfterExtractorStats) { + if c.filesExtracted != nil { + c.filesExtracted = map[string]struct{}{} + } + c.filesExtracted[extractorstats.Path] = struct{}{} if extractorstats.Error != nil { // Don't log scanned if error occurred return From d6eacdad639679a3a595186d34fbf328ba8b6f10 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 14:59:41 +1100 Subject: [PATCH 15/22] Fix windows --- pkg/osvscanner/scan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index bd89e8a412a..e94ecb6dfc4 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -254,7 +254,7 @@ SBOMLoop: // This allows us to error if a specific file provided by the user failed to extract, and return an error for them. for _, path := range specificPaths { key, _ := filepath.Rel(root, path) - if _, ok := statsCollector.filesExtracted[key]; !ok { + if _, ok := statsCollector.filesExtracted[filepath.ToSlash(key)]; !ok { return nil, fmt.Errorf("%w: %q", ErrExtractorNotFound, path) } } From b7ddd73f18b0daaf257e7d77991f5a88dd20197c Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 15:06:51 +1100 Subject: [PATCH 16/22] Fix for macos --- internal/testlogger/handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/testlogger/handler.go b/internal/testlogger/handler.go index da3e2eb36bb..d54878ed86a 100644 --- a/internal/testlogger/handler.go +++ b/internal/testlogger/handler.go @@ -82,7 +82,9 @@ func (tl *Handler) Handle(ctx context.Context, record slog.Record) error { "Neither CPE nor PURL found for package", "Invalid PURL", "os-release[ID] not set, fallback to", + // TODO(another-rex): We should allow overriding of these values to avoid this issue. "VERSION_ID not set in os-release", + "VERSION_CODENAME and VERSION_ID not set in os-release", "osrelease.ParseOsRelease(): file does not exist", "Status: new inodes:", "Created image content file:", From 42af31d2e13054810e72dcf1a2b3f533c242b4ba Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 15:07:30 +1100 Subject: [PATCH 17/22] Fix formatting --- pkg/osvscanner/internal/scanners/lockfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osvscanner/internal/scanners/lockfile.go b/pkg/osvscanner/internal/scanners/lockfile.go index a3bff8bde5e..d8b440949d2 100644 --- a/pkg/osvscanner/internal/scanners/lockfile.go +++ b/pkg/osvscanner/internal/scanners/lockfile.go @@ -82,7 +82,7 @@ func ParseLockfilePath(scanArg string) (string, string) { } parseAs, path, found := strings.Cut(scanArg, ":") -if !found { + if !found { path = parseAs parseAs = "" } From 8669795f62da1f72e754d9938070c44134efd560 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 15:10:50 +1100 Subject: [PATCH 18/22] Make to be more idomatic --- pkg/osvscanner/stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osvscanner/stats.go b/pkg/osvscanner/stats.go index d22171ae863..b803276ea49 100644 --- a/pkg/osvscanner/stats.go +++ b/pkg/osvscanner/stats.go @@ -18,7 +18,7 @@ var _ stats.Collector = &fileOpenedPrinter{} func (c *fileOpenedPrinter) AfterExtractorRun(_ string, extractorstats *stats.AfterExtractorStats) { if c.filesExtracted != nil { - c.filesExtracted = map[string]struct{}{} + c.filesExtracted = make(map[string]struct{}) } c.filesExtracted[extractorstats.Path] = struct{}{} From 67e26eb90ed81d2b4119c0b38e0f5023c0b2828b Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 15:27:51 +1100 Subject: [PATCH 19/22] What was I doing --- pkg/osvscanner/stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/osvscanner/stats.go b/pkg/osvscanner/stats.go index b803276ea49..1226c5cf521 100644 --- a/pkg/osvscanner/stats.go +++ b/pkg/osvscanner/stats.go @@ -17,7 +17,7 @@ type fileOpenedPrinter struct { var _ stats.Collector = &fileOpenedPrinter{} func (c *fileOpenedPrinter) AfterExtractorRun(_ string, extractorstats *stats.AfterExtractorStats) { - if c.filesExtracted != nil { + if c.filesExtracted == nil { c.filesExtracted = make(map[string]struct{}) } From 08694a07ad8847f0d1a6e758ebb7437281622e73 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 3 Nov 2025 15:48:37 +1100 Subject: [PATCH 20/22] Remove cast.go --- internal/utility/types/cast.go | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 internal/utility/types/cast.go diff --git a/internal/utility/types/cast.go b/internal/utility/types/cast.go deleted file mode 100644 index 7fa8a974792..00000000000 --- a/internal/utility/types/cast.go +++ /dev/null @@ -1,11 +0,0 @@ -// Package types provides type conversion utility functions -package types - -func MustCastSlice[OUT, IN any](a []IN) []OUT { - out := make([]OUT, len(a)) - for i := range a { - out[i] = any(a[i]).(OUT) - } - - return out -} From 77cba09eb9bc8c10f6723acaaafa75103d19405e Mon Sep 17 00:00:00 2001 From: Rex P Date: Tue, 4 Nov 2025 11:26:16 +1100 Subject: [PATCH 21/22] Attempt at root based checking --- pkg/osvscanner/scan.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index e94ecb6dfc4..3e587ed1d67 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -236,8 +236,8 @@ SBOMLoop: builder.WriteString(fmt.Sprintf("%s: %s", fileError.FilePath, fileError.ErrorMessage)) // Check if the erroring file was a path specifically passed in (not a result of a file walk) - for _, path := range specificPaths { - if strings.Contains(filepath.ToSlash(path), filepath.ToSlash(fileError.FilePath)) { + for _, p := range specificPaths { + if p == filepath.Join(root, fileError.FilePath) { criticalError = true break } From 99e1d9a21060a349ae5b789d5db671dd9aed6147 Mon Sep 17 00:00:00 2001 From: Rex P Date: Tue, 4 Nov 2025 11:31:09 +1100 Subject: [PATCH 22/22] Fix logic for multi root scanning --- pkg/osvscanner/scan.go | 17 ++++++++--------- pkg/osvscanner/stats.go | 5 +++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index 3e587ed1d67..1d4a9f2fae7 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -250,15 +250,6 @@ SBOMLoop: } } - // Check if specific paths have been extracted. - // This allows us to error if a specific file provided by the user failed to extract, and return an error for them. - for _, path := range specificPaths { - key, _ := filepath.Rel(root, path) - if _, ok := statsCollector.filesExtracted[filepath.ToSlash(key)]; !ok { - return nil, fmt.Errorf("%w: %q", ErrExtractorNotFound, path) - } - } - slices.SortFunc(sr.Inventory.Packages, inventorySort) invsCompact := slices.CompactFunc(sr.Inventory.Packages, func(a, b *extractor.Package) bool { return inventorySort(a, b) == 0 @@ -271,6 +262,14 @@ SBOMLoop: testlogger.EndDirScanMarker() + // Check if specific paths have been extracted. + // This allows us to error if a specific file provided by the user failed to extract, and return an error for them. + for _, path := range specificPaths { + if _, ok := statsCollector.filesExtracted[path]; !ok { + return nil, fmt.Errorf("%w: %q", ErrExtractorNotFound, path) + } + } + if len(scannedInventories) == 0 { return nil, ErrNoPackagesFound } diff --git a/pkg/osvscanner/stats.go b/pkg/osvscanner/stats.go index 1226c5cf521..897c3afeb1d 100644 --- a/pkg/osvscanner/stats.go +++ b/pkg/osvscanner/stats.go @@ -21,7 +21,8 @@ func (c *fileOpenedPrinter) AfterExtractorRun(_ string, extractorstats *stats.Af c.filesExtracted = make(map[string]struct{}) } - c.filesExtracted[extractorstats.Path] = struct{}{} + systemPath := filepath.Join(extractorstats.Root, filepath.FromSlash(extractorstats.Path)) + c.filesExtracted[systemPath] = struct{}{} if extractorstats.Error != nil { // Don't log scanned if error occurred return } @@ -30,7 +31,7 @@ func (c *fileOpenedPrinter) AfterExtractorRun(_ string, extractorstats *stats.Af cmdlogger.Infof( "Scanned %s file and found %d %s", - filepath.Join(extractorstats.Root, extractorstats.Path), + systemPath, pkgsFound, output.Form(pkgsFound, "package", "packages"), )