Skip to content

Commit f224de3

Browse files
DmitriyLewenDmitriyLewen
authored
fix: migrate from *.list to *.md5sums files for dpkg (#9131)
1 parent 2807478 commit f224de3

5 files changed

Lines changed: 58 additions & 82 deletions

File tree

‎integration/testdata/debian-buster-ignore-unfixed.json.golden‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@
7070
"PkgName": "libidn2-0",
7171
"PkgIdentifier": {
7272
"PURL": "pkg:deb/debian/libidn2-0@2.0.5-1?arch=amd64\u0026distro=debian-10.1",
73-
"UID": "473f5eb9e3d4a2f2"
73+
"UID": "24f9b08969c58720"
7474
},
7575
"InstalledVersion": "2.0.5-1",
7676
"FixedVersion": "2.0.5-1+deb10u1",

‎pkg/fanal/analyzer/pkg/dpkg/dpkg.go‎

Lines changed: 29 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ const (
4747
statusDir = "var/lib/dpkg/status.d/"
4848
infoDir = "var/lib/dpkg/info/"
4949
availableFile = "var/lib/dpkg/available"
50+
51+
md5sumsExtension = ".md5sums"
5052
)
5153

5254
var (
@@ -72,14 +74,14 @@ func (a dpkgAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis
7274

7375
// parse other files
7476
err = fsutils.WalkDir(input.FS, ".", required, func(path string, _ fs.DirEntry, r io.Reader) error {
75-
// parse list files
76-
if a.isListFile(filepath.Split(path)) {
77+
// parse *md5sums files
78+
if a.isMd5SumsFile(filepath.Split(path)) {
7779
scanner := bufio.NewScanner(r)
78-
systemFiles, err := a.parseDpkgInfoList(scanner)
80+
systemFiles, err := a.parseDpkgMd5sums(scanner)
7981
if err != nil {
80-
return err
82+
return xerrors.Errorf("failed to parse %s file: %w", path, err)
8183
}
82-
packageFiles[strings.TrimSuffix(filepath.Base(path), ".list")] = systemFiles
84+
packageFiles[strings.TrimSuffix(filepath.Base(path), md5sumsExtension)] = systemFiles
8385
systemInstalledFiles = append(systemInstalledFiles, systemFiles...)
8486
return nil
8587
}
@@ -113,46 +115,31 @@ func (a dpkgAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis
113115

114116
}
115117

116-
// parseDpkgInfoList parses /var/lib/dpkg/info/*.list
117-
func (a dpkgAnalyzer) parseDpkgInfoList(scanner *bufio.Scanner) ([]string, error) {
118-
var (
119-
allLines []string
120-
installedFiles []string
121-
previous string
122-
)
123-
118+
// parseDpkgMd5sums parses `/var/lib/dpkg/*/*.md5sums` file.
119+
//
120+
// `*.md5sums` files don't contain links (see https://github.com/aquasecurity/trivy/pull/9131#discussion_r2182557288).
121+
// But Trivy doesn't support links, so this will not cause problems.
122+
// TODO use `*.list` files instead of `*.md5sums` files when Trivy will support links.
123+
func (a dpkgAnalyzer) parseDpkgMd5sums(scanner *bufio.Scanner) ([]string, error) {
124+
var installedFiles []string
124125
for scanner.Scan() {
125126
current := scanner.Text()
126-
if current == "/." {
127-
continue
127+
128+
// md5sums file use the following format:
129+
// <digest> <filepath> (2 spaces)
130+
// cf. https://man7.org/linux/man-pages/man5/deb-md5sums.5.html
131+
_, file, ok := strings.Cut(current, " ")
132+
if !ok {
133+
return nil, xerrors.Errorf("invalid md5sums line format: %s", current)
128134
}
129-
allLines = append(allLines, current)
135+
installedFiles = append(installedFiles, "/"+file) // md5sums files don't contain leading slash
130136
}
131137

132138
if err := scanner.Err(); err != nil {
133139
return nil, xerrors.Errorf("scan error: %w", err)
134140
}
135141

136-
// Add the file if it is not directory.
137-
// e.g.
138-
// /usr/sbin
139-
// /usr/sbin/tarcat
140-
//
141-
// In the above case, we should take only /usr/sbin/tarcat since /usr/sbin is a directory
142-
// sort first,see here:https://github.com/aquasecurity/trivy/discussions/6543
143-
sort.Strings(allLines)
144-
for _, current := range allLines {
145-
if !strings.HasPrefix(current, previous+"/") {
146-
installedFiles = append(installedFiles, previous)
147-
}
148-
previous = current
149-
}
150-
151-
// // Add the last file
152-
if previous != "" && !strings.HasSuffix(previous, "/") {
153-
installedFiles = append(installedFiles, previous)
154-
}
155-
142+
sort.Strings(installedFiles)
156143
return installedFiles, nil
157144
}
158145

@@ -290,14 +277,10 @@ func (a dpkgAnalyzer) parseDpkgPkg(header textproto.MIMEHeader) *types.Package {
290277

291278
func (a dpkgAnalyzer) Required(filePath string, _ os.FileInfo) bool {
292279
dir, fileName := filepath.Split(filePath)
293-
if a.isListFile(dir, fileName) || filePath == statusFile || filePath == availableFile {
280+
if a.isMd5SumsFile(dir, fileName) || filePath == statusFile || filePath == availableFile || dir == statusDir {
294281
return true
295282
}
296283

297-
// skip `*.md5sums` files from `status.d` directory
298-
if dir == statusDir && filepath.Ext(fileName) != ".md5sums" {
299-
return true
300-
}
301284
return false
302285
}
303286

@@ -356,12 +339,14 @@ func (a dpkgAnalyzer) consolidateDependencies(pkgs map[string]*types.Package, pk
356339
}
357340
}
358341

359-
func (a dpkgAnalyzer) isListFile(dir, fileName string) bool {
360-
if dir != infoDir {
342+
func (a dpkgAnalyzer) isMd5SumsFile(dir, fileName string) bool {
343+
// - var/lib/dpkg/info/*.md5sums is default path
344+
// - var/lib/dpkg/status.d/*.md5sums path in distroless images (see https://github.com/GoogleContainerTools/distroless/blob/5c119701429fb742ab45682cfc3073f911bad4bf/PACKAGE_METADATA.md#omitted-files)
345+
if dir != infoDir && dir != statusDir {
361346
return false
362347
}
363348

364-
return strings.HasSuffix(fileName, ".list")
349+
return strings.HasSuffix(fileName, md5sumsExtension)
365350
}
366351

367352
func (a dpkgAnalyzer) Type() analyzer.Type {

‎pkg/fanal/analyzer/pkg/dpkg/dpkg_test.go‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ func Test_dpkgAnalyzer_Analyze(t *testing.T) {
1818
tests := []struct {
1919
name string
2020
// testFiles contains path in testdata and path in OS
21-
// e.g. tar.list => var/lib/dpkg/info/tar.list
21+
// e.g. tar.md5sums => var/lib/dpkg/info/tar.md5sums
2222
testFiles map[string]string
2323
want *analyzer.AnalysisResult
2424
wantErr bool
@@ -1372,7 +1372,9 @@ func Test_dpkgAnalyzer_Analyze(t *testing.T) {
13721372
Packages: []types.Package{
13731373
{
13741374
ID: "apt@1.6.3ubuntu0.1", Name: "apt", Version: "1.6.3ubuntu0.1",
1375-
SrcName: "apt", SrcVersion: "1.6.3ubuntu0.1", Maintainer: "Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>", Arch: "amd64"},
1375+
SrcName: "apt", SrcVersion: "1.6.3ubuntu0.1",
1376+
Maintainer: "Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>", Arch: "amd64",
1377+
},
13761378
},
13771379
},
13781380
},
@@ -1418,12 +1420,11 @@ func Test_dpkgAnalyzer_Analyze(t *testing.T) {
14181420
},
14191421
},
14201422
{
1421-
name: "info list",
1422-
testFiles: map[string]string{"./testdata/tar.list": "var/lib/dpkg/info/tar.list"},
1423+
name: "md5sums",
1424+
testFiles: map[string]string{"./testdata/tar.md5sums": "var/lib/dpkg/info/tar.md5sums"},
14231425
want: &analyzer.AnalysisResult{
14241426
SystemInstalledFiles: []string{
1425-
"/bin/tar",
1426-
"/etc/rmt",
1427+
"/usr/bin/tar",
14271428
"/usr/lib/mime/packages/tar",
14281429
"/usr/sbin/rmt-tar",
14291430
"/usr/sbin/tarcat",
@@ -1488,12 +1489,17 @@ func Test_dpkgAnalyzer_Required(t *testing.T) {
14881489
{
14891490
name: "*.md5sums file in status dir",
14901491
filePath: "var/lib/dpkg/status.d/base-files.md5sums",
1491-
want: false,
1492+
want: true,
1493+
},
1494+
{
1495+
name: "*.md5sums file in info dir",
1496+
filePath: "var/lib/dpkg/info/bash.md5sums",
1497+
want: true,
14921498
},
14931499
{
14941500
name: "list file",
14951501
filePath: "var/lib/dpkg/info/bash.list",
1496-
want: true,
1502+
want: false,
14971503
},
14981504
{
14991505
name: "available file",
@@ -1502,7 +1508,7 @@ func Test_dpkgAnalyzer_Required(t *testing.T) {
15021508
},
15031509
{
15041510
name: "sad path",
1505-
filePath: "var/lib/dpkg/status/bash.list",
1511+
filePath: "var/lib/dpkg/status/bash.md5sums",
15061512
want: false,
15071513
},
15081514
}

‎pkg/fanal/analyzer/pkg/dpkg/testdata/tar.list‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎pkg/fanal/analyzer/pkg/dpkg/testdata/tar.md5sums‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
25de5fcdc3c8ebd9c9f599fb7a899b40 usr/bin/tar
2+
5bf0e62990e0b668830ceb2c8615b497 usr/lib/mime/packages/tar
3+
de1096fbccdc14324196fc6829324ebc usr/sbin/rmt-tar
4+
fd2fab77cf4da2288c11a4de2c0c7fe0 usr/sbin/tarcat
5+
020511595e8fb6a77b29032b3ec4ab10 usr/share/doc/tar/AUTHORS
6+
5386c77b19064999a340c62ec50b9307 usr/share/doc/tar/NEWS.gz
7+
882d20d9ef9c5baf1557a921bb9c6251 usr/share/doc/tar/README.Debian
8+
63b372367a085f8d07c4127828c56999 usr/share/doc/tar/THANKS.gz
9+
e9c4ef9b5dffd7b5b94f019d28bf86d6 usr/share/doc/tar/changelog.Debian.gz
10+
4546fba65a596c8793dca393b32ad9ee usr/share/doc/tar/copyright
11+
0502616f764289571b974ecc5adc1209 usr/share/man/man1/tar.1.gz
12+
9376d82eb54e507863d32114dddd3de6 usr/share/man/man1/tarcat.1.gz
13+
4892b2039586e61ef7b290591e9743e5 usr/share/man/man8/rmt-tar.8.gz

0 commit comments

Comments
 (0)