Skip to content

Commit def351d

Browse files
authored
Merge pull request #5 from scanoss/hpsm
added support to enable/disable HPSM
2 parents f0a4cf6 + 9f2ba79 commit def351d

6 files changed

Lines changed: 260 additions & 1 deletion

File tree

CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
99
### Added
1010
- Upcoming changes...
1111

12+
## [1.2.1] - 2023-08-09
13+
### Added
14+
- Added option to enable/disable HPSM processing (`HPSMEnabled`)
15+
1216
## [1.2.0] - 2023-05-18
1317
### Added
1418
- Added support for password protected TLS Key Files (`Password`)
@@ -56,3 +60,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
5660
[1.0.0]: https://github.com/scanoss/api.go/compare/v0.7.0...v1.0.0
5761
[1.1.0]: https://github.com/scanoss/api.go/compare/v1.0.0...v1.1.0
5862
[1.2.0]: https://github.com/scanoss/api.go/compare/v1.1.0...v1.2.0
63+
[1.2.1]: https://github.com/scanoss/api.go/compare/v1.2.0...v1.2.1

config/app-config-prod.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@
2424
"WfpGrouping": 3,
2525
"Workers": 2,
2626
"TmpFileDelete": true,
27-
"KeepFailedWfps": true
27+
"KeepFailedWfps": true,
28+
"HPSMEnabled": true
2829
},
2930
"TLS": {
3031
"CertFile": "",

pkg/config/server_config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ type ServerConfig struct {
5858
TmpFileDelete bool `env:"SCAN_TMP_DELETE"` // true/false
5959
KeepFailedWfps bool `env:"SCAN_KEEP_FAILED_WFP"` // true/false
6060
ScanningURL string `env:"SCANOSS_API_URL"` // URL to present back in API responses - default https://osskb.org/api
61+
HPSMEnabled bool `env:"SCAN_HPSM_ENABLED"` // Enable HPSM (High Precision Snippet Matching) or not (default true)
6162
}
6263
TLS struct {
6364
CertFile string `env:"SCAN_TLS_CERT"` // TLS Certificate
@@ -101,6 +102,7 @@ func setServerConfigDefaults(cfg *ServerConfig) {
101102
cfg.Scanning.Workers = 1 // Default to single threaded scanning
102103
cfg.Scanning.ScanTimeout = 120 // Default scan engine timeout to 2 minutes
103104
cfg.Scanning.WfpGrouping = 3 // Default number of WFPs to group into a single scan request (when Workers > 1)
105+
cfg.Scanning.HPSMEnabled = true
104106
}
105107

106108
// LoadFile loads the specified file and returns its contents in a string array.

pkg/service/scanning_service.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"go.uber.org/zap"
3232
)
3333

34+
// getFlags extracts the form values from a request returns the flags, scan type, and sbom data if detected.
3435
func (s APIService) getFlags(r *http.Request, zs *zap.SugaredLogger) (string, string, string) {
3536
flags := strings.TrimSpace(r.FormValue("flags")) // Check form for Scanning flags
3637
scanType := strings.TrimSpace(r.FormValue("type")) // Check form for SBOM type
@@ -198,6 +199,9 @@ func (s APIService) ScanDirect(w http.ResponseWriter, r *http.Request) {
198199
http.Error(w, "ERROR no WFP file contents (file=...) supplied", http.StatusBadRequest)
199200
return
200201
}
202+
if !s.validateHPSM(contentsTrimmed, zs, w) {
203+
return
204+
}
201205
counters.incRequestAmount("files", int64(wfpCount))
202206
zs.Debugf("Need to scan %v files", wfpCount)
203207
// Only one worker selected, so send the whole WFP in a single command
@@ -208,6 +212,21 @@ func (s APIService) ScanDirect(w http.ResponseWriter, r *http.Request) {
208212
}
209213
}
210214

215+
// validateHPSM checks if HPSM is enabled or not. If it's not and HPSM is detected. Fail the scan request.
216+
func (s APIService) validateHPSM(contents []byte, zs *zap.SugaredLogger, w http.ResponseWriter) bool {
217+
if !s.config.Scanning.HPSMEnabled {
218+
if s.config.App.Trace {
219+
zs.Debugf("Checking if HPSM is present in the submitted WFP...")
220+
}
221+
if strings.Contains(string(contents), "hpsm=") {
222+
zs.Errorf("HPSM (hpsm=...) detected in WFPs and HPSM support is disabled")
223+
http.Error(w, "ERROR HPSM detected in WFP. HPSM is disabled", http.StatusForbidden)
224+
return false
225+
}
226+
}
227+
return true
228+
}
229+
211230
// workerScan attempts to process all incoming scanning jobs and dumps the results into the subsequent results channel.
212231
func (s APIService) workerScan(id string, jobs <-chan string, results chan<- string, flags, sbomType, sbomFile string, zs *zap.SugaredLogger) {
213232
if s.config.App.Trace {

pkg/service/scanning_service_test.go

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,13 @@ func TestScanDirectThreaded(t *testing.T) {
251251
file: "./tests/fingers.wfp",
252252
want: http.StatusOK,
253253
},
254+
{
255+
name: "Scanning - HPSM success",
256+
binary: "../../test-support/scanoss.sh",
257+
fieldName: "file",
258+
file: "./tests/fingers-hpsm.wfp",
259+
want: http.StatusOK,
260+
},
254261
}
255262
for _, test := range tests {
256263
t.Run(test.name, func(t *testing.T) {
@@ -295,3 +302,96 @@ func TestScanDirectThreaded(t *testing.T) {
295302
})
296303
}
297304
}
305+
306+
func TestScanDirectSingleHPSM(t *testing.T) {
307+
err := zlog.NewSugaredDevLogger()
308+
if err != nil {
309+
t.Fatalf("an error '%s' was not expected when opening a sugared logger", err)
310+
}
311+
defer zlog.SyncZap()
312+
myConfig := setupConfig(t)
313+
myConfig.App.Trace = true
314+
myConfig.Scanning.ScanDebug = true
315+
myConfig.Scanning.HPSMEnabled = false
316+
apiService := NewAPIService(myConfig)
317+
318+
tests := []struct {
319+
name string
320+
fieldName string
321+
file string
322+
binary string
323+
scanType string
324+
assets string
325+
want int
326+
}{
327+
{
328+
name: "Scanning - success 1",
329+
binary: "../../test-support/scanoss.sh",
330+
fieldName: "file",
331+
file: "./tests/fingers.wfp",
332+
want: http.StatusOK,
333+
},
334+
{
335+
name: "Scanning - HPSM fail",
336+
binary: "../../test-support/scanoss.sh",
337+
fieldName: "filename",
338+
file: "./tests/fingers-hpsm.wfp",
339+
want: http.StatusForbidden,
340+
},
341+
}
342+
for _, test := range tests {
343+
t.Run(test.name, func(t *testing.T) {
344+
if test.want == http.StatusOK { // flip between Trace for some successful calls
345+
if myConfig.App.Trace {
346+
myConfig.App.Trace = false
347+
} else {
348+
myConfig.App.Trace = true
349+
}
350+
} else {
351+
myConfig.App.Trace = true
352+
}
353+
myConfig.Scanning.ScanBinary = test.binary
354+
filePath := test.file
355+
fieldName := test.fieldName
356+
postBody := new(bytes.Buffer)
357+
mw := multipart.NewWriter(postBody)
358+
file, err := os.Open(filePath)
359+
if err != nil {
360+
t.Fatal(err)
361+
}
362+
writer, err := mw.CreateFormFile(fieldName, filePath)
363+
if err != nil {
364+
t.Fatal(err)
365+
}
366+
if _, err = io.Copy(writer, file); err != nil {
367+
t.Fatal(err)
368+
}
369+
if len(test.scanType) > 0 && len(test.assets) > 0 {
370+
fmt.Println("Adding asset options: ", test.scanType, test.assets)
371+
err = mw.WriteField("type", test.scanType)
372+
if err != nil {
373+
t.Fatal(err)
374+
}
375+
err = mw.WriteField("assets", test.assets)
376+
if err != nil {
377+
t.Fatal(err)
378+
}
379+
}
380+
_ = mw.Close() // close the writer before making the request
381+
382+
req := httptest.NewRequest(http.MethodPost, "http://localhost/api/api/scan/direct", postBody)
383+
w := httptest.NewRecorder()
384+
req.Header.Add("Content-Type", mw.FormDataContentType())
385+
apiService.ScanDirect(w, req)
386+
resp := w.Result()
387+
body, err := io.ReadAll(resp.Body)
388+
if err != nil {
389+
t.Fatalf("an error was not expected when reading from request: %v", err)
390+
}
391+
assert.Equal(t, test.want, resp.StatusCode)
392+
fmt.Println("Status: ", resp.StatusCode)
393+
fmt.Println("Type: ", resp.Header.Get("Content-Type"))
394+
fmt.Println("Body: ", string(body))
395+
})
396+
}
397+
}

pkg/service/tests/fingers-hpsm.wfp

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
file=37f7cd1e657aa3c30ece35995b4c59e5,3114,scancodedeps-test.py
2+
hpsm=00d2fffaffc62d1801413bff9bacfffab0442798c29400f883ff9693acffff4400410099ff0400da000a14fed320ffba0085000aff04943ad320ff2c008500d154b5f75fb623a70ae0b0ad6f20ffff8b0a
3+
6=d5e54c33,b03faabe
4+
7=23bfe641
5+
8=8f1a259c
6+
9=96e0f2da
7+
10=86238bbc,57fd17f4
8+
11=3df15217
9+
13=49b901c6,76c8f7b2,5e608a23
10+
14=d18cc549,a6a72e20
11+
16=86ebf3c6
12+
17=6127c6f8
13+
18=85d7719c,b0bcdb7d,8b4fa021
14+
20=b05b10fa,7e399b18,d6e25bf1,45de0420
15+
21=32c89370,66902c56,8c6aa9e2
16+
27=0eed4641
17+
29=3b805377,e3580ff7,6d55823a
18+
32=5f3a1717
19+
34=5695ea50,165cff28
20+
36=5658134e
21+
38=830584a7
22+
40=31042d6c
23+
42=f6c592f5,c520eb6d
24+
45=8bb31fff,397e1cf4
25+
50=88c082fe,f24ae766
26+
54=ba9495af,ee4967e4
27+
55=00472355
28+
57=206fa2e1
29+
58=61d30512
30+
62=ac19a9d6,f24ae766
31+
65=1b440eb6
32+
66=3ea07e21
33+
68=fb6a4afb
34+
70=b0540d65
35+
71=a2a1db62
36+
73=cc335ef4,aab22912
37+
74=88cfbe2a
38+
75=cf2c92f6,c4af7521,ed7201d8,d90e1ea1
39+
77=6cef3503,9616a9c7,f1da437a
40+
81=aac97cfe
41+
file=3d61f8e5b082897cb9b193510284e5b5,2232,winnowing-test.py
42+
hpsm=00d2fffaffc62d1801413bff9bacfffab0442798c2940083fffcffff44004c00640d990888cdcb83253fcb83ffc90d255d88cdcb83ffff8b0a
43+
6=d5e54c33,b03faabe
44+
7=23bfe641
45+
8=8f1a259c
46+
9=96e0f2da
47+
10=86238bbc,57fd17f4
48+
11=3df15217
49+
13=49b901c6,76c8f7b2,5e608a23
50+
14=d18cc549,a6a72e20
51+
16=86ebf3c6
52+
17=6127c6f8
53+
18=85d7719c,b0bcdb7d,8b4fa021
54+
20=b05b10fa,7e399b18,d6e25bf1,45de0420
55+
21=32c89370,66902c56,8c6aa9e2
56+
26=c9fcb62d
57+
29=45d77b0e
58+
33=e8fae880
59+
35=d003b44d
60+
36=02fd799d
61+
37=168c5416
62+
38=7cabb204
63+
39=8f91aea2
64+
41=9eabe731
65+
43=fe04b2d8
66+
46=81e903d7
67+
47=1f285981
68+
50=ab03eaec,9a784fa2,d003b44d,ed34ad2f,168c5416
69+
51=7cabb204
70+
52=8f91aea2
71+
file=17324fe73aaee770c416b91404f1bff7,1861,csvoutput-test.py
72+
hpsm=00d2fffaffc62d1801413bff9bacfffab0442798c2940083fffcffff44004c002d0d990888cdcb83253fcb83ffff8b0a
73+
6=d5e54c33,b03faabe
74+
7=23bfe641
75+
8=8f1a259c
76+
9=96e0f2da
77+
10=86238bbc,57fd17f4
78+
11=3df15217
79+
13=49b901c6,76c8f7b2,5e608a23
80+
14=d18cc549,a6a72e20
81+
16=86ebf3c6
82+
17=6127c6f8
83+
18=85d7719c,b0bcdb7d,8b4fa021
84+
20=b05b10fa,7e399b18,d6e25bf1,45de0420
85+
21=32c89370,66902c56,8c6aa9e2
86+
26=c9fcb62d
87+
29=45d77b0e
88+
34=1c403291,5bc0fdd3
89+
35=d003b44d
90+
36=02fd799d
91+
37=168c5416
92+
38=7cabb204
93+
39=8f91aea2
94+
41=9eabe731
95+
43=fe04b2d8
96+
48=584afedf
97+
file=d8fdce51fb00df7bdce9c845b3dfda6a,2919,grpc-client-test.py
98+
hpsm=00d2fffaffc62d1801413bff9bacfffab0442798c29400f883ff9693ffff4400ab0099ff2400b500b5f75fb623a78e903eff9500ad000a14fed320b5f75fb623a7c8aaa6ff85d2e76f21ff
99+
6=d5e54c33,b03faabe
100+
7=23bfe641
101+
8=8f1a259c
102+
9=96e0f2da
103+
10=86238bbc,57fd17f4
104+
11=3df15217
105+
13=49b901c6,76c8f7b2,5e608a23
106+
14=d18cc549,a6a72e20
107+
16=86ebf3c6
108+
17=6127c6f8
109+
18=85d7719c,b0bcdb7d,8b4fa021
110+
20=b05b10fa,7e399b18,d6e25bf1,45de0420
111+
21=32c89370,66902c56,8c6aa9e2
112+
27=0eed4641
113+
31=3b805377,eb84d927
114+
35=91b67e77,0009c10f,830584a7
115+
39=1d66bd98
116+
43=bc811c12,fb6a4afb
117+
45=b0540d65
118+
46=a2a1db62
119+
48=f9c65f50,412ffca4,3ceb36a7
120+
51=7386e388
121+
53=b0a251fd,f1e92f1a,ae7c2c39,c02cbeb1
122+
56=f72ca1c2
123+
57=8bb31fff
124+
58=397e1cf4
125+
61=61d30512
126+
62=bf2da6c3,fb6a4afb
127+
64=b0540d65
128+
65=a2a1db62
129+
67=bae84449
130+
71=178e2453,888ef9e6
131+
73=afdfb29f
132+

0 commit comments

Comments
 (0)