KB Details

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Upcoming changes...
 
+## [1.2.2] - 2023-08-24
+### Added
+- Added support for `/api/kb/details` endpoint
+
 ## [1.2.1] - 2023-08-09
 ### Added
 - Added option to enable/disable HPSM processing (`HPSMEnabled`)
@@ -61,3 +65,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [1.1.0]: https://github.com/scanoss/api.go/compare/v1.0.0...v1.1.0
 [1.2.0]: https://github.com/scanoss/api.go/compare/v1.1.0...v1.2.0
 [1.2.1]: https://github.com/scanoss/api.go/compare/v1.2.0...v1.2.1
+[1.2.2]: https://github.com/scanoss/api.go/compare/v1.2.1...v1.2.2

go.mod

@@ -3,6 +3,7 @@ module scanoss.com/go-api
 go 1.19
 
 require (
+	github.com/go-co-op/gocron v1.31.0
 	github.com/golobby/config/v3 v3.4.2
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
@@ -20,8 +21,9 @@ require (
 	github.com/golobby/env/v2 v2.2.4 // indirect
 	github.com/phuslu/iploc v1.0.20230201 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce // indirect
-	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -5,6 +5,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-co-op/gocron v1.31.0 h1:8VaWk7ARDpsVYFP8SmjvHrBZQkcPQ7HyAzF7acG57yE=
+github.com/go-co-op/gocron v1.31.0/go.mod h1:39f6KNSGVOU1LO/ZOoZfcSxwlsJDQOKSu8erN0SH48Y=
 github.com/golobby/cast v1.3.3 h1:s2Lawb9RMz7YyYf8IrfMQY4IFmA1R/lgfmj97Vc6fig=
 github.com/golobby/cast v1.3.3/go.mod h1:0oDO5IT84HTXcbLDf1YXuk0xtg/cRDrxhbpWKxwtJCY=
 github.com/golobby/config/v3 v3.4.2 h1:oIOSo24mC0A8f93ZTL24NDNw0hZ3Tbb34wc1ckn2CsA=
@@ -19,7 +21,9 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/jpillora/ipfilter v1.2.9 h1:vjjcI1JpxZ6HvIj1MZfomhrfzXW/67QNdE449ZZfon8=
 github.com/jpillora/ipfilter v1.2.9/go.mod h1:QUYQLXQU0myCdxZVbYBZ5+An/qtSB2m1OBRiwqTa9pk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -32,6 +36,10 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/scanoss/zap-logging-helper v0.2.0 h1:r1HM/lalRZkkZU/3RQoIQJmZtzWsoAQr6Eqy/FRxWR8=
@@ -43,20 +51,24 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
-go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/protocol/rest/server.go

@@ -54,6 +54,7 @@ func RunServer(config *myconfig.ServerConfig) error {
 		zlog.S.Warnf("Scanning engine test failed. Scan requests are likely to fail.")
 		zlog.S.Warnf("Please make sure that %v is accessible", config.Scanning.ScanBinary)
 	}
+	apiService.SetupKBDetailsCron()
 	// Set up the endpoint routing
 	router := mux.NewRouter().StrictSlash(true)
 	router.HandleFunc("/", service.WelcomeMsg).Methods(http.MethodGet)
@@ -62,6 +63,7 @@ func RunServer(config *myconfig.ServerConfig) error {
 	router.HandleFunc("/api/health-check", service.HealthCheck).Methods(http.MethodGet)
 	router.HandleFunc("/api/metrics/{type}", service.MetricsHandler).Methods(http.MethodGet)
 	router.HandleFunc("/api/file_contents/{md5}", apiService.FileContents).Methods(http.MethodGet)
+	router.HandleFunc("/api/kb/details", apiService.KBDetails).Methods(http.MethodGet)
 	router.HandleFunc("/api/license/obligations/{license}", apiService.LicenseDetails).Methods(http.MethodGet)
 	router.HandleFunc("/api/scan/direct", apiService.ScanDirect).Methods(http.MethodPost)
 	router.HandleFunc("/api/sbom/attribution", apiService.SbomAttribution).Methods(http.MethodPost)

pkg/service/kb_details.go

@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2018-2023 SCANOSS.COM
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/go-co-op/gocron"
+	zlog "github.com/scanoss/zap-logging-helper/pkg/logger"
+)
+
+// Structure for parsing KB & Engine version from scan response.
+type matchStructure []struct {
+	Server struct {
+		Hostname  string `json:"hostname"`
+		KbVersion struct {
+			Daily   string `json:"daily"`
+			Monthly string `json:"monthly"`
+		} `json:"kb_version"`
+		Version string `json:"version"`
+	} `json:"server"`
+}
+
+var kbDetails string // KB Details JSON string
+
+// SetupKBDetailsCron sets up a background cron to update the KB version once an hour.
+func (s APIService) SetupKBDetailsCron() {
+	scheduler := gocron.NewScheduler(time.UTC)
+	_, err := scheduler.Every(60).Minutes().Do(s.loadKBDetails)
+	if err != nil {
+		zlog.S.Warnf("Problem setting up KB details cron: %v", err)
+		return
+	}
+	scheduler.StartAsync()
+}
+
+// KBDetails retrieves the KB details and send back to the requester.
+func (s APIService) KBDetails(w http.ResponseWriter, r *http.Request) {
+	reqID := getReqID(r)
+	w.Header().Set(ResponseIDKey, reqID)
+	zs := sugaredLogger(context.WithValue(r.Context(), RequestContextKey{}, reqID)) // Setup logger with context
+	zs.Infof("%v request from %v", r.URL.Path, r.RemoteAddr)
+	w.Header().Set(ContentTypeKey, ApplicationJSON)
+	w.WriteHeader(http.StatusOK)
+	printResponse(w, fmt.Sprintf("%s\n", kbDetails), zlog.S, true)
+}
+
+// loadKBDetails attempts to scan a file to load the latest KB details from the server.
+func (s APIService) loadKBDetails() {
+	zs := sugaredLogger(context.TODO()) // Setup logger without context
+	zs.Debugf("Loading latest KB details...")
+	if len(kbDetails) == 0 {
+		kbDetails = fmt.Sprintf(`{"kb_version": { "monthly": "%v", "daily": "%v"}}`, "unknown", "unknown")
+	}
+	// Load a random (hopefully non-existent) file match to extract the KB version details
+	result, err := s.scanWfp("file=7c53a2de7dfeaa20d057db98468d6670,2321,path/to/dummy/file.txt", "", "", "", zs)
+	if err != nil {
+		zs.Warnf("Failed to detect KB version from eninge: %v", err)
+		return
+	}
+	if len(result) > 0 {
+		if !json.Valid([]byte(result)) {
+			zs.Warnf("Invalid JSON response from engine for KB version: %v", result)
+			return
+		}
+		resDataAny := map[string]interface{}{}
+		err = json.Unmarshal([]byte(result), &resDataAny) // parse the response JSON into an interface map
+		if err != nil {
+			zs.Warnf("Failed to parse KB version from eninge response: %v - %v", result, err)
+			return
+		}
+		if s.config.App.Trace {
+			zs.Debugf("KB details JSON: %v", resDataAny)
+		}
+		var ms matchStructure
+		// Go through the list of file results and extract one set of KB details
+		for _, key := range resDataAny {
+			data, err := json.Marshal(key) // convert the given interface to JSON
+			if err != nil {
+				zs.Warnf("Failed to convert KB version map to json: %v - %v", key, err)
+				return
+			}
+			err = json.Unmarshal(data, &ms)
+			if err != nil {
+				zs.Warnf("Failed to parse KB version from eninge result: %v - %v", data, err)
+				return
+			}
+		}
+		if len(ms) > 0 {
+			kbDetails = fmt.Sprintf(`{"kb_version": { "monthly": "%v", "daily": "%v"}}`, ms[0].Server.KbVersion.Monthly, ms[0].Server.KbVersion.Daily)
+		}
+	}
+}

pkg/service/kb_details_test.go

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2018-2023 SCANOSS.COM
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package service
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	zlog "github.com/scanoss/zap-logging-helper/pkg/logger"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestKBDetails(t *testing.T) {
+	err := zlog.NewSugaredDevLogger()
+	if err != nil {
+		t.Fatalf("an error '%s' was not expected when opening a sugared logger", err)
+	}
+	defer zlog.SyncZap()
+
+	myConfig := setupConfig(t)
+	myConfig.App.Trace = true
+	apiService := NewAPIService(myConfig)
+	apiService.SetupKBDetailsCron()
+	time.Sleep(time.Duration(5) * time.Second) // Sleep a little to allow the KB details to be loaded
+	req := httptest.NewRequest(http.MethodGet, "http://localhost/", nil)
+	w := httptest.NewRecorder()
+	apiService.KBDetails(w, req)
+
+	resp := w.Result()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("an error was not expected when reading from request: %v", err)
+	}
+	bodyStr := string(body)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	expected := `{"kb_version": { "monthly": "23.07", "daily": "23.08.09"}}`
+	assert.Equal(t, expected, strings.TrimSpace(bodyStr))
+	fmt.Println("Status: ", resp.StatusCode)
+	fmt.Println("Type: ", resp.Header.Get("Content-Type"))
+	fmt.Println("Body: ", bodyStr)
+
+	// Test the version loading to fail
+	myConfig.Scanning.ScanBinary = "../path/to/does-not-exist.sh"
+	apiService.loadKBDetails()
+}

test-support/scanoss.sh

@@ -55,7 +55,7 @@ fi
 if [ "$1" == "-w" ] || [ "$2" == "-w" ] || [ "$3" == "-w" ] || [ "$4" == "-w" ] || [ "$5" == "-w" ] || [ "$6" == "-w" ] || [ "$7" == "-w" ] || [ "$8" == "-w" ]; then
   for i in "$@"; do :; done
   scf=$i
-  echo " {\"$scf\":[{\"id\": \"none\"}]}  "
+  echo " {\"$scf\":[{\"id\": \"none\", \"server\": { \"kb_version\": {\"daily\": \"23.08.09\", \"monthly\": \"23.07\"}, \"version\": \"5.2.7\"}}]}  "
   exit 0
 fi
 

tests/kb_details_test.go

@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2018-2023 SCANOSS.COM
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package tests
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/suite"
+	"io"
+	"net/http"
+	"testing"
+)
+
+type E2EKBDetailsSuite struct {
+	suite.Suite
+}
+
+func TestE2EKBDetailsSuite(t *testing.T) {
+	suite.Run(t, new(E2EKBDetailsSuite))
+}
+
+func (s *E2EKBDetailsSuite) TestHappyKBDetails() {
+	c := http.Client{}
+	resp, err := c.Get(fmt.Sprintf("%v/api/kb/details", hostPort))
+	if err != nil {
+		s.Failf("an error was not expected when sending request.", "error: %v", err)
+	}
+	s.Equal(http.StatusOK, resp.StatusCode)
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		s.Failf("an error was not expected when reading response body.", "error: %v", err)
+	}
+	bodyStr := string(body)
+	fmt.Println("Status: ", resp.StatusCode)
+	fmt.Println("Type: ", resp.Header.Get("Content-Type"))
+	fmt.Println("Body: ", bodyStr)
+}