Skip to content

Commit 084a437

Browse files
williamlin-susewilliamlin-suse
committed
NVSHAS-9553-9979-9987
1 parent 5f4973f commit 084a437

28 files changed

+1756
-350
lines changed

‎controller/api/apis.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3865,13 +3865,14 @@ type RESTUserRoleConfigData struct {
38653865

38663866
// Import task
38673867
type RESTImportTask struct {
3868-
TID string `json:"tid"`
3869-
CtrlerID string `json:"ctrler_id"`
3870-
LastUpdateTime time.Time `json:"last_update_time,omitempty"`
3871-
Percentage int `json:"percentage"`
3872-
TriggeredBy string `json:"triggered_by,omitempty"` // fullname of the user who triggers import
3873-
Status string `json:"status,omitempty"`
3874-
TempToken string `json:"temp_token,omitempty"`
3868+
TID string `json:"tid"`
3869+
CtrlerID string `json:"ctrler_id"`
3870+
LastUpdateTime time.Time `json:"last_update_time,omitempty"`
3871+
Percentage int `json:"percentage"`
3872+
TriggeredBy string `json:"triggered_by,omitempty"` // fullname of the user who triggers import
3873+
Status string `json:"status,omitempty"`
3874+
TempToken string `json:"temp_token,omitempty"`
3875+
FailToDecryptKeyFields map[string][]string `json:"fail_to_decrypt_key_fields"` // key : []fields
38753876
}
38763877

38773878
type RESTImportTaskData struct {

‎controller/api/apis.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7519,6 +7519,14 @@ definitions:
75197519
temp_token:
75207520
type: string
75217521
example: ""
7522+
fail_to_decrypt_key_fields:
7523+
type: object
7524+
description: Object key is kv key and value is array of cloaked fields that cannot be decrypted
7525+
additionalProperties:
7526+
type: array
7527+
items:
7528+
type: string
7529+
example: ["x509_cert", "signing_cert"]
75227530
RESTImportTaskData:
75237531
type: object
75247532
required:

‎controller/api/internal_apis.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ type AlertType string
180180
const (
181181
AlertTypeRBAC AlertType = "RBAC"
182182
AlertTypeTlsCertificate AlertType = "TLS_CERTIFICATE"
183+
AlertTypeOthers AlertType = "Others"
183184
)
184185

185186
type RESTNvAlertGroup struct {

‎controller/api/log_apis.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,10 @@ const (
161161
EventNameGroupMetricViolation = "Group.Metric.Violation"
162162
EventNameKvRestored = "Configuration.Restore"
163163
EventNameScanDataRestored = "Scan.Data.Restore"
164+
EventNameMismatchedDEKSeed = "Security.DEK.Seed.Mismatch"
165+
EventNameDEKSeedUnavailable = "Security.DEK.Seed.Unavailable"
166+
EventNameReEncryptWithDEK = "Security.DEK.Encrypt"
167+
EventNameEncryptionSecretSet = "Security.Encryption.Secret.Set"
164168
)
165169

166170
// TODO: these are not events but incidents

‎controller/cache/cache.go‎

Lines changed: 90 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
package cache
22

33
import (
4+
"crypto/sha256"
5+
"encoding/hex"
46
"encoding/json"
57
"fmt"
68
"net"
79
"os"
810
"reflect"
911
"sort"
12+
"strconv"
1013
"strings"
1114
"sync"
1215
"sync/atomic"
@@ -178,11 +181,13 @@ const (
178181
type Context struct {
179182
k8sVersion string
180183
ocVersion string
181-
RancherEP string // from yaml/helm chart
182-
RancherSSO bool // from yaml/helm chart
183-
TelemetryFreq uint // from yaml
184-
CheckDefAdminFreq uint // from yaml, in minutes
185-
CspPauseInterval uint // from yaml, in minutes
184+
RancherEP string // from yaml/helm chart
185+
RancherSSO bool // from yaml/helm chart
186+
TelemetryFreq uint // from yaml
187+
KeyRotationDuration time.Duration // from yaml
188+
CheckKeyRotationDuration time.Duration // from yaml
189+
CheckDefAdminFreq uint // from yaml, in minutes
190+
CspPauseInterval uint // from yaml, in minutes
186191
LocalDev *common.LocalDevice
187192
EvQueue cluster.ObjectQueueInterface
188193
AuditQueue cluster.ObjectQueueInterface
@@ -1679,6 +1684,9 @@ func startWorkerThread(ctx *Context) {
16791684
groupMetricCheckTicker := time.NewTicker(groupMetricCheckPeriod)
16801685
policyMetricCheckTicker := time.NewTicker(policyMetricCheckPeriod)
16811686

1687+
keyRotationCheckTicker := time.NewTicker(cctx.CheckKeyRotationDuration)
1688+
log.WithFields(log.Fields{"CheckKeyRotationDuration": cctx.CheckKeyRotationDuration, "KeyRotationDuration": cctx.KeyRotationDuration}).Info()
1689+
16821690
wlSuspected := utils.NewSet() // supicious workload ids
16831691
pruneKvTicker := time.NewTicker(pruneKVPeriod)
16841692
pruneWorkloadKV(wlSuspected) // the first scan
@@ -1742,6 +1750,8 @@ func startWorkerThread(ctx *Context) {
17421750
cacheMutexUnlock()
17431751
case <-pruneKvTicker.C:
17441752
pruneWorkloadKV(wlSuspected)
1753+
case <-keyRotationCheckTicker.C:
1754+
checkKeyRotation()
17451755
case <-scannerTicker.C:
17461756
if isScanner() {
17471757
// Remove stalled scanner
@@ -2031,6 +2041,33 @@ func startWorkerThread(ctx *Context) {
20312041
log.WithFields(log.Fields{"name": o.Name, "domain": o.Domain}).Info("deleted")
20322042
}
20332043
}
2044+
case resource.RscTypeSecret:
2045+
var err error
2046+
var lock cluster.LockInterface
2047+
for i := 0; i < 4; i++ {
2048+
var encKeys common.EncKeys
2049+
var currEncKeyVer string
2050+
lock, err = clusHelper.AcquireLock(share.CLUSStoreSecretKey, time.Duration(time.Second)*4)
2051+
if err == nil {
2052+
if encKeys, currEncKeyVer, _, err = resource.RetrieveStorePassphrases(); err == nil {
2053+
if len(encKeys) > 0 {
2054+
if err = common.InitAesGcmKey(encKeys, currEncKeyVer); err == nil {
2055+
log.Info("store passphrases reloaded")
2056+
clusHelper.ReleaseLock(lock)
2057+
break
2058+
}
2059+
log.WithFields(log.Fields{"i": i, "lead": isLeader(), "err": err}).Error("Failed to init AesGcm")
2060+
}
2061+
} else {
2062+
log.WithFields(log.Fields{"i": i, "err": err}).Error("Failed to reloaded store passphrases")
2063+
}
2064+
clusHelper.ReleaseLock(lock)
2065+
}
2066+
time.Sleep(time.Second)
2067+
}
2068+
if err != nil {
2069+
log.WithFields(log.Fields{"err": err}).Error("Failed to reloaded store passphrases")
2070+
}
20342071
/*
20352072
case resource.RscTypeMutatingWebhookConfiguration:
20362073
var n, o *resource.AdmissionWebhookConfiguration
@@ -2108,6 +2145,54 @@ func startWorkerThread(ctx *Context) {
21082145
}()
21092146
}
21102147

2148+
func checkKeyRotation() {
2149+
if !isLeader() {
2150+
return
2151+
}
2152+
2153+
if valueBackup, _ := cluster.Get(share.CLUSSystemEncMigratedKey); len(valueBackup) > 0 {
2154+
var cfgEncBackup share.CLUSSystemConfigEncMigrated
2155+
if err := json.Unmarshal(valueBackup, &cfgEncBackup); err == nil {
2156+
if time.Now().UTC().After(cfgEncBackup.EncDataExpirationTime) {
2157+
// the backup is expired
2158+
_ = cluster.Delete(share.CLUSSystemEncMigratedKey)
2159+
log.Info("encryption-migrated config backup is deleted")
2160+
}
2161+
}
2162+
}
2163+
2164+
value, _ := cluster.Get(share.CLUSNextKeyRotationTSKey)
2165+
if len(value) == 0 {
2166+
_ = kv.SetNextKeyRotationTime(cctx.KeyRotationDuration)
2167+
return
2168+
}
2169+
i, err := strconv.ParseInt(string(value), 10, 64)
2170+
if err != nil {
2171+
return
2172+
}
2173+
nextKeyRotationTime := time.Unix(i, 0)
2174+
if time.Now().After(nextKeyRotationTime) {
2175+
if err := resource.AddStorePassphrase(); err == nil {
2176+
if err = kv.SetNextKeyRotationTime(cctx.KeyRotationDuration); err == nil {
2177+
alert := "Important: Please backup the data in Kubernetes secret neuvector-store-secret reset by NeuVector"
2178+
b := sha256.Sum256([]byte(alert))
2179+
key := hex.EncodeToString(b[:])
2180+
allUser := clusHelper.GetAllUsers(access.NewReaderAccessControl())
2181+
for _, user := range allUser {
2182+
accepted := utils.NewSetFromStringSlice(user.AcceptedAlerts)
2183+
if accepted.Contains(key) {
2184+
accepted.Remove(key)
2185+
user.AcceptedAlerts = accepted.ToStringSlice()
2186+
if err = clusHelper.PutUser(user); err != nil {
2187+
log.WithFields(log.Fields{"": user.Fullname, "err": err}).Error()
2188+
}
2189+
}
2190+
}
2191+
}
2192+
}
2193+
}
2194+
}
2195+
21112196
// handler of K8s resource watcher calls cbResourceWatcher() which sends to orchObjChan/objChan
21122197
// [2021-02-15] CRD-related resource changes do not call this function.
21132198
//

‎controller/cache/config.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ type webhookCache struct {
3232

3333
const DefaultScannerConfigUpdateTimeout = time.Minute * 5
3434

35+
var encMigratedConfigRestored bool
3536
var systemConfigCache share.CLUSSystemConfig = common.DefaultSystemConfig
3637
var fedSystemConfigCache share.CLUSSystemConfig = share.CLUSSystemConfig{CfgType: share.FederalCfg}
3738
var webhookCacheMap map[string]*webhookCache = make(map[string]*webhookCache, 0) // Only the enabled webhooks
@@ -467,6 +468,42 @@ func (m CacheMethod) GetSystemConfigClusterName(acc *access.AccessControl) strin
467468
return systemConfigCache.ClusterName
468469
}
469470

471+
func encMigrateSystemConfig(valueBackup, value []byte) {
472+
controllers := cacher.GetAllControllers(access.NewAdminAccessControl())
473+
for _, ctrler := range controllers {
474+
if ctrler.Ver != cctx.CtrlerVersion {
475+
log.WithFields(log.Fields{"target": ctrler.Ver, "me": cctx.CtrlerVersion}).Info()
476+
// the system config backup for variant encryption key migration is found.
477+
// it means sensitive fields in system config are re-encrypted for variant encryption key migration.
478+
var dec common.DecryptUnmarshaller
479+
var cfg share.CLUSSystemConfig
480+
if err := dec.Unmarshal(value, &cfg); err == nil && dec.GetDecryptedFieldsNumber() == 0 {
481+
// however, it's unexpected that no sensitive field is decrypted in dec.Unmarshal() call.
482+
// it means all sensitive fields in current system config(cfg) were set to empty string that we need to restore system config from the system config backup
483+
var cfgEncBackup share.CLUSSystemConfigEncMigrated
484+
if err = json.Unmarshal(valueBackup, &cfgEncBackup); err == nil {
485+
if time.Now().UTC().Before(cfgEncBackup.EncDataExpirationTime) {
486+
// the backup is not expired yet
487+
if err = cluster.PutBinary(share.CLUSConfigSystemKey, []byte(cfgEncBackup.EncMigratedSystemConfig)); err == nil {
488+
log.Info("encryption-migrated config is restored")
489+
encMigratedConfigRestored = true
490+
}
491+
} else {
492+
_ = cluster.Delete(share.CLUSSystemEncMigratedKey)
493+
log.Info("encryption-migrated config backup is deleted")
494+
}
495+
} else {
496+
_ = cluster.Delete(share.CLUSSystemEncMigratedKey)
497+
log.WithFields(log.Fields{"err": err}).Error("failed to json unmarshal encryption-migrated config")
498+
}
499+
} else if err != nil {
500+
log.WithFields(log.Fields{"err": err}).Error("failed to dec unmarshal system config")
501+
}
502+
break
503+
}
504+
}
505+
}
506+
470507
func systemConfigUpdate(nType cluster.ClusterNotifyType, key string, value []byte) {
471508
log.WithFields(log.Fields{"type": cluster.ClusterNotifyName[nType], "key": key}).Debug()
472509

@@ -477,6 +514,14 @@ func systemConfigUpdate(nType cluster.ClusterNotifyType, key string, value []byt
477514
_ = json.Unmarshal(value, &cfg)
478515
log.WithFields(log.Fields{"config": cfg}).Debug()
479516

517+
if nType == cluster.ClusterNotifyModify {
518+
if valueBackup, _ := cluster.Get(share.CLUSSystemEncMigratedKey); len(valueBackup) > 0 {
519+
if !encMigratedConfigRestored {
520+
encMigrateSystemConfig(valueBackup, value)
521+
}
522+
}
523+
}
524+
480525
if cfg.IBMSAConfigNV.EpEnabled && cfg.IBMSAConfigNV.EpStart == 1 {
481526
if isLeader() {
482527
var param interface{} = &cfg.IBMSAConfig

‎controller/cache/object.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1646,6 +1646,7 @@ func ObjectUpdateHandler(nType cluster.ClusterNotifyType, key string, value []by
16461646
case "uniconf":
16471647
uniconfUpdate(nType, key, value)
16481648
case "cert":
1649+
value, _, _ = kv.UpgradeAndConvert(key, value)
16491650
certObjectUpdate(nType, key, value)
16501651
case "throttled", "telemetry":
16511652
default:

‎controller/cluster.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ func leadChangeHandler(newLead, oldLead string) {
141141
}
142142

143143
if emptyKvFound {
144-
if _, _, restored, restoredKvVersion, err := kv.GetConfigHelper().Restore(); restored && err == nil {
144+
if _, _, restored, restoredKvVersion, err := kv.GetConfigHelper().Restore(Host, Ctrler); restored && err == nil {
145145
clog := share.CLUSEventLog{
146146
Event: share.CLUSEvKvRestored,
147147
HostID: Host.ID,
@@ -216,7 +216,7 @@ func ctlrMemberUpdateHandler(nType cluster.ClusterNotifyType, memberAddr string,
216216
selfRejoin = false
217217

218218
ctlrPutLocalInfo()
219-
logController(share.CLUSEvControllerJoin)
219+
logController(share.CLUSEvControllerJoin, "")
220220
}
221221
} else if nType == cluster.ClusterNotifyDelete {
222222
if Ctrler.ClusterIP == memberAddr {
@@ -247,13 +247,14 @@ func clusterStart(clusterCfg *cluster.ClusterConfig) (string, string, error) {
247247
return cluster.GetSelfAddress(), lead, nil
248248
}
249249

250-
func logController(ev share.TLogEvent) {
250+
func logController(ev share.TLogEvent, msg string) {
251251
clog := share.CLUSEventLog{
252252
Event: ev,
253253
HostID: Host.ID,
254254
HostName: Host.Name,
255255
ControllerID: Ctrler.ID,
256256
ControllerName: Ctrler.Name,
257+
Msg: msg,
257258
}
258259
switch ev {
259260
case share.CLUSEvControllerStart:

‎controller/common/common.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ var ErrObjectExists error = errors.New("Object exists")
8282
var ErrAtomicWriteFail error = errors.New("Atomic write failed")
8383
var ErrUnsupported error = errors.New("Unsupported action")
8484
var ErrClusterWriteFail error = errors.New("Failed to write cluster")
85+
var ErrInvalidPassphrase error = errors.New("Invalid passphrase for data encryption key")
86+
var ErrDEKSeedUnavailable error = errors.New("store passphrase unavailable")
87+
var ErrEmptyValue error = errors.New("Empty value")
8588

8689
var defaultWebhookCategory []string = []string{}
8790
var defaultSyslogCategory []string = []string{
@@ -340,6 +343,10 @@ var LogEventMap = map[share.TLogEvent]LogEventInfo{
340343
share.CLUSEvGroupMetricViolation: {api.EventNameGroupMetricViolation, api.EventCatGroup, api.LogLevelWARNING},
341344
share.CLUSEvKvRestored: {api.EventNameKvRestored, api.EventCatConfig, api.LogLevelINFO},
342345
share.CLUSEvScanDataRestored: {api.EventNameScanDataRestored, api.EventCatScan, api.LogLevelINFO},
346+
share.CLUSEvMismatchedDEKSeed: {api.EventNameMismatchedDEKSeed, api.EventCatConfig, api.LogLevelERR},
347+
share.CLUSEvDEKSeedUnavailable: {api.EventNameDEKSeedUnavailable, api.EventCatConfig, api.LogLevelWARNING},
348+
share.CLUSEvReEncryptWithDEK: {api.EventNameReEncryptWithDEK, api.EventCatConfig, api.LogLevelINFO},
349+
share.CLUSEvEncryptionSecretSet: {api.EventNameEncryptionSecretSet, api.EventCatConfig, api.LogLevelNOTICE},
343350
}
344351

345352
type LogIncidentInfo struct {

0 commit comments

Comments
 (0)