diff --git a/config/transformer/models_list b/config/transformer/models_list index 89ba79e9c..133a3bfa3 100644 --- a/config/transformer/models_list +++ b/config/transformer/models_list @@ -11,3 +11,7 @@ openconfig-if-aggregate.yang openconfig-mclag.yang openconfig-mclag-annot.yang openconfig-vlan.yang +gnsi-authz.yang +gnsi-pathz.yang +gnsi-certz.yang +gnsi-credentialz.yang diff --git a/models/yang/annotations/openconfig-system-annot.yang b/models/yang/annotations/openconfig-system-annot.yang new file mode 100644 index 000000000..dec5a4bf2 --- /dev/null +++ b/models/yang/annotations/openconfig-system-annot.yang @@ -0,0 +1,36 @@ +module openconfig-system-annot { + + yang-version "1"; + + namespace "http://openconfig.net/yang/openconfig-system-annot"; + prefix "oc-sys-annot"; + + import openconfig-system { prefix oc-sys; } + import openconfig-yang-types { prefix oc-yang; } + import openconfig-system { prefix oc-sys; } + import gnsi-credentialz { prefix gnsi-credz; } + import gnsi-pathz { prefix gnsi-pathz; } + import sonic-extensions {prefix sonic-ext; } + import openconfig-system-grpc { prefix oc-sys-grpc; } + + deviation /oc-sys:system/oc-sys:aaa/oc-sys:authorization/oc-sys:state { + deviate add { + sonic-ext:db-name "STATE_DB"; + sonic-ext:subtree-transformer "authz_policy_xfmr"; + } + } + + deviation /oc-sys:system/oc-sys-grpc:grpc-servers { + deviate add { + sonic-ext:key-transformer "grpc_server_key_xfmr"; + sonic-ext:subtree-transformer "grpc_server_xfmr"; + } + } + + deviation /oc-sys:system/gnsi-pathz:gnmi-pathz-policies { + deviate add { + sonic-ext:key-transformer "pathz_policies_key_xfmr"; + sonic-ext:subtree-transformer "pathz_policies_xfmr"; + } + } +} diff --git a/models/yang/gnsi-authz.yang b/models/yang/gnsi-authz.yang new file mode 100644 index 000000000..cc1a3337d --- /dev/null +++ b/models/yang/gnsi-authz.yang @@ -0,0 +1,155 @@ +module gnsi-authz { + yang-version 1.1; + namespace "https://github.com/openconfig/gnsi/authz/yang"; + prefix gnsi-authz; + import openconfig-system { + prefix oc-sys; + } + import openconfig-system-grpc { + prefix oc-sys-grpc; + } + import openconfig-types { + prefix oc-types; + } + import openconfig-yang-types { + prefix oc-yang; + } + organization + "Google LLC"; + contact + "Google LLC"; + description + "This module provides a data model for the metadata of the gRPC + authorization policies installed on a networking device."; + revision 2022-10-30 { + description + "Adds success/failure counters."; + reference "0.3.0"; + } + revision 2022-08-01 { + description + "Single authz policy."; + reference "0.2.0"; + } + revision 2022-01-17 { + description + "Initial revision."; + reference "0.1.0"; + } + typedef version { + type string; + description + "The version ID of the gRPC authorization policy as provided by + the gRPC Authorization Policy Manager when the policy was pushed. + This leaf persists through a reboot."; + } + typedef created-on { + type oc-types:timeticks64; + description + "The creation time of the gRPC authorization policy as reported by + the gRPC Authorization Policy manager when the policy was pushed + to the device. This value is reported as nanoseconds since epoch + (January 1st, 1970 00:00:00 GMT). This leaf persists through + a reboot."; + } + // gRPC server authorization policy related definitions. + grouping counters { + description + "A collection of counters that were collected by the gNSI.authz + module while evaluating access to a RPC."; + leaf access-rejects { + type oc-yang:counter64; + description + "The total number of times the gNSI.authz module denied access + to a RPC."; + } + leaf last-access-reject { + type oc-types:timeticks64; + description + "A timestamp of the last time the gNSI.authz denied access to + a RPC."; + } + leaf access-accepts { + type oc-yang:counter64; + description + "The total number of times the gNSI.authz module allowed access + to a RPC."; + } + leaf last-access-accept { + type oc-types:timeticks64; + description + "A timestamp of the last time the gNSI.authz allowed access to + a RPC."; + } + } + grouping grpc-server-user-authz-policy-success-failure-counters { + description + "A collection of counters collected by the gNSI.authz module."; + container rpcs { + description + "A collection of counters collected by the gNSI.authz module + for each RPC separately."; + list rpc { + description + "A collection of counters collected by the gNSI.authz module + for a RPC identified by the `name`."; + key name; + leaf name { + type leafref { + path "../state/name"; + } + description + "The name of the RPC the counters were collected for."; + } + container state { + leaf name { + type string; + description + "The name of the RPC the counters were collected + for."; + } + uses counters; + } + } + } + } + grouping grpc-server-authz-policy-success-failure-counters { + description + "A collection of counters collected by the gNSI.authz module."; + container authz-policy-counters { + description + "A collection of counters collected by the gNSI.authz module."; + config false; + uses grpc-server-user-authz-policy-success-failure-counters; + } + } + grouping grpc-server-authz-policy-state { + description + "gNMI server's gRPC authorization policy freshness-related data."; + leaf grpc-authz-policy-version { + type version; + description + "The version of the gRPC authorization policy that is used by + this system."; + } + leaf grpc-authz-policy-created-on { + type created-on; + description + "The timestamp of the moment when the gRPC authorization policy + that is currently used by this system was created."; + } + } + // Augments section. + augment "/oc-sys:system/oc-sys:aaa/oc-sys:authorization/" + + "oc-sys:state" { + description + "A system's gRPC authorization policy freshness information."; + uses grpc-server-authz-policy-state; + } + augment "/oc-sys:system/oc-sys-grpc:grpc-servers/oc-sys-grpc:grpc-server" { + description + "Counters collected while evaluating access to a gRPC server using + the gNSI.authz authorization policy."; + uses grpc-server-authz-policy-success-failure-counters; + } +} diff --git a/models/yang/gnsi-certz.yang b/models/yang/gnsi-certz.yang new file mode 100644 index 000000000..fcc36375e --- /dev/null +++ b/models/yang/gnsi-certz.yang @@ -0,0 +1,187 @@ +module gnsi-certz { + yang-version 1.1; + namespace "https://github.com/openconfig/gnsi/certz/yang"; + prefix gnsi-certz; + + import openconfig-system { + prefix oc-sys; + } + import openconfig-system-grpc { + prefix oc-sys-grpc; + } + import openconfig-types { + prefix oc-types; + } + import openconfig-yang-types { + prefix oc-yang; + } + organization + "Google LLC"; + + contact + "Google LLC"; + + description + "This module provides a data model for the metadata of gRPC credentials + installed on a networking device."; + + revision 2023-02-13 { + description + "rename access/reject counters"; + reference "0.5.0"; + } + + revision 2023-08-24 { + description + "Adds ssl-profile-id leaf"; + reference "0.4.0"; + } + + revision 2023-05-10 { + description + "Adds authentication policy freshness information."; + reference "0.3.0"; + } + + revision 2022-10-30 { + description + "Adds success/failure counters."; + reference "0.2.0"; + } + + revision 2022-09-20 { + description + "Initial revision."; + reference "0.1.0"; + } + + typedef version { + type string; + description + "The version ID of the credential as provided by the credential + manager when the credential was pushed. This leaf persists through + a reboot."; + } + + typedef created-on { + type oc-types:timeticks64; + description + "The creation time of the credential as reported by the credential + manager when the credential was pushed to the device. This value is + reported as nanoseconds since epoch (January 1st, 1970 00:00:00 GMT). + This leaf persists through a reboot."; + } + // gRPC server related definitions. + // Success/failure counters. + grouping counters { + description + "A collection of counters that were collected while attempting + to establish connections to the gRPC server."; + + container counters { + config false; + description + "A collection of counters that were collected by the gRPC during + the authentication process."; + + leaf connection-rejects { + type oc-yang:counter64; + description + "The total number of times that gRPC clients have failed + in establishing a connection to the server."; + } + leaf last-connection-reject { + type oc-types:timeticks64; + description + "A timestamp of the last time a gRPC client failed + in establishing a connection to the server."; + } + leaf connection-accepts { + type oc-yang:counter64; + description + "The total number of times that gRPC clients have succeeded + in establishing a connection to the server."; + } + leaf last-connection-accept { + type oc-types:timeticks64; + description + "A timestamp of the last time a gRPC client succeeded + in establishing a connection to the server."; + } + } + } + + grouping grpc-server-credentials-state { + description + "gRPC server credentials freshness-related data."; + + leaf certificate-version { + type version; + description + "The version of the certificate (and associated + private key) that is used by this gRPC server."; + } + leaf certificate-created-on { + type created-on; + description + "The timestamp of the moment when the certificate + (and associated private key) that is currently used + by this gRPC server was created."; + } + leaf ca-trust-bundle-version { + type version; + description + "The version of the bundle of the Certificate + Authority certificates a.k.a. trust bundle used by + this gRPC server."; + } + leaf ca-trust-bundle-created-on { + type created-on; + description + "The timestamp of the moment when the bundle of + the Certificate Authority certificates (a.k.a. + trust bundle) was created."; + } + leaf certificate-revocation-list-bundle-version { + type version; + description + "The version of the Certificate Revocation List bundle used by + this gRPC server."; + } + leaf certificate-revocation-list-bundle-created-on { + type created-on; + description + "The timestamp of the moment when the Certificate Revocation + List bundle was created."; + } + leaf authentication-policy-version { + type version; + description + "The version of the authentication policy that is used by + this gRPC server."; + } + leaf authentication-policy-created-on { + type created-on; + description + "The timestamp of the moment when the authentication policy + that is currently used by this gRPC server was created."; + } + leaf ssl-profile-id { + type string; + description + "The ID of this gRPC server's SSL profile + as used by the gNSI Certz service"; + } + } + + // Augments section. + + augment "/oc-sys:system/oc-sys-grpc:grpc-servers/oc-sys-grpc:grpc-server/" + + "oc-sys-grpc:state" { + description + "A gRPC server credentials freshness information."; + + uses grpc-server-credentials-state; + uses counters; + } +} diff --git a/models/yang/gnsi-credentialz.yang b/models/yang/gnsi-credentialz.yang new file mode 100644 index 000000000..56f319e64 --- /dev/null +++ b/models/yang/gnsi-credentialz.yang @@ -0,0 +1,299 @@ +module gnsi-credentialz { + yang-version 1.1; + namespace "https://github.com/openconfig/gnsi/credentialz/yang"; + prefix gnsi-credz; + + import openconfig-system { + prefix oc-sys; + } + import openconfig-types { + prefix oc-types; + } + import openconfig-yang-types { + prefix oc-yang; + } + organization + "Google LLC"; + + contact + "Google LLC"; + + description + "This module provides a data model for the metadata of SSH and console + credentials installed on a networking device."; + + revision 2024-01-05 { + description + "Fix typo in YANG leaves"; + reference + "0.5.0"; + } + + revision 2023-10-03 { + description + "Added state leaves for admin-user"; + reference + "0.4.0"; + } + + revision 2023-08-18 { + description + "Fixed the canonical order of config field."; + reference + "0.3.0"; + } + + revision 2022-10-30 { + description + "Adds success/failure counters."; + reference + "0.2.0"; + } + + revision 2022-08-22 { + description + "Initial revision."; + reference + "0.1.0"; + } + + typedef version { + type string; + description + "The version ID of the credential as provided by the credential + manager when the credential was pushed. This leaf persists through + a reboot."; + } + + typedef created-on { + type oc-types:timeticks64; + description + "The creation time of the credential as reported by the credential + manager when the credential was pushed to the device. This value is + reported as nanoseconds since epoch (January 1st, 1970 00:00:00 GMT). + This leaf persists through a reboot."; + } + + // SSH server related definitions. + grouping ssh-server-credentials-version { + description + "SSH server credentials freshness-related data."; + + leaf active-trusted-user-ca-keys-version { + type version; + description + "The version of the Certificate Authority keys."; + } + + leaf active-trusted-user-ca-keys-created-on { + type created-on; + description + "The timestamp of the moment when the trusted user CA keys + were created."; + } + + leaf active-host-certificate-version { + type version; + description + "The version of the host certificate."; + } + + leaf active-host-certificate-created-on { + type created-on; + description + "The timestamp of the moment when the host certificate + was created."; + } + + leaf active-host-key-version { + type version; + description + "The version of the host public key."; + } + + leaf active-host-key-created-on { + type created-on; + description + "The timestamp of the moment when the host key was + created."; + } + } + + // Success/failure counters. + grouping counters { + description + "A collection of counters that were collected while evaluating + access to the target."; + + container counters { + config false; + description + "A collection of counters collected while authorizing users + accessing the target."; + leaf access-rejects { + type oc-yang:counter64; + description + "The total number of times access to the target has been + denied."; + } + leaf last-access-reject { + type oc-types:timeticks64; + description + "A timestamp of the last time access to the target has been + denied."; + } + leaf access-accepts { + type oc-yang:counter64; + description + "The total number of times access to the target has been + allowed."; + } + leaf last-access-accept { + type oc-types:timeticks64; + description + "A timestamp of the last time access to the target has been + allowed."; + } + } + } + + // GLOME related definitions. + + grouping glome-key-version { + description + "Version identifier for the configured GLOME key."; + + leaf active-glome-key-version { + type version; + description + "The version of the GLOME key."; + } + + leaf active-glome-key-created-on { + type created-on; + description + "The timestamp of the moment when the GLOME key + was created."; + } + } + + // System role SSH related definitions. + grouping user-ssh-credentials-version { + description + "System role credentials freshness-related data."; + + leaf authorized-principals-list-version { + type version; + description + "The version of the list of authorized principals currently + associated with this system role."; + } + + leaf authorized-principals-list-created-on { + type created-on; + description + "The timestamp of the moment the currently used list of + authorized principals has been created."; + } + + leaf authorized-keys-list-version { + type version; + description + "The version of the list of authorized keys that is currently + associated with this system role."; + } + + leaf authorized-keys-list-created-on { + type created-on; + description + "The timestamp of the moment the currently used list of + authorized keys has been created."; + } + } + + grouping console-config-state { + description + "Console-related configuration and state."; + container console { + description + "Console-related configuration and state."; + + container config { + description + "Console-related configuration."; + } + + container state { + config false; + description + "Console-related state."; + + uses counters; + + leaf enabled { + type boolean; + description + "Whether GLOME is enabled or not."; + } + } + } + } + // System role console related definitions. + grouping user-console-credentials-version { + description + "System role credentials freshness-related data."; + + leaf password-version { + type version; + description + "The version of the password that is currently used to + authenticate this user account."; + } + + leaf password-created-on { + type created-on; + description + "The timestamp of the moment the currently used password has + been created."; + } + } + + // Augments section. + augment "/oc-sys:system" { + description + "Console credentials freshness data."; + + uses console-config-state; + } + augment "/oc-sys:system/oc-sys:ssh-server/oc-sys:state" { + description + "SSH server credentials freshness data."; + + uses ssh-server-credentials-version; + uses counters; + } + augment "/oc-sys:system/oc-sys:aaa/oc-sys:authentication/oc-sys:users/" + + "oc-sys:user/oc-sys:state" { + description + "A system role credentials freshness information."; + + uses user-console-credentials-version; + uses user-ssh-credentials-version; + } + augment "/oc-sys:system/oc-sys:aaa/oc-sys:authentication/oc-sys:admin-user/" + + "oc-sys:state" { + description + "A system role credentials freshness information."; + + uses user-console-credentials-version; + uses user-ssh-credentials-version; + } + deviation "/oc-sys:system/oc-sys:aaa/oc-sys:authentication/oc-sys:users/" + + "oc-sys:user/oc-sys:config/oc-sys:ssh-key" { + deviate not-supported; + } + deviation "/oc-sys:system/oc-sys:aaa/oc-sys:authentication/oc-sys:users/" + + "oc-sys:user/oc-sys:state/oc-sys:ssh-key" { + deviate not-supported; + } +} diff --git a/models/yang/gnsi-pathz.yang b/models/yang/gnsi-pathz.yang new file mode 100644 index 000000000..ca9fa9239 --- /dev/null +++ b/models/yang/gnsi-pathz.yang @@ -0,0 +1,298 @@ +module gnsi-pathz { + yang-version 1.1; + namespace "https://github.com/openconfig/gnsi/pathz/yang"; + prefix gnsi-pathz; + + import openconfig-system { + prefix oc-sys; + } + import openconfig-system-grpc { + prefix oc-sys-grpc; + } + import openconfig-types { + prefix oc-types; + } + import openconfig-yang-types { + prefix oc-yang; + } + + organization + "Google LLC"; + + contact + "Google LLC"; + + description + "This module provides a data model for the metadata of + OpenConfig-path-based authorization policies installed on a networking + device."; + + revision 2022-10-30 { + description + "Adds success/failure counters."; + reference "0.2.0"; + } + + revision 2022-01-17 { + description + "Initial revision."; + reference "0.1.0"; + } + + typedef version { + type string; + description + "The version ID of the OpenConfig-path-based authorization policy + as provided by the OpenConfig-path-based Authorization Policy + Manager when the policy was pushed. This leaf persists through + a reboot."; + } + + typedef created-on { + type oc-types:timeticks64; + description + "The creation time of the OpenConfig-path-based authorization policy + as reported by the OpenConfig-path-based Authorization Policy + manager when the policy was pushed to the device. This value is + reported as nanoseconds since epoch (January 1st, 1970 00:00:00 GMT). + This leaf persists through a reboot."; + } + + // gRPC server related definitions. + grouping counters { + description + "A collection of counters that were collected by the gNSI.pathz + module while evaluating access to an OpenConfig path."; + + leaf access-rejects { + type oc-yang:counter64; + description + "The total number of times the gNSI.pathz module denied access + to an OpenConfig path."; + } + leaf last-access-reject { + type oc-types:timeticks64; + description + "A timestamp of the last time the gNSI.pathz denied access to + an OpenConfig path"; + } + leaf access-accepts { + type oc-yang:counter64; + description + "The total number of times the gNSI.pathz module allowed access + to an OpenConfig path."; + } + leaf last-access-accept { + type oc-types:timeticks64; + description + "A timestamp of the last time the gNSI.pathz allowed access to + an OpenConfig path"; + } + } + + grouping gnmi-pathz-policy-success-failure-counters { + description + "A collection of counters collected by the gNSI.pathz module."; + + container gnmi-pathz-policy-counters { + config false; + + uses gnmi-pathz-policy-xpath-success-failure-counters; + } + } + + grouping gnmi-pathz-policy-xpath-success-failure-counters { + description + "A collection of per-OpenConfig path counters."; + + container paths { + description + "A collection of per-OpenConfig path counters."; + + list path { + key xpath; + leaf xpath { + type leafref { + path "../state/xpath"; + } + description + "A OpenConfig schema path (xpath) the counter were + collected for."; + } + container state { + leaf xpath { + type string; + description + "A OpenConfig schema path (xpath) the counter were + collected for."; + } + container reads { + description + "The counter were collected while + performing a read operation on the + `xpath`."; + uses counters; + } + container writes { + description + "The counter were collected while + performing a write operation on the + `xpath`."; + uses counters; + } + } + } + } + } + + grouping grpc-server-gnmi-pathz-policy-state { + description + "gNMI server OpenConfig-path-based authorization policy + freshness-related data."; + + leaf gnmi-pathz-policy-version { + type version; + description + "The version of the OpenConfig-path-based authorization policy + that is used by this gNMI server."; + } + leaf gnmi-pathz-policy-created-on { + type created-on; + description + "The timestamp of the moment when the OpenConfig-path-based + authorization policy that is currently used by this gNMI server + was created."; + } + } + + grouping gnmi-pathz-policy-state { + description + "Operational state data for a gNMI OpenConfig-path-based + authorization policy."; + leaf instance { + type enumeration { + enum ACTIVE { + value 1; + description + "The policy that is currently used by the gNMI service + to authorize access."; + } + enum SANDBOX { + value 2; + description + "The most recent policy that has been uploaded during + the Rotation() RPC. If there is no Rotate() RPC in + progress, then referring to this instance of the policy + will result in an error."; + } + } + description + "The instance identifier of the gNMI OpenConfig-path-based + authorization policy."; + } + leaf version { + type version; + description + "The version of the gNMI OpenConfig-path-based authorization + policy."; + } + leaf created-on { + type created-on; + description + "The timestamp of the moment when the policy was + created."; + } + } + + grouping gnmi-pathz-policies { + description + "Collection of OpenConfig-path-based authorization policies that + have been installed on the device using the gNSI OpenConfig-path- + based authorization policy management service. + Each OpenConfig-path-based authorization policy listed here is + identified by its status (either ACTIVE or SANDBOX) and has its + version and creation date/time listed."; + + container policies { + config false; + description + "Information about freshness of an OpenConfig-path-based + authorization policy that have been installed + on the device using the gNSI OpenConfig-path-based + authorization policy management service."; + + list policy { + key instance; + ordered-by system; + description + "Information about the OpenConfig-path-based authorization + policy that is identified by the `instance`."; + leaf instance { + type leafref { + path "../state/instance"; + } + description + "The ID of the OpenConfig-path-based authorization + policy."; + } + container state { + description + "Operational state data for an OpenConfig-path-based + authorization policies."; + + uses gnmi-pathz-policy-state; + } + } + } + } + + grouping system-gnmi-pathz-policies { + description + "Collection of OpenConfig-path-based authorization policies that + have been installed on the device using the gNSI OpenConfig-path- + based authorization policy management service. + Each policy listed here is identified by its status (either ACTIVE + or SANDBOX) and has its version and creation date/time listed."; + + container gnmi-pathz-policies { + config false; + description + "Collection of OpenConfig-path-based authorization policies that + have been installed on the device using the gNSI OpenConfig- + path-based authorization policy management service. + Each policy listed here is identified by its status (either + ACTIVE or SANDBOX) and has its version and creation date/time + listed."; + + uses gnmi-pathz-policies; + } + } + + // Augments section. + + augment "/oc-sys:system" { + description + "Collection of OpenConfig-path-based authorization policies that + have been installed on the device using the gNSI OpenConfig-path- + based authorization policy management service. + Each policy listed here is identified by its status (either ACTIVE + or SANDBOX) and has its version and creation date/time listed."; + + uses system-gnmi-pathz-policies; + } + augment "/oc-sys:system/oc-sys-grpc:grpc-servers/oc-sys-grpc:grpc-server/" + + "oc-sys-grpc:state" { + description + "A gNMI server OpenConfig-path-based authorization policy freshness + information."; + + uses grpc-server-gnmi-pathz-policy-state; + } + augment "/oc-sys:system/oc-sys-grpc:grpc-servers/oc-sys-grpc:grpc-server" { + description + "A gNMI server OpenConfig-path-based authorization policy + success/failure counters."; + + uses gnmi-pathz-policy-success-failure-counters; + } +} diff --git a/translib/db/db_redis_opts.go b/translib/db/db_redis_opts.go index 023fd6072..27aa1193e 100644 --- a/translib/db/db_redis_opts.go +++ b/translib/db/db_redis_opts.go @@ -144,7 +144,7 @@ func adjustRedisOpts(dbOpt *Options) *redis.Options { return &redisOpts } -func init() { +func initializeRedisOpts() { flag.StringVar(&goRedisOpts, "go_redis_opts", "", "Options for go-redis") } diff --git a/translib/db/rcm.go b/translib/db/rcm.go new file mode 100644 index 000000000..1c188100b --- /dev/null +++ b/translib/db/rcm.go @@ -0,0 +1,197 @@ +package db + +import ( + "flag" + "fmt" + "sync" + "sync/atomic" + + lvl "github.com/Azure/sonic-mgmt-common/translib/log" + "github.com/go-redis/redis/v7" + log "github.com/golang/glog" +) + +var usePools = flag.Bool("use_connection_pools", true, "use connection pools for Redis Clients") + +const ( + POOL_SIZE = 25 +) + +var rcm *redisClientManager +var initMu = &sync.Mutex{} + +type redisClientManager struct { + // clients holds one Redis Client for each DBNum + clients [MaxDB + 1]*redis.Client + mu *sync.RWMutex + curTransactionalClients atomic.Int32 + totalPoolClientsRequested atomic.Uint64 + totalTransactionalClientsRequested atomic.Uint64 +} + +type RedisCounters struct { + CurTransactionalClients uint32 // The number of transactional clients currently opened. + TotalPoolClientsRequested uint64 // The total number of Redis Clients using a connection pool requested. + TotalTransactionalClientsRequested uint64 // The total number of Transactional Redis Clients requested. + PoolStatsPerDB map[string]*redis.PoolStats // The pool counters for each Redis Client in the cache. +} + +func init() { + initializeRedisOpts() + initializeRedisClientManager() +} + +func initializeRedisClientManager() { + initMu.Lock() + defer initMu.Unlock() + if rcm != nil { + return + } + rcm = &redisClientManager{ + clients: [MaxDB + 1]*redis.Client{}, + mu: &sync.RWMutex{}, + curTransactionalClients: atomic.Int32{}, + totalPoolClientsRequested: atomic.Uint64{}, + totalTransactionalClientsRequested: atomic.Uint64{}, + } + rcm.mu.Lock() + defer rcm.mu.Unlock() + for dbNum := DBNum(0); dbNum < MaxDB; dbNum++ { + if len(getDBInstName(dbNum)) == 0 { + continue + } + // Create a Redis Client for each database. + rcm.clients[int(dbNum)] = createRedisClient(dbNum, POOL_SIZE) + } +} + +func createRedisClient(db DBNum, poolSize int) *redis.Client { + opts := adjustRedisOpts(&Options{DBNo: db}) + opts.PoolSize = poolSize + client := redis.NewClient(opts) + if _, err := client.Ping().Result(); err != nil { + log.V(lvl.ERROR).Infof("RCM error during Redis Client creation for DBNum=%v: %v", db, err) + } + return client +} + +func createRedisClientWithOpts(opts *redis.Options) *redis.Client { + client := redis.NewClient(opts) + if _, err := client.Ping().Result(); err != nil { + log.V(lvl.ERROR).Infof("RCM error during Redis Client creation for DBNum=%v: %v", opts.DB, err) + } + return client +} + +func getClient(db DBNum) *redis.Client { + rcm.mu.RLock() + defer rcm.mu.RUnlock() + return rcm.clients[int(db)] +} + +// RedisClient will return a Redis Client that can be used for non-transactional Redis operations. +// The client returned by this function is shared among many DB readers/writers and uses +// a connection pool. For transactional Redis operations, please use GetRedisClientForTransaction(). +func RedisClient(db DBNum) *redis.Client { + if rcm == nil { + initializeRedisClientManager() + } + if !(*usePools) { // Connection Pooling is disabled. + return TransactionalRedisClient(db) + } + if len(getDBInstName(db)) == 0 { + log.V(lvl.ERROR).Infof("Invalid DBNum requested: %v", db) + return nil + } + rcm.totalPoolClientsRequested.Add(1) + rc := getClient(db) + if rc == nil { + log.V(lvl.ERROR).Infof("RCM Redis client for DBNum=%v is nil!", db) + rcm.mu.Lock() + defer rcm.mu.Unlock() + if rc = rcm.clients[int(db)]; rc != nil { + return rc + } + rc = createRedisClient(db, POOL_SIZE) + rcm.clients[int(db)] = rc + } + return rc +} + +// TransactionalRedisClient will create and return a unique Redis client. This client can be used +// for transactional operations. These operations include MULTI, PSUBSCRIBE (PubSub), and SCAN. This +// client must be closed using CloseRedisClient when it is no longer needed. +func TransactionalRedisClient(db DBNum) *redis.Client { + if rcm == nil { + initializeRedisClientManager() + } + if len(getDBInstName(db)) == 0 { + log.V(lvl.ERROR).Infof("Invalid DBNum requested: %v", db) + return nil + } + rcm.totalTransactionalClientsRequested.Add(1) + client := createRedisClient(db, 1) + rcm.curTransactionalClients.Add(1) + return client +} + +func TransactionalRedisClientWithOpts(opts *redis.Options) *redis.Client { + if rcm == nil { + initializeRedisClientManager() + } + rcm.totalTransactionalClientsRequested.Add(1) + opts.PoolSize = 1 + client := createRedisClientWithOpts(opts) + rcm.curTransactionalClients.Add(1) + return client +} + +// CloseUniqueRedisClient will close the Redis client that is passed in. +func CloseRedisClient(rc *redis.Client) error { + if rcm == nil { + return fmt.Errorf("RCM is nil when trying to close Redis Client: %v", rc) + } + if rc == nil { + return nil + } + // Closing a Redis Client with a connection pool is a no-op because these clients need to stay open. + if !IsTransactionalClient(rc) { + return nil + } + if err := rc.Close(); err != nil { + return err + } + rcm.curTransactionalClients.Add(-1) + return nil +} + +// IsTransactionalClient returns true if rc is a transactional client and false otherwise. +func IsTransactionalClient(rc *redis.Client) bool { + if rc == nil { + return false + } + return rc.Options().PoolSize == 1 +} + +// RedisClientManagerCounters returns the counters stored in the RCM. +func RedisClientManagerCounters() *RedisCounters { + if rcm == nil { + initializeRedisClientManager() + } + counters := &RedisCounters{ + CurTransactionalClients: uint32(rcm.curTransactionalClients.Load()), + TotalPoolClientsRequested: rcm.totalPoolClientsRequested.Load(), + TotalTransactionalClientsRequested: rcm.totalTransactionalClientsRequested.Load(), + PoolStatsPerDB: map[string]*redis.PoolStats{}, + } + rcm.mu.RLock() + defer rcm.mu.RUnlock() + for db, client := range rcm.clients { + dbName := getDBInstName(DBNum(db)) + if dbName == "" || client == nil { + continue + } + counters.PoolStatsPerDB[dbName] = client.PoolStats() + } + return counters +} diff --git a/translib/log/level.go b/translib/log/level.go new file mode 100644 index 000000000..4e057b6e2 --- /dev/null +++ b/translib/log/level.go @@ -0,0 +1,10 @@ +package log + +import "github.com/golang/glog" + +const ( + ERROR glog.Level = iota + WARNING + INFO + DEBUG +) diff --git a/translib/tlerr/tlerr.go b/translib/tlerr/tlerr.go index 93bdbeb05..c30bdb8b2 100644 --- a/translib/tlerr/tlerr.go +++ b/translib/tlerr/tlerr.go @@ -31,10 +31,12 @@ package tlerr import ( // "fmt" + "errors" "github.com/Azure/sonic-mgmt-common/cvl" + lvl "github.com/Azure/sonic-mgmt-common/translib/log" + "github.com/golang/glog" "golang.org/x/text/language" "golang.org/x/text/message" - // "errors" // "strings" ) @@ -190,3 +192,32 @@ type TranslibBusy struct { func (e TranslibBusy) Error() string { return p.Sprintf("Translib Busy") } + +func IsTranslibRedisClientEntryNotExist(err error) bool { + switch err.(type) { + case *TranslibRedisClientEntryNotExist, TranslibRedisClientEntryNotExist: + return true + } + return false +} + +// isDBEntryNotExistError returns `true` if `err` is (or wraps around) an error +// of type `TranslibRedisClientEntryNotExist`. +func isDBEntryNotExistError(err error) bool { + if IsTranslibRedisClientEntryNotExist(err) { + return true + } + pdberr := &TranslibRedisClientEntryNotExist{} + return errors.As(err, &TranslibRedisClientEntryNotExist{}) || errors.As(err, &pdberr) +} + +// ErrorSeverity based on `err` calculates the VLOG level. +func ErrorSeverity(err error) glog.Level { + if err == nil { + return lvl.DEBUG + } + if isDBEntryNotExistError(err) { + return lvl.DEBUG + } + return lvl.ERROR +} diff --git a/translib/transformer/subscribe_req_xlate.go b/translib/transformer/subscribe_req_xlate.go index bc2a8cf13..27ab34136 100644 --- a/translib/transformer/subscribe_req_xlate.go +++ b/translib/transformer/subscribe_req_xlate.go @@ -478,7 +478,7 @@ func (pathXltr *subscribePathXlator) handleSubtreeNodeXlate() error { log.Info(pathXltr.subReq.reqLogId, "handleSubtreeNodeXlate: handleSubtreeNodeXlate: uriSubtree: ", uriSubtree) } - subInParam := XfmrSubscInParams{uriSubtree, pathXltr.subReq.dbs, make(RedisDbMap), TRANSLATE_SUBSCRIBE} + subInParam := XfmrSubscInParams{uri: uriSubtree, requestURI: pathXltr.subReq.reqUri, dbs: pathXltr.subReq.dbs, dbDataMap: make(RedisDbMap), subscProc: TRANSLATE_SUBSCRIBE} subOutPram, subErr := xfmrSubscSubtreeHandler(subInParam, ygXpathInfo.xfmrFunc) if log.V(dbLgLvl) { diff --git a/translib/transformer/xfmr_interface.go b/translib/transformer/xfmr_interface.go index 5994a7956..ad48ca770 100644 --- a/translib/transformer/xfmr_interface.go +++ b/translib/transformer/xfmr_interface.go @@ -76,10 +76,11 @@ type notificationOpts struct { // XfmrSubscInParams represents input to subscribe subtree callbacks - request uri, DBs info access-pointers, DB info for request uri and subscription process type from translib. type XfmrSubscInParams struct { - uri string - dbs [db.MaxDB]*db.DB - dbDataMap RedisDbMap - subscProc SubscProcType + uri string + requestURI string + dbs [db.MaxDB]*db.DB + dbDataMap RedisDbMap + subscProc SubscProcType } // XfmrSubscOutParams represents output from subscribe subtree callback - DB data for request uri, Need cache, OnChange, subscription preference and interval. diff --git a/translib/transformer/xfmr_system.go b/translib/transformer/xfmr_system.go new file mode 100644 index 000000000..e63a31a02 --- /dev/null +++ b/translib/transformer/xfmr_system.go @@ -0,0 +1,367 @@ +package transformer + +import ( + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/Azure/sonic-mgmt-common/translib/db" + lvl "github.com/Azure/sonic-mgmt-common/translib/log" + "github.com/Azure/sonic-mgmt-common/translib/ocbinds" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" + log "github.com/golang/glog" + ygot "github.com/openconfig/ygot/ygot" +) + +const ( + PATHZ_TBL = "PATHZ_TABLE" + AUTHZ_TBL = "AUTHZ_TABLE" + BOOT_INFO_TBL = "BOOT_INFO" + READS_GET = "get" + READS_SUB = "subscribe" + WRITES = "set" + ACCEPTS = "permitted" + REJECTS = "denied" + GNXI_ID = "gnxi" + cntResult = "cntResult" + tsResult = "tsResult" + + /** Credential Tables **/ + CREDENTIALS_TBL = "CREDENTIALS" + CRED_AUTHZ_TBL = "CREDENTIALS|AUTHZ_POLICY" + CERT_TBL = "CREDENTIALS|CERT" + CONSOLE_TBL = "CREDENTIALS|CONSOLE_ACCOUNT" + CRED_PATHZ_TBL = "CREDENTIALS|PATHZ_POLICY" + SSH_TBL = "CREDENTIALS|SSH_HOST" + + /** System Root paths **/ + SYSTEM_ROOT = "/openconfig-system:system" + + /** Pathz paths **/ + GRPC_OC_SERVERS = SYSTEM_ROOT + "/openconfig-system-grpc:grpc-servers" + GRPC_SERVERS = SYSTEM_ROOT + "/grpc-servers" + GRPC_SERVER = GRPC_OC_SERVERS + "/grpc-server" + PATHZ_POLICY_COUNTERS = GRPC_SERVER + "/gnsi-pathz:gnmi-pathz-policy-counters" + ALL_PATHZ = PATHZ_POLICY_COUNTERS + "/paths" + SINGLE_PATHZ = ALL_PATHZ + "/path" + + PATHZ_STATE = SINGLE_PATHZ + "/state" + PATHZ_READS = PATHZ_STATE + "/reads" + PATHZ_WRITES = PATHZ_STATE + "/writes" + + PATHZ_READ_SUCCESS = PATHZ_READS + "/access-accepts" + PATHZ_READ_SUCCESS_TIMESTAMP = PATHZ_READS + "/last-access-accept" + PATHZ_READ_FAILED = PATHZ_READS + "/access-rejects" + PATHZ_READ_FAILED_TIMESTAMP = PATHZ_READS + "/last-access-reject" + PATHZ_WRITE_SUCCESS = PATHZ_WRITES + "/access-accepts" + PATHZ_WRITE_SUCCESS_TIMESTAMP = PATHZ_WRITES + "/last-access-accept" + PATHZ_WRITE_FAILED = PATHZ_WRITES + "/access-rejects" + PATHZ_WRITE_FAILED_TIMESTAMP = PATHZ_WRITES + "/last-access-reject" + + /** Authz paths **/ + AUTHZ_POLICY_COUNTERS = GRPC_SERVER + "/authz-policy-counters" + ALL_AUTHZ = AUTHZ_POLICY_COUNTERS + "/rpcs" + SINGLE_AUTHZ = ALL_AUTHZ + "/rpc" + AUTHZ_STATE = SINGLE_AUTHZ + "/state" + AUTHZ_SUCCESS = AUTHZ_STATE + "/access-accepts" + AUTHZ_SUCCESS_TIMESTAMP = AUTHZ_STATE + "/last-access-accept" + AUTHZ_FAILED = AUTHZ_STATE + "/access-rejects" + AUTHZ_FAILED_TIMESTAMP = AUTHZ_STATE + "/last-access-reject" +) + +// XfmrCache a sync.Map for storing path values that need to be cached +var XfmrCache sync.Map + +var pathzOpers = [][]string{[]string{READS_GET, ACCEPTS}, []string{READS_GET, REJECTS}, []string{READS_SUB, ACCEPTS}, []string{READS_SUB, REJECTS}, []string{WRITES, ACCEPTS}, []string{WRITES, REJECTS}} +var pathzMap = &pathzCounters{ + mu: sync.Mutex{}, + updated: make(map[string]time.Time), + data: make(map[string]map[string]map[string]*uint64), +} + +func init() { + XlateFuncBind("DbToYang_pathz_policies_xfmr", DbToYang_pathz_policies_xfmr) + XlateFuncBind("Subscribe_pathz_policies_xfmr", Subscribe_pathz_policies_xfmr) + XlateFuncBind("DbToYang_pathz_policies_key_xfmr", DbToYang_pathz_policies_key_xfmr) +} + +type pathzCounters struct { + mu sync.Mutex + updated map[string]time.Time + data map[string]map[string]map[string]*uint64 +} + +type grpcState struct { + name string + certVersion string + certCreated uint64 + caVersion string + caCreated uint64 + crlVersion string + crlCreated uint64 + authPolVersion string + authPolCreated uint64 + pathzVersion string + pathzCreated uint64 + profileId string +} + +type policyState struct { + instance ocbinds.E_OpenconfigSystem_System_GnmiPathzPolicies_Policies_Policy_State_Instance + version string + created uint64 +} + +var dbToYangPathzInstanceMap = map[string]ocbinds.E_OpenconfigSystem_System_GnmiPathzPolicies_Policies_Policy_State_Instance{ + "ACTIVE": ocbinds.OpenconfigSystem_System_GnmiPathzPolicies_Policies_Policy_State_Instance_ACTIVE, + "SANDBOX": ocbinds.OpenconfigSystem_System_GnmiPathzPolicies_Policies_Policy_State_Instance_SANDBOX, +} + +func getAppRootObject(inParams XfmrParams) *ocbinds.OpenconfigSystem_System { + deviceObj := (*inParams.ygRoot).(*ocbinds.Device) + return deviceObj.System +} + +func (m *pathzCounters) getCounters(pathzTables db.Table, xpath string) map[string]map[string]*uint64 { + result := make(map[string]map[string]*uint64) + m.mu.Lock() + defer m.mu.Unlock() + if m.updated == nil || m.data == nil { + m.updated = make(map[string]time.Time) + m.data = make(map[string]map[string]map[string]*uint64) + } + + // Update the map if necessary + updateTime, ok := m.updated[xpath] + if !ok { + result = GetPathzPolicyCounter(pathzTables, xpath) + if len(m.data) < 50 { + m.data[xpath] = result + m.updated[xpath] = time.Now() + } + } else if time.Now().After(updateTime.Add(30 * time.Second)) { + m.data[xpath] = GetPathzPolicyCounter(pathzTables, xpath) + m.updated[xpath] = time.Now() + } + + // Fetch the result or return the previously calculated result + if data, ok := m.data[xpath]; ok { + result = data + } + return result +} + +func GetPathzPolicyCounter(pathzTables db.Table, path string) map[string]map[string]*uint64 { + cntMap := make(map[string]*uint64) + tsMap := make(map[string]*uint64) + + for _, tmp := range pathzOpers { + pattern := PatternGenerator(tmp, path) + if pattern == "" { + log.V(lvl.DEBUG).Infof("Invalid pathz counter key pattern.") + continue + } + key := db.NewKey(tmp[0], path, tmp[1]) + + // Sum the data collected + value, err := pathzTables.GetEntry(*key) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Cannot get value from %v table for %v, err: %v", PATHZ_TBL, key, err) + continue + } + + c := value.Get("count") + if c == "" { + continue + } + dbCnt, err := strconv.ParseUint(c, 10, 64) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Failed to convert counters from DB for pathz, err: %v", err) + continue + } + tsval := value.Get("timestamp") + if tsval == "" { + continue + } + dbTs, err := strconv.ParseUint(tsval, 10, 64) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Failed to convert timestamp for counters from DB for pathz, err: %v", err) + continue + } + + cnt, cntExists := cntMap[pattern] + if cntExists && cnt != nil { + cntUpdate, err := strconv.ParseUint(strconv.FormatUint((*cnt+dbCnt), 10), 10, 64) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Failed to convert counters for pathz, err: %v", err) + continue + } + cntMap[pattern] = &cntUpdate + } else { + cntMap[pattern] = &dbCnt + } + + ts, tsExists := tsMap[pattern] + if !tsExists || ts == nil || *ts < dbTs { + tsMap[pattern] = &dbTs + } + } + return map[string]map[string]*uint64{cntResult: cntMap, tsResult: tsMap} +} + +func getAllXpaths(pathzTables db.Table) ([]string, error) { + var res []string + check := make(map[string]bool) + pathzTableKeys, err := pathzTables.GetKeys() + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Cannot get all keys from %v table, err: %v", PATHZ_TBL, err) + return []string{}, err + } + for _, pathzTableKey := range pathzTableKeys { + if len(pathzTableKey.Comp) != 3 { + log.V(lvl.DEBUG).Infof("invalid number of Comps for pathzTableKey %v.", pathzTableKey) + continue + } + if pathzTableKey.Comp[1] != "" { + key := pathzTableKey.Comp[1] + if val, ok := check[key]; !ok || !val { + res = append(res, key) + check[key] = true + } + } + } + + return res, nil +} + +var pathToPatternKeysMap = map[string][]string{ + PATHZ_READ_SUCCESS: []string{"reads", ACCEPTS}, + PATHZ_READ_SUCCESS_TIMESTAMP: []string{"reads", ACCEPTS}, + PATHZ_READ_FAILED: []string{"reads", REJECTS}, + PATHZ_READ_FAILED_TIMESTAMP: []string{"reads", REJECTS}, + PATHZ_WRITE_SUCCESS: []string{"writes", ACCEPTS}, + PATHZ_WRITE_SUCCESS_TIMESTAMP: []string{"writes", ACCEPTS}, + PATHZ_WRITE_FAILED: []string{"writes", REJECTS}, + PATHZ_WRITE_FAILED_TIMESTAMP: []string{"writes", REJECTS}, +} + +func PatternGenerator(params []string, xpath string) string { + if len(params) != 2 { + log.V(lvl.DEBUG).Infof("Invalid params for patternGenerator %#v", params) + return "" + } + + if params[0] == READS_GET || params[0] == READS_SUB || params[0] == "reads" { + return "*|reads|" + xpath + "|" + params[1] + } + + if params[0] == WRITES || params[0] == "writes" { + return "*|writes|" + xpath + "|" + params[1] + } + + log.V(lvl.DEBUG).Infof("Invalid operation %v", params[0]) + return "" +} + +var DbToYang_pathz_policies_xfmr SubTreeXfmrDbToYang = func(inParams XfmrParams) error { + pathInfo := NewPathInfo(inParams.uri) + instances := []string{pathInfo.Var("instance")} + targetUriPath, _ := getYangPathFromUri(pathInfo.Path) + log.V(lvl.DEBUG).Infof("DbToYang_pathz_policies_xfmr: targetUriPath: %s instances: %v", targetUriPath, instances) + + stateDb := inParams.dbs[db.StateDB] + if len(instances) == 0 || len(instances[0]) == 0 { + var err error + if instances, err = getAllKeys(stateDb, CRED_PATHZ_TBL); err != nil { + return err + } + } + sysObj := getAppRootObject(inParams) + ygot.BuildEmptyTree(sysObj) + ygot.BuildEmptyTree(sysObj.GnmiPathzPolicies) + ygot.BuildEmptyTree(sysObj.GnmiPathzPolicies.Policies) + + for _, instance := range instances { + log.V(lvl.DEBUG).Infof("instance: %v", instance) + i, ok := dbToYangPathzInstanceMap[instance] + if !ok { + log.V(lvl.ERROR).Infof("Pathz Policy Instance not found: %v", instance) + continue + } + policyObj, ok := sysObj.GnmiPathzPolicies.Policies.Policy[i] + if !ok { + var err error + policyObj, err = sysObj.GnmiPathzPolicies.Policies.NewPolicy(i) + if err != nil { + log.V(lvl.ERROR).Infof("sysObj.GnmiPathzPolicies.Policies.NewPolicy failed: %v", err) + continue + } + } + table, err := stateDb.GetEntry(&db.TableSpec{Name: CRED_PATHZ_TBL}, db.Key{Comp: []string{instance}}) + if err != nil { + log.V(lvl.ERROR).Infof("Failed to read from StateDB %v, id: %v, err: %v", inParams.table, instance, err) + return err + } + var state policyState + + state.instance = i + state.version = table.Get("pathz_version") + time := table.Get("pathz_created_on") + if state.created, err = strconv.ParseUint(time, 10, 64); err != nil && time != "" { + return err + } + ygot.BuildEmptyTree(policyObj) + policyObj.State.Instance = state.instance + policyObj.State.CreatedOn = &state.created + policyObj.State.Version = &state.version + } + return nil +} +var DbToYang_pathz_policies_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[string]interface{}, error) { + log.V(lvl.DEBUG).Info("DbToYang_pathz_policies_key_xfmr root, uri: ", inParams.ygRoot, inParams.uri) + + return map[string]interface{}{"instance": NewPathInfo(inParams.uri).Var("instance")}, nil +} + +var Subscribe_pathz_policies_xfmr SubTreeXfmrSubscribe = func(inParams XfmrSubscInParams) (XfmrSubscOutParams, error) { + pathInfo := NewPathInfo(inParams.uri) + instance := pathInfo.Var("instance") + if instance == "" { + instance = "*" + } + targetUriPath, _ := getYangPathFromUri(pathInfo.Path) + log.V(lvl.DEBUG).Infof("Subscribe_pathz_policies_xfmr: targetUriPath: %s instance: %s", targetUriPath, instance) + + key := strings.Join([]string{"PATHZ_POLICY", instance}, "|") + return XfmrSubscOutParams{ + dbDataMap: RedisDbSubscribeMap{ + db.StateDB: {CREDENTIALS_TBL: {key: {}}}}, + onChange: OnchangeEnable, + nOpts: ¬ificationOpts{mInterval: 0, pType: OnChange}, + }, nil +} + +func getAllKeys(sdb *db.DB, tblName string) ([]string, error) { + tbl, err := sdb.GetTable(&db.TableSpec{Name: tblName}) + if err != nil { + return nil, fmt.Errorf("Can't get table: %v, err: %v", tblName, err) + } + log.V(lvl.DEBUG).Infof("tbl: %v", tbl) + keys, err := tbl.GetKeys() + if err != nil { + return nil, fmt.Errorf("Can't get keys from %v, err: %v", tblName, err) + } + log.V(lvl.DEBUG).Infof("tbl keys: %v", keys) + ret := []string{} + for _, key := range keys { + if len(key.Comp) != 3 { + // This is a fanthom key. Ignore it. + continue + } + ret = append(ret, key.Comp[2]) + } + log.V(lvl.DEBUG).Infof("keys: %v", ret) + return ret, nil +}