Merge pull request #25 from vimeo/add-gsm-support

Adds support for Google Secrets Manager

Author: willroberts

.github/workflows/go.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       matrix:
         os: [macOS-latest, ubuntu-latest]
-        goversion: [1.19, "1.20"]
+        goversion: ["1.24"]
     steps:
       - name: Set up Go ${{matrix.goversion}} on ${{matrix.os}}
         uses: actions/setup-go@v3

Dockerfile

@@ -1,12 +1,9 @@
-FROM golang:1.18.3-alpine as builder
+FROM golang:1.24-alpine AS builder
 
 RUN apk add --no-cache ca-certificates libc-dev git make gcc
 RUN adduser -D pentagon
 USER pentagon
 
-# Enable go modules
-ENV GO111MODULE on
-
 # The golang docker images configure GOPATH=/go
 RUN mkdir -p /go/src/github.com/vimeo/pentagon /go/pkg/
 COPY --chown=pentagon . /go/src/github.com/vimeo/pentagon/

README.md

@@ -2,15 +2,13 @@
 [![GoDoc](https://godoc.org/github.com/vimeo/pentagon?status.svg)](https://godoc.org/github.com/vimeo/pentagon) [![Go Report Card](https://goreportcard.com/badge/github.com/vimeo/pentagon)](https://goreportcard.com/report/github.com/vimeo/pentagon) 
 
 # Pentagon
-Pentagon is a small application designed to run as a Kubernetes CronJob to periodically copy secrets stored in [Vault](https://www.vaultproject.io) into equivalent [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/), keeping them synchronized.  Naturally, this should be used with care as "standard" Kubernetes Secrets are simply obfuscated as base64-encoded strings.  However, one can and should use more secure methods of securing secrets including Google's [KMS](https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets) and restricting roles and service accounts appropriately.
-
-Use at your own risk...
+Pentagon is a small application designed to run as a Kubernetes CronJob to periodically copy secrets stored in [Vault](https://www.vaultproject.io) or Google Secrets Manager into equivalent [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/), keeping them synchronized.  Naturally, this should be used with care as "standard" Kubernetes Secrets are simply obfuscated as base64-encoded strings.  However, one can and should use more secure methods of securing secrets including Google's [KMS](https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets) and restricting roles and service accounts appropriately.
 
 ## Why not just query Vault?
 That's a good question.  If you have a highly-available Vault setup that is stable and performant and you're able to modify your applications to query Vault, that's a completely reasonable approach to take.  If you don't have such a setup, Pentagon provides a way to cache things securely in Kubernetes secrets which can then be provided to applications without directly introducing a Vault dependency.
 
 ## Configuration
-Pentagon requires a simple YAML configuration file, the path to which should be passed as the first and only argument to the application.  It is recommended that you store this configuration in a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) and reference it in the CronJob specification.  A sample configuration follows:
+Pentagon requires a YAML configuration file, the path to which should be passed as the first and only argument to the application.  It is recommended that you store this configuration in a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) and reference it in the CronJob specification.  A sample configuration follows:
 
 ```yaml
 vault:
@@ -28,6 +26,10 @@ mappings:
     secretName: k8s-secretname
     vaultEngineType: # optionally "kv" or "kv-v2" to override the defaultEngineType specified above
     secretType: Opaque # optionally - default "Opaque" e.g.: "kubernetes.io/tls"
+  # mappings from google secrets manager paths to kubernetes secret names
+  - sourceType: gsm
+    path: projects/my-project/secrets/my-secret/versions/latest
+    secretName: my-secret
 ```
 
 ### Labels and Reconciliation
@@ -92,6 +94,7 @@ The application will return 0 on success (when all keys were copied/updated succ
 | 22 | Configuration error. |
 | 30 | Unable to instantiate vault client. |
 | 31 | Unable to instantiate kubernetes client. |
+| 32 | Unable to instantiate Google Secrets Manager client. |
 | 40 | Error copying keys. |
 
 ## Kubernetes Configuration

config.go

@@ -2,23 +2,32 @@ package pentagon
 
 import (
 	"fmt"
+	"log"
 
 	"github.com/hashicorp/vault/api"
 	"github.com/vimeo/pentagon/vault"
 
 	corev1 "k8s.io/api/core/v1"
 )
 
-// DefaultNamespace is the default kubernetes namespace.
-const DefaultNamespace = "default"
+const (
+	// DefaultNamespace is the default kubernetes namespace.
+	DefaultNamespace = "default"
 
-// DefaultLabelValue is the default label value that will be applied to secrets
-// created by pentagon.
-const DefaultLabelValue = "default"
+	// DefaultLabelValue is the default label value that will be applied to secrets
+	// created by pentagon.
+	DefaultLabelValue = "default"
 
-// Config describes the configuration for vaultofsecrets
+	// VaultSourceType indicates a mapping sourced from Hashicorp Vault.
+	VaultSourceType = "vault"
+
+	// GSMSourceType indicates a mapping sourced from Google Secrets Manager.
+	GSMSourceType = "gsm"
+)
+
+// Config describes the configuration for Pentagon.
 type Config struct {
-	// VaultURL is the URL used to connect to vault.
+	// Vault is the configuration used to connect to vault.
 	Vault VaultConfig `yaml:"vault"`
 
 	// Namespace is the k8s namespace that the secrets will be created in.
@@ -51,6 +60,17 @@ func (c *Config) SetDefaults() {
 	// set all the underlying mapping engine types to their default
 	// if unspecified
 	for i, m := range c.Mappings {
+		// default to vault source type for backward compatibility
+		if m.SourceType == "" {
+			c.Mappings[i].SourceType = VaultSourceType
+		}
+
+		// copy VaultPath to Path for backward compatibility
+		if m.Path == "" && m.VaultPath != "" {
+			log.Println("WARNING: Use mapping.Path instead of mapping.VaultPath (deprecated)")
+			c.Mappings[i].Path = m.VaultPath
+		}
+
 		if m.VaultEngineType == "" {
 			c.Mappings[i].VaultEngineType = c.Vault.DefaultEngineType
 		}
@@ -66,6 +86,21 @@ func (c *Config) Validate() error {
 		return fmt.Errorf("no mappings provided")
 	}
 
+	validSourceTypes := map[string]struct{}{
+		"":              {},
+		VaultSourceType: {},
+		GSMSourceType:   {},
+	}
+
+	for _, m := range c.Mappings {
+		if _, ok := validSourceTypes[m.SourceType]; !ok {
+			return fmt.Errorf("invalid source type: %+v", m.SourceType)
+		}
+		if m.Path == "" {
+			return fmt.Errorf("path should not be empty: %+v", m)
+		}
+	}
+
 	return nil
 }
 
@@ -99,7 +134,16 @@ type VaultConfig struct {
 
 // Mapping is a single mapping for a vault secret to a k8s secret.
 type Mapping struct {
-	// VaultPath is the path to the vault secret.
+	// SourceType is the source of a secret: Vault or GSM. Defaults to Vault.
+	SourceType string `yaml:"sourceType"`
+
+	// Path is the path to a Vault or GSM secret.
+	// GSM secrets can use one of the following forms;
+	// - projects/*/secrets/*/versions/*
+	// - projects/*/locations/*/secrets/*/versions/*
+	Path string `yaml:"path"`
+
+	// [DEPRECATED] VaultPath is the path to a vault secret. Use Path instead.
 	VaultPath string `yaml:"vaultPath"`
 
 	// SecretName is the name of the k8s secret that the vault contents should

config_test.go

@@ -13,7 +13,11 @@ func TestSetDefaults(t *testing.T) {
 		},
 		Mappings: []Mapping{
 			{
-				VaultPath:  "vaultPath",
+				Path:       "path",
+				SecretName: "theSecret",
+			},
+			{
+				VaultPath:  "path",
 				SecretName: "theSecret",
 			},
 		},
@@ -33,6 +37,12 @@ func TestSetDefaults(t *testing.T) {
 	}
 
 	for _, m := range c.Mappings {
+		if m.SourceType != VaultSourceType {
+			t.Fatalf("source type should have defaulted to vault: %+v", m)
+		}
+		if m.Path == "" {
+			t.Fatalf("empty path for vault secret: %+v", m)
+		}
 		if m.VaultEngineType == "" {
 			t.Fatalf("empty vault engine type for mapping: %+v", m)
 		}
@@ -80,7 +90,7 @@ func TestValidate(t *testing.T) {
 	c = &Config{
 		Mappings: []Mapping{
 			{
-				VaultPath:  "foo",
+				Path:       "foo",
 				SecretName: "bar",
 			},
 		},
@@ -91,3 +101,27 @@ func TestValidate(t *testing.T) {
 		t.Fatalf("configuration should have been valid: %s", err)
 	}
 }
+
+func TestValidSourceTypes(t *testing.T) {
+	c := &Config{
+		Mappings: []Mapping{
+			{SourceType: "", Path: "foo"},
+			{SourceType: VaultSourceType, Path: "foo"},
+			{SourceType: GSMSourceType, Path: "foo"},
+		},
+	}
+	if err := c.Validate(); err != nil {
+		t.Fatalf("mappings should have been valid: %s", err)
+	}
+}
+
+func TestInvalidSourceType(t *testing.T) {
+	c := &Config{
+		Mappings: []Mapping{
+			{SourceType: "foo"},
+		},
+	}
+	if err := c.Validate(); err == nil {
+		t.Fatalf("failed to detect invalid mapping source type")
+	}
+}

go.mod

@@ -1,9 +1,13 @@
 module github.com/vimeo/pentagon
 
-go 1.18
+go 1.23.0
+
+toolchain go1.24.2
 
 require (
-	cloud.google.com/go/compute/metadata v0.2.3
+	cloud.google.com/go/compute/metadata v0.7.0
+	cloud.google.com/go/secretmanager v1.15.0
+	github.com/googleapis/gax-go/v2 v2.14.2
 	github.com/hashicorp/vault/api v1.9.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.26.3
@@ -12,22 +16,28 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute v1.19.0 // indirect
+	cloud.google.com/go/auth v0.16.2 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/iam v1.5.2 // indirect
 	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/fatih/color v1.15.0 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
@@ -50,15 +60,26 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.7.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/oauth2 v0.6.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	golang.org/x/crypto v0.39.0 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/sync v0.15.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
+	google.golang.org/api v0.237.0 // indirect
+	google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
+	google.golang.org/grpc v1.73.0 // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect