Merge pull request #8 from replicatedhq/redact2

Adding basic redaction functionality

Author: divolgin

Makefile

@@ -1,6 +1,7 @@
 
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
+export GO111MODULE=on
 
 all: test manager
 
@@ -93,3 +94,9 @@ run-preflight: preflight
 		--collector-pullpolicy=Always \
 		--image=localhost:32000/troubleshoot:alpha \
 		--pullpolicy=Always
+
+.PHONY: run-troubleshoot
+run-troubleshoot: troubleshoot
+	./bin/troubleshoot run \
+		--image=localhost:32000/troubleshoot:alpha \
+		--pullpolicy=Always

go.mod

@@ -14,6 +14,9 @@ require (
 	github.com/nwaples/rardecode v1.0.0 // indirect
 	github.com/onsi/gomega v1.5.0
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/pmezard/go-difflib v1.0.0
+	github.com/sergi/go-diff v1.0.0
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.3.0

go.sum

@@ -85,6 +85,7 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ=
+github.com/gobuffalo/envy v1.6.15 h1:OsV5vOpHYUpP7ZLS6sem1y40/lNX1BZj+ynMiRi21lQ=
 github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
 github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
 github.com/gogo/protobuf v1.0.0 h1:2jyBKDKU/8v3v2xVR2PtiWQviFUyiaGk2rpfyFT8rTM=
@@ -162,6 +163,7 @@ github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
 github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -190,6 +192,7 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/manifoldco/promptui v0.3.2 h1:rir7oByTERac6jhpHUPErHuopoRDvO3jxS+FdadEns8=
 github.com/manifoldco/promptui v0.3.2/go.mod h1:8JU+igZ+eeiiRku4T5BjtKh2ms8sziGpSYl1gN8Bazw=
+github.com/markbates/inflect v1.0.4 h1:5fh1gzTFhfae06u3hzHYO9xe3l3v3nW5Pwt3naLTP5g=
 github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -279,9 +282,12 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2 h1:J7U/N7eRtzjhs26d6GqMh2HBuXP8/Z64Densiiieafo=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -425,6 +431,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190501045030-23463209683d h1:D7DVZUZEUgsSIDTivnUtVeGfN5AvhDIKtdIZAqx0ieE=
 golang.org/x/tools v0.0.0-20190501045030-23463209683d/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -533,6 +540,7 @@ sigs.k8s.io/controller-runtime v0.1.12/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHE
 sigs.k8s.io/controller-runtime v0.2.0-beta.2 h1:hOWldx1qmGI9TsU+uUsq1xTgVmUV7AZo08VAYX0dwGI=
 sigs.k8s.io/controller-runtime v0.2.0-beta.2/go.mod h1:TSH2R0nSz4WAlUUlNnOFcOR/VUhfwBLlmtq2X6AiQCA=
 sigs.k8s.io/controller-tools v0.1.11/go.mod h1:6g08p9m9G/So3sBc1AOQifHfhxH/mb6Sc4z0LMI8XMw=
+sigs.k8s.io/controller-tools v0.2.0-beta.2 h1:ucniFzEuW7PFfFDuUxacdY4Fy4q065wPguVl+BE2/t0=
 sigs.k8s.io/controller-tools v0.2.0-beta.2/go.mod h1:gC5UAnK1jbxWnDaqTi0yxKIsRsRwshzeRtTUGbM9vos=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/testing_frameworks v0.1.1 h1:cP2l8fkA3O9vekpy5Ks8mmA0NW/F7yBdXf8brkWhVrs=

pkg/collect/cluster_resources.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/replicatedhq/troubleshoot/pkg/redact"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1beta1clientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,7 +33,7 @@ func ClusterResources() error {
 		return err
 	}
 
-	clusterResourcesOutput := ClusterResourcesOutput{}
+	clusterResourcesOutput := &ClusterResourcesOutput{}
 
 	// namespaces
 	namespaces, namespaceList, err := namespaces(client)
@@ -91,7 +92,12 @@ func ClusterResources() error {
 	}
 	clusterResourcesOutput.CustomResourceDefinitions = customResourceDefinitions
 
-	b, err := json.MarshalIndent(clusterResourcesOutput, "", "  ")
+	redacted, err := clusterResourcesOutput.Redact()
+	if err != nil {
+		return err
+	}
+
+	b, err := json.MarshalIndent(redacted, "", "  ")
 	if err != nil {
 		return err
 	}
@@ -222,3 +228,55 @@ func crds(client *apiextensionsv1beta1clientset.ApiextensionsV1beta1Client) ([]b
 
 	return b, nil
 }
+
+func (c *ClusterResourcesOutput) Redact() (*ClusterResourcesOutput, error) {
+	namespaces, err := redact.Redact(c.Namespaces)
+	if err != nil {
+		return nil, err
+	}
+	pods, err := redactMap(c.Pods)
+	if err != nil {
+		return nil, err
+	}
+	services, err := redactMap(c.Services)
+	if err != nil {
+		return nil, err
+	}
+	deployments, err := redactMap(c.Deployments)
+	if err != nil {
+		return nil, err
+	}
+	ingress, err := redactMap(c.Ingress)
+	if err != nil {
+		return nil, err
+	}
+	storageClasses, err := redact.Redact(c.StorageClasses)
+	if err != nil {
+		return nil, err
+	}
+	crds, err := redact.Redact(c.CustomResourceDefinitions)
+	if err != nil {
+		return nil, err
+	}
+	return &ClusterResourcesOutput{
+		Namespaces:                namespaces,
+		Pods:                      pods,
+		Services:                  services,
+		Deployments:               deployments,
+		Ingress:                   ingress,
+		StorageClasses:            storageClasses,
+		CustomResourceDefinitions: crds,
+	}, nil
+}
+
+func redactMap(input map[string][]byte) (map[string][]byte, error) {
+	result := make(map[string][]byte)
+	for k, v := range input {
+		redacted, err := redact.Redact(v)
+		if err != nil {
+			return nil, err
+		}
+		result[k] = redacted
+	}
+	return result, nil
+}

pkg/redact/multi_line.go

@@ -0,0 +1,97 @@
+package redact
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"regexp"
+)
+
+type MultiLineRedactor struct {
+	re1      *regexp.Regexp
+	re2      *regexp.Regexp
+	maskText string
+}
+
+func NewMultiLineRedactor(re1, re2, maskText string) (*MultiLineRedactor, error) {
+	compiled1, err := regexp.Compile(re1)
+	if err != nil {
+		return nil, err
+	}
+	compiled2, err := regexp.Compile(re2)
+	if err != nil {
+		return nil, err
+	}
+	return &MultiLineRedactor{re1: compiled1, re2: compiled2, maskText: maskText}, nil
+}
+
+func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
+	reader, writer := io.Pipe()
+	go func() {
+		var err error
+		defer func() {
+			writer.CloseWithError(err)
+		}()
+
+		substStr := getReplacementPattern(r.re2, r.maskText)
+
+		reader := bufio.NewReader(input)
+		line1, line2, err := getNextTwoLines(reader, nil)
+		if err != nil {
+			// this will print 2 blank lines for empty input...
+			fmt.Fprintf(writer, "%s\n", line1)
+			fmt.Fprintf(writer, "%s\n", line2)
+			return
+		}
+
+		flushLastLine := false
+		for err == nil {
+			// If line1 matches re1, then transform line2 using re2
+			if !r.re1.MatchString(line1) {
+				fmt.Fprintf(writer, "%s\n", line1)
+				line1, line2, err = getNextTwoLines(reader, &line2)
+				flushLastLine = true
+				continue
+			}
+			flushLastLine = false
+
+			clean := r.re2.ReplaceAllString(line2, substStr)
+
+			// io.WriteString would be nicer, but reader strips new lines
+			fmt.Fprintf(writer, "%s\n%s\n", line1, clean)
+			if err != nil {
+				return
+			}
+
+			line1, line2, err = getNextTwoLines(reader, nil)
+		}
+
+		if flushLastLine {
+			fmt.Fprintf(writer, "%s\n", line1)
+		}
+	}()
+	return reader
+}
+
+func getNextTwoLines(reader *bufio.Reader, curLine2 *string) (line1 string, line2 string, err error) {
+	line1 = ""
+	line2 = ""
+
+	if curLine2 == nil {
+		line1, err = readLine(reader)
+		if err != nil {
+			return
+		}
+
+		line2, err = readLine(reader)
+		return
+	}
+
+	line1 = *curLine2
+	line2, err = readLine(reader)
+	if err != nil {
+		return
+	}
+
+	return
+}