Skip to content

Commit 56150c3

Browse files
laveryalaverya
committed
add support for redacting items within yaml documents
1 parent 8f594e8 commit 56150c3

File tree

7 files changed

+313
-6
lines changed

7 files changed

+313
-6
lines changed

‎pkg/collect/collector_test.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,32 @@ pwd=somethinggoeshere;`,
176176
"data/data/collectorname": `***HIDDEN*** ***HIDDEN***
177177
***HIDDEN*** line here
178178
pwd=***HIDDEN***;
179+
`,
180+
},
181+
},
182+
{
183+
name: "data with custom yaml redactor",
184+
Collect: &troubleshootv1beta1.Collect{
185+
Data: &troubleshootv1beta1.Data{
186+
CollectorMeta: troubleshootv1beta1.CollectorMeta{
187+
CollectorName: "datacollectorname",
188+
Exclude: multitype.BoolOrString{},
189+
},
190+
Name: "data",
191+
Data: `abc 123
192+
another line here`,
193+
},
194+
},
195+
Redactors: []*troubleshootv1beta1.Redact{
196+
{
197+
Yaml: []string{
198+
`abc`,
199+
},
200+
},
201+
},
202+
want: map[string]string{
203+
"data/datacollectorname": `abc 123
204+
another line here
179205
`,
180206
},
181207
},

‎pkg/redact/literal.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ func literalString(matchString string) Redactor {
1616
}
1717

1818
func (r literalRedactor) Redact(input io.Reader) io.Reader {
19-
reader, writer := io.Pipe()
19+
out, writer := io.Pipe()
2020

2121
go func() {
2222
var err error
@@ -43,5 +43,5 @@ func (r literalRedactor) Redact(input io.Reader) io.Reader {
4343
}
4444
}
4545
}()
46-
return reader
46+
return out
4747
}

‎pkg/redact/multi_line.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ func NewMultiLineRedactor(re1, re2, maskText string) (*MultiLineRedactor, error)
2626
}
2727

2828
func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
29-
reader, writer := io.Pipe()
29+
out, writer := io.Pipe()
3030
go func() {
3131
var err error
3232
defer func() {
@@ -70,7 +70,7 @@ func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
7070
fmt.Fprintf(writer, "%s\n", line1)
7171
}
7272
}()
73-
return reader
73+
return out
7474
}
7575

7676
func getNextTwoLines(reader *bufio.Reader, curLine2 *string) (line1 string, line2 string, err error) {

‎pkg/redact/redact.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact
8181
}
8282
additionalRedactors = append(additionalRedactors, r)
8383
}
84+
85+
for _, yaml := range redact.Yaml {
86+
r := NewYamlRedactor(yaml)
87+
additionalRedactors = append(additionalRedactors, r)
88+
}
8489
}
8590
return additionalRedactors, nil
8691
}

‎pkg/redact/single_line.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ func NewSingleLineRedactor(re, maskText string) (*SingleLineRedactor, error) {
2121
}
2222

2323
func (r *SingleLineRedactor) Redact(input io.Reader) io.Reader {
24-
reader, writer := io.Pipe()
24+
out, writer := io.Pipe()
2525

2626
go func() {
2727
var err error
@@ -57,5 +57,5 @@ func (r *SingleLineRedactor) Redact(input io.Reader) io.Reader {
5757
}
5858
}
5959
}()
60-
return reader
60+
return out
6161
}

‎pkg/redact/yaml.go‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package redact
2+
3+
import (
4+
"bufio"
5+
"bytes"
6+
"io"
7+
"io/ioutil"
8+
"strconv"
9+
"strings"
10+
11+
"gopkg.in/yaml.v2"
12+
)
13+
14+
type YamlRedactor struct {
15+
maskPath []string
16+
foundMatch bool
17+
}
18+
19+
func NewYamlRedactor(yamlPath string) *YamlRedactor {
20+
pathComponents := strings.Split(yamlPath, ".")
21+
return &YamlRedactor{maskPath: pathComponents}
22+
}
23+
24+
func (r *YamlRedactor) Redact(input io.Reader) io.Reader {
25+
reader, writer := io.Pipe()
26+
go func() {
27+
var err error
28+
defer func() {
29+
if err == io.EOF {
30+
writer.Close()
31+
} else {
32+
writer.CloseWithError(err)
33+
}
34+
}()
35+
reader := bufio.NewReader(input)
36+
37+
var doc []byte
38+
doc, err = ioutil.ReadAll(reader)
39+
var yamlInterface interface{}
40+
err = yaml.Unmarshal(doc, &yamlInterface)
41+
if err != nil {
42+
buf := bytes.NewBuffer(doc)
43+
buf.WriteTo(writer)
44+
err = nil // this is not a fatal error
45+
return
46+
}
47+
48+
newYaml := r.redactYaml(yamlInterface, r.maskPath)
49+
if !r.foundMatch {
50+
// no match found, so make no changes
51+
buf := bytes.NewBuffer(doc)
52+
buf.WriteTo(writer)
53+
return
54+
}
55+
56+
var newBytes []byte
57+
newBytes, err = yaml.Marshal(newYaml)
58+
if err != nil {
59+
return
60+
}
61+
62+
buf := bytes.NewBuffer(newBytes)
63+
buf.WriteTo(writer)
64+
return
65+
}()
66+
return reader
67+
}
68+
69+
func (r *YamlRedactor) redactYaml(in interface{}, path []string) interface{} {
70+
if len(path) == 0 {
71+
r.foundMatch = true
72+
return MASK_TEXT
73+
}
74+
switch typed := in.(type) {
75+
case []interface{}:
76+
// check if first path element is * - if it is, run redact on all children
77+
if path[0] == "*" {
78+
var newArr []interface{}
79+
for _, child := range typed {
80+
newChild := r.redactYaml(child, path[1:])
81+
newArr = append(newArr, newChild)
82+
}
83+
return newArr
84+
}
85+
// check if first path element is an integer - if it is, run redact on that child
86+
pathIdx, err := strconv.Atoi(path[0])
87+
if err != nil {
88+
return typed
89+
}
90+
if len(typed) > pathIdx {
91+
child := typed[pathIdx]
92+
typed[pathIdx] = r.redactYaml(child, path[1:])
93+
return typed
94+
}
95+
return typed
96+
case map[interface{}]interface{}:
97+
child, ok := typed[path[0]]
98+
if ok {
99+
newChild := r.redactYaml(child, path[1:])
100+
typed[path[0]] = newChild
101+
}
102+
return typed
103+
default:
104+
return typed
105+
}
106+
}

‎pkg/redact/yaml_test.go‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
package redact
2+
3+
import (
4+
"bytes"
5+
"io/ioutil"
6+
"testing"
7+
8+
"github.com/stretchr/testify/require"
9+
"go.undefinedlabs.com/scopeagent"
10+
)
11+
12+
func TestNewYamlRedactor(t *testing.T) {
13+
tests := []struct {
14+
name string
15+
path []string
16+
inputString string
17+
wantString string
18+
}{
19+
{
20+
name: "object paths",
21+
path: []string{"abc", "xyz"},
22+
inputString: `
23+
abc:
24+
xyz:
25+
- a
26+
- b
27+
xyz:
28+
hello: {}`,
29+
wantString: `abc:
30+
xyz: '***HIDDEN***'
31+
xyz:
32+
hello: {}
33+
`,
34+
},
35+
{
36+
name: "one index in array",
37+
path: []string{"abc", "xyz", "0"},
38+
inputString: `
39+
abc:
40+
xyz:
41+
- a
42+
- b
43+
xyz:
44+
hello: {}`,
45+
wantString: `abc:
46+
xyz:
47+
- '***HIDDEN***'
48+
- b
49+
xyz:
50+
hello: {}
51+
`,
52+
},
53+
{
54+
name: "index after end of array",
55+
path: []string{"abc", "xyz", "10"},
56+
inputString: `
57+
abc:
58+
xyz:
59+
- a
60+
- b
61+
xyz:
62+
hello: {}`,
63+
wantString: `
64+
abc:
65+
xyz:
66+
- a
67+
- b
68+
xyz:
69+
hello: {}`,
70+
},
71+
{
72+
name: "non-integer index",
73+
path: []string{"abc", "xyz", "non-int"},
74+
inputString: `
75+
abc:
76+
xyz:
77+
- a
78+
- b
79+
xyz:
80+
hello: {}`,
81+
wantString: `
82+
abc:
83+
xyz:
84+
- a
85+
- b
86+
xyz:
87+
hello: {}`,
88+
},
89+
{
90+
name: "object paths, no matches",
91+
path: []string{"notexist", "xyz"},
92+
inputString: `
93+
abc:
94+
xyz:
95+
- a
96+
- b
97+
xyz:
98+
hello: {}`,
99+
wantString: `
100+
abc:
101+
xyz:
102+
- a
103+
- b
104+
xyz:
105+
hello: {}`,
106+
},
107+
{
108+
name: "star index in array",
109+
path: []string{"abc", "xyz", "*"},
110+
inputString: `
111+
abc:
112+
xyz:
113+
- a
114+
- b
115+
xyz:
116+
hello: {}`,
117+
wantString: `abc:
118+
xyz:
119+
- '***HIDDEN***'
120+
- '***HIDDEN***'
121+
xyz:
122+
hello: {}
123+
`,
124+
},
125+
{
126+
name: "objects within array index in array",
127+
path: []string{"abc", "xyz", "0", "a"},
128+
inputString: `
129+
abc:
130+
xyz:
131+
- a: hello
132+
- b
133+
xyz:
134+
hello: {}`,
135+
wantString: `abc:
136+
xyz:
137+
- a: '***HIDDEN***'
138+
- b
139+
xyz:
140+
hello: {}
141+
`,
142+
},
143+
{
144+
name: "non-yaml file",
145+
path: []string{""},
146+
inputString: `hello world, this is not valid yaml: {`,
147+
wantString: `hello world, this is not valid yaml: {`,
148+
},
149+
{
150+
name: "no matches",
151+
path: []string{"abc"},
152+
inputString: `improperly-formatted: yaml`,
153+
wantString: `improperly-formatted: yaml`,
154+
},
155+
}
156+
for _, tt := range tests {
157+
t.Run(tt.name, func(t *testing.T) {
158+
scopetest := scopeagent.StartTest(t)
159+
defer scopetest.End()
160+
161+
req := require.New(t)
162+
yamlRunner := YamlRedactor{maskPath: tt.path}
163+
164+
outReader := yamlRunner.Redact(bytes.NewReader([]byte(tt.inputString)))
165+
gotBytes, err := ioutil.ReadAll(outReader)
166+
req.NoError(err)
167+
req.Equal(tt.wantString, string(gotBytes))
168+
})
169+
}
170+
}

0 commit comments

Comments
 (0)