Skip to content

Commit 130a339

Browse files
sudo-suhassudo-suhas
authored
fix(gen): format, fix sample configs used for generating recipes (#406)
- Introduce and use template helper function on sample configs of sinks, extractors and processors to replace tabs, if present, with spaces in sample configs. If tabs are present, parsing the file as YAML fails. - Fix sample configs for sinks/http, extractors/{kafka, tableau}. - Reformat sample configs for consistency.
1 parent 11a0192 commit 130a339

21 files changed

Lines changed: 73 additions & 67 deletions

File tree

‎generator/recipe.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ type templateData struct {
2727
}
2828

2929
var templateFuncs = map[string]interface{}{
30-
"indent": indent,
30+
"indent": indent,
31+
"fmtSampleConfig": fmtSampleConfig,
3132
}
3233

3334
var recipeVersions = [1]string{"v1beta1"}
@@ -93,6 +94,14 @@ func indent(spaces int, v string) string {
9394
return pad + strings.Replace(v, "\n", "\n"+pad, -1)
9495
}
9596

97+
func fmtSampleConfig(s string) string {
98+
if !strings.HasPrefix(s, "\n") {
99+
s = "\n" + s
100+
}
101+
102+
return strings.ReplaceAll(s, "\t", " ")
103+
}
104+
96105
func GetRecipeVersions() [1]string {
97106
return recipeVersions
98107
}

‎generator/recipe.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@ source:
44
{{- with .Source }}
55
name: {{.Name}}
66
scope: {{.Scope}}
7-
config: {{.SampleConfig | indent 4}}
7+
config: {{.SampleConfig | fmtSampleConfig | indent 4}}
88
{{- end }}
99
{{- if ne (len .Sinks) 0 }}
1010
sinks:
1111
{{- range $key, $value := .Sinks }}
1212
- name: {{$key}}
1313
{{- if $value}}
14-
config: {{$value | indent 6}}
14+
config: {{$value | fmtSampleConfig | indent 6}}
1515
{{- end }}
1616
{{- end }}
1717
{{- end }}
@@ -20,7 +20,7 @@ processors:
2020
{{- range $key, $value := .Processors }}
2121
- name: {{$key}}
2222
{{- if $value}}
23-
config: {{$value | indent 6}}
23+
config: {{$value | fmtSampleConfig | indent 6}}
2424
{{- end }}
2525
{{- end }}
2626
{{- end }}

‎plugins/extractors/clickhouse/clickhouse.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ type Config struct {
2525
ConnectionURL string `mapstructure:"connection_url" validate:"required"`
2626
}
2727

28-
var sampleConfig = `
29-
connection_url: "tcp://localhost:3306?username=admin&password=pass123&debug=true"`
28+
var sampleConfig = `connection_url: "tcp://localhost:3306?username=admin&password=pass123&debug=true"`
3029

3130
var info = plugins.Info{
3231
Description: "Column-oriented DBMS for online analytical processing.",
@@ -108,10 +107,10 @@ func (e *Extractor) extractTables(emit plugins.Emit) (err error) {
108107

109108
asset := v1beta2.Asset{
110109
Urn: models.NewURN("clickhouse", e.UrnScope, "table", fmt.Sprintf("%s.%s", dbName, tableName)),
111-
Name: tableName,
112-
Type: "table",
110+
Name: tableName,
111+
Type: "table",
113112
Service: "clickhouse",
114-
Data: table,
113+
Data: table,
115114
}
116115
emit(models.NewRecord(&asset))
117116
}

‎plugins/extractors/csv/csv.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package csv
33
import (
44
"context"
55
_ "embed" // used to print the embedded assets
6+
"encoding/csv"
67
"fmt"
78
"io"
89
"io/ioutil"
@@ -16,8 +17,6 @@ import (
1617
"github.com/pkg/errors"
1718
"google.golang.org/protobuf/types/known/anypb"
1819

19-
"encoding/csv"
20-
2120
"github.com/odpf/meteor/plugins"
2221
"github.com/odpf/salt/log"
2322
)
@@ -30,8 +29,7 @@ type Config struct {
3029
Path string `mapstructure:"path" validate:"required"`
3130
}
3231

33-
var sampleConfig = `
34-
path: ./path-to-a-file-or-a-directory`
32+
var sampleConfig = `path: ./path-to-a-file-or-a-directory`
3533

3634
var info = plugins.Info{
3735
Description: "Comma separated file",

‎plugins/extractors/elastic/elastic.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ type Config struct {
2828
}
2929

3030
var sampleConfig = `
31-
user: "elastic"
32-
password: "changeme"
33-
host: elastic_server`
31+
user: "elastic"
32+
password: "changeme"
33+
host: elastic_server`
3434

3535
var info = plugins.Info{
3636
Description: "Search engine based on the Lucene library.",
@@ -63,7 +63,7 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error)
6363
return err
6464
}
6565

66-
//build elasticsearch client
66+
// build elasticsearch client
6767
cfg := elasticsearch.Config{
6868
Addresses: []string{
6969
e.config.Host,

‎plugins/extractors/kafka/kafka.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2"
1212
"github.com/odpf/meteor/plugins"
1313
"github.com/odpf/meteor/registry"
14-
kafka "github.com/segmentio/kafka-go"
14+
"github.com/segmentio/kafka-go"
1515

1616
"github.com/odpf/salt/log"
1717
)
@@ -30,8 +30,7 @@ type Config struct {
3030
Broker string `mapstructure:"broker" validate:"required"`
3131
}
3232

33-
var sampleConfig = `
34-
broker: "localhost:9092"`
33+
var sampleConfig = `broker: "localhost:9092"`
3534

3635
var info = plugins.Info{
3736
Description: "Topic list from Apache Kafka.",

‎plugins/extractors/metabase/metabase.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ var summary string
2525
var sampleConfig = `
2626
host: http://localhost:3000
2727
instance_label: my-metabase
28-
user_id: meteor_tester
28+
username: meteor_tester
2929
password: meteor_pass_1234`
3030

3131
var info = plugins.Info{

‎plugins/extractors/mongodb/mongodb.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,7 @@ type Config struct {
3434
ConnectionURL string `mapstructure:"connection_url" validate:"required"`
3535
}
3636

37-
var sampleConfig = `
38-
connection_url: "mongodb://admin:pass123@localhost:3306"`
37+
var sampleConfig = `connection_url: "mongodb://admin:pass123@localhost:3306"`
3938

4039
var info = plugins.Info{
4140
Description: "Collection metadata from MongoDB Server",
@@ -144,10 +143,10 @@ func (e *Extractor) buildTable(ctx context.Context, db *mongo.Database, collecti
144143
//
145144
table = &v1beta2.Asset{
146145
Urn: models.NewURN("mongodb", e.UrnScope, "collection", fmt.Sprintf("%s.%s", db.Name(), collectionName)),
147-
Name: collectionName,
146+
Name: collectionName,
148147
Service: "mongodb",
149-
Type: "table",
150-
Data: data,
148+
Type: "table",
149+
Data: data,
151150
}
152151

153152
return

‎plugins/extractors/mssql/mssql.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,7 @@ type Config struct {
3636
ConnectionURL string `mapstructure:"connection_url" validate:"required"`
3737
}
3838

39-
var sampleConfig = `
40-
connection_url: "sqlserver://admin:pass123@localhost:3306/"`
39+
var sampleConfig = `connection_url: "sqlserver://admin:pass123@localhost:3306/"`
4140

4241
var info = plugins.Info{
4342
Description: "Table metdata from MSSQL server",
@@ -131,10 +130,10 @@ func (e *Extractor) processTable(database string, tableName string) (err error)
131130
// push table to channel
132131
e.emit(models.NewRecord(&v1beta2.Asset{
133132
Urn: models.NewURN("mssql", e.UrnScope, "table", fmt.Sprintf("%s.%s", database, tableName)),
134-
Name: tableName,
135-
Type: "table",
133+
Name: tableName,
134+
Type: "table",
136135
Service: "mssql",
137-
Data: table,
136+
Data: table,
138137
}))
139138

140139
return

‎plugins/extractors/mysql/mysql.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,7 @@ type Config struct {
3535
ConnectionURL string `mapstructure:"connection_url" validate:"required"`
3636
}
3737

38-
var sampleConfig = `
39-
connection_url: "admin:pass123@tcp(localhost:3306)/"`
38+
var sampleConfig = `connection_url: "admin:pass123@tcp(localhost:3306)/"`
4039

4140
var info = plugins.Info{
4241
Description: "Table metadata from MySQL server.",
@@ -144,10 +143,10 @@ func (e *Extractor) processTable(database string, tableName string) (err error)
144143
// push table to channel
145144
e.emit(models.NewRecord(&v1beta2.Asset{
146145
Urn: models.NewURN("mysql", e.UrnScope, "table", fmt.Sprintf("%s.%s", database, tableName)),
147-
Name: tableName,
148-
Type: "table",
146+
Name: tableName,
147+
Type: "table",
149148
Service: "mysql",
150-
Data: table,
149+
Data: table,
151150
}))
152151

153152
return

0 commit comments

Comments
 (0)