diff --git a/generator/recipe.go b/generator/recipe.go index 40aabc0aa..e4fd71907 100644 --- a/generator/recipe.go +++ b/generator/recipe.go @@ -27,7 +27,8 @@ type templateData struct { } var templateFuncs = map[string]interface{}{ - "indent": indent, + "indent": indent, + "fmtSampleConfig": fmtSampleConfig, } var recipeVersions = [1]string{"v1beta1"} @@ -93,6 +94,14 @@ func indent(spaces int, v string) string { return pad + strings.Replace(v, "\n", "\n"+pad, -1) } +func fmtSampleConfig(s string) string { + if !strings.HasPrefix(s, "\n") { + s = "\n" + s + } + + return strings.ReplaceAll(s, "\t", " ") +} + func GetRecipeVersions() [1]string { return recipeVersions } diff --git a/generator/recipe.yaml b/generator/recipe.yaml index 685dd5fa3..ba5b0653a 100644 --- a/generator/recipe.yaml +++ b/generator/recipe.yaml @@ -4,14 +4,14 @@ source: {{- with .Source }} name: {{.Name}} scope: {{.Scope}} - config: {{.SampleConfig | indent 4}} + config: {{.SampleConfig | fmtSampleConfig | indent 4}} {{- end }} {{- if ne (len .Sinks) 0 }} sinks: {{- range $key, $value := .Sinks }} - name: {{$key}} {{- if $value}} - config: {{$value | indent 6}} + config: {{$value | fmtSampleConfig | indent 6}} {{- end }} {{- end }} {{- end }} @@ -20,7 +20,7 @@ processors: {{- range $key, $value := .Processors }} - name: {{$key}} {{- if $value}} - config: {{$value | indent 6}} + config: {{$value | fmtSampleConfig | indent 6}} {{- end }} {{- end }} {{- end }} diff --git a/plugins/extractors/clickhouse/clickhouse.go b/plugins/extractors/clickhouse/clickhouse.go index 12e7ab1aa..62e9284cc 100644 --- a/plugins/extractors/clickhouse/clickhouse.go +++ b/plugins/extractors/clickhouse/clickhouse.go @@ -25,8 +25,7 @@ type Config struct { ConnectionURL string `mapstructure:"connection_url" validate:"required"` } -var sampleConfig = ` -connection_url: "tcp://localhost:3306?username=admin&password=pass123&debug=true"` +var sampleConfig = `connection_url: "tcp://localhost:3306?username=admin&password=pass123&debug=true"` var info = plugins.Info{ Description: "Column-oriented DBMS for online analytical processing.", @@ -108,10 +107,10 @@ func (e *Extractor) extractTables(emit plugins.Emit) (err error) { asset := v1beta2.Asset{ Urn: models.NewURN("clickhouse", e.UrnScope, "table", fmt.Sprintf("%s.%s", dbName, tableName)), - Name: tableName, - Type: "table", + Name: tableName, + Type: "table", Service: "clickhouse", - Data: table, + Data: table, } emit(models.NewRecord(&asset)) } diff --git a/plugins/extractors/csv/csv.go b/plugins/extractors/csv/csv.go index 8a4e0dd0c..505c351c1 100644 --- a/plugins/extractors/csv/csv.go +++ b/plugins/extractors/csv/csv.go @@ -3,6 +3,7 @@ package csv import ( "context" _ "embed" // used to print the embedded assets + "encoding/csv" "fmt" "io" "io/ioutil" @@ -16,8 +17,6 @@ import ( "github.com/pkg/errors" "google.golang.org/protobuf/types/known/anypb" - "encoding/csv" - "github.com/odpf/meteor/plugins" "github.com/odpf/salt/log" ) @@ -30,8 +29,7 @@ type Config struct { Path string `mapstructure:"path" validate:"required"` } -var sampleConfig = ` -path: ./path-to-a-file-or-a-directory` +var sampleConfig = `path: ./path-to-a-file-or-a-directory` var info = plugins.Info{ Description: "Comma separated file", diff --git a/plugins/extractors/elastic/elastic.go b/plugins/extractors/elastic/elastic.go index 7720a200c..841a976b0 100644 --- a/plugins/extractors/elastic/elastic.go +++ b/plugins/extractors/elastic/elastic.go @@ -28,9 +28,9 @@ type Config struct { } var sampleConfig = ` - user: "elastic" - password: "changeme" - host: elastic_server` +user: "elastic" +password: "changeme" +host: elastic_server` var info = plugins.Info{ Description: "Search engine based on the Lucene library.", @@ -63,7 +63,7 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error) return err } - //build elasticsearch client + // build elasticsearch client cfg := elasticsearch.Config{ Addresses: []string{ e.config.Host, diff --git a/plugins/extractors/kafka/kafka.go b/plugins/extractors/kafka/kafka.go index 9d99c0284..a25f14e1a 100644 --- a/plugins/extractors/kafka/kafka.go +++ b/plugins/extractors/kafka/kafka.go @@ -11,7 +11,7 @@ import ( v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" "github.com/odpf/meteor/plugins" "github.com/odpf/meteor/registry" - kafka "github.com/segmentio/kafka-go" + "github.com/segmentio/kafka-go" "github.com/odpf/salt/log" ) @@ -30,8 +30,7 @@ type Config struct { Broker string `mapstructure:"broker" validate:"required"` } -var sampleConfig = ` -broker: "localhost:9092"` +var sampleConfig = `broker: "localhost:9092"` var info = plugins.Info{ Description: "Topic list from Apache Kafka.", diff --git a/plugins/extractors/metabase/metabase.go b/plugins/extractors/metabase/metabase.go index 6705a9483..d712df02f 100644 --- a/plugins/extractors/metabase/metabase.go +++ b/plugins/extractors/metabase/metabase.go @@ -25,7 +25,7 @@ var summary string var sampleConfig = ` host: http://localhost:3000 instance_label: my-metabase -user_id: meteor_tester +username: meteor_tester password: meteor_pass_1234` var info = plugins.Info{ diff --git a/plugins/extractors/mongodb/mongodb.go b/plugins/extractors/mongodb/mongodb.go index 49c7da3ce..c850c5fcb 100644 --- a/plugins/extractors/mongodb/mongodb.go +++ b/plugins/extractors/mongodb/mongodb.go @@ -34,8 +34,7 @@ type Config struct { ConnectionURL string `mapstructure:"connection_url" validate:"required"` } -var sampleConfig = ` -connection_url: "mongodb://admin:pass123@localhost:3306"` +var sampleConfig = `connection_url: "mongodb://admin:pass123@localhost:3306"` var info = plugins.Info{ Description: "Collection metadata from MongoDB Server", @@ -144,10 +143,10 @@ func (e *Extractor) buildTable(ctx context.Context, db *mongo.Database, collecti // table = &v1beta2.Asset{ Urn: models.NewURN("mongodb", e.UrnScope, "collection", fmt.Sprintf("%s.%s", db.Name(), collectionName)), - Name: collectionName, + Name: collectionName, Service: "mongodb", - Type: "table", - Data: data, + Type: "table", + Data: data, } return diff --git a/plugins/extractors/mssql/mssql.go b/plugins/extractors/mssql/mssql.go index 926c2a919..885b05c60 100644 --- a/plugins/extractors/mssql/mssql.go +++ b/plugins/extractors/mssql/mssql.go @@ -36,8 +36,7 @@ type Config struct { ConnectionURL string `mapstructure:"connection_url" validate:"required"` } -var sampleConfig = ` -connection_url: "sqlserver://admin:pass123@localhost:3306/"` +var sampleConfig = `connection_url: "sqlserver://admin:pass123@localhost:3306/"` var info = plugins.Info{ Description: "Table metdata from MSSQL server", @@ -131,10 +130,10 @@ func (e *Extractor) processTable(database string, tableName string) (err error) // push table to channel e.emit(models.NewRecord(&v1beta2.Asset{ Urn: models.NewURN("mssql", e.UrnScope, "table", fmt.Sprintf("%s.%s", database, tableName)), - Name: tableName, - Type: "table", + Name: tableName, + Type: "table", Service: "mssql", - Data: table, + Data: table, })) return diff --git a/plugins/extractors/mysql/mysql.go b/plugins/extractors/mysql/mysql.go index 759a9803d..34ddf1352 100644 --- a/plugins/extractors/mysql/mysql.go +++ b/plugins/extractors/mysql/mysql.go @@ -35,8 +35,7 @@ type Config struct { ConnectionURL string `mapstructure:"connection_url" validate:"required"` } -var sampleConfig = ` -connection_url: "admin:pass123@tcp(localhost:3306)/"` +var sampleConfig = `connection_url: "admin:pass123@tcp(localhost:3306)/"` var info = plugins.Info{ Description: "Table metadata from MySQL server.", @@ -144,10 +143,10 @@ func (e *Extractor) processTable(database string, tableName string) (err error) // push table to channel e.emit(models.NewRecord(&v1beta2.Asset{ Urn: models.NewURN("mysql", e.UrnScope, "table", fmt.Sprintf("%s.%s", database, tableName)), - Name: tableName, - Type: "table", + Name: tableName, + Type: "table", Service: "mysql", - Data: table, + Data: table, })) return diff --git a/plugins/extractors/optimus/optimus.go b/plugins/extractors/optimus/optimus.go index 58d151719..7d5fa9dbe 100644 --- a/plugins/extractors/optimus/optimus.go +++ b/plugins/extractors/optimus/optimus.go @@ -26,8 +26,7 @@ type Config struct { MaxSizeInMB int `mapstructure:"max_size_in_mb"` } -var sampleConfig = ` -host: optimus.com:80` +var sampleConfig = `host: optimus.com:80` var info = plugins.Info{ Description: "Optimus' jobs metadata", diff --git a/plugins/extractors/oracle/oracle.go b/plugins/extractors/oracle/oracle.go index b9b3a4f0b..84e2141d9 100644 --- a/plugins/extractors/oracle/oracle.go +++ b/plugins/extractors/oracle/oracle.go @@ -26,8 +26,7 @@ type Config struct { ConnectionURL string `mapstructure:"connection_url" validate:"required"` } -var sampleConfig = ` -connection_url: oracle://username:passwd@localhost:1521/xe` +var sampleConfig = `connection_url: oracle://username:passwd@localhost:1521/xe` var info = plugins.Info{ Description: "Table metadata oracle SQL Database.", diff --git a/plugins/extractors/shield/shield.go b/plugins/extractors/shield/shield.go index d70c8fe9c..d0b3eb0d8 100644 --- a/plugins/extractors/shield/shield.go +++ b/plugins/extractors/shield/shield.go @@ -22,8 +22,7 @@ type Config struct { Host string `mapstructure:"host" validate:"required"` } -var sampleConfig = ` -host: shield.com:80` +var sampleConfig = `host: shield.com:80` var info = plugins.Info{ Description: "Shield' users metadata", diff --git a/plugins/extractors/tableau/tableau.go b/plugins/extractors/tableau/tableau.go index e041048ed..53083e51a 100644 --- a/plugins/extractors/tableau/tableau.go +++ b/plugins/extractors/tableau/tableau.go @@ -21,7 +21,7 @@ var summary string var sampleConfig = ` host: https://server.tableau.com -version: 3.12 +version: "3.12" identifier: my-tableau username: meteor_user password: xxxxxxxxxx diff --git a/plugins/processors/enrich/processor.go b/plugins/processors/enrich/processor.go index 7bdd40401..7a1969f66 100644 --- a/plugins/processors/enrich/processor.go +++ b/plugins/processors/enrich/processor.go @@ -27,10 +27,10 @@ type Processor struct { } var sampleConfig = ` - # Enrichment configuration - # attributes: - # fieldA: valueA - # fieldB: valueB` +# Enrichment configuration +# attributes: +# fieldA: valueA +# fieldB: valueB` var info = plugins.Info{ Description: "Append custom fields to records", diff --git a/plugins/processors/labels/labels.go b/plugins/processors/labels/labels.go index 36d981a27..107e5c6b6 100644 --- a/plugins/processors/labels/labels.go +++ b/plugins/processors/labels/labels.go @@ -26,10 +26,10 @@ type Processor struct { } var sampleConfig = ` - # Append labels to asset - # labels: - # fieldA: valueA - # fieldB: valueB` +# Append labels to asset +# labels: +# fieldA: valueA +# fieldB: valueB` var info = plugins.Info{ Description: "Append labels to assets", diff --git a/plugins/sinks/compass/sink.go b/plugins/sinks/compass/sink.go index b8b382dba..808b00008 100644 --- a/plugins/sinks/compass/sink.go +++ b/plugins/sinks/compass/sink.go @@ -10,6 +10,7 @@ import ( "net/http" "strings" + "github.com/MakeNowJust/heredoc" "github.com/odpf/meteor/models" v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" "github.com/odpf/meteor/plugins" @@ -34,18 +35,18 @@ var info = plugins.Info{ Description: "Send metadata to compass http service", Summary: summary, Tags: []string{"http", "sink"}, - SampleConfig: ` + SampleConfig: heredoc.Doc(` # The hostname of the compass service host: https://compass.com # Additional HTTP headers send to compass, multiple headers value are separated by a comma headers: - Compass-User-Email: meteor@odpf.io - X-Other-Header: value1, value2 + Compass-User-Email: meteor@odpf.io + X-Other-Header: value1, value2 # The labels to pass as payload label of the patch api labels: - myCustom: $properties.attributes.myCustomField - sampleLabel: $properties.labels.sampleLabelField - `, + myCustom: $properties.attributes.myCustomField + sampleLabel: $properties.labels.sampleLabelField + `), } type httpClient interface { diff --git a/plugins/sinks/file/file.go b/plugins/sinks/file/file.go index 6c5887164..fd8019bf9 100644 --- a/plugins/sinks/file/file.go +++ b/plugins/sinks/file/file.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/MakeNowJust/heredoc" "github.com/odpf/meteor/models" assetsv1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" "github.com/odpf/meteor/plugins" @@ -28,10 +29,10 @@ var info = plugins.Info{ Description: "save output to a file", Summary: summary, Tags: []string{"file", "json", "yaml", "sink"}, - SampleConfig: ` + SampleConfig: heredoc.Doc(` path: ./output-filename.txt format: ndjson - `, + `), } type Sink struct { diff --git a/plugins/sinks/http/http.go b/plugins/sinks/http/http.go index e94974c21..f01f443ae 100644 --- a/plugins/sinks/http/http.go +++ b/plugins/sinks/http/http.go @@ -10,6 +10,7 @@ import ( "net/http" "strings" + "github.com/MakeNowJust/heredoc" "github.com/odpf/meteor/models" "github.com/odpf/meteor/plugins" "github.com/odpf/meteor/registry" @@ -31,13 +32,14 @@ var info = plugins.Info{ Description: "Send metadata to http service", Summary: summary, Tags: []string{"http", "sink"}, - SampleConfig: ` + SampleConfig: heredoc.Doc(` # The url (hostname and route) of the http service url: https://compass.com/route + method: "PUT" # Additional HTTP headers, multiple headers value are separated by a comma headers: - X-Other-Header: value1, value2 - `, + X-Other-Header: value1, value2 + `), } type httpClient interface { diff --git a/plugins/sinks/kafka/sink.go b/plugins/sinks/kafka/sink.go index acb8f53ac..2a1fc9442 100644 --- a/plugins/sinks/kafka/sink.go +++ b/plugins/sinks/kafka/sink.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" + "github.com/MakeNowJust/heredoc" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/dynamicpb" @@ -14,7 +15,7 @@ import ( "github.com/odpf/meteor/registry" "github.com/odpf/salt/log" "github.com/pkg/errors" - kafka "github.com/segmentio/kafka-go" + "github.com/segmentio/kafka-go" "google.golang.org/protobuf/proto" ) @@ -31,13 +32,14 @@ var info = plugins.Info{ Description: "Sink metadata to Apache Kafka topic", Summary: summary, Tags: []string{"kafka", "topic", "sink"}, - SampleConfig: ` + SampleConfig: heredoc.Doc(` # Kafka broker addresses brokers: "localhost:9092" # The Kafka topic to write to topic: sample-topic-name # The path to the key field in the payload - key_path: xxx`, + key_path: xxx + `), } type ProtoReflector interface { diff --git a/plugins/sinks/stencil/stencil.go b/plugins/sinks/stencil/stencil.go index 5ca7d7d4c..9c7827a78 100644 --- a/plugins/sinks/stencil/stencil.go +++ b/plugins/sinks/stencil/stencil.go @@ -10,6 +10,7 @@ import ( "net/http" "strings" + "github.com/MakeNowJust/heredoc" "github.com/odpf/meteor/models" v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" "github.com/odpf/meteor/plugins" @@ -32,14 +33,14 @@ var info = plugins.Info{ Description: "Send metadata to stencil http service", Summary: summary, Tags: []string{"http", "sink"}, - SampleConfig: ` + SampleConfig: heredoc.Doc(` # The hostname of the stencil service host: https://stencil.com # The namespace ID of the stencil service namespace_id: myNamespace # The schema format in which data will sink to stencil format: avro - `, + `), } // httpClient holds the set of methods require for creating request