fix(gen): format, fix sample configs used for generating recipes

generator/recipe.go

@@ -27,7 +27,8 @@ type templateData struct {
 }
 
 var templateFuncs = map[string]interface{}{
-	"indent": indent,
+	"indent":          indent,
+	"fmtSampleConfig": fmtSampleConfig,
 }
 
 var recipeVersions = [1]string{"v1beta1"}
@@ -93,6 +94,14 @@ func indent(spaces int, v string) string {
 	return pad + strings.Replace(v, "\n", "\n"+pad, -1)
 }
 
+func fmtSampleConfig(s string) string {
+	if !strings.HasPrefix(s, "\n") {
+		s = "\n" + s
+	}
+
+	return strings.ReplaceAll(s, "\t", "  ")
+}
+
 func GetRecipeVersions() [1]string {
 	return recipeVersions
 }

generator/recipe.yaml

@@ -4,14 +4,14 @@ source:
 {{- with .Source }}
   name: {{.Name}}
   scope: {{.Scope}}
-  config: {{.SampleConfig | indent 4}}
+  config: {{.SampleConfig | fmtSampleConfig | indent 4}}
 {{- end }}
 {{- if ne (len .Sinks) 0 }}
 sinks:
 {{- range $key, $value := .Sinks }}
   - name: {{$key}}
     {{- if  $value}}
-    config: {{$value | indent 6}}
+    config: {{$value | fmtSampleConfig | indent 6}}
     {{- end }}    
 {{- end }}
 {{- end }}
@@ -20,7 +20,7 @@ processors:
 {{- range $key, $value := .Processors }}
   - name: {{$key}}
     {{- if  $value}}
-    config: {{$value | indent 6}}
+    config: {{$value | fmtSampleConfig | indent 6}}
     {{- end }}
 {{- end }}
 {{- end }}

plugins/extractors/clickhouse/clickhouse.go

@@ -25,8 +25,7 @@ type Config struct {
 	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
 }
 
-var sampleConfig = `
-connection_url: "tcp://localhost:3306?username=admin&password=pass123&debug=true"`
+var sampleConfig = `connection_url: "tcp://localhost:3306?username=admin&password=pass123&debug=true"`
 
 var info = plugins.Info{
 	Description:  "Column-oriented DBMS for online analytical processing.",
@@ -108,10 +107,10 @@ func (e *Extractor) extractTables(emit plugins.Emit) (err error) {
 
 		asset := v1beta2.Asset{
 			Urn:     models.NewURN("clickhouse", e.UrnScope, "table", fmt.Sprintf("%s.%s", dbName, tableName)),
-			Name: tableName,
-			Type: "table",
+			Name:    tableName,
+			Type:    "table",
 			Service: "clickhouse",
-			Data: table,
+			Data:    table,
 		}
 		emit(models.NewRecord(&asset))
 	}

plugins/extractors/csv/csv.go

@@ -3,6 +3,7 @@ package csv
 import (
 	"context"
 	_ "embed" // used to print the embedded assets
+	"encoding/csv"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -16,8 +17,6 @@ import (
 	"github.com/pkg/errors"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	"encoding/csv"
-
 	"github.com/odpf/meteor/plugins"
 	"github.com/odpf/salt/log"
 )
@@ -30,8 +29,7 @@ type Config struct {
 	Path string `mapstructure:"path" validate:"required"`
 }
 
-var sampleConfig = `
-path: ./path-to-a-file-or-a-directory`
+var sampleConfig = `path: ./path-to-a-file-or-a-directory`
 
 var info = plugins.Info{
 	Description:  "Comma separated file",

plugins/extractors/elastic/elastic.go

@@ -28,9 +28,9 @@ type Config struct {
 }
 
 var sampleConfig = `
- user: "elastic"
- password: "changeme"
- host: elastic_server`
+user: "elastic"
+password: "changeme"
+host: elastic_server`
 
 var info = plugins.Info{
 	Description:  "Search engine based on the Lucene library.",
@@ -63,7 +63,7 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error)
 		return err
 	}
 
-	//build elasticsearch client
+	// build elasticsearch client
 	cfg := elasticsearch.Config{
 		Addresses: []string{
 			e.config.Host,

plugins/extractors/kafka/kafka.go

@@ -11,7 +11,7 @@ import (
 	v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2"
 	"github.com/odpf/meteor/plugins"
 	"github.com/odpf/meteor/registry"
-	kafka "github.com/segmentio/kafka-go"
+	"github.com/segmentio/kafka-go"
 
 	"github.com/odpf/salt/log"
 )
@@ -30,8 +30,7 @@ type Config struct {
 	Broker string `mapstructure:"broker" validate:"required"`
 }
 
-var sampleConfig = `
-broker: "localhost:9092"`
+var sampleConfig = `broker: "localhost:9092"`
 
 var info = plugins.Info{
 	Description:  "Topic list from Apache Kafka.",

plugins/extractors/metabase/metabase.go

@@ -25,7 +25,7 @@ var summary string
 var sampleConfig = `
 host: http://localhost:3000
 instance_label: my-metabase
-user_id: meteor_tester
+username: meteor_tester
 password: meteor_pass_1234`
 
 var info = plugins.Info{

plugins/extractors/mongodb/mongodb.go

@@ -34,8 +34,7 @@ type Config struct {
 	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
 }
 
-var sampleConfig = `
-connection_url: "mongodb://admin:pass123@localhost:3306"`
+var sampleConfig = `connection_url: "mongodb://admin:pass123@localhost:3306"`
 
 var info = plugins.Info{
 	Description:  "Collection metadata from MongoDB Server",
@@ -144,10 +143,10 @@ func (e *Extractor) buildTable(ctx context.Context, db *mongo.Database, collecti
 	//
 	table = &v1beta2.Asset{
 		Urn:     models.NewURN("mongodb", e.UrnScope, "collection", fmt.Sprintf("%s.%s", db.Name(), collectionName)),
-		Name: collectionName,
+		Name:    collectionName,
 		Service: "mongodb",
-		Type: "table",
-		Data: data,
+		Type:    "table",
+		Data:    data,
 	}
 
 	return

plugins/extractors/mssql/mssql.go

@@ -36,8 +36,7 @@ type Config struct {
 	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
 }
 
-var sampleConfig = `
-connection_url: "sqlserver://admin:pass123@localhost:3306/"`
+var sampleConfig = `connection_url: "sqlserver://admin:pass123@localhost:3306/"`
 
 var info = plugins.Info{
 	Description:  "Table metdata from MSSQL server",
@@ -131,10 +130,10 @@ func (e *Extractor) processTable(database string, tableName string) (err error)
 	// push table to channel
 	e.emit(models.NewRecord(&v1beta2.Asset{
 		Urn:     models.NewURN("mssql", e.UrnScope, "table", fmt.Sprintf("%s.%s", database, tableName)),
-		Name: tableName,
-		Type: "table",
+		Name:    tableName,
+		Type:    "table",
 		Service: "mssql",
-		Data: table,
+		Data:    table,
 	}))
 
 	return

plugins/extractors/mysql/mysql.go

@@ -35,8 +35,7 @@ type Config struct {
 	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
 }
 
-var sampleConfig = `
-connection_url: "admin:pass123@tcp(localhost:3306)/"`
+var sampleConfig = `connection_url: "admin:pass123@tcp(localhost:3306)/"`
 
 var info = plugins.Info{
 	Description:  "Table metadata from MySQL server.",
@@ -144,10 +143,10 @@ func (e *Extractor) processTable(database string, tableName string) (err error)
 	// push table to channel
 	e.emit(models.NewRecord(&v1beta2.Asset{
 		Urn:     models.NewURN("mysql", e.UrnScope, "table", fmt.Sprintf("%s.%s", database, tableName)),
-		Name: tableName,
-		Type: "table",
+		Name:    tableName,
+		Type:    "table",
 		Service: "mysql",
-		Data: table,
+		Data:    table,
 	}))
 
 	return

plugins/extractors/optimus/optimus.go

@@ -26,8 +26,7 @@ type Config struct {
 	MaxSizeInMB int    `mapstructure:"max_size_in_mb"`
 }
 
-var sampleConfig = `
-host: optimus.com:80`
+var sampleConfig = `host: optimus.com:80`
 
 var info = plugins.Info{
 	Description:  "Optimus' jobs metadata",

plugins/extractors/oracle/oracle.go

@@ -26,8 +26,7 @@ type Config struct {
 	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
 }
 
-var sampleConfig = `
-connection_url: oracle://username:passwd@localhost:1521/xe`
+var sampleConfig = `connection_url: oracle://username:passwd@localhost:1521/xe`
 
 var info = plugins.Info{
 	Description:  "Table metadata oracle SQL Database.",

plugins/extractors/shield/shield.go

@@ -22,8 +22,7 @@ type Config struct {
 	Host string `mapstructure:"host" validate:"required"`
 }
 
-var sampleConfig = `
-host: shield.com:80`
+var sampleConfig = `host: shield.com:80`
 
 var info = plugins.Info{
 	Description:  "Shield' users metadata",

plugins/extractors/tableau/tableau.go

@@ -21,7 +21,7 @@ var summary string
 
 var sampleConfig = `
 host: https://server.tableau.com
-version: 3.12
+version: "3.12"
 identifier: my-tableau
 username: meteor_user
 password: xxxxxxxxxx

plugins/processors/enrich/processor.go

@@ -27,10 +27,10 @@ type Processor struct {
 }
 
 var sampleConfig = `
- # Enrichment configuration
- # attributes:
- #	fieldA: valueA
- # 	fieldB: valueB`
+# Enrichment configuration
+# attributes:
+#   fieldA: valueA
+#   fieldB: valueB`
 
 var info = plugins.Info{
 	Description:  "Append custom fields to records",

plugins/processors/labels/labels.go

@@ -26,10 +26,10 @@ type Processor struct {
 }
 
 var sampleConfig = `
- # Append labels to asset
- # labels:
- #	fieldA: valueA
- # 	fieldB: valueB`
+# Append labels to asset
+# labels:
+#   fieldA: valueA
+#   fieldB: valueB`
 
 var info = plugins.Info{
 	Description:  "Append labels to assets",

plugins/sinks/compass/sink.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/MakeNowJust/heredoc"
 	"github.com/odpf/meteor/models"
 	v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2"
 	"github.com/odpf/meteor/plugins"
@@ -34,18 +35,18 @@ var info = plugins.Info{
 	Description: "Send metadata to compass http service",
 	Summary:     summary,
 	Tags:        []string{"http", "sink"},
-	SampleConfig: `
+	SampleConfig: heredoc.Doc(`
 	# The hostname of the compass service
 	host: https://compass.com
 	# Additional HTTP headers send to compass, multiple headers value are separated by a comma
 	headers:
-		Compass-User-Email: [email protected]
-		X-Other-Header: value1, value2
+	  Compass-User-Email: [email protected]
+	  X-Other-Header: value1, value2
 	# The labels to pass as payload label of the patch api
 	labels:
-		myCustom: $properties.attributes.myCustomField
-		sampleLabel: $properties.labels.sampleLabelField
-	`,
+	  myCustom: $properties.attributes.myCustomField
+	  sampleLabel: $properties.labels.sampleLabelField
+	`),
 }
 
 type httpClient interface {

plugins/sinks/file/file.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/MakeNowJust/heredoc"
 	"github.com/odpf/meteor/models"
 	assetsv1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2"
 	"github.com/odpf/meteor/plugins"
@@ -28,10 +29,10 @@ var info = plugins.Info{
 	Description: "save output to a file",
 	Summary:     summary,
 	Tags:        []string{"file", "json", "yaml", "sink"},
-	SampleConfig: `
+	SampleConfig: heredoc.Doc(`
 	path: ./output-filename.txt
 	format: ndjson
-	`,
+	`),
 }
 
 type Sink struct {

plugins/sinks/http/http.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/MakeNowJust/heredoc"
 	"github.com/odpf/meteor/models"
 	"github.com/odpf/meteor/plugins"
 	"github.com/odpf/meteor/registry"
@@ -31,13 +32,14 @@ var info = plugins.Info{
 	Description: "Send metadata to http service",
 	Summary:     summary,
 	Tags:        []string{"http", "sink"},
-	SampleConfig: `
+	SampleConfig: heredoc.Doc(`
 	# The url (hostname and route) of the http service
 	url: https://compass.com/route
+	method: "PUT"
 	# Additional HTTP headers, multiple headers value are separated by a comma
 	headers:
-		X-Other-Header: value1, value2
-	`,
+	  X-Other-Header: value1, value2
+	`),
 }
 
 type httpClient interface {