feat: allow agent level config with --config flags

.gitignore

@@ -25,6 +25,7 @@
 config.yaml
 meteor
 _recipes
+meteor.yaml
 
 # plugins
 meteor-plugin-*

Makefile

@@ -15,6 +15,9 @@ build-dev:
 clean:
 	rm -rf dist/
 
+copy-config:
+	cp ./config/meteor.yaml.sample ./meteor.yaml
+
 test:
 	go test ./... -coverprofile=coverage.out
 

cmd/root.go

@@ -1,16 +1,42 @@
 package cmd
 
 import (
+	"fmt"
+	"os"
+
 	"github.com/MakeNowJust/heredoc"
 	"github.com/odpf/meteor/config"
 	"github.com/odpf/meteor/metrics"
+	"github.com/odpf/meteor/plugins"
 	"github.com/odpf/salt/cmdx"
 	"github.com/odpf/salt/log"
 	"github.com/spf13/cobra"
 )
 
+const exitError = 1
+
 // New adds all child commands to the root command and sets flags appropriately.
-func New(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra.Command {
+func New() *cobra.Command {
+	cfg, err := config.Load("./meteor.yaml")
+	if err != nil {
+		fmt.Printf("ERROR: %s\n", err.Error())
+		os.Exit(1)
+	}
+
+	lg := log.NewLogrus(log.LogrusWithLevel(cfg.LogLevel))
+	plugins.SetLog(lg)
+
+	// Setup statsd monitor to collect monitoring metrics
+	var mt *metrics.StatsdMonitor
+	if cfg.StatsdEnabled {
+		client, err := metrics.NewStatsdClient(cfg.StatsdHost)
+		if err != nil {
+			fmt.Printf("ERROR: %s\n", err.Error())
+			os.Exit(exitError)
+		}
+		mt = metrics.NewStatsdMonitor(client, cfg.StatsdPrefix)
+	}
+
 	var cmd = &cobra.Command{
 		Use:           "meteor <command> <subcommand> [flags]",
 		Short:         "Metadata CLI",

cmd/run.go

@@ -29,6 +29,7 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra.
 		pathToConfig string
 		success      = 0
 		failures     = 0
+		configFile   string
 	)
 
 	cmd := &cobra.Command{
@@ -56,6 +57,13 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra.
 			"group:core": "true",
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if configFile != "" {
+				var err error
+				cfg, err = config.Load(configFile)
+				if err != nil {
+					return err
+				}
+			}
 
 			cs := term.NewColorScheme()
 			runner := agent.NewAgent(agent.Config{
@@ -123,6 +131,7 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra.
 	}
 
 	cmd.Flags().StringVar(&pathToConfig, "var", "", "Path to Config file with env variables for recipe")
+	cmd.Flags().StringVarP(&configFile, "config", "c", "./meteor.yaml", "file path for agent level config")
 
 	return cmd
 }

config/config.go

@@ -18,10 +18,11 @@ type Config struct {
 	StopOnSinkError             bool   `mapstructure:"STOP_ON_SINK_ERROR" default:"false"`
 }
 
-func Load() (cfg Config, err error) {
+func Load(configFile string) (cfg Config, err error) {
 	err = config.
-		NewLoader(config.WithPath("./")).
+		NewLoader(config.WithFile(configFile)).
 		Load(&cfg)
+
 	if errors.As(err, &config.ConfigFileNotFoundError{}) {
 		log.Println(err)
 		err = nil

config/meteor.yaml.sample

@@ -0,0 +1,7 @@
+LOG_LEVEL: info
+STATSD_ENABLED: false
+STATSD_HOST: "localhost:8125"
+STATSD_PREFIX: meteor
+MAX_RETRIES: 5
+RETRY_INITIAL_INTERVAL_SECONDS: 5
+STOP_ON_SINK_ERROR: false

main.go

@@ -6,15 +6,11 @@ import (
 	"strings"
 
 	"github.com/odpf/meteor/cmd"
-	"github.com/odpf/meteor/config"
-	"github.com/odpf/meteor/metrics"
-	"github.com/odpf/meteor/plugins"
 
 	_ "github.com/odpf/meteor/plugins/extractors"
 	_ "github.com/odpf/meteor/plugins/processors"
 	_ "github.com/odpf/meteor/plugins/sinks"
 	"github.com/odpf/salt/cmdx"
-	"github.com/odpf/salt/log"
 )
 
 const (
@@ -23,28 +19,8 @@ const (
 )
 
 func main() {
-	cfg, err := config.Load()
-	if err != nil {
-		fmt.Printf("ERROR: %s\n", err.Error())
-		os.Exit(1)
-	}
-
-	lg := log.NewLogrus(log.LogrusWithLevel(cfg.LogLevel))
-	plugins.SetLog(lg)
-
-	// Setup statsd monitor to collect monitoring metrics
-	var monitor *metrics.StatsdMonitor
-	if cfg.StatsdEnabled {
-		client, err := metrics.NewStatsdClient(cfg.StatsdHost)
-		if err != nil {
-			fmt.Printf("ERROR: %s\n", err.Error())
-			os.Exit(exitError)
-		}
-		monitor = metrics.NewStatsdMonitor(client, cfg.StatsdPrefix)
-	}
-
 	// Execute the root command
-	root := cmd.New(lg, monitor, cfg)
+	root := cmd.New()
 	cmd, err := root.ExecuteC()
 
 	if err == nil {