diff --git a/.gitignore b/.gitignore index f683253ad..7495d827e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ config.yaml meteor _recipes +meteor.yaml # plugins meteor-plugin-* diff --git a/Makefile b/Makefile index ea695accd..ed551e742 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,9 @@ build-dev: clean: rm -rf dist/ +copy-config: + cp ./config/meteor.yaml.sample ./meteor.yaml + test: go test ./... -coverprofile=coverage.out diff --git a/cmd/root.go b/cmd/root.go index 6cea0826e..a028205c4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,16 +1,42 @@ package cmd import ( + "fmt" + "os" + "github.com/MakeNowJust/heredoc" "github.com/odpf/meteor/config" "github.com/odpf/meteor/metrics" + "github.com/odpf/meteor/plugins" "github.com/odpf/salt/cmdx" "github.com/odpf/salt/log" "github.com/spf13/cobra" ) +const exitError = 1 + // New adds all child commands to the root command and sets flags appropriately. -func New(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra.Command { +func New() *cobra.Command { + cfg, err := config.Load("./meteor.yaml") + if err != nil { + fmt.Printf("ERROR: %s\n", err.Error()) + os.Exit(1) + } + + lg := log.NewLogrus(log.LogrusWithLevel(cfg.LogLevel)) + plugins.SetLog(lg) + + // Setup statsd monitor to collect monitoring metrics + var mt *metrics.StatsdMonitor + if cfg.StatsdEnabled { + client, err := metrics.NewStatsdClient(cfg.StatsdHost) + if err != nil { + fmt.Printf("ERROR: %s\n", err.Error()) + os.Exit(exitError) + } + mt = metrics.NewStatsdMonitor(client, cfg.StatsdPrefix) + } + var cmd = &cobra.Command{ Use: "meteor [flags]", Short: "Metadata CLI", diff --git a/cmd/run.go b/cmd/run.go index d107fbe6e..a77fbbcfc 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -29,6 +29,7 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra. pathToConfig string success = 0 failures = 0 + configFile string ) cmd := &cobra.Command{ @@ -56,6 +57,13 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra. "group:core": "true", }, RunE: func(cmd *cobra.Command, args []string) error { + if configFile != "" { + var err error + cfg, err = config.Load(configFile) + if err != nil { + return err + } + } cs := term.NewColorScheme() runner := agent.NewAgent(agent.Config{ @@ -123,6 +131,7 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra. } cmd.Flags().StringVar(&pathToConfig, "var", "", "Path to Config file with env variables for recipe") + cmd.Flags().StringVarP(&configFile, "config", "c", "./meteor.yaml", "file path for agent level config") return cmd } diff --git a/config/config.go b/config/config.go index c59acb22c..0c5e0f467 100644 --- a/config/config.go +++ b/config/config.go @@ -18,10 +18,11 @@ type Config struct { StopOnSinkError bool `mapstructure:"STOP_ON_SINK_ERROR" default:"false"` } -func Load() (cfg Config, err error) { +func Load(configFile string) (cfg Config, err error) { err = config. - NewLoader(config.WithPath("./")). + NewLoader(config.WithFile(configFile)). Load(&cfg) + if errors.As(err, &config.ConfigFileNotFoundError{}) { log.Println(err) err = nil diff --git a/config/meteor.yaml.sample b/config/meteor.yaml.sample new file mode 100644 index 000000000..b2edb1b3b --- /dev/null +++ b/config/meteor.yaml.sample @@ -0,0 +1,7 @@ +LOG_LEVEL: info +STATSD_ENABLED: false +STATSD_HOST: "localhost:8125" +STATSD_PREFIX: meteor +MAX_RETRIES: 5 +RETRY_INITIAL_INTERVAL_SECONDS: 5 +STOP_ON_SINK_ERROR: false \ No newline at end of file diff --git a/main.go b/main.go index 6b22f733f..9827c858d 100644 --- a/main.go +++ b/main.go @@ -6,15 +6,11 @@ import ( "strings" "github.com/odpf/meteor/cmd" - "github.com/odpf/meteor/config" - "github.com/odpf/meteor/metrics" - "github.com/odpf/meteor/plugins" _ "github.com/odpf/meteor/plugins/extractors" _ "github.com/odpf/meteor/plugins/processors" _ "github.com/odpf/meteor/plugins/sinks" "github.com/odpf/salt/cmdx" - "github.com/odpf/salt/log" ) const ( @@ -23,28 +19,8 @@ const ( ) func main() { - cfg, err := config.Load() - if err != nil { - fmt.Printf("ERROR: %s\n", err.Error()) - os.Exit(1) - } - - lg := log.NewLogrus(log.LogrusWithLevel(cfg.LogLevel)) - plugins.SetLog(lg) - - // Setup statsd monitor to collect monitoring metrics - var monitor *metrics.StatsdMonitor - if cfg.StatsdEnabled { - client, err := metrics.NewStatsdClient(cfg.StatsdHost) - if err != nil { - fmt.Printf("ERROR: %s\n", err.Error()) - os.Exit(exitError) - } - monitor = metrics.NewStatsdMonitor(client, cfg.StatsdPrefix) - } - // Execute the root command - root := cmd.New(lg, monitor, cfg) + root := cmd.New() cmd, err := root.ExecuteC() if err == nil {