Skip to content

Commit 093d8a6

Browse files
sudo-suhassudo-suhas
authored
fix: skip config load for cmds that don't require it (#396)
- Load config only for 'lint' and 'run' commands. This would ensure that there are no warnings logs related to config file not being found for commands that do not require it. - In case the config file is not present, print the error but proceed with defaults. - Remove metrics/instrumentation for lint command since it does not provide any value.
1 parent 761d7bc commit 093d8a6

File tree

10 files changed

+63
-74
lines changed

10 files changed

+63
-74
lines changed

‎cmd/gen.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,12 @@ import (
66

77
"github.com/MakeNowJust/heredoc"
88
"github.com/odpf/meteor/recipe"
9-
"github.com/odpf/salt/log"
109
"github.com/spf13/cobra"
1110
"gopkg.in/yaml.v3"
1211
)
1312

1413
// GenCmd creates a command object for the "gen" action
15-
func GenCmd(lg log.Logger) *cobra.Command {
14+
func GenCmd() *cobra.Command {
1615
var (
1716
outputDirPath string
1817
dataFilePath string

‎cmd/info.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,13 @@ import (
66
"github.com/AlecAivazis/survey/v2"
77
"github.com/MakeNowJust/heredoc"
88
"github.com/odpf/meteor/registry"
9-
"github.com/odpf/salt/log"
109
"github.com/odpf/salt/printer"
1110
"github.com/odpf/salt/term"
1211
"github.com/spf13/cobra"
1312
)
1413

1514
// InfoCmd creates a command object for get info about a plugin
16-
func InfoCmd(lg log.Logger) *cobra.Command {
15+
func InfoCmd() *cobra.Command {
1716
cmd := &cobra.Command{
1817
Use: "info <command>",
1918
Short: "Display plugin information",

‎cmd/lint.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,20 @@ import (
55
"fmt"
66
"os"
77

8-
"github.com/odpf/meteor/plugins"
9-
108
"github.com/MakeNowJust/heredoc"
119
"github.com/odpf/meteor/agent"
12-
"github.com/odpf/meteor/metrics"
10+
"github.com/odpf/meteor/config"
11+
"github.com/odpf/meteor/plugins"
1312
"github.com/odpf/meteor/recipe"
1413
"github.com/odpf/meteor/registry"
1514
"github.com/odpf/salt/log"
1615
"github.com/odpf/salt/printer"
1716
"github.com/odpf/salt/term"
18-
1917
"github.com/spf13/cobra"
2018
)
2119

2220
// LintCmd creates a command object for linting recipes
23-
func LintCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command {
21+
func LintCmd() *cobra.Command {
2422
var (
2523
report [][]string
2624
success = 0
@@ -50,12 +48,19 @@ func LintCmd(lg log.Logger, mt *metrics.StatsdMonitor) *cobra.Command {
5048
"group:core": "true",
5149
},
5250
RunE: func(cmd *cobra.Command, args []string) error {
51+
cfg, err := config.Load("./meteor.yaml")
52+
if err != nil {
53+
return err
54+
}
55+
56+
lg := log.NewLogrus(log.LogrusWithLevel(cfg.LogLevel))
57+
plugins.SetLog(lg)
58+
5359
cs := term.NewColorScheme()
5460
runner := agent.NewAgent(agent.Config{
5561
ExtractorFactory: registry.Extractors,
5662
ProcessorFactory: registry.Processors,
5763
SinkFactory: registry.Sinks,
58-
Monitor: mt,
5964
Logger: lg,
6065
})
6166

‎cmd/list.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,14 @@ import (
77

88
"github.com/MakeNowJust/heredoc"
99
"github.com/odpf/meteor/registry"
10-
"github.com/odpf/salt/log"
1110
"github.com/odpf/salt/printer"
1211
"github.com/odpf/salt/term"
1312

1413
"github.com/spf13/cobra"
1514
)
1615

1716
// ListCmd creates a command object for linting recipes
18-
func ListCmd(lg log.Logger) *cobra.Command {
17+
func ListCmd() *cobra.Command {
1918
cmd := &cobra.Command{
2019
Use: "list <command>",
2120
Short: "List available plugins",

‎cmd/new.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,11 @@ import (
88
"github.com/MakeNowJust/heredoc"
99
"github.com/odpf/meteor/generator"
1010
"github.com/odpf/meteor/registry"
11-
"github.com/odpf/salt/log"
1211
"github.com/spf13/cobra"
1312
)
1413

1514
// NewCmd creates a command object for the "new" action
16-
func NewCmd(lg log.Logger) *cobra.Command {
15+
func NewCmd() *cobra.Command {
1716
cmd := &cobra.Command{
1817
Use: "new",
1918
Short: "Bootstrap new recipes",

‎cmd/root.go‎

Lines changed: 6 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,13 @@
11
package cmd
22

33
import (
4-
"fmt"
5-
"os"
6-
74
"github.com/MakeNowJust/heredoc"
8-
"github.com/odpf/meteor/config"
9-
"github.com/odpf/meteor/metrics"
10-
"github.com/odpf/meteor/plugins"
115
"github.com/odpf/salt/cmdx"
12-
"github.com/odpf/salt/log"
136
"github.com/spf13/cobra"
147
)
158

16-
const exitError = 1
17-
189
// New adds all child commands to the root command and sets flags appropriately.
1910
func New() *cobra.Command {
20-
cfg, err := config.Load("./meteor.yaml")
21-
if err != nil {
22-
fmt.Printf("ERROR: %s\n", err.Error())
23-
os.Exit(1)
24-
}
25-
26-
lg := log.NewLogrus(log.LogrusWithLevel(cfg.LogLevel))
27-
plugins.SetLog(lg)
28-
29-
// Setup statsd monitor to collect monitoring metrics
30-
var mt *metrics.StatsdMonitor
31-
if cfg.StatsdEnabled {
32-
client, err := metrics.NewStatsdClient(cfg.StatsdHost)
33-
if err != nil {
34-
fmt.Printf("ERROR: %s\n", err.Error())
35-
os.Exit(exitError)
36-
}
37-
mt = metrics.NewStatsdMonitor(client, cfg.StatsdPrefix)
38-
}
39-
4011
var cmd = &cobra.Command{
4112
Use: "meteor <command> <subcommand> [flags]",
4213
Short: "Metadata CLI",
@@ -66,12 +37,12 @@ func New() *cobra.Command {
6637
cmd.AddCommand(cmdx.SetRefCmd(cmd))
6738

6839
cmd.AddCommand(VersionCmd())
69-
cmd.AddCommand(GenCmd(lg))
70-
cmd.AddCommand(ListCmd(lg))
71-
cmd.AddCommand(InfoCmd(lg))
72-
cmd.AddCommand(RunCmd(lg, mt, cfg))
73-
cmd.AddCommand(LintCmd(lg, mt))
74-
cmd.AddCommand(NewCmd(lg))
40+
cmd.AddCommand(GenCmd())
41+
cmd.AddCommand(ListCmd())
42+
cmd.AddCommand(InfoCmd())
43+
cmd.AddCommand(RunCmd())
44+
cmd.AddCommand(LintCmd())
45+
cmd.AddCommand(NewCmd())
7546

7647
return cmd
7748
}

‎cmd/run.go‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/odpf/meteor/agent"
1414
"github.com/odpf/meteor/config"
1515
"github.com/odpf/meteor/metrics"
16+
"github.com/odpf/meteor/plugins"
1617
"github.com/odpf/meteor/recipe"
1718
"github.com/odpf/meteor/registry"
1819
"github.com/odpf/salt/log"
@@ -23,7 +24,7 @@ import (
2324
)
2425

2526
// RunCmd creates a command object for the "run" action.
26-
func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra.Command {
27+
func RunCmd() *cobra.Command {
2728
var (
2829
report [][]string
2930
pathToConfig string
@@ -57,12 +58,17 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra.
5758
"group:core": "true",
5859
},
5960
RunE: func(cmd *cobra.Command, args []string) error {
60-
if configFile != "" {
61-
var err error
62-
cfg, err = config.Load(configFile)
63-
if err != nil {
64-
return err
65-
}
61+
cfg, err := config.Load(configFile)
62+
if err != nil {
63+
return err
64+
}
65+
66+
lg := log.NewLogrus(log.LogrusWithLevel(cfg.LogLevel))
67+
plugins.SetLog(lg)
68+
69+
mt, err := newStatsdMonitor(cfg)
70+
if err != nil {
71+
return err
6672
}
6773

6874
cs := term.NewColorScheme()
@@ -135,3 +141,15 @@ func RunCmd(lg log.Logger, mt *metrics.StatsdMonitor, cfg config.Config) *cobra.
135141

136142
return cmd
137143
}
144+
145+
func newStatsdMonitor(cfg config.Config) (*metrics.StatsdMonitor, error) {
146+
if !cfg.StatsdEnabled {
147+
return nil, nil
148+
}
149+
150+
client, err := metrics.NewStatsdClient(cfg.StatsdHost)
151+
if err != nil {
152+
return nil, err
153+
}
154+
return metrics.NewStatsdMonitor(client, cfg.StatsdPrefix), nil
155+
}

‎config/config.go‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package config
22

33
import (
44
"errors"
5-
"log"
5+
"fmt"
66

77
"github.com/odpf/salt/config"
88
)
@@ -18,15 +18,17 @@ type Config struct {
1818
StopOnSinkError bool `mapstructure:"STOP_ON_SINK_ERROR" default:"false"`
1919
}
2020

21-
func Load(configFile string) (cfg Config, err error) {
22-
err = config.
23-
NewLoader(config.WithFile(configFile)).
21+
func Load(configFile string) (Config, error) {
22+
var cfg Config
23+
err := config.NewLoader(config.WithFile(configFile)).
2424
Load(&cfg)
25-
26-
if errors.As(err, &config.ConfigFileNotFoundError{}) {
27-
log.Println(err)
28-
err = nil
25+
if err != nil {
26+
if errors.As(err, &config.ConfigFileNotFoundError{}) {
27+
fmt.Println(err)
28+
return cfg, nil
29+
}
30+
return Config{}, err
2931
}
3032

31-
return
33+
return cfg, nil
3234
}

‎main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ const (
2121
func main() {
2222
// Execute the root command
2323
root := cmd.New()
24+
2425
cmd, err := root.ExecuteC()
2526

2627
if err == nil {

‎test/e2e/e2e_test.go‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ import (
77
"context"
88
"database/sql"
99
"fmt"
10-
v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2"
11-
"google.golang.org/protobuf/types/known/anypb"
1210
"log"
1311
"net"
1412
"os"
@@ -17,12 +15,14 @@ import (
1715
"testing"
1816
"time"
1917

18+
v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2"
19+
"google.golang.org/protobuf/types/known/anypb"
20+
2021
"github.com/odpf/meteor/test/utils"
2122
"github.com/ory/dockertest/v3"
2223
"github.com/ory/dockertest/v3/docker"
2324

2425
"github.com/odpf/meteor/cmd"
25-
"github.com/odpf/meteor/config"
2626
_ "github.com/odpf/meteor/plugins/extractors"
2727
_ "github.com/odpf/meteor/plugins/processors"
2828
_ "github.com/odpf/meteor/plugins/sinks"
@@ -105,13 +105,9 @@ func TestMySqlToKafka(t *testing.T) {
105105
}
106106
}()
107107

108-
// run mysql_kafka.yml file
109-
cfg, err := config.Load("mysql_kafka.yml")
110-
if err != nil {
111-
t.Error(err)
112-
}
113-
command := cmd.RunCmd(utils.Logger, nil, cfg)
108+
command := cmd.RunCmd()
114109

110+
// run mysql_kafka.yml file
115111
command.SetArgs([]string{"mysql_kafka.yml"})
116112
if err := command.Execute(); err != nil {
117113
if strings.HasPrefix(err.Error(), "unknown command ") {

0 commit comments

Comments
 (0)