diff --git a/.github/workflows/plugin_test.yml b/.github/workflows/plugin_test.yml index 99364d9f1..204528270 100644 --- a/.github/workflows/plugin_test.yml +++ b/.github/workflows/plugin_test.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - fix-plugin-tests workflow_dispatch: jobs: diff --git a/plugins/extractors/bigtable/bigtable_test.go b/plugins/extractors/bigtable/bigtable_test.go index 3a4baba9e..94eb8a91f 100644 --- a/plugins/extractors/bigtable/bigtable_test.go +++ b/plugins/extractors/bigtable/bigtable_test.go @@ -1,58 +1,19 @@ -//go:build integration -// +build integration +//go:build plugins +// +build plugins package bigtable_test import ( "context" - "log" - "os" "testing" "github.com/odpf/meteor/test/utils" - "cloud.google.com/go/bigtable" "github.com/odpf/meteor/plugins" bt "github.com/odpf/meteor/plugins/extractors/bigtable" - "github.com/ory/dockertest/v3" - "github.com/ory/dockertest/v3/docker" "github.com/stretchr/testify/assert" ) -func TestMain(m *testing.M) { - // setup test - opts := dockertest.RunOptions{ - Repository: "shopify/bigtable-emulator", - Env: []string{ - "BIGTABLE_EMULATOR_HOST=localhost:9035", - }, - ExposedPorts: []string{"9035"}, - PortBindings: map[docker.Port][]docker.PortBinding{ - "9035": { - {HostIP: "0.0.0.0", HostPort: "9035"}, - }, - }, - Cmd: []string{"-cf", "dev.records.data,dev.records.metadata"}, - } - // exponential backoff-retry, because the application in the container might not be ready to accept connections yet - retryFn := func(resource *dockertest.Resource) (err error) { - _, err = bigtable.NewAdminClient(context.Background(), "dev", "dev") - return - } - purgeFn, err := utils.CreateContainer(opts, retryFn) - if err != nil { - log.Fatal("", err) - } - - // run tests - code := m.Run() - - if err := purgeFn(); err != nil { - log.Fatal("", err) - } - os.Exit(code) -} - func TestInit(t *testing.T) { t.Run("should return error if no project_id in config", func(t *testing.T) { err := bt.New(utils.Logger).Init(context.TODO(), map[string]interface{}{ @@ -67,6 +28,6 @@ func TestInit(t *testing.T) { "project_id": "", }) - assert.EqualError(t, err, "invalid extractor config") + assert.Equal(t, plugins.InvalidConfigError{}, err) }) } diff --git a/plugins/extractors/clickhouse/clickhouse_test.go b/plugins/extractors/clickhouse/clickhouse_test.go index 88169f89c..d41aee099 100644 --- a/plugins/extractors/clickhouse/clickhouse_test.go +++ b/plugins/extractors/clickhouse/clickhouse_test.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build plugins +// +build plugins package clickhouse_test @@ -51,7 +51,7 @@ func TestMain(m *testing.M) { Tag: "21.7.4-alpine", ExposedPorts: []string{"9000", port}, Mounts: []string{ - fmt.Sprintf("%s/localConfig/users.xml:/etc/clickhouse-server/users.xml:rw", pwd), + fmt.Sprintf("%s/localConfig/users.xml:/etc/clickhouse-server/users.d/user.xml:rw", pwd), }, PortBindings: map[docker.Port][]docker.PortBinding{ "9000": { diff --git a/plugins/extractors/couchdb/couchdb_test.go b/plugins/extractors/couchdb/couchdb_test.go index f321ebe80..1e311a7eb 100644 --- a/plugins/extractors/couchdb/couchdb_test.go +++ b/plugins/extractors/couchdb/couchdb_test.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build plugins +// +build plugins package couchdb_test @@ -7,6 +7,7 @@ import ( "context" "fmt" "log" + "net/http" "os" "strconv" "testing" @@ -38,6 +39,10 @@ var ( ) func TestMain(m *testing.M) { + pwd, err := os.Getwd() + if err != nil { + log.Fatal(err) + } // setup test opts := dockertest.RunOptions{ Repository: "docker.io/bitnami/couchdb", @@ -46,7 +51,10 @@ func TestMain(m *testing.M) { "COUCHDB_USER=" + user, "COUCHDB_PASSWORD=" + pass, }, - ExposedPorts: []string{"4369", "5984", port}, + Mounts: []string{ + fmt.Sprintf("%s/localConfig:/opt/bitnami/couchdb/etc/local.d:rw", pwd), + }, + ExposedPorts: []string{port}, PortBindings: map[docker.Port][]docker.PortBinding{ "5984": { {HostIP: "0.0.0.0", HostPort: "5984"}, @@ -57,18 +65,15 @@ func TestMain(m *testing.M) { retryFn := func(resource *dockertest.Resource) (err error) { client, err = kivik.New("couch", fmt.Sprintf("http://%s:%s@%s/", user, pass, host)) if err != nil { - return err + return } - _, err = client.Ping(context.TODO()) + err = setup() return } purgeFn, err := utils.CreateContainer(opts, retryFn) if err != nil { log.Fatal(err) } - if err := setup(); err != nil { - log.Fatal(err) - } // run tests code := m.Run() @@ -115,6 +120,10 @@ func setup() (err error) { for _, database := range dbs { // create database err = client.CreateDB(context.TODO(), database) + // DB already created + if kivik.StatusCode(err) == http.StatusPreconditionFailed { + err = nil + } if err != nil { return } @@ -131,6 +140,9 @@ func setup() (err error) { func execute(queries []map[string]interface{}, db *kivik.DB) (err error) { for _, query := range queries { _, err := db.Put(context.TODO(), query["_id"].(string), query) + if kivik.StatusCode(err) == http.StatusConflict { + err = nil + } if err != nil { return err } diff --git a/plugins/extractors/couchdb/localConfig/local.ini b/plugins/extractors/couchdb/localConfig/local.ini new file mode 100644 index 000000000..adcfedcaa --- /dev/null +++ b/plugins/extractors/couchdb/localConfig/local.ini @@ -0,0 +1,7 @@ +[couchdb] single_node=true + +[admins] +meteor_test_user = -pbkdf2-5ab0d9d0d782edcd1353a0b058a4ea48618e363a,11726daaa26701e6ae8eccb6748f6abb,10 + +[chttpd_auth] +secret = ba1f0b5979e7e969f603da7710f8d7bf diff --git a/plugins/extractors/elastic/elastic_test.go b/plugins/extractors/elastic/elastic_test.go index f456881b0..962f24ef6 100644 --- a/plugins/extractors/elastic/elastic_test.go +++ b/plugins/extractors/elastic/elastic_test.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build plugins +// +build plugins package elastic_test diff --git a/plugins/extractors/kafka/kafka_test.go b/plugins/extractors/kafka/kafka_test.go index 274f7e0c3..b8bfd3fb7 100644 --- a/plugins/extractors/kafka/kafka_test.go +++ b/plugins/extractors/kafka/kafka_test.go @@ -1,10 +1,11 @@ -//go:build integration -// +build integration +//go:build plugins +// +build plugins package kafka_test import ( "context" + "errors" "log" "net" @@ -54,6 +55,17 @@ func TestMain(m *testing.M) { if err != nil { return } + + // healthcheck + brokerList, err := conn.Brokers() + if err != nil { + return + } + if len(brokerList) == 0 { + err = errors.New("not ready") + return + } + broker, err = conn.Controller() if err != nil { conn.Close() diff --git a/plugins/extractors/redshift/redshift_test.go b/plugins/extractors/redshift/redshift_test.go index ba6142c1e..064fd2826 100644 --- a/plugins/extractors/redshift/redshift_test.go +++ b/plugins/extractors/redshift/redshift_test.go @@ -1,3 +1,6 @@ +//go:build plugins +// +build plugins + package redshift_test import ( diff --git a/plugins/extractors/superset/localConfig/.gitignore b/plugins/extractors/superset/localConfig/.gitignore new file mode 100644 index 000000000..ed8ebf583 --- /dev/null +++ b/plugins/extractors/superset/localConfig/.gitignore @@ -0,0 +1 @@ +__pycache__ \ No newline at end of file diff --git a/plugins/extractors/superset/localConfig/superset_config.py b/plugins/extractors/superset/localConfig/superset_config.py new file mode 100644 index 000000000..89d8de40c --- /dev/null +++ b/plugins/extractors/superset/localConfig/superset_config.py @@ -0,0 +1,2 @@ +# Flask-WTF flag for CSRF +WTF_CSRF_ENABLED = False \ No newline at end of file diff --git a/plugins/extractors/superset/superset_test.go b/plugins/extractors/superset/superset_test.go index e24399678..455c8146a 100644 --- a/plugins/extractors/superset/superset_test.go +++ b/plugins/extractors/superset/superset_test.go @@ -1,5 +1,5 @@ -//go:build integration -// +build integration +//go:build plugins +// +build plugins package superset_test @@ -24,14 +24,13 @@ import ( "github.com/odpf/meteor/test/mocks" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) const ( user = "admin" pass = "admin" - port = "8080" + port = "9999" provider = "db" dashboardTitle = "random dashboard" mockChart = "random chart" @@ -53,18 +52,25 @@ type responseToken struct { } type securityToken struct { - CsrfToken string `json:"csrf_token"` + CsrfToken string `json:"result"` } func TestMain(m *testing.M) { + pwd, err := os.Getwd() + if err != nil { + log.Fatal(err) + } // setup test opts := dockertest.RunOptions{ Repository: "apache/superset", - Tag: "latest", + Tag: "6b136c2bc9a6c9756e5319b045e3c42da06243cb", ExposedPorts: []string{"8088"}, + Mounts: []string{ + fmt.Sprintf("%s/localConfig:/app/pythonpath:rw", pwd), + }, PortBindings: map[docker.Port][]docker.PortBinding{ "8088": { - {HostIP: "0.0.0.0", HostPort: "8088"}, + {HostIP: "0.0.0.0", HostPort: port}, }, }, } @@ -74,13 +80,44 @@ func TestMain(m *testing.M) { if err != nil { return } - body, err := io.ReadAll(res.Body) - if err := res.Body.Close(); err != nil { - return err - } if res.StatusCode != http.StatusOK { - return errors.Wrapf(err, "Response failed with status code: %d and\nbody: %s\n", res.StatusCode, body) + err = fmt.Errorf("Response failed with status code: %d", res.StatusCode) + return } + + var stdout bytes.Buffer + _, err = resource.Exec( + []string{ + "superset", "fab", "create-admin", + "--username", user, + "--firstname", "Superset", + "--lastname", "Admin", + "--email", "admin@superset.com", + "--password", pass, + }, + dockertest.ExecOptions{StdOut: &stdout}, + ) + + _, err = resource.Exec( + []string{ + "superset", "db", "upgrade", + }, + dockertest.ExecOptions{StdOut: &stdout}, + ) + + _, err = resource.Exec( + []string{ + "superset", "load_examples", "--load-test-data", + }, + dockertest.ExecOptions{StdOut: &stdout}, + ) + + _, err = resource.Exec( + []string{ + "superset", "init", + }, + dockertest.ExecOptions{StdOut: &stdout}, + ) return } @@ -89,6 +126,7 @@ func TestMain(m *testing.M) { if err != nil { log.Fatal(err) } + if err := setup(); err != nil { log.Fatal(err) } @@ -134,12 +172,11 @@ func TestExtract(t *testing.T) { assert.NoError(t, err) var urns []string - fmt.Println(emitter.Get()) for _, record := range emitter.Get() { dashboard := record.Data().(*assetsv1beta1.Dashboard) urns = append(urns, dashboard.Resource.Urn) } - assert.Equal(t, 10, len(urns)) + assert.Equal(t, 4, len(urns)) }) } @@ -149,10 +186,6 @@ func setup() (err error) { if err != nil { return } - err = addDashboard() - if err != nil { - return - } return } @@ -187,53 +220,16 @@ func setCsrfToken() (err error) { return } -// addDashboard adds a dashboard to the database -func addDashboard() (err error) { - type responseID struct { - ID int `json:"id"` - } - payload := map[string]interface{}{ - "dashboard_title": dashboardTitle, - } - var data responseID - if err = makeRequest("POST", host+"/api/v1/dashboard", payload, &data); err != nil { - return - } - dashboardID = data.ID - if err = addChart(dashboardID); err != nil { - return - } - return -} - -// addChart adds a chart to the dashboard -func addChart(id int) (err error) { - payload := map[string]interface{}{ - "dashboards": id, - "slice_name": mockChart, - } - type response struct { - ID int `json:"id"` - } - var data response - err = makeRequest("POST", host+"/api/v1/chart/", payload, &data) - if err != nil { - return - } - chartID = data.ID - return -} - // makeRequest helper function to avoid rewriting a request func makeRequest(method, url string, payload interface{}, data interface{}) (err error) { jsonifyPayload, err := json.Marshal(payload) if err != nil { - return errors.Wrap(err, "failed to encode the payload JSON") + return fmt.Errorf("failed to encode the payload JSON: %w", err) } body := bytes.NewBuffer(jsonifyPayload) req, err := http.NewRequest(method, url, body) if err != nil { - return errors.Wrap(err, "failed to create request") + return fmt.Errorf("failed to create request: %w", err) } var bearer = "Bearer " + accessToken req.Header.Set("Content-Type", "application/json") @@ -243,17 +239,22 @@ func makeRequest(method, url string, payload interface{}, data interface{}) (err res, err := client.Do(req) if err != nil { - return errors.Wrap(err, "failed to generate response") + return fmt.Errorf("failed to generate response: %w", err) } if res.StatusCode < 200 || res.StatusCode >= 300 { - return errors.Wrapf(err, "response failed with status code: %d", res.StatusCode) + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return err + } + bodyString := string(bodyBytes) + return fmt.Errorf("response failed with status code: %d and body: %s", res.StatusCode, bodyString) } b, err := ioutil.ReadAll(res.Body) if err != nil { - return errors.Wrap(err, "failed to read response body") + return fmt.Errorf("failed to read response body: %w", err) } if err = json.Unmarshal(b, &data); err != nil { - return errors.Wrapf(err, "failed to parse: %s", string(b)) + return fmt.Errorf("failed to parse: %s, err: %w", string(b), err) } return } diff --git a/test/utils/dockertest.go b/test/utils/dockertest.go index 736370056..95fb72b56 100644 --- a/test/utils/dockertest.go +++ b/test/utils/dockertest.go @@ -15,7 +15,7 @@ import ( func CreateContainer(opts dockertest.RunOptions, retryOp func(r *dockertest.Resource) error) (purgeFn func() error, err error) { pool, err := dockertest.NewPool("") // exponential backoff-retry, because the application in the container might not be ready to accept connections yet - pool.MaxWait = 120 * time.Second + pool.MaxWait = 300 * time.Second if err != nil { return purgeFn, fmt.Errorf("could not create dockertest pool: %s", err) }