diff --git a/.github/workflows/plugin_test.yml b/.github/workflows/plugin_test.yml deleted file mode 100644 index d70b37ce6..000000000 --- a/.github/workflows/plugin_test.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Plugins Test - -on: - push: - branches: - - main - pull_request: - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: '1.18' - cache: true - - name: Install dependencies - run: sudo apt-get install build-essential - - name: Run Test - run: make test-plugins diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9220c5f81..d7bc43f97 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,9 @@ name: Test -on: [push, pull_request] + +on: + push: + pull_request: + workflow_dispatch: jobs: test: @@ -16,9 +20,58 @@ jobs: run: sudo apt-get install build-essential - name: Run Test run: make test + - name: Upload coverage artifact + uses: actions/upload-artifact@v3 + with: + name: coverage + path: coverage.out + + plugins-test: + runs-on: ubuntu-latest + if: | + github.ref == 'refs/heads/main' || + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.18' + cache: true + - name: Install dependencies + run: sudo apt-get install build-essential + - name: Run Test + run: make test-plugins + - name: Upload coverage artifact + uses: actions/upload-artifact@v3 + with: + name: coverage-plugins + path: coverage-plugins.out + + coverage: + runs-on: ubuntu-latest + needs: [test, plugins-test] + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.18' + cache: true + - name: Download coverage + uses: actions/download-artifact@v3 + with: + name: coverage + - name: Download plugins coverage + uses: actions/download-artifact@v3 + with: + name: coverage-plugins - name: Install goveralls and send coverage env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | go install github.com/mattn/goveralls@v0.0.11 - goveralls -coverprofile=coverage.out -service=github + goveralls -coverprofile=coverage.out,coverage-plugins.out -service=github diff --git a/Makefile b/Makefile index 6e84f6bf7..aacb70119 100644 --- a/Makefile +++ b/Makefile @@ -21,15 +21,17 @@ copy-config: test: go test ./... -coverprofile=coverage.out -test-coverage: test - go tool cover -html=coverage.out - test-e2e: go test ./test/e2e -tags=integration -count=1 test-plugins: @echo " > Testing plugins with tag 'plugins'" - go test ./plugins... -tags=plugins -count=1 + go test ./plugins... -tags=plugins -coverprofile=coverage-plugins.out -parallel=1 + +test-coverage: # test test-plugins + cp coverage.out coverage-all.out + tail -n +2 coverage-plugins.out >> coverage-all.out + go tool cover -html=coverage-all.out generate-proto: ## regenerate protos @echo " > cloning protobuf from odpf/proton" diff --git a/plugins/base_extractor_test.go b/plugins/base_extractor_test.go index 36ec5ce03..4043a96a4 100644 --- a/plugins/base_extractor_test.go +++ b/plugins/base_extractor_test.go @@ -1,3 +1,6 @@ +//go:build plugins +// +build plugins + package plugins_test import ( diff --git a/plugins/extractors/bigquery/auditlog/auditlog_test.go b/plugins/extractors/bigquery/auditlog/auditlog_test.go index 68d4ab36e..4c6e84a06 100644 --- a/plugins/extractors/bigquery/auditlog/auditlog_test.go +++ b/plugins/extractors/bigquery/auditlog/auditlog_test.go @@ -1,3 +1,6 @@ +//go:build plugins +// +build plugins + package auditlog import ( diff --git a/plugins/extractors/bigquery/auditlog/logdata_test.go b/plugins/extractors/bigquery/auditlog/logdata_test.go index 9b6ce0ca8..ed0c55a7b 100644 --- a/plugins/extractors/bigquery/auditlog/logdata_test.go +++ b/plugins/extractors/bigquery/auditlog/logdata_test.go @@ -1,3 +1,6 @@ +//go:build plugins +// +build plugins + package auditlog import ( diff --git a/plugins/extractors/bigquery/auditlog/stats_test.go b/plugins/extractors/bigquery/auditlog/stats_test.go index 52f813fca..013faf430 100644 --- a/plugins/extractors/bigquery/auditlog/stats_test.go +++ b/plugins/extractors/bigquery/auditlog/stats_test.go @@ -1,3 +1,6 @@ +//go:build plugins +// +build plugins + package auditlog import ( diff --git a/plugins/extractors/bigquery/auditlog/testdata.go b/plugins/extractors/bigquery/auditlog/testdata_test.go similarity index 99% rename from plugins/extractors/bigquery/auditlog/testdata.go rename to plugins/extractors/bigquery/auditlog/testdata_test.go index 3da1e4cb6..5e5420879 100644 --- a/plugins/extractors/bigquery/auditlog/testdata.go +++ b/plugins/extractors/bigquery/auditlog/testdata_test.go @@ -1,3 +1,6 @@ +//go:build plugins +// +build plugins + package auditlog import ( diff --git a/plugins/extractors/bigquery/sqlparser/parser_test.go b/plugins/extractors/bigquery/sqlparser/parser_test.go index 5c4c12513..9666dedcd 100644 --- a/plugins/extractors/bigquery/sqlparser/parser_test.go +++ b/plugins/extractors/bigquery/sqlparser/parser_test.go @@ -1,3 +1,6 @@ +//go:build plugins +// +build plugins + package sqlparser import ( @@ -208,3 +211,60 @@ func TestComplexQueries(t *testing.T) { }) } } + +const testDataSQL1 = ` +SELECT +COUNT(DISTINCT user_id) AS purchasers_count +FROM +-- PLEASE REPLACE WITH YOUR TABLE NAME. +` + "`" + "YOUR_TABLE.events_*" + "`" + ` +WHERE +event_name IN ('in_app_purchase', 'purchase') +-- PLEASE REPLACE WITH YOUR DESIRED DATE RANGE +AND _TABLE_SUFFIX BETWEEN '20180501' AND '20240131' +AND ` + "`" + "_TABLE_SUFFIX" + "`" + ` BETWEEN '1' AND '2';` + +const testDataSQL2 = ` +USING ( + SELECT + t.column_1, + t.column_2, + CAST(count(distinct(id)) AS BIGNUMERIC) as total, + TIMESTAMP("2021-10-29", "UTC") as window_start_time, + CAST("2021-11-22T02:01:06Z" AS TIMESTAMP) as event_timestamp + FROM` + "`" + `a-dataset.a-specific_table-2021` + "`" + `, + UNNEST(trial) as t + WHERE + t.column_type = 'tester' AND + t.param_id = "280481a2-2384-4b81-aa3e-214ac60b31db" AND + event_timestamp >= TIMESTAMP("2021-10-29", "UTC") AND + event_timestamp < TIMESTAMP("2021-11-22T02:01:06Z") + GROUP BY t.column_1, t.column_2 + ) source + ON + target.column_1 = source.column_1 + and target.variant_name = source.variant_name + and DATE(target.event_timestamp) = DATE(source.event_timestamp) + WHEN matched then update set + target.total = source.total, + target.event_timestamp = source.event_timestamp + WHEN NOT MATCHED then insert(param1,param2,param3,param4,param5_ext) values( + source.param1, + source.param2, + source.param3, + source.param4, + source.param5_ext + )` + +const testDataSQL3 = ` +SELECT + COUNT(DISTINCT user_id) AS acquired_users_count +FROM + -- PLEASE REPLACE WITH YOUR TABLE NAME. + ` + "`" + "YOUR_TABLE.events_*" + "`" + ` +WHERE + traffic_source.source = 'google' + AND traffic_source.medium = 'cpc' + AND traffic_source.name = 'VTA-Test-Android' + -- PLEASE REPLACE YOUR DESIRED DATE RANGE. + AND _TABLE_SUFFIX BETWEEN '20180521' AND '20240131';` diff --git a/plugins/extractors/bigquery/sqlparser/testdatasql.go b/plugins/extractors/bigquery/sqlparser/testdatasql.go deleted file mode 100644 index 896bc1e7e..000000000 --- a/plugins/extractors/bigquery/sqlparser/testdatasql.go +++ /dev/null @@ -1,58 +0,0 @@ -package sqlparser - -const testDataSQL1 = ` -SELECT -COUNT(DISTINCT user_id) AS purchasers_count -FROM --- PLEASE REPLACE WITH YOUR TABLE NAME. -` + "`" + "YOUR_TABLE.events_*" + "`" + ` -WHERE -event_name IN ('in_app_purchase', 'purchase') --- PLEASE REPLACE WITH YOUR DESIRED DATE RANGE -AND _TABLE_SUFFIX BETWEEN '20180501' AND '20240131' -AND ` + "`" + "_TABLE_SUFFIX" + "`" + ` BETWEEN '1' AND '2';` - -const testDataSQL2 = ` -USING ( - SELECT - t.column_1, - t.column_2, - CAST(count(distinct(id)) AS BIGNUMERIC) as total, - TIMESTAMP("2021-10-29", "UTC") as window_start_time, - CAST("2021-11-22T02:01:06Z" AS TIMESTAMP) as event_timestamp - FROM` + "`" + `a-dataset.a-specific_table-2021` + "`" + `, - UNNEST(trial) as t - WHERE - t.column_type = 'tester' AND - t.param_id = "280481a2-2384-4b81-aa3e-214ac60b31db" AND - event_timestamp >= TIMESTAMP("2021-10-29", "UTC") AND - event_timestamp < TIMESTAMP("2021-11-22T02:01:06Z") - GROUP BY t.column_1, t.column_2 - ) source - ON - target.column_1 = source.column_1 - and target.variant_name = source.variant_name - and DATE(target.event_timestamp) = DATE(source.event_timestamp) - WHEN matched then update set - target.total = source.total, - target.event_timestamp = source.event_timestamp - WHEN NOT MATCHED then insert(param1,param2,param3,param4,param5_ext) values( - source.param1, - source.param2, - source.param3, - source.param4, - source.param5_ext - )` - -const testDataSQL3 = ` -SELECT - COUNT(DISTINCT user_id) AS acquired_users_count -FROM - -- PLEASE REPLACE WITH YOUR TABLE NAME. - ` + "`" + "YOUR_TABLE.events_*" + "`" + ` -WHERE - traffic_source.source = 'google' - AND traffic_source.medium = 'cpc' - AND traffic_source.name = 'VTA-Test-Android' - -- PLEASE REPLACE YOUR DESIRED DATE RANGE. - AND _TABLE_SUFFIX BETWEEN '20180521' AND '20240131';` diff --git a/plugins/extractors/elastic/elastic_test.go b/plugins/extractors/elastic/elastic_test.go index bb20bce09..a002398c2 100644 --- a/plugins/extractors/elastic/elastic_test.go +++ b/plugins/extractors/elastic/elastic_test.go @@ -240,7 +240,7 @@ func getExpectedVal(t *testing.T) []models.Record { func matchRecords(t *testing.T, expected, actual []models.Record) { if actual[0].Data().Name == "index2" { - //swap index order + // swap index order temp := actual[0] actual[0] = actual[1] actual[1] = temp diff --git a/plugins/extractors/gsuite/gsuite_test.go b/plugins/extractors/gsuite/gsuite_test.go index ac17752f2..59e556574 100644 --- a/plugins/extractors/gsuite/gsuite_test.go +++ b/plugins/extractors/gsuite/gsuite_test.go @@ -1,3 +1,6 @@ +//go:build plugins +// +build plugins + package gsuite_test import ( diff --git a/plugins/extractors/redash/redash_test.go b/plugins/extractors/redash/redash_test.go index ad44ff2dc..0edf93ffc 100644 --- a/plugins/extractors/redash/redash_test.go +++ b/plugins/extractors/redash/redash_test.go @@ -1,3 +1,6 @@ +//go:build plugins +// +build plugins + package redash_test import (