diff --git a/plugins/extractors/bigquery/README.md b/plugins/extractors/bigquery/README.md index 780fb57ec..d2ef2b27e 100644 --- a/plugins/extractors/bigquery/README.md +++ b/plugins/extractors/bigquery/README.md @@ -9,7 +9,8 @@ source: project_id: google-project-id table_pattern: gofood.fact_ profile_column: true - credentials_json: + service_account_base64: _________BASE64_ENCODED_SERVICE_ACCOUNT_________________ + service_account_json: { "type": "service_account", "private_key_id": "xxxxxxx", @@ -33,7 +34,8 @@ source: | Key | Value | Example | Description | | | :-- | :---- | :------ | :---------- | :- | | `project_id` | `string` | `my-project` | BigQuery Project ID | *required* | -| `credentials_json` | `string` | `{"private_key": .., "private_id": ...}` | Service Account in JSON string | *optional* | +| `service_account_base64` | `string` | `____BASE64_ENCODED_SERVICE_ACCOUNT____` | Service Account in base64 encoded string. Takes precedence over `service_account_json` value | *optional* | +| `service_account_json` | `string` | `{"private_key": .., "private_id": ...}` | Service Account in JSON string | *optional* | | `table_pattern` | `string` | `gofood.fact_` | Regex pattern to filter which bigquery table to scan (whitelist) | *optional* | | `include_column_profile` | `bool` | `true` | true if you want to profile the column value such min, max, med, avg, top, and freq | *optional* | | `max_preview_rows` | `int` | `30` | max number of preview rows to fetch, `0` will skip preview fetching. Default to `30`. | *optional* | diff --git a/plugins/extractors/bigquery/bigquery.go b/plugins/extractors/bigquery/bigquery.go index df8d4c3ed..b36eebc72 100644 --- a/plugins/extractors/bigquery/bigquery.go +++ b/plugins/extractors/bigquery/bigquery.go @@ -3,6 +3,7 @@ package bigquery import ( "context" _ "embed" // used to print the embedded assets + "encoding/base64" "encoding/json" "html/template" "strings" @@ -30,7 +31,9 @@ var summary string // Config holds the set of configuration for the bigquery extractor type Config struct { - ProjectID string `mapstructure:"project_id" validate:"required"` + ProjectID string `mapstructure:"project_id" validate:"required"` + // ServiceAccountBase64 takes precedence over ServiceAccountJSON field + ServiceAccountBase64 string `mapstructure:"service_account_base64"` ServiceAccountJSON string `mapstructure:"service_account_json"` TablePattern string `mapstructure:"table_pattern"` IncludeColumnProfile bool `mapstructure:"include_column_profile"` @@ -44,6 +47,9 @@ var sampleConfig = ` project_id: google-project-id table_pattern: gofood.fact_ include_column_profile: true +# Only one of service_account_base64 / service_account_json is needed. +# If both are present, service_account_base64 takes precedence +service_account_base64: ____base64_encoded_service_account____ service_account_json: |- { "type": "service_account", @@ -137,11 +143,20 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) // Create big query client func (e *Extractor) createClient(ctx context.Context) (*bigquery.Client, error) { - if e.config.ServiceAccountJSON == "" { + if e.config.ServiceAccountBase64 == "" && e.config.ServiceAccountJSON == "" { e.logger.Info("credentials are not specified, creating bigquery client using default credentials...") return bigquery.NewClient(ctx, e.config.ProjectID) } + if e.config.ServiceAccountBase64 != "" { + serviceAccountJSON, err := base64.StdEncoding.DecodeString(e.config.ServiceAccountBase64) + if err != nil || len(serviceAccountJSON) == 0 { + return nil, errors.Wrap(err, "failed to decode base64 service account") + } + // overwrite ServiceAccountJSON with credentials from ServiceAccountBase64 value + e.config.ServiceAccountJSON = string(serviceAccountJSON) + } + return bigquery.NewClient(ctx, e.config.ProjectID, option.WithCredentialsJSON([]byte(e.config.ServiceAccountJSON))) } diff --git a/plugins/extractors/bigquery/bigquery_test.go b/plugins/extractors/bigquery/bigquery_test.go index 6b7a50a72..ef458f7b2 100644 --- a/plugins/extractors/bigquery/bigquery_test.go +++ b/plugins/extractors/bigquery/bigquery_test.go @@ -39,4 +39,17 @@ func TestInit(t *testing.T) { assert.NotEqual(t, plugins.InvalidConfigError{}, err) }) + t.Run("should return error if service_account_base64 config is invalid", func(t *testing.T) { + extr := bigquery.New(utils.Logger) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := extr.Init(ctx, plugins.Config{ + URNScope: "test-bigquery", + RawConfig: map[string]interface{}{ + "project_id": "google-project-id", + "service_account_base64": "----", // invalid + }}) + + assert.ErrorContains(t, err, "failed to decode base64 service account") + }) }