feat(bigquery): use service_account_base64 to pass credentials (#438)

* feat(bigquery): use service_account_base64 to pass credentials

* use `service_account_base64` config that will take precedence over service_account_json in bigquery extractor

Co-authored-by: Suhas Karanth <sudo-suhas@users.noreply.github.com>

Author: bsushmith

plugins/extractors/bigquery/README.md

@@ -10,7 +10,8 @@ source:
     table_pattern: gofood.fact_
     max_page_size: 100
     profile_column: true
-    credentials_json:
+    service_account_base64: _________BASE64_ENCODED_SERVICE_ACCOUNT_________________
+    service_account_json:
       {
         "type": "service_account",
         "private_key_id": "xxxxxxx",
@@ -34,7 +35,8 @@ source:
 | Key | Value | Example | Description |    |
 | :-- | :---- | :------ | :---------- | :-- |
 | `project_id` | `string` | `my-project` | BigQuery Project ID | *required* |
-| `credentials_json` | `string` | `{"private_key": .., "private_id": ...}` | Service Account in JSON string | *optional* |
+| `service_account_base64` | `string` | `____BASE64_ENCODED_SERVICE_ACCOUNT____` | Service Account in base64 encoded string. Takes precedence over `service_account_json` value | *optional* |
+| `service_account_json` | `string` | `{"private_key": .., "private_id": ...}` | Service Account in JSON string | *optional* |
 | `table_pattern` | `string` | `gofood.fact_` | Regex pattern to filter which bigquery table to scan (whitelist) | *optional* |
 | `max_page_size` | `int` | `100` | max page size hint used for fetching datasets/tables/rows from bigquery | *optional* |
 | `include_column_profile` | `bool` | `true` | true if you want to profile the column value such min, max, med, avg, top, and freq | *optional* |

plugins/extractors/bigquery/bigquery.go

@@ -3,6 +3,7 @@ package bigquery
 import (
 	"context"
 	_ "embed" // used to print the embedded assets
+	"encoding/base64"
 	"encoding/json"
 	"html/template"
 	"strings"
@@ -30,7 +31,9 @@ var summary string
 
 // Config holds the set of configuration for the bigquery extractor
 type Config struct {
-	ProjectID            string   `mapstructure:"project_id" validate:"required"`
+	ProjectID string `mapstructure:"project_id" validate:"required"`
+	// ServiceAccountBase64 takes precedence over ServiceAccountJSON field
+	ServiceAccountBase64 string   `mapstructure:"service_account_base64"`
 	ServiceAccountJSON   string   `mapstructure:"service_account_json"`
 	MaxPageSize          int      `mapstructure:"max_page_size"`
 	TablePattern         string   `mapstructure:"table_pattern"`
@@ -50,6 +53,9 @@ project_id: google-project-id
 table_pattern: gofood.fact_
 max_page_size: 100
 include_column_profile: true
+# Only one of service_account_base64 / service_account_json is needed. 
+# If both are present, service_account_base64 takes precedence
+service_account_base64: ____base64_encoded_service_account____
 service_account_json: |-
   {
     "type": "service_account",
@@ -144,11 +150,20 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error)
 
 // Create big query client
 func (e *Extractor) createClient(ctx context.Context) (*bigquery.Client, error) {
-	if e.config.ServiceAccountJSON == "" {
+	if e.config.ServiceAccountBase64 == "" && e.config.ServiceAccountJSON == "" {
 		e.logger.Info("credentials are not specified, creating bigquery client using default credentials...")
 		return bigquery.NewClient(ctx, e.config.ProjectID)
 	}
 
+	if e.config.ServiceAccountBase64 != "" {
+		serviceAccountJSON, err := base64.StdEncoding.DecodeString(e.config.ServiceAccountBase64)
+		if err != nil || len(serviceAccountJSON) == 0 {
+			return nil, errors.Wrap(err, "failed to decode base64 service account")
+		}
+		// overwrite ServiceAccountJSON with credentials from ServiceAccountBase64 value
+		e.config.ServiceAccountJSON = string(serviceAccountJSON)
+	}
+
 	return bigquery.NewClient(ctx, e.config.ProjectID, option.WithCredentialsJSON([]byte(e.config.ServiceAccountJSON)))
 }
 

plugins/extractors/bigquery/bigquery_test.go

@@ -39,4 +39,17 @@ func TestInit(t *testing.T) {
 
 		assert.NotEqual(t, plugins.InvalidConfigError{}, err)
 	})
+	t.Run("should return error if service_account_base64 config is invalid", func(t *testing.T) {
+		extr := bigquery.New(utils.Logger)
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		err := extr.Init(ctx, plugins.Config{
+			URNScope: "test-bigquery",
+			RawConfig: map[string]interface{}{
+				"project_id":             "google-project-id",
+				"service_account_base64": "----", // invalid
+			}})
+
+		assert.ErrorContains(t, err, "failed to decode base64 service account")
+	})
 }