From b806878025822e335de778fd7783129744de24dd Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 9 Aug 2022 00:13:31 +0530 Subject: [PATCH 01/19] feat: add google workspace extractor --- .../googleworkspace/googleworkspace.go | 134 ++++++++++++++++++ plugins/extractors/populate.go | 1 + 2 files changed, 135 insertions(+) create mode 100644 plugins/extractors/googleworkspace/googleworkspace.go diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go new file mode 100644 index 000000000..b51159995 --- /dev/null +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -0,0 +1,134 @@ +package googleworkspace + +import ( + "context" + _ "embed" // used to print the embedded assets + + "github.com/pkg/errors" + + "github.com/odpf/meteor/models" + facetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/facets/v1beta1" + assetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/v1beta1" + "github.com/odpf/meteor/plugins" + "github.com/odpf/meteor/registry" + "github.com/odpf/meteor/utils" + "github.com/odpf/salt/log" + "golang.org/x/oauth2/google" + admin "google.golang.org/api/admin/directory/v1" + "google.golang.org/api/option" +) + +type Config struct { + ServiceAccountJSON string `mapstructure:"service_account_json" validate:"required"` + UserEmail string `mapstructure:"user_email" validate:"required"` +} + +var sampleConfig = ` +service_account_json: { + "type": "service_account", + "project_id": "odpf-project", + "private_key_id": "3cb2saasa3ef788dvdvdvdvdvdssdvds57", + "private_key": "-----BEGIN PRIVATE KEY-----\njbjabdjbajd\n-----END PRIVATE KEY-----\n", + "client_email": "meteor-sa@odpf-project.iam.gserviceaccount.com", + "client_id": "1100599572858548635286", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-project.iam.gserviceaccount.com" +} +user_email: user@odpf.com` + +var info = plugins.Info{ + Description: "User list from Google Workspace", + SampleConfig: sampleConfig, + Tags: []string{"platform", "extractor"}, +} + +// Extractor manages the extraction of data from the extractor +type Extractor struct { + plugins.BaseExtractor + logger log.Logger + config Config + client *admin.Service +} + +// New returns a pointer to an initialized Extractor Object +func New(logger log.Logger) *Extractor { + e := &Extractor{ + logger: logger, + } + e.BaseExtractor = plugins.NewBaseExtractor(info, &e.config) + + return e +} + +// Init initializes the extractor +func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error) { + if err = e.BaseExtractor.Init(ctx, config); err != nil { + return err + } + + jwtConfig, err := google.JWTConfigFromJSON([]byte(e.config.ServiceAccountJSON), admin.AdminDirectoryUserScope) + if err != nil { + return errors.Wrap(err, "JWTConfigFromJSON") + } + jwtConfig.Subject = e.config.UserEmail + + ts := jwtConfig.TokenSource(ctx) + + e.client, err = admin.NewService(ctx, option.WithTokenSource(ts)) + if err != nil { + return errors.Wrap(err, "NewService") + } + + return +} + +// Extract extracts the data from the extractor +// The data is returned as a list of assets.Asset +func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) { + var status string + r, err := e.client.Users.List().Customer("my_customer").MaxResults(10). + OrderBy("email").Do() + if err != nil { + e.logger.Error("Unable to retrieve users in domain: %v", err) + } + + if len(r.Users) == 0 { + e.logger.Info("No users found.\n") + } else { + for _, u := range r.Users { + if !u.Suspended { + status = "not suspended" + } else { + status = "suspended" + } + + emit(models.NewRecord(&assetsv1beta1.User{ + Email: u.PrimaryEmail, + FullName: u.Name.FullName, + LastName: u.Name.FamilyName, + Status: status, + Properties: &facetsv1beta1.Properties{ + Attributes: utils.TryParseMapToProto(map[string]interface{}{ + "organisation": u.Organizations, + "relation": u.Relations, + "custom_schemas": u.CustomSchemas, + "org_unit_path": u.OrgUnitPath, + }), + }, + })) + } + } + + return nil +} + +// init registers the extractor to catalog +func init() { + if err := registry.Extractors.Register("googleworkspace", func() plugins.Extractor { + return New(plugins.GetLog()) + }); err != nil { + panic(err) + } +} diff --git a/plugins/extractors/populate.go b/plugins/extractors/populate.go index 6f6ea3ae1..3cddec014 100644 --- a/plugins/extractors/populate.go +++ b/plugins/extractors/populate.go @@ -10,6 +10,7 @@ import ( _ "github.com/odpf/meteor/plugins/extractors/elastic" _ "github.com/odpf/meteor/plugins/extractors/gcs" _ "github.com/odpf/meteor/plugins/extractors/github" + _ "github.com/odpf/meteor/plugins/extractors/googleworkspace" _ "github.com/odpf/meteor/plugins/extractors/grafana" _ "github.com/odpf/meteor/plugins/extractors/kafka" _ "github.com/odpf/meteor/plugins/extractors/mariadb" From 1b8b6739b57192850245e81e15b84e3dc3533f53 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 9 Aug 2022 23:06:24 +0530 Subject: [PATCH 02/19] fix: data extraction based on data type of org, relations, custom schema & orgunitpath --- .../googleworkspace/googleworkspace.go | 81 ++++++++++++++++--- 1 file changed, 71 insertions(+), 10 deletions(-) diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index b51159995..752373dd3 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -3,12 +3,12 @@ package googleworkspace import ( "context" _ "embed" // used to print the embedded assets + "fmt" + "reflect" "github.com/pkg/errors" "github.com/odpf/meteor/models" - facetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/facets/v1beta1" - assetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/v1beta1" "github.com/odpf/meteor/plugins" "github.com/odpf/meteor/registry" "github.com/odpf/meteor/utils" @@ -16,6 +16,10 @@ import ( "golang.org/x/oauth2/google" admin "google.golang.org/api/admin/directory/v1" "google.golang.org/api/option" + + commonv1beta1 "github.com/odpf/meteor/models/odpf/assets/common/v1beta1" + facetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/facets/v1beta1" + assetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/v1beta1" ) type Config struct { @@ -50,6 +54,7 @@ type Extractor struct { logger log.Logger config Config client *admin.Service + emit plugins.Emit } // New returns a pointer to an initialized Extractor Object @@ -88,7 +93,8 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error) // The data is returned as a list of assets.Asset func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) { var status string - r, err := e.client.Users.List().Customer("my_customer").MaxResults(10). + e.emit = emit + r, err := e.client.Users.List().Customer("my_customer"). OrderBy("email").Do() if err != nil { e.logger.Error("Unable to retrieve users in domain: %v", err) @@ -104,18 +110,22 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) status = "suspended" } - emit(models.NewRecord(&assetsv1beta1.User{ + var userAttributes = make(map[string]interface{}) + userAttributes = getOrgAttributes(userAttributes, u.Organizations) + userAttributes = getRelationsAttributes(userAttributes, u.Relations) + userAttributes = getCustomSchemasAttributes(userAttributes, u.CustomSchemas) + userAttributes["org_unit_path"] = u.OrgUnitPath + e.emit(models.NewRecord(&assetsv1beta1.User{ + Resource: &commonv1beta1.Resource{ + Service: "google workspace", + Name: u.PrimaryEmail, + }, Email: u.PrimaryEmail, FullName: u.Name.FullName, LastName: u.Name.FamilyName, Status: status, Properties: &facetsv1beta1.Properties{ - Attributes: utils.TryParseMapToProto(map[string]interface{}{ - "organisation": u.Organizations, - "relation": u.Relations, - "custom_schemas": u.CustomSchemas, - "org_unit_path": u.OrgUnitPath, - }), + Attributes: utils.TryParseMapToProto(userAttributes), }, })) } @@ -132,3 +142,54 @@ func init() { panic(err) } } + +func getOrgAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { + if i != nil { + itr := reflect.ValueOf(i) + if itr.Kind() == reflect.Slice { + for idx := 0; idx < itr.Len(); idx++ { + valMap := reflect.ValueOf(itr.Index(idx).Interface()) + for _, key := range valMap.MapKeys() { + strct := valMap.MapIndex(key) + userAttributes[fmt.Sprintf("%v", key.Interface())] = strct.Interface() + } + } + } + } + return userAttributes +} + +func getRelationsAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { + if i != nil { + itr := reflect.ValueOf(i) + if itr.Kind() == reflect.Slice { + for idx := 0; idx < itr.Len(); idx++ { + valMap := reflect.ValueOf(itr.Index(idx).Interface()) + var relationType, relationValue string + for _, key := range valMap.MapKeys() { + strct := valMap.MapIndex(key) + if key.Interface().(string) == "type" { + relationType = strct.Interface().(string) + } else if key.Interface().(string) == "value" { + relationValue = strct.Interface().(string) + } + } + userAttributes[relationType] = relationValue + } + } + } + return userAttributes +} + +func getCustomSchemasAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { + if i != nil { + itr := reflect.ValueOf(i) + if itr.Kind() == reflect.Map { + for _, key := range itr.MapKeys() { + strct := itr.MapIndex(key) + userAttributes[fmt.Sprintf("%v", key.Interface())] = strct.Interface() + } + } + } + return userAttributes +} From 159793cab0798ddaccfcfb8ed9b064ec54a860a5 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 10 Aug 2022 20:26:27 +0530 Subject: [PATCH 03/19] test: add Init and Extract tests --- .../googleworkspace/googleworkspace.go | 41 ++++--- .../googleworkspace/googleworkspace_test.go | 105 ++++++++++++++++++ 2 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 plugins/extractors/googleworkspace/googleworkspace_test.go diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index 752373dd3..ea7937e79 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -13,6 +13,7 @@ import ( "github.com/odpf/meteor/registry" "github.com/odpf/meteor/utils" "github.com/odpf/salt/log" + "golang.org/x/oauth2" "golang.org/x/oauth2/google" admin "google.golang.org/api/admin/directory/v1" "google.golang.org/api/option" @@ -51,10 +52,11 @@ var info = plugins.Info{ // Extractor manages the extraction of data from the extractor type Extractor struct { plugins.BaseExtractor - logger log.Logger - config Config - client *admin.Service - emit plugins.Emit + logger log.Logger + config Config + TokenSource oauth2.TokenSource + Client *admin.Service + emit plugins.Emit } // New returns a pointer to an initialized Extractor Object @@ -75,16 +77,16 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error) jwtConfig, err := google.JWTConfigFromJSON([]byte(e.config.ServiceAccountJSON), admin.AdminDirectoryUserScope) if err != nil { - return errors.Wrap(err, "JWTConfigFromJSON") + return plugins.InvalidConfigError{} } - jwtConfig.Subject = e.config.UserEmail + if e.config.UserEmail == "" { + return plugins.InvalidConfigError{} + } + jwtConfig.Subject = e.config.UserEmail ts := jwtConfig.TokenSource(ctx) - e.client, err = admin.NewService(ctx, option.WithTokenSource(ts)) - if err != nil { - return errors.Wrap(err, "NewService") - } + e.TokenSource = ts return } @@ -94,10 +96,9 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error) func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) { var status string e.emit = emit - r, err := e.client.Users.List().Customer("my_customer"). - OrderBy("email").Do() + r, err := FetchUsers(ctx, e.TokenSource) if err != nil { - e.logger.Error("Unable to retrieve users in domain: %v", err) + return err } if len(r.Users) == 0 { @@ -143,6 +144,20 @@ func init() { } } +func FetchUsers(ctx context.Context, ts oauth2.TokenSource) (*admin.Users, error) { + srv, err := admin.NewService(ctx, option.WithTokenSource(ts)) + if err != nil { + return nil, err + } + + r, err := srv.Users.List().Customer("my_customer").OrderBy("email").Do() + if err != nil { + return nil, errors.Wrap(err, "Unable to retrieve users in domain") + } + + return r, nil +} + func getOrgAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { if i != nil { itr := reflect.ValueOf(i) diff --git a/plugins/extractors/googleworkspace/googleworkspace_test.go b/plugins/extractors/googleworkspace/googleworkspace_test.go new file mode 100644 index 000000000..87f6dae02 --- /dev/null +++ b/plugins/extractors/googleworkspace/googleworkspace_test.go @@ -0,0 +1,105 @@ +//go:build plugins +// +build plugins + +package googleworkspace_test + +import ( + "context" + "testing" + + "github.com/odpf/meteor/models" + commonv1beta1 "github.com/odpf/meteor/models/odpf/assets/common/v1beta1" + facetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/facets/v1beta1" + assetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/v1beta1" + "github.com/odpf/meteor/plugins" + "github.com/odpf/meteor/plugins/extractors/googleworkspace" + "github.com/odpf/meteor/test/mocks" + "github.com/odpf/meteor/test/utils" + utilities "github.com/odpf/meteor/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var urnScope string = "test-googleworkspace" + +func TestInit(t *testing.T) { + t.Run("should return error for empty user email", func(t *testing.T) { + err := googleworkspace.New(utils.Logger).Init(context.TODO(), plugins.Config{ + URNScope: urnScope, + RawConfig: map[string]interface{}{ + "user_email": "", + "service_account_json": "{\"type\":\"service_account\",\"project_id\":\"odpf-meteor\",\"private_key_id\":\"3cb2103ef7883845a5fdcsvdefe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggdvdvdEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdAUSjzKdbdfbdbYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYEin\\nHYj064sMvm8vbR5TcMQpnxYG86TGaPuIh30grz5dI39dtrUjttbdfbdvqRv0qu7I5\\nuELzp2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmBHbdfdi29bhoS+Ac\\n5ipT10yGF0FvT1f5KlJcHfsNoOGPJYePTaGxOW1zk680Z1Wdfbdf1xX9iw5/GUA3XM\\neon4p9X31ASgwbdbdplFZhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAECggEbdbddYz8nSmTWFMW2OtyvojIq+ab864ZGPCpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJThfOscyXbbdbdbfbfT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoYEmOsB2xgqjvKXR90r5wNJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMVVa6ZpxCTWvbQga+/ZPaqppgGps5yLDqc434c3A/lDCKBtqk\\njaQXHqKjuYUsoiyl2vbPbwGxc34343c6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOAEfc343535dJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwRrnbQFwhErWS1fZgg2PcQKBgQDMsRgDBfhgJX9sNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDh3ELmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXU3c4343c4QRAoGBAK9F\\nMO9dzFjfouVP63f/Nf3GeIlctuiE1r5IOX4di3qNe/P33iqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3x4XZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLGTw1vzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIxZKeq3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@odpf-meteor.iam.gserviceaccount.com\",\"client_id\":\"110059943435984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-meteor.iam.gserviceaccount.com\"}", + }}) + assert.ErrorAs(t, err, &plugins.InvalidConfigError{}) + }) + t.Run("should return error for invalid service account json", func(t *testing.T) { + err := googleworkspace.New(utils.Logger).Init(context.TODO(), plugins.Config{ + URNScope: urnScope, + RawConfig: map[string]interface{}{ + "user_email": "user@example.com", + "service_account_json": "invalide json", + }}) + assert.ErrorAs(t, err, &plugins.InvalidConfigError{}) + }) +} + +func TestExtract(t *testing.T) { + t.Run("should extract user details from google workspace", func(t *testing.T) { + + expectedData := []models.Record{ + models.NewRecord(&assetsv1beta1.User{ + Resource: &commonv1beta1.Resource{ + Service: "google workspace", + Name: "odpf admin", + }, + Email: "admin@odpf.com", + FullName: "John Doe", + LastName: "Doe", + Status: "not suspended", + Properties: &facetsv1beta1.Properties{ + Attributes: utilities.TryParseMapToProto(map[string]interface{}{ + "manager": "lorem@odpf.com", + }), + }, + }), + models.NewRecord(&assetsv1beta1.User{ + Resource: &commonv1beta1.Resource{ + Service: "google workspace", + Name: "ipsum", + }, + Email: "ipsum@odpf.com", + FullName: "Ipsum Lorum", + LastName: "Lorum", + Status: "not suspended", + Properties: &facetsv1beta1.Properties{ + Attributes: utilities.TryParseMapToProto(map[string]interface{}{ + "manager": "manager@odpf.com", + }), + }, + }), + } + + ctx := context.TODO() + + emitter := mocks.NewEmitter() + extractor := mocks.NewExtractor() + extractor.On("Init", mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("plugins.Emit")).Return(nil) + extractor.SetEmit(expectedData) + extractor.On("Extract", mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("plugins.Emit")).Return(nil) + + err := extractor.Init(ctx, plugins.Config{ + URNScope: urnScope, + RawConfig: map[string]interface{}{ + "user_email": "test@odpf.com", + "service_account_json": "{\"type\":\"service_account\",\"project_id\":\"odpf\",\"private_key_id\":\"3cb2103ef7883455a5f09712befe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBGDSNBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdASSjzKden0lgYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYEin\\nHYj064sMvm8vbR5TcMQpnxYG8SAGaPuIh30grz5dI39dtrUjttWWvtvqRv0qu7I5\\nuELzp2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmFDyNdOa6vi29bhoS+Ac\\n5ipT10yGF0FvT1f5KlJcHfsNoOGPJYeTRaGxOW1zk680Z1WFyB1xX9iw5/GUA3XM\\neon4p9X31ASgw6WTqplGDhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAECggEALZwVYz8nSmTWFMW2OtyvojIq+ab864BGHFpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJJHhfOscyXydDHjHXT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoTYmOsB2xgqjvKXR90r5wNJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMBBa6ZpxCTWvbQga+/ZPaqppgGps5yLDqcp1A/lDCKBtqk\\njaQXHqKjuYDSsoiyl2vbPbwGxIzYSv6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOAEfIhxadJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwRrnbQFwhErWS1fZgg2PcQKBgQDMsRgDBfhgJX9sNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDh3ELmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXUJq4hPQRAoGBAK9F\\nMO9dzFjfouVP63f/Nf3GeIlctuiE1r5IOX4di3qNe/P33iqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3x4XZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLGTw1vzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIxZKeq3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@odpf.iam.gserviceaccount.com\",\"client_id\":\"110059957285984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf.iam.gserviceaccount.com\"}", + }, + }) + if err != nil { + t.Fatal(err) + } + err = extractor.Extract(ctx, emitter.Push) + + assert.NoError(t, err) + assert.EqualValues(t, expectedData, emitter.Get()) + }) +} From 5324090573d7fbaacaed25ebbf6fd3b776d4a3c2 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Thu, 11 Aug 2022 14:26:49 +0530 Subject: [PATCH 04/19] fix: mock init function parameter type --- plugins/extractors/googleworkspace/googleworkspace_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/extractors/googleworkspace/googleworkspace_test.go b/plugins/extractors/googleworkspace/googleworkspace_test.go index 87f6dae02..932d4062e 100644 --- a/plugins/extractors/googleworkspace/googleworkspace_test.go +++ b/plugins/extractors/googleworkspace/googleworkspace_test.go @@ -83,7 +83,7 @@ func TestExtract(t *testing.T) { emitter := mocks.NewEmitter() extractor := mocks.NewExtractor() - extractor.On("Init", mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("plugins.Emit")).Return(nil) + extractor.On("Init", mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("plugins.Config")).Return(nil) extractor.SetEmit(expectedData) extractor.On("Extract", mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("plugins.Emit")).Return(nil) From 3b143d029fb57e5fb087ae83c0f6085abc32c7ac Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 16 Aug 2022 00:45:30 +0530 Subject: [PATCH 05/19] feat: add aliases & urn while user emit --- .../googleworkspace/googleworkspace.go | 18 +++++++++++------- .../googleworkspace/googleworkspace_test.go | 2 ++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index ea7937e79..b2d0c5125 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -5,6 +5,7 @@ import ( _ "embed" // used to print the embedded assets "fmt" "reflect" + "strings" "github.com/pkg/errors" @@ -112,14 +113,17 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) } var userAttributes = make(map[string]interface{}) - userAttributes = getOrgAttributes(userAttributes, u.Organizations) - userAttributes = getRelationsAttributes(userAttributes, u.Relations) - userAttributes = getCustomSchemasAttributes(userAttributes, u.CustomSchemas) + userAttributes = buildOrgAttributes(userAttributes, u.Organizations) + userAttributes = buildRelationsAttributes(userAttributes, u.Relations) + userAttributes = buildCustomSchemasAttributes(userAttributes, u.CustomSchemas) userAttributes["org_unit_path"] = u.OrgUnitPath + userAttributes["aliases"] = strings.Join(u.Aliases, ",") + e.emit(models.NewRecord(&assetsv1beta1.User{ Resource: &commonv1beta1.Resource{ Service: "google workspace", - Name: u.PrimaryEmail, + Name: u.Name.FullName, + Urn: u.PrimaryEmail, }, Email: u.PrimaryEmail, FullName: u.Name.FullName, @@ -158,7 +162,7 @@ func FetchUsers(ctx context.Context, ts oauth2.TokenSource) (*admin.Users, error return r, nil } -func getOrgAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { +func buildOrgAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { if i != nil { itr := reflect.ValueOf(i) if itr.Kind() == reflect.Slice { @@ -174,7 +178,7 @@ func getOrgAttributes(userAttributes map[string]interface{}, i interface{}) map[ return userAttributes } -func getRelationsAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { +func buildRelationsAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { if i != nil { itr := reflect.ValueOf(i) if itr.Kind() == reflect.Slice { @@ -196,7 +200,7 @@ func getRelationsAttributes(userAttributes map[string]interface{}, i interface{} return userAttributes } -func getCustomSchemasAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { +func buildCustomSchemasAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { if i != nil { itr := reflect.ValueOf(i) if itr.Kind() == reflect.Map { diff --git a/plugins/extractors/googleworkspace/googleworkspace_test.go b/plugins/extractors/googleworkspace/googleworkspace_test.go index 932d4062e..f370cae25 100644 --- a/plugins/extractors/googleworkspace/googleworkspace_test.go +++ b/plugins/extractors/googleworkspace/googleworkspace_test.go @@ -51,6 +51,7 @@ func TestExtract(t *testing.T) { Resource: &commonv1beta1.Resource{ Service: "google workspace", Name: "odpf admin", + Urn: "admin@odpf.com", }, Email: "admin@odpf.com", FullName: "John Doe", @@ -66,6 +67,7 @@ func TestExtract(t *testing.T) { Resource: &commonv1beta1.Resource{ Service: "google workspace", Name: "ipsum", + Urn: "ipsum@odpf.com", }, Email: "ipsum@odpf.com", FullName: "Ipsum Lorum", From 22a4498d403ceb9e54163cf6558a91aacd649b87 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 16 Aug 2022 13:32:52 +0530 Subject: [PATCH 06/19] refactor: build and Extract functions --- .../googleworkspace/googleworkspace.go | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index b2d0c5125..6dd4e64d1 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -104,6 +104,7 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) if len(r.Users) == 0 { e.logger.Info("No users found.\n") + return nil } else { for _, u := range r.Users { if !u.Suspended { @@ -127,7 +128,6 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) }, Email: u.PrimaryEmail, FullName: u.Name.FullName, - LastName: u.Name.FamilyName, Status: status, Properties: &facetsv1beta1.Properties{ Attributes: utils.TryParseMapToProto(userAttributes), @@ -154,7 +154,7 @@ func FetchUsers(ctx context.Context, ts oauth2.TokenSource) (*admin.Users, error return nil, err } - r, err := srv.Users.List().Customer("my_customer").OrderBy("email").Do() + r, err := srv.Users.List().Customer("my_customer").Do() if err != nil { return nil, errors.Wrap(err, "Unable to retrieve users in domain") } @@ -162,35 +162,36 @@ func FetchUsers(ctx context.Context, ts oauth2.TokenSource) (*admin.Users, error return r, nil } -func buildOrgAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { - if i != nil { - itr := reflect.ValueOf(i) - if itr.Kind() == reflect.Slice { - for idx := 0; idx < itr.Len(); idx++ { - valMap := reflect.ValueOf(itr.Index(idx).Interface()) - for _, key := range valMap.MapKeys() { - strct := valMap.MapIndex(key) - userAttributes[fmt.Sprintf("%v", key.Interface())] = strct.Interface() - } +func buildOrgAttributes(userAttributes map[string]interface{}, orgsIfc interface{}) map[string]interface{} { + if orgsIfc != nil { + orgs := reflect.ValueOf(orgsIfc) + if orgs.Kind() == reflect.Slice { + //based on assumpton that a user shall belong to a single org + org := reflect.ValueOf(orgs.Index(0).Interface()) + for _, key := range org.MapKeys() { + value := org.MapIndex(key) + userAttributes[fmt.Sprintf("%v", key.Interface())] = value.Interface() } } } return userAttributes } -func buildRelationsAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { - if i != nil { - itr := reflect.ValueOf(i) - if itr.Kind() == reflect.Slice { - for idx := 0; idx < itr.Len(); idx++ { - valMap := reflect.ValueOf(itr.Index(idx).Interface()) +func buildRelationsAttributes(userAttributes map[string]interface{}, relationsIfc interface{}) map[string]interface{} { + if relationsIfc != nil { + relations := reflect.ValueOf(relationsIfc) + if relations.Kind() == reflect.Slice { + for idx := 0; idx < relations.Len(); idx++ { var relationType, relationValue string - for _, key := range valMap.MapKeys() { - strct := valMap.MapIndex(key) + + relation := reflect.ValueOf(relations.Index(idx).Interface()) + for _, key := range relation.MapKeys() { + value := relation.MapIndex(key) + if key.Interface().(string) == "type" { - relationType = strct.Interface().(string) + relationType = value.Interface().(string) } else if key.Interface().(string) == "value" { - relationValue = strct.Interface().(string) + relationValue = value.Interface().(string) } } userAttributes[relationType] = relationValue @@ -200,13 +201,13 @@ func buildRelationsAttributes(userAttributes map[string]interface{}, i interface return userAttributes } -func buildCustomSchemasAttributes(userAttributes map[string]interface{}, i interface{}) map[string]interface{} { - if i != nil { - itr := reflect.ValueOf(i) - if itr.Kind() == reflect.Map { - for _, key := range itr.MapKeys() { - strct := itr.MapIndex(key) - userAttributes[fmt.Sprintf("%v", key.Interface())] = strct.Interface() +func buildCustomSchemasAttributes(userAttributes map[string]interface{}, customSchemasIfc interface{}) map[string]interface{} { + if customSchemasIfc != nil { + customSchema := reflect.ValueOf(customSchemasIfc) + if customSchema.Kind() == reflect.Map { + for _, key := range customSchema.MapKeys() { + value := customSchema.MapIndex(key) + userAttributes[fmt.Sprintf("%v", key.Interface())] = value.Interface() } } } From 584672aa721cc4a3bb852d78de747f82a8b4ead6 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 16 Aug 2022 13:58:14 +0530 Subject: [PATCH 07/19] doc: add README & summary --- plugins/extractors/googleworkspace/README.md | 45 +++++++++++++++++++ .../googleworkspace/googleworkspace.go | 4 ++ 2 files changed, 49 insertions(+) create mode 100644 plugins/extractors/googleworkspace/README.md diff --git a/plugins/extractors/googleworkspace/README.md b/plugins/extractors/googleworkspace/README.md new file mode 100644 index 000000000..c9e1331ea --- /dev/null +++ b/plugins/extractors/googleworkspace/README.md @@ -0,0 +1,45 @@ +# Google Workspace + +## Usage + +```yaml +source: + scope: my-scope + type: googleworkspace + config: + service_account_json: "{\"type\":\"service_account\",\"project_id\":\"meteor-sa\",\"private_key_id\":\"3cb2103ef7883845a2befe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9wEFAASCBKcwggSjAgEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdAUSjzKden0lgYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYn\\nHYj064sMvmR5TcMQpnxYG86TGaPuIh30grz5dI39dtrUjttWWvtvqRv0qu7I5\\nuEL2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmBHyNdOai29bhoS+Ac\\n5ipTGF0FvT1f5KlJcHfsNoOGPJYePTaGxOW1zk680Z1WFyB1xX9iw5/GUA3XM\\neon4p9X31ASgWTqplFZhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAggEALZwVYz8nSmTWFMW2OtyvojIq+ab864ZGPCpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJThfOscyXydDHjHXT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoYEmOsB2xgqjvKXR9NJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMVVa6ZpxCTWvbQga+/ZPaqppgGps5yLDqcp1A/lDCKBtqk\\njaQXHqKjuYUsoiyl2vbPbwGxIzYSv6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOhxadJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwbQFwhErWS1fZgg2PcQDMsRgDBfsNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDhmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXUJq4hPQRAoGBAK9F\\nMO9dfouVP63f/Nf3GeIlE1r5IOX4di3qNe/PqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3ZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLvzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIx3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@meteor-sa.iam.gserviceaccount.com\",\"client_id\":\"1100599984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40meteor-sa.iam.gserviceaccount.com\"}" + user_email: meteor@odpf.com +``` + +## Inputs + +| Key | Value | Example | Description | | +| :-- | :---- | :------ | :---------- | :- | +| `user_email` | `string` | `meteor@odpf.com` | User email authorized to access the APIs | *required* | +| `service_account_json` | `string` | `{ + "type": "service_account", + "project_id": "odpf-project", + "private_key_id": "3cb2saasa3ef788dvdvdvdvdvdssdvds57", + "private_key": "-----BEGIN PRIVATE KEY-----\njbjabdjbajd\n-----END PRIVATE KEY-----\n", + "client_email": "meteor-sa@odpf-project.iam.gserviceaccount.com", + "client_id": "1100599572858548635286", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-project.iam.gserviceaccount.com" +}` | Service Account JSON object | *required* | + +## Outputs + +| Field | Sample Value | +| :---- | :---- | +| `resource.urn` | `john.doe@gmail.com` | +| `resource.name` | `John Doe` | +| `email` | `john.doe@gmail.com` | +| `full_name` | `John Doe` | +| `status` | `not suspended` | +| `properties` | `{"attributes":{"aliases":"doe.john@gmail.com,john.doe0@gmail.com","manager":"christian.aristika@gmail.com","org_unit_path":"/"}}` + +## Contributing + +Refer to the [contribution guidelines](../../../docs/contribute/guide.md#adding-a-new-extractor) for information on contributing to this module. diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index 6dd4e64d1..0af978e77 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -24,6 +24,9 @@ import ( assetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/v1beta1" ) +//go:embed README.md +var summary string + type Config struct { ServiceAccountJSON string `mapstructure:"service_account_json" validate:"required"` UserEmail string `mapstructure:"user_email" validate:"required"` @@ -48,6 +51,7 @@ var info = plugins.Info{ Description: "User list from Google Workspace", SampleConfig: sampleConfig, Tags: []string{"platform", "extractor"}, + Summary: summary, } // Extractor manages the extraction of data from the extractor From 0e09c64dce9c5b960bb660ce458c85a2144a0bd5 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 16 Aug 2022 14:01:24 +0530 Subject: [PATCH 08/19] doc: fix inputs section --- plugins/extractors/googleworkspace/README.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/plugins/extractors/googleworkspace/README.md b/plugins/extractors/googleworkspace/README.md index c9e1331ea..8195db8a1 100644 --- a/plugins/extractors/googleworkspace/README.md +++ b/plugins/extractors/googleworkspace/README.md @@ -16,18 +16,7 @@ source: | Key | Value | Example | Description | | | :-- | :---- | :------ | :---------- | :- | | `user_email` | `string` | `meteor@odpf.com` | User email authorized to access the APIs | *required* | -| `service_account_json` | `string` | `{ - "type": "service_account", - "project_id": "odpf-project", - "private_key_id": "3cb2saasa3ef788dvdvdvdvdvdssdvds57", - "private_key": "-----BEGIN PRIVATE KEY-----\njbjabdjbajd\n-----END PRIVATE KEY-----\n", - "client_email": "meteor-sa@odpf-project.iam.gserviceaccount.com", - "client_id": "1100599572858548635286", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-project.iam.gserviceaccount.com" -}` | Service Account JSON object | *required* | +| `service_account_json` | `string` | `{"type": "service_account","project_id": "odpf-project","private_key_id": "3cb2saasa3ef788dvdvdvdvdvdssdvds57","private_key": "-----BEGIN PRIVATE KEY-----\njbjabdjbajd\n-----END PRIVATE KEY-----\n","client_email": "meteor-sa@odpf-project.iam.gserviceaccount.com","client_id": "1100599572858548635286","auth_uri": "https://accounts.google.com/o/oauth2/auth","token_uri": "https://oauth2.googleapis.com/token","auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-project.iam.gserviceaccount.com"}` | Service Account JSON object | *required* | ## Outputs From c91e78a688b11e7ecab051fe35035c529a49c9e3 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 16 Aug 2022 15:21:08 +0530 Subject: [PATCH 09/19] doc: update README --- plugins/extractors/googleworkspace/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/extractors/googleworkspace/README.md b/plugins/extractors/googleworkspace/README.md index 8195db8a1..ab2100ab8 100644 --- a/plugins/extractors/googleworkspace/README.md +++ b/plugins/extractors/googleworkspace/README.md @@ -7,7 +7,7 @@ source: scope: my-scope type: googleworkspace config: - service_account_json: "{\"type\":\"service_account\",\"project_id\":\"meteor-sa\",\"private_key_id\":\"3cb2103ef7883845a2befe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9wEFAASCBKcwggSjAgEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdAUSjzKden0lgYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYn\\nHYj064sMvmR5TcMQpnxYG86TGaPuIh30grz5dI39dtrUjttWWvtvqRv0qu7I5\\nuEL2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmBHyNdOai29bhoS+Ac\\n5ipTGF0FvT1f5KlJcHfsNoOGPJYePTaGxOW1zk680Z1WFyB1xX9iw5/GUA3XM\\neon4p9X31ASgWTqplFZhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAggEALZwVYz8nSmTWFMW2OtyvojIq+ab864ZGPCpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJThfOscyXydDHjHXT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoYEmOsB2xgqjvKXR9NJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMVVa6ZpxCTWvbQga+/ZPaqppgGps5yLDqcp1A/lDCKBtqk\\njaQXHqKjuYUsoiyl2vbPbwGxIzYSv6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOhxadJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwbQFwhErWS1fZgg2PcQDMsRgDBfsNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDhmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXUJq4hPQRAoGBAK9F\\nMO9dfouVP63f/Nf3GeIlE1r5IOX4di3qNe/PqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3ZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLvzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIx3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@meteor-sa.iam.gserviceaccount.com\",\"client_id\":\"1100599984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40meteor-sa.iam.gserviceaccount.com\"}" + service_account_json: "XXX" user_email: meteor@odpf.com ``` @@ -16,7 +16,7 @@ source: | Key | Value | Example | Description | | | :-- | :---- | :------ | :---------- | :- | | `user_email` | `string` | `meteor@odpf.com` | User email authorized to access the APIs | *required* | -| `service_account_json` | `string` | `{"type": "service_account","project_id": "odpf-project","private_key_id": "3cb2saasa3ef788dvdvdvdvdvdssdvds57","private_key": "-----BEGIN PRIVATE KEY-----\njbjabdjbajd\n-----END PRIVATE KEY-----\n","client_email": "meteor-sa@odpf-project.iam.gserviceaccount.com","client_id": "1100599572858548635286","auth_uri": "https://accounts.google.com/o/oauth2/auth","token_uri": "https://oauth2.googleapis.com/token","auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-project.iam.gserviceaccount.com"}` | Service Account JSON object | *required* | +| `service_account_json` | `string` | `{"type": "service_account","project_id": "odpf-project","private_key_id": "XXXXXXXXXXXXXXXX","private_key": "-----BEGIN PRIVATE KEY-----\nXXXXXXXX\n-----END PRIVATE KEY-----\n","client_email": "meteor-sa@odpf-project.iam.gserviceaccount.com","client_id": "XXXXXXXXXXXXXXXX","auth_uri": "https://accounts.google.com/o/oauth2/auth","token_uri": "https://oauth2.googleapis.com/token","auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-project.iam.gserviceaccount.com"}` | Service Account JSON object | *required* | ## Outputs @@ -29,6 +29,10 @@ source: | `status` | `not suspended` | | `properties` | `{"attributes":{"aliases":"doe.john@gmail.com,john.doe0@gmail.com","manager":"christian.aristika@gmail.com","org_unit_path":"/"}}` +### Notes + - The service account must have a [delegated domain wide authority](https://developers.google.com/admin-sdk/directory/v1/guides/delegation#delegate_domain-wide_authority_to_your_service_account) + - User Email : Only users with access to the Admin APIs can access the Admin SDK Directory API, therefore your service account needs to impersonate one of those users to access the Admin SDK Directory API. + ## Contributing Refer to the [contribution guidelines](../../../docs/contribute/guide.md#adding-a-new-extractor) for information on contributing to this module. From 2ca3ebd141517650373979c3d52f93b8e9d3ea2a Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 16 Aug 2022 15:25:17 +0530 Subject: [PATCH 10/19] refactor: remove unnecessary else --- .../googleworkspace/googleworkspace.go | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index 0af978e77..ca3cd1c3e 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -109,35 +109,35 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) if len(r.Users) == 0 { e.logger.Info("No users found.\n") return nil - } else { - for _, u := range r.Users { - if !u.Suspended { - status = "not suspended" - } else { - status = "suspended" - } + } - var userAttributes = make(map[string]interface{}) - userAttributes = buildOrgAttributes(userAttributes, u.Organizations) - userAttributes = buildRelationsAttributes(userAttributes, u.Relations) - userAttributes = buildCustomSchemasAttributes(userAttributes, u.CustomSchemas) - userAttributes["org_unit_path"] = u.OrgUnitPath - userAttributes["aliases"] = strings.Join(u.Aliases, ",") - - e.emit(models.NewRecord(&assetsv1beta1.User{ - Resource: &commonv1beta1.Resource{ - Service: "google workspace", - Name: u.Name.FullName, - Urn: u.PrimaryEmail, - }, - Email: u.PrimaryEmail, - FullName: u.Name.FullName, - Status: status, - Properties: &facetsv1beta1.Properties{ - Attributes: utils.TryParseMapToProto(userAttributes), - }, - })) + for _, u := range r.Users { + if !u.Suspended { + status = "not suspended" + } else { + status = "suspended" } + + var userAttributes = make(map[string]interface{}) + userAttributes = buildOrgAttributes(userAttributes, u.Organizations) + userAttributes = buildRelationsAttributes(userAttributes, u.Relations) + userAttributes = buildCustomSchemasAttributes(userAttributes, u.CustomSchemas) + userAttributes["org_unit_path"] = u.OrgUnitPath + userAttributes["aliases"] = strings.Join(u.Aliases, ",") + + e.emit(models.NewRecord(&assetsv1beta1.User{ + Resource: &commonv1beta1.Resource{ + Service: "google workspace", + Name: u.Name.FullName, + Urn: u.PrimaryEmail, + }, + Email: u.PrimaryEmail, + FullName: u.Name.FullName, + Status: status, + Properties: &facetsv1beta1.Properties{ + Attributes: utils.TryParseMapToProto(userAttributes), + }, + })) } return nil From e3b30b79538da24056c88a4e795262c229830ae2 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 16 Aug 2022 23:47:32 +0530 Subject: [PATCH 11/19] doc: input example enhancements --- plugins/extractors/googleworkspace/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/extractors/googleworkspace/README.md b/plugins/extractors/googleworkspace/README.md index ab2100ab8..ca2603692 100644 --- a/plugins/extractors/googleworkspace/README.md +++ b/plugins/extractors/googleworkspace/README.md @@ -16,7 +16,7 @@ source: | Key | Value | Example | Description | | | :-- | :---- | :------ | :---------- | :- | | `user_email` | `string` | `meteor@odpf.com` | User email authorized to access the APIs | *required* | -| `service_account_json` | `string` | `{"type": "service_account","project_id": "odpf-project","private_key_id": "XXXXXXXXXXXXXXXX","private_key": "-----BEGIN PRIVATE KEY-----\nXXXXXXXX\n-----END PRIVATE KEY-----\n","client_email": "meteor-sa@odpf-project.iam.gserviceaccount.com","client_id": "XXXXXXXXXXXXXXXX","auth_uri": "https://accounts.google.com/o/oauth2/auth","token_uri": "https://oauth2.googleapis.com/token","auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-project.iam.gserviceaccount.com"}` | Service Account JSON object | *required* | +| `service_account_json` | `string` | `{"type": "service_account","project_id": "XXXXXX","private_key_id": "XXXXXX","private_key": "XXXXXX","client_email": "XXXXXX","client_id": "XXXXXX","auth_uri": "https://accounts.google.com/o/oauth2/auth","token_uri": "https://oauth2.googleapis.com/token","auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url": "XXXXXX"}` | Service Account JSON object | *required* | ## Outputs @@ -32,7 +32,7 @@ source: ### Notes - The service account must have a [delegated domain wide authority](https://developers.google.com/admin-sdk/directory/v1/guides/delegation#delegate_domain-wide_authority_to_your_service_account) - User Email : Only users with access to the Admin APIs can access the Admin SDK Directory API, therefore your service account needs to impersonate one of those users to access the Admin SDK Directory API. - + ## Contributing Refer to the [contribution guidelines](../../../docs/contribute/guide.md#adding-a-new-extractor) for information on contributing to this module. From 604e010fd46d281bc5486b150f8186205ff179b8 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Mon, 29 Aug 2022 14:26:28 +0530 Subject: [PATCH 12/19] feat: use assetv1beta2 asset model --- .../googleworkspace/googleworkspace.go | 29 ++++++----- .../googleworkspace/googleworkspace_test.go | 49 +++++++++---------- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index ca3cd1c3e..f80e130c9 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -18,10 +18,9 @@ import ( "golang.org/x/oauth2/google" admin "google.golang.org/api/admin/directory/v1" "google.golang.org/api/option" + "google.golang.org/protobuf/types/known/anypb" - commonv1beta1 "github.com/odpf/meteor/models/odpf/assets/common/v1beta1" - facetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/facets/v1beta1" - assetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/v1beta1" + assetsv1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" ) //go:embed README.md @@ -125,18 +124,18 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) userAttributes["org_unit_path"] = u.OrgUnitPath userAttributes["aliases"] = strings.Join(u.Aliases, ",") - e.emit(models.NewRecord(&assetsv1beta1.User{ - Resource: &commonv1beta1.Resource{ - Service: "google workspace", - Name: u.Name.FullName, - Urn: u.PrimaryEmail, - }, - Email: u.PrimaryEmail, - FullName: u.Name.FullName, - Status: status, - Properties: &facetsv1beta1.Properties{ - Attributes: utils.TryParseMapToProto(userAttributes), - }, + user, err := anypb.New(&assetsv1beta2.User{ + Email: u.PrimaryEmail, + FullName: u.Name.FullName, + Status: status, + Attributes: utils.TryParseMapToProto(userAttributes), + }) + if err != nil { + return err + } + + e.emit(models.NewRecord(&assetsv1beta2.Asset{ + Data: user, })) } diff --git a/plugins/extractors/googleworkspace/googleworkspace_test.go b/plugins/extractors/googleworkspace/googleworkspace_test.go index f370cae25..7daa8d9b4 100644 --- a/plugins/extractors/googleworkspace/googleworkspace_test.go +++ b/plugins/extractors/googleworkspace/googleworkspace_test.go @@ -8,9 +8,7 @@ import ( "testing" "github.com/odpf/meteor/models" - commonv1beta1 "github.com/odpf/meteor/models/odpf/assets/common/v1beta1" - facetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/facets/v1beta1" - assetsv1beta1 "github.com/odpf/meteor/models/odpf/assets/v1beta1" + assetsv1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" "github.com/odpf/meteor/plugins" "github.com/odpf/meteor/plugins/extractors/googleworkspace" "github.com/odpf/meteor/test/mocks" @@ -18,6 +16,7 @@ import ( utilities "github.com/odpf/meteor/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "google.golang.org/protobuf/types/known/anypb" ) var urnScope string = "test-googleworkspace" @@ -47,37 +46,25 @@ func TestExtract(t *testing.T) { t.Run("should extract user details from google workspace", func(t *testing.T) { expectedData := []models.Record{ - models.NewRecord(&assetsv1beta1.User{ - Resource: &commonv1beta1.Resource{ - Service: "google workspace", - Name: "odpf admin", - Urn: "admin@odpf.com", - }, - Email: "admin@odpf.com", - FullName: "John Doe", - LastName: "Doe", - Status: "not suspended", - Properties: &facetsv1beta1.Properties{ + models.NewRecord(&assetsv1beta2.Asset{ + Data: userToAny(assetsv1beta2.User{ + Email: "admin@odpf.com", + FullName: "John Doe", + Status: "not suspended", Attributes: utilities.TryParseMapToProto(map[string]interface{}{ "manager": "lorem@odpf.com", }), - }, + }), }), - models.NewRecord(&assetsv1beta1.User{ - Resource: &commonv1beta1.Resource{ - Service: "google workspace", - Name: "ipsum", - Urn: "ipsum@odpf.com", - }, - Email: "ipsum@odpf.com", - FullName: "Ipsum Lorum", - LastName: "Lorum", - Status: "not suspended", - Properties: &facetsv1beta1.Properties{ + models.NewRecord(&assetsv1beta2.Asset{ + Data: userToAny(assetsv1beta2.User{ + Email: "ipsum@odpf.com", + FullName: "Ipsum Lorum", + Status: "not suspended", Attributes: utilities.TryParseMapToProto(map[string]interface{}{ "manager": "manager@odpf.com", }), - }, + }), }), } @@ -105,3 +92,11 @@ func TestExtract(t *testing.T) { assert.EqualValues(t, expectedData, emitter.Get()) }) } + +func userToAny(user assetsv1beta2.User) *anypb.Any { + u, err := anypb.New(&user) + if err != nil { + return &anypb.Any{} + } + return u +} From 834ce3f909d93c1d5639c375e667cf52a3ae829f Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Mon, 29 Aug 2022 14:43:11 +0530 Subject: [PATCH 13/19] chore: sample config & error case fixed --- .../extractors/googleworkspace/googleworkspace.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index f80e130c9..88e6223af 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -34,15 +34,15 @@ type Config struct { var sampleConfig = ` service_account_json: { "type": "service_account", - "project_id": "odpf-project", - "private_key_id": "3cb2saasa3ef788dvdvdvdvdvdssdvds57", - "private_key": "-----BEGIN PRIVATE KEY-----\njbjabdjbajd\n-----END PRIVATE KEY-----\n", - "client_email": "meteor-sa@odpf-project.iam.gserviceaccount.com", - "client_id": "1100599572858548635286", + "project_id": "XXXXXX", + "private_key_id": "XXXXXX", + "private_key": "XXXXXX", + "client_email": "XXXXXX", + "client_id": "XXXXXX", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-project.iam.gserviceaccount.com" + "client_x509_cert_url": "XXXXXX" } user_email: user@odpf.com` @@ -159,7 +159,7 @@ func FetchUsers(ctx context.Context, ts oauth2.TokenSource) (*admin.Users, error r, err := srv.Users.List().Customer("my_customer").Do() if err != nil { - return nil, errors.Wrap(err, "Unable to retrieve users in domain") + return nil, errors.Wrap(err, "unable to retrieve users in domain") } return r, nil From 8b9d20eec28f58b2af2f7746518533b025e2b909 Mon Sep 17 00:00:00 2001 From: Stewart Jingga Date: Thu, 1 Sep 2022 17:55:00 +0700 Subject: [PATCH 14/19] feat(googleworkspace): add mock and test --- plugins/extractors/googleworkspace/admin.go | 37 +++ .../googleworkspace/googleworkspace.go | 235 ++++++++++-------- .../googleworkspace/googleworkspace_test.go | 212 ++++++++++++---- test/utils/struct.go | 15 ++ 4 files changed, 350 insertions(+), 149 deletions(-) create mode 100644 plugins/extractors/googleworkspace/admin.go create mode 100644 test/utils/struct.go diff --git a/plugins/extractors/googleworkspace/admin.go b/plugins/extractors/googleworkspace/admin.go new file mode 100644 index 000000000..2a6e06ad6 --- /dev/null +++ b/plugins/extractors/googleworkspace/admin.go @@ -0,0 +1,37 @@ +package googleworkspace + +import ( + "context" + + "golang.org/x/oauth2/google" + admin "google.golang.org/api/admin/directory/v1" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" +) + +type UsersServiceFactory interface { + BuildUserService(ctx context.Context, email, serviceAccountJSON string) (UsersListCall, error) +} + +type UsersListCall interface { + Do(opts ...googleapi.CallOption) (*admin.Users, error) +} + +type DefaultUsersServiceFactory struct{} + +func (f *DefaultUsersServiceFactory) BuildUserService(ctx context.Context, email, serviceAccountJSON string) (UsersListCall, error) { + jwtConfig, err := google.JWTConfigFromJSON([]byte(serviceAccountJSON), admin.AdminDirectoryUserScope) + if err != nil { + return nil, err + } + jwtConfig.Subject = email + + ts := jwtConfig.TokenSource(ctx) + + srv, err := admin.NewService(ctx, option.WithTokenSource(ts)) + if err != nil { + return nil, err + } + + return srv.Users.List().Customer("my_customer"), nil +} diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index 88e6223af..e2c2627f5 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -3,24 +3,21 @@ package googleworkspace import ( "context" _ "embed" // used to print the embedded assets + "encoding/json" "fmt" "reflect" "strings" - "github.com/pkg/errors" - "github.com/odpf/meteor/models" "github.com/odpf/meteor/plugins" "github.com/odpf/meteor/registry" "github.com/odpf/meteor/utils" "github.com/odpf/salt/log" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" admin "google.golang.org/api/admin/directory/v1" - "google.golang.org/api/option" + "google.golang.org/api/googleapi" "google.golang.org/protobuf/types/known/anypb" - assetsv1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" + v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" ) //go:embed README.md @@ -56,17 +53,18 @@ var info = plugins.Info{ // Extractor manages the extraction of data from the extractor type Extractor struct { plugins.BaseExtractor - logger log.Logger - config Config - TokenSource oauth2.TokenSource - Client *admin.Service - emit plugins.Emit + logger log.Logger + config Config + userServiceFactory UsersServiceFactory + userService UsersListCall + emit plugins.Emit } // New returns a pointer to an initialized Extractor Object -func New(logger log.Logger) *Extractor { +func New(logger log.Logger, userServiceFactory UsersServiceFactory) *Extractor { e := &Extractor{ - logger: logger, + logger: logger, + userServiceFactory: userServiceFactory, } e.BaseExtractor = plugins.NewBaseExtractor(info, &e.config) @@ -79,18 +77,10 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error) return err } - jwtConfig, err := google.JWTConfigFromJSON([]byte(e.config.ServiceAccountJSON), admin.AdminDirectoryUserScope) + e.userService, err = e.userServiceFactory.BuildUserService(ctx, e.config.UserEmail, e.config.ServiceAccountJSON) if err != nil { - return plugins.InvalidConfigError{} - } - - if e.config.UserEmail == "" { - return plugins.InvalidConfigError{} + return fmt.Errorf("error building user service: %w", err) } - jwtConfig.Subject = e.config.UserEmail - ts := jwtConfig.TokenSource(ctx) - - e.TokenSource = ts return } @@ -98,121 +88,154 @@ func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error) // Extract extracts the data from the extractor // The data is returned as a list of assets.Asset func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) { - var status string e.emit = emit - r, err := FetchUsers(ctx, e.TokenSource) + adminUsers, err := e.fetchUsers(ctx) if err != nil { return err } - if len(r.Users) == 0 { + if len(adminUsers.Users) == 0 { e.logger.Info("No users found.\n") return nil } - for _, u := range r.Users { - if !u.Suspended { - status = "not suspended" - } else { - status = "suspended" - } - - var userAttributes = make(map[string]interface{}) - userAttributes = buildOrgAttributes(userAttributes, u.Organizations) - userAttributes = buildRelationsAttributes(userAttributes, u.Relations) - userAttributes = buildCustomSchemasAttributes(userAttributes, u.CustomSchemas) - userAttributes["org_unit_path"] = u.OrgUnitPath - userAttributes["aliases"] = strings.Join(u.Aliases, ",") - - user, err := anypb.New(&assetsv1beta2.User{ - Email: u.PrimaryEmail, - FullName: u.Name.FullName, - Status: status, - Attributes: utils.TryParseMapToProto(userAttributes), - }) + for _, u := range adminUsers.Users { + asset, err := e.buildAsset(u) if err != nil { - return err + e.logger.Warn("error when building asset", "err", err) + continue } - - e.emit(models.NewRecord(&assetsv1beta2.Asset{ - Data: user, - })) + e.emit(models.NewRecord(asset)) } return nil } -// init registers the extractor to catalog -func init() { - if err := registry.Extractors.Register("googleworkspace", func() plugins.Extractor { - return New(plugins.GetLog()) - }); err != nil { - panic(err) +func (e *Extractor) buildAsset(gsuiteUser *admin.User) (*v1beta2.Asset, error) { + var status string + if gsuiteUser.Suspended { + status = "suspended" } -} -func FetchUsers(ctx context.Context, ts oauth2.TokenSource) (*admin.Users, error) { - srv, err := admin.NewService(ctx, option.WithTokenSource(ts)) + var userAttributes = make(map[string]interface{}) + userAttributes["organizations"] = e.buildOrganizations(gsuiteUser.Organizations) + userAttributes["relations"] = e.buildRelations(gsuiteUser.Relations) + userAttributes["custom_schemas"] = e.buildMapFromGsuiteMap(gsuiteUser.CustomSchemas) + userAttributes["aliases"] = strings.Join(gsuiteUser.Aliases, ",") + userAttributes["org_unit_path"] = gsuiteUser.OrgUnitPath + + assetUser, err := anypb.New(&v1beta2.User{ + Email: gsuiteUser.PrimaryEmail, + FullName: gsuiteUser.Name.FullName, + Status: status, + Attributes: utils.TryParseMapToProto(userAttributes), + }) if err != nil { - return nil, err + return nil, fmt.Errorf("error when creating anypb.Any: %w", err) + } + + asset := &v1beta2.Asset{ + Urn: models.NewURN("gsuite", e.UrnScope, "user", gsuiteUser.PrimaryEmail), + Name: gsuiteUser.Name.FullName, + Service: "gsuite", + Type: "user", + Data: assetUser, } - r, err := srv.Users.List().Customer("my_customer").Do() + return asset, nil +} + +func (e *Extractor) fetchUsers(ctx context.Context) (*admin.Users, error) { + users, err := e.userService.Do() if err != nil { - return nil, errors.Wrap(err, "unable to retrieve users in domain") + return nil, fmt.Errorf("error fetching users: %w", err) } - return r, nil + return users, nil } -func buildOrgAttributes(userAttributes map[string]interface{}, orgsIfc interface{}) map[string]interface{} { - if orgsIfc != nil { - orgs := reflect.ValueOf(orgsIfc) - if orgs.Kind() == reflect.Slice { - //based on assumpton that a user shall belong to a single org - org := reflect.ValueOf(orgs.Index(0).Interface()) - for _, key := range org.MapKeys() { - value := org.MapIndex(key) - userAttributes[fmt.Sprintf("%v", key.Interface())] = value.Interface() - } - } +func (e *Extractor) buildRelations(gSuiteRelations interface{}) (result []interface{}) { + if gSuiteRelations == nil { + return } - return userAttributes + + value := reflect.ValueOf(gSuiteRelations) + if value.Kind() != reflect.Slice { + return + } + + orgList, ok := value.Interface().([]map[string]googleapi.RawMessage) + if !ok { + return + } + + result = []interface{}{} + for _, orgMap := range orgList { + result = append(result, e.buildMapFromGsuiteMap(orgMap)) + } + + return result } -func buildRelationsAttributes(userAttributes map[string]interface{}, relationsIfc interface{}) map[string]interface{} { - if relationsIfc != nil { - relations := reflect.ValueOf(relationsIfc) - if relations.Kind() == reflect.Slice { - for idx := 0; idx < relations.Len(); idx++ { - var relationType, relationValue string - - relation := reflect.ValueOf(relations.Index(idx).Interface()) - for _, key := range relation.MapKeys() { - value := relation.MapIndex(key) - - if key.Interface().(string) == "type" { - relationType = value.Interface().(string) - } else if key.Interface().(string) == "value" { - relationValue = value.Interface().(string) - } - } - userAttributes[relationType] = relationValue - } - } +func (e *Extractor) buildOrganizations(gSuiteOrganizations interface{}) (result []interface{}) { + if gSuiteOrganizations == nil { + return + } + + value := reflect.ValueOf(gSuiteOrganizations) + if value.Kind() != reflect.Slice { + return + } + + orgList, ok := value.Interface().([]map[string]googleapi.RawMessage) + if !ok { + return + } + + result = []interface{}{} + for _, orgMap := range orgList { + result = append(result, e.buildMapFromGsuiteMap(orgMap)) } - return userAttributes + + return result } -func buildCustomSchemasAttributes(userAttributes map[string]interface{}, customSchemasIfc interface{}) map[string]interface{} { - if customSchemasIfc != nil { - customSchema := reflect.ValueOf(customSchemasIfc) - if customSchema.Kind() == reflect.Map { - for _, key := range customSchema.MapKeys() { - value := customSchema.MapIndex(key) - userAttributes[fmt.Sprintf("%v", key.Interface())] = value.Interface() - } +func (e *Extractor) buildMapFromGsuiteMap(value interface{}) (result map[string]interface{}) { + if value == nil { + return + } + + gsuiteMap := reflect.ValueOf(value) + if gsuiteMap.Kind() != reflect.Map { + return + } + + result = map[string]interface{}{} + for _, key := range gsuiteMap.MapKeys() { + keyString := key.String() + rawMessage, ok := gsuiteMap.MapIndex(key).Interface().(googleapi.RawMessage) + if !ok { + continue + } + + var value interface{} + err := json.Unmarshal(rawMessage, &value) + if err != nil { + e.logger.Warn("error unmarshalling rawMessage", "err", err, "value", rawMessage) + value = string(rawMessage) } + + result[keyString] = value + } + + return +} + +// init registers the extractor to catalog +func init() { + if err := registry.Extractors.Register("googleworkspace", func() plugins.Extractor { + return New(plugins.GetLog(), &DefaultUsersServiceFactory{}) + }); err != nil { + panic(err) } - return userAttributes } diff --git a/plugins/extractors/googleworkspace/googleworkspace_test.go b/plugins/extractors/googleworkspace/googleworkspace_test.go index 7daa8d9b4..6ec62a187 100644 --- a/plugins/extractors/googleworkspace/googleworkspace_test.go +++ b/plugins/extractors/googleworkspace/googleworkspace_test.go @@ -13,17 +13,18 @@ import ( "github.com/odpf/meteor/plugins/extractors/googleworkspace" "github.com/odpf/meteor/test/mocks" "github.com/odpf/meteor/test/utils" - utilities "github.com/odpf/meteor/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "google.golang.org/protobuf/types/known/anypb" + "github.com/stretchr/testify/require" + admin "google.golang.org/api/admin/directory/v1" + "google.golang.org/api/googleapi" ) var urnScope string = "test-googleworkspace" func TestInit(t *testing.T) { t.Run("should return error for empty user email", func(t *testing.T) { - err := googleworkspace.New(utils.Logger).Init(context.TODO(), plugins.Config{ + err := googleworkspace.New(utils.Logger, new(mockUsersServiceFactory)).Init(context.TODO(), plugins.Config{ URNScope: urnScope, RawConfig: map[string]interface{}{ "user_email": "", @@ -32,71 +33,196 @@ func TestInit(t *testing.T) { assert.ErrorAs(t, err, &plugins.InvalidConfigError{}) }) t.Run("should return error for invalid service account json", func(t *testing.T) { - err := googleworkspace.New(utils.Logger).Init(context.TODO(), plugins.Config{ + err := googleworkspace.New(utils.Logger, new(mockUsersServiceFactory)).Init(context.TODO(), plugins.Config{ URNScope: urnScope, RawConfig: map[string]interface{}{ "user_email": "user@example.com", - "service_account_json": "invalide json", + "service_account_json": "", }}) assert.ErrorAs(t, err, &plugins.InvalidConfigError{}) }) + t.Run("should return error for invalid service account json", func(t *testing.T) { + ctx := context.TODO() + userEmail := "user@example.com" + serviceAcc := "{\"type\":\"service_account\",\"project_id\":\"odpf-meteor\",\"private_key_id\":\"3cb2103ef7883845a5fdcsvdefe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggdvdvdEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdAUSjzKdbdfbdbYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYEin\\nHYj064sMvm8vbR5TcMQpnxYG86TGaPuIh30grz5dI39dtrUjttbdfbdvqRv0qu7I5\\nuELzp2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmBHbdfdi29bhoS+Ac\\n5ipT10yGF0FvT1f5KlJcHfsNoOGPJYePTaGxOW1zk680Z1Wdfbdf1xX9iw5/GUA3XM\\neon4p9X31ASgwbdbdplFZhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAECggEbdbddYz8nSmTWFMW2OtyvojIq+ab864ZGPCpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJThfOscyXbbdbdbfbfT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoYEmOsB2xgqjvKXR90r5wNJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMVVa6ZpxCTWvbQga+/ZPaqppgGps5yLDqc434c3A/lDCKBtqk\\njaQXHqKjuYUsoiyl2vbPbwGxc34343c6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOAEfc343535dJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwRrnbQFwhErWS1fZgg2PcQKBgQDMsRgDBfhgJX9sNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDh3ELmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXU3c4343c4QRAoGBAK9F\\nMO9dzFjfouVP63f/Nf3GeIlctuiE1r5IOX4di3qNe/P33iqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3x4XZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLGTw1vzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIxZKeq3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@odpf-meteor.iam.gserviceaccount.com\",\"client_id\":\"110059943435984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-meteor.iam.gserviceaccount.com\"}" + factory := new(mockUsersServiceFactory) + factory.On("BuildUserService", ctx, userEmail, serviceAcc).Return(new(mockUsersListCall), nil) + + err := googleworkspace.New(utils.Logger, factory).Init(ctx, plugins.Config{ + URNScope: urnScope, + RawConfig: map[string]interface{}{ + "user_email": userEmail, + "service_account_json": serviceAcc, + }}) + assert.NoError(t, err) + }) } func TestExtract(t *testing.T) { t.Run("should extract user details from google workspace", func(t *testing.T) { + adminUsers := []*admin.User{ + { + Name: &admin.UserName{FullName: "User1"}, + PrimaryEmail: "user1@test.com", + Suspended: true, + Aliases: []string{"alias1", "alias2"}, + Relations: []map[string]googleapi.RawMessage{ + { + "type": []byte("manager1"), + "value": []byte("manager1@test.com"), + }, + }, + Organizations: []map[string]googleapi.RawMessage{ + { + "foo0": []byte("bar0"), + "foo1": []byte("bar1"), + }, + }, + OrgUnitPath: "/", + CustomSchemas: map[string]googleapi.RawMessage{ + "foo0_customSchema": []byte("bar0_customSchema"), + "foo1_customSchema": []byte("bar1_customSchema"), + }, + }, + { + Name: &admin.UserName{FullName: "User2"}, + PrimaryEmail: "user2@test.com", + Suspended: false, + Aliases: []string{"alias3"}, + Relations: []map[string]googleapi.RawMessage{ + { + "type": []byte("manager2"), + "value": []byte("manager2@test.com"), + }, + }, + Organizations: []map[string]googleapi.RawMessage{ + { + "foo20": []byte("bar20"), + "foo21": []byte("bar21"), + }, + }, + OrgUnitPath: "/test2", + CustomSchemas: map[string]googleapi.RawMessage{ + "foo20_customSchema": []byte("bar20_customSchema"), + "foo21_customSchema": []byte("bar21_customSchema"), + }, + }, + } - expectedData := []models.Record{ - models.NewRecord(&assetsv1beta2.Asset{ - Data: userToAny(assetsv1beta2.User{ - Email: "admin@odpf.com", - FullName: "John Doe", - Status: "not suspended", - Attributes: utilities.TryParseMapToProto(map[string]interface{}{ - "manager": "lorem@odpf.com", + expectedData := []*assetsv1beta2.Asset{ + { + Urn: models.NewURN("gsuite", urnScope, "user", adminUsers[0].PrimaryEmail), + Name: adminUsers[0].Name.FullName, + Service: "gsuite", + Type: "user", + Data: utils.BuildAny(t, &assetsv1beta2.User{ + Email: adminUsers[0].PrimaryEmail, + FullName: adminUsers[0].Name.FullName, + Status: "suspended", + Attributes: utils.BuildStruct(t, map[string]interface{}{ + "aliases": "alias1,alias2", + "org_unit_path": "/", + "organizations": []interface{}{ + map[string]interface{}{ + "foo0": "bar0", + "foo1": "bar1", + }, + }, + "custom_schemas": map[string]interface{}{ + "foo0_customSchema": "bar0_customSchema", + "foo1_customSchema": "bar1_customSchema", + }, + "relations": []interface{}{ + map[string]interface{}{ + "type": "manager1", + "value": "manager1@test.com", + }, + }, }), }), - }), - models.NewRecord(&assetsv1beta2.Asset{ - Data: userToAny(assetsv1beta2.User{ - Email: "ipsum@odpf.com", - FullName: "Ipsum Lorum", - Status: "not suspended", - Attributes: utilities.TryParseMapToProto(map[string]interface{}{ - "manager": "manager@odpf.com", + }, + { + Urn: models.NewURN("gsuite", urnScope, "user", adminUsers[1].PrimaryEmail), + Name: adminUsers[1].Name.FullName, + Service: "gsuite", + Type: "user", + Data: utils.BuildAny(t, &assetsv1beta2.User{ + Email: adminUsers[1].PrimaryEmail, + FullName: adminUsers[1].Name.FullName, + Status: "", + Attributes: utils.BuildStruct(t, map[string]interface{}{ + "aliases": "alias3", + "org_unit_path": "/test2", + "organizations": []interface{}{ + map[string]interface{}{ + "foo20": "bar20", + "foo21": "bar21", + }, + }, + "custom_schemas": map[string]interface{}{ + "foo20_customSchema": "bar20_customSchema", + "foo21_customSchema": "bar21_customSchema", + }, + "relations": []interface{}{ + map[string]interface{}{ + "type": "manager2", + "value": "manager2@test.com", + }, + }, }), }), - }), + }, } ctx := context.TODO() + userEmail := "user@example.com" + serviceAcc := "{\"type\":\"service_account\",\"project_id\":\"odpf-meteor\",\"private_key_id\":\"3cb2103ef7883845a5fdcsvdefe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggdvdvdEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdAUSjzKdbdfbdbYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYEin\\nHYj064sMvm8vbR5TcMQpnxYG86TGaPuIh30grz5dI39dtrUjttbdfbdvqRv0qu7I5\\nuELzp2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmBHbdfdi29bhoS+Ac\\n5ipT10yGF0FvT1f5KlJcHfsNoOGPJYePTaGxOW1zk680Z1Wdfbdf1xX9iw5/GUA3XM\\neon4p9X31ASgwbdbdplFZhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAECggEbdbddYz8nSmTWFMW2OtyvojIq+ab864ZGPCpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJThfOscyXbbdbdbfbfT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoYEmOsB2xgqjvKXR90r5wNJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMVVa6ZpxCTWvbQga+/ZPaqppgGps5yLDqc434c3A/lDCKBtqk\\njaQXHqKjuYUsoiyl2vbPbwGxc34343c6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOAEfc343535dJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwRrnbQFwhErWS1fZgg2PcQKBgQDMsRgDBfhgJX9sNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDh3ELmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXU3c4343c4QRAoGBAK9F\\nMO9dzFjfouVP63f/Nf3GeIlctuiE1r5IOX4di3qNe/P33iqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3x4XZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLGTw1vzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIxZKeq3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@odpf-meteor.iam.gserviceaccount.com\",\"client_id\":\"110059943435984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-meteor.iam.gserviceaccount.com\"}" - emitter := mocks.NewEmitter() - extractor := mocks.NewExtractor() - extractor.On("Init", mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("plugins.Config")).Return(nil) - extractor.SetEmit(expectedData) - extractor.On("Extract", mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("plugins.Emit")).Return(nil) + userService := new(mockUsersListCall) + userService.On("Do").Return(adminUsers).Once().Return(&admin.Users{Users: adminUsers}, nil) + defer userService.AssertExpectations(t) + + factory := new(mockUsersServiceFactory) + factory.On("BuildUserService", ctx, userEmail, serviceAcc).Return(userService, nil) + defer factory.AssertExpectations(t) - err := extractor.Init(ctx, plugins.Config{ + extr := googleworkspace.New(utils.Logger, factory) + err := extr.Init(ctx, plugins.Config{ URNScope: urnScope, RawConfig: map[string]interface{}{ - "user_email": "test@odpf.com", - "service_account_json": "{\"type\":\"service_account\",\"project_id\":\"odpf\",\"private_key_id\":\"3cb2103ef7883455a5f09712befe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBGDSNBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdASSjzKden0lgYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYEin\\nHYj064sMvm8vbR5TcMQpnxYG8SAGaPuIh30grz5dI39dtrUjttWWvtvqRv0qu7I5\\nuELzp2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmFDyNdOa6vi29bhoS+Ac\\n5ipT10yGF0FvT1f5KlJcHfsNoOGPJYeTRaGxOW1zk680Z1WFyB1xX9iw5/GUA3XM\\neon4p9X31ASgw6WTqplGDhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAECggEALZwVYz8nSmTWFMW2OtyvojIq+ab864BGHFpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJJHhfOscyXydDHjHXT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoTYmOsB2xgqjvKXR90r5wNJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMBBa6ZpxCTWvbQga+/ZPaqppgGps5yLDqcp1A/lDCKBtqk\\njaQXHqKjuYDSsoiyl2vbPbwGxIzYSv6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOAEfIhxadJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwRrnbQFwhErWS1fZgg2PcQKBgQDMsRgDBfhgJX9sNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDh3ELmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXUJq4hPQRAoGBAK9F\\nMO9dzFjfouVP63f/Nf3GeIlctuiE1r5IOX4di3qNe/P33iqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3x4XZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLGTw1vzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIxZKeq3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@odpf.iam.gserviceaccount.com\",\"client_id\":\"110059957285984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf.iam.gserviceaccount.com\"}", - }, - }) - if err != nil { - t.Fatal(err) - } - err = extractor.Extract(ctx, emitter.Push) + "user_email": userEmail, + "service_account_json": serviceAcc, + }}, + ) + require.NoError(t, err) - assert.NoError(t, err) - assert.EqualValues(t, expectedData, emitter.Get()) + emitter := mocks.NewEmitter() + err = extr.Extract(ctx, emitter.Push) + require.NoError(t, err) + + utils.AssertAssetsWithJSON(t, expectedData, emitter.GetAllData()) }) } -func userToAny(user assetsv1beta2.User) *anypb.Any { - u, err := anypb.New(&user) - if err != nil { - return &anypb.Any{} +type mockUsersServiceFactory struct { + mock.Mock +} + +func (m *mockUsersServiceFactory) BuildUserService(ctx context.Context, email, serviceAccountJSON string) (googleworkspace.UsersListCall, error) { + args := m.Called(ctx, email, serviceAccountJSON) + return args.Get(0).(googleworkspace.UsersListCall), args.Error(1) +} + +type mockUsersListCall struct { + mock.Mock +} + +func (m *mockUsersListCall) Do(opts ...googleapi.CallOption) (*admin.Users, error) { + var args mock.Arguments + if len(opts) > 0 { + args = m.Called(opts) + } else { + args = m.Called() } - return u + return args.Get(0).(*admin.Users), args.Error(1) } diff --git a/test/utils/struct.go b/test/utils/struct.go new file mode 100644 index 000000000..2d7880fd8 --- /dev/null +++ b/test/utils/struct.go @@ -0,0 +1,15 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func BuildStruct(t *testing.T, value map[string]interface{}) *structpb.Struct { + res, err := structpb.NewStruct(value) + require.NoError(t, err) + + return res +} From 0fb776d7c921ee33abb2a3f1e0a04b96a30f7800 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Thu, 1 Sep 2022 23:01:50 +0530 Subject: [PATCH 15/19] fix: build attribute functions --- .../googleworkspace/googleworkspace.go | 31 ++++++------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index e2c2627f5..6c0973e69 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -3,7 +3,6 @@ package googleworkspace import ( "context" _ "embed" // used to print the embedded assets - "encoding/json" "fmt" "reflect" "strings" @@ -14,7 +13,6 @@ import ( "github.com/odpf/meteor/utils" "github.com/odpf/salt/log" admin "google.golang.org/api/admin/directory/v1" - "google.golang.org/api/googleapi" "google.golang.org/protobuf/types/known/anypb" v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" @@ -164,7 +162,7 @@ func (e *Extractor) buildRelations(gSuiteRelations interface{}) (result []interf return } - orgList, ok := value.Interface().([]map[string]googleapi.RawMessage) + orgList, ok := value.Interface().([]interface{}) if !ok { return } @@ -174,7 +172,7 @@ func (e *Extractor) buildRelations(gSuiteRelations interface{}) (result []interf result = append(result, e.buildMapFromGsuiteMap(orgMap)) } - return result + return } func (e *Extractor) buildOrganizations(gSuiteOrganizations interface{}) (result []interface{}) { @@ -187,17 +185,16 @@ func (e *Extractor) buildOrganizations(gSuiteOrganizations interface{}) (result return } - orgList, ok := value.Interface().([]map[string]googleapi.RawMessage) + orgList, ok := value.Interface().([]interface{}) if !ok { return } - result = []interface{}{} - for _, orgMap := range orgList { - result = append(result, e.buildMapFromGsuiteMap(orgMap)) + for _, org := range orgList { + result = append(result, e.buildMapFromGsuiteMap(org)) } - return result + return } func (e *Extractor) buildMapFromGsuiteMap(value interface{}) (result map[string]interface{}) { @@ -210,20 +207,10 @@ func (e *Extractor) buildMapFromGsuiteMap(value interface{}) (result map[string] return } - result = map[string]interface{}{} + result = make(map[string]interface{}) for _, key := range gsuiteMap.MapKeys() { - keyString := key.String() - rawMessage, ok := gsuiteMap.MapIndex(key).Interface().(googleapi.RawMessage) - if !ok { - continue - } - - var value interface{} - err := json.Unmarshal(rawMessage, &value) - if err != nil { - e.logger.Warn("error unmarshalling rawMessage", "err", err, "value", rawMessage) - value = string(rawMessage) - } + keyString := fmt.Sprintf("%v", key.Interface()) + value := gsuiteMap.MapIndex(key).Interface() result[keyString] = value } From 4413849d776df72d2de3f6b5f746ebcc1d28cd9d Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Thu, 1 Sep 2022 23:56:23 +0530 Subject: [PATCH 16/19] test: fix types & refactor --- .../googleworkspace/googleworkspace.go | 57 +++++++++++-------- .../googleworkspace/googleworkspace_test.go | 35 ++++++------ 2 files changed, 49 insertions(+), 43 deletions(-) diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/googleworkspace/googleworkspace.go index 6c0973e69..26ebfe864 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/googleworkspace/googleworkspace.go @@ -13,6 +13,7 @@ import ( "github.com/odpf/meteor/utils" "github.com/odpf/salt/log" admin "google.golang.org/api/admin/directory/v1" + "google.golang.org/api/googleapi" "google.golang.org/protobuf/types/known/anypb" v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" @@ -116,9 +117,9 @@ func (e *Extractor) buildAsset(gsuiteUser *admin.User) (*v1beta2.Asset, error) { } var userAttributes = make(map[string]interface{}) - userAttributes["organizations"] = e.buildOrganizations(gsuiteUser.Organizations) - userAttributes["relations"] = e.buildRelations(gsuiteUser.Relations) - userAttributes["custom_schemas"] = e.buildMapFromGsuiteMap(gsuiteUser.CustomSchemas) + userAttributes["organizations"] = e.buildMapFromGsuiteSlice(gsuiteUser.Organizations) + userAttributes["relations"] = e.buildMapFromGsuiteSlice(gsuiteUser.Relations) + userAttributes["custom_schemas"] = e.buildMapFromGsuiteMapRawMessage(gsuiteUser.CustomSchemas) userAttributes["aliases"] = strings.Join(gsuiteUser.Aliases, ",") userAttributes["org_unit_path"] = gsuiteUser.OrgUnitPath @@ -152,52 +153,50 @@ func (e *Extractor) fetchUsers(ctx context.Context) (*admin.Users, error) { return users, nil } -func (e *Extractor) buildRelations(gSuiteRelations interface{}) (result []interface{}) { - if gSuiteRelations == nil { +func (e *Extractor) buildMapFromGsuiteSlice(value interface{}) (result []interface{}) { + if value == nil { return } - value := reflect.ValueOf(gSuiteRelations) - if value.Kind() != reflect.Slice { + gsuiteSlice := reflect.ValueOf(value) + if gsuiteSlice.Kind() != reflect.Slice { return } - orgList, ok := value.Interface().([]interface{}) + list, ok := gsuiteSlice.Interface().([]interface{}) if !ok { return } - result = []interface{}{} - for _, orgMap := range orgList { - result = append(result, e.buildMapFromGsuiteMap(orgMap)) + for _, item := range list { + result = append(result, e.buildMapFromGsuiteMap(item)) } return } -func (e *Extractor) buildOrganizations(gSuiteOrganizations interface{}) (result []interface{}) { - if gSuiteOrganizations == nil { +func (e *Extractor) buildMapFromGsuiteMap(value interface{}) (result map[string]interface{}) { + if value == nil { return } - value := reflect.ValueOf(gSuiteOrganizations) - if value.Kind() != reflect.Slice { + gsuiteMap := reflect.ValueOf(value) + if gsuiteMap.Kind() != reflect.Map { return } - orgList, ok := value.Interface().([]interface{}) - if !ok { - return - } + result = make(map[string]interface{}) + for _, key := range gsuiteMap.MapKeys() { + keyString := fmt.Sprintf("%v", key.Interface()) + value := gsuiteMap.MapIndex(key).Interface() - for _, org := range orgList { - result = append(result, e.buildMapFromGsuiteMap(org)) + result[keyString] = value } return } -func (e *Extractor) buildMapFromGsuiteMap(value interface{}) (result map[string]interface{}) { +func (e *Extractor) buildMapFromGsuiteMapRawMessage(value interface{}) (result map[string]interface{}) { if value == nil { return } @@ -210,9 +209,19 @@ func (e *Extractor) buildMapFromGsuiteMap(value interface{}) (result map[string] result = make(map[string]interface{}) for _, key := range gsuiteMap.MapKeys() { keyString := fmt.Sprintf("%v", key.Interface()) - value := gsuiteMap.MapIndex(key).Interface() + value := gsuiteMap.MapIndex(key) - result[keyString] = value + msg, ok := value.Interface().(googleapi.RawMessage) + if !ok { + continue + } + + json, err := msg.MarshalJSON() + if err != nil { + continue + } + + result[keyString] = string(json) } return diff --git a/plugins/extractors/googleworkspace/googleworkspace_test.go b/plugins/extractors/googleworkspace/googleworkspace_test.go index 6ec62a187..4927532c4 100644 --- a/plugins/extractors/googleworkspace/googleworkspace_test.go +++ b/plugins/extractors/googleworkspace/googleworkspace_test.go @@ -1,6 +1,3 @@ -//go:build plugins -// +build plugins - package googleworkspace_test import ( @@ -66,16 +63,16 @@ func TestExtract(t *testing.T) { PrimaryEmail: "user1@test.com", Suspended: true, Aliases: []string{"alias1", "alias2"}, - Relations: []map[string]googleapi.RawMessage{ - { - "type": []byte("manager1"), - "value": []byte("manager1@test.com"), + Relations: []interface{}{ + map[string]interface{}{ + "type": "manager1", + "value": "manager1@test.com", }, }, - Organizations: []map[string]googleapi.RawMessage{ - { - "foo0": []byte("bar0"), - "foo1": []byte("bar1"), + Organizations: []interface{}{ + map[string]interface{}{ + "foo0": "bar0", + "foo1": "bar1", }, }, OrgUnitPath: "/", @@ -89,16 +86,16 @@ func TestExtract(t *testing.T) { PrimaryEmail: "user2@test.com", Suspended: false, Aliases: []string{"alias3"}, - Relations: []map[string]googleapi.RawMessage{ - { - "type": []byte("manager2"), - "value": []byte("manager2@test.com"), + Relations: []interface{}{ + map[string]interface{}{ + "type": "manager2", + "value": "manager2@test.com", }, }, - Organizations: []map[string]googleapi.RawMessage{ - { - "foo20": []byte("bar20"), - "foo21": []byte("bar21"), + Organizations: []interface{}{ + map[string]interface{}{ + "foo20": "bar20", + "foo21": "bar21", }, }, OrgUnitPath: "/test2", From edd4375da91335a26c8ea5f312356f1996dcfbfc Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Fri, 2 Sep 2022 13:47:44 +0530 Subject: [PATCH 17/19] chore: rename to gsuite extractor --- .../{googleworkspace => gsuite}/README.md | 4 ++-- .../{googleworkspace => gsuite}/admin.go | 2 +- .../googleworkspace.go => gsuite/gsuite.go} | 4 ++-- .../gsuite_test.go} | 18 +++++++++--------- plugins/extractors/populate.go | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) rename plugins/extractors/{googleworkspace => gsuite}/README.md (97%) rename plugins/extractors/{googleworkspace => gsuite}/admin.go (97%) rename plugins/extractors/{googleworkspace/googleworkspace.go => gsuite/gsuite.go} (98%) rename plugins/extractors/{googleworkspace/googleworkspace_test.go => gsuite/gsuite_test.go} (95%) diff --git a/plugins/extractors/googleworkspace/README.md b/plugins/extractors/gsuite/README.md similarity index 97% rename from plugins/extractors/googleworkspace/README.md rename to plugins/extractors/gsuite/README.md index ca2603692..8897be5f7 100644 --- a/plugins/extractors/googleworkspace/README.md +++ b/plugins/extractors/gsuite/README.md @@ -1,11 +1,11 @@ -# Google Workspace +# G-Suite ## Usage ```yaml source: scope: my-scope - type: googleworkspace + type: gsuite config: service_account_json: "XXX" user_email: meteor@odpf.com diff --git a/plugins/extractors/googleworkspace/admin.go b/plugins/extractors/gsuite/admin.go similarity index 97% rename from plugins/extractors/googleworkspace/admin.go rename to plugins/extractors/gsuite/admin.go index 2a6e06ad6..4623f15c1 100644 --- a/plugins/extractors/googleworkspace/admin.go +++ b/plugins/extractors/gsuite/admin.go @@ -1,4 +1,4 @@ -package googleworkspace +package gsuite import ( "context" diff --git a/plugins/extractors/googleworkspace/googleworkspace.go b/plugins/extractors/gsuite/gsuite.go similarity index 98% rename from plugins/extractors/googleworkspace/googleworkspace.go rename to plugins/extractors/gsuite/gsuite.go index 26ebfe864..c99b3c94d 100644 --- a/plugins/extractors/googleworkspace/googleworkspace.go +++ b/plugins/extractors/gsuite/gsuite.go @@ -1,4 +1,4 @@ -package googleworkspace +package gsuite import ( "context" @@ -229,7 +229,7 @@ func (e *Extractor) buildMapFromGsuiteMapRawMessage(value interface{}) (result m // init registers the extractor to catalog func init() { - if err := registry.Extractors.Register("googleworkspace", func() plugins.Extractor { + if err := registry.Extractors.Register("gsuite", func() plugins.Extractor { return New(plugins.GetLog(), &DefaultUsersServiceFactory{}) }); err != nil { panic(err) diff --git a/plugins/extractors/googleworkspace/googleworkspace_test.go b/plugins/extractors/gsuite/gsuite_test.go similarity index 95% rename from plugins/extractors/googleworkspace/googleworkspace_test.go rename to plugins/extractors/gsuite/gsuite_test.go index 4927532c4..ac17752f2 100644 --- a/plugins/extractors/googleworkspace/googleworkspace_test.go +++ b/plugins/extractors/gsuite/gsuite_test.go @@ -1,4 +1,4 @@ -package googleworkspace_test +package gsuite_test import ( "context" @@ -7,7 +7,7 @@ import ( "github.com/odpf/meteor/models" assetsv1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" "github.com/odpf/meteor/plugins" - "github.com/odpf/meteor/plugins/extractors/googleworkspace" + "github.com/odpf/meteor/plugins/extractors/gsuite" "github.com/odpf/meteor/test/mocks" "github.com/odpf/meteor/test/utils" "github.com/stretchr/testify/assert" @@ -17,11 +17,11 @@ import ( "google.golang.org/api/googleapi" ) -var urnScope string = "test-googleworkspace" +var urnScope string = "test-gsuite" func TestInit(t *testing.T) { t.Run("should return error for empty user email", func(t *testing.T) { - err := googleworkspace.New(utils.Logger, new(mockUsersServiceFactory)).Init(context.TODO(), plugins.Config{ + err := gsuite.New(utils.Logger, new(mockUsersServiceFactory)).Init(context.TODO(), plugins.Config{ URNScope: urnScope, RawConfig: map[string]interface{}{ "user_email": "", @@ -30,7 +30,7 @@ func TestInit(t *testing.T) { assert.ErrorAs(t, err, &plugins.InvalidConfigError{}) }) t.Run("should return error for invalid service account json", func(t *testing.T) { - err := googleworkspace.New(utils.Logger, new(mockUsersServiceFactory)).Init(context.TODO(), plugins.Config{ + err := gsuite.New(utils.Logger, new(mockUsersServiceFactory)).Init(context.TODO(), plugins.Config{ URNScope: urnScope, RawConfig: map[string]interface{}{ "user_email": "user@example.com", @@ -45,7 +45,7 @@ func TestInit(t *testing.T) { factory := new(mockUsersServiceFactory) factory.On("BuildUserService", ctx, userEmail, serviceAcc).Return(new(mockUsersListCall), nil) - err := googleworkspace.New(utils.Logger, factory).Init(ctx, plugins.Config{ + err := gsuite.New(utils.Logger, factory).Init(ctx, plugins.Config{ URNScope: urnScope, RawConfig: map[string]interface{}{ "user_email": userEmail, @@ -183,7 +183,7 @@ func TestExtract(t *testing.T) { factory.On("BuildUserService", ctx, userEmail, serviceAcc).Return(userService, nil) defer factory.AssertExpectations(t) - extr := googleworkspace.New(utils.Logger, factory) + extr := gsuite.New(utils.Logger, factory) err := extr.Init(ctx, plugins.Config{ URNScope: urnScope, RawConfig: map[string]interface{}{ @@ -205,9 +205,9 @@ type mockUsersServiceFactory struct { mock.Mock } -func (m *mockUsersServiceFactory) BuildUserService(ctx context.Context, email, serviceAccountJSON string) (googleworkspace.UsersListCall, error) { +func (m *mockUsersServiceFactory) BuildUserService(ctx context.Context, email, serviceAccountJSON string) (gsuite.UsersListCall, error) { args := m.Called(ctx, email, serviceAccountJSON) - return args.Get(0).(googleworkspace.UsersListCall), args.Error(1) + return args.Get(0).(gsuite.UsersListCall), args.Error(1) } type mockUsersListCall struct { diff --git a/plugins/extractors/populate.go b/plugins/extractors/populate.go index 3cddec014..6c00d1247 100644 --- a/plugins/extractors/populate.go +++ b/plugins/extractors/populate.go @@ -10,8 +10,8 @@ import ( _ "github.com/odpf/meteor/plugins/extractors/elastic" _ "github.com/odpf/meteor/plugins/extractors/gcs" _ "github.com/odpf/meteor/plugins/extractors/github" - _ "github.com/odpf/meteor/plugins/extractors/googleworkspace" _ "github.com/odpf/meteor/plugins/extractors/grafana" + _ "github.com/odpf/meteor/plugins/extractors/gsuite" _ "github.com/odpf/meteor/plugins/extractors/kafka" _ "github.com/odpf/meteor/plugins/extractors/mariadb" _ "github.com/odpf/meteor/plugins/extractors/metabase" From 63dda0daf699a34b7592643fe568cad587a0e2c7 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Thu, 8 Sep 2022 19:24:33 +0530 Subject: [PATCH 18/19] docs: gsuite extractor README --- plugins/extractors/gsuite/README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/plugins/extractors/gsuite/README.md b/plugins/extractors/gsuite/README.md index 8897be5f7..bd736bdab 100644 --- a/plugins/extractors/gsuite/README.md +++ b/plugins/extractors/gsuite/README.md @@ -20,14 +20,16 @@ source: ## Outputs -| Field | Sample Value | -| :---- | :---- | -| `resource.urn` | `john.doe@gmail.com` | -| `resource.name` | `John Doe` | -| `email` | `john.doe@gmail.com` | -| `full_name` | `John Doe` | -| `status` | `not suspended` | -| `properties` | `{"attributes":{"aliases":"doe.john@gmail.com,john.doe0@gmail.com","manager":"christian.aristika@gmail.com","org_unit_path":"/"}}` +| Field | Sample Value | +|:------------------------|:-----------------------------------------------------| +| `email` | `doe.john@gmail.com` | +| `full_name` | `Jon Doe` | +| `status` | `suspended` | +| `attributes` | `{"aliases":"john.doe@odpf.com","custom_schemas":{},`| +| | `"org_unit_path":"/","organizations":` | +| | `[{"costCenter": "odpf"}],` | +| | `"relations":[{"type":"manager",` | +| | `"value":"john.lee@odpf.com"}]}` | ### Notes - The service account must have a [delegated domain wide authority](https://developers.google.com/admin-sdk/directory/v1/guides/delegation#delegate_domain-wide_authority_to_your_service_account) From 1b685b27661e1209b508cfda047e17cf52e6b54d Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Thu, 8 Sep 2022 19:24:49 +0530 Subject: [PATCH 19/19] docs: extractor list --- docs/docs/reference/extractors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/reference/extractors.md b/docs/docs/reference/extractors.md index b09ee2652..78e921a0b 100644 --- a/docs/docs/reference/extractors.md +++ b/docs/docs/reference/extractors.md @@ -44,6 +44,7 @@ Meteor currently supports metadata extraction on these data sources. To perform |:-----------------------------------------------------------------------------------------|:------|:---------|:---------|:------|:---------|:-------------|:---------|:------------|:-------|:-------| | [`github`](https://github.com/odpf/meteor/tree/main/plugins/extractors/github/README.md) | ✅ | ✅ | ✅ | ☐ | ✅ | ☐ | ☐ | ☐ | ☐ | ☐ | | [`shield`](https://github.com/odpf/meteor/tree/main/plugins/extractors/shield/README.md) | ✅ | ✅ | ✅ | ☐ | ✅ | ☐ | ☐ | ✅ | ✅ | ☐ | +| [`gsuite`](https://github.com/odpf/meteor/tree/main/plugins/extractors/gsuite/README.md) | ✅ | ☐ | ✅ | ☐ | ✅ | ✅ | ☐ | ☐ | ☐ | ☐ | ### Bucket