diff --git a/docs/docs/reference/extractors.md b/docs/docs/reference/extractors.md index b09ee2652..78e921a0b 100644 --- a/docs/docs/reference/extractors.md +++ b/docs/docs/reference/extractors.md @@ -44,6 +44,7 @@ Meteor currently supports metadata extraction on these data sources. To perform |:-----------------------------------------------------------------------------------------|:------|:---------|:---------|:------|:---------|:-------------|:---------|:------------|:-------|:-------| | [`github`](https://github.com/odpf/meteor/tree/main/plugins/extractors/github/README.md) | ✅ | ✅ | ✅ | ☐ | ✅ | ☐ | ☐ | ☐ | ☐ | ☐ | | [`shield`](https://github.com/odpf/meteor/tree/main/plugins/extractors/shield/README.md) | ✅ | ✅ | ✅ | ☐ | ✅ | ☐ | ☐ | ✅ | ✅ | ☐ | +| [`gsuite`](https://github.com/odpf/meteor/tree/main/plugins/extractors/gsuite/README.md) | ✅ | ☐ | ✅ | ☐ | ✅ | ✅ | ☐ | ☐ | ☐ | ☐ | ### Bucket diff --git a/plugins/extractors/gsuite/README.md b/plugins/extractors/gsuite/README.md new file mode 100644 index 000000000..bd736bdab --- /dev/null +++ b/plugins/extractors/gsuite/README.md @@ -0,0 +1,40 @@ +# G-Suite + +## Usage + +```yaml +source: + scope: my-scope + type: gsuite + config: + service_account_json: "XXX" + user_email: meteor@odpf.com +``` + +## Inputs + +| Key | Value | Example | Description | | +| :-- | :---- | :------ | :---------- | :- | +| `user_email` | `string` | `meteor@odpf.com` | User email authorized to access the APIs | *required* | +| `service_account_json` | `string` | `{"type": "service_account","project_id": "XXXXXX","private_key_id": "XXXXXX","private_key": "XXXXXX","client_email": "XXXXXX","client_id": "XXXXXX","auth_uri": "https://accounts.google.com/o/oauth2/auth","token_uri": "https://oauth2.googleapis.com/token","auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url": "XXXXXX"}` | Service Account JSON object | *required* | + +## Outputs + +| Field | Sample Value | +|:------------------------|:-----------------------------------------------------| +| `email` | `doe.john@gmail.com` | +| `full_name` | `Jon Doe` | +| `status` | `suspended` | +| `attributes` | `{"aliases":"john.doe@odpf.com","custom_schemas":{},`| +| | `"org_unit_path":"/","organizations":` | +| | `[{"costCenter": "odpf"}],` | +| | `"relations":[{"type":"manager",` | +| | `"value":"john.lee@odpf.com"}]}` | + +### Notes + - The service account must have a [delegated domain wide authority](https://developers.google.com/admin-sdk/directory/v1/guides/delegation#delegate_domain-wide_authority_to_your_service_account) + - User Email : Only users with access to the Admin APIs can access the Admin SDK Directory API, therefore your service account needs to impersonate one of those users to access the Admin SDK Directory API. + +## Contributing + +Refer to the [contribution guidelines](../../../docs/contribute/guide.md#adding-a-new-extractor) for information on contributing to this module. diff --git a/plugins/extractors/gsuite/admin.go b/plugins/extractors/gsuite/admin.go new file mode 100644 index 000000000..4623f15c1 --- /dev/null +++ b/plugins/extractors/gsuite/admin.go @@ -0,0 +1,37 @@ +package gsuite + +import ( + "context" + + "golang.org/x/oauth2/google" + admin "google.golang.org/api/admin/directory/v1" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" +) + +type UsersServiceFactory interface { + BuildUserService(ctx context.Context, email, serviceAccountJSON string) (UsersListCall, error) +} + +type UsersListCall interface { + Do(opts ...googleapi.CallOption) (*admin.Users, error) +} + +type DefaultUsersServiceFactory struct{} + +func (f *DefaultUsersServiceFactory) BuildUserService(ctx context.Context, email, serviceAccountJSON string) (UsersListCall, error) { + jwtConfig, err := google.JWTConfigFromJSON([]byte(serviceAccountJSON), admin.AdminDirectoryUserScope) + if err != nil { + return nil, err + } + jwtConfig.Subject = email + + ts := jwtConfig.TokenSource(ctx) + + srv, err := admin.NewService(ctx, option.WithTokenSource(ts)) + if err != nil { + return nil, err + } + + return srv.Users.List().Customer("my_customer"), nil +} diff --git a/plugins/extractors/gsuite/gsuite.go b/plugins/extractors/gsuite/gsuite.go new file mode 100644 index 000000000..c99b3c94d --- /dev/null +++ b/plugins/extractors/gsuite/gsuite.go @@ -0,0 +1,237 @@ +package gsuite + +import ( + "context" + _ "embed" // used to print the embedded assets + "fmt" + "reflect" + "strings" + + "github.com/odpf/meteor/models" + "github.com/odpf/meteor/plugins" + "github.com/odpf/meteor/registry" + "github.com/odpf/meteor/utils" + "github.com/odpf/salt/log" + admin "google.golang.org/api/admin/directory/v1" + "google.golang.org/api/googleapi" + "google.golang.org/protobuf/types/known/anypb" + + v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" +) + +//go:embed README.md +var summary string + +type Config struct { + ServiceAccountJSON string `mapstructure:"service_account_json" validate:"required"` + UserEmail string `mapstructure:"user_email" validate:"required"` +} + +var sampleConfig = ` +service_account_json: { + "type": "service_account", + "project_id": "XXXXXX", + "private_key_id": "XXXXXX", + "private_key": "XXXXXX", + "client_email": "XXXXXX", + "client_id": "XXXXXX", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "XXXXXX" +} +user_email: user@odpf.com` + +var info = plugins.Info{ + Description: "User list from Google Workspace", + SampleConfig: sampleConfig, + Tags: []string{"platform", "extractor"}, + Summary: summary, +} + +// Extractor manages the extraction of data from the extractor +type Extractor struct { + plugins.BaseExtractor + logger log.Logger + config Config + userServiceFactory UsersServiceFactory + userService UsersListCall + emit plugins.Emit +} + +// New returns a pointer to an initialized Extractor Object +func New(logger log.Logger, userServiceFactory UsersServiceFactory) *Extractor { + e := &Extractor{ + logger: logger, + userServiceFactory: userServiceFactory, + } + e.BaseExtractor = plugins.NewBaseExtractor(info, &e.config) + + return e +} + +// Init initializes the extractor +func (e *Extractor) Init(ctx context.Context, config plugins.Config) (err error) { + if err = e.BaseExtractor.Init(ctx, config); err != nil { + return err + } + + e.userService, err = e.userServiceFactory.BuildUserService(ctx, e.config.UserEmail, e.config.ServiceAccountJSON) + if err != nil { + return fmt.Errorf("error building user service: %w", err) + } + + return +} + +// Extract extracts the data from the extractor +// The data is returned as a list of assets.Asset +func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) { + e.emit = emit + adminUsers, err := e.fetchUsers(ctx) + if err != nil { + return err + } + + if len(adminUsers.Users) == 0 { + e.logger.Info("No users found.\n") + return nil + } + + for _, u := range adminUsers.Users { + asset, err := e.buildAsset(u) + if err != nil { + e.logger.Warn("error when building asset", "err", err) + continue + } + e.emit(models.NewRecord(asset)) + } + + return nil +} + +func (e *Extractor) buildAsset(gsuiteUser *admin.User) (*v1beta2.Asset, error) { + var status string + if gsuiteUser.Suspended { + status = "suspended" + } + + var userAttributes = make(map[string]interface{}) + userAttributes["organizations"] = e.buildMapFromGsuiteSlice(gsuiteUser.Organizations) + userAttributes["relations"] = e.buildMapFromGsuiteSlice(gsuiteUser.Relations) + userAttributes["custom_schemas"] = e.buildMapFromGsuiteMapRawMessage(gsuiteUser.CustomSchemas) + userAttributes["aliases"] = strings.Join(gsuiteUser.Aliases, ",") + userAttributes["org_unit_path"] = gsuiteUser.OrgUnitPath + + assetUser, err := anypb.New(&v1beta2.User{ + Email: gsuiteUser.PrimaryEmail, + FullName: gsuiteUser.Name.FullName, + Status: status, + Attributes: utils.TryParseMapToProto(userAttributes), + }) + if err != nil { + return nil, fmt.Errorf("error when creating anypb.Any: %w", err) + } + + asset := &v1beta2.Asset{ + Urn: models.NewURN("gsuite", e.UrnScope, "user", gsuiteUser.PrimaryEmail), + Name: gsuiteUser.Name.FullName, + Service: "gsuite", + Type: "user", + Data: assetUser, + } + + return asset, nil +} + +func (e *Extractor) fetchUsers(ctx context.Context) (*admin.Users, error) { + users, err := e.userService.Do() + if err != nil { + return nil, fmt.Errorf("error fetching users: %w", err) + } + + return users, nil +} + +func (e *Extractor) buildMapFromGsuiteSlice(value interface{}) (result []interface{}) { + if value == nil { + return + } + + gsuiteSlice := reflect.ValueOf(value) + if gsuiteSlice.Kind() != reflect.Slice { + return + } + + list, ok := gsuiteSlice.Interface().([]interface{}) + if !ok { + return + } + + for _, item := range list { + result = append(result, e.buildMapFromGsuiteMap(item)) + } + + return +} + +func (e *Extractor) buildMapFromGsuiteMap(value interface{}) (result map[string]interface{}) { + if value == nil { + return + } + + gsuiteMap := reflect.ValueOf(value) + if gsuiteMap.Kind() != reflect.Map { + return + } + + result = make(map[string]interface{}) + for _, key := range gsuiteMap.MapKeys() { + keyString := fmt.Sprintf("%v", key.Interface()) + value := gsuiteMap.MapIndex(key).Interface() + + result[keyString] = value + } + + return +} + +func (e *Extractor) buildMapFromGsuiteMapRawMessage(value interface{}) (result map[string]interface{}) { + if value == nil { + return + } + + gsuiteMap := reflect.ValueOf(value) + if gsuiteMap.Kind() != reflect.Map { + return + } + + result = make(map[string]interface{}) + for _, key := range gsuiteMap.MapKeys() { + keyString := fmt.Sprintf("%v", key.Interface()) + value := gsuiteMap.MapIndex(key) + + msg, ok := value.Interface().(googleapi.RawMessage) + if !ok { + continue + } + + json, err := msg.MarshalJSON() + if err != nil { + continue + } + + result[keyString] = string(json) + } + + return +} + +// init registers the extractor to catalog +func init() { + if err := registry.Extractors.Register("gsuite", func() plugins.Extractor { + return New(plugins.GetLog(), &DefaultUsersServiceFactory{}) + }); err != nil { + panic(err) + } +} diff --git a/plugins/extractors/gsuite/gsuite_test.go b/plugins/extractors/gsuite/gsuite_test.go new file mode 100644 index 000000000..ac17752f2 --- /dev/null +++ b/plugins/extractors/gsuite/gsuite_test.go @@ -0,0 +1,225 @@ +package gsuite_test + +import ( + "context" + "testing" + + "github.com/odpf/meteor/models" + assetsv1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" + "github.com/odpf/meteor/plugins" + "github.com/odpf/meteor/plugins/extractors/gsuite" + "github.com/odpf/meteor/test/mocks" + "github.com/odpf/meteor/test/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + admin "google.golang.org/api/admin/directory/v1" + "google.golang.org/api/googleapi" +) + +var urnScope string = "test-gsuite" + +func TestInit(t *testing.T) { + t.Run("should return error for empty user email", func(t *testing.T) { + err := gsuite.New(utils.Logger, new(mockUsersServiceFactory)).Init(context.TODO(), plugins.Config{ + URNScope: urnScope, + RawConfig: map[string]interface{}{ + "user_email": "", + "service_account_json": "{\"type\":\"service_account\",\"project_id\":\"odpf-meteor\",\"private_key_id\":\"3cb2103ef7883845a5fdcsvdefe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggdvdvdEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdAUSjzKdbdfbdbYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYEin\\nHYj064sMvm8vbR5TcMQpnxYG86TGaPuIh30grz5dI39dtrUjttbdfbdvqRv0qu7I5\\nuELzp2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmBHbdfdi29bhoS+Ac\\n5ipT10yGF0FvT1f5KlJcHfsNoOGPJYePTaGxOW1zk680Z1Wdfbdf1xX9iw5/GUA3XM\\neon4p9X31ASgwbdbdplFZhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAECggEbdbddYz8nSmTWFMW2OtyvojIq+ab864ZGPCpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJThfOscyXbbdbdbfbfT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoYEmOsB2xgqjvKXR90r5wNJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMVVa6ZpxCTWvbQga+/ZPaqppgGps5yLDqc434c3A/lDCKBtqk\\njaQXHqKjuYUsoiyl2vbPbwGxc34343c6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOAEfc343535dJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwRrnbQFwhErWS1fZgg2PcQKBgQDMsRgDBfhgJX9sNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDh3ELmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXU3c4343c4QRAoGBAK9F\\nMO9dzFjfouVP63f/Nf3GeIlctuiE1r5IOX4di3qNe/P33iqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3x4XZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLGTw1vzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIxZKeq3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@odpf-meteor.iam.gserviceaccount.com\",\"client_id\":\"110059943435984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-meteor.iam.gserviceaccount.com\"}", + }}) + assert.ErrorAs(t, err, &plugins.InvalidConfigError{}) + }) + t.Run("should return error for invalid service account json", func(t *testing.T) { + err := gsuite.New(utils.Logger, new(mockUsersServiceFactory)).Init(context.TODO(), plugins.Config{ + URNScope: urnScope, + RawConfig: map[string]interface{}{ + "user_email": "user@example.com", + "service_account_json": "", + }}) + assert.ErrorAs(t, err, &plugins.InvalidConfigError{}) + }) + t.Run("should return error for invalid service account json", func(t *testing.T) { + ctx := context.TODO() + userEmail := "user@example.com" + serviceAcc := "{\"type\":\"service_account\",\"project_id\":\"odpf-meteor\",\"private_key_id\":\"3cb2103ef7883845a5fdcsvdefe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggdvdvdEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdAUSjzKdbdfbdbYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYEin\\nHYj064sMvm8vbR5TcMQpnxYG86TGaPuIh30grz5dI39dtrUjttbdfbdvqRv0qu7I5\\nuELzp2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmBHbdfdi29bhoS+Ac\\n5ipT10yGF0FvT1f5KlJcHfsNoOGPJYePTaGxOW1zk680Z1Wdfbdf1xX9iw5/GUA3XM\\neon4p9X31ASgwbdbdplFZhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAECggEbdbddYz8nSmTWFMW2OtyvojIq+ab864ZGPCpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJThfOscyXbbdbdbfbfT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoYEmOsB2xgqjvKXR90r5wNJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMVVa6ZpxCTWvbQga+/ZPaqppgGps5yLDqc434c3A/lDCKBtqk\\njaQXHqKjuYUsoiyl2vbPbwGxc34343c6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOAEfc343535dJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwRrnbQFwhErWS1fZgg2PcQKBgQDMsRgDBfhgJX9sNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDh3ELmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXU3c4343c4QRAoGBAK9F\\nMO9dzFjfouVP63f/Nf3GeIlctuiE1r5IOX4di3qNe/P33iqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3x4XZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLGTw1vzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIxZKeq3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@odpf-meteor.iam.gserviceaccount.com\",\"client_id\":\"110059943435984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-meteor.iam.gserviceaccount.com\"}" + factory := new(mockUsersServiceFactory) + factory.On("BuildUserService", ctx, userEmail, serviceAcc).Return(new(mockUsersListCall), nil) + + err := gsuite.New(utils.Logger, factory).Init(ctx, plugins.Config{ + URNScope: urnScope, + RawConfig: map[string]interface{}{ + "user_email": userEmail, + "service_account_json": serviceAcc, + }}) + assert.NoError(t, err) + }) +} + +func TestExtract(t *testing.T) { + t.Run("should extract user details from google workspace", func(t *testing.T) { + adminUsers := []*admin.User{ + { + Name: &admin.UserName{FullName: "User1"}, + PrimaryEmail: "user1@test.com", + Suspended: true, + Aliases: []string{"alias1", "alias2"}, + Relations: []interface{}{ + map[string]interface{}{ + "type": "manager1", + "value": "manager1@test.com", + }, + }, + Organizations: []interface{}{ + map[string]interface{}{ + "foo0": "bar0", + "foo1": "bar1", + }, + }, + OrgUnitPath: "/", + CustomSchemas: map[string]googleapi.RawMessage{ + "foo0_customSchema": []byte("bar0_customSchema"), + "foo1_customSchema": []byte("bar1_customSchema"), + }, + }, + { + Name: &admin.UserName{FullName: "User2"}, + PrimaryEmail: "user2@test.com", + Suspended: false, + Aliases: []string{"alias3"}, + Relations: []interface{}{ + map[string]interface{}{ + "type": "manager2", + "value": "manager2@test.com", + }, + }, + Organizations: []interface{}{ + map[string]interface{}{ + "foo20": "bar20", + "foo21": "bar21", + }, + }, + OrgUnitPath: "/test2", + CustomSchemas: map[string]googleapi.RawMessage{ + "foo20_customSchema": []byte("bar20_customSchema"), + "foo21_customSchema": []byte("bar21_customSchema"), + }, + }, + } + + expectedData := []*assetsv1beta2.Asset{ + { + Urn: models.NewURN("gsuite", urnScope, "user", adminUsers[0].PrimaryEmail), + Name: adminUsers[0].Name.FullName, + Service: "gsuite", + Type: "user", + Data: utils.BuildAny(t, &assetsv1beta2.User{ + Email: adminUsers[0].PrimaryEmail, + FullName: adminUsers[0].Name.FullName, + Status: "suspended", + Attributes: utils.BuildStruct(t, map[string]interface{}{ + "aliases": "alias1,alias2", + "org_unit_path": "/", + "organizations": []interface{}{ + map[string]interface{}{ + "foo0": "bar0", + "foo1": "bar1", + }, + }, + "custom_schemas": map[string]interface{}{ + "foo0_customSchema": "bar0_customSchema", + "foo1_customSchema": "bar1_customSchema", + }, + "relations": []interface{}{ + map[string]interface{}{ + "type": "manager1", + "value": "manager1@test.com", + }, + }, + }), + }), + }, + { + Urn: models.NewURN("gsuite", urnScope, "user", adminUsers[1].PrimaryEmail), + Name: adminUsers[1].Name.FullName, + Service: "gsuite", + Type: "user", + Data: utils.BuildAny(t, &assetsv1beta2.User{ + Email: adminUsers[1].PrimaryEmail, + FullName: adminUsers[1].Name.FullName, + Status: "", + Attributes: utils.BuildStruct(t, map[string]interface{}{ + "aliases": "alias3", + "org_unit_path": "/test2", + "organizations": []interface{}{ + map[string]interface{}{ + "foo20": "bar20", + "foo21": "bar21", + }, + }, + "custom_schemas": map[string]interface{}{ + "foo20_customSchema": "bar20_customSchema", + "foo21_customSchema": "bar21_customSchema", + }, + "relations": []interface{}{ + map[string]interface{}{ + "type": "manager2", + "value": "manager2@test.com", + }, + }, + }), + }), + }, + } + + ctx := context.TODO() + userEmail := "user@example.com" + serviceAcc := "{\"type\":\"service_account\",\"project_id\":\"odpf-meteor\",\"private_key_id\":\"3cb2103ef7883845a5fdcsvdefe6ff83d616757\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggdvdvdEAAoIBAQDF/cDQ++JnH9+9\\n3YBm4APqPbvfj6eHSdAUSjzKdbdfbdbYGgdxC7xPS1PVo+ENw+pBAH3NoRwQWYEin\\nHYj064sMvm8vbR5TcMQpnxYG86TGaPuIh30grz5dI39dtrUjttbdfbdvqRv0qu7I5\\nuELzp2OLUz509Q3AvuqvQVCZc7sDjNr2TPOsLeuCkpmcmBHbdfdi29bhoS+Ac\\n5ipT10yGF0FvT1f5KlJcHfsNoOGPJYePTaGxOW1zk680Z1Wdfbdf1xX9iw5/GUA3XM\\neon4p9X31ASgwbdbdplFZhwvcpoaYpxcuxyvefR44emnfveUY91h6wLvF/mPBElO\\npXOiVJ3lAgMBAAECggEbdbddYz8nSmTWFMW2OtyvojIq+ab864ZGPCpW4zfzF4BI\\n7o5TSIsNOMQMrawFUz0xZkgofJThfOscyXbbdbdbfbfT3wXI9JTWT8l275ssvFQVy1\\nVyAJI/Kize9ru5GnnEzV2sZoYEmOsB2xgqjvKXR90r5wNJ6wFp8Ubp9/+v2lTv1n\\nUCBBYPsPyVmUq677HfMVVa6ZpxCTWvbQga+/ZPaqppgGps5yLDqc434c3A/lDCKBtqk\\njaQXHqKjuYUsoiyl2vbPbwGxc34343c6gQfe7aeCouf8bI4GzCPmoyVPMRFpQJ6Ahp\\nMnCE96KfVVUARh1goxEEwMmSFyBPYFbmvXLPUGNfcQKBgQD3nrDHeWxW+0MjnaYD\\novXKvpnv1NiBCywOAEfc343535dJfgMZX0cfpnTDGXKPBI5ZbUywxk0sewu382JoArM\\n1w2wEIqH+73FGiMVpAuN2DpNX5mOC+z/zjFdOFZ28jkRUy8T+PTkajj7rkB7VDOr\\nIiCZwRrnbQFwhErWS1fZgg2PcQKBgQDMsRgDBfhgJX9sNRX3FHzIEZU94PP1KOc2\\nEUUzcwIV0cNOVzSyOUn2qrcYNg/hZZpGeRBBwyOcDGsqxmz5FAzk0OtbSCaMxybF\\n8NXFDh3ELmnfIyVBjvNBWPckcR1LCZcKGTqVLH/rhPiNhyzH3NQ0c3Gl15GPgzkD\\nboLfFN3jtQKBgG++blpmYkzScNb2wr9rX+5Rm1hOvjFl4EilOb+1rq/WPZ0ig5ZD\\nT5mdQ6ZC+5ppWp8AyjQsgsAYgUG1NoqAFg45OLrrERWMmP6gHBKz3IOkO8CNgzNh\\nUoeV7/cXkkdOObWSqLkXcoWpejHtqq905C9epIyBdZ/YI4mXU3c4343c4QRAoGBAK9F\\nMO9dzFjfouVP63f/Nf3GeIlctuiE1r5IOX4di3qNe/P33iqBvaCWe2Mi36Q78MdJ\\nYK8+3Z4AUD93WtZI4eWIMw+dj0zaNowldZZfSQO0Tnl/yaYCNq8M88pjhRa8pnVC\\nNxSG3x4XZREi3yhgIeCrvXOpS32celRC65MDdiBFAoGAHbURTEkQDZaWPAmVv+0q\\nYaT7x+UzQDGKy/By9QLGM/U2gvLGTw1vzmoeh99BTsQopPB/QuAfJNIHk9h0ohXJ\\nfA/X4T3F2LGhZ9+bujVyCQc0tTxuh41t2ipJPWtDP52rXk1AkCnIeWD+UHI0u5Ba\\nhI1dzLIxZKeq3bESrc/9tmM=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"meteor-sa@odpf-meteor.iam.gserviceaccount.com\",\"client_id\":\"110059943435984635286\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/meteor-sa%40odpf-meteor.iam.gserviceaccount.com\"}" + + userService := new(mockUsersListCall) + userService.On("Do").Return(adminUsers).Once().Return(&admin.Users{Users: adminUsers}, nil) + defer userService.AssertExpectations(t) + + factory := new(mockUsersServiceFactory) + factory.On("BuildUserService", ctx, userEmail, serviceAcc).Return(userService, nil) + defer factory.AssertExpectations(t) + + extr := gsuite.New(utils.Logger, factory) + err := extr.Init(ctx, plugins.Config{ + URNScope: urnScope, + RawConfig: map[string]interface{}{ + "user_email": userEmail, + "service_account_json": serviceAcc, + }}, + ) + require.NoError(t, err) + + emitter := mocks.NewEmitter() + err = extr.Extract(ctx, emitter.Push) + require.NoError(t, err) + + utils.AssertAssetsWithJSON(t, expectedData, emitter.GetAllData()) + }) +} + +type mockUsersServiceFactory struct { + mock.Mock +} + +func (m *mockUsersServiceFactory) BuildUserService(ctx context.Context, email, serviceAccountJSON string) (gsuite.UsersListCall, error) { + args := m.Called(ctx, email, serviceAccountJSON) + return args.Get(0).(gsuite.UsersListCall), args.Error(1) +} + +type mockUsersListCall struct { + mock.Mock +} + +func (m *mockUsersListCall) Do(opts ...googleapi.CallOption) (*admin.Users, error) { + var args mock.Arguments + if len(opts) > 0 { + args = m.Called(opts) + } else { + args = m.Called() + } + return args.Get(0).(*admin.Users), args.Error(1) +} diff --git a/plugins/extractors/populate.go b/plugins/extractors/populate.go index 6f6ea3ae1..6c00d1247 100644 --- a/plugins/extractors/populate.go +++ b/plugins/extractors/populate.go @@ -11,6 +11,7 @@ import ( _ "github.com/odpf/meteor/plugins/extractors/gcs" _ "github.com/odpf/meteor/plugins/extractors/github" _ "github.com/odpf/meteor/plugins/extractors/grafana" + _ "github.com/odpf/meteor/plugins/extractors/gsuite" _ "github.com/odpf/meteor/plugins/extractors/kafka" _ "github.com/odpf/meteor/plugins/extractors/mariadb" _ "github.com/odpf/meteor/plugins/extractors/metabase" diff --git a/test/utils/struct.go b/test/utils/struct.go new file mode 100644 index 000000000..2d7880fd8 --- /dev/null +++ b/test/utils/struct.go @@ -0,0 +1,15 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func BuildStruct(t *testing.T, value map[string]interface{}) *structpb.Struct { + res, err := structpb.NewStruct(value) + require.NoError(t, err) + + return res +}