From 50145c7d759123d72e26c090c012462e04e7b810 Mon Sep 17 00:00:00 2001 From: Vishal Bihani Date: Wed, 5 Apr 2023 01:27:08 +0530 Subject: [PATCH 01/10] make bulk cd trigger async --- go.mod | 2 +- go.sum | 18 +---- pkg/bulkAction/BulkUpdateService.go | 70 +++++++++++++++++-- .../go-winio/.gitignore | 0 .../go-winio/CODEOWNERS | 0 .../{microsoft => Microsoft}/go-winio/LICENSE | 0 .../go-winio/README.md | 0 .../go-winio/backup.go | 0 .../{microsoft => Microsoft}/go-winio/ea.go | 0 .../{microsoft => Microsoft}/go-winio/file.go | 0 .../go-winio/fileinfo.go | 0 .../go-winio/hvsock.go | 0 .../{microsoft => Microsoft}/go-winio/pipe.go | 0 .../go-winio/pkg/guid/guid.go | 0 .../go-winio/pkg/guid/guid_nonwindows.go | 0 .../go-winio/pkg/guid/guid_windows.go | 0 .../go-winio/privilege.go | 0 .../go-winio/reparse.go | 0 .../{microsoft => Microsoft}/go-winio/sd.go | 0 .../go-winio/syscall.go | 0 .../go-winio/zsyscall_windows.go | 0 .../blob-storage/BlobStorageService.go | 2 +- .../common-lib/pubsub-lib/JetStreamUtil.go | 20 +++--- vendor/modules.txt | 2 +- 24 files changed, 82 insertions(+), 32 deletions(-) rename vendor/github.com/{microsoft => Microsoft}/go-winio/.gitignore (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/CODEOWNERS (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/LICENSE (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/README.md (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/backup.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/ea.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/file.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/fileinfo.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/hvsock.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/pipe.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/pkg/guid/guid.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/pkg/guid/guid_nonwindows.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/pkg/guid/guid_windows.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/privilege.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/reparse.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/sd.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/syscall.go (100%) rename vendor/github.com/{microsoft => Microsoft}/go-winio/zsyscall_windows.go (100%) diff --git a/go.mod b/go.mod index cdec164342..e0dca699ca 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/davecgh/go-spew v1.1.1 github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 - github.com/devtron-labs/common-lib v0.0.0-20230228120609-58fa7d3de8e2 + github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-pg/pg v6.15.1+incompatible diff --git a/go.sum b/go.sum index 373ab84e29..a78ddc640e 100644 --- a/go.sum +++ b/go.sum @@ -282,22 +282,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 h1:oHDpsCsuYiL+u5TEhilKNLGhH9lwRNkIqrzhHQA6d3c= github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.0-20230109070754-ff4dca200a2c h1:jz7yPUlIJXFg9AvJh2fb0QW7JT6+RKj8LOl1mWM/HQA= -github.com/devtron-labs/common-lib v0.0.0-20230109070754-ff4dca200a2c/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20230202071119-47116fd18242 h1:FfS2R86Ft/WFShnwGu4z14iOKZAPhfDV7tvvlaJfRmI= -github.com/devtron-labs/common-lib v0.0.0-20230202071119-47116fd18242/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20230202123846-6a2fbc3adf9a h1:W7iFtOdOj/pYLBLkVxkpXIGX8La5nCq3tZey1wtSeCo= -github.com/devtron-labs/common-lib v0.0.0-20230202123846-6a2fbc3adf9a/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20230202130856-d4ef0e6ba405 h1:PHtTohiRePmGWkNCykvLY6Mpebke4GlW4fvO9zHzn5E= -github.com/devtron-labs/common-lib v0.0.0-20230202130856-d4ef0e6ba405/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20230202133846-0be4c43dd7f2 h1:O0NgzYI+dF2OxncbzfcsscIPV6Kf1G6O3SXec7yhs7E= -github.com/devtron-labs/common-lib v0.0.0-20230202133846-0be4c43dd7f2/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20230220060258-e0dba0258e97 h1:3gzBIn0k/4/Cal0px0HuzOcbDP/12C8U+nUN0GzolIA= -github.com/devtron-labs/common-lib v0.0.0-20230220060258-e0dba0258e97/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20230220122715-29f6f63d39e9 h1:JKaG846oVWXdx+1zqb777sYEhvmYOEJOI+Wroc4AQaI= -github.com/devtron-labs/common-lib v0.0.0-20230220122715-29f6f63d39e9/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20230228120609-58fa7d3de8e2 h1:Z4OIQU6mTBZREYBN0+GEl7E2Y4ffWCWPbu9OSowbZlE= -github.com/devtron-labs/common-lib v0.0.0-20230228120609-58fa7d3de8e2/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= +github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c h1:Zjr8yvDyK/NTWcObcnPYzBverONH1108gwKAXyEZKH4= +github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index 027876e781..f4df29b0b1 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/devtron/api/bean" client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" @@ -84,6 +85,7 @@ type BulkUpdateServiceImpl struct { ciPipelineRepository pipelineConfig.CiPipelineRepository appWorkflowRepository appWorkflow.AppWorkflowRepository appWorkflowService appWorkflow2.AppWorkflowService + pubsubClient *pubsub.PubSubClientServiceImpl } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, @@ -111,8 +113,9 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito enforcerUtilHelm rbac.EnforcerUtilHelm, ciHandler pipeline.CiHandler, ciPipelineRepository pipelineConfig.CiPipelineRepository, appWorkflowRepository appWorkflow.AppWorkflowRepository, - appWorkflowService appWorkflow2.AppWorkflowService) *BulkUpdateServiceImpl { - return &BulkUpdateServiceImpl{ + appWorkflowService appWorkflow2.AppWorkflowService, + pubsubClient *pubsub.PubSubClientServiceImpl) (*BulkUpdateServiceImpl, error) { + impl := &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, chartRepository: chartRepository, logger: logger, @@ -143,7 +146,11 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito ciPipelineRepository: ciPipelineRepository, appWorkflowRepository: appWorkflowRepository, appWorkflowService: appWorkflowService, + pubsubClient: pubsubClient, } + + err := impl.SubscribeToCdBulkTriggerTopic() + return impl, err } func (impl BulkUpdateServiceImpl) FindBulkUpdateReadme(operation string) (*BulkUpdateSeeExampleResponse, error) { @@ -1198,14 +1205,32 @@ func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironm UserId: request.UserId, CdWorkflowType: bean.CD_WORKFLOW_TYPE_DEPLOY, } - _, err = impl.workflowDagExecutor.ManualCdTrigger(overrideRequest, ctx) + + payload, err := json.Marshal(overrideRequest) if err != nil { - impl.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest) + impl.logger.Errorw("failed to marshal override request", + "request", overrideRequest, + "err", err) + pipelineResponse := response[appKey] pipelineResponse[pipelineKey] = false response[appKey] = pipelineResponse - //return nil, err + continue + } + + err = impl.pubsubClient.Publish(pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, string(payload)) + if err != nil { + impl.logger.Errorw("failed to publish trigger request event", + "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, + "request", overrideRequest, + "err", err) + + pipelineResponse := response[appKey] + pipelineResponse[pipelineKey] = false + response[appKey] = pipelineResponse + continue } + pipelineResponse := response[appKey] pipelineResponse[pipelineKey] = success response[appKey] = pipelineResponse @@ -1216,6 +1241,41 @@ func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironm return bulkOperationResponse, nil } +func (impl BulkUpdateServiceImpl) SubscribeToCdBulkTriggerTopic() error { + + callback := func(msg *pubsub.PubSubMsg) { + impl.logger.Debugw("Event received", + "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC) + + overrideReq := &bean.ValuesOverrideRequest{} + err := json.Unmarshal([]byte(msg.Data), overrideReq) + if err != nil { + impl.logger.Errorw("Error unmarshalling received event", + "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, + "msg", msg.Data, + "err", err) + return + } + + // trigger + _, err = impl.workflowDagExecutor.ManualCdTrigger(overrideReq, context.Background()) + if err != nil { + impl.logger.Errorw("Error triggering CD", + "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, + "msg", msg.Data, + "err", err) + } + } + err := impl.pubsubClient.Subscribe(pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, callback) + if err != nil { + impl.logger.Error("failed to subscribe to NATS topic", + "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, + "err", err) + return err + } + return nil +} + func (impl BulkUpdateServiceImpl) BulkBuildTrigger(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) { var pipelines []*pipelineConfig.Pipeline var err error diff --git a/vendor/github.com/microsoft/go-winio/.gitignore b/vendor/github.com/Microsoft/go-winio/.gitignore similarity index 100% rename from vendor/github.com/microsoft/go-winio/.gitignore rename to vendor/github.com/Microsoft/go-winio/.gitignore diff --git a/vendor/github.com/microsoft/go-winio/CODEOWNERS b/vendor/github.com/Microsoft/go-winio/CODEOWNERS similarity index 100% rename from vendor/github.com/microsoft/go-winio/CODEOWNERS rename to vendor/github.com/Microsoft/go-winio/CODEOWNERS diff --git a/vendor/github.com/microsoft/go-winio/LICENSE b/vendor/github.com/Microsoft/go-winio/LICENSE similarity index 100% rename from vendor/github.com/microsoft/go-winio/LICENSE rename to vendor/github.com/Microsoft/go-winio/LICENSE diff --git a/vendor/github.com/microsoft/go-winio/README.md b/vendor/github.com/Microsoft/go-winio/README.md similarity index 100% rename from vendor/github.com/microsoft/go-winio/README.md rename to vendor/github.com/Microsoft/go-winio/README.md diff --git a/vendor/github.com/microsoft/go-winio/backup.go b/vendor/github.com/Microsoft/go-winio/backup.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/backup.go rename to vendor/github.com/Microsoft/go-winio/backup.go diff --git a/vendor/github.com/microsoft/go-winio/ea.go b/vendor/github.com/Microsoft/go-winio/ea.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/ea.go rename to vendor/github.com/Microsoft/go-winio/ea.go diff --git a/vendor/github.com/microsoft/go-winio/file.go b/vendor/github.com/Microsoft/go-winio/file.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/file.go rename to vendor/github.com/Microsoft/go-winio/file.go diff --git a/vendor/github.com/microsoft/go-winio/fileinfo.go b/vendor/github.com/Microsoft/go-winio/fileinfo.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/fileinfo.go rename to vendor/github.com/Microsoft/go-winio/fileinfo.go diff --git a/vendor/github.com/microsoft/go-winio/hvsock.go b/vendor/github.com/Microsoft/go-winio/hvsock.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/hvsock.go rename to vendor/github.com/Microsoft/go-winio/hvsock.go diff --git a/vendor/github.com/microsoft/go-winio/pipe.go b/vendor/github.com/Microsoft/go-winio/pipe.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/pipe.go rename to vendor/github.com/Microsoft/go-winio/pipe.go diff --git a/vendor/github.com/microsoft/go-winio/pkg/guid/guid.go b/vendor/github.com/Microsoft/go-winio/pkg/guid/guid.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/pkg/guid/guid.go rename to vendor/github.com/Microsoft/go-winio/pkg/guid/guid.go diff --git a/vendor/github.com/microsoft/go-winio/pkg/guid/guid_nonwindows.go b/vendor/github.com/Microsoft/go-winio/pkg/guid/guid_nonwindows.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/pkg/guid/guid_nonwindows.go rename to vendor/github.com/Microsoft/go-winio/pkg/guid/guid_nonwindows.go diff --git a/vendor/github.com/microsoft/go-winio/pkg/guid/guid_windows.go b/vendor/github.com/Microsoft/go-winio/pkg/guid/guid_windows.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/pkg/guid/guid_windows.go rename to vendor/github.com/Microsoft/go-winio/pkg/guid/guid_windows.go diff --git a/vendor/github.com/microsoft/go-winio/privilege.go b/vendor/github.com/Microsoft/go-winio/privilege.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/privilege.go rename to vendor/github.com/Microsoft/go-winio/privilege.go diff --git a/vendor/github.com/microsoft/go-winio/reparse.go b/vendor/github.com/Microsoft/go-winio/reparse.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/reparse.go rename to vendor/github.com/Microsoft/go-winio/reparse.go diff --git a/vendor/github.com/microsoft/go-winio/sd.go b/vendor/github.com/Microsoft/go-winio/sd.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/sd.go rename to vendor/github.com/Microsoft/go-winio/sd.go diff --git a/vendor/github.com/microsoft/go-winio/syscall.go b/vendor/github.com/Microsoft/go-winio/syscall.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/syscall.go rename to vendor/github.com/Microsoft/go-winio/syscall.go diff --git a/vendor/github.com/microsoft/go-winio/zsyscall_windows.go b/vendor/github.com/Microsoft/go-winio/zsyscall_windows.go similarity index 100% rename from vendor/github.com/microsoft/go-winio/zsyscall_windows.go rename to vendor/github.com/Microsoft/go-winio/zsyscall_windows.go diff --git a/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobStorageService.go b/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobStorageService.go index ebd8fe4ce2..3c7cca2506 100644 --- a/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobStorageService.go +++ b/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobStorageService.go @@ -46,7 +46,7 @@ func (impl *BlobStorageServiceImpl) PutWithCommand(request *BlobStorageRequest) if err != nil { log.Println(" -----> push err", err) } - return nil + return err } func (impl *BlobStorageServiceImpl) Get(request *BlobStorageRequest) (bool, int64, error) { diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go index 9b0915376e..3f748471b9 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go @@ -80,6 +80,9 @@ const ( ARGO_PIPELINE_STATUS_UPDATE_TOPIC string = "ARGO_PIPELINE_STATUS_UPDATE" ARGO_PIPELINE_STATUS_UPDATE_GROUP string = "ARGO_PIPELINE_STATUS_UPDATE_GROUP-1" ARGO_PIPELINE_STATUS_UPDATE_DURABLE string = "ARGO_PIPELINE_STATUS_UPDATE_DURABLE-1" + CD_BULK_DEPLOY_TRIGGER_TOPIC string = "CD-BULK-DEPLOY-TRIGGER" + CD_BULK_DEPLOY_TRIGGER_GROUP string = "CD-BULK-DEPLOY-TRIGGER-GROUP-1" + CD_BULK_DEPLOY_TRIGGER_DURABLE string = "CD-BULK-DEPLOY-TRIGGER-DURABLE-1" ) type NatsTopic struct { @@ -95,11 +98,12 @@ type ConfigJson struct { var natsTopicMapping = map[string]NatsTopic{ - BULK_APPSTORE_DEPLOY_TOPIC: {topicName: BULK_APPSTORE_DEPLOY_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: BULK_APPSTORE_DEPLOY_GROUP, consumerName: BULK_APPSTORE_DEPLOY_DURABLE}, - BULK_DEPLOY_TOPIC: {topicName: BULK_DEPLOY_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: BULK_DEPLOY_GROUP, consumerName: BULK_DEPLOY_DURABLE}, - BULK_HIBERNATE_TOPIC: {topicName: BULK_HIBERNATE_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: BULK_HIBERNATE_GROUP, consumerName: BULK_HIBERNATE_DURABLE}, - CD_SUCCESS: {topicName: CD_SUCCESS, streamName: ORCHESTRATOR_STREAM, queueName: CD_TRIGGER_GROUP, consumerName: CD_TRIGGER_DURABLE}, - WEBHOOK_EVENT_TOPIC: {topicName: WEBHOOK_EVENT_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: WEBHOOK_EVENT_GROUP, consumerName: WEBHOOK_EVENT_DURABLE}, + BULK_APPSTORE_DEPLOY_TOPIC: {topicName: BULK_APPSTORE_DEPLOY_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: BULK_APPSTORE_DEPLOY_GROUP, consumerName: BULK_APPSTORE_DEPLOY_DURABLE}, + BULK_DEPLOY_TOPIC: {topicName: BULK_DEPLOY_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: BULK_DEPLOY_GROUP, consumerName: BULK_DEPLOY_DURABLE}, + BULK_HIBERNATE_TOPIC: {topicName: BULK_HIBERNATE_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: BULK_HIBERNATE_GROUP, consumerName: BULK_HIBERNATE_DURABLE}, + CD_SUCCESS: {topicName: CD_SUCCESS, streamName: ORCHESTRATOR_STREAM, queueName: CD_TRIGGER_GROUP, consumerName: CD_TRIGGER_DURABLE}, + WEBHOOK_EVENT_TOPIC: {topicName: WEBHOOK_EVENT_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: WEBHOOK_EVENT_GROUP, consumerName: WEBHOOK_EVENT_DURABLE}, + CD_BULK_DEPLOY_TRIGGER_TOPIC: {topicName: CD_BULK_DEPLOY_TRIGGER_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: CD_BULK_DEPLOY_TRIGGER_GROUP, consumerName: CD_BULK_DEPLOY_TRIGGER_DURABLE}, CI_COMPLETE_TOPIC: {topicName: CI_COMPLETE_TOPIC, streamName: CI_RUNNER_STREAM, queueName: CI_COMPLETE_GROUP, consumerName: CI_COMPLETE_DURABLE}, CD_STAGE_COMPLETE_TOPIC: {topicName: CD_STAGE_COMPLETE_TOPIC, streamName: CI_RUNNER_STREAM, queueName: CD_COMPLETE_GROUP, consumerName: CD_COMPLETE_DURABLE}, @@ -215,14 +219,14 @@ func ParseAndFillStreamWiseAndConsumerWiseConfigMaps() { for key, _ := range NatsConsumerWiseConfigMapping { defaultValue := defaultConsumerConfigVal - // Setting AckWait config. Only for BULK CD TRIGGER topic. Can be used for other topics + // Setting AckWait config. Only for BULK CD TRIGGER topics. Can be used for other topics // if required to be made configurable - if key == BULK_DEPLOY_DURABLE { + if key == BULK_DEPLOY_DURABLE || key == CD_BULK_DEPLOY_TRIGGER_DURABLE { defaultValue.AckWaitInSecs = defaultConsumerConfigForBulkCdTriggerTopic.AckWaitInSecs } // Overriding default config with explicitly provided topic-specific config - if _, ok := consumerConfigMap[key]; ok && (key != BULK_DEPLOY_DURABLE) { + if _, ok := consumerConfigMap[key]; ok && (key != BULK_DEPLOY_DURABLE && key != CD_BULK_DEPLOY_TRIGGER_DURABLE) { defaultValue = consumerConfigMap[key] } diff --git a/vendor/modules.txt b/vendor/modules.txt index 077a569279..aa0b23fc92 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -327,7 +327,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.0-20230228120609-58fa7d3de8e2 +# github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c ## explicit; go 1.18 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/pubsub-lib From 74f4d46caeec5beb26eab440da54ec976f3dbc49 Mon Sep 17 00:00:00 2001 From: Vishal Bihani Date: Wed, 5 Apr 2023 01:35:22 +0530 Subject: [PATCH 02/10] fix dependency injection issue --- wire_gen.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/wire_gen.go b/wire_gen.go index 6072c0ac81..c2cb48e436 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,7 +1,8 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate wire -//+build !wireinject +//go:generate go run github.com/google/wire/cmd/wire +//go:build !wireinject +// +build !wireinject package main @@ -613,7 +614,10 @@ func InitializeApp() (*App, error) { telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended, enforcerImpl, userServiceImpl) telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) - bulkUpdateServiceImpl := bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, httpClient, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, cdWorkflowRepositoryImpl, pipelineBuilderImpl, helmAppServiceImpl, enforcerUtilImpl, enforcerUtilHelmImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl) + bulkUpdateServiceImpl, err := bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, httpClient, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, cdWorkflowRepositoryImpl, pipelineBuilderImpl, helmAppServiceImpl, enforcerUtilImpl, enforcerUtilHelmImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, pubSubClientServiceImpl) + if err != nil { + return nil, err + } bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, gitSensorClientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, argoUserServiceImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := git.NewWebhookSecretValidatorImpl(sugaredLogger) From 8a3ac206f39207c3de770b9a2ed6007e366d1f88 Mon Sep 17 00:00:00 2001 From: Vishal Bihani Date: Wed, 5 Apr 2023 14:37:52 +0530 Subject: [PATCH 03/10] add log --- pkg/bulkAction/BulkUpdateService.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index f4df29b0b1..4b291cc376 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -1244,8 +1244,9 @@ func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironm func (impl BulkUpdateServiceImpl) SubscribeToCdBulkTriggerTopic() error { callback := func(msg *pubsub.PubSubMsg) { - impl.logger.Debugw("Event received", - "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC) + impl.logger.Infow("Event received", + "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, + "msg", msg.Data) overrideReq := &bean.ValuesOverrideRequest{} err := json.Unmarshal([]byte(msg.Data), overrideReq) From 85cd203986ed5601dbfb07c09e8e3b71576e1994 Mon Sep 17 00:00:00 2001 From: vikramdevtron Date: Wed, 5 Apr 2023 16:57:22 +0530 Subject: [PATCH 04/10] batch authorization check added for bulk deploy --- api/restHandler/BulkUpdateRestHandler.go | 14 +++++++++- pkg/bulkAction/BulkUpdateService.go | 33 +++++++++++++++++++----- wire_gen.go | 5 ++-- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/api/restHandler/BulkUpdateRestHandler.go b/api/restHandler/BulkUpdateRestHandler.go index 945f35e534..52642115e5 100644 --- a/api/restHandler/BulkUpdateRestHandler.go +++ b/api/restHandler/BulkUpdateRestHandler.go @@ -354,7 +354,7 @@ func (handler BulkUpdateRestHandlerImpl) BulkDeploy(w http.ResponseWriter, r *ht } ctx := context.WithValue(r.Context(), "token", acdToken) token := r.Header.Get("token") - response, err := handler.bulkUpdateService.BulkDeploy(&request, ctx, w, token, handler.checkAuthForBulkActions) + response, err := handler.bulkUpdateService.BulkDeploy(&request, ctx, w, token, handler.checkAuthBatch) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return @@ -489,3 +489,15 @@ func (handler BulkUpdateRestHandlerImpl) HandleCdPipelineBulkAction(w http.Respo } common.WriteJsonResp(w, nil, resp, http.StatusOK) } + +func (handler BulkUpdateRestHandlerImpl) checkAuthBatch(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool) { + var appResult map[string]bool + var envResult map[string]bool + if len(appObject) > 0 { + appResult = handler.enforcer.EnforceByEmailInBatch(emailId, casbin.ResourceApplications, casbin.ActionGet, appObject) + } + if len(envObject) > 0 { + envResult = handler.enforcer.EnforceByEmailInBatch(emailId, casbin.ResourceEnvironment, casbin.ActionGet, envObject) + } + return appResult, envResult +} diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index 4b291cc376..b5228044f0 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -47,7 +47,7 @@ type BulkUpdateService interface { BulkHibernate(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) BulkUnHibernate(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) - BulkDeploy(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) + BulkDeploy(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) (*BulkApplicationForEnvironmentResponse, error) BulkBuildTrigger(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) GetBulkActionImpactedPipelinesAndWfs(dto *CdBulkActionRequestDto) ([]*pipelineConfig.Pipeline, []int, []int, error) @@ -1145,7 +1145,7 @@ func (impl BulkUpdateServiceImpl) BulkUnHibernate(request *BulkApplicationForEnv bulkOperationResponse.Response = response return bulkOperationResponse, nil } -func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) { +func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, emailId string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) (*BulkApplicationForEnvironmentResponse, error) { var pipelines []*pipelineConfig.Pipeline var err error if len(request.AppIdIncludes) > 0 { @@ -1159,6 +1159,26 @@ func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironm impl.logger.Errorw("error in fetching pipelines", "envId", request.EnvId, "err", err) return nil, err } + + pipelineIds := make([]int, 0) + for _, pipeline := range pipelines { + pipelineIds = append(pipelineIds, pipeline.Id) + } + if len(pipelineIds) == 0 { + return nil, fmt.Errorf("no pipeline found for this environment") + } + //authorization block starts here + var appObjectArr []string + var envObjectArr []string + objects := impl.enforcerUtil.GetAppAndEnvObjectByPipelineIds(pipelineIds) + pipelineIds = []int{} + for _, object := range objects { + appObjectArr = append(appObjectArr, object[0]) + envObjectArr = append(envObjectArr, object[1]) + } + appResults, envResults := checkAuthBatch(emailId, appObjectArr, envObjectArr) + //authorization block ends here + response := make(map[string]map[string]bool) for _, pipeline := range pipelines { appKey := fmt.Sprintf("%d_%s", pipeline.AppId, pipeline.App.AppName) @@ -1169,11 +1189,10 @@ func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironm pResponse[pipelineKey] = false response[appKey] = pResponse } - appObject := impl.enforcerUtil.GetAppRBACNameByAppId(pipeline.AppId) - envObject := impl.enforcerUtil.GetEnvRBACNameByAppId(pipeline.AppId, pipeline.EnvironmentId) - isValidAuth := checkAuthForBulkActions(token, appObject, envObject) - if !isValidAuth { - //skip hibernate for the app if user does not have access on that + appObject := objects[pipeline.Id][0] + envObject := objects[pipeline.Id][1] + if !(appResults[appObject] && envResults[envObject]) { + //if user unauthorized, skip items pipelineResponse := response[appKey] pipelineResponse[pipelineKey] = false response[appKey] = pipelineResponse diff --git a/wire_gen.go b/wire_gen.go index c2cb48e436..decd190d7a 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,8 +1,7 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire -//go:build !wireinject -// +build !wireinject +//go:generate wire +//+build !wireinject package main From ee0bad147773f45518fe3dc64eec829ce5b72e3f Mon Sep 17 00:00:00 2001 From: Vishal Bihani Date: Wed, 5 Apr 2023 17:19:46 +0530 Subject: [PATCH 05/10] upgrade common-lib dependency --- go.mod | 2 +- go.sum | 2 ++ vendor/modules.txt | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e0dca699ca..85bc067b86 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/davecgh/go-spew v1.1.1 github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 - github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c + github.com/devtron-labs/common-lib v0.0.0-20230405114314-16f9d9a4b397 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-pg/pg v6.15.1+incompatible diff --git a/go.sum b/go.sum index a78ddc640e..5032c569f0 100644 --- a/go.sum +++ b/go.sum @@ -284,6 +284,8 @@ github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 h1:o github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c h1:Zjr8yvDyK/NTWcObcnPYzBverONH1108gwKAXyEZKH4= github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= +github.com/devtron-labs/common-lib v0.0.0-20230405114314-16f9d9a4b397 h1:cF9MaOIovkR9d94WJgCFwsZJwFkKkUeGGp9GNBZSa1k= +github.com/devtron-labs/common-lib v0.0.0-20230405114314-16f9d9a4b397/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= diff --git a/vendor/modules.txt b/vendor/modules.txt index aa0b23fc92..98e89eb766 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -327,7 +327,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c +# github.com/devtron-labs/common-lib v0.0.0-20230405114314-16f9d9a4b397 ## explicit; go 1.18 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/pubsub-lib From f6a15f100710503f34ae0ca46d6c58efc2ba2c90 Mon Sep 17 00:00:00 2001 From: vikramdevtron Date: Thu, 6 Apr 2023 16:37:24 +0530 Subject: [PATCH 06/10] fixed auth on bulk update --- api/restHandler/BulkUpdateRestHandler.go | 16 +++++++--------- pkg/bulkAction/BulkUpdateService.go | 4 ++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/api/restHandler/BulkUpdateRestHandler.go b/api/restHandler/BulkUpdateRestHandler.go index 52642115e5..cf52b3bfe8 100644 --- a/api/restHandler/BulkUpdateRestHandler.go +++ b/api/restHandler/BulkUpdateRestHandler.go @@ -332,6 +332,12 @@ func (handler BulkUpdateRestHandlerImpl) BulkDeploy(w http.ResponseWriter, r *ht common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } + user, err := handler.userAuthService.GetById(userId) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + userEmailId := strings.ToLower(user.EmailId) decoder := json.NewDecoder(r.Body) var request bulkAction.BulkApplicationForEnvironmentPayload err = decoder.Decode(&request) @@ -346,15 +352,7 @@ func (handler BulkUpdateRestHandlerImpl) BulkDeploy(w http.ResponseWriter, r *ht common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - acdToken, err := handler.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - handler.logger.Errorw("error in getting acd token", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - ctx := context.WithValue(r.Context(), "token", acdToken) - token := r.Header.Get("token") - response, err := handler.bulkUpdateService.BulkDeploy(&request, ctx, w, token, handler.checkAuthBatch) + response, err := handler.bulkUpdateService.BulkDeploy(&request, userEmailId, handler.checkAuthBatch) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index b5228044f0..f003506efd 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -47,7 +47,7 @@ type BulkUpdateService interface { BulkHibernate(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) BulkUnHibernate(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) - BulkDeploy(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) (*BulkApplicationForEnvironmentResponse, error) + BulkDeploy(request *BulkApplicationForEnvironmentPayload, emailId string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) (*BulkApplicationForEnvironmentResponse, error) BulkBuildTrigger(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) GetBulkActionImpactedPipelinesAndWfs(dto *CdBulkActionRequestDto) ([]*pipelineConfig.Pipeline, []int, []int, error) @@ -1145,7 +1145,7 @@ func (impl BulkUpdateServiceImpl) BulkUnHibernate(request *BulkApplicationForEnv bulkOperationResponse.Response = response return bulkOperationResponse, nil } -func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, emailId string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) (*BulkApplicationForEnvironmentResponse, error) { +func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironmentPayload, emailId string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) (*BulkApplicationForEnvironmentResponse, error) { var pipelines []*pipelineConfig.Pipeline var err error if len(request.AppIdIncludes) > 0 { From 2804655d329361916cfc4ff3f1fe1f0e49f6f788 Mon Sep 17 00:00:00 2001 From: Vishal Bihani Date: Fri, 7 Apr 2023 06:08:58 +0530 Subject: [PATCH 07/10] fix slow consumer error --- go.mod | 2 +- go.sum | 6 ++---- .../devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go | 1 + vendor/modules.txt | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 85bc067b86..f3972dabd0 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/davecgh/go-spew v1.1.1 github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 - github.com/devtron-labs/common-lib v0.0.0-20230405114314-16f9d9a4b397 + github.com/devtron-labs/common-lib v0.0.0-20230407002957-fc2f65dc518e github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-pg/pg v6.15.1+incompatible diff --git a/go.sum b/go.sum index 5032c569f0..040de33198 100644 --- a/go.sum +++ b/go.sum @@ -282,10 +282,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 h1:oHDpsCsuYiL+u5TEhilKNLGhH9lwRNkIqrzhHQA6d3c= github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c h1:Zjr8yvDyK/NTWcObcnPYzBverONH1108gwKAXyEZKH4= -github.com/devtron-labs/common-lib v0.0.0-20230404194633-b6b38b25594c/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20230405114314-16f9d9a4b397 h1:cF9MaOIovkR9d94WJgCFwsZJwFkKkUeGGp9GNBZSa1k= -github.com/devtron-labs/common-lib v0.0.0-20230405114314-16f9d9a4b397/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= +github.com/devtron-labs/common-lib v0.0.0-20230407002957-fc2f65dc518e h1:3TSKEN4RM61qDXh9Ri0AOdmt2D2lehWMzK/g8lldK2M= +github.com/devtron-labs/common-lib v0.0.0-20230407002957-fc2f65dc518e/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go index 3f748471b9..cddf055e2d 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go @@ -145,6 +145,7 @@ var NatsConsumerWiseConfigMapping = map[string]NatsConsumerConfig{ BULK_HIBERNATE_DURABLE: {}, BULK_DEPLOY_DURABLE: {}, BULK_APPSTORE_DEPLOY_DURABLE: {}, + CD_BULK_DEPLOY_TRIGGER_DURABLE: {}, } func getConsumerConfigMap(jsonString string) map[string]NatsConsumerConfig { diff --git a/vendor/modules.txt b/vendor/modules.txt index 98e89eb766..964202757b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -327,7 +327,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.0-20230405114314-16f9d9a4b397 +# github.com/devtron-labs/common-lib v0.0.0-20230407002957-fc2f65dc518e ## explicit; go 1.18 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/pubsub-lib From 94e30af13862acc9f220aaf16bae19a688fa4e24 Mon Sep 17 00:00:00 2001 From: Vishal Bihani Date: Fri, 7 Apr 2023 13:18:09 +0530 Subject: [PATCH 08/10] upgrade common-lib dependency --- go.mod | 2 +- go.sum | 4 ++-- vendor/modules.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index f3972dabd0..665203d499 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/davecgh/go-spew v1.1.1 github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 - github.com/devtron-labs/common-lib v0.0.0-20230407002957-fc2f65dc518e + github.com/devtron-labs/common-lib v0.0.0-20230407072229-d4f665f5ca12 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-pg/pg v6.15.1+incompatible diff --git a/go.sum b/go.sum index 040de33198..91e10a3db3 100644 --- a/go.sum +++ b/go.sum @@ -282,8 +282,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 h1:oHDpsCsuYiL+u5TEhilKNLGhH9lwRNkIqrzhHQA6d3c= github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.0-20230407002957-fc2f65dc518e h1:3TSKEN4RM61qDXh9Ri0AOdmt2D2lehWMzK/g8lldK2M= -github.com/devtron-labs/common-lib v0.0.0-20230407002957-fc2f65dc518e/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= +github.com/devtron-labs/common-lib v0.0.0-20230407072229-d4f665f5ca12 h1:tcwNgAWTzalItF5XT0WxicRvY4wzz/o0HYLMVLxsxjg= +github.com/devtron-labs/common-lib v0.0.0-20230407072229-d4f665f5ca12/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= diff --git a/vendor/modules.txt b/vendor/modules.txt index 964202757b..99d7543aee 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -327,7 +327,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.0-20230407002957-fc2f65dc518e +# github.com/devtron-labs/common-lib v0.0.0-20230407072229-d4f665f5ca12 ## explicit; go 1.18 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/pubsub-lib From 68750bb1bde81c726e462806ca66e2cac8ea3740 Mon Sep 17 00:00:00 2001 From: Vishal Bihani Date: Fri, 7 Apr 2023 14:27:57 +0530 Subject: [PATCH 09/10] fix created_by not null constraint violation --- api/bean/ValuesOverrideRequest.go | 5 +++++ pkg/bulkAction/BulkUpdateService.go | 17 +++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/api/bean/ValuesOverrideRequest.go b/api/bean/ValuesOverrideRequest.go index 8c6e70248a..1de7317f82 100644 --- a/api/bean/ValuesOverrideRequest.go +++ b/api/bean/ValuesOverrideRequest.go @@ -54,6 +54,11 @@ type ValuesOverrideRequest struct { DeploymentType models.DeploymentType `json:"-"` } +type BulkCdDeployEvent struct { + ValuesOverrideRequest *ValuesOverrideRequest `json:"valuesOverrideRequest"` + UserId int32 `json:"userId"` +} + type ReleaseStatusUpdateRequest struct { RequestId string `json:"requestId"` NewStatus models.ChartStatus `json:"newStatus"` diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index f003506efd..ca54c2424f 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -1224,11 +1224,15 @@ func (impl BulkUpdateServiceImpl) BulkDeploy(request *BulkApplicationForEnvironm UserId: request.UserId, CdWorkflowType: bean.CD_WORKFLOW_TYPE_DEPLOY, } + event := &bean.BulkCdDeployEvent{ + ValuesOverrideRequest: overrideRequest, + UserId: overrideRequest.UserId, + } - payload, err := json.Marshal(overrideRequest) + payload, err := json.Marshal(event) if err != nil { - impl.logger.Errorw("failed to marshal override request", - "request", overrideRequest, + impl.logger.Errorw("failed to marshal cd bulk deploy event request", + "request", event, "err", err) pipelineResponse := response[appKey] @@ -1267,8 +1271,8 @@ func (impl BulkUpdateServiceImpl) SubscribeToCdBulkTriggerTopic() error { "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, "msg", msg.Data) - overrideReq := &bean.ValuesOverrideRequest{} - err := json.Unmarshal([]byte(msg.Data), overrideReq) + event := &bean.BulkCdDeployEvent{} + err := json.Unmarshal([]byte(msg.Data), event) if err != nil { impl.logger.Errorw("Error unmarshalling received event", "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, @@ -1276,9 +1280,10 @@ func (impl BulkUpdateServiceImpl) SubscribeToCdBulkTriggerTopic() error { "err", err) return } + event.ValuesOverrideRequest.UserId = event.UserId // trigger - _, err = impl.workflowDagExecutor.ManualCdTrigger(overrideReq, context.Background()) + _, err = impl.workflowDagExecutor.ManualCdTrigger(event.ValuesOverrideRequest, context.Background()) if err != nil { impl.logger.Errorw("Error triggering CD", "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, From c2b1b7425eb32bb899d7214fec70681b97e9bdd2 Mon Sep 17 00:00:00 2001 From: Vishal Bihani Date: Fri, 7 Apr 2023 16:54:51 +0530 Subject: [PATCH 10/10] fix invalid json document error --- api/bean/ValuesOverrideRequest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/bean/ValuesOverrideRequest.go b/api/bean/ValuesOverrideRequest.go index 1de7317f82..1c723536c3 100644 --- a/api/bean/ValuesOverrideRequest.go +++ b/api/bean/ValuesOverrideRequest.go @@ -42,7 +42,7 @@ type ValuesOverrideRequest struct { PipelineId int `json:"pipelineId" validate:"required"` AppId int `json:"appId" validate:"required"` CiArtifactId int `json:"ciArtifactId" validate:"required"` - AdditionalOverride json.RawMessage `json:"additionalOverride"` + AdditionalOverride json.RawMessage `json:"additionalOverride,omitempty"` TargetDbVersion int `json:"targetDbVersion"` ForceTrigger bool `json:"forceTrigger,notnull"` DeploymentTemplate string `json:"strategy,omitempty"` // validate:"oneof=BLUE-GREEN ROLLING"`