feat: Audit ci trigger, precd and post cd trigger so that retrigger can happen from last failed config snapshot

env_gen.md

@@ -290,10 +290,10 @@
  | DASHBOARD_PORT | string |3000 | Port for dashboard micro-service |  | false |
  | DEX_HOST | string |http://localhost |  |  | false |
  | DEX_PORT | string |5556 |  |  | false |
- | GIT_SENSOR_PROTOCOL | string |REST | Protocol to connect with git-sensor micro-service |  | false |
+ | GIT_SENSOR_PROTOCOL | string |GRPC | Protocol to connect with git-sensor micro-service |  | false |
  | GIT_SENSOR_SERVICE_CONFIG | string |{"loadBalancingPolicy":"pick_first"} | git-sensor grpc service config |  | false |
  | GIT_SENSOR_TIMEOUT | int |0 | Timeout for getting response from the git-sensor |  | false |
- | GIT_SENSOR_URL | string |127.0.0.1:7070 | git-sensor micro-service url  |  | false |
+ | GIT_SENSOR_URL | string |127.0.0.1:7071 | git-sensor micro-service url  |  | false |
  | HELM_CLIENT_URL | string |127.0.0.1:50051 | Kubelink micro-service url  |  | false |
  | KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE | int |20 |  |  | false |
  | KUBELINK_GRPC_MAX_SEND_MSG_SIZE | int |4 |  |  | false |

pkg/build/trigger/HandlerService_ent.go

@@ -99,6 +99,6 @@ func (impl *HandlerServiceImpl) updateCIBuildConfig(ciBuildConfigBean *bean2.CiB
 	return ciBuildConfigBean
 }
 
-func updateBuildPrePostStepDataReq(req *pipelineConfigBean.BuildPrePostStepDataRequest, trigger types.Trigger) *pipelineConfigBean.BuildPrePostStepDataRequest {
+func updateBuildPrePostStepDataReq(req *pipelineConfigBean.BuildPrePostStepDataRequest, trigger *types.CiTriggerRequest) *pipelineConfigBean.BuildPrePostStepDataRequest {
 	return req
 }

pkg/build/trigger/adaptor/adaptor.go

@@ -0,0 +1,23 @@
+package adaptor
+
+import (
+	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
+	"time"
+)
+
+func GetCiWorkflowFromRefCiWorkflow(refCiWorkflow *pipelineConfig.CiWorkflow, workflowStatus string, triggeredBy int32) *pipelineConfig.CiWorkflow {
+	return &pipelineConfig.CiWorkflow{
+		Name:                  refCiWorkflow.Name,
+		Status:                workflowStatus, // starting CIStage
+		StartedOn:             time.Now(),
+		CiPipelineId:          refCiWorkflow.CiPipelineId,
+		Namespace:             refCiWorkflow.Namespace,
+		BlobStorageEnabled:    refCiWorkflow.BlobStorageEnabled,
+		GitTriggers:           refCiWorkflow.GitTriggers,
+		CiBuildType:           refCiWorkflow.CiBuildType,
+		TriggeredBy:           triggeredBy,
+		ReferenceCiWorkflowId: refCiWorkflow.Id, // Reference to original workflow
+		ExecutorType:          refCiWorkflow.ExecutorType,
+		EnvironmentId:         refCiWorkflow.EnvironmentId,
+	}
+}

pkg/deployment/trigger/devtronApps/HandlerService.go

@@ -20,6 +20,7 @@ import (
 	"bufio"
 	"context"
 	"github.com/devtron-labs/common-lib/async"
+	service2 "github.com/devtron-labs/devtron/pkg/workflow/trigger/audit/service"
 	"os"
 	"time"
 
@@ -90,15 +91,15 @@ prePostWfAndLogsCode.go - code containing pre/post wf handling(abort) and logs r
 */
 
 type HandlerService interface {
-	TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error)
-	TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error)
+	TriggerPostStage(request bean.CdTriggerRequest) (*bean4.ManifestPushTemplate, error)
+	TriggerPreStage(request bean.CdTriggerRequest) (*bean4.ManifestPushTemplate, error)
 
 	TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error
 
-	TriggerStageForBulk(triggerRequest bean.TriggerRequest) error
+	TriggerStageForBulk(triggerRequest bean.CdTriggerRequest) error
 
 	ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest, userMetadata *userBean.UserMetadata) (int, string, *bean4.ManifestPushTemplate, error)
-	TriggerAutomaticDeployment(request bean.TriggerRequest) error
+	TriggerAutomaticDeployment(request bean.CdTriggerRequest) error
 
 	TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error)
 
@@ -170,6 +171,7 @@ type HandlerServiceImpl struct {
 	blobConfigStorageService            pipeline.BlobStorageConfigService
 	deploymentEventHandler              app.DeploymentEventHandler
 	asyncRunnable                       *async.Runnable
+	workflowTriggerAuditService         service2.WorkflowTriggerAuditService
 }
 
 func NewHandlerServiceImpl(logger *zap.SugaredLogger,
@@ -232,7 +234,9 @@ func NewHandlerServiceImpl(logger *zap.SugaredLogger,
 	workflowService executor.WorkflowService,
 	blobConfigStorageService pipeline.BlobStorageConfigService,
 	deploymentEventHandler app.DeploymentEventHandler,
-	asyncRunnable *async.Runnable) (*HandlerServiceImpl, error) {
+	asyncRunnable *async.Runnable,
+	workflowTriggerAuditService service2.WorkflowTriggerAuditService,
+) (*HandlerServiceImpl, error) {
 	impl := &HandlerServiceImpl{
 		logger:                              logger,
 		cdWorkflowCommonService:             cdWorkflowCommonService,
@@ -293,13 +297,14 @@ func NewHandlerServiceImpl(logger *zap.SugaredLogger,
 		attributeService:         attributeService,
 		cdWorkflowRunnerService:  cdWorkflowRunnerService,
 
-		clusterRepository:        clusterRepository,
-		clusterService:           clusterService,
-		ciLogService:             ciLogService,
-		workflowService:          workflowService,
-		blobConfigStorageService: blobConfigStorageService,
-		deploymentEventHandler:   deploymentEventHandler,
-		asyncRunnable:            asyncRunnable,
+		clusterRepository:           clusterRepository,
+		clusterService:              clusterService,
+		ciLogService:                ciLogService,
+		workflowService:             workflowService,
+		blobConfigStorageService:    blobConfigStorageService,
+		deploymentEventHandler:      deploymentEventHandler,
+		asyncRunnable:               asyncRunnable,
+		workflowTriggerAuditService: workflowTriggerAuditService,
 	}
 	config, err := types.GetCdConfig()
 	if err != nil {

pkg/deployment/trigger/devtronApps/bean/bean.go

@@ -40,7 +40,7 @@ type TriggerEvent struct {
 	TriggeredAt                time.Time
 }
 
-type TriggerRequest struct {
+type CdTriggerRequest struct {
 	CdWf                   *pipelineConfig.CdWorkflow
 	Pipeline               *pipelineConfig.Pipeline
 	Artifact               *repository.CiArtifact
@@ -51,6 +51,8 @@ type TriggerRequest struct {
 	WorkflowType           bean.WorkflowType
 	CdWorkflowRunnerId     int
 	TriggerContext
+	// below fields used for retrigger flow
+	IsRetrigger bool
 }
 
 type TriggerContext struct {
@@ -82,7 +84,7 @@ const (
 )
 
 type TriggerRequirementRequestDto struct {
-	TriggerRequest TriggerRequest
+	TriggerRequest CdTriggerRequest
 }
 
 type VulnerabilityCheckRequest struct {

pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go

@@ -67,7 +67,7 @@ import (
 	"helm.sh/helm/v3/pkg/chart"
 )
 
-func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error {
+func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.CdTriggerRequest) error {
 
 	preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false)
 	if err != nil && err != pg.ErrNoRows {
@@ -228,7 +228,7 @@ func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte
 		overrideRequest.CdWorkflowId = cdWf.Id
 
 		_, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPreStage")
-		triggerRequest := bean.TriggerRequest{
+		triggerRequest := bean.CdTriggerRequest{
 			CdWf:                  cdWf,
 			Artifact:              artifact,
 			Pipeline:              cdPipeline,
@@ -357,7 +357,7 @@ func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte
 			}
 		}
 		_, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPostStage")
-		triggerRequest := bean.TriggerRequest{
+		triggerRequest := bean.CdTriggerRequest{
 			CdWf:                  cdWf,
 			Pipeline:              cdPipeline,
 			TriggeredBy:           overrideRequest.UserId,
@@ -383,7 +383,7 @@ func isNotHibernateRequest(deploymentType models.DeploymentType) bool {
 }
 
 // TODO: write a wrapper to handle auto and manual trigger
-func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error {
+func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.CdTriggerRequest) error {
 	// in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here
 	triggeredBy := request.TriggeredBy
 	pipeline := request.Pipeline

pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go

@@ -63,7 +63,7 @@ func (impl *HandlerServiceImpl) getManifestPushService(storageType string) publi
 	return manifestPushService
 }
 
-func (impl *HandlerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error {
+func (impl *HandlerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.CdTriggerRequest) error {
 	return nil
 }
 

pkg/deployment/trigger/devtronApps/postStageHandlerCode.go

@@ -23,19 +23,19 @@ import (
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow"
 	bean4 "github.com/devtron-labs/devtron/pkg/app/bean"
 	"github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean"
-	bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean"
 	repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository"
 	"github.com/devtron-labs/devtron/pkg/pipeline/types"
 	util2 "github.com/devtron-labs/devtron/util/event"
 	"time"
 )
 
-func (impl *HandlerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) {
+func (impl *HandlerServiceImpl) TriggerPostStage(request bean.CdTriggerRequest) (*bean4.ManifestPushTemplate, error) {
 	request.WorkflowType = bean2.CD_WORKFLOW_TYPE_POST
 	// setting triggeredAt variable to have consistent data for various audit log places in db for deployment time
 	triggeredAt := time.Now()
 	triggeredBy := request.TriggeredBy
 	pipeline := request.Pipeline
+	artifact := request.Artifact
 	cdWf := request.CdWf
 	ctx := context.Background() //before there was only one context. To check why here we are not using ctx from request.TriggerContext
 	env, namespace, err := impl.getEnvAndNsIfRunStageInEnv(ctx, request)
@@ -104,25 +104,24 @@ func (impl *HandlerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*
 		impl.logger.Errorw("error, checkVulnerabilityStatusAndFailWfIfNeeded", "err", err, "runner", runner)
 		return nil, err
 	}
-	cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, envDevploymentConfig, triggeredBy)
-	if err != nil {
-		return impl.buildWfRequestErrorHandler(runner, err, triggeredBy)
-	}
-	cdStageWorkflowRequest.StageType = types.POST
-	cdStageWorkflowRequest.Pipeline = pipeline
-	cdStageWorkflowRequest.Env = env
-	cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE
-	// handling plugin specific logic
 
-	pluginImagePathReservationIds, err := impl.setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest, pipeline.Id, types.POST, cdWf.CiArtifact)
-	if err != nil {
-		runner.Status = cdWorkflow.WorkflowFailed
-		runner.Message = err.Error()
-		runner.FinishedOn = time.Now()
-		_ = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(runner)
-		return nil, err
+	var cdStageWorkflowRequest *types.WorkflowRequest
+	if request.IsRetrigger {
+		// Retrieve workflow request from snapshot
+		workflowRequest, err := impl.workflowTriggerAuditService.GetWorkflowRequestFromSnapshotForRetrigger(runner.Id, types.POST_CD_WORKFLOW_TYPE)
+		if err != nil {
+			impl.logger.Errorw("error retrieving workflow request from snapshot for post cd workflow type", "cdWorkflowId", runner.CdWorkflow.Id, "err", err)
+			return nil, err
+		}
+		cdStageWorkflowRequest = workflowRequest
+		// Update dynamic fields in the workflow request for retrigger
+		impl.updateWorkflowRequestForCdRetrigger(cdStageWorkflowRequest, runner)
+	} else {
+		cdStageWorkflowRequest, err = impl.getPrePostCdStageWorkflowRequest(ctx, runner, cdWf, pipeline, env, artifact, types.POST, envDevploymentConfig, triggeredBy)
+		if err != nil {
+			return impl.buildWfRequestErrorHandler(runner, err, triggeredBy)
+		}
 	}
-
 	_, jobHelmPackagePath, err := impl.workflowService.SubmitWorkflow(cdStageWorkflowRequest)
 	if err != nil {
 		impl.logger.Errorw("error in submitting workflow", "err", err, "workflowId", cdStageWorkflowRequest.WorkflowId, "pipeline", pipeline, "env", env)
@@ -137,20 +136,9 @@ func (impl *HandlerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*
 		impl.logger.Errorw("error in getting manifest push template", "err", err)
 		return nil, err
 	}
-	wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), cdWf.Id, bean2.CD_WORKFLOW_TYPE_POST)
-	if err != nil {
-		impl.logger.Errorw("error in getting wfr by workflowId and runnerType", "err", err, "wfId", cdWf.Id)
-		return nil, err
-	}
-	wfr.ImagePathReservationIds = pluginImagePathReservationIds
-	err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(&wfr)
-	if err != nil {
-		impl.logger.Error("error in updating image path reservation ids in cd workflow runner", "err", "err")
-	}
-
 	event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, &pipeline.EnvironmentId, util2.CD)
-	impl.logger.Debugw("event Cd Post Trigger", "event", event)
-	event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, bean2.CD_WORKFLOW_TYPE_POST)
+	impl.logger.Debugw("event Cd Post CiTriggerRequest", "event", event)
+	event = impl.eventFactory.BuildExtraCDData(event, runner, 0, bean2.CD_WORKFLOW_TYPE_POST)
 	_, evtErr := impl.eventClient.WriteNotificationEvent(event)
 	if evtErr != nil {
 		impl.logger.Errorw("CD trigger event not sent", "error", evtErr)

pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go

@@ -26,13 +26,13 @@ import (
 	"time"
 )
 
-func (impl *HandlerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest,
+func (impl *HandlerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.CdTriggerRequest,
 	env *repository.Environment, cdWf *pipelineConfig.CdWorkflow, triggeredBy int32) (interface{}, error) {
 	//here return type is interface as ResourceFilterEvaluationAudit is not present in this version
 	return nil, nil
 }
 
-func (impl *HandlerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig,
+func (impl *HandlerServiceImpl) getManifestPushTemplateForPostStage(request bean.CdTriggerRequest, envDevploymentConfig *bean5.DeploymentConfig,
 	jobHelmPackagePath string, cdStageWorkflowRequest *types.WorkflowRequest, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner,
 	pipeline *pipelineConfig.Pipeline, triggeredBy int32, triggeredAt time.Time) (*bean4.ManifestPushTemplate, error) {
 	return nil, nil