feat: jira issue validator plugin added (#2968)

* first commit

* second commit

* jira issue validator plugin sql script

* sending env variables on cd

* sql refactoring

* icon for plugin-jira-validation added

* extra bracket removed from dtrack plugin name

* jiraId env variable changed

* icon deleted

* wire generated

* jira issue updater and validator plugin

* jira status fix

* plugin fix

* plugin metadata updated

* message change

* jira issue updater plugin updated

Author: manish-agrawal-ai

client/gitSensor/GitSensorClient.go

@@ -60,12 +60,13 @@ type HeadRequest struct {
 type SourceType string
 
 type CiPipelineMaterial struct {
-	Id            int
-	GitMaterialId int
-	Type          SourceType
-	Value         string
-	Active        bool
-	GitCommit     GitCommit
+	Id                        int
+	GitMaterialId             int
+	Type                      SourceType
+	Value                     string
+	Active                    bool
+	GitCommit                 GitCommit
+	ExtraEnvironmentVariables map[string]string // extra env variables which will be used for CI
 }
 
 type GitMaterial struct {
@@ -100,6 +101,11 @@ type GitCommit struct {
 	WebhookData *WebhookData
 }
 
+type WebhookAndCiData struct {
+	ExtraEnvironmentVariables map[string]string `json:"extraEnvironmentVariables"` // extra env variables which will be used for CI
+	WebhookData               *WebhookData      `json:"webhookData"`
+}
+
 type WebhookData struct {
 	Id              int               `json:"id"`
 	EventActionType string            `json:"eventActionType"`
@@ -133,7 +139,8 @@ type RefreshGitMaterialResponse struct {
 }
 
 type WebhookDataRequest struct {
-	Id int `json:"id"`
+	Id                   int `json:"id"`
+	CiPipelineMaterialId int `json:"ciPipelineMaterialId"`
 }
 
 type WebhookEventConfigRequest struct {
@@ -219,7 +226,7 @@ type GitSensorClient interface {
 	SavePipelineMaterial(material []*CiPipelineMaterial) (materialRes []*CiPipelineMaterial, err error)
 	RefreshGitMaterial(req *RefreshGitMaterialRequest) (refreshRes *RefreshGitMaterialResponse, err error)
 
-	GetWebhookData(req *WebhookDataRequest) (*WebhookData, error)
+	GetWebhookData(req *WebhookDataRequest) (*WebhookAndCiData, error)
 
 	GetAllWebhookEventConfigForHost(req *WebhookEventConfigRequest) (webhookEvents []*WebhookEventConfig, err error)
 	GetWebhookEventConfig(req *WebhookEventConfigRequest) (webhookEvent *WebhookEventConfig, err error)
@@ -376,8 +383,8 @@ func (session GitSensorClientImpl) RefreshGitMaterial(req *RefreshGitMaterialReq
 	return refreshRes, err
 }
 
-func (session GitSensorClientImpl) GetWebhookData(req *WebhookDataRequest) (*WebhookData, error) {
-	webhookData := new(WebhookData)
+func (session GitSensorClientImpl) GetWebhookData(req *WebhookDataRequest) (*WebhookAndCiData, error) {
+	webhookData := new(WebhookAndCiData)
 	request := &ClientRequest{ResponseBody: webhookData, Method: "GET", RequestBody: req, Path: "webhook/data"}
 	_, _, err := session.doRequest(request)
 	return webhookData, err

cmd/external-app/wire_gen.go

@@ -215,8 +215,9 @@ type CiRegexPatchRequest struct {
 }
 
 type GitCiTriggerRequest struct {
-	CiPipelineMaterial CiPipelineMaterial `json:"ciPipelineMaterial" validate:"required"`
-	TriggeredBy        int32              `json:"triggeredBy"`
+	CiPipelineMaterial        CiPipelineMaterial `json:"ciPipelineMaterial" validate:"required"`
+	TriggeredBy               int32              `json:"triggeredBy"`
+	ExtraEnvironmentVariables map[string]string  `json:"extraEnvironmentVariables"` // extra env variables which will be used for CI
 }
 
 type GitCommit struct {

pkg/bean/app.go

@@ -70,8 +70,9 @@ func (impl *GitWebhookServiceImpl) HandleGitWebhook(gitWebhookRequest gitSensor.
 	}
 
 	resp, err := impl.ciHandler.HandleCIWebhook(bean.GitCiTriggerRequest{
-		CiPipelineMaterial: ciPipelineMaterial,
-		TriggeredBy:        1, // Automatic trigger, userId is 1
+		CiPipelineMaterial:        ciPipelineMaterial,
+		TriggeredBy:               1, // Automatic trigger, userId is 1
+		ExtraEnvironmentVariables: gitWebhookRequest.ExtraEnvironmentVariables,
 	})
 	if err != nil {
 		impl.logger.Errorw("failed HandleCIWebhook", "err", err)

pkg/git/GitWebhookService.go

@@ -170,11 +170,12 @@ type GitTriggerInfoResponse struct {
 }
 
 type Trigger struct {
-	PipelineId      int
-	CommitHashes    map[int]bean.GitCommit
-	CiMaterials     []*pipelineConfig.CiPipelineMaterial
-	TriggeredBy     int32
-	InvalidateCache bool
+	PipelineId                int
+	CommitHashes              map[int]bean.GitCommit
+	CiMaterials               []*pipelineConfig.CiPipelineMaterial
+	TriggeredBy               int32
+	InvalidateCache           bool
+	ExtraEnvironmentVariables map[string]string // extra env variables which will be used for CI
 }
 
 const WorkflowCancel = "CANCELLED"
@@ -184,16 +185,17 @@ const Starting = "Starting"
 
 func (impl *CiHandlerImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) {
 	impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId)
-	commitHashes, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest)
+	commitHashes, extraEnvironmentVariables, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest)
 	if err != nil {
 		return 0, err
 	}
 	trigger := Trigger{
-		PipelineId:      ciTriggerRequest.PipelineId,
-		CommitHashes:    commitHashes,
-		CiMaterials:     nil,
-		TriggeredBy:     ciTriggerRequest.TriggeredBy,
-		InvalidateCache: ciTriggerRequest.InvalidateCache,
+		PipelineId:                ciTriggerRequest.PipelineId,
+		CommitHashes:              commitHashes,
+		CiMaterials:               nil,
+		TriggeredBy:               ciTriggerRequest.TriggeredBy,
+		InvalidateCache:           ciTriggerRequest.InvalidateCache,
+		ExtraEnvironmentVariables: extraEnvironmentVariables,
 	}
 	id, err := impl.ciService.TriggerCiPipeline(trigger)
 	if err != nil {
@@ -230,10 +232,11 @@ func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTrigger
 	}
 
 	trigger := Trigger{
-		PipelineId:   ciPipeline.Id,
-		CommitHashes: commitHashes,
-		CiMaterials:  ciMaterials,
-		TriggeredBy:  gitCiTriggerRequest.TriggeredBy,
+		PipelineId:                ciPipeline.Id,
+		CommitHashes:              commitHashes,
+		CiMaterials:               ciMaterials,
+		TriggeredBy:               gitCiTriggerRequest.TriggeredBy,
+		ExtraEnvironmentVariables: gitCiTriggerRequest.ExtraEnvironmentVariables,
 	}
 	id, err := impl.ciService.TriggerCiPipeline(trigger)
 	if err != nil {
@@ -868,37 +871,38 @@ func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipeli
 	return newGitCommit
 }
 
-func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]bean.GitCommit, error) {
+func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]bean.GitCommit, map[string]string, error) {
 	commitHashes := map[int]bean.GitCommit{}
+	extraEnvironmentVariables := make(map[string]string)
 	for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial {
 
 		pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id)
 		if err != nil {
 			impl.Logger.Errorw("err in fetching pipeline material by id", "err", err)
-			return map[int]bean.GitCommit{}, err
+			return map[int]bean.GitCommit{}, nil, err
 		}
 
 		pipelineType := pipeLineMaterialFromDb.Type
 		if pipelineType == pipelineConfig.SOURCE_TYPE_BRANCH_FIXED {
 			gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb)
 			if err != nil {
 				impl.Logger.Errorw("err", "err", err)
-				return map[int]bean.GitCommit{}, err
+				return map[int]bean.GitCommit{}, nil, err
 			}
 			commitHashes[ciPipelineMaterial.Id] = gitCommit
 
 		} else if pipelineType == pipelineConfig.SOURCE_TYPE_WEBHOOK {
-			gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb)
+			gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb)
 			if err != nil {
 				impl.Logger.Errorw("err", "err", err)
-				return map[int]bean.GitCommit{}, err
+				return map[int]bean.GitCommit{}, nil, err
 			}
 			commitHashes[ciPipelineMaterial.Id] = gitCommit
-
+			extraEnvironmentVariables = extraEnvVariables
 		}
 
 	}
-	return commitHashes, nil
+	return commitHashes, extraEnvironmentVariables, nil
 }
 
 func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (bean.GitCommit, error) {
@@ -931,19 +935,21 @@ func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(
 	return gitCommit, nil
 }
 
-func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (bean.GitCommit, error) {
+func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (bean.GitCommit, map[string]string, error) {
 	webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData
 
 	// fetch webhook data on the basis of Id
 	webhookDataRequest := &gitSensor.WebhookDataRequest{
-		Id: webhookDataInput.Id,
+		Id:                   webhookDataInput.Id,
+		CiPipelineMaterialId: ciPipelineMaterial.Id,
 	}
 
-	webhookData, err := impl.gitSensorClient.GetWebhookData(webhookDataRequest)
+	webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(webhookDataRequest)
 	if err != nil {
 		impl.Logger.Errorw("err", "err", err)
-		return bean.GitCommit{}, err
+		return bean.GitCommit{}, nil, err
 	}
+	webhookData := webhookAndCiData.WebhookData
 
 	// if webhook event is of merged type, then fetch latest commit for target branch
 	if webhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE {
@@ -952,7 +958,7 @@ func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ci
 		targetBranchName := webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_BRANCH_NAME_NAME]
 		if targetBranchName == "" {
 			impl.Logger.Error("target branch not found from webhook data")
-			return bean.GitCommit{}, err
+			return bean.GitCommit{}, nil, err
 		}
 
 		// get latest commit hash for target branch
@@ -965,7 +971,7 @@ func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ci
 
 		if err != nil {
 			impl.Logger.Errorw("err", "err", err)
-			return bean.GitCommit{}, err
+			return bean.GitCommit{}, nil, err
 		}
 
 		// update webhookData (local) with target latest hash
@@ -986,7 +992,7 @@ func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ci
 		},
 	}
 
-	return gitCommit, nil
+	return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil
 }
 
 func (impl *CiHandlerImpl) getLastSeenCommit(ciMaterialId int) (bean.GitCommit, error) {

pkg/pipeline/CiHandler.go

@@ -482,6 +482,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.
 		IgnoreDockerCachePush:      impl.ciConfig.IgnoreDockerCacheForCI,
 		IgnoreDockerCachePull:      impl.ciConfig.IgnoreDockerCacheForCI,
 		CacheInvalidate:            trigger.InvalidateCache,
+		ExtraEnvironmentVariables:  trigger.ExtraEnvironmentVariables,
 	}
 
 	if ciWorkflowConfig.LogsBucket == "" {

pkg/pipeline/CiService.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"github.com/argoproj/gitops-engine/pkg/health"
 	blob_storage "github.com/devtron-labs/common-lib/blob-storage"
+	"github.com/devtron-labs/devtron/client/gitSensor"
 	"github.com/devtron-labs/devtron/util/argo"
 	"go.opentelemetry.io/otel"
 	"strconv"
@@ -101,6 +102,7 @@ type WorkflowDagExecutorImpl struct {
 	CiTemplateRepository          pipelineConfig.CiTemplateRepository
 	ciWorkflowRepository          pipelineConfig.CiWorkflowRepository
 	appLabelRepository            pipelineConfig.AppLabelRepository
+	gitSensorClient               gitSensor.GitSensorClient
 }
 
 const (
@@ -165,7 +167,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi
 	pipelineStatusTimelineService app.PipelineStatusTimelineService,
 	CiTemplateRepository pipelineConfig.CiTemplateRepository,
 	ciWorkflowRepository pipelineConfig.CiWorkflowRepository,
-	appLabelRepository pipelineConfig.AppLabelRepository) *WorkflowDagExecutorImpl {
+	appLabelRepository pipelineConfig.AppLabelRepository, gitSensorClient gitSensor.GitSensorClient) *WorkflowDagExecutorImpl {
 	wde := &WorkflowDagExecutorImpl{logger: Logger,
 		pipelineRepository:            pipelineRepository,
 		cdWorkflowRepository:          cdWorkflowRepository,
@@ -196,6 +198,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi
 		CiTemplateRepository:          CiTemplateRepository,
 		ciWorkflowRepository:          ciWorkflowRepository,
 		appLabelRepository:            appLabelRepository,
+		gitSensorClient:               gitSensorClient,
 	}
 	err := wde.Subscribe()
 	if err != nil {
@@ -738,10 +741,33 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor
 
 	if ciWf != nil && ciWf.GitTriggers != nil {
 		i := 1
-		for _, gitTrigger := range ciWf.GitTriggers {
+		for ciPipelineMaterialId, gitTrigger := range ciWf.GitTriggers {
 			extraEnvVariables[fmt.Sprintf("%s_%d", GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit
 			extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType)
 			extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue
+
+			// CODE-BLOCK starts - store extra environment variables if webhook
+			if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK {
+				webhookDataId := gitTrigger.WebhookData.Id
+				if webhookDataId > 0 {
+					webhookDataRequest := &gitSensor.WebhookDataRequest{
+						Id:                   webhookDataId,
+						CiPipelineMaterialId: ciPipelineMaterialId,
+					}
+					webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(webhookDataRequest)
+					if err != nil {
+						impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest)
+						return nil, err
+					}
+					if webhookAndCiData != nil {
+						for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables {
+							extraEnvVariables[extEnvVariableKey] = extEnvVariableVal
+						}
+					}
+				}
+			}
+			// CODE_BLOCK ends
+
 			i++
 		}
 		extraEnvVariables[GIT_SOURCE_COUNT] = strconv.Itoa(len(ciWf.GitTriggers))

pkg/pipeline/WorkflowDagExecutor.go

@@ -116,6 +116,7 @@ type WorkflowRequest struct {
 	IgnoreDockerCachePull      bool                              `json:"ignoreDockerCachePull"`
 	CacheInvalidate            bool                              `json:"cacheInvalidate"`
 	IsPvcMounted               bool                              `json:"IsPvcMounted"`
+	ExtraEnvironmentVariables  map[string]string                 `json:"extraEnvironmentVariables"`
 }
 
 const (

pkg/pipeline/WorkflowService.go

@@ -0,0 +1,4 @@
+DELETE FROM plugin_step_variable WHERE name = 'JiraUsername';
+DELETE FROM plugin_step_variable WHERE name = 'JiraPassword';
+DELETE FROM plugin_step_variable WHERE name = 'JiraBaseUrl';
+DELETE FROM plugin_step_variable WHERE name = 'JiraId';