diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go
index d64fd0cfa7..671a295207 100644
--- a/api/router/pubsub/CiEventHandler.go
+++ b/api/router/pubsub/CiEventHandler.go
@@ -65,6 +65,7 @@ type CiCompleteEvent struct {
Metrics util.CIMetrics `json:"metrics"`
AppName string `json:"appName"`
IsArtifactUploaded bool `json:"isArtifactUploaded"`
+ FailureReason string `json:"failureReason"`
}
func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, ciEventConfig *CiEventConfig) *CiEventHandlerImpl {
@@ -92,19 +93,31 @@ func (impl *CiEventHandlerImpl) Subscribe() error {
impl.logger.Error("error while unmarshalling json data", "error", err)
return
}
- util.TriggerCIMetrics(ciCompleteEvent.Metrics, impl.ciEventConfig.ExposeCiMetrics, ciCompleteEvent.PipelineName, ciCompleteEvent.AppName)
impl.logger.Debugw("ci complete event for ci", "ciPipelineId", ciCompleteEvent.PipelineId)
req, err := impl.BuildCiArtifactRequest(ciCompleteEvent)
if err != nil {
return
}
- resp, err := impl.webhookService.HandleCiSuccessEvent(ciCompleteEvent.PipelineId, req)
- if err != nil {
- impl.logger.Error(err)
- return
+ if ciCompleteEvent.FailureReason != "" {
+ req.FailureReason = ciCompleteEvent.FailureReason
+ err := impl.webhookService.HandleCiStepFailedEvent(ciCompleteEvent.PipelineId, req)
+ if err != nil {
+ impl.logger.Error("Error while sending event for CI failure for pipelineID: ",
+ ciCompleteEvent.PipelineId, "request: ", req, "error: ", err)
+ return
+ }
+ } else {
+ util.TriggerCIMetrics(ciCompleteEvent.Metrics, impl.ciEventConfig.ExposeCiMetrics, ciCompleteEvent.PipelineName, ciCompleteEvent.AppName)
+
+ resp, err := impl.webhookService.HandleCiSuccessEvent(ciCompleteEvent.PipelineId, req)
+ if err != nil {
+ impl.logger.Error("Error while sending event for CI success for pipelineID: ",
+ ciCompleteEvent.PipelineId, "request: ", req, "error: ", err)
+ return
+ }
+ impl.logger.Debug(resp)
}
- impl.logger.Debug(resp)
}
err := impl.pubsubClient.Subscribe(pubsub.CI_COMPLETE_TOPIC, callback)
if err != nil {
diff --git a/client/events/EventClient.go b/client/events/EventClient.go
index e396a0d42e..9fa5e73591 100644
--- a/client/events/EventClient.go
+++ b/client/events/EventClient.go
@@ -89,6 +89,7 @@ type Payload struct {
DownloadLink string `json:"downloadLink"`
BuildHistoryLink string `json:"buildHistoryLink"`
MaterialTriggerInfo *MaterialTriggerInfo `json:"material"`
+ FailureReason string `json:"failureReason"`
}
type CiPipelineMaterialResponse struct {
diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go
index fb1c68857e..bfe053c206 100644
--- a/pkg/pipeline/CiHandler.go
+++ b/pkg/pipeline/CiHandler.go
@@ -34,6 +34,7 @@ import (
"net/http"
"os"
"path/filepath"
+ "regexp"
"strconv"
"strings"
"time"
@@ -762,6 +763,8 @@ func (impl *CiHandlerImpl) extractWorkfowStatus(workflowStatus v1alpha1.Workflow
return workflowName, status, podStatus, message, logLocation, podName
}
+const CiStageFailErrorCode = 2
+
func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) {
workflowName, status, podStatus, message, logLocation, podName := impl.extractWorkfowStatus(workflowStatus)
if workflowName == "" {
@@ -812,7 +815,12 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus
}
if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status {
impl.Logger.Warnw("ci failed for workflow: ", "wfId", savedWorkflow.Id)
- go impl.WriteCIFailEvent(savedWorkflow, ciWorkflowConfig.CiImage)
+
+ if extractErrorCode(savedWorkflow.Message) != CiStageFailErrorCode {
+ go impl.WriteCIFailEvent(savedWorkflow, ciWorkflowConfig.CiImage)
+ } else {
+ impl.Logger.Infof("Step failed notification received for wfID %d with message %s", savedWorkflow.Id, savedWorkflow.Message)
+ }
impl.WriteToCreateTestSuites(savedWorkflow.CiPipelineId, workflowId, int(savedWorkflow.TriggeredBy))
}
@@ -820,14 +828,26 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus
return savedWorkflow.Id, nil
}
+func extractErrorCode(msg string) int {
+ re := regexp.MustCompile(`\d+`)
+ matches := re.FindAllString(msg, -1)
+ if len(matches) > 0 {
+ code, err := strconv.Atoi(matches[0])
+ if err == nil {
+ return code
+ }
+ }
+ return -1
+}
+
func (impl *CiHandlerImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow, ciImage string) {
event := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI)
material := &client.MaterialTriggerInfo{}
material.GitTriggers = ciWorkflow.GitTriggers
event.CiWorkflowRunnerId = ciWorkflow.Id
+ event.UserId = int(ciWorkflow.TriggeredBy)
event = impl.eventFactory.BuildExtraCIData(event, material, ciImage)
event.CiArtifactId = 0
- event.UserId = int(ciWorkflow.TriggeredBy)
_, evtErr := impl.eventClient.WriteNotificationEvent(event)
if evtErr != nil {
impl.Logger.Errorw("error in writing event", "err", evtErr)
diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go
index cd918fc49d..0fb2c3ffe7 100644
--- a/pkg/pipeline/WebhookService.go
+++ b/pkg/pipeline/WebhookService.go
@@ -46,12 +46,14 @@ type CiArtifactWebhookRequest struct {
WorkflowId *int `json:"workflowId"`
UserId int32 `json:"userId"`
IsArtifactUploaded bool `json:"isArtifactUploaded"`
+ FailureReason string `json:"failureReason"`
}
type WebhookService interface {
AuthenticateExternalCiWebhook(apiKey string) (int, error)
HandleCiSuccessEvent(ciPipelineId int, request *CiArtifactWebhookRequest) (id int, err error)
HandleExternalCiWebhook(externalCiId int, request *CiArtifactWebhookRequest, auth func(token string, projectObject string, envObject string) bool) (id int, err error)
+ HandleCiStepFailedEvent(ciPipelineId int, request *CiArtifactWebhookRequest) (err error)
}
type WebhookServiceImpl struct {
@@ -115,6 +117,24 @@ func (impl WebhookServiceImpl) AuthenticateExternalCiWebhook(apiKey string) (int
return id, nil
}
+func (impl WebhookServiceImpl) HandleCiStepFailedEvent(ciPipelineId int, request *CiArtifactWebhookRequest) (err error) {
+
+ savedWorkflow, err := impl.ciWorkflowRepository.FindById(*request.WorkflowId)
+ if err != nil {
+ impl.logger.Errorw("cannot get saved wf", "wf ID: ", *request.WorkflowId, "err", err)
+ return err
+ }
+
+ pipeline, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId)
+ if err != nil {
+ impl.logger.Errorw("unable to find pipeline", "ID", ciPipelineId, "err", err)
+ return err
+ }
+
+ go impl.WriteCIStepFailedEvent(pipeline, request, savedWorkflow)
+ return nil
+}
+
func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *CiArtifactWebhookRequest) (id int, err error) {
impl.logger.Infow("webhook for artifact save", "req", request)
if request.WorkflowId != nil {
@@ -210,6 +230,7 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C
}
ciArtifactArr = append(ciArtifactArr, artifact)
go impl.WriteCISuccessEvent(request, pipeline, artifact)
+
isCiManual := true
if request.UserId == 1 {
impl.logger.Debugw("Trigger (auto) by system user", "userId", request.UserId)
@@ -286,6 +307,21 @@ func (impl WebhookServiceImpl) HandleExternalCiWebhook(externalCiId int, request
return artifact.Id, err
}
+func (impl *WebhookServiceImpl) WriteCIStepFailedEvent(pipeline *pipelineConfig.CiPipeline, request *CiArtifactWebhookRequest, ciWorkflow *pipelineConfig.CiWorkflow) {
+ event := impl.eventFactory.Build(util.Fail, &pipeline.Id, pipeline.AppId, nil, util.CI)
+ material := &client.MaterialTriggerInfo{}
+ material.GitTriggers = ciWorkflow.GitTriggers
+ event.CiWorkflowRunnerId = ciWorkflow.Id
+ event.UserId = int(ciWorkflow.TriggeredBy)
+ event = impl.eventFactory.BuildExtraCIData(event, material, request.Image)
+ event.CiArtifactId = 0
+ event.Payload.FailureReason = request.FailureReason
+ _, evtErr := impl.eventClient.WriteNotificationEvent(event)
+ if evtErr != nil {
+ impl.logger.Errorw("error in writing event: ", event, "error: ", evtErr)
+ }
+}
+
func (impl *WebhookServiceImpl) WriteCISuccessEvent(request *CiArtifactWebhookRequest, pipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact) {
event := impl.eventFactory.Build(util.Success, &pipeline.Id, pipeline.AppId, nil, util.CI)
event.CiArtifactId = artifact.Id
diff --git a/scripts/sql/141_update_ci_notification_template.down.sql b/scripts/sql/141_update_ci_notification_template.down.sql
new file mode 100644
index 0000000000..173d6d0b11
--- /dev/null
+++ b/scripts/sql/141_update_ci_notification_template.down.sql
@@ -0,0 +1,139 @@
+---- revert notification template for CI fail ses/smtp
+UPDATE notification_templates
+set template_payload = '{"from": "{{fromEmail}}",
+ "to": "{{toEmail}}",
+ "subject": "CI failed for app: {{appName}}",
+ "html": "Build Pipeline Failed
{{eventTime}}
Triggered by {{triggeredBy}}
View Pipeline
Application: {{appName}} | Pipeline: {{pipelineName}}
Source Code
{{#ciMaterials}}{{^webhookType}}Branch: {{appName}}/{{branch}} | Commit: {{commit}}
{{/webhookType}}{{#webhookType}}{{#webhookData.mergedType}}Title: {{webhookData.data.title}} | Git URL: View
Source Branch: {{webhookData.data.sourcebranchname}} | Source Commit: {{webhookData.data.sourcecheckout}}
Target Branch: {{webhookData.data.targetbranchname}} | Target Commit: {{webhookData.data.targetcheckout}}
{{/webhookData.mergedType}}{{^webhookData.mergedType}}Target Checkout: {{webhookData.data.targetcheckout}}
{{/webhookData.mergedType}}{{/webhookType}}{{/ciMaterials}}
"}'
+where channel_type = 'ses'
+ and node_type = 'CI'
+ and event_type_id = 3;
+
+
+---- revert notification template for CI fail slack
+UPDATE notification_templates
+set template_payload = '{
+ "text": ":x: Build pipeline Failed | {{#ciMaterials}} Branch > {{branch}} {{/ciMaterials}} | Application > {{appName}}",
+ "blocks": [{
+ "type": "section",
+ "text": {
+ "type": "mrkdwn",
+ "text": "\n"
+ }
+ },
+ {
+ "type": "divider"
+ },
+ {
+ "type": "section",
+ "text": {
+ "type": "mrkdwn",
+ "text": ":x: *Build Pipeline failed*\n \n Triggered by {{triggeredBy}}"
+ },
+ "accessory": {
+ "type": "image",
+ "image_url": "https://github.com/devtron-labs/notifier/assets/image/img_build_notification.png",
+ "alt_text": "calendar thumbnail"
+ }
+ },
+ {
+ "type": "section",
+ "fields": [{
+ "type": "mrkdwn",
+ "text": "*Application*\n{{appName}}"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Pipeline*\n{{pipelineName}}"
+ }
+ ]
+ },
+ {{#ciMaterials}}
+ {{^webhookType}}
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Branch*\n`{{appName}}/{{branch}}`"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Commit*\n<{{& commitLink}}|{{commit}}>"
+ }
+ ]
+ },
+ {{/webhookType}}
+ {{#webhookType}}
+ {{#webhookData.mergedType}}
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Title*\n{{webhookData.data.title}}"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Git URL*\n<{{& webhookData.data.giturl}}|View>"
+ }
+ ]
+ },
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Source Branch*\n{{webhookData.data.sourcebranchname}}"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Source Commit*\n<{{& webhookData.data.sourcecheckoutlink}}|{{webhookData.data.sourcecheckout}}>"
+ }
+ ]
+ },
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Target Branch*\n{{webhookData.data.targetbranchname}}"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Target Commit*\n<{{& webhookData.data.targetcheckoutlink}}|{{webhookData.data.targetcheckout}}>"
+ }
+ ]
+ },
+ {{/webhookData.mergedType}}
+ {{^webhookData.mergedType}}
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Target Checkout*\n{{webhookData.data.targetcheckout}}"
+ }
+ ]
+ },
+ {{/webhookData.mergedType}}
+ {{/webhookType}}
+ {{/ciMaterials}}
+ {
+ "type": "actions",
+ "elements": [{
+ "type": "button",
+ "text": {
+ "type": "plain_text",
+ "text": "View Details"
+ }
+ {{#buildHistoryLink}}
+ ,
+ "url": "{{& buildHistoryLink}}"
+ {{/buildHistoryLink}}
+ }]
+ }
+ ]
+}'
+where channel_type = 'slack'
+ and node_type = 'CI'
+ and event_type_id = 3;
\ No newline at end of file
diff --git a/scripts/sql/141_update_ci_notification_template.up.sql b/scripts/sql/141_update_ci_notification_template.up.sql
new file mode 100644
index 0000000000..8fd1aaa900
--- /dev/null
+++ b/scripts/sql/141_update_ci_notification_template.up.sql
@@ -0,0 +1,148 @@
+---- update notification template for CI fail ses/smtp
+UPDATE notification_templates
+set template_payload = '{"from": "{{fromEmail}}",
+ "to": "{{toEmail}}",
+ "subject": "CI failed for app: {{appName}}",
+ "html": "Build Pipeline Failed
{{eventTime}}
Triggered by {{triggeredBy}}
View Pipeline
Application: {{appName}} | Pipeline: {{pipelineName}}
Source Code
{{#ciMaterials}}{{^webhookType}}Branch: {{appName}}/{{branch}} | Commit: {{commit}}
{{#failureReason}}Reason: {{failureReason}}{{/failureReason}}{{/webhookType}}{{#webhookType}}{{#webhookData.mergedType}}Title: {{webhookData.data.title}} | Git URL: View
Source Branch: {{webhookData.data.sourcebranchname}} | Source Commit: {{webhookData.data.sourcecheckout}}
Target Branch: {{webhookData.data.targetbranchname}} | Target Commit: {{webhookData.data.targetcheckout}}
{{/webhookData.mergedType}}{{^webhookData.mergedType}}Target Checkout: {{webhookData.data.targetcheckout}}
{{/webhookData.mergedType}}{{/webhookType}}{{/ciMaterials}}
"}'
+where channel_type = 'ses'
+ and node_type = 'CI'
+ and event_type_id = 3;
+
+---- update notification template for CI fail slack
+UPDATE notification_templates
+set template_payload = '{
+ "text": ":x: Build pipeline Failed | {{#ciMaterials}} Branch > {{branch}} {{/ciMaterials}} | Application > {{appName}}",
+ "blocks": [{
+ "type": "section",
+ "text": {
+ "type": "mrkdwn",
+ "text": "\n"
+ }
+ },
+ {
+ "type": "divider"
+ },
+ {
+ "type": "section",
+ "text": {
+ "type": "mrkdwn",
+ "text": ":x: *Build Pipeline failed*\n \n Triggered by {{triggeredBy}}"
+ },
+ "accessory": {
+ "type": "image",
+ "image_url": "https://github.com/devtron-labs/notifier/assets/image/img_build_notification.png",
+ "alt_text": "calendar thumbnail"
+ }
+ },
+ {
+ "type": "section",
+ "fields": [{
+ "type": "mrkdwn",
+ "text": "*Application*\n{{appName}}"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Pipeline*\n{{pipelineName}}"
+ }
+ ]
+ },
+ {{#ciMaterials}}
+ {{^webhookType}}
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Branch*\n`{{appName}}/{{branch}}`"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Commit*\n<{{& commitLink}}|{{commit}}>"
+ }
+ ]
+ },
+ {{#failureReason}}
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Reason*\n{{failureReason}}"
+ }
+ ]
+ },{{/failureReason}}
+ {{/webhookType}}
+ {{#webhookType}}
+ {{#webhookData.mergedType}}
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Title*\n{{webhookData.data.title}}"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Git URL*\n<{{& webhookData.data.giturl}}|View>"
+ }
+ ]
+ },
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Source Branch*\n{{webhookData.data.sourcebranchname}}"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Source Commit*\n<{{& webhookData.data.sourcecheckoutlink}}|{{webhookData.data.sourcecheckout}}>"
+ }
+ ]
+ },
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Target Branch*\n{{webhookData.data.targetbranchname}}"
+ },
+ {
+ "type": "mrkdwn",
+ "text": "*Target Commit*\n<{{& webhookData.data.targetcheckoutlink}}|{{webhookData.data.targetcheckout}}>"
+ }
+ ]
+ },
+ {{/webhookData.mergedType}}
+ {{^webhookData.mergedType}}
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": "*Target Checkout*\n{{webhookData.data.targetcheckout}}"
+ }
+ ]
+ },
+ {{/webhookData.mergedType}}
+ {{/webhookType}}
+ {{/ciMaterials}}
+ {
+ "type": "actions",
+ "elements": [{
+ "type": "button",
+ "text": {
+ "type": "plain_text",
+ "text": "View Details"
+ }
+ {{#buildHistoryLink}}
+ ,
+ "url": "{{& buildHistoryLink}}"
+ {{/buildHistoryLink}}
+ }]
+ }
+ ]
+}'
+where channel_type = 'slack'
+ and node_type = 'CI'
+ and event_type_id = 3;
\ No newline at end of file