fix: stage artifact logic

Dockerfile

@@ -6,6 +6,8 @@
 RUN go install github.com/google/wire/cmd/wire@latest
 WORKDIR /go/src/github.com/devtron-labs/devtron
 ADD . /go/src/github.com/devtron-labs/devtron/
+# TODO Asutosh: revert after dev test
+ADD ./vendor/github.com/Microsoft/ /go/src/github.com/devtron-labs/devtron/vendor/github.com/microsoft/
 RUN GOOS=linux make build-all
 
 # uncomment this post build arg

api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go

@@ -51,10 +51,10 @@ import (
 )
 
 type DeploymentHistoryResp struct {
-	CdWorkflows                []pipelineConfig.CdWorkflowWithArtifact `json:"cdWorkflows"`
-	TagsEdiatable              bool                                    `json:"tagsEditable"`
-	AppReleaseTagNames         []string                                `json:"appReleaseTagNames"` //unique list of tags exists in the app
-	HideImageTaggingHardDelete bool                                    `json:"hideImageTaggingHardDelete"`
+	CdWorkflows                []pipelineBean.CdWorkflowWithArtifact `json:"cdWorkflows"`
+	TagsEdiatable              bool                                  `json:"tagsEditable"`
+	AppReleaseTagNames         []string                              `json:"appReleaseTagNames"` //unique list of tags exists in the app
+	HideImageTaggingHardDelete bool                                  `json:"hideImageTaggingHardDelete"`
 }
 
 type DevtronAppDeploymentRestHandler interface {

internal/sql/repository/CiArtifactRepository.go

@@ -73,7 +73,7 @@ type CiArtifact struct {
 	ScanEnabled           bool      `sql:"scan_enabled,notnull"`
 	Scanned               bool      `sql:"scanned,notnull"`
 	ExternalCiPipelineId  int       `sql:"external_ci_pipeline_id"`
-	IsArtifactUploaded    bool      `sql:"is_artifact_uploaded"`
+	IsArtifactUploaded    bool      `sql:"is_artifact_uploaded"` // Deprecated; Use pipelineConfig.CiWorkflow instead.
 	CredentialsSourceType string    `sql:"credentials_source_type"`
 	CredentialSourceValue string    `sql:"credentials_source_value"`
 	ComponentId           int       `sql:"component_id"`

internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go

@@ -23,7 +23,6 @@ import (
 	apiBean "github.com/devtron-labs/devtron/api/bean"
 	"github.com/devtron-labs/devtron/client/gitSensor"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
-	repository2 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging"
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow"
 	"github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/sql"
@@ -43,6 +42,7 @@ type CdWorkflowRepository interface {
 	FindArtifactByPipelineIdAndRunnerType(pipelineId int, runnerType apiBean.WorkflowType, limit int, runnerStatuses []string) ([]CdWorkflowRunner, error)
 	SaveWorkFlowRunner(wfr *CdWorkflowRunner) (*CdWorkflowRunner, error)
 	UpdateWorkFlowRunner(wfr *CdWorkflowRunner) error
+	UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error
 	GetPreviousQueuedRunners(cdWfrId, pipelineId int) ([]*CdWorkflowRunner, error)
 	UpdateRunnerStatusToFailedForIds(errMsg string, triggeredBy int32, cdWfrIds ...int) error
 	UpdateWorkFlowRunnersWithTxn(wfrs []*CdWorkflowRunner, tx *pg.Tx) error
@@ -134,6 +134,8 @@ type CdWorkflowRunner struct {
 	FinishedOn              time.Time                       `sql:"finished_on"`
 	Namespace               string                          `sql:"namespace"`
 	LogLocation             string                          `sql:"log_file_path"`
+	CdArtifactLocation      string                          `sql:"cd_artifact_location"`
+	IsArtifactUploaded      *bool                           `sql:"is_artifact_uploaded"`
 	TriggeredBy             int32                           `sql:"triggered_by"`
 	CdWorkflowId            int                             `sql:"cd_workflow_id"`
 	PodName                 string                          `sql:"pod_name"`
@@ -173,34 +175,6 @@ type CiPipelineMaterialResponse struct {
 	Regex           string                 `json:"regex"`
 }
 
-type CdWorkflowWithArtifact struct {
-	Id                    int                          `json:"id"`
-	CdWorkflowId          int                          `json:"cd_workflow_id"`
-	Name                  string                       `json:"name"`
-	Status                string                       `json:"status"`
-	PodStatus             string                       `json:"pod_status"`
-	Message               string                       `json:"message"`
-	StartedOn             time.Time                    `json:"started_on"`
-	FinishedOn            time.Time                    `json:"finished_on"`
-	PipelineId            int                          `json:"pipeline_id"`
-	Namespace             string                       `json:"namespace"`
-	LogFilePath           string                       `json:"log_file_path"`
-	TriggeredBy           int32                        `json:"triggered_by"`
-	EmailId               string                       `json:"email_id"`
-	Image                 string                       `json:"image"`
-	MaterialInfo          string                       `json:"material_info,omitempty"`
-	DataSource            string                       `json:"data_source,omitempty"`
-	CiArtifactId          int                          `json:"ci_artifact_id,omitempty"`
-	WorkflowType          string                       `json:"workflow_type,omitempty"`
-	ExecutorType          string                       `json:"executor_type,omitempty"`
-	BlobStorageEnabled    bool                         `json:"blobStorageEnabled"`
-	GitTriggers           map[int]GitCommit            `json:"gitTriggers"`
-	CiMaterials           []CiPipelineMaterialResponse `json:"ciMaterials"`
-	ImageReleaseTags      []*repository2.ImageTag      `json:"imageReleaseTags"`
-	ImageComment          *repository2.ImageComment    `json:"imageComment"`
-	RefCdWorkflowRunnerId int                          `json:"referenceCdWorkflowRunnerId"`
-}
-
 type TriggerWorkflowStatus struct {
 	CdWorkflowStatus []*CdWorkflowStatus `json:"cdWorkflowStatus"`
 	CiWorkflowStatus []*CiWorkflowStatus `json:"ciWorkflowStatus"`
@@ -502,6 +476,14 @@ func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunner(wfr *CdWorkflowRunner
 	return err
 }
 
+func (impl *CdWorkflowRepositoryImpl) UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error {
+	_, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)).
+		Set("is_artifact_uploaded = ?", isArtifactUploaded).
+		Where("id = ?", wfrId).
+		Update()
+	return err
+}
+
 func (impl *CdWorkflowRepositoryImpl) GetPreviousQueuedRunners(cdWfrId, pipelineId int) ([]*CdWorkflowRunner, error) {
 	var cdWfrs []*CdWorkflowRunner
 	err := impl.dbConnection.Model(&cdWfrs).

internal/sql/repository/pipelineConfig/CiWorkflowRepository.go

@@ -32,6 +32,7 @@ type CiWorkflowRepository interface {
 	SaveWorkFlow(wf *CiWorkflow) error
 	FindLastTriggeredWorkflow(pipelineId int) (*CiWorkflow, error)
 	UpdateWorkFlow(wf *CiWorkflow) error
+	UpdateArtifactUploaded(id int, isUploaded bool) error
 	FindByStatusesIn(activeStatuses []string) ([]*CiWorkflow, error)
 	FindByPipelineId(pipelineId int, offset int, size int) ([]WorkflowWithArtifact, error)
 	FindById(id int) (*CiWorkflow, error)
@@ -72,6 +73,7 @@ type CiWorkflow struct {
 	GitTriggers             map[int]GitCommit               `sql:"git_triggers"`
 	TriggeredBy             int32                           `sql:"triggered_by"`
 	CiArtifactLocation      string                          `sql:"ci_artifact_location"`
+	IsArtifactUploaded      *bool                           `sql:"is_artifact_uploaded"`
 	PodName                 string                          `sql:"pod_name"`
 	CiBuildType             string                          `sql:"ci_build_type"`
 	EnvironmentId           int                             `sql:"environment_id"`
@@ -111,7 +113,8 @@ type WorkflowWithArtifact struct {
 	CiArtifactId            int                             `json:"ci_artifact_d"`
 	BlobStorageEnabled      bool                            `json:"blobStorageEnabled"`
 	CiBuildType             string                          `json:"ci_build_type"`
-	IsArtifactUploaded      bool                            `json:"is_artifact_uploaded"`
+	IsArtifactUploadedV2    *bool                           `json:"is_artifact_uploaded"`     // IsArtifactUploadedV2 is the new column from ci_workflow table, IsArtifactUploaded is Deprecated and will be removed in future
+	IsArtifactUploaded      bool                            `json:"old_is_artifact_uploaded"` // Deprecated; Use IsArtifactUploadedV2 instead. IsArtifactUploaded is the column from ci_artifact table
 	EnvironmentId           int                             `json:"environmentId"`
 	EnvironmentName         string                          `json:"environmentName"`
 	RefCiWorkflowId         int                             `json:"referenceCiWorkflowId"`
@@ -191,7 +194,7 @@ func (impl *CiWorkflowRepositoryImpl) FindByStatusesIn(activeStatuses []string)
 // FindByPipelineId gets only those workflowWithArtifact whose parent_ci_workflow_id is null, this is done to accommodate multiple ci_artifacts through a single workflow(parent), making child workflows for other ci_artifacts (this has been done due to design understanding and db constraint) single workflow single ci-artifact
 func (impl *CiWorkflowRepositoryImpl) FindByPipelineId(pipelineId int, offset int, limit int) ([]WorkflowWithArtifact, error) {
 	var wfs []WorkflowWithArtifact
-	queryTemp := "select cia.id as ci_artifact_id, env.environment_name, cia.image, cia.is_artifact_uploaded, wf.*, u.email_id from ci_workflow wf left join users u on u.id = wf.triggered_by left join ci_artifact cia on wf.id = cia.ci_workflow_id left join environment env on env.id = wf.environment_id where wf.ci_pipeline_id = ? and parent_ci_workflow_id is null order by wf.started_on desc offset ? limit ?;"
+	queryTemp := "select cia.id as ci_artifact_id, env.environment_name, cia.image, cia.is_artifact_uploaded as old_is_artifact_uploaded, wf.*, u.email_id from ci_workflow wf left join users u on u.id = wf.triggered_by left join ci_artifact cia on wf.id = cia.ci_workflow_id left join environment env on env.id = wf.environment_id where wf.ci_pipeline_id = ? and parent_ci_workflow_id is null order by wf.started_on desc offset ? limit ?;"
 	_, err := impl.dbConnection.Query(&wfs, queryTemp, pipelineId, offset, limit)
 	if err != nil {
 		return nil, err
@@ -269,6 +272,14 @@ func (impl *CiWorkflowRepositoryImpl) UpdateWorkFlow(wf *CiWorkflow) error {
 	return err
 }
 
+func (impl *CiWorkflowRepositoryImpl) UpdateArtifactUploaded(id int, isUploaded bool) error {
+	_, err := impl.dbConnection.Model(&CiWorkflow{}).
+		Set("is_artifact_uploaded = ?", isUploaded).
+		Where("id = ?", id).
+		Update()
+	return err
+}
+
 func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) {
 	err = impl.dbConnection.Model(&ciWorkflow).
 		Column("ci_workflow.*", "CiPipeline").

internal/util/ValidateUtil.go

@@ -18,6 +18,7 @@ package util
 
 import (
 	"github.com/devtron-labs/devtron/pkg/auth/user/bean"
+	"github.com/devtron-labs/devtron/util/urlUtil"
 	"regexp"
 	"strings"
 
@@ -65,7 +66,7 @@ func validateNonEmptyUrl(fl validator.FieldLevel) bool {
 	if value == "" {
 		return true
 	}
-	return IsValidUrl(value)
+	return urlUtil.IsValidUrl(value)
 }
 
 func IntValidator() (*validator.Validate, error) {

pkg/deployment/trigger/devtronApps/PreStageTriggerService.go

@@ -51,6 +51,7 @@ import (
 	"github.com/go-pg/pg"
 	"go.opentelemetry.io/otel"
 	"maps"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -936,23 +937,23 @@ func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineC
 
 func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) {
 	cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat
-	if cdArtifactLocationFormat == "" {
+	if len(cdArtifactLocationFormat) == 0 {
 		cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat()
 	}
 	if cdWorkflowConfig.LogsBucket == "" {
 		cdWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket()
 	}
-	ArtifactLocation := fmt.Sprintf("s3://%s/"+impl.config.GetDefaultArtifactKeyPrefix()+"/"+cdArtifactLocationFormat, cdWorkflowConfig.LogsBucket, cdWf.Id, runner.Id)
-	artifactFileName := fmt.Sprintf(impl.config.GetDefaultArtifactKeyPrefix()+"/"+cdArtifactLocationFormat, cdWf.Id, runner.Id)
+	ArtifactLocation := fmt.Sprintf("s3://"+path.Join(cdWorkflowConfig.LogsBucket, cdArtifactLocationFormat), cdWf.Id, runner.Id)
+	artifactFileName := fmt.Sprintf(cdArtifactLocationFormat, cdWf.Id, runner.Id)
 	return ArtifactLocation, cdWorkflowConfig.LogsBucket, artifactFileName
 }
 
 func (impl *TriggerServiceImpl) buildDefaultArtifactLocation(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string {
 	cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat
-	if cdArtifactLocationFormat == "" {
+	if len(cdArtifactLocationFormat) == 0 {
 		cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat()
 	}
-	ArtifactLocation := fmt.Sprintf("%s/"+cdArtifactLocationFormat, impl.config.GetDefaultArtifactKeyPrefix(), savedWf.Id, runner.Id)
+	ArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWf.Id, runner.Id)
 	return ArtifactLocation
 }
 

pkg/devtronResource/bean/history/cdPipelineDeploymentHistory.go

@@ -1,6 +1,8 @@
 package history
 
-import "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
+import (
+	"github.com/devtron-labs/devtron/pkg/pipeline/bean"
+)
 
 type CdPipelineDeploymentHistoryListReq struct {
 	PipelineId int
@@ -18,8 +20,8 @@ type CdPipelineDeploymentHistoryConfigListReq struct {
 }
 
 type DeploymentHistoryResp struct {
-	CdWorkflows                []pipelineConfig.CdWorkflowWithArtifact `json:"cdWorkflows"`
-	TagsEditable               bool                                    `json:"tagsEditable"`
-	AppReleaseTagNames         []string                                `json:"appReleaseTagNames"` // unique list of tags exists in the app
-	HideImageTaggingHardDelete bool                                    `json:"hideImageTaggingHardDelete"`
+	CdWorkflows                []bean.CdWorkflowWithArtifact `json:"cdWorkflows"`
+	TagsEditable               bool                          `json:"tagsEditable"`
+	AppReleaseTagNames         []string                      `json:"appReleaseTagNames"` // unique list of tags exists in the app
+	HideImageTaggingHardDelete bool                          `json:"hideImageTaggingHardDelete"`
 }

pkg/devtronResource/history/deployment/cdPipeline/DeploymentHistoryService.go

@@ -4,6 +4,7 @@ import (
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	historyBean "github.com/devtron-labs/devtron/pkg/devtronResource/bean/history"
 	"github.com/devtron-labs/devtron/pkg/pipeline"
+	"github.com/devtron-labs/devtron/pkg/pipeline/bean"
 	"github.com/devtron-labs/devtron/pkg/pipeline/history"
 	"go.uber.org/zap"
 )
@@ -36,7 +37,7 @@ func NewDeploymentHistoryServiceImpl(logger *zap.SugaredLogger,
 }
 
 func (impl *DeploymentHistoryServiceImpl) GetCdPipelineDeploymentHistory(req *historyBean.CdPipelineDeploymentHistoryListReq) (resp historyBean.DeploymentHistoryResp, err error) {
-	var wfs []pipelineConfig.CdWorkflowWithArtifact
+	var wfs []bean.CdWorkflowWithArtifact
 	wfs, err = impl.cdHandler.GetCdBuildHistory(req.AppId, req.EnvId, req.PipelineId, req.Offset, req.Limit)
 	if err != nil {
 		impl.logger.Errorw("service err, List", "err", err, "req", req)

pkg/eventProcessor/bean/workflowEventBean.go

@@ -39,6 +39,7 @@ type CdStageCompleteEvent struct {
 	CiArtifactDTO                 pipelineConfig.CiArtifactDTO `json:"ciArtifactDTO"`
 	PluginRegistryArtifactDetails map[string][]string          `json:"PluginRegistryArtifactDetails"`
 	PluginArtifacts               *PluginArtifacts             `json:"pluginArtifacts"`
+	IsArtifactUploaded            bool                         `json:"isArtifactUploaded"`
 }
 
 type UserDeploymentRequest struct {
@@ -77,12 +78,12 @@ type CiCompleteEvent struct {
 	Metrics                       util.CIMetrics           `json:"metrics"`
 	AppName                       string                   `json:"appName"`
 	IsArtifactUploaded            bool                     `json:"isArtifactUploaded"`
-	FailureReason                 string                   `json:"failureReason"`
+	FailureReason                 string                   `json:"failureReason"` // FailureReason is used for notifying the failure reason to the user. Should be short and user-friendly
 	ImageDetailsFromCR            json.RawMessage          `json:"imageDetailsFromCR"`
 	PluginRegistryArtifactDetails map[string][]string      `json:"PluginRegistryArtifactDetails"`
 	PluginArtifactStage           string                   `json:"pluginArtifactStage"`
 	pluginImageDetails            *registry.ImageDetailsFromCR
-	PluginArtifacts               *PluginArtifacts         `json:"pluginArtifacts"`
+	PluginArtifacts               *PluginArtifacts `json:"pluginArtifacts"`
 }
 
 func (c *CiCompleteEvent) GetPluginImageDetails() *registry.ImageDetailsFromCR {

pkg/eventProcessor/in/WorkflowEventProcessorService.go

@@ -165,14 +165,21 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDStageCompleteEvent() error {
 			impl.logger.Errorw("could not get wf runner", "err", err)
 			return
 		}
-
+		wfr.IsArtifactUploaded = cdStageCompleteEvent.IsArtifactUploaded
 		if wfr.Status != string(v1alpha1.NodeSucceeded) {
 			impl.logger.Debugw("event received from ci runner, updating workflow runner status as succeeded", "savedWorkflowRunnerId", wfr.Id, "oldStatus", wfr.Status, "podStatus", wfr.PodStatus)
-			err = impl.cdWorkflowRunnerService.UpdateWfrStatus(wfr, string(v1alpha1.NodeSucceeded), 1)
+			wfr.Status = string(v1alpha1.NodeSucceeded)
+			err = impl.cdWorkflowRunnerService.UpdateWfr(wfr, 1)
 			if err != nil {
 				impl.logger.Errorw("update cd-wf-runner failed for id ", "cdWfrId", wfr.Id, "err", err)
 				return
 			}
+		} else if cdStageCompleteEvent.IsArtifactUploaded {
+			err = impl.cdWorkflowRunnerService.UpdateIsArtifactUploaded(wfr.Id, cdStageCompleteEvent.IsArtifactUploaded)
+			if err != nil {
+				impl.logger.Errorw("error in updating isArtifactUploaded", "cdWfrId", wfr.Id, "err", err)
+				return
+			}
 		}
 
 		triggerContext := triggerBean.TriggerContext{
@@ -514,7 +521,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCICompleteEvent() error {
 			ReferenceId: pointer.String(msg.MsgId),
 		}
 
-		if ciCompleteEvent.FailureReason != "" {
+		if len(ciCompleteEvent.FailureReason) != 0 {
 			req.FailureReason = ciCompleteEvent.FailureReason
 			err := impl.workflowDagExecutor.HandleCiStepFailedEvent(ciCompleteEvent.PipelineId, req)
 			if err != nil {