fix: stage artifact logic

api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go

@@ -1016,7 +1016,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon
 		return
 	}
 	//RBAC
-	resp, err := handler.ciHandler.GetHistoricBuildLogs(pipelineId, workflowId, nil)
+	resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil)
 	if err != nil {
 		handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId)
 		common.WriteJsonResp(w, err, resp, http.StatusInternalServerError)
@@ -1155,7 +1155,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetBuildLogs(w http.ResponseWriter
 			return
 		}
 	}
-	logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(pipelineId, workflowId)
+	logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(workflowId)
 	if err != nil {
 		handler.Logger.Errorw("service err, GetBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId, "lastEventId", lastEventId)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go

@@ -51,10 +51,10 @@ import (
 )
 
 type DeploymentHistoryResp struct {
-	CdWorkflows                []pipelineConfig.CdWorkflowWithArtifact `json:"cdWorkflows"`
-	TagsEdiatable              bool                                    `json:"tagsEditable"`
-	AppReleaseTagNames         []string                                `json:"appReleaseTagNames"` //unique list of tags exists in the app
-	HideImageTaggingHardDelete bool                                    `json:"hideImageTaggingHardDelete"`
+	CdWorkflows                []pipelineBean.CdWorkflowWithArtifact `json:"cdWorkflows"`
+	TagsEdiatable              bool                                  `json:"tagsEditable"`
+	AppReleaseTagNames         []string                              `json:"appReleaseTagNames"` //unique list of tags exists in the app
+	HideImageTaggingHardDelete bool                                  `json:"hideImageTaggingHardDelete"`
 }
 
 type DevtronAppDeploymentRestHandler interface {
@@ -1903,7 +1903,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadArtifacts(w http.ResponseW
 	}
 	//RBAC CHECK
 
-	file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(pipelineId, buildId)
+	file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(buildId)
 	defer file.Close()
 
 	if err != nil {

client/events/EventBuilder.go

@@ -37,7 +37,7 @@ import (
 type EventFactory interface {
 	Build(eventType util.EventType, sourceId *int, appId int, envId *int, pipelineType util.PipelineType) Event
 	BuildExtraCDData(event Event, wfr *pipelineConfig.CdWorkflowRunner, pipelineOverrideId int, stage bean2.WorkflowType) Event
-	BuildExtraCIData(event Event, material *MaterialTriggerInfo, dockerImage string) Event
+	BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event
 	//BuildFinalData(event Event) *Payload
 }
 
@@ -163,7 +163,7 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCDData(event Event, wfr *pipelineC
 	return event
 }
 
-func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo, dockerImage string) Event {
+func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event {
 	if material == nil {
 		materialInfo, err := impl.getCiMaterialInfo(event.PipelineId, event.CiArtifactId)
 		if err != nil {

env_gen.md

@@ -40,7 +40,6 @@
  | BUILD_LOG_TTL_VALUE_IN_SECS | 3600 |  | 
  | CACHE_LIMIT | 5000000000 |  | 
  | CASBIN_DATABASE | casbin |  | 
- | CD_ARTIFACT_LOCATION_FORMAT | %d/%d.zip |  | 
  | CD_DEFAULT_ADDRESS_POOL_BASE_CIDR |  |  | 
  | CD_DEFAULT_ADDRESS_POOL_SIZE |  |  | 
  | CD_HELM_PIPELINE_STATUS_CRON_TIME | */2 * * * * |  | 
@@ -59,7 +58,6 @@
  | CD_WORKFLOW_EXECUTOR_TYPE | AWF |  | 
  | CD_WORKFLOW_SERVICE_ACCOUNT | cd-runner |  | 
  | CExpirationTime | 600 |  | 
- | CI_ARTIFACT_LOCATION_FORMAT | %d/%d.zip |  | 
  | CI_DEFAULT_ADDRESS_POOL_BASE_CIDR |  |  | 
  | CI_DEFAULT_ADDRESS_POOL_SIZE |  |  | 
  | CI_IGNORE_DOCKER_CACHE |  |  | 

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/deckarep/golang-set v1.8.0
 	github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8
-	github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214
+	github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6
 	github.com/devtron-labs/go-bitbucket v0.9.60-beta
 	github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80
 	github.com/evanphx/json-patch v5.7.0+incompatible

go.sum

@@ -794,8 +794,8 @@ github.com/devtron-labs/argo-workflows/v3 v3.5.10 h1:6rxQOesOzDz6SgQCMDQNHaehsKF
 github.com/devtron-labs/argo-workflows/v3 v3.5.10/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA=
 github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 h1:2+Q7Jdhpo/uMiaQiZZzAh+ZX7wEJIFuMFG6DEiMuo64=
 github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8/go.mod h1:702R6WIf5y9UzKGoCGxQ+x3l5Ws+l0fXg2xlCpSGFZI=
-github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214 h1:QBBienyRjnNw/IwVt8O4WxIQvQJUbqvLp51mQqfwcTM=
-github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E=
+github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6 h1:ikMVToilPandd3iKEGkISMcyvWva440n/zAlMZT2M3E=
+github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
 github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys=

internal/sql/repository/CiArtifactRepository.go

@@ -73,7 +73,7 @@ type CiArtifact struct {
 	ScanEnabled           bool      `sql:"scan_enabled,notnull"`
 	Scanned               bool      `sql:"scanned,notnull"`
 	ExternalCiPipelineId  int       `sql:"external_ci_pipeline_id"`
-	IsArtifactUploaded    bool      `sql:"is_artifact_uploaded"`
+	IsArtifactUploaded    bool      `sql:"is_artifact_uploaded"` // Deprecated; Use pipelineConfig.CiWorkflow instead.
 	CredentialsSourceType string    `sql:"credentials_source_type"`
 	CredentialSourceValue string    `sql:"credentials_source_value"`
 	ComponentId           int       `sql:"component_id"`

internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go

@@ -23,8 +23,8 @@ import (
 	apiBean "github.com/devtron-labs/devtron/api/bean"
 	"github.com/devtron-labs/devtron/client/gitSensor"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
-	repository2 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging"
-	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow"
+	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow"
+	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow"
 	"github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/sql"
 	"github.com/go-pg/pg"
@@ -43,13 +43,13 @@ type CdWorkflowRepository interface {
 	FindArtifactByPipelineIdAndRunnerType(pipelineId int, runnerType apiBean.WorkflowType, limit int, runnerStatuses []string) ([]CdWorkflowRunner, error)
 	SaveWorkFlowRunner(wfr *CdWorkflowRunner) (*CdWorkflowRunner, error)
 	UpdateWorkFlowRunner(wfr *CdWorkflowRunner) error
+	UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded workflow.ArtifactUploadedType) error
 	GetPreviousQueuedRunners(cdWfrId, pipelineId int) ([]*CdWorkflowRunner, error)
 	UpdateRunnerStatusToFailedForIds(errMsg string, triggeredBy int32, cdWfrIds ...int) error
 	UpdateWorkFlowRunnersWithTxn(wfrs []*CdWorkflowRunner, tx *pg.Tx) error
 	UpdateWorkFlowRunners(wfr []*CdWorkflowRunner) error
 	FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([]*CdWorkflowRunner, error)
 	FindPreviousCdWfRunnerByStatus(pipelineId int, currentWFRunnerId int, status []string) ([]*CdWorkflowRunner, error)
-	FindConfigByPipelineId(pipelineId int) (*CdWorkflowConfig, error)
 	FindWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error)
 	FindBasicWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error)
 	FindRetriedWorkflowCountByReferenceId(wfrId int) (int, error)
@@ -76,6 +76,9 @@ type CdWorkflowRepository interface {
 	FetchArtifactsByCdPipelineId(pipelineId int, runnerType apiBean.WorkflowType, offset, limit int, searchString string) ([]CdWorkflowRunner, error)
 	GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses(getPipelineDeployedWithinHours int) ([]*CdWorkflowRunner, error)
 	FindLatestRunnerByPipelineIdsAndRunnerType(ctx context.Context, pipelineIds []int, runnerType apiBean.WorkflowType) ([]CdWorkflowRunner, error)
+
+	MigrateIsArtifactUploaded(wfrId int, isArtifactUploaded bool)
+	MigrateCdArtifactLocation(wfrId int, cdArtifactLocation string)
 }
 
 type CdWorkflowRepositoryImpl struct {
@@ -95,27 +98,6 @@ type CdWorkflow struct {
 	sql.AuditLog
 }
 
-type CdWorkflowConfig struct {
-	tableName                struct{} `sql:"cd_workflow_config" pg:",discard_unknown_columns"`
-	Id                       int      `sql:"id,pk"`
-	CdTimeout                int64    `sql:"cd_timeout"`
-	MinCpu                   string   `sql:"min_cpu"`
-	MaxCpu                   string   `sql:"max_cpu"`
-	MinMem                   string   `sql:"min_mem"`
-	MaxMem                   string   `sql:"max_mem"`
-	MinStorage               string   `sql:"min_storage"`
-	MaxStorage               string   `sql:"max_storage"`
-	MinEphStorage            string   `sql:"min_eph_storage"`
-	MaxEphStorage            string   `sql:"max_eph_storage"`
-	CdCacheBucket            string   `sql:"cd_cache_bucket"`
-	CdCacheRegion            string   `sql:"cd_cache_region"`
-	CdImage                  string   `sql:"cd_image"`
-	Namespace                string   `sql:"wf_namespace"`
-	CdPipelineId             int      `sql:"cd_pipeline_id"`
-	LogsBucket               string   `sql:"logs_bucket"`
-	CdArtifactLocationFormat string   `sql:"cd_artifact_location_format"`
-}
-
 type CdWorkflowRunnerWithExtraFields struct {
 	CdWorkflowRunner
 	TotalCount int
@@ -134,6 +116,8 @@ type CdWorkflowRunner struct {
 	FinishedOn              time.Time                       `sql:"finished_on"`
 	Namespace               string                          `sql:"namespace"`
 	LogLocation             string                          `sql:"log_file_path"`
+	CdArtifactLocation      string                          `sql:"cd_artifact_location"`
+	IsArtifactUploaded      workflow.ArtifactUploadedType   `sql:"is_artifact_uploaded"`
 	TriggeredBy             int32                           `sql:"triggered_by"`
 	CdWorkflowId            int                             `sql:"cd_workflow_id"`
 	PodName                 string                          `sql:"pod_name"`
@@ -145,6 +129,15 @@ type CdWorkflowRunner struct {
 	sql.AuditLog
 }
 
+func (c *CdWorkflowRunner) GetIsArtifactUploaded() (isArtifactUploaded bool, isMigrationRequired bool) {
+	return workflow.IsArtifactUploaded(c.IsArtifactUploaded)
+}
+
+func (c *CdWorkflowRunner) WithIsArtifactUploaded(isArtifactUploaded bool) *CdWorkflowRunner {
+	c.IsArtifactUploaded = workflow.GetArtifactUploadedType(isArtifactUploaded)
+	return c
+}
+
 func (c *CdWorkflowRunner) IsExternalRun() bool {
 	var isExtCluster bool
 	if c.WorkflowType == cdWorkflow.WorkflowTypePre {
@@ -173,34 +166,6 @@ type CiPipelineMaterialResponse struct {
 	Regex           string                 `json:"regex"`
 }
 
-type CdWorkflowWithArtifact struct {
-	Id                    int                          `json:"id"`
-	CdWorkflowId          int                          `json:"cd_workflow_id"`
-	Name                  string                       `json:"name"`
-	Status                string                       `json:"status"`
-	PodStatus             string                       `json:"pod_status"`
-	Message               string                       `json:"message"`
-	StartedOn             time.Time                    `json:"started_on"`
-	FinishedOn            time.Time                    `json:"finished_on"`
-	PipelineId            int                          `json:"pipeline_id"`
-	Namespace             string                       `json:"namespace"`
-	LogFilePath           string                       `json:"log_file_path"`
-	TriggeredBy           int32                        `json:"triggered_by"`
-	EmailId               string                       `json:"email_id"`
-	Image                 string                       `json:"image"`
-	MaterialInfo          string                       `json:"material_info,omitempty"`
-	DataSource            string                       `json:"data_source,omitempty"`
-	CiArtifactId          int                          `json:"ci_artifact_id,omitempty"`
-	WorkflowType          string                       `json:"workflow_type,omitempty"`
-	ExecutorType          string                       `json:"executor_type,omitempty"`
-	BlobStorageEnabled    bool                         `json:"blobStorageEnabled"`
-	GitTriggers           map[int]GitCommit            `json:"gitTriggers"`
-	CiMaterials           []CiPipelineMaterialResponse `json:"ciMaterials"`
-	ImageReleaseTags      []*repository2.ImageTag      `json:"imageReleaseTags"`
-	ImageComment          *repository2.ImageComment    `json:"imageComment"`
-	RefCdWorkflowRunnerId int                          `json:"referenceCdWorkflowRunnerId"`
-}
-
 type TriggerWorkflowStatus struct {
 	CdWorkflowStatus []*CdWorkflowStatus `json:"cdWorkflowStatus"`
 	CiWorkflowStatus []*CiWorkflowStatus `json:"ciWorkflowStatus"`
@@ -277,12 +242,6 @@ func (impl *CdWorkflowRepositoryImpl) FindById(wfId int) (*CdWorkflow, error) {
 	return ddWorkflow, err
 }
 
-func (impl *CdWorkflowRepositoryImpl) FindConfigByPipelineId(pipelineId int) (*CdWorkflowConfig, error) {
-	cdWorkflowConfig := &CdWorkflowConfig{}
-	err := impl.dbConnection.Model(cdWorkflowConfig).Where("cd_pipeline_id = ?", pipelineId).Select()
-	return cdWorkflowConfig, err
-}
-
 func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineId(pipelineIds []int) (*CdWorkflow, error) {
 	cdWorkflow := &CdWorkflow{}
 	err := impl.dbConnection.Model(cdWorkflow).Where("pipeline_id in (?)", pg.In(pipelineIds)).Order("id DESC").Limit(1).Select()
@@ -502,6 +461,14 @@ func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunner(wfr *CdWorkflowRunner
 	return err
 }
 
+func (impl *CdWorkflowRepositoryImpl) UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded workflow.ArtifactUploadedType) error {
+	_, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)).
+		Set("is_artifact_uploaded = ?", isArtifactUploaded).
+		Where("id = ?", wfrId).
+		Update()
+	return err
+}
+
 func (impl *CdWorkflowRepositoryImpl) GetPreviousQueuedRunners(cdWfrId, pipelineId int) ([]*CdWorkflowRunner, error) {
 	var cdWfrs []*CdWorkflowRunner
 	err := impl.dbConnection.Model(&cdWfrs).
@@ -773,3 +740,23 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestRunnerByPipelineIdsAndRunnerType
 	}
 	return latestWfrs, err
 }
+
+func (impl *CdWorkflowRepositoryImpl) MigrateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) {
+	_, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)).
+		Set("is_artifact_uploaded = ?", workflow.GetArtifactUploadedType(isArtifactUploaded)).
+		Where("id = ?", wfrId).
+		Update()
+	if err != nil {
+		impl.logger.Errorw("error in updating is artifact uploaded", "wfrId", wfrId, "err", err)
+	}
+}
+
+func (impl *CdWorkflowRepositoryImpl) MigrateCdArtifactLocation(wfrId int, cdArtifactLocation string) {
+	_, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)).
+		Set("cd_artifact_location = ?", cdArtifactLocation).
+		Where("id = ?", wfrId).
+		Update()
+	if err != nil {
+		impl.logger.Errorw("error in updating cd artifact location", "wfrId", wfrId, "err", err)
+	}
+}