Merge branch 'main' into release-bot

Author: kamal-devtron

env_gen.json

@@ -187,6 +187,7 @@
  | ECR_REPO_NAME_PREFIX | string |test/ | Prefix for ECR repo to be created in does not exist |  | false |
  | ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART | bool |false | To enable async installation of gitops application |  | false |
  | ENABLE_ASYNC_INSTALL_DEVTRON_CHART | bool |false | To enable async installation of no-gitops application |  | false |
+ | ENABLE_LINKED_CI_ARTIFACT_COPY | bool |false | Enable copying artifacts from parent CI pipeline to linked CI pipeline during creation |  | false |
  | EPHEMERAL_SERVER_VERSION_REGEX | string |v[1-9]\.\b(2[3-9]\|[3-9][0-9])\b.* | ephemeral containers support version regex that is compared with k8sServerVersion |  | false |
  | EVENT_URL | string |http://localhost:3000/notify | Notifier service url |  | false |
  | EXECUTE_WIRE_NIL_CHECKER | bool |false | checks for any nil pointer in wire.go |  | false |
@@ -225,6 +226,7 @@
  | LENS_URL | string |http://lens-milandevtron-service:80 | Lens micro-service URL |  | false |
  | LIMIT_CI_CPU | string |0.5 |  |  | false |
  | LIMIT_CI_MEM | string |3G |  |  | false |
+ | LINKED_CI_ARTIFACT_COPY_LIMIT | int |10 | Maximum number of artifacts to copy from parent CI pipeline to linked CI pipeline |  | false |
  | LOGGER_DEV_MODE | bool |false | Enables a different logger theme. |  | false |
  | LOG_LEVEL | int |-1 |  |  | false |
  | MAX_SESSION_PER_USER | int |5 | max no of cluster terminal pods can be created by an user |  | false |

env_gen.md

@@ -124,6 +124,7 @@ type CiArtifactRepository interface {
 	GetArtifactsByCDPipelineAndRunnerType(cdPipelineId int, runnerType bean.WorkflowType) ([]CiArtifact, error)
 	SaveAll(artifacts []*CiArtifact) ([]*CiArtifact, error)
 	GetArtifactsByCiPipelineId(ciPipelineId int) ([]CiArtifact, error)
+	GetLatestArtifactsByCiPipelineId(ciPipelineId, limit int) ([]CiArtifact, error)
 	GetArtifactsByCiPipelineIds(ciPipelineIds []int) ([]CiArtifact, error)
 	FinDByParentCiArtifactAndCiId(parentCiArtifact int, ciPipelineIds []int) ([]*CiArtifact, error)
 	GetLatest(cdPipelineId int) (int, error)
@@ -680,6 +681,19 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCiPipelineId(ciPipelineId int
 	return artifacts, err
 }
 
+func (impl CiArtifactRepositoryImpl) GetLatestArtifactsByCiPipelineId(ciPipelineId, limit int) ([]CiArtifact, error) {
+	var artifacts []CiArtifact
+	err := impl.dbConnection.
+		Model(&artifacts).
+		Column("ci_artifact.*").
+		Join("INNER JOIN ci_pipeline cp on cp.id=ci_artifact.pipeline_id").
+		Where("ci_artifact.pipeline_id = ?", ciPipelineId).
+		Where("cp.deleted = ?", false).
+		Order("ci_artifact.id DESC").Limit(limit).
+		Select()
+	return artifacts, err
+}
+
 func (impl CiArtifactRepositoryImpl) GetArtifactsByCiPipelineIds(ciPipelineIds []int) ([]CiArtifact, error) {
 	var artifacts []CiArtifact
 	if len(ciPipelineIds) == 0 {

internal/sql/repository/CiArtifactRepository.go

@@ -292,9 +292,6 @@ func (impl ChartTemplateServiceImpl) overrideChartMetaDataInDir(chartDir string,
 		impl.logger.Errorw("error in loading template chart", "chartPath", chartDir, "err", err)
 		return nil, err
 	}
-	if len(chartMetaData.APIVersion) > 0 {
-		chart.Metadata.APIVersion = chartMetaData.APIVersion
-	}
 	if len(chartMetaData.Name) > 0 {
 		chart.Metadata.Name = chartMetaData.Name
 	}

internal/util/ChartTemplateService.go

@@ -154,6 +154,7 @@ type CiCdPipelineOrchestratorImpl struct {
 	deploymentConfigReadService   read.DeploymentConfigReadService
 	workflowCacheConfig           types.WorkflowCacheConfig
 	chartReadService              read2.ChartReadService
+	globalEnvVariables            *util2.GlobalEnvVariables
 }
 
 func NewCiCdPipelineOrchestrator(
@@ -185,7 +186,8 @@ func NewCiCdPipelineOrchestrator(
 	gitOpsConfigReadService config.GitOpsConfigReadService,
 	deploymentConfigService common.DeploymentConfigService,
 	deploymentConfigReadService read.DeploymentConfigReadService,
-	chartReadService read2.ChartReadService) *CiCdPipelineOrchestratorImpl {
+	chartReadService read2.ChartReadService,
+	envVariables *util2.EnvironmentVariables) *CiCdPipelineOrchestratorImpl {
 	_, workflowCacheConfig, err := types.GetCiConfigWithWorkflowCacheConfig()
 	if err != nil {
 		logger.Errorw("Error in getting workflow cache config, continuing with default values", "err", err)
@@ -223,6 +225,7 @@ func NewCiCdPipelineOrchestrator(
 		deploymentConfigReadService:   deploymentConfigReadService,
 		workflowCacheConfig:           workflowCacheConfig,
 		chartReadService:              chartReadService,
+		globalEnvVariables:            envVariables.GlobalEnvVariables,
 	}
 }
 
@@ -1160,6 +1163,19 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf
 				r.GitMaterialName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:]
 			}
 		}
+
+		// Copy artifacts from parent CI pipeline if this is a linked CI pipeline
+		if ciPipeline.ParentCiPipeline > 0 && ciPipeline.PipelineType == buildCommonBean.LINKED {
+			go func() {
+				err = impl.copyArtifactsFromParentCiPipeline(ciPipeline.ParentCiPipeline, ciPipeline.Id, createRequest.UserId)
+				if err != nil {
+					impl.logger.Errorw("error in copying artifacts from parent CI pipeline",
+						"parentCiPipelineId", ciPipeline.ParentCiPipeline,
+						"childCiPipelineId", ciPipeline.Id,
+						"err", err)
+				}
+			}()
+		}
 	}
 	return createRequest, nil
 }
@@ -2511,3 +2527,63 @@ func (impl *CiCdPipelineOrchestratorImpl) GetWorkflowCacheConfig(appType helper.
 		}
 	}
 }
+
+// copyArtifactsFromParentCiPipeline copies existing artifacts from parent CI pipeline to linked CI pipeline during pipeline creation
+func (impl *CiCdPipelineOrchestratorImpl) copyArtifactsFromParentCiPipeline(parentCiPipelineId, childCiPipelineId int, userId int32) error {
+	if !impl.globalEnvVariables.EnableLinkedCiArtifactCopy {
+		impl.logger.Debugw("Linked CI artifact copying is disabled", "parentCiPipelineId", parentCiPipelineId, "childCiPipelineId", childCiPipelineId)
+		return nil
+	}
+	limit := impl.globalEnvVariables.LinkedCiArtifactCopyLimit
+	if limit <= 0 {
+		return nil
+	}
+
+	impl.logger.Infow("Starting, copyArtifactsFromParentCiPipeline", "parentCiPipelineId", parentCiPipelineId,
+		"childCiPipelineId", childCiPipelineId, "limit", limit)
+
+	parentArtifacts, err := impl.CiArtifactRepository.GetLatestArtifactsByCiPipelineId(parentCiPipelineId, limit)
+	if err != nil && !util.IsErrNoRows(err) {
+		impl.logger.Errorw("error in fetching artifacts from parent CI pipeline", "parentCiPipelineId", parentCiPipelineId, "err", err)
+		return err
+	}
+
+	if len(parentArtifacts) == 0 {
+		impl.logger.Infow("No artifacts found in parent CI pipeline", "parentCiPipelineId", parentCiPipelineId)
+		return nil
+	}
+
+	var childArtifacts []*repository.CiArtifact
+	for i := len(parentArtifacts); i > 0; i-- {
+		parentArtifact := parentArtifacts[i-1] //reversing order into new array because postgres will save the array as it is thus causing our actual order to reverse
+		childArtifact := &repository.CiArtifact{
+			Image:              parentArtifact.Image,
+			ImageDigest:        parentArtifact.ImageDigest,
+			MaterialInfo:       parentArtifact.MaterialInfo,
+			DataSource:         parentArtifact.DataSource,
+			PipelineId:         childCiPipelineId,
+			ParentCiArtifact:   parentArtifact.Id,
+			IsArtifactUploaded: parentArtifact.IsArtifactUploaded, // for backward compatibility
+			ScanEnabled:        parentArtifact.ScanEnabled,
+			TargetPlatforms:    parentArtifact.TargetPlatforms,
+			AuditLog: sql.AuditLog{
+				CreatedOn: time.Now(),
+				CreatedBy: userId,
+				UpdatedOn: time.Now(),
+				UpdatedBy: userId,
+			},
+		}
+		if parentArtifact.ScanEnabled {
+			childArtifact.Scanned = parentArtifact.Scanned
+		}
+		childArtifacts = append(childArtifacts, childArtifact)
+	}
+	if len(childArtifacts) > 0 {
+		_, err = impl.CiArtifactRepository.SaveAll(childArtifacts)
+		if err != nil {
+			impl.logger.Errorw("error in saving copied artifacts for child ci pipeline", "childCiPipelineId", childCiPipelineId, "err", err)
+			return err
+		}
+	}
+	return nil
+}

pkg/pipeline/CiCdPipelineOrchestrator.go

@@ -1,2 +1,4 @@
+apiVersion: v2
+description: Proxy chart for devtron reference app
 name: sample
 version: 1.0.0

scripts/devtron-reference-helm-charts/reference-chart-proxy/Chart.yaml

@@ -0,0 +1,67 @@
+BEGIN;
+
+-- Delete plugin step variable (for step index=1)
+DELETE FROM plugin_step_variable 
+WHERE plugin_step_id = (
+    SELECT ps.id 
+    FROM plugin_metadata p
+    INNER JOIN plugin_step ps ON ps.plugin_id = p.id
+    WHERE p.plugin_version = '1.0.1'
+      AND p.name = 'Terraform CLI v1.0.0'
+      AND p.deleted = false
+      AND ps."index" = 1
+      AND ps.deleted = false
+);
+
+-- Delete plugin steps
+DELETE FROM plugin_step 
+WHERE plugin_id = (
+    SELECT id 
+    FROM plugin_metadata 
+    WHERE plugin_version = '1.0.1' 
+      AND name = 'Terraform CLI v1.0.0' 
+      AND deleted = false
+);
+
+-- Delete pipeline script
+DELETE FROM plugin_pipeline_script 
+WHERE id = (
+    SELECT script_id 
+    FROM plugin_step 
+    WHERE plugin_id = (
+        SELECT id 
+        FROM plugin_metadata 
+        WHERE plugin_version = '1.0.1' 
+          AND name = 'Terraform CLI v1.0.0' 
+          AND deleted = false
+    )
+);
+
+-- Delete stage mappings
+DELETE FROM plugin_stage_mapping 
+WHERE plugin_id = (
+    SELECT id 
+    FROM plugin_metadata 
+    WHERE plugin_version = '1.0.1' 
+      AND name = 'Terraform CLI v1.0.0' 
+      AND deleted = false
+);
+
+-- Delete plugin metadata entry
+DELETE FROM plugin_metadata 
+WHERE plugin_version = '1.0.1' 
+  AND name = 'Terraform CLI v1.0.0' 
+  AND deleted = false;
+
+-- Mark previous version as latest
+UPDATE plugin_metadata 
+SET is_latest = true 
+WHERE id = (
+    SELECT id 
+    FROM plugin_metadata 
+    WHERE name = 'Terraform CLI v1.0.0'
+      AND is_latest = false
+      AND plugin_version = '1.0.0'
+);
+
+COMMIT;

scripts/sql/34504202_terraform_cli_v1_1_0.down.sql

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2025. Devtron Inc.
+ */
+BEGIN;
+
+-- Update the existing entry is_latest false for terrfaorm plugin
+UPDATE plugin_metadata SET is_latest = false WHERE id = (SELECT id FROM plugin_metadata WHERE name= 'Terraform CLI v1.0.0' and is_latest= true);
+
+-- Insert an entry in plugin_metadata for new version v1.0.1
+INSERT INTO "plugin_metadata" ("id", "name", "description","deleted", "created_on", "created_by", "updated_on", "updated_by","plugin_parent_metadata_id","plugin_version","is_deprecated","is_latest")
+VALUES (nextval('id_seq_plugin_metadata'), 'Terraform CLI v1.0.0','Terraform: Simplify infrastructure as code, manage resources effortlessly, and deploy with ease using this plugins','f', 'now()', 1, 'now()', 1, (SELECT id FROM plugin_parent_metadata WHERE identifier='terraform-cli-v1-0-0'),'1.0.1', false, true);
+
+-- Insert the plugin stage mapping 
+INSERT INTO "plugin_stage_mapping" ("plugin_id","stage_type","created_on", "created_by", "updated_on", "updated_by")
+VALUES ((SELECT id FROM plugin_metadata WHERE plugin_version='1.0.1' and name='Terraform CLI v1.0.0' and deleted= false),0,'now()', 1, 'now()', 1);
+
+-- Insert updated script 
+INSERT INTO "plugin_pipeline_script" ("id", "script","type","deleted","created_on", "created_by", "updated_on", "updated_by")
+VALUES (nextval('id_seq_plugin_pipeline_script'),E'
+export DEFAULT_TF_IMAGE=docker.io/hashicorp/terraform:latest
+if [ -n "$TERRAFORM_IMAGE" ]; then
+    echo "Using $TERRAFORM_IMAGE as the custom image."
+    DEFAULT_TF_IMAGE="$TERRAFORM_IMAGE"
+else
+    echo "Using the default image --> $DEFAULT_TF_IMAGE"
+fi
+
+# RUNNING Terraform init 
+if [ $RUN_TERRAFORM_INIT == "TRUE" ]; then 
+    #RUNNING Terraform init 
+    docker run -v $PWD:$PWD -w $PWD/$WORKINGDIR  -e HTTP_PROXY=$HTTP_PROXY -e HTTPS_PROXY=$HTTPS_PROXY -e NO_PROXY=$NO_PROXY $DEFAULT_TF_IMAGE init
+fi
+
+
+# exporting all the env variables 
+echo "$ADDITIONALPARAMS" > devtron-custom-values.env
+
+# RUNNING Terraform command 
+echo "docker run -v $PWD:$PWD -w $PWD/$WORKINGDIR --env-file devtron-custom-values.env -e HTTP_PROXY=$HTTP_PROXY -e HTTPS_PROXY=$HTTPS_PROXY -e NO_PROXY=$NO_PROXY $DEFAULT_TF_IMAGE $ARGS"
+docker run -v $PWD:$PWD -w $PWD/$WORKINGDIR  --env-file devtron-custom-values.env -e HTTP_PROXY=$HTTP_PROXY -e HTTPS_PROXY=$HTTPS_PROXY -e NO_PROXY=$NO_PROXY $DEFAULT_TF_IMAGE $ARGS','SHELL','f','now()',1,'now()',1);
+
+INSERT INTO "plugin_step" ("id", "plugin_id","name","description","index","step_type","script_id","deleted", "created_on", "created_by", "updated_on", "updated_by")
+VALUES (nextval('id_seq_plugin_step'), (SELECT id FROM plugin_metadata WHERE name='Terraform CLI v1.0.0' and plugin_version='1.0.1'),'Step 1','Step 1 - Terraform CLI v1.0.0','1','INLINE',(SELECT last_value FROM id_seq_plugin_pipeline_script),'f','now()', 1, 'now()', 1);
+
+INSERT INTO plugin_step_variable (id,plugin_step_id,name,format,description,is_exposed,allow_empty_value,default_value,value,variable_type,value_type,previous_step_index,variable_step_index,variable_step_index_in_plugin,reference_variable_name,deleted,created_on,created_by,updated_on,updated_by) 
+VALUES
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Terraform CLI v1.0.0' and ps."index"=1 and p.plugin_version='1.0.1' and ps.deleted=false),'HTTP_PROXY','STRING','Specify the HTTP proxy server for non-SSL requests.','t','t',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Terraform CLI v1.0.0' and ps."index"=1 and p.plugin_version='1.0.1' and ps.deleted=false),'HTTPS_PROXY','STRING','Specify the HTTPS proxy server for SSL requests.','t','t',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Terraform CLI v1.0.0' and ps."index"=1 and p.plugin_version='1.0.1' and ps.deleted=false),'NO_PROXY','STRING','Opt out of proxying HTTP/HTTPS requests. Use this to specify hosts that should bypass the proxy.','t','t',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Terraform CLI v1.0.0' and ps."index"=1 and p.plugin_version='1.0.1' and ps.deleted=false),'TERRAFORM_IMAGE','STRING','Specify a custom Terraform container image to use for the execution of Terraform commands.','t','t',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Terraform CLI v1.0.0' and ps."index"=1 and p.plugin_version='1.0.1' and ps.deleted=false),'WORKINGDIR','STRING','Set the source directory for Terraform execution.Example: /path/to/terraform/project','t','t',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Terraform CLI v1.0.0' and ps."index"=1 and p.plugin_version='1.0.1' and ps.deleted=false),'ARGS','STRING','Specifies Terraform CLI commands to run. Example: plan -var-file=myvars.tfvars','t','t','--help',null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Terraform CLI v1.0.0' and ps."index"=1 and p.plugin_version='1.0.1' and ps.deleted=false),'RUN_TERRAFORM_INIT','BOOL','Determines whether to run the Terraform initialization command.','t','t','TRUE',null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Terraform CLI v1.0.0' and ps."index"=1 and p.plugin_version='1.0.1' and ps.deleted=false),'ADDITIONALPARAMS','STRING','Provides key-value pairs to inject into the Terraform container.Example: VAR1=value1 VAR2=value2c','t','t',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1);
+COMMIT;

scripts/sql/34504202_terraform_cli_v1_1_0.up.sql

@@ -0,0 +1 @@
+--- No down migration required

scripts/sql/34604601_deployment_chart_cache.down.sql

@@ -0,0 +1,13 @@
+--- There was an issue with Deployment chart versions 4.18.0, 4.19.0, 1.0.0, 1.1.0
+--- Which was fixed in PR https://github.com/devtron-labs/devtron/pull/5215/files
+--- But as we cache the reference chart in DB, we need to clear that cache for those charts
+UPDATE charts
+SET reference_chart = NULL
+WHERE id IN (
+    SELECT charts.id FROM charts
+    INNER JOIN chart_ref ON (charts.chart_ref_id = chart_ref.id)
+    INNER JOIN app ON (charts.app_id = app.id AND app.active = true)
+    WHERE charts.active = true
+      AND chart_ref.version IN ('4.18.0', '4.19.0', '1.0.0', '1.1.0')
+      AND chart_ref.name = 'Deployment'
+);