fix: pvc mounted on pods for cache handling (#2912)

* first cut pvc for ci

* isPvcMounted flag introduced

* cache invalidate and global invalidate handling

* cache path made configurable

* pvc mounted false

* PVC mounted at two new paths for storing build and buildx cache

* mnor fix

* removed redundant objects from struct

* chenged names of env variables for paths

* buildx cache path restored after doing POC

---------

Co-authored-by: Prashant Ghildiyal <[email protected]>
Co-authored-by: Kripansh <[email protected]>

Author: 3 people

pkg/pipeline/CiConfig.go

@@ -79,6 +79,9 @@ type CiConfig struct {
 	CiRunnerDockerMTUValue         int                          `env:"CI_RUNNER_DOCKER_MTU_VALUE" envDefault:"-1"`
 	IgnoreDockerCacheForCI         bool                         `env:"CI_IGNORE_DOCKER_CACHE"`
 	VolumeMountsForCiJson          string                       `env:"CI_VOLUME_MOUNTS_JSON"`
+	BuildPvcCachePath              string                       `env:"PRE_CI_CACHE_PATH" envDefault:"/devtroncd-cache"`
+	DefaultPvcCachePath            string                       `env:"DOCKER_BUILD_CACHE_PATH" envDefault:"/var/lib/docker"`
+	BuildxPvcCachePath             string                       `env:"BUILDX_CACHE_PATH" envDefault:"/var/lib/devtron/buildx"`
 	ClusterConfig                  *rest.Config
 	NodeLabel                      map[string]string
 }

pkg/pipeline/CiService.go

@@ -480,7 +480,8 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.
 		CiBuildConfig:              ciBuildConfigBean,
 		CiBuildDockerMtuValue:      impl.ciConfig.CiRunnerDockerMTUValue,
 		IgnoreDockerCachePush:      impl.ciConfig.IgnoreDockerCacheForCI,
-		IgnoreDockerCachePull:      impl.ciConfig.IgnoreDockerCacheForCI || trigger.InvalidateCache,
+		IgnoreDockerCachePull:      impl.ciConfig.IgnoreDockerCacheForCI,
+		CacheInvalidate:            trigger.InvalidateCache,
 	}
 
 	if ciWorkflowConfig.LogsBucket == "" {

pkg/pipeline/WorkflowService.go

@@ -20,6 +20,7 @@ package pipeline
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	"github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned"
 	v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
@@ -37,6 +38,7 @@ import (
 	"k8s.io/client-go/rest"
 	"net/url"
 	"strconv"
+	"strings"
 )
 
 type WorkflowService interface {
@@ -112,6 +114,8 @@ type WorkflowRequest struct {
 	CiBuildDockerMtuValue      int                               `json:"ciBuildDockerMtuValue"`
 	IgnoreDockerCachePush      bool                              `json:"ignoreDockerCachePush"`
 	IgnoreDockerCachePull      bool                              `json:"ignoreDockerCachePull"`
+	CacheInvalidate            bool                              `json:"cacheInvalidate"`
+	IsPvcMounted               bool                              `json:"IsPvcMounted"`
 }
 
 const (
@@ -122,6 +126,8 @@ const (
 	CI_WORKFLOW_NAME               = "ci"
 	CI_WORKFLOW_WITH_STAGES        = "ci-stages-with-env"
 	CI_NODE_SELECTOR_APP_LABEL_KEY = "devtron.ai/node-selector"
+	CI_NODE_PVC_ALL_ENV            = "devtron.ai/ci-pvc-all"
+	CI_NODE_PVC_PIPELINE_PREFIX    = "devtron.ai/ci-pvc"
 )
 
 type ContainerResources struct {
@@ -194,7 +200,15 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest
 		miniCred := []v12.EnvVar{{Name: "AWS_ACCESS_KEY_ID", Value: impl.ciConfig.BlobStorageS3AccessKey}, {Name: "AWS_SECRET_ACCESS_KEY", Value: impl.ciConfig.BlobStorageS3SecretKey}}
 		containerEnvVariables = append(containerEnvVariables, miniCred...)
 	}
-
+	pvc := appLabels[strings.ToLower(fmt.Sprintf("%s-%s", CI_NODE_PVC_PIPELINE_PREFIX, workflowRequest.PipelineName))]
+	if len(pvc) == 0 {
+		pvc = appLabels[CI_NODE_PVC_ALL_ENV]
+	}
+	if len(pvc) != 0 {
+		workflowRequest.IsPvcMounted = true
+		workflowRequest.IgnoreDockerCachePush = true
+		workflowRequest.IgnoreDockerCachePull = true
+	}
 	ciCdTriggerEvent := CiCdTriggerEvent{
 		Type:      ciEvent,
 		CiRequest: workflowRequest,
@@ -543,6 +557,36 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest
 		}
 	}
 
+	// pvc mounting starts
+	if len(pvc) != 0 {
+		buildPvcCachePath := impl.ciConfig.BuildPvcCachePath
+		buildxPvcCachePath := impl.ciConfig.BuildxPvcCachePath
+		defaultPvcCachePath := impl.ciConfig.DefaultPvcCachePath
+
+		ciTemplate.Volumes = append(ciTemplate.Volumes, v12.Volume{
+			Name: "root-vol",
+			VolumeSource: v12.VolumeSource{
+				PersistentVolumeClaim: &v12.PersistentVolumeClaimVolumeSource{
+					ClaimName: pvc,
+					ReadOnly:  false,
+				},
+			},
+		})
+		ciTemplate.Container.VolumeMounts = append(ciTemplate.Container.VolumeMounts,
+			v12.VolumeMount{
+				Name:      "root-vol",
+				MountPath: buildPvcCachePath,
+			},
+			v12.VolumeMount{
+				Name:      "root-vol",
+				MountPath: buildxPvcCachePath,
+			},
+			v12.VolumeMount{
+				Name:      "root-vol",
+				MountPath: defaultPvcCachePath,
+			})
+	}
+
 	// node selector
 	if val, ok := appLabels[CI_NODE_SELECTOR_APP_LABEL_KEY]; ok {
 		var nodeSelectors map[string]string