feat: Reload materials api added

client/gitSensor/GitSensorClient.go

@@ -34,6 +34,7 @@ type Client interface {
 	GetCommitMetadata(ctx context.Context, req *CommitMetadataRequest) (*GitCommit, error)
 	GetCommitMetadataForPipelineMaterial(ctx context.Context, req *CommitMetadataRequest) (*GitCommit, error)
 	RefreshGitMaterial(ctx context.Context, req *RefreshGitMaterialRequest) (*RefreshGitMaterialResponse, error)
+	ReloadMaterials(ctx context.Context, reloadMaterials *ReloadMaterialsDto) error
 
 	GetWebhookData(ctx context.Context, req *WebhookDataRequest) (*WebhookAndCiData, error)
 	GetAllWebhookEventConfigForHost(ctx context.Context, req *WebhookEventConfigRequest) ([]*WebhookEventConfig, error)
@@ -145,3 +146,6 @@ func (c *ClientImpl) GetWebhookPayloadDataForPipelineMaterialId(ctx context.Cont
 func (c *ClientImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(ctx context.Context, req *WebhookPayloadFilterDataRequest) (*WebhookPayloadFilterDataResponse, error) {
 	return c.apiClient.GetWebhookPayloadFilterDataForPipelineMaterialId(ctx, req)
 }
+func (c *ClientImpl) ReloadMaterials(ctx context.Context, req *ReloadMaterialsDto) error {
+	return c.apiClient.ReloadMaterials(ctx, req)
+}

client/gitSensor/GitSensorGrpcClient.go

@@ -44,6 +44,7 @@ type ApiClient interface {
 	GetCommitMetadata(ctx context.Context, req *CommitMetadataRequest) (*GitCommit, error)
 	GetCommitMetadataForPipelineMaterial(ctx context.Context, req *CommitMetadataRequest) (*GitCommit, error)
 	RefreshGitMaterial(ctx context.Context, req *RefreshGitMaterialRequest) (*RefreshGitMaterialResponse, error)
+	ReloadMaterials(ctx context.Context, reloadMaterials *ReloadMaterialsDto) error
 
 	GetWebhookData(ctx context.Context, req *WebhookDataRequest) (*WebhookAndCiData, error)
 	GetAllWebhookEventConfigForHost(ctx context.Context, req *WebhookEventConfigRequest) ([]*WebhookEventConfig, error)
@@ -154,6 +155,7 @@ func (client *GrpcApiClientImpl) AddRepo(ctx context.Context, materials []*GitMa
 				CheckoutMsgAny:   item.CheckoutMsgAny,
 				Deleted:          item.Deleted,
 				FilterPattern:    item.FilterPattern,
+				CloningMode:      item.CloningMode,
 			})
 		}
 	}
@@ -184,6 +186,7 @@ func (client *GrpcApiClientImpl) UpdateRepo(ctx context.Context, material *GitMa
 		CheckoutMsgAny:   material.CheckoutMsgAny,
 		Deleted:          material.Deleted,
 		FilterPattern:    material.FilterPattern,
+		CloningMode:      material.CloningMode,
 	}
 
 	_, err = serviceClient.UpdateRepo(ctx, mappedMaterial)
@@ -756,3 +759,21 @@ func (client *GrpcApiClientImpl) mapGitCommitToProtoType(commit *GitCommit) (*pb
 	}
 	return mappedRes, nil
 }
+
+func (client *GrpcApiClientImpl) ReloadMaterials(ctx context.Context, reloadMaterials *ReloadMaterialsDto) error {
+
+	serviceClient, err := client.getGitSensorServiceClient()
+	if err != nil {
+		return err
+	}
+	req := pb.ReloadMaterialsRequest{}
+	for _, reloadMaterial := range reloadMaterials.ReloadMaterial {
+		tmpRel := pb.ReloadMaterial{MaterialId: reloadMaterial.GitmaterialId, CloningMode: reloadMaterial.CloningMode}
+		req.ReloadMaterials = append(req.ReloadMaterials, &tmpRel)
+	}
+	_, err = serviceClient.ReloadMaterials(ctx, &req)
+	if err != nil {
+		return err
+	}
+	return nil
+}

client/gitSensor/GitSensorRestClient.go

@@ -87,6 +87,7 @@ type GitMaterial struct {
 	Deleted          bool
 	FetchSubmodules  bool
 	FilterPattern    []string
+	CloningMode      string
 }
 type GitProvider struct {
 	Id            int
@@ -110,6 +111,16 @@ type GitCommit struct {
 	Excluded    bool
 }
 
+type ReloadMaterialsDto struct {
+	ReloadMaterial []ReloadMaterialDto
+}
+
+type ReloadMaterialDto struct {
+	AppId         int    `json:"appId"`
+	GitmaterialId int64  `json:"gitmaterialId"`
+	CloningMode   string `json:"cloningMode"`
+}
+
 type WebhookAndCiData struct {
 	ExtraEnvironmentVariables map[string]string `json:"extraEnvironmentVariables"` // extra env variables which will be used for CI
 	WebhookData               *WebhookData      `json:"webhookData"`
@@ -399,3 +410,9 @@ func (session RestClientImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(c
 	_, _, err = session.doRequest(request)
 	return response, err
 }
+
+func (session RestClientImpl) ReloadMaterials(ctx context.Context, reloadMaterials *ReloadMaterialsDto) error {
+	request := &ClientRequest{Method: GET, RequestBody: reloadMaterials, Path: "/admin/reload/materials"}
+	_, _, err := session.doRequest(request)
+	return err
+}

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/devtron-labs/authenticator v0.4.35-0.20240607135426-c86e868ecee1
 	github.com/devtron-labs/common-lib v0.0.20
 	github.com/devtron-labs/go-bitbucket v0.9.60-beta
-	github.com/devtron-labs/protos v0.0.3-0.20240326053929-48e42d9d4534
+	github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1
 	github.com/evanphx/json-patch v5.7.0+incompatible
 	github.com/gammazero/workerpool v1.1.3
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32

go.sum

@@ -189,8 +189,8 @@ github.com/devtron-labs/common-lib v0.0.20 h1:PH33VGmXZFx7U+rwXYnNXkB90sGbBTW0KH
 github.com/devtron-labs/common-lib v0.0.20/go.mod h1:UZGPt1ep9Tnd9Ak2sibGSiLr7p3ijO2/JLT+h+pqBuU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
-github.com/devtron-labs/protos v0.0.3-0.20240326053929-48e42d9d4534 h1:TElPRU69QedW7DIQiiQxtjwSQ6cK0fCTAMGvSLhP0ac=
-github.com/devtron-labs/protos v0.0.3-0.20240326053929-48e42d9d4534/go.mod h1:ypUknVph8Ph4dxSlrFoouf7wLedQxHku2LQwgRrdgS4=
+github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1 h1:R6qVeFaayqstBSu4w+ipWQqJyMKDqBVV3a11qoA2IaM=
+github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1/go.mod h1:ypUknVph8Ph4dxSlrFoouf7wLedQxHku2LQwgRrdgS4=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=

pkg/pipeline/CiCdPipelineOrchestrator.go

@@ -1230,7 +1230,7 @@ func (impl CiCdPipelineOrchestratorImpl) DeleteApp(appId int, userId int32) erro
 
 	impl.logger.Debug("deleting materials in git_sensor")
 	for _, m := range materials {
-		err = impl.updateRepositoryToGitSensor(m)
+		err = impl.updateRepositoryToGitSensor(m, "")
 		if err != nil {
 			impl.logger.Errorw("error in updating to git-sensor", "err", err)
 			return err
@@ -1308,7 +1308,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateMaterials(createMaterialRequest *
 		}
 		materials = append(materials, inputMaterial)
 	}
-	err = impl.addRepositoryToGitSensor(materials)
+	err = impl.addRepositoryToGitSensor(materials, "")
 	if err != nil {
 		impl.logger.Errorw("error in updating to sensor", "err", err)
 		return nil, err
@@ -1334,7 +1334,7 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateMaterial(updateMaterialDTO *bean.
 		return nil, err
 	}
 
-	err = impl.updateRepositoryToGitSensor(updatedMaterial)
+	err = impl.updateRepositoryToGitSensor(updatedMaterial, "")
 	if err != nil {
 		impl.logger.Errorw("error in updating to git-sensor", "err", err)
 		return nil, err
@@ -1347,7 +1347,7 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateMaterial(updateMaterialDTO *bean.
 	return updateMaterialDTO, nil
 }
 
-func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *pipelineConfig.GitMaterial) error {
+func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *pipelineConfig.GitMaterial, cloningMode string) error {
 	sensorMaterial := &gitSensor.GitMaterial{
 		Name:             material.Name,
 		Url:              material.Url,
@@ -1357,11 +1357,12 @@ func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *p
 		Deleted:          !material.Active,
 		FetchSubmodules:  material.FetchSubmodules,
 		FilterPattern:    material.FilterPattern,
+		CloningMode:      cloningMode,
 	}
 	return impl.GitSensorClient.UpdateRepo(context.Background(), sensorMaterial)
 }
 
-func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*bean.GitMaterial) error {
+func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*bean.GitMaterial, cloningMode string) error {
 	var sensorMaterials []*gitSensor.GitMaterial
 	for _, material := range materials {
 		sensorMaterial := &gitSensor.GitMaterial{
@@ -1372,6 +1373,7 @@ func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*b
 			Deleted:         false,
 			FetchSubmodules: material.FetchSubmodules,
 			FilterPattern:   material.FilterPattern,
+			CloningMode:     cloningMode,
 		}
 		sensorMaterials = append(sensorMaterials, sensorMaterial)
 	}

pkg/resourceQualifiers/serviceBean.go

@@ -120,7 +120,7 @@ func GetValuesFromSelectionIdentifier(selector QualifierSelector, selectionIdent
 		return 0, ""
 	}
 }
-func getAuditLog(userid int32) sql.AuditLog {
+func GetAuditLog(userid int32) sql.AuditLog {
 	auditLog := sql.AuditLog{
 		CreatedOn: time.Now(),
 		CreatedBy: userid,
@@ -140,6 +140,6 @@ func (selection *ResourceMappingSelection) toResourceMapping(selector QualifierS
 		IdentifierValueString: valueString,
 		Active:                true,
 		CompositeKey:          compositeString,
-		AuditLog:              getAuditLog(userId),
+		AuditLog:              GetAuditLog(userId),
 	}
 }

pkg/variables/ScopedVariableService.go

@@ -36,7 +36,6 @@ import (
 	"regexp"
 	"strings"
 	"sync"
-	"time"
 )
 
 type ScopedVariableService interface {
@@ -45,6 +44,8 @@ type ScopedVariableService interface {
 	GetJsonForVariables() (*models.Payload, error)
 	CheckForSensitiveVariables(variableNames []string) (map[string]bool, error)
 	GetFormattedVariableForName(name string) string
+	GetMatchedScopedVariables(varScope []*resourceQualifiers.QualifierMapping) map[int][]*resourceQualifiers.QualifierMapping
+	GetScopeWithPriority(variableIdToVariableScopes map[int][]*resourceQualifiers.QualifierMapping) map[int]int
 }
 
 type ScopedVariableServiceImpl struct {
@@ -151,7 +152,7 @@ func (impl *ScopedVariableServiceImpl) CreateVariables(payload models.Payload) e
 		return err
 	}
 
-	auditLog := getAuditLog(payload)
+	auditLog := resourceQualifiers.GetAuditLog(payload.UserId)
 	// Begin Transaction
 	tx, err := impl.scopedVariableRepository.StartTx()
 	if err != nil {
@@ -281,7 +282,7 @@ func (impl *ScopedVariableServiceImpl) createVariableScopes(payload models.Paylo
 	return scopeIdToVarData, nil
 }
 
-func (impl *ScopedVariableServiceImpl) getMatchedScopedVariables(varScope []*resourceQualifiers.QualifierMapping) map[int][]*resourceQualifiers.QualifierMapping {
+func (impl *ScopedVariableServiceImpl) GetMatchedScopedVariables(varScope []*resourceQualifiers.QualifierMapping) map[int][]*resourceQualifiers.QualifierMapping {
 	variableIdToVariableScopes := make(map[int][]*resourceQualifiers.QualifierMapping)
 	for _, vScope := range varScope {
 		variableId := vScope.ResourceId
@@ -315,7 +316,7 @@ func (impl *ScopedVariableServiceImpl) getMatchedScopedVariables(varScope []*res
 
 }
 
-func (impl *ScopedVariableServiceImpl) getScopeWithPriority(variableIdToVariableScopes map[int][]*resourceQualifiers.QualifierMapping) map[int]int {
+func (impl *ScopedVariableServiceImpl) GetScopeWithPriority(variableIdToVariableScopes map[int][]*resourceQualifiers.QualifierMapping) map[int]int {
 	variableIdToSelectedScopeId := make(map[int]int)
 	var minScope *resourceQualifiers.QualifierMapping
 	for variableId, scopes := range variableIdToVariableScopes {
@@ -337,7 +338,7 @@ func (impl *ScopedVariableServiceImpl) selectScopeForCompoundQualifier(scopes []
 		if scope.ParentIdentifier > 0 {
 			parentIdToChildScopes[scope.ParentIdentifier] = append(parentIdToChildScopes[scope.ParentIdentifier], scope)
 		} else {
-			//is parent so collect IDs and put it in a map for easy retrieval
+			// is parent so collect IDs and put it in a map for easy retrieval
 			parentScopeIds = append(parentScopeIds, scope.Id)
 			parentScopeIdToScope[scope.Id] = scope
 		}
@@ -419,8 +420,8 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie
 		return nil, err
 	}
 
-	matchedScopes := impl.getMatchedScopedVariables(varScope)
-	variableIdToSelectedScopeId := impl.getScopeWithPriority(matchedScopes)
+	matchedScopes := impl.GetMatchedScopedVariables(varScope)
+	variableIdToSelectedScopeId := impl.GetScopeWithPriority(matchedScopes)
 
 	scopeIds := make([]int, 0)
 	foundVarIds := make([]int, 0) // the variable IDs which have data
@@ -684,13 +685,3 @@ func (impl *ScopedVariableServiceImpl) getVariableScopeData(scopeIds []int) (map
 	}
 	return scopeIdVsVarDataMap, nil
 }
-
-func getAuditLog(payload models.Payload) sql.AuditLog {
-	auditLog := sql.AuditLog{
-		CreatedOn: time.Now(),
-		CreatedBy: payload.UserId,
-		UpdatedOn: time.Now(),
-		UpdatedBy: payload.UserId,
-	}
-	return auditLog
-}

pkg/variables/helper/priorityManager.go

@@ -16,9 +16,7 @@
 
 package helper
 
-import (
-	"github.com/devtron-labs/devtron/pkg/resourceQualifiers"
-)
+import "github.com/devtron-labs/devtron/pkg/resourceQualifiers"
 
 func QualifierComparator(a, b resourceQualifiers.Qualifier) bool {
 	return GetPriority(a) < GetPriority(b)