feat: git material include exclude feature and partial commit search

api/restHandler/app/BuildPipelineRestHandler.go

@@ -52,6 +52,7 @@ type DevtronAppBuildMaterialRestHandler interface {
 	CreateMaterial(w http.ResponseWriter, r *http.Request)
 	UpdateMaterial(w http.ResponseWriter, r *http.Request)
 	FetchMaterials(w http.ResponseWriter, r *http.Request)
+	FetchMaterialsByMaterialId(w http.ResponseWriter, r *http.Request)
 	RefreshMaterials(w http.ResponseWriter, r *http.Request)
 	FetchMaterialInfo(w http.ResponseWriter, r *http.Request)
 	FetchChanges(w http.ResponseWriter, r *http.Request)
@@ -196,7 +197,8 @@ func (handler PipelineConfigRestHandlerImpl) UpdateBranchCiPipelinesWithRegex(w
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
 		return
 	}
-	resp, err := handler.ciHandler.FetchMaterialsByPipelineId(patchRequest.Id)
+	//if include/exclude configured showAll will include excluded materials also in list, if not configured it will ignore this flag
+	resp, err := handler.ciHandler.FetchMaterialsByPipelineId(patchRequest.Id, false)
 	if err != nil {
 		handler.Logger.Errorw("service err, FetchMaterials", "err", err, "pipelineId", patchRequest.Id)
 		common.WriteJsonResp(w, err, resp, http.StatusInternalServerError)
@@ -459,6 +461,7 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr
 	response["apiResponse"] = strconv.Itoa(resp)
 	common.WriteJsonResp(w, err, response, http.StatusOK)
 }
+
 func (handler PipelineConfigRestHandlerImpl) FetchMaterials(w http.ResponseWriter, r *http.Request) {
 	userId, err := handler.userAuthService.GetLoggedInUser(r)
 	if userId == 0 || err != nil {
@@ -471,6 +474,69 @@ func (handler PipelineConfigRestHandlerImpl) FetchMaterials(w http.ResponseWrite
 		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
 		return
 	}
+	v := r.URL.Query()
+	showAll := false
+	show := v.Get("showAll")
+	if len(show) > 0 {
+		showAll, err = strconv.ParseBool(show)
+		if err != nil {
+			showAll = true
+			err = nil
+			//ignore error, apply rbac by default
+		}
+	}
+	handler.Logger.Infow("request payload, FetchMaterials", "pipelineId", pipelineId)
+	ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId)
+	if err != nil {
+		handler.Logger.Errorw("service err, UpdateCiTemplate", "err", err, "pipelineId", pipelineId)
+		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
+	}
+	//RBAC
+	token := r.Header.Get("token")
+	object := handler.enforcerUtil.GetAppRBACNameByAppId(ciPipeline.AppId)
+	if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, object); !ok {
+		common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden)
+		return
+	}
+	//RBAC
+	resp, err := handler.ciHandler.FetchMaterialsByPipelineId(pipelineId, showAll)
+	if err != nil {
+		handler.Logger.Errorw("service err, FetchMaterials", "err", err, "pipelineId", pipelineId)
+		common.WriteJsonResp(w, err, resp, http.StatusInternalServerError)
+		return
+	}
+	common.WriteJsonResp(w, err, resp, http.StatusOK)
+}
+
+func (handler PipelineConfigRestHandlerImpl) FetchMaterialsByMaterialId(w http.ResponseWriter, r *http.Request) {
+	userId, err := handler.userAuthService.GetLoggedInUser(r)
+	if userId == 0 || err != nil {
+		common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized)
+		return
+	}
+	vars := mux.Vars(r)
+	pipelineId, err := strconv.Atoi(vars["pipelineId"])
+	if err != nil {
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	gitMaterialId, err := strconv.Atoi(vars["gitMaterialId"])
+	if err != nil {
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	v := r.URL.Query()
+	showAll := false
+	show := v.Get("showAll")
+	if len(show) > 0 {
+		showAll, err = strconv.ParseBool(show)
+		if err != nil {
+			showAll = true
+			err = nil
+			//ignore error, apply rbac by default
+		}
+	}
 	handler.Logger.Infow("request payload, FetchMaterials", "pipelineId", pipelineId)
 	ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId)
 	if err != nil {
@@ -486,7 +552,7 @@ func (handler PipelineConfigRestHandlerImpl) FetchMaterials(w http.ResponseWrite
 		return
 	}
 	//RBAC
-	resp, err := handler.ciHandler.FetchMaterialsByPipelineId(pipelineId)
+	resp, err := handler.ciHandler.FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId, gitMaterialId, showAll)
 	if err != nil {
 		handler.Logger.Errorw("service err, FetchMaterials", "err", err, "pipelineId", pipelineId)
 		common.WriteJsonResp(w, err, resp, http.StatusInternalServerError)
@@ -1191,6 +1257,17 @@ func (handler PipelineConfigRestHandlerImpl) FetchChanges(w http.ResponseWriter,
 		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
 		return
 	}
+	showAll := false
+	v := r.URL.Query()
+	show := v.Get("showAll")
+	if len(show) > 0 {
+		showAll, err = strconv.ParseBool(show)
+		if err != nil {
+			showAll = true
+			err = nil
+			//ignore error, apply rbac by default
+		}
+	}
 	handler.Logger.Infow("request payload, FetchChanges", "ciMaterialId", ciMaterialId, "pipelineId", pipelineId)
 	ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId)
 	if err != nil {
@@ -1209,6 +1286,7 @@ func (handler PipelineConfigRestHandlerImpl) FetchChanges(w http.ResponseWriter,
 
 	changeRequest := &gitSensor.FetchScmChangesRequest{
 		PipelineMaterialId: ciMaterialId,
+		ShowAll:            showAll,
 	}
 	changes, err := handler.gitSensorClient.FetchChanges(context.Background(), changeRequest)
 	if err != nil {

api/router/PipelineConfigRouter.go

@@ -98,6 +98,7 @@ func (router PipelineConfigRouterImpl) initPipelineConfigRouter(configRouter *mu
 
 	configRouter.Path("/{appId}/ci-pipeline/min").HandlerFunc(router.restHandler.GetCiPipelineMin).Methods("GET")
 	configRouter.Path("/ci-pipeline/{pipelineId}/material").HandlerFunc(router.restHandler.FetchMaterials).Methods("GET")
+	configRouter.Path("/ci-pipeline/{pipelineId}/material/{gitMaterialId}").HandlerFunc(router.restHandler.FetchMaterialsByMaterialId).Methods("GET")
 	configRouter.Path("/ci-pipeline/refresh-material/{gitMaterialId}").HandlerFunc(router.restHandler.RefreshMaterials).Methods("GET")
 
 	configRouter.Path("/{appId}/ci-pipeline/{pipelineId}/workflow/{workflowId}").HandlerFunc(router.restHandler.FetchWorkflowDetails).Methods("GET")

client/gitSensor/GitSensorGrpcClient.go

@@ -137,6 +137,7 @@ func (client *GrpcApiClientImpl) AddRepo(ctx context.Context, materials []*GitMa
 				CheckoutStatus:   item.CheckoutStatus,
 				CheckoutMsgAny:   item.CheckoutMsgAny,
 				Deleted:          item.Deleted,
+				FilterPattern:    item.FilterPattern,
 			})
 		}
 	}
@@ -166,6 +167,7 @@ func (client *GrpcApiClientImpl) UpdateRepo(ctx context.Context, material *GitMa
 		CheckoutStatus:   material.CheckoutStatus,
 		CheckoutMsgAny:   material.CheckoutMsgAny,
 		Deleted:          material.Deleted,
+		FilterPattern:    material.FilterPattern,
 	}
 
 	_, err = serviceClient.UpdateRepo(ctx, mappedMaterial)
@@ -214,6 +216,7 @@ func (client *GrpcApiClientImpl) FetchChanges(ctx context.Context, req *FetchScm
 		PipelineMaterialId: int64(req.PipelineMaterialId),
 		From:               req.From,
 		To:                 req.To,
+		ShowAll:            req.ShowAll,
 	})
 	if err != nil {
 		return nil, err

client/gitSensor/GitSensorRestClient.go

@@ -59,6 +59,7 @@ type FetchScmChangesRequest struct {
 	PipelineMaterialId int    `json:"pipelineMaterialId"`
 	From               string `json:"from"`
 	To                 string `json:"to"`
+	ShowAll            bool   `json:"showAll"`
 }
 type HeadRequest struct {
 	MaterialIds []int `json:"materialIds"`
@@ -86,6 +87,7 @@ type GitMaterial struct {
 	CheckoutMsgAny   string
 	Deleted          bool
 	FetchSubmodules  bool
+	FilterPattern    []string
 }
 type GitProvider struct {
 	Id            int
@@ -106,6 +108,7 @@ type GitCommit struct {
 	Message     string
 	Changes     []string
 	WebhookData *WebhookData
+	Excluded    bool
 }
 
 type WebhookAndCiData struct {

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53
 	github.com/devtron-labs/common-lib v0.0.0-20230407072229-d4f665f5ca12
-	github.com/devtron-labs/protos v0.0.0-20230307051313-ecb763b443a8
+	github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2
 	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/go-pg/pg v6.15.1+incompatible
@@ -26,6 +26,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github v17.0.0+incompatible
+	github.com/google/uuid v1.3.0
 	github.com/google/wire v0.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/schema v1.1.0
@@ -144,7 +145,6 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.6.0 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect

go.sum

@@ -284,8 +284,8 @@ github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 h1:o
 github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU=
 github.com/devtron-labs/common-lib v0.0.0-20230407072229-d4f665f5ca12 h1:tcwNgAWTzalItF5XT0WxicRvY4wzz/o0HYLMVLxsxjg=
 github.com/devtron-labs/common-lib v0.0.0-20230407072229-d4f665f5ca12/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k=
-github.com/devtron-labs/protos v0.0.0-20230307051313-ecb763b443a8 h1:B/mf/ojkLxUsLPnSzY8c5ffFbkU8RS11C1BbtB4jJd0=
-github.com/devtron-labs/protos v0.0.0-20230307051313-ecb763b443a8/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto=
+github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 h1:/IEIsJTxDZ3hv8uOoCaqdWCXqcv7nCAgX9AP/v84dUY=
+github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=

internal/sql/repository/pipelineConfig/CiPipelineMaterial.go

@@ -54,6 +54,7 @@ type CiPipelineMaterialRepository interface {
 	GetRegexByPipelineId(id int) ([]*CiPipelineMaterial, error)
 	CheckRegexExistsForMaterial(id int) bool
 	GetByPipelineIdForRegexAndFixed(id int) ([]*CiPipelineMaterial, error)
+	GetByPipelineIdAndGitMaterialId(id int, gitMaterialId int) ([]*CiPipelineMaterial, error)
 }
 
 type CiPipelineMaterialRepositoryImpl struct {
@@ -88,6 +89,19 @@ func (impl CiPipelineMaterialRepositoryImpl) GetByPipelineId(id int) ([]*CiPipel
 		Select()
 	return ciPipelineMaterials, err
 }
+
+func (impl CiPipelineMaterialRepositoryImpl) GetByPipelineIdAndGitMaterialId(id int, gitMaterialId int) ([]*CiPipelineMaterial, error) {
+	var ciPipelineMaterials []*CiPipelineMaterial
+	err := impl.dbConnection.Model(&ciPipelineMaterials).
+		Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "CiPipeline.CiTemplate.CiBuildConfig", "GitMaterial", "GitMaterial.GitProvider").
+		Where("ci_pipeline_material.ci_pipeline_id = ?", id).
+		Where("ci_pipeline_material.active = ?", true).
+		Where("ci_pipeline_material.type != ?", SOURCE_TYPE_BRANCH_REGEX).
+		Where("ci_pipeline_material.git_material_id =?", gitMaterialId).
+		Select()
+	return ciPipelineMaterials, err
+}
+
 func (impl CiPipelineMaterialRepositoryImpl) GetByPipelineIdForRegexAndFixed(id int) ([]*CiPipelineMaterial, error) {
 	var ciPipelineMaterials []*CiPipelineMaterial
 	err := impl.dbConnection.Model(&ciPipelineMaterials).

internal/sql/repository/pipelineConfig/MaterialRepository.go

@@ -44,6 +44,7 @@ type GitMaterial struct {
 	Name            string   `sql:"name, omitempty"`
 	CheckoutPath    string   `sql:"checkout_path, omitempty"`
 	FetchSubmodules bool     `sql:"fetch_submodules,notnull"`
+	FilterPattern   []string `sql:"filter_pattern"`
 	sql.AuditLog
 	App         *app.App
 	GitProvider *repository.GitProvider

pkg/bean/app.go

@@ -71,6 +71,7 @@ type GitMaterial struct {
 	CheckoutPath     string `json:"checkoutPath" validate:"checkout-path-component"`
 	FetchSubmodules  bool   `json:"fetchSubmodules"`
 	IsUsedInCiConfig bool   `json:"isUsedInCiConfig"`
+	FilterPattern    []string `json:"filterPattern"`
 }
 
 type CiMaterial struct {

pkg/bulkAction/BulkUpdateService.go

@@ -1357,7 +1357,9 @@ func (impl BulkUpdateServiceImpl) BulkBuildTrigger(request *BulkApplicationForEn
 				}
 				ciPipelineId = ciPipeline.ParentCiPipeline
 			}
-			materialResponse, err := impl.ciHandler.FetchMaterialsByPipelineId(ciPipelineId)
+
+			//if include/exclude configured showAll will include excluded materials also in list, if not configured it will ignore this flag
+			materialResponse, err := impl.ciHandler.FetchMaterialsByPipelineId(ciPipelineId, false)
 			if err != nil {
 				impl.logger.Errorw("error in fetching ci pipeline materials", "CiPipelineId", ciPipelineId, "err", err)
 				return nil, err

pkg/pipeline/CiCdPipelineOrchestrator.go

@@ -951,6 +951,7 @@ func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *p
 		CheckoutLocation: material.CheckoutPath,
 		Deleted:          !material.Active,
 		FetchSubmodules:  material.FetchSubmodules,
+		FilterPattern:    material.FilterPattern,
 	}
 	return impl.GitSensorClient.UpdateRepo(context.Background(), sensorMaterial)
 }
@@ -965,6 +966,7 @@ func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*b
 			GitProviderId:   material.GitProviderId,
 			Deleted:         false,
 			FetchSubmodules: material.FetchSubmodules,
+			FilterPattern:   material.FilterPattern,
 		}
 		sensorMaterials = append(sensorMaterials, sensorMaterial)
 	}
@@ -1107,6 +1109,7 @@ func (impl CiCdPipelineOrchestratorImpl) updateMaterial(updateMaterialDTO *bean.
 	currentMaterial.GitProviderId = updateMaterialDTO.Material.GitProviderId
 	currentMaterial.CheckoutPath = updateMaterialDTO.Material.CheckoutPath
 	currentMaterial.FetchSubmodules = updateMaterialDTO.Material.FetchSubmodules
+	currentMaterial.FilterPattern = updateMaterialDTO.Material.FilterPattern
 	currentMaterial.AuditLog = sql.AuditLog{UpdatedBy: updateMaterialDTO.UserId, CreatedBy: currentMaterial.CreatedBy, UpdatedOn: time.Now(), CreatedOn: currentMaterial.CreatedOn}
 
 	err = impl.materialRepository.UpdateMaterial(currentMaterial)
@@ -1132,6 +1135,7 @@ func (impl CiCdPipelineOrchestratorImpl) createMaterial(inputMaterial *bean.GitM
 		Active:          true,
 		CheckoutPath:    inputMaterial.CheckoutPath,
 		FetchSubmodules: inputMaterial.FetchSubmodules,
+		FilterPattern:   inputMaterial.FilterPattern,
 		AuditLog:        sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()},
 	}
 	err := impl.materialRepository.SaveMaterial(material)