Skip to content

Commit ad1b072

Browse files
ayu-devtronayu-devtron
authored
feat: Reload materials api added (#5182)
* reload materials api added
1 parent 48432fa commit ad1b072

File tree

13 files changed

+1057
-793
lines changed

13 files changed

+1057
-793
lines changed

‎client/gitSensor/GitSensorClient.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ type Client interface {
3434
GetCommitMetadata(ctx context.Context, req *CommitMetadataRequest) (*GitCommit, error)
3535
GetCommitMetadataForPipelineMaterial(ctx context.Context, req *CommitMetadataRequest) (*GitCommit, error)
3636
RefreshGitMaterial(ctx context.Context, req *RefreshGitMaterialRequest) (*RefreshGitMaterialResponse, error)
37+
ReloadMaterials(ctx context.Context, reloadMaterials *ReloadMaterialsDto) error
3738

3839
GetWebhookData(ctx context.Context, req *WebhookDataRequest) (*WebhookAndCiData, error)
3940
GetAllWebhookEventConfigForHost(ctx context.Context, req *WebhookEventConfigRequest) ([]*WebhookEventConfig, error)
@@ -145,3 +146,6 @@ func (c *ClientImpl) GetWebhookPayloadDataForPipelineMaterialId(ctx context.Cont
145146
func (c *ClientImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(ctx context.Context, req *WebhookPayloadFilterDataRequest) (*WebhookPayloadFilterDataResponse, error) {
146147
return c.apiClient.GetWebhookPayloadFilterDataForPipelineMaterialId(ctx, req)
147148
}
149+
func (c *ClientImpl) ReloadMaterials(ctx context.Context, req *ReloadMaterialsDto) error {
150+
return c.apiClient.ReloadMaterials(ctx, req)
151+
}

‎client/gitSensor/GitSensorGrpcClient.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ type ApiClient interface {
4444
GetCommitMetadata(ctx context.Context, req *CommitMetadataRequest) (*GitCommit, error)
4545
GetCommitMetadataForPipelineMaterial(ctx context.Context, req *CommitMetadataRequest) (*GitCommit, error)
4646
RefreshGitMaterial(ctx context.Context, req *RefreshGitMaterialRequest) (*RefreshGitMaterialResponse, error)
47+
ReloadMaterials(ctx context.Context, reloadMaterials *ReloadMaterialsDto) error
4748

4849
GetWebhookData(ctx context.Context, req *WebhookDataRequest) (*WebhookAndCiData, error)
4950
GetAllWebhookEventConfigForHost(ctx context.Context, req *WebhookEventConfigRequest) ([]*WebhookEventConfig, error)
@@ -154,6 +155,7 @@ func (client *GrpcApiClientImpl) AddRepo(ctx context.Context, materials []*GitMa
154155
CheckoutMsgAny: item.CheckoutMsgAny,
155156
Deleted: item.Deleted,
156157
FilterPattern: item.FilterPattern,
158+
CloningMode: item.CloningMode,
157159
})
158160
}
159161
}
@@ -184,6 +186,7 @@ func (client *GrpcApiClientImpl) UpdateRepo(ctx context.Context, material *GitMa
184186
CheckoutMsgAny: material.CheckoutMsgAny,
185187
Deleted: material.Deleted,
186188
FilterPattern: material.FilterPattern,
189+
CloningMode: material.CloningMode,
187190
}
188191

189192
_, err = serviceClient.UpdateRepo(ctx, mappedMaterial)
@@ -756,3 +759,21 @@ func (client *GrpcApiClientImpl) mapGitCommitToProtoType(commit *GitCommit) (*pb
756759
}
757760
return mappedRes, nil
758761
}
762+
763+
func (client *GrpcApiClientImpl) ReloadMaterials(ctx context.Context, reloadMaterials *ReloadMaterialsDto) error {
764+
765+
serviceClient, err := client.getGitSensorServiceClient()
766+
if err != nil {
767+
return err
768+
}
769+
req := pb.ReloadMaterialsRequest{}
770+
for _, reloadMaterial := range reloadMaterials.ReloadMaterial {
771+
tmpRel := pb.ReloadMaterial{MaterialId: reloadMaterial.GitmaterialId, CloningMode: reloadMaterial.CloningMode}
772+
req.ReloadMaterials = append(req.ReloadMaterials, &tmpRel)
773+
}
774+
_, err = serviceClient.ReloadMaterials(ctx, &req)
775+
if err != nil {
776+
return err
777+
}
778+
return nil
779+
}

‎client/gitSensor/GitSensorRestClient.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ type GitMaterial struct {
8787
Deleted bool
8888
FetchSubmodules bool
8989
FilterPattern []string
90+
CloningMode string
9091
}
9192
type GitProvider struct {
9293
Id int
@@ -110,6 +111,16 @@ type GitCommit struct {
110111
Excluded bool
111112
}
112113

114+
type ReloadMaterialsDto struct {
115+
ReloadMaterial []ReloadMaterialDto
116+
}
117+
118+
type ReloadMaterialDto struct {
119+
AppId int `json:"appId"`
120+
GitmaterialId int64 `json:"gitmaterialId"`
121+
CloningMode string `json:"cloningMode"`
122+
}
123+
113124
type WebhookAndCiData struct {
114125
ExtraEnvironmentVariables map[string]string `json:"extraEnvironmentVariables"` // extra env variables which will be used for CI
115126
WebhookData *WebhookData `json:"webhookData"`
@@ -399,3 +410,9 @@ func (session RestClientImpl) GetWebhookPayloadFilterDataForPipelineMaterialId(c
399410
_, _, err = session.doRequest(request)
400411
return response, err
401412
}
413+
414+
func (session RestClientImpl) ReloadMaterials(ctx context.Context, reloadMaterials *ReloadMaterialsDto) error {
415+
request := &ClientRequest{Method: GET, RequestBody: reloadMaterials, Path: "/admin/reload/materials"}
416+
_, _, err := session.doRequest(request)
417+
return err
418+
}

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ require (
2222
github.com/devtron-labs/authenticator v0.4.35-0.20240607135426-c86e868ecee1
2323
github.com/devtron-labs/common-lib v0.0.20
2424
github.com/devtron-labs/go-bitbucket v0.9.60-beta
25-
github.com/devtron-labs/protos v0.0.3-0.20240326053929-48e42d9d4534
25+
github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1
2626
github.com/evanphx/json-patch v5.7.0+incompatible
2727
github.com/gammazero/workerpool v1.1.3
2828
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,8 @@ github.com/devtron-labs/common-lib v0.0.20 h1:PH33VGmXZFx7U+rwXYnNXkB90sGbBTW0KH
189189
github.com/devtron-labs/common-lib v0.0.20/go.mod h1:UZGPt1ep9Tnd9Ak2sibGSiLr7p3ijO2/JLT+h+pqBuU=
190190
github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
191191
github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
192-
github.com/devtron-labs/protos v0.0.3-0.20240326053929-48e42d9d4534 h1:TElPRU69QedW7DIQiiQxtjwSQ6cK0fCTAMGvSLhP0ac=
193-
github.com/devtron-labs/protos v0.0.3-0.20240326053929-48e42d9d4534/go.mod h1:ypUknVph8Ph4dxSlrFoouf7wLedQxHku2LQwgRrdgS4=
192+
github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1 h1:R6qVeFaayqstBSu4w+ipWQqJyMKDqBVV3a11qoA2IaM=
193+
github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1/go.mod h1:ypUknVph8Ph4dxSlrFoouf7wLedQxHku2LQwgRrdgS4=
194194
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
195195
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
196196
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=

‎pkg/pipeline/CiCdPipelineOrchestrator.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1230,7 +1230,7 @@ func (impl CiCdPipelineOrchestratorImpl) DeleteApp(appId int, userId int32) erro
12301230

12311231
impl.logger.Debug("deleting materials in git_sensor")
12321232
for _, m := range materials {
1233-
err = impl.updateRepositoryToGitSensor(m)
1233+
err = impl.updateRepositoryToGitSensor(m, "")
12341234
if err != nil {
12351235
impl.logger.Errorw("error in updating to git-sensor", "err", err)
12361236
return err
@@ -1308,7 +1308,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateMaterials(createMaterialRequest *
13081308
}
13091309
materials = append(materials, inputMaterial)
13101310
}
1311-
err = impl.addRepositoryToGitSensor(materials)
1311+
err = impl.addRepositoryToGitSensor(materials, "")
13121312
if err != nil {
13131313
impl.logger.Errorw("error in updating to sensor", "err", err)
13141314
return nil, err
@@ -1334,7 +1334,7 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateMaterial(updateMaterialDTO *bean.
13341334
return nil, err
13351335
}
13361336

1337-
err = impl.updateRepositoryToGitSensor(updatedMaterial)
1337+
err = impl.updateRepositoryToGitSensor(updatedMaterial, "")
13381338
if err != nil {
13391339
impl.logger.Errorw("error in updating to git-sensor", "err", err)
13401340
return nil, err
@@ -1347,7 +1347,7 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateMaterial(updateMaterialDTO *bean.
13471347
return updateMaterialDTO, nil
13481348
}
13491349

1350-
func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *pipelineConfig.GitMaterial) error {
1350+
func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *pipelineConfig.GitMaterial, cloningMode string) error {
13511351
sensorMaterial := &gitSensor.GitMaterial{
13521352
Name: material.Name,
13531353
Url: material.Url,
@@ -1357,11 +1357,12 @@ func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *p
13571357
Deleted: !material.Active,
13581358
FetchSubmodules: material.FetchSubmodules,
13591359
FilterPattern: material.FilterPattern,
1360+
CloningMode: cloningMode,
13601361
}
13611362
return impl.GitSensorClient.UpdateRepo(context.Background(), sensorMaterial)
13621363
}
13631364

1364-
func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*bean.GitMaterial) error {
1365+
func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*bean.GitMaterial, cloningMode string) error {
13651366
var sensorMaterials []*gitSensor.GitMaterial
13661367
for _, material := range materials {
13671368
sensorMaterial := &gitSensor.GitMaterial{
@@ -1372,6 +1373,7 @@ func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*b
13721373
Deleted: false,
13731374
FetchSubmodules: material.FetchSubmodules,
13741375
FilterPattern: material.FilterPattern,
1376+
CloningMode: cloningMode,
13751377
}
13761378
sensorMaterials = append(sensorMaterials, sensorMaterial)
13771379
}

‎pkg/resourceQualifiers/serviceBean.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ func GetValuesFromSelectionIdentifier(selector QualifierSelector, selectionIdent
120120
return 0, ""
121121
}
122122
}
123-
func getAuditLog(userid int32) sql.AuditLog {
123+
func GetAuditLog(userid int32) sql.AuditLog {
124124
auditLog := sql.AuditLog{
125125
CreatedOn: time.Now(),
126126
CreatedBy: userid,
@@ -140,6 +140,6 @@ func (selection *ResourceMappingSelection) toResourceMapping(selector QualifierS
140140
IdentifierValueString: valueString,
141141
Active: true,
142142
CompositeKey: compositeString,
143-
AuditLog: getAuditLog(userId),
143+
AuditLog: GetAuditLog(userId),
144144
}
145145
}

‎pkg/variables/ScopedVariableService.go‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ import (
3636
"regexp"
3737
"strings"
3838
"sync"
39-
"time"
4039
)
4140

4241
type ScopedVariableService interface {
@@ -45,6 +44,8 @@ type ScopedVariableService interface {
4544
GetJsonForVariables() (*models.Payload, error)
4645
CheckForSensitiveVariables(variableNames []string) (map[string]bool, error)
4746
GetFormattedVariableForName(name string) string
47+
GetMatchedScopedVariables(varScope []*resourceQualifiers.QualifierMapping) map[int][]*resourceQualifiers.QualifierMapping
48+
GetScopeWithPriority(variableIdToVariableScopes map[int][]*resourceQualifiers.QualifierMapping) map[int]int
4849
}
4950

5051
type ScopedVariableServiceImpl struct {
@@ -151,7 +152,7 @@ func (impl *ScopedVariableServiceImpl) CreateVariables(payload models.Payload) e
151152
return err
152153
}
153154

154-
auditLog := getAuditLog(payload)
155+
auditLog := resourceQualifiers.GetAuditLog(payload.UserId)
155156
// Begin Transaction
156157
tx, err := impl.scopedVariableRepository.StartTx()
157158
if err != nil {
@@ -281,7 +282,7 @@ func (impl *ScopedVariableServiceImpl) createVariableScopes(payload models.Paylo
281282
return scopeIdToVarData, nil
282283
}
283284

284-
func (impl *ScopedVariableServiceImpl) getMatchedScopedVariables(varScope []*resourceQualifiers.QualifierMapping) map[int][]*resourceQualifiers.QualifierMapping {
285+
func (impl *ScopedVariableServiceImpl) GetMatchedScopedVariables(varScope []*resourceQualifiers.QualifierMapping) map[int][]*resourceQualifiers.QualifierMapping {
285286
variableIdToVariableScopes := make(map[int][]*resourceQualifiers.QualifierMapping)
286287
for _, vScope := range varScope {
287288
variableId := vScope.ResourceId
@@ -315,7 +316,7 @@ func (impl *ScopedVariableServiceImpl) getMatchedScopedVariables(varScope []*res
315316

316317
}
317318

318-
func (impl *ScopedVariableServiceImpl) getScopeWithPriority(variableIdToVariableScopes map[int][]*resourceQualifiers.QualifierMapping) map[int]int {
319+
func (impl *ScopedVariableServiceImpl) GetScopeWithPriority(variableIdToVariableScopes map[int][]*resourceQualifiers.QualifierMapping) map[int]int {
319320
variableIdToSelectedScopeId := make(map[int]int)
320321
var minScope *resourceQualifiers.QualifierMapping
321322
for variableId, scopes := range variableIdToVariableScopes {
@@ -337,7 +338,7 @@ func (impl *ScopedVariableServiceImpl) selectScopeForCompoundQualifier(scopes []
337338
if scope.ParentIdentifier > 0 {
338339
parentIdToChildScopes[scope.ParentIdentifier] = append(parentIdToChildScopes[scope.ParentIdentifier], scope)
339340
} else {
340-
//is parent so collect IDs and put it in a map for easy retrieval
341+
// is parent so collect IDs and put it in a map for easy retrieval
341342
parentScopeIds = append(parentScopeIds, scope.Id)
342343
parentScopeIdToScope[scope.Id] = scope
343344
}
@@ -419,8 +420,8 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie
419420
return nil, err
420421
}
421422

422-
matchedScopes := impl.getMatchedScopedVariables(varScope)
423-
variableIdToSelectedScopeId := impl.getScopeWithPriority(matchedScopes)
423+
matchedScopes := impl.GetMatchedScopedVariables(varScope)
424+
variableIdToSelectedScopeId := impl.GetScopeWithPriority(matchedScopes)
424425

425426
scopeIds := make([]int, 0)
426427
foundVarIds := make([]int, 0) // the variable IDs which have data
@@ -684,13 +685,3 @@ func (impl *ScopedVariableServiceImpl) getVariableScopeData(scopeIds []int) (map
684685
}
685686
return scopeIdVsVarDataMap, nil
686687
}
687-
688-
func getAuditLog(payload models.Payload) sql.AuditLog {
689-
auditLog := sql.AuditLog{
690-
CreatedOn: time.Now(),
691-
CreatedBy: payload.UserId,
692-
UpdatedOn: time.Now(),
693-
UpdatedBy: payload.UserId,
694-
}
695-
return auditLog
696-
}

‎pkg/variables/helper/priorityManager.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,7 @@
1616

1717
package helper
1818

19-
import (
20-
"github.com/devtron-labs/devtron/pkg/resourceQualifiers"
21-
)
19+
import "github.com/devtron-labs/devtron/pkg/resourceQualifiers"
2220

2321
func QualifierComparator(a, b resourceQualifiers.Qualifier) bool {
2422
return GetPriority(a) < GetPriority(b)

0 commit comments

Comments
 (0)