Skip to content

misc: GetDockerRegistryIdForCiPipeline #6231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 30, 2024
Merged

misc: GetDockerRegistryIdForCiPipeline #6231

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
970d0d1
fix: GetDockerRegistryIdForCiPipeline
prkhrkat Dec 30, 2024
758f179
nil handling
prkhrkat Dec 30, 2024
7dbf1df
nil handling
prkhrkat Dec 30, 2024
626ee68
Merge branch 'develop' into registry-fix
prkhrkat Dec 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/external-app/wire_gen.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

37 changes: 25 additions & 12 deletions pkg/app/AppListingService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,27 @@ package app
import (
"context"
"fmt"
argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean"
"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow"
repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository"
"github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics"
util2 "github.com/devtron-labs/devtron/util"
"net/http"
"strconv"
"strings"
"time"
"github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read"
"github.com/devtron-labs/common-lib/utils/k8s/health"
argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean"
"github.com/devtron-labs/devtron/internal/middleware"
"github.com/devtron-labs/devtron/internal/sql/repository/app"
"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow"
userrepository "github.com/devtron-labs/devtron/pkg/auth/user/repository"
ciConfig "github.com/devtron-labs/devtron/pkg/build/pipeline/read"
chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository"
repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository"
"github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics"
"github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read"
"github.com/devtron-labs/devtron/pkg/dockerRegistry"
util2 "github.com/devtron-labs/devtron/util"
"github.com/devtron-labs/devtron/util/argo"
errors2 "github.com/juju/errors"
"go.opentelemetry.io/otel"
"golang.org/x/exp/slices"
"net/http"
"strconv"
"strings"
"time"

"github.com/devtron-labs/devtron/api/bean"
application2 "github.com/devtron-labs/devtron/client/argocdServer/application"
Expand Down Expand Up @@ -145,6 +146,7 @@ type AppListingServiceImpl struct {
deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService
ciArtifactRepository repository.CiArtifactRepository
envConfigOverrideReadService read.EnvConfigOverrideService
ciPipelineConfigReadService ciConfig.CiPipelineConfigReadService
}

func NewAppListingServiceImpl(Logger *zap.SugaredLogger, appListingRepository repository.AppListingRepository,
Expand All @@ -156,7 +158,8 @@ func NewAppListingServiceImpl(Logger *zap.SugaredLogger, appListingRepository re
chartRepository chartRepoRepository.ChartRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository,
dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService, userRepository userrepository.UserRepository,
deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, ciArtifactRepository repository.CiArtifactRepository,
envConfigOverrideReadService read.EnvConfigOverrideService) *AppListingServiceImpl {
envConfigOverrideReadService read.EnvConfigOverrideService,
ciPipelineConfigReadService ciConfig.CiPipelineConfigReadService) *AppListingServiceImpl {
serviceImpl := &AppListingServiceImpl{
Logger: Logger,
appListingRepository: appListingRepository,
Expand All @@ -176,6 +179,7 @@ func NewAppListingServiceImpl(Logger *zap.SugaredLogger, appListingRepository re
deployedAppMetricsService: deployedAppMetricsService,
ciArtifactRepository: ciArtifactRepository,
envConfigOverrideReadService: envConfigOverrideReadService,
ciPipelineConfigReadService: ciPipelineConfigReadService,
}
return serviceImpl
}
Expand Down Expand Up @@ -780,7 +784,16 @@ func (impl AppListingServiceImpl) setIpAccessProvidedData(ctx context.Context, a
}

if ciPipeline != nil && ciPipeline.CiTemplate != nil && len(*ciPipeline.CiTemplate.DockerRegistryId) > 0 {
dockerRegistryId := ciPipeline.CiTemplate.DockerRegistryId
artifact, err := impl.ciArtifactRepository.Get(appDetailContainer.CiArtifactId)
if err != nil {
impl.Logger.Errorw("error in fetching ci artifact", "ciArtifactId", appDetailContainer.CiArtifactId, "error", err)
return bean.AppDetailContainer{}, err
}
dockerRegistryId, err := impl.ciPipelineConfigReadService.GetDockerRegistryIdForCiPipeline(ciPipelineId, artifact)
if err != nil {
impl.Logger.Errorw("error in fetching docker registry id", "ciPipelineId", ciPipelineId, "error", err)
return bean.AppDetailContainer{}, err
}
appDetailContainer.DockerRegistryId = *dockerRegistryId
if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 {
appDetailContainer.IsExternalCi = false
Expand Down
60 changes: 56 additions & 4 deletions pkg/build/pipeline/read/CiPipelineReadService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package read

import (
"errors"
"github.com/devtron-labs/devtron/internal/sql/repository"
"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
"github.com/devtron-labs/devtron/pkg/build/pipeline/read/adapter"
"github.com/devtron-labs/devtron/pkg/build/pipeline/read/bean"
Expand All @@ -32,20 +33,24 @@ type CiPipelineConfigReadService interface {
FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error)
GetChildrenCiCount(parentCiPipelineId int) (int, error)
GetCiPipelineById(ciPipelineId int) (*bean.CiPipelineMin, error)
GetDockerRegistryIdForCiPipeline(ciPipelineId int, artifact *repository.CiArtifact) (*string, error)
}

type CiPipelineConfigReadServiceImpl struct {
logger *zap.SugaredLogger
ciPipelineRepository pipelineConfig.CiPipelineRepository
logger *zap.SugaredLogger
ciPipelineRepository pipelineConfig.CiPipelineRepository
ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository
}

func NewCiPipelineConfigReadServiceImpl(
logger *zap.SugaredLogger,
ciPipelineRepository pipelineConfig.CiPipelineRepository,
ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository,
) *CiPipelineConfigReadServiceImpl {
return &CiPipelineConfigReadServiceImpl{
logger: logger,
ciPipelineRepository: ciPipelineRepository,
logger: logger,
ciPipelineRepository: ciPipelineRepository,
ciTemplateOverrideRepository: ciTemplateOverrideRepository,
}
}

Expand Down Expand Up @@ -85,3 +90,50 @@ func (impl *CiPipelineConfigReadServiceImpl) GetCiPipelineById(ciPipelineId int)
}
return adapter.NewCiPipelineMin(ciPipeline)
}

func (impl *CiPipelineConfigReadServiceImpl) GetDockerRegistryIdForCiPipeline(ciPipelineId int, artifact *repository.CiArtifact) (*string, error) {
ciPipeline, err := impl.ciPipelineRepository.FindById(ciPipelineId)
if err != nil {
impl.logger.Errorw("error in fetching ciPipeline", "ciPipelineId", ciPipelineId, "error", err)
return nil, err
}

if ciPipeline.IsExternal && ciPipeline.ParentCiPipeline == 0 {
impl.logger.Warn("Ignoring for external ci")
return nil, nil
}

if ciPipeline.CiTemplate == nil {
impl.logger.Warn("returning as ciPipeline.CiTemplate is found nil")
return nil, nil
}
var dockerRegistryId string
if artifact.DataSource == repository.POST_CI || artifact.DataSource == repository.PRE_CD || artifact.DataSource == repository.POST_CD {
// if image is generated by plugin at these stages
if artifact.CredentialsSourceType == repository.GLOBAL_CONTAINER_REGISTRY {
dockerRegistryId = artifact.CredentialSourceValue
}
} else if artifact.DataSource == repository.CI_RUNNER {
// if image is created by ci build
dockerRegistryId = *ciPipeline.CiTemplate.DockerRegistryId
if len(dockerRegistryId) == 0 {
impl.logger.Warn("returning as dockerRegistryId is found empty")
return nil, nil
}

if ciPipeline.IsDockerConfigOverridden {
//set dockerRegistryId value with the DockerRegistryId of the overridden dockerRegistry
ciPipId := ciPipelineId
if ciPipeline.ParentCiPipeline != 0 {
ciPipId = ciPipeline.ParentCiPipeline
}
ciTemplateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(ciPipId)
if err != nil {
impl.logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "ciPipelineId", ciPipelineId, "error", err)
return nil, err
}
dockerRegistryId = ciTemplateOverride.DockerRegistryId
}
}
return &dockerRegistryId, nil
}
61 changes: 7 additions & 54 deletions pkg/dockerRegistry/DockerRegistryIpsConfigService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ import (
"github.com/devtron-labs/common-lib/utils/k8s"
repository3 "github.com/devtron-labs/devtron/internal/sql/repository"
"github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry"
"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
util2 "github.com/devtron-labs/devtron/internal/util"
ciConfig "github.com/devtron-labs/devtron/pkg/build/pipeline/read"
"github.com/devtron-labs/devtron/pkg/cluster"
repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository"
"github.com/go-pg/pg"
Expand All @@ -46,22 +46,21 @@ type DockerRegistryIpsConfigServiceImpl struct {
dockerRegistryIpsConfigRepository repository.DockerRegistryIpsConfigRepository
k8sUtil *k8s.K8sServiceImpl
clusterService cluster.ClusterService
ciPipelineRepository pipelineConfig.CiPipelineRepository
dockerArtifactStoreRepository repository.DockerArtifactStoreRepository
ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository
ciPipelineConfigReadService ciConfig.CiPipelineConfigReadService
}

func NewDockerRegistryIpsConfigServiceImpl(logger *zap.SugaredLogger, dockerRegistryIpsConfigRepository repository.DockerRegistryIpsConfigRepository,
k8sUtil *k8s.K8sServiceImpl, clusterService cluster.ClusterService, ciPipelineRepository pipelineConfig.CiPipelineRepository,
dockerArtifactStoreRepository repository.DockerArtifactStoreRepository, ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository) *DockerRegistryIpsConfigServiceImpl {
k8sUtil *k8s.K8sServiceImpl, clusterService cluster.ClusterService,
dockerArtifactStoreRepository repository.DockerArtifactStoreRepository,
ciPipelineConfigReadService ciConfig.CiPipelineConfigReadService) *DockerRegistryIpsConfigServiceImpl {
return &DockerRegistryIpsConfigServiceImpl{
logger: logger,
dockerRegistryIpsConfigRepository: dockerRegistryIpsConfigRepository,
k8sUtil: k8sUtil,
clusterService: clusterService,
ciPipelineRepository: ciPipelineRepository,
dockerArtifactStoreRepository: dockerArtifactStoreRepository,
ciTemplateOverrideRepository: ciTemplateOverrideRepository,
ciPipelineConfigReadService: ciPipelineConfigReadService,
}
}

Expand Down Expand Up @@ -91,7 +90,7 @@ func (impl DockerRegistryIpsConfigServiceImpl) HandleImagePullSecretOnApplicatio
return valuesFileContent, nil
}

dockerRegistryId, err := impl.getDockerRegistryIdForCiPipeline(ciPipelineId, artifact)
dockerRegistryId, err := impl.ciPipelineConfigReadService.GetDockerRegistryIdForCiPipeline(ciPipelineId, artifact)
if err != nil {
impl.logger.Errorw("error in getting docker registry", "dockerRegistryId", dockerRegistryId, "error", err)
return valuesFileContent, err
Expand Down Expand Up @@ -144,52 +143,6 @@ func (impl DockerRegistryIpsConfigServiceImpl) HandleImagePullSecretOnApplicatio
return updatedValuesFileContent, nil
}

func (impl DockerRegistryIpsConfigServiceImpl) getDockerRegistryIdForCiPipeline(ciPipelineId int, artifact *repository3.CiArtifact) (*string, error) {
ciPipeline, err := impl.ciPipelineRepository.FindById(ciPipelineId)
if err != nil {
impl.logger.Errorw("error in fetching ciPipeline", "ciPipelineId", ciPipelineId, "error", err)
return nil, err
}

if ciPipeline.IsExternal && ciPipeline.ParentCiPipeline == 0 {
impl.logger.Warn("Ignoring for external ci")
return nil, nil
}

if ciPipeline.CiTemplate == nil {
impl.logger.Warn("returning as ciPipeline.CiTemplate is found nil")
return nil, nil
}
var dockerRegistryId string
if artifact.DataSource == repository3.POST_CI || artifact.DataSource == repository3.PRE_CD || artifact.DataSource == repository3.POST_CD {
// if image is generated by plugin at these stages
if artifact.CredentialsSourceType == repository3.GLOBAL_CONTAINER_REGISTRY {
dockerRegistryId = artifact.CredentialSourceValue
}
} else if artifact.DataSource == repository3.CI_RUNNER {
// if image is created by ci build
dockerRegistryId = *ciPipeline.CiTemplate.DockerRegistryId
if len(dockerRegistryId) == 0 {
impl.logger.Warn("returning as dockerRegistryId is found empty")
return nil, nil
}

if ciPipeline.IsDockerConfigOverridden {
//set dockerRegistryId value with the DockerRegistryId of the overridden dockerRegistry
ciPipId := ciPipelineId
if ciPipeline.ParentCiPipeline != 0 {
ciPipId = ciPipeline.ParentCiPipeline
}
ciTemplateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(ciPipId)
if err != nil {
impl.logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "ciPipelineId", ciPipelineId, "error", err)
return nil, err
}
dockerRegistryId = ciTemplateOverride.DockerRegistryId
}
}
return &dockerRegistryId, nil
}
func (impl DockerRegistryIpsConfigServiceImpl) createOrUpdateDockerRegistryImagePullSecret(clusterId int, namespace string, ipsName string, dockerRegistryBean *repository.DockerArtifactStore) error {
impl.logger.Infow("creating/updating ips", "ipsName", ipsName, "clusterId", clusterId)

Expand Down
Loading
Loading