From 56d3a3bce8fda332754c806b7fc1e6a1b028b209 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 24 Jan 2025 20:44:34 +0530 Subject: [PATCH 1/2] wip --- .../sql/repository/AppListingRepository.go | 4 +-- .../CiArtifactsListingQueryBuilder.go | 27 ++++++++++++------- .../AppListingRepositoryQueryBuilder.go | 8 +++--- .../pipelineConfig/CdWorfkflowRepository.go | 6 ++--- .../pipelineConfig/CiWorkflowRepository.go | 12 +++------ pkg/auth/user/repository/UserRepository.go | 10 ++++--- .../helper/UserRepositoryQueryBuilder.go | 15 ++++++----- 7 files changed, 46 insertions(+), 36 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 864f87921e..f8456932b2 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -308,10 +308,10 @@ func (impl AppListingRepositoryImpl) FetchAppsByEnvironmentV2(appListingFilter h // if any pipeline found get the latest deployment time if len(pipelineIds) > 0 { - query := impl.appListingRepositoryQueryBuilder.BuildAppListingQueryLastDeploymentTimeV2(pipelineIds) + query, queryParams := impl.appListingRepositoryQueryBuilder.BuildAppListingQueryLastDeploymentTimeV2(pipelineIds) impl.Logger.Debugw("basic app detail query: ", query) start := time.Now() - _, err := impl.dbConnection.Query(&lastDeployedTimeDTO, query) + _, err := impl.dbConnection.Query(&lastDeployedTimeDTO, query, queryParams...) middleware.AppListingDuration.WithLabelValues("buildAppListingQueryLastDeploymentTime", "devtron").Observe(time.Since(start).Seconds()) if err != nil { impl.Logger.Errorw("error in getting latest deployment time for given pipelines", "err", err, "pipelines", pipelineIds, "query", query) diff --git a/internal/sql/repository/CiArtifactsListingQueryBuilder.go b/internal/sql/repository/CiArtifactsListingQueryBuilder.go index f262fc3732..1227ade98a 100644 --- a/internal/sql/repository/CiArtifactsListingQueryBuilder.go +++ b/internal/sql/repository/CiArtifactsListingQueryBuilder.go @@ -20,24 +20,28 @@ import ( "fmt" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/helper" + "github.com/go-pg/pg" ) const EmptyLikeRegex = "%%" func BuildQueryForParentTypeCIOrWebhook(listingFilterOpts bean.ArtifactsListFilterOptions) string { - commonPaginatedQueryPart := fmt.Sprintf(" cia.image LIKE '%v'", listingFilterOpts.SearchString) + commonPaginatedQueryPart, commonPaginatedQueryParams := " cia.image LIKE '?'", []interface{}{listingFilterOpts.SearchString} orderByClause := " ORDER BY cia.id DESC" - limitOffsetQueryPart := fmt.Sprintf(" LIMIT %v OFFSET %v", listingFilterOpts.Limit, listingFilterOpts.Offset) + limitOffsetQueryPart, limitOffsetQueryParams := fmt.Sprintf(" LIMIT ? OFFSET ?"), []interface{}{listingFilterOpts.Limit, listingFilterOpts.Offset} finalQuery := "" + var finalQueryParams []interface{} + var remainingQueryParams []interface{} if listingFilterOpts.ParentStageType == bean.CI_WORKFLOW_TYPE { selectQuery := " SELECT cia.* " remainingQuery := " FROM ci_artifact cia" + " INNER JOIN ci_pipeline cp ON (cp.id=cia.pipeline_id or (cp.id=cia.component_id and cia.data_source='post_ci' ) )" + - " INNER JOIN pipeline p ON (p.ci_pipeline_id = cp.id and p.id=%v )" + + " INNER JOIN pipeline p ON (p.ci_pipeline_id = cp.id and p.id=? )" + " WHERE " - remainingQuery = fmt.Sprintf(remainingQuery, listingFilterOpts.PipelineId) + remainingQueryParams = []interface{}{listingFilterOpts.PipelineId} if len(listingFilterOpts.ExcludeArtifactIds) > 0 { - remainingQuery += fmt.Sprintf("cia.id NOT IN (%s) AND ", helper.GetCommaSepratedString(listingFilterOpts.ExcludeArtifactIds)) + remainingQuery += "cia.id NOT IN (?) AND " + remainingQueryParams = append(remainingQueryParams, pg.In(listingFilterOpts.ExcludeArtifactIds)) } countQuery := " SELECT count(cia.id) as total_count" @@ -47,18 +51,23 @@ func BuildQueryForParentTypeCIOrWebhook(listingFilterOpts bean.ArtifactsListFilt } else if listingFilterOpts.ParentStageType == bean.WEBHOOK_WORKFLOW_TYPE { selectQuery := " SELECT cia.* " remainingQuery := " FROM ci_artifact cia " + - " WHERE cia.external_ci_pipeline_id = %v AND " - remainingQuery = fmt.Sprintf(remainingQuery, listingFilterOpts.ParentId) + " WHERE cia.external_ci_pipeline_id = ? AND " + remainingQueryParams = []interface{}{listingFilterOpts.ParentId} if len(listingFilterOpts.ExcludeArtifactIds) > 0 { - remainingQuery += fmt.Sprintf("cia.id NOT IN (%s) AND ", helper.GetCommaSepratedString(listingFilterOpts.ExcludeArtifactIds)) + remainingQuery += "cia.id NOT IN (?) AND " + remainingQueryParams = append(remainingQueryParams, pg.In(listingFilterOpts.ExcludeArtifactIds)) } countQuery := " SELECT count(cia.id) as total_count" totalCountQuery := countQuery + remainingQuery + commonPaginatedQueryPart selectQuery = fmt.Sprintf("%s,(%s) ", selectQuery, totalCountQuery) finalQuery = selectQuery + remainingQuery + commonPaginatedQueryPart + orderByClause + limitOffsetQueryPart - } + finalQueryParams = append(finalQueryParams, remainingQueryParams...) + finalQueryParams = append(finalQueryParams, commonPaginatedQueryParams...) + finalQueryParams = append(finalQueryParams, remainingQueryParams...) + finalQueryParams = append(finalQueryParams, commonPaginatedQueryParams...) + finalQueryParams = append(finalQueryParams, limitOffsetQueryParams...) return finalQuery } diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index ae7d013696..b527185e19 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -160,15 +160,17 @@ func (impl AppListingRepositoryQueryBuilder) TestForCommonAppFilter(appListingFi return query, queryParams } -func (impl AppListingRepositoryQueryBuilder) BuildAppListingQueryLastDeploymentTimeV2(pipelineIDs []int) string { +func (impl AppListingRepositoryQueryBuilder) BuildAppListingQueryLastDeploymentTimeV2(pipelineIDs []int) (string, []interface{}) { whereCondition := "" + queryParams := []interface{}{} if len(pipelineIDs) > 0 { - whereCondition += fmt.Sprintf(" Where pco.pipeline_id IN (%s) ", GetCommaSepratedString(pipelineIDs)) + whereCondition += " Where pco.pipeline_id IN (?) " + queryParams = append(queryParams, pg.In(pipelineIDs)) } query := "select pco.pipeline_id , MAX(pco.created_on) as last_deployed_time" + " from pipeline_config_override pco" + whereCondition + " GROUP BY pco.pipeline_id;" - return query + return query, queryParams } func (impl AppListingRepositoryQueryBuilder) GetAppIdsQueryWithPaginationForLastDeployedSearch(appListingFilter AppListingFilter) (string, []interface{}) { diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 906dd3d326..6802285c73 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -19,7 +19,6 @@ package pipelineConfig import ( "context" "errors" - "fmt" apiBean "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -539,10 +538,9 @@ func (impl *CdWorkflowRepositoryImpl) FindBasicWorkflowRunnerById(wfrId int) (*C func (impl *CdWorkflowRepositoryImpl) FindRetriedWorkflowCountByReferenceId(wfrId int) (int, error) { retryCount := 0 - query := fmt.Sprintf("select count(id) "+ - "from cd_workflow_runner where ref_cd_workflow_runner_id = %v", wfrId) + query := "select count(id) from cd_workflow_runner where ref_cd_workflow_runner_id = ?" - _, err := impl.dbConnection.Query(&retryCount, query) + _, err := impl.dbConnection.Query(&retryCount, query, wfrId) return retryCount, err } diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 9593094a1f..5376228a3f 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -17,9 +17,7 @@ package pipelineConfig import ( - "fmt" "github.com/devtron-labs/devtron/internal/sql/constants" - "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/go-pg/pg" @@ -234,10 +232,9 @@ func (impl *CiWorkflowRepositoryImpl) FindById(id int) (*CiWorkflow, error) { func (impl *CiWorkflowRepositoryImpl) FindRetriedWorkflowCountByReferenceId(id int) (int, error) { retryCount := 0 - query := fmt.Sprintf("select count(*) "+ - "from ci_workflow where ref_ci_workflow_id = %v", id) + query := "select count(*) from ci_workflow where ref_ci_workflow_id = ?" - _, err := impl.dbConnection.Query(&retryCount, query) + _, err := impl.dbConnection.Query(&retryCount, query, id) return retryCount, err } @@ -337,9 +334,8 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowGitTriggersByArti } query := "SELECT cw.git_triggers,cw.id,cw.triggered_by,cw.ci_pipeline_id,cia.id as ci_artifact_id" + " FROM ci_workflow cw INNER JOIN ci_artifact cia on cia.ci_workflow_id = cw.id " + - " WHERE cia.id IN (%s)" - query = fmt.Sprintf(query, helper.GetCommaSepratedString(ciArtifactIds)) - _, err := impl.dbConnection.Query(&workflows, query) + " WHERE cia.id IN (?)" + _, err := impl.dbConnection.Query(&workflows, query, pg.In(ciArtifactIds)) return workflows, err } diff --git a/pkg/auth/user/repository/UserRepository.go b/pkg/auth/user/repository/UserRepository.go index 99274d9221..b5c3cc625b 100644 --- a/pkg/auth/user/repository/UserRepository.go +++ b/pkg/auth/user/repository/UserRepository.go @@ -181,9 +181,10 @@ func (impl UserRepositoryImpl) GetAllExecutingQuery(query string, queryParams [] func (impl UserRepositoryImpl) FetchActiveUserByEmail(email string) (bean.UserInfo, error) { var users bean.UserInfo + emailSearchQuery, queryParams := helper.GetEmailSearchQuery("u", email) query := fmt.Sprintf("SELECT u.id, u.email_id, u.access_token, u.user_type FROM users u"+ - " WHERE u.active = true and %s order by u.updated_on desc", helper.GetEmailSearchQuery("u", email)) - _, err := impl.dbConnection.Query(&users, query, email) + " WHERE u.active = true and %s order by u.updated_on desc", emailSearchQuery) + _, err := impl.dbConnection.Query(&users, query, queryParams...) if err != nil { impl.Logger.Errorw("Exception caught:", "err", err) return users, err @@ -197,12 +198,13 @@ func (impl UserRepositoryImpl) FetchUserDetailByEmail(email string) (bean.UserIn var users []bean.UserRole var userFinal bean.UserInfo + emailSearchQuery, queryParams := helper.GetEmailSearchQuery("u", email) query := fmt.Sprintf("SELECT u.id, u.email_id, u.user_type, r.role FROM users u"+ " INNER JOIN user_roles ur ON ur.user_id=u.id"+ " INNER JOIN roles r ON r.id=ur.role_id"+ " WHERE %s and u.active = true"+ - " ORDER BY u.updated_on desc;", helper.GetEmailSearchQuery("u", email)) - _, err := impl.dbConnection.Query(&users, query, email) + " ORDER BY u.updated_on desc;", emailSearchQuery) + _, err := impl.dbConnection.Query(&users, query, queryParams...) if err != nil { return userFinal, err } diff --git a/pkg/auth/user/repository/helper/UserRepositoryQueryBuilder.go b/pkg/auth/user/repository/helper/UserRepositoryQueryBuilder.go index 9dbf55d439..27a868770b 100644 --- a/pkg/auth/user/repository/helper/UserRepositoryQueryBuilder.go +++ b/pkg/auth/user/repository/helper/UserRepositoryQueryBuilder.go @@ -84,9 +84,11 @@ func GetQueryForGroupListingWithFilters(req *bean.ListingRequest) (string, []int orderCondition := "" if len(req.SortBy) > 0 && !req.CountCheck { - orderCondition += fmt.Sprintf(" order by %s ", req.SortBy) + orderCondition += " order by ? " + queryParams = append(queryParams, req.SortBy) if req.SortOrder == bean2.Desc { - orderCondition += fmt.Sprintf(" %s ", bean2.Desc) + orderCondition += " ? " + queryParams = append(queryParams, bean2.Desc) } } if req.Size > 0 && !req.CountCheck && !req.ShowAll { @@ -103,9 +105,10 @@ func GetQueryForGroupListingWithFilters(req *bean.ListingRequest) (string, []int } -func GetEmailSearchQuery(usersTableAlias string, emailId string) string { +func GetEmailSearchQuery(usersTableAlias string, emailId string) (string, []interface{}) { + queryParams := []interface{}{emailId, emailId} expression := fmt.Sprintf( - "( (%s.user_type is NULL and %s.email_id ILIKE '%s' ) or (%s.user_type='apiToken' and %s.email_id='%s') )", - usersTableAlias, usersTableAlias, emailId, usersTableAlias, usersTableAlias, emailId) - return expression + "( (%s.user_type is NULL and %s.email_id ILIKE '?' ) or (%s.user_type='apiToken' and %s.email_id='?') )", + usersTableAlias, usersTableAlias, usersTableAlias, usersTableAlias) + return expression, queryParams } From 4617e215203775ba22f0c02f15a82af6cdc5b967 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 29 Jan 2025 23:09:09 +0530 Subject: [PATCH 2/2] develop sync change --- internal/sql/repository/CiArtifactRepository.go | 4 ++-- internal/sql/repository/CiArtifactsListingQueryBuilder.go | 6 +++--- .../user/repository/helper/UserRepositoryQueryBuilder.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 2f3c41ae37..4f765c8eee 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -362,8 +362,8 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineV3(listingFilterOpt artifactsResp := make([]*CiArtifactWithExtraData, 0, listingFilterOpts.Limit) var artifacts []*CiArtifact totalCount := 0 - finalQuery := BuildQueryForParentTypeCIOrWebhook(*listingFilterOpts) - _, err := impl.dbConnection.Query(&artifactsResp, finalQuery) + finalQuery, finalQueryParams := BuildQueryForParentTypeCIOrWebhook(*listingFilterOpts) + _, err := impl.dbConnection.Query(&artifactsResp, finalQuery, finalQueryParams...) if err != nil { return nil, totalCount, err } diff --git a/internal/sql/repository/CiArtifactsListingQueryBuilder.go b/internal/sql/repository/CiArtifactsListingQueryBuilder.go index 1227ade98a..e13ea4632d 100644 --- a/internal/sql/repository/CiArtifactsListingQueryBuilder.go +++ b/internal/sql/repository/CiArtifactsListingQueryBuilder.go @@ -25,8 +25,8 @@ import ( const EmptyLikeRegex = "%%" -func BuildQueryForParentTypeCIOrWebhook(listingFilterOpts bean.ArtifactsListFilterOptions) string { - commonPaginatedQueryPart, commonPaginatedQueryParams := " cia.image LIKE '?'", []interface{}{listingFilterOpts.SearchString} +func BuildQueryForParentTypeCIOrWebhook(listingFilterOpts bean.ArtifactsListFilterOptions) (string, []interface{}) { + commonPaginatedQueryPart, commonPaginatedQueryParams := " cia.image LIKE ?", []interface{}{listingFilterOpts.SearchString} orderByClause := " ORDER BY cia.id DESC" limitOffsetQueryPart, limitOffsetQueryParams := fmt.Sprintf(" LIMIT ? OFFSET ?"), []interface{}{listingFilterOpts.Limit, listingFilterOpts.Offset} finalQuery := "" @@ -68,7 +68,7 @@ func BuildQueryForParentTypeCIOrWebhook(listingFilterOpts bean.ArtifactsListFilt finalQueryParams = append(finalQueryParams, remainingQueryParams...) finalQueryParams = append(finalQueryParams, commonPaginatedQueryParams...) finalQueryParams = append(finalQueryParams, limitOffsetQueryParams...) - return finalQuery + return finalQuery, finalQueryParams } func BuildQueryForArtifactsForCdStage(listingFilterOptions bean.ArtifactsListFilterOptions) string { diff --git a/pkg/auth/user/repository/helper/UserRepositoryQueryBuilder.go b/pkg/auth/user/repository/helper/UserRepositoryQueryBuilder.go index 27a868770b..6f344a87bf 100644 --- a/pkg/auth/user/repository/helper/UserRepositoryQueryBuilder.go +++ b/pkg/auth/user/repository/helper/UserRepositoryQueryBuilder.go @@ -108,7 +108,7 @@ func GetQueryForGroupListingWithFilters(req *bean.ListingRequest) (string, []int func GetEmailSearchQuery(usersTableAlias string, emailId string) (string, []interface{}) { queryParams := []interface{}{emailId, emailId} expression := fmt.Sprintf( - "( (%s.user_type is NULL and %s.email_id ILIKE '?' ) or (%s.user_type='apiToken' and %s.email_id='?') )", + "( (%s.user_type is NULL and %s.email_id ILIKE ? ) or (%s.user_type='apiToken' and %s.email_id=?) )", usersTableAlias, usersTableAlias, usersTableAlias, usersTableAlias) return expression, queryParams }