Skip to content

feat: added support for in app logging archival from CI runner #3467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Jun 8, 2023
Merged

feat: added support for in app logging archival from CI runner #3467

Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
3918bc3
archive logs sync
subhashish-devtron May 23, 2023
d983e69
added type for cdstage event
subhashish-devtron May 23, 2023
abbde0f
test
subhashish-devtron May 24, 2023
c86a9e7
cleanup
subhashish-devtron Jun 2, 2023
0151db1
adding log file path to db on trigger
subhashish-devtron May 25, 2023
6480248
cleanup
subhashish-devtron Jun 2, 2023
3ddae08
cleanup
subhashish-devtron Jun 2, 2023
ed91114
cleanup
subhashish-devtron Jun 2, 2023
68e830e
cleanup
subhashish-devtron Jun 2, 2023
11bb803
exposed pod name in cd workflow details
subhashish-devtron Jun 2, 2023
e8a8678
log path fix
subhashish-devtron Jun 6, 2023
bc7acec
fix
subhashish-devtron Jun 6, 2023
fa9cb50
addresing pr comments
subhashish-devtron Jun 7, 2023
3192aca
addresing pr comments2
subhashish-devtron Jun 7, 2023
e71f2b8
addresing pr comments3
subhashish-devtron Jun 7, 2023
d0ac413
fix
subhashish-devtron Jun 7, 2023
5343ee7
Merge branch 'main' into archive-logs-final
subhashish-devtron Jun 7, 2023
8b52a16
Update CiHandler.go
subhashish-devtron Jun 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/pipeline/ArgoWorkflowExecutor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func (impl *ArgoWorkflowExecutorImpl) ExecuteWorkflow(workflowTemplate bean.Work

func (impl *ArgoWorkflowExecutorImpl) updateBlobStorageConfig(workflowTemplate bean.WorkflowTemplate, cdTemplate *v1alpha1.Template) {
cdTemplate.ArchiveLocation = &v1alpha1.ArtifactLocation{
ArchiveLogs: &workflowTemplate.BlobStorageConfigured,
ArchiveLogs: &workflowTemplate.ArchiveLogs,
}
if workflowTemplate.BlobStorageConfigured {
var s3Artifact *v1alpha1.S3Artifact
Expand Down
1 change: 1 addition & 0 deletions pkg/pipeline/CdConfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ type CdConfig struct {
UseBlobStorageConfigInCdWorkflow bool `env:"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW" envDefault:"true"`
BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/"`
CdWorkflowExecutorType pipelineConfig.WorkflowExecutorType `env:"CD_WORKFLOW_EXECUTOR_TYPE" envDefault:"AWF"`
InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false"`
}

func GetCdConfig() (*CdConfig, error) {
Expand Down
3 changes: 2 additions & 1 deletion pkg/pipeline/CdHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -627,7 +627,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus
savedWorkflow.Message = message
savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time
savedWorkflow.Name = workflowName
savedWorkflow.LogLocation = wfStatusRs.LogLocation
// removed log location from here since we are saving it at trigger
savedWorkflow.PodName = podName
savedWorkflow.UpdatedOn = time.Now()
savedWorkflow.UpdatedBy = 1
Expand Down Expand Up @@ -902,6 +902,7 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int,
GitTriggers: gitTriggers,
BlobStorageEnabled: workflow.BlobStorageEnabled,
IsVirtualEnvironment: workflowR.CdWorkflow.Pipeline.Environment.IsVirtualEnvironment,
PodName: workflowR.PodName,
}
return workflowResponse, nil

Expand Down
19 changes: 15 additions & 4 deletions pkg/pipeline/CdWorkflowService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,9 @@ type CdWorkflowRequest struct {
CloudProvider blob_storage.BlobStorageType `json:"cloudProvider"`
AzureBlobConfig *blob_storage.AzureBlobConfig `json:"azureBlobConfig"`
GcpBlobConfig *blob_storage.GcpBlobConfig `json:"gcpBlobConfig"`
BlobStorageLogsKey string `json:"blobStorageLogsKey"`
InAppLoggingEnabled bool `json:"inAppLoggingEnabled"`
WorkflowPrefixForLog string `json:"workflowPrefixForLog"`
DefaultAddressPoolBaseCidr string `json:"defaultAddressPoolBaseCidr"`
DefaultAddressPoolSize int `json:"defaultAddressPoolSize"`
DeploymentTriggeredBy string `json:"deploymentTriggeredBy,omitempty"`
Expand Down Expand Up @@ -146,8 +149,13 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq
workflowRequest.IsExtRun = true
}
ciCdTriggerEvent := CiCdTriggerEvent{
Type: cdStage,
CdRequest: workflowRequest,
}

// key will be used for log archival through in-app logging
ciCdTriggerEvent.CdRequest.BlobStorageLogsKey = fmt.Sprintf("%s/%s", impl.cdConfig.DefaultBuildLogsKeyPrefix, workflowRequest.WorkflowPrefixForLog)
ciCdTriggerEvent.CdRequest.InAppLoggingEnabled = impl.cdConfig.InAppLoggingEnabled || (workflowRequest.WorkflowExecutor == pipelineConfig.WORKFLOW_EXECUTOR_TYPE_SYSTEM)
workflowJson, err := json.Marshal(&ciCdTriggerEvent)
if err != nil {
impl.Logger.Errorw("error occurred while marshalling ciCdTriggerEvent", "error", err)
Expand Down Expand Up @@ -223,6 +231,7 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq
workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.TaintKey, Value: impl.cdConfig.TaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}}
workflowTemplate.Volumes = ExtractVolumesFromCmCs(workflowConfigMaps, workflowSecrets)
workflowTemplate.ArchiveLogs = storageConfigured
workflowTemplate.ArchiveLogs = workflowTemplate.ArchiveLogs && !ciCdTriggerEvent.CdRequest.InAppLoggingEnabled
workflowTemplate.RestartPolicy = v12.RestartPolicyNever

if len(impl.cdConfig.NodeLabel) > 0 {
Expand All @@ -234,11 +243,13 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq
reqCpu := impl.cdConfig.ReqCpu
reqMem := impl.cdConfig.ReqMem

eventEnv := v12.EnvVar{Name: "CI_CD_EVENT", Value: string(workflowJson)}
inAppLoggingEnv := v12.EnvVar{Name: "IN_APP_LOGGING", Value: strconv.FormatBool(impl.cdConfig.InAppLoggingEnabled)}
containerEnvVariables = append(containerEnvVariables, eventEnv, inAppLoggingEnv)
workflowMainContainer := v12.Container{
Env: containerEnvVariables,
Name: common.MainContainerName,
Image: workflowRequest.CdImage,
Args: []string{string(workflowJson)},
SecurityContext: &v12.SecurityContext{
Privileged: &privileged,
},
Expand All @@ -255,7 +266,7 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq
}
UpdateContainerEnvsFromCmCs(&workflowMainContainer, workflowConfigMaps, workflowSecrets)

impl.updateBlobStorageConfig(workflowRequest, &workflowTemplate, storageConfigured)
impl.updateBlobStorageConfig(workflowRequest, &workflowTemplate, storageConfigured, ciCdTriggerEvent.CdRequest.BlobStorageLogsKey)
workflowTemplate.Containers = []v12.Container{workflowMainContainer}
workflowTemplate.WorkflowNamePrefix = workflowRequest.WorkflowNamePrefix
workflowTemplate.WfControllerInstanceID = impl.cdConfig.WfControllerInstanceID
Expand All @@ -273,12 +284,12 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq
return workflowExecutor.ExecuteWorkflow(workflowTemplate)
}

func (impl *CdWorkflowServiceImpl) updateBlobStorageConfig(workflowRequest *CdWorkflowRequest, workflowTemplate *bean3.WorkflowTemplate, storageConfigured bool) {
func (impl *CdWorkflowServiceImpl) updateBlobStorageConfig(workflowRequest *CdWorkflowRequest, workflowTemplate *bean3.WorkflowTemplate, storageConfigured bool, blobStorageKey string) {
workflowTemplate.BlobStorageConfigured = storageConfigured && (impl.cdConfig.UseBlobStorageConfigInCdWorkflow || !workflowRequest.IsExtRun)
workflowTemplate.BlobStorageS3Config = workflowRequest.BlobStorageS3Config
workflowTemplate.AzureBlobConfig = workflowRequest.AzureBlobConfig
workflowTemplate.GcpBlobConfig = workflowRequest.GcpBlobConfig
workflowTemplate.CloudStorageKey = impl.cdConfig.DefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix
workflowTemplate.CloudStorageKey = blobStorageKey
}

func (impl *CdWorkflowServiceImpl) getWorkflowExecutor(executorType pipelineConfig.WorkflowExecutorType) WorkflowExecutor {
Expand Down
1 change: 1 addition & 0 deletions pkg/pipeline/CiConfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ type CiConfig struct {
BuildxPvcCachePath string `env:"BUILDX_CACHE_PATH" envDefault:"/var/lib/devtron/buildx"`
UseBlobStorageConfigInCiWorkflow bool `env:"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW" envDefault:"true"`
BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/"`
InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false"`
ClusterConfig *rest.Config
NodeLabel map[string]string
EnableBuildContext bool `env:"ENABLE_BUILD_CONTEXT" envDefault:"false"`
Expand Down
5 changes: 3 additions & 2 deletions pkg/pipeline/CiHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ type WorkflowResponse struct {
ArtifactId int `json:"artifactId"`
IsArtifactUploaded bool `json:"isArtifactUploaded"`
IsVirtualEnvironment bool `json:"isVirtualEnvironment"`
PodName string `json:"podName"`
}

type GitTriggerInfoResponse struct {
Expand Down Expand Up @@ -845,7 +846,7 @@ func (impl *CiHandlerImpl) extractWorkfowStatus(workflowStatus v1alpha1.Workflow
const CiStageFailErrorCode = 2

func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) {
workflowName, status, podStatus, message, logLocation, podName := impl.extractWorkfowStatus(workflowStatus)
workflowName, status, podStatus, message, _, podName := impl.extractWorkfowStatus(workflowStatus)
if workflowName == "" {
impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "podStatus", podStatus, "message", message)
return 0, errors.New("invalid wf name")
Expand Down Expand Up @@ -883,7 +884,7 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus
savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time
savedWorkflow.Name = workflowName
//savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object
savedWorkflow.LogLocation = logLocation
//savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger
savedWorkflow.CiArtifactLocation = ciArtifactLocation
savedWorkflow.PodName = podName
impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow)
Expand Down
2 changes: 2 additions & 0 deletions pkg/pipeline/CiService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) {
return 0, err
}

//savedCiWf.LogLocation = impl.ciConfig.DefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log"
savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.ciConfig.DefaultBuildLogsKeyPrefix, workflowRequest.WorkflowNamePrefix)
err = impl.updateCiWorkflow(workflowRequest, savedCiWf)

appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId)
Expand Down
3 changes: 3 additions & 0 deletions pkg/pipeline/WorkflowDagExecutor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerPreStage(ctx context.Context, cdWf *
Namespace: impl.cdConfig.DefaultNamespace,
BlobStorageEnabled: impl.cdConfig.BlobStorageEnabled,
CdWorkflowId: cdWf.Id,
LogLocation: fmt.Sprintf("%s/%s%s-%s/main.log", impl.cdConfig.DefaultBuildLogsKeyPrefix, strconv.Itoa(cdWf.Id), string(bean.CD_WORKFLOW_TYPE_PRE), pipeline.Name),
AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1},
}
var env *repository2.Environment
Expand Down Expand Up @@ -522,6 +523,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerPostStage(cdWf *pipelineConfig.CdWor
Namespace: impl.cdConfig.DefaultNamespace,
BlobStorageEnabled: impl.cdConfig.BlobStorageEnabled,
CdWorkflowId: cdWf.Id,
LogLocation: fmt.Sprintf("%s/%s%s-%s/main.log", impl.cdConfig.DefaultBuildLogsKeyPrefix, strconv.Itoa(cdWf.Id), string(bean.CD_WORKFLOW_TYPE_POST), pipeline.Name),
AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy},
}
var env *repository2.Environment
Expand Down Expand Up @@ -712,6 +714,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor
WorkflowId: cdWf.Id,
WorkflowRunnerId: runner.Id,
WorkflowNamePrefix: strconv.Itoa(runner.Id) + "-" + runner.Name,
WorkflowPrefixForLog: strconv.Itoa(cdWf.Id) + string(runner.WorkflowType) + "-" + runner.Name,
CdImage: impl.cdConfig.DefaultImage,
CdPipelineId: cdWf.PipelineId,
TriggeredBy: triggeredBy,
Expand Down
10 changes: 8 additions & 2 deletions pkg/pipeline/WorkflowService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ type WorkflowRequest struct {
BlobStorageS3Config *blob_storage.BlobStorageS3Config `json:"blobStorageS3Config"`
AzureBlobConfig *blob_storage.AzureBlobConfig `json:"azureBlobConfig"`
GcpBlobConfig *blob_storage.GcpBlobConfig `json:"gcpBlobConfig"`
BlobStorageLogsKey string `json:"blobStorageLogsKey"`
InAppLoggingEnabled bool `json:"inAppLoggingEnabled"`
DefaultAddressPoolBaseCidr string `json:"defaultAddressPoolBaseCidr"`
DefaultAddressPoolSize int `json:"defaultAddressPoolSize"`
PreCiSteps []*bean2.StepObject `json:"preCiSteps"`
Expand Down Expand Up @@ -216,6 +218,8 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest
CiRequest: workflowRequest,
}

ciCdTriggerEvent.CiRequest.BlobStorageLogsKey = fmt.Sprintf("%s/%s", impl.ciConfig.DefaultBuildLogsKeyPrefix, workflowRequest.WorkflowNamePrefix)
ciCdTriggerEvent.CiRequest.InAppLoggingEnabled = impl.ciConfig.InAppLoggingEnabled
workflowJson, err := json.Marshal(&ciCdTriggerEvent)
if err != nil {
impl.Logger.Errorw("err", err)
Expand All @@ -231,7 +235,7 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest

privileged := true
blobStorageConfigured := workflowRequest.BlobStorageConfigured
archiveLogs := blobStorageConfigured
archiveLogs := blobStorageConfigured && !impl.ciConfig.InAppLoggingEnabled

limitCpu := impl.ciConfig.LimitCpu
limitMem := impl.ciConfig.LimitMem
Expand Down Expand Up @@ -276,12 +280,14 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest
Steps: steps,
})

eventEnv := v12.EnvVar{Name: "CI_CD_EVENT", Value: string(workflowJson)}
inAppLoggingEnv := v12.EnvVar{Name: "IN_APP_LOGGING", Value: strconv.FormatBool(impl.ciConfig.InAppLoggingEnabled)}
containerEnvVariables = append(containerEnvVariables, eventEnv, inAppLoggingEnv)
ciTemplate := v1alpha1.Template{
Name: CI_WORKFLOW_NAME,
Container: &v12.Container{
Env: containerEnvVariables,
Image: workflowRequest.CiImage, //TODO need to check whether trigger buildx image or normal image
Args: []string{string(workflowJson)},
SecurityContext: &v12.SecurityContext{
Privileged: &privileged,
},
Expand Down