Skip to content

feat: added support for in app logging archival from CI runner #3467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Jun 8, 2023
Merged

feat: added support for in app logging archival from CI runner #3467

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
3918bc3
archive logs sync
subhashish-devtron May 23, 2023
d983e69
added type for cdstage event
subhashish-devtron May 23, 2023
abbde0f
test
subhashish-devtron May 24, 2023
c86a9e7
cleanup
subhashish-devtron Jun 2, 2023
0151db1
adding log file path to db on trigger
subhashish-devtron May 25, 2023
6480248
cleanup
subhashish-devtron Jun 2, 2023
3ddae08
cleanup
subhashish-devtron Jun 2, 2023
ed91114
cleanup
subhashish-devtron Jun 2, 2023
68e830e
cleanup
subhashish-devtron Jun 2, 2023
11bb803
exposed pod name in cd workflow details
subhashish-devtron Jun 2, 2023
e8a8678
log path fix
subhashish-devtron Jun 6, 2023
bc7acec
fix
subhashish-devtron Jun 6, 2023
fa9cb50
addresing pr comments
subhashish-devtron Jun 7, 2023
3192aca
addresing pr comments2
subhashish-devtron Jun 7, 2023
e71f2b8
addresing pr comments3
subhashish-devtron Jun 7, 2023
d0ac413
fix
subhashish-devtron Jun 7, 2023
5343ee7
Merge branch 'main' into archive-logs-final
subhashish-devtron Jun 7, 2023
8b52a16
Update CiHandler.go
subhashish-devtron Jun 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/pipeline/ArgoWorkflowExecutor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func (impl *ArgoWorkflowExecutorImpl) ExecuteWorkflow(workflowTemplate bean.Work

func (impl *ArgoWorkflowExecutorImpl) updateBlobStorageConfig(workflowTemplate bean.WorkflowTemplate, cdTemplate *v1alpha1.Template) {
cdTemplate.ArchiveLocation = &v1alpha1.ArtifactLocation{
ArchiveLogs: &workflowTemplate.BlobStorageConfigured,
ArchiveLogs: &workflowTemplate.ArchiveLogs,
}
if workflowTemplate.BlobStorageConfigured {
var s3Artifact *v1alpha1.S3Artifact
Expand Down
1 change: 1 addition & 0 deletions pkg/pipeline/CdConfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ type CdConfig struct {
UseBlobStorageConfigInCdWorkflow bool `env:"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW" envDefault:"true"`
BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/"`
CdWorkflowExecutorType pipelineConfig.WorkflowExecutorType `env:"CD_WORKFLOW_EXECUTOR_TYPE" envDefault:"AWF"`
InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false"`
}

func GetCdConfig() (*CdConfig, error) {
Expand Down
1 change: 0 additions & 1 deletion pkg/pipeline/CdHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -627,7 +627,6 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus
savedWorkflow.Message = message
savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time
savedWorkflow.Name = workflowName
savedWorkflow.LogLocation = wfStatusRs.LogLocation
savedWorkflow.PodName = podName
savedWorkflow.UpdatedOn = time.Now()
savedWorkflow.UpdatedBy = 1
Expand Down
8 changes: 8 additions & 0 deletions pkg/pipeline/CdWorkflowService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ type CdWorkflowRequest struct {
CloudProvider blob_storage.BlobStorageType `json:"cloudProvider"`
AzureBlobConfig *blob_storage.AzureBlobConfig `json:"azureBlobConfig"`
GcpBlobConfig *blob_storage.GcpBlobConfig `json:"gcpBlobConfig"`
BlobStorageLogsKey string `json:"blobStorageLogsKey"`
InAppLoggingEnabled bool `json:"inAppLoggingEnabled"`
DefaultAddressPoolBaseCidr string `json:"defaultAddressPoolBaseCidr"`
DefaultAddressPoolSize int `json:"defaultAddressPoolSize"`
DeploymentTriggeredBy string `json:"deploymentTriggeredBy,omitempty"`
Expand Down Expand Up @@ -146,8 +148,12 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq
workflowRequest.IsExtRun = true
}
ciCdTriggerEvent := CiCdTriggerEvent{
Type: cdStage,
CdRequest: workflowRequest,
}

ciCdTriggerEvent.CdRequest.BlobStorageLogsKey = impl.cdConfig.DefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix
ciCdTriggerEvent.CdRequest.InAppLoggingEnabled = impl.cdConfig.InAppLoggingEnabled
workflowJson, err := json.Marshal(&ciCdTriggerEvent)
if err != nil {
impl.Logger.Errorw("error occurred while marshalling ciCdTriggerEvent", "error", err)
Expand Down Expand Up @@ -223,6 +229,7 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq
workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.TaintKey, Value: impl.cdConfig.TaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}}
workflowTemplate.Volumes = ExtractVolumesFromCmCs(workflowConfigMaps, workflowSecrets)
workflowTemplate.ArchiveLogs = storageConfigured
workflowTemplate.ArchiveLogs = workflowTemplate.ArchiveLogs && !impl.cdConfig.InAppLoggingEnabled
workflowTemplate.RestartPolicy = v12.RestartPolicyNever

if len(impl.cdConfig.NodeLabel) > 0 {
Expand All @@ -234,6 +241,7 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq
reqCpu := impl.cdConfig.ReqCpu
reqMem := impl.cdConfig.ReqMem

containerEnvVariables = append(containerEnvVariables, []v12.EnvVar{{Name: "CI_CD_EVENT", Value: string(workflowJson)}}...)
workflowMainContainer := v12.Container{
Env: containerEnvVariables,
Name: common.MainContainerName,
Expand Down
1 change: 1 addition & 0 deletions pkg/pipeline/CiConfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ type CiConfig struct {
BuildxPvcCachePath string `env:"BUILDX_CACHE_PATH" envDefault:"/var/lib/devtron/buildx"`
UseBlobStorageConfigInCiWorkflow bool `env:"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW" envDefault:"true"`
BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/"`
InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false"`
ClusterConfig *rest.Config
NodeLabel map[string]string
EnableBuildContext bool `env:"ENABLE_BUILD_CONTEXT" envDefault:"false"`
Expand Down
4 changes: 2 additions & 2 deletions pkg/pipeline/CiHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -845,7 +845,7 @@ func (impl *CiHandlerImpl) extractWorkfowStatus(workflowStatus v1alpha1.Workflow
const CiStageFailErrorCode = 2

func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) {
workflowName, status, podStatus, message, logLocation, podName := impl.extractWorkfowStatus(workflowStatus)
workflowName, status, podStatus, message, _, podName := impl.extractWorkfowStatus(workflowStatus)
if workflowName == "" {
impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "podStatus", podStatus, "message", message)
return 0, errors.New("invalid wf name")
Expand Down Expand Up @@ -883,7 +883,7 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus
savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time
savedWorkflow.Name = workflowName
//savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object
savedWorkflow.LogLocation = logLocation
//savedWorkflow.LogLocation = logLocation
savedWorkflow.CiArtifactLocation = ciArtifactLocation
savedWorkflow.PodName = podName
impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow)
Expand Down
1 change: 1 addition & 0 deletions pkg/pipeline/CiService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) {
return 0, err
}

savedCiWf.LogLocation = impl.ciConfig.DefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log"
err = impl.updateCiWorkflow(workflowRequest, savedCiWf)

appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId)
Expand Down
1 change: 1 addition & 0 deletions pkg/pipeline/WorkflowDagExecutor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerPreStage(ctx context.Context, cdWf *
Namespace: impl.cdConfig.DefaultNamespace,
BlobStorageEnabled: impl.cdConfig.BlobStorageEnabled,
CdWorkflowId: cdWf.Id,
LogLocation: impl.cdConfig.DefaultBuildLogsKeyPrefix + "/" + strconv.Itoa(cdWf.Id) + "-" + pipeline.Name + "/main.log",
AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1},
}
var env *repository2.Environment
Expand Down
7 changes: 6 additions & 1 deletion pkg/pipeline/WorkflowService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ type WorkflowRequest struct {
BlobStorageS3Config *blob_storage.BlobStorageS3Config `json:"blobStorageS3Config"`
AzureBlobConfig *blob_storage.AzureBlobConfig `json:"azureBlobConfig"`
GcpBlobConfig *blob_storage.GcpBlobConfig `json:"gcpBlobConfig"`
BlobStorageLogsKey string `json:"blobStorageLogsKey"`
InAppLoggingEnabled bool `json:"inAppLoggingEnabled"`
DefaultAddressPoolBaseCidr string `json:"defaultAddressPoolBaseCidr"`
DefaultAddressPoolSize int `json:"defaultAddressPoolSize"`
PreCiSteps []*bean2.StepObject `json:"preCiSteps"`
Expand Down Expand Up @@ -216,6 +218,8 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest
CiRequest: workflowRequest,
}

ciCdTriggerEvent.CiRequest.BlobStorageLogsKey = impl.ciConfig.DefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix
ciCdTriggerEvent.CiRequest.InAppLoggingEnabled = impl.ciConfig.InAppLoggingEnabled
workflowJson, err := json.Marshal(&ciCdTriggerEvent)
if err != nil {
impl.Logger.Errorw("err", err)
Expand All @@ -231,7 +235,7 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest

privileged := true
blobStorageConfigured := workflowRequest.BlobStorageConfigured
archiveLogs := blobStorageConfigured
archiveLogs := blobStorageConfigured && !impl.ciConfig.InAppLoggingEnabled

limitCpu := impl.ciConfig.LimitCpu
limitMem := impl.ciConfig.LimitMem
Expand Down Expand Up @@ -276,6 +280,7 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest
Steps: steps,
})

containerEnvVariables = append(containerEnvVariables, []v12.EnvVar{{Name: "CI_CD_EVENT", Value: string(workflowJson)}}...)
ciTemplate := v1alpha1.Template{
Name: CI_WORKFLOW_NAME,
Container: &v12.Container{
Expand Down