diff --git a/pkg/util/collection/collection.go b/pkg/util/collection/collection.go index 6f657bc8..864d7067 100644 --- a/pkg/util/collection/collection.go +++ b/pkg/util/collection/collection.go @@ -1,6 +1,10 @@ package collection -// Returns symbol times, with between each one. +import ( + "reflect" +) + +// RepeatSymbol returns symbol times, with between each one. // if symbol = ?, separator = , and count = 5 // this returns: "?,?,?,?,?" func RepeatSymbol(count int, symbol, separator string) string { @@ -15,3 +19,39 @@ func RepeatSymbol(count int, symbol, separator string) string { return result } + +// RemoveBlanks goes through the data, assumed to be an array or map of some kind, +// and removes any data that is a nil or zero value. Maps with no keys are also removed. +// +// Note that this will not check the data again. So if you have the following +// parent: { +// child: {} +// } +// +// The result will be +// parent: {} +// +// it will not go through it again and remove parent. +func RemoveBlanks(data interface{}) { + if mapping, ok := data.(map[string]interface{}); ok { + keysToDelete := make([]string, 0) + for key, v := range mapping { + rv := reflect.ValueOf(v) + if v == nil || rv.IsZero() { + keysToDelete = append(keysToDelete, key) + } else if vMap, vMapOk := v.(map[string]interface{}); vMapOk && len(vMap) == 0 { + keysToDelete = append(keysToDelete, key) + } else { + RemoveBlanks(v) + } + } + + for _, keyToDelete := range keysToDelete { + delete(mapping, keyToDelete) + } + } else if list, ok := data.([]interface{}); ok { + for _, listItem := range list { + RemoveBlanks(listItem) + } + } +} diff --git a/pkg/workflow_execution.go b/pkg/workflow_execution.go index 8939d65f..fb80e19f 100644 --- a/pkg/workflow_execution.go +++ b/pkg/workflow_execution.go @@ -288,7 +288,7 @@ func (c *Client) injectAutomatedFields(namespace string, wf *wfv1.Workflow, opts if opts.PodGCStrategy == nil { if wf.Spec.PodGC == nil { //TODO - Load this data from onepanel config-map or secret - podGCStrategy := env.GetEnv("ARGO_POD_GC_STRATEGY", "OnPodCompletion") + podGCStrategy := env.Get("ARGO_POD_GC_STRATEGY", "OnPodCompletion") strategy := PodGCStrategy(podGCStrategy) wf.Spec.PodGC = &wfv1.PodGC{ Strategy: strategy, @@ -499,6 +499,10 @@ func (c *Client) createWorkflow(namespace string, workflowTemplateID uint64, wor } wf.Spec.Arguments.Parameters = newParameters + if err = injectFilesyncerSidecar(wf); err != nil { + return nil, err + } + if err = injectWorkflowExecutionStatusCaller(wf, wfv1.NodeRunning); err != nil { return nil, err } @@ -2063,6 +2067,38 @@ func getCURLNodeTemplate(name, curlMethod, curlPath, curlBody string, inputs wfv return } +func injectFilesyncerSidecar(wf *wfv1.Workflow) error { + filesyncer := wfv1.UserContainer{ + Container: corev1.Container{ + Name: "sys-filesyncer", + Image: "onepanel/filesyncer:v0.19.0", + Args: []string{"server", "-server-prefix=/sys/filesyncer", "-backend=local-storage"}, + Env: []corev1.EnvVar{ + { + Name: "ONEPANEL_INTERACTIVE_SIDECAR", + Value: "true", + }, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 8888, + }, + }, + }, + } + + for i := range wf.Spec.Templates { + template := &wf.Spec.Templates[i] + + if (template.Container != nil && len(template.Container.VolumeMounts) != 0) || + (template.Script != nil && len(template.Script.VolumeMounts) != 0) { + template.Sidecars = append(template.Sidecars, filesyncer) + } + } + + return nil +} + func injectExitHandlerWorkflowExecutionStatistic(wf *wfv1.Workflow, workflowTemplateId *uint64) error { curlPath := "/apis/v1beta1/{{workflow.namespace}}/workflow_executions/{{workflow.name}}/statistics" statistics := map[string]interface{}{ diff --git a/server/workflow_server.go b/server/workflow_server.go index 62d4ff0a..db933dcc 100644 --- a/server/workflow_server.go +++ b/server/workflow_server.go @@ -2,9 +2,11 @@ package server import ( "context" + "encoding/json" "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" "github.com/onepanelio/core/pkg/util" + "github.com/onepanelio/core/pkg/util/collection" "github.com/onepanelio/core/pkg/util/request" "github.com/onepanelio/core/pkg/util/request/pagination" "github.com/onepanelio/core/pkg/util/router" @@ -35,15 +37,68 @@ func NewWorkflowServer() *WorkflowServer { return &WorkflowServer{} } +// removedUnusedManifestFields removes any fields not necessary in a Workflow Manfiest. +// this is used to cut down the size for more efficient data transmission +func removedUnusedManifestFields(manifest string) (string, error) { + if manifest == "" { + return "", nil + } + + result := make(map[string]map[string]interface{}) + if err := json.Unmarshal([]byte(manifest), &result); err != nil { + return "", err + } + + for key := range result { + collection.RemoveBlanks(result[key]) + } + + delete(result["metadata"], "managedFields") + delete(result["status"], "resourcesDuration") + + templatesRaw, ok := result["spec"]["templates"] + if ok { + templatesArray := templatesRaw.([]interface{}) + + for _, template := range templatesArray { + templateMap := template.(map[string]interface{}) + delete(templateMap, "metadata") + } + } + + nodeStatusRaw, ok := result["status"]["nodes"] + if ok { + nodeStatusArray := nodeStatusRaw.(map[string]interface{}) + + for _, nodeStatus := range nodeStatusArray { + nodeStatusMap := nodeStatus.(map[string]interface{}) + delete(nodeStatusMap, "resourcesDuration") + } + } + + finalManifestBytes, err := json.Marshal(result) + if err != nil { + return "", err + } + + return string(finalManifestBytes), nil +} + // apiWorkflowExecution converts a package workflow execution to the api version // router is optional func apiWorkflowExecution(wf *v1.WorkflowExecution, router router.Web) (workflow *api.WorkflowExecution) { + manifest, err := removedUnusedManifestFields(wf.Manifest) + if err != nil { + log.Printf("error trimming manifest %v", err) + return nil + } + workflow = &api.WorkflowExecution{ CreatedAt: wf.CreatedAt.Format(time.RFC3339), Uid: wf.UID, Name: wf.Name, Phase: string(wf.Phase), - Manifest: wf.Manifest, + Manifest: manifest, Labels: converter.MappingToKeyValue(wf.Labels), Metrics: converter.MetricsToAPI(wf.Metrics), }