Oss sync (#12)

* feat: cluster bearer token hide from dashboard (#2894)

* cluster token config removed mandatory

* api spec added for cluster update and create, and cluster list api changes for token

* fix check config for cluster token

* docs: mount pvc (#2941)

* added pvc

* added pvc

* added pvc

* added pvc

* added pvc

* feat: Resource browser child ref (#2913)

* child rbac handling init commit

* resource manifest validate handling added

* handle rbac case

* wire gen fix

* gvk passed and ap resource handling

* code cleaning

* dead code cleaning

* removed unused func

* fix: k8s log stream cpu issue (#2929)

* updated buffer size

* refactored code for getting logs from k8s

* updated Dockerfile

* updated Dockerfile

* downgraded go version

* removed redundant log

---------

Co-authored-by: Vikram <[email protected]>
Co-authored-by: SNe789 <[email protected]>
Co-authored-by: kripanshdevtron <[email protected]>

Author: 4 people

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19  AS build-env
+FROM golang:1.18  AS build-env
 
 RUN echo $GOPATH
 RUN apt update

DockerfileEA

@@ -1,4 +1,4 @@
-FROM golang:1.19  AS build-env
+FROM golang:1.18  AS build-env
 
 RUN echo $GOPATH
 RUN apt update

api/connector/Connector.go

@@ -33,6 +33,7 @@ import (
 	"net/http"
 	"regexp"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 )
@@ -104,28 +105,39 @@ func (impl PumpImpl) StartK8sStreamWithHeartBeat(w http.ResponseWriter, isReconn
 		done <- true
 	}()
 
-	// heartbeat end
-	for {
-		sc := bufio.NewScanner(stream)
-		for sc.Scan() {
-			log := sc.Text()
-			a := regexp.MustCompile(" ")
-			splitLog := a.Split(log, 2)
-			timeParsed, err := time.Parse(time.RFC3339, splitLog[0])
-			if err != nil {
-				impl.logger.Errorw("error in writing data over sse", "err", err)
-				return
-			}
-			mux.Lock()
-			err = impl.sendEvent([]byte(strconv.FormatInt(timeParsed.UnixNano(), 10)), nil, []byte(splitLog[1]), w)
-			mux.Unlock()
-			if err != nil {
-				impl.logger.Errorw("error in writing data over sse", "err", err)
+	bufReader := bufio.NewReader(stream)
+	eof := false
+	for !eof {
+		log, err := bufReader.ReadString('\n')
+		if err == io.EOF {
+			eof = true
+			// stop if we reached end of stream and the next line is empty
+			if log == "" {
 				return
 			}
-			f.Flush()
+		} else if err != nil && err != io.EOF {
+			impl.logger.Errorw("error in reading buffer string, StartK8sStreamWithHeartBeat", "err", err)
+			return
+		}
+		log = strings.TrimSpace(log) // Remove trailing line ending
+		a := regexp.MustCompile(" ")
+		splitLog := a.Split(log, 2)
+		parsedTime, err := time.Parse(time.RFC3339, splitLog[0])
+		if err != nil {
+			impl.logger.Errorw("error in writing data over sse", "err", err)
+			return
 		}
+		eventId := strconv.FormatInt(parsedTime.UnixNano(), 10)
+		mux.Lock()
+		err = impl.sendEvent([]byte(eventId), nil, []byte(splitLog[1]), w)
+		mux.Unlock()
+		if err != nil {
+			impl.logger.Errorw("error in writing data over sse", "err", err)
+			return
+		}
+		f.Flush()
 	}
+	// heartbeat end
 }
 
 func (impl PumpImpl) StartStreamWithHeartBeat(w http.ResponseWriter, isReconnect bool, recv func() (*application.LogEntry, error), err error) {
@@ -223,11 +235,9 @@ func (impl *PumpImpl) sendEvent(eventId []byte, eventName []byte, payload []byte
 		res = append(res, payload...)
 	}
 	res = append(res, '\n', '\n')
-	if i, err := w.Write(res); err != nil {
+	if _, err := w.Write(res); err != nil {
 		impl.logger.Errorf("Failed to send response chunk: %v", err)
 		return err
-	} else {
-		impl.logger.Debugw("msg written", "count", i)
 	}
 
 	return nil

cmd/external-app/wire_gen.go

@@ -47,4 +47,63 @@ Click **Save**. The application will be moved to the selected project.
 * To remove the tags from propagation, click the symbol <img src="https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/creating-application/propagate-dark.jpg" height="10"> again.
 * Click `Save`.
 
-The changes in the tags will be reflected in the `Tags` on the `Overview` section.
+The changes in the tags will be reflected in the `Tags` on the `Overview` section.
+
+
+## Configure PersistentVolumeClaim (PVC) for Build Time Optimization
+
+ A PersistentVolumeClaim (PVC) volume is a request for storage, which is used to mount a PersistentVolume (PV) into a Pod. In order to optimize build time, you can configure PVC in your application.
+
+If you want to optimize build time for the multiple target platforms (e.g., arm64, amd64), mounting a PVC will provide volume directly to a pod which helps in shorter build time by storing build cache. Mounting a PVC into a pod will provide storage for build cache which will not impact the normal build where the image is built on the basis of architecture and operating system of the K8s node on which CI is running.
+
+### Create PVC file
+
+* The following configuration file describes persistent volume claim e.g.,`cache-pvc.yaml`, where you have to define the metadata `name` and `storageClassname`.
+
+```bash
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: cache-pvc # here comes the name of PVC
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: # here comes storage class name
+  resources:
+    requests:
+      storage: 30Gi
+```
+
+* Create the PersistentVolumeClaim by running the following command:
+
+```bash
+kubectl apply -f https://k8s.io/examples/pods/storage/pv-claim.yaml -n {namespace}
+```
+
+For more detail, refer [Kubernetes PVC](https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim).
+
+
+## Configure PVC
+
+In order to configure PVC:
+* Go to the `Overview` section of your application.
+* On the right-corner, click `Edit Tags`.
+
+![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/creating-application/overview/pvc-edit-tags.jpg)
+
+![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/creating-application/overview/manage-tags-pvc.jpg)
+
+* For app level PVC mounting, enter the following:<ul><li>key:`devtron.ai/ci-pvc-all`</li><li>value: metadata name (e.g., `cache-pvc)` which you define on the [PVC template](#create-pvc-file).</li></ul>`Note`: This PVC mounting will impact all the build pipilines of the application.
+* For pipeline level, enter the following:<ul><li>key:`devtron.ai/ci-pvc-{pipelinename}`</li><li>value: metadata name which you define on the [PVC template](#create-pvc-file).</li></ul>`Note`: This PVC mounting will impact only the particular build pipeline.
+
+To know the `pipelinename` detail, go to the `App Configutation`, click `Workflow Editor` the pipeline name will be on the `Build` pipeline as shown below.
+
+![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/creating-application/overview/pipeline-name-pvc.jpg)
+
+* Click `Save`.
+
+
+
+
+
+ 

docs/user-guide/creating-application/overview.md

@@ -23,8 +23,10 @@ import (
 	error2 "errors"
 	"flag"
 	"fmt"
+	"github.com/argoproj/gitops-engine/pkg/utils/kube"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"net/http"
 	"os/user"
 	"path/filepath"
@@ -546,11 +548,12 @@ func (impl K8sUtil) GetPodByName(namespace string, name string, client *v12.Core
 	}
 }
 
-func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.UnstructuredList, namespaced bool, kind string, validateResourceAccess func(namespace, resourceName string) bool) (*ClusterResourceListMap, error) {
+func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.UnstructuredList, namespaced bool, gvk schema.GroupVersionKind, validateResourceAccess func(namespace string, group string, kind string, resourceName string) bool) (*ClusterResourceListMap, error) {
 	clusterResourceListMap := &ClusterResourceListMap{}
 	// build headers
 	var headers []string
 	columnIndexes := make(map[int]string)
+	kind := gvk.Kind
 	if kind == "Event" {
 		headers, columnIndexes = impl.getEventKindHeader()
 	} else {
@@ -587,13 +590,11 @@ func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.Unstructu
 	// build rows
 	rowsMapping := make([]map[string]interface{}, 0)
 	rowsDataUncast := manifest.Object[K8sClusterResourceRowsKey]
-	var resourceName string
 	var namespace string
 	var allowed bool
 	if rowsDataUncast != nil {
 		rows := rowsDataUncast.([]interface{})
 		for _, row := range rows {
-			resourceName = ""
 			namespace = ""
 			allowed = true
 			rowIndex := make(map[string]interface{})
@@ -614,11 +615,10 @@ func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.Unstructu
 				rowIndex[columnName] = cell
 			}
 
-			// set namespace
-
 			cellObjUncast := rowMap[K8sClusterResourceObjectKey]
+			var cellObj map[string]interface{}
 			if cellObjUncast != nil {
-				cellObj := cellObjUncast.(map[string]interface{})
+				cellObj = cellObjUncast.(map[string]interface{})
 				if cellObj != nil && cellObj[K8sClusterResourceMetadataKey] != nil {
 					metadata := cellObj[K8sClusterResourceMetadataKey].(map[string]interface{})
 					if metadata[K8sClusterResourceNamespaceKey] != nil {
@@ -627,14 +627,9 @@ func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.Unstructu
 							rowIndex[K8sClusterResourceNamespaceKey] = namespace
 						}
 					}
-					if metadata[K8sClusterResourceMetadataNameKey] != nil {
-						resourceName = metadata[K8sClusterResourceMetadataNameKey].(string)
-					}
 				}
 			}
-			if resourceName != "" {
-				allowed = validateResourceAccess(namespace, resourceName)
-			}
+			allowed = impl.ValidateResource(cellObj, gvk, validateResourceAccess)
 			if allowed {
 				rowsMapping = append(rowsMapping, rowIndex)
 			}
@@ -647,6 +642,89 @@ func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.Unstructu
 	return clusterResourceListMap, nil
 }
 
+func (impl K8sUtil) ValidateResource(resourceObj map[string]interface{}, gvk schema.GroupVersionKind, validateCallback func(namespace string, group string, kind string, resourceName string) bool) bool {
+	resKind := gvk.Kind
+	groupName := gvk.Group
+	metadata := resourceObj[K8sClusterResourceMetadataKey]
+	if metadata == nil {
+		return false
+	}
+	metadataMap := metadata.(map[string]interface{})
+	var namespace, resourceName string
+	var ownerReferences []interface{}
+	if metadataMap[K8sClusterResourceNamespaceKey] != nil {
+		namespace = metadataMap[K8sClusterResourceNamespaceKey].(string)
+	}
+	if metadataMap[K8sClusterResourceMetadataNameKey] != nil {
+		resourceName = metadataMap[K8sClusterResourceMetadataNameKey].(string)
+	}
+	if metadataMap[K8sClusterResourceOwnerReferenceKey] != nil {
+		ownerReferences = metadataMap[K8sClusterResourceOwnerReferenceKey].([]interface{})
+	}
+	if len(ownerReferences) > 0 {
+		for _, ownerRef := range ownerReferences {
+			allowed := impl.validateForResource(namespace, ownerRef, validateCallback)
+			if allowed {
+				return allowed
+			}
+		}
+	}
+	// check current RBAC in case not matched with above one
+	return validateCallback(namespace, groupName, resKind, resourceName)
+}
+
+func (impl K8sUtil) validateForResource(namespace string, resourceRef interface{}, validateCallback func(namespace string, group string, kind string, resourceName string) bool) bool {
+	resourceReference := resourceRef.(map[string]interface{})
+	resKind := resourceReference[K8sClusterResourceKindKey].(string)
+	apiVersion := resourceReference[K8sClusterResourceApiVersionKey].(string)
+	groupName := ""
+	if strings.Contains(apiVersion, "/") {
+		groupName = apiVersion[:strings.LastIndex(apiVersion, "/")] // extracting group from this apiVersion
+	}
+	resName := ""
+	if resourceReference["name"] != "" {
+		resName = resourceReference["name"].(string)
+		switch resKind {
+		case kube.ReplicaSetKind:
+			// check deployment first, then RO and then RS
+			if strings.Contains(resName, "-") {
+				deploymentName := resName[:strings.LastIndex(resName, "-")]
+				allowed := validateCallback(namespace, groupName, kube.DeploymentKind, deploymentName)
+				if allowed {
+					return true
+				}
+				allowed = validateCallback(namespace, K8sClusterResourceRolloutGroup, K8sClusterResourceRolloutKind, deploymentName)
+				if allowed {
+					return true
+				}
+			}
+			allowed := validateCallback(namespace, groupName, resKind, resName)
+			if allowed {
+				return true
+			}
+		case kube.JobKind:
+			// check CronJob first, then Job
+			if strings.Contains(resName, "-") {
+				cronJobName := resName[:strings.LastIndex(resName, "-")]
+				allowed := validateCallback(namespace, groupName, K8sClusterResourceCronJobKind, cronJobName)
+				if allowed {
+					return true
+				}
+			}
+			allowed := validateCallback(namespace, groupName, resKind, resName)
+			if allowed {
+				return true
+			}
+		case kube.DeploymentKind, K8sClusterResourceCronJobKind, kube.StatefulSetKind, kube.DaemonSetKind, K8sClusterResourceRolloutKind, K8sClusterResourceReplicationControllerKind:
+			allowed := validateCallback(namespace, groupName, resKind, resName)
+			if allowed {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func (impl K8sUtil) getEventKindHeader() ([]string, map[int]string) {
 	headers := []string{"type", "message", "namespace", "involved object", "source", "count", "age", "last seen"}
 	columnIndexes := make(map[int]string)