feat: add cvat migration template

Dockerfile

@@ -5,11 +5,13 @@ COPY . .
 
 RUN go get -d -v ./...
 RUN go install -v ./...
-
+RUN go get -u github.com/pressly/goose/cmd/goose
+RUN go build -o /go/bin/goose ./cmd/goose.go
 
 FROM golang:1.13.10
 COPY --from=builder /go/bin/core .
 COPY --from=builder /go/src/db ./db
+COPY --from=builder /go/bin/goose .
 
 EXPOSE 8888
 EXPOSE 8887

cmd/goose.go

@@ -0,0 +1,56 @@
+// This is custom goose binary to support .go migration files in ./db dir
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/onepanelio/core/db"
+	v1 "github.com/onepanelio/core/pkg"
+	"log"
+	"os"
+
+	"github.com/pressly/goose"
+)
+
+var (
+	flags = flag.NewFlagSet("goose", flag.ExitOnError)
+	dir   = flags.String("dir", ".", "directory with migration files")
+)
+
+func main() {
+	flags.Parse(os.Args[1:])
+	args := flags.Args()
+
+	if len(args) < 1 {
+		flags.Usage()
+		return
+	}
+
+	kubeConfig := v1.NewConfig()
+	client, err := v1.NewClient(kubeConfig, nil)
+	if err != nil {
+		log.Fatalf("Failed to connect to Kubernetes cluster: %v", err)
+	}
+	config, err := client.GetSystemConfig()
+	if err != nil {
+		log.Fatalf("Failed to get system config: %v", err)
+	}
+
+	databaseDataSourceName := fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable",
+		config["databaseHost"], config["databaseUsername"], config["databasePassword"], config["databaseName"])
+
+	db := sqlx.MustConnect(config["databaseDriverName"], databaseDataSourceName)
+
+	command := args[0]
+
+	arguments := []string{}
+	if len(args) > 2 {
+		arguments = append(arguments, args[2:]...)
+	}
+
+	if err := goose.Run(command, db.DB, *dir, arguments...); err != nil {
+		log.Fatalf("goose %v: %v", command, err)
+	}
+}

db/20200525160514_add_jupyter_workspace_template.go

@@ -1,8 +1,11 @@
+// Package migration is for carrying out migrations against the database.
+// To support Onepanel Core operations.
 package migration
 
 import (
 	"database/sql"
 	v1 "github.com/onepanelio/core/pkg"
+	uid2 "github.com/onepanelio/core/pkg/util/uid"
 	"github.com/pressly/goose"
 	"log"
 )
@@ -48,22 +51,23 @@ routes:
       port:
         number: 80
 # DAG Workflow to be executed once a Workspace action completes
-postExecutionWorkflow:
-  entrypoint: main
-  templates:
-  - name: main
-    dag:
-       tasks:
-       - name: slack-notify
-         template: slack-notify
-  -  name: slack-notify
-     container:
-       image: technosophos/slack-notify
-       args:
-       - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
-       command:
-       - sh
-       - -c`
+# postExecutionWorkflow:
+#   entrypoint: main
+#   templates:
+#   - name: main
+#     dag:
+#        tasks:
+#        - name: slack-notify
+#          template: slack-notify
+#   -  name: slack-notify
+#      container:
+#        image: technosophos/slack-notify
+#        args:
+#        - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
+#        command:
+#        - sh
+#        - -c
+`
 
 const jupyterLabTemplateName = "JupyterLab"
 
@@ -89,7 +93,7 @@ func Up20200525160514(tx *sql.Tx) error {
 
 	for _, namespace := range namespaces {
 		if _, err := client.CreateWorkspaceTemplate(namespace.Name, workspaceTemplate); err != nil {
-			log.Printf("error %v", err.Error())
+			log.Fatalf("error %v", err.Error())
 		}
 	}
 
@@ -106,10 +110,13 @@ func Down20200525160514(tx *sql.Tx) error {
 	if err != nil {
 		return err
 	}
-
+	uid, err := uid2.GenerateUID(jupyterLabTemplateName, 30)
+	if err != nil {
+		return err
+	}
 	for _, namespace := range namespaces {
-		if err := client.DeleteWorkspace(namespace.Name, jupyterLabTemplateName); err != nil {
-			log.Printf("error %v", err.Error())
+		if _, err := client.ArchiveWorkspaceTemplate(namespace.Name, uid); err != nil {
+			log.Fatalf("error %v", err.Error())
 		}
 	}
 

db/20200528140124_add_cvat_workspace_template.go

@@ -0,0 +1,176 @@
+package migration
+
+import (
+	"database/sql"
+	v1 "github.com/onepanelio/core/pkg"
+	uid2 "github.com/onepanelio/core/pkg/util/uid"
+	"github.com/pressly/goose"
+	"log"
+)
+
+const cvatWorkspaceTemplate = `# Docker containers that are part of the Workspace
+containers:
+- name: cvat-db
+  image: postgres:10-alpine
+  env:
+  - name: POSTGRES_USER
+    value: root
+  - name: POSTGRES_DB
+    value: cvat
+  - name: POSTGRES_HOST_AUTH_METHOD
+    value: trust
+  - name: PGDATA
+    value: /var/lib/psql/data
+  ports:
+  - containerPort: 5432
+    name: tcp
+  volumeMounts:
+  - name: db
+    mountPath: /var/lib/psql
+- name: cvat-redis
+  image: redis:4.0-alpine
+  ports:
+  - containerPort: 6379
+    name: tcp
+- name: cvat
+  image: onepanel/cvat:v0.7.0
+  env:
+  - name: DJANGO_MODWSGI_EXTRA_ARGS
+    value: ""
+  - name: ALLOWED_HOSTS
+    value: '*'
+  - name: CVAT_REDIS_HOST
+    value: localhost
+  - name: CVAT_POSTGRES_HOST
+    value: localhost
+  - name: CVAT_SHARE_URL
+    value: /home/django/data
+  ports:
+  - containerPort: 8080
+    name: http
+  volumeMounts:
+  - name: data
+    mountPath: /home/django/data
+  - name: keys
+    mountPath: /home/django/keys
+  - name: logs
+    mountPath: /home/django/logs
+  - name: models
+    mountPath: /home/django/models
+- name: cvat-ui
+  image: onepanel/cvat-ui:v0.7.0
+  ports:
+  - containerPort: 80
+    name: http
+ports:
+- name: cvat-ui
+  port: 80
+  protocol: TCP
+  targetPort: 80
+- name: cvat
+  port: 8080
+  protocol: TCP
+  targetPort: 8080
+routes:
+- match:
+  - uri:
+      regex: /api/.*|/git/.*|/tensorflow/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
+  - queryParams:
+      id:
+        regex: \d+.*
+  route:
+  - destination:
+      port:
+        number: 8080
+- match:
+  - uri:
+      prefix: /
+  route:
+  - destination:
+      port:
+        number: 80
+# DAG Workflow to be executed once a Workspace action completes
+# postExecutionWorkflow:
+#   entrypoint: main
+#   templates:
+#   - name: main
+#     dag:
+#        tasks:
+#        - name: slack-notify
+#          template: slack-notify
+#   -  name: slack-notify
+#      container:
+#        image: technosophos/slack-notify
+#        args:
+#        - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
+#        command:
+#        - sh
+#        - -c
+`
+
+const cvatTemplateName = "CVAT"
+
+func init() {
+	goose.AddMigration(Up20200528140124, Down20200528140124)
+}
+
+// Up20200528140124 will insert the cvatTemplate to each user.
+// Each user is determined by onepanel enabled namespaces.
+// Any errors reported are logged as fatal.
+func Up20200528140124(tx *sql.Tx) error {
+	// This code is executed when the migration is applied.
+
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	workspaceTemplate := &v1.WorkspaceTemplate{
+		Name:     cvatTemplateName,
+		Manifest: cvatWorkspaceTemplate,
+	}
+
+	for _, namespace := range namespaces {
+		if _, err := client.CreateWorkspaceTemplate(namespace.Name, workspaceTemplate); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+
+	return nil
+}
+
+// Down20200528140124 will attempt to remove cvatTemplate from each user.
+// Each user is determined by onepanel enabled namespaces.
+// DB entries are archived, K8S components are deleted.
+// Active workspaces with that template are terminated.
+// Any errors reported are logged as fatal.
+func Down20200528140124(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	uid, err := uid2.GenerateUID(cvatTemplateName, 30)
+	if err != nil {
+		return err
+	}
+	for _, namespace := range namespaces {
+		if _, err := client.ArchiveWorkspaceTemplate(namespace.Name, uid); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+
+	return nil
+}

pkg/workspace.go

@@ -266,6 +266,40 @@ func (c *Client) UpdateWorkspaceStatus(namespace, uid string, status *WorkspaceS
 	return
 }
 
+// ListWorkspacesByTemplateID will return all the workspaces for a given workspace template id.
+// Sourced from database.
+func (c *Client) ListWorkspacesByTemplateID(namespace string, templateID uint64) (workspaces []*Workspace, err error) {
+	sb := sb.Select(getWorkspaceColumns("w", "")...).
+		From("workspaces w").
+		Where(sq.And{
+			sq.Eq{
+				"w.namespace":             namespace,
+				"w.workspace_template_id": templateID,
+			},
+			sq.NotEq{
+				"phase": WorkspaceTerminated,
+			},
+		})
+	query, args, err := sb.ToSql()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := c.DB.Select(&workspaces, query, args...); err != nil {
+		return nil, err
+	}
+
+	labelMap, err := c.GetDbLabelsMapped(TypeWorkspace, WorkspacesToIds(workspaces)...)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, workspace := range workspaces {
+		workspace.Labels = labelMap[workspace.ID]
+	}
+	return
+}
+
 func (c *Client) ListWorkspaces(namespace string, paginator *pagination.PaginationRequest) (workspaces []*Workspace, err error) {
 	sb := sb.Select(getWorkspaceColumns("w", "")...).
 		Columns(getWorkspaceStatusColumns("w", "status")...).
@@ -328,7 +362,9 @@ func (c *Client) updateWorkspace(namespace, uid, workspaceAction, resourceAction
 	if err != nil {
 		return util.NewUserError(codes.NotFound, "Workspace not found.")
 	}
-
+	if workspace == nil {
+		return nil
+	}
 	config, err := c.GetSystemConfig()
 	if err != nil {
 		return
@@ -407,3 +443,9 @@ func (c *Client) ResumeWorkspace(namespace, uid string) (err error) {
 func (c *Client) DeleteWorkspace(namespace, uid string) (err error) {
 	return c.updateWorkspace(namespace, uid, "delete", "delete", &WorkspaceStatus{Phase: WorkspaceTerminating})
 }
+
+// ArchiveWorkspace archives by setting the workspace to delete or terminate.
+// Kicks off DB archiving and k8s cleaning.
+func (c *Client) ArchiveWorkspace(namespace, uid string) (err error) {
+	return c.updateWorkspace(namespace, uid, "delete", "delete", &WorkspaceStatus{Phase: WorkspaceTerminating})
+}