diff --git a/db/data/cvat_20201016170415.yaml b/db/data/cvat_20201016170415.yaml new file mode 100644 index 00000000..a2065991 --- /dev/null +++ b/db/data/cvat_20201016170415.yaml @@ -0,0 +1,147 @@ +# Workspace arguments +arguments: + parameters: + - name: sync-directory + displayName: Directory to sync raw input and training output + value: workflow-data + hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace. +containers: + - name: cvat-db + image: postgres:10-alpine + env: + - name: POSTGRES_USER + value: root + - name: POSTGRES_DB + value: cvat + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/psql/data + ports: + - containerPort: 5432 + name: tcp + volumeMounts: + - name: db + mountPath: /var/lib/psql + - name: cvat-redis + image: redis:4.0-alpine + ports: + - containerPort: 6379 + name: tcp + - name: cvat + image: onepanel/cvat:0.14.0_cvat.1.0.0 + env: + - name: DJANGO_MODWSGI_EXTRA_ARGS + value: "" + - name: ALLOWED_HOSTS + value: '*' + - name: CVAT_REDIS_HOST + value: localhost + - name: CVAT_POSTGRES_HOST + value: localhost + - name: CVAT_SHARE_URL + value: /home/django/data + - name: ONEPANEL_SYNC_DIRECTORY + value: '{{workspace.parameters.sync-directory}}' + - name: NVIDIA_VISIBLE_DEVICES + value: all + - name: NVIDIA_DRIVER_CAPABILITIES + value: compute,utility + - name: NVIDIA_REQUIRE_CUDA + value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411" + ports: + - containerPort: 8080 + name: http + volumeMounts: + - name: data + mountPath: /home/django/data + - name: keys + mountPath: /home/django/keys + - name: logs + mountPath: /home/django/logs + - name: models + mountPath: /home/django/models + - name: share + mountPath: /home/django/share + - name: sys-namespace-config + mountPath: /etc/onepanel + readOnly: true + - name: cvat-ui + image: onepanel/cvat-ui:0.14.0_cvat.1.0.0 + ports: + - containerPort: 80 + name: http + # You can add multiple FileSyncer sidecar containers if needed + - name: filesyncer + image: onepanel/filesyncer:s3 + imagePullPolicy: Always + args: + - download + - -server-prefix=/sys/filesyncer + env: + - name: FS_PATH + value: /mnt/share + - name: FS_PREFIX + value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}' + volumeMounts: + - name: share + mountPath: /mnt/share + - name: sys-namespace-config + mountPath: /etc/onepanel + readOnly: true +ports: + - name: cvat-ui + port: 80 + protocol: TCP + targetPort: 80 + - name: cvat + port: 8080 + protocol: TCP + targetPort: 8080 + - name: fs + port: 8888 + protocol: TCP + targetPort: 8888 +routes: + - match: + - uri: + prefix: /sys/filesyncer + route: + - destination: + port: + number: 8888 + - match: + - uri: + regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.* + - queryParams: + id: + regex: \d+.* + route: + - destination: + port: + number: 8080 + - match: + - uri: + prefix: / + route: + - destination: + port: + number: 80 +# DAG Workflow to be executed once a Workspace action completes (optional) +# Uncomment the lines below if you want to send Slack notifications +#postExecutionWorkflow: +# entrypoint: main +# templates: +# - name: main +# dag: +# tasks: +# - name: slack-notify +# template: slack-notify +# - name: slack-notify +# container: +# image: technosophos/slack-notify +# args: +# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify +# command: +# - sh +# - -c \ No newline at end of file diff --git a/db/go/20201016170415_update_cvat.go b/db/go/20201016170415_update_cvat.go new file mode 100644 index 00000000..11533123 --- /dev/null +++ b/db/go/20201016170415_update_cvat.go @@ -0,0 +1,25 @@ +package migration + +import ( + "database/sql" + "github.com/pressly/goose" +) + +func initialize20201016170415() { + if _, ok := initializedMigrations[20201016170415]; !ok { + goose.AddMigration(Up20201016170415, Down20201016170415) + initializedMigrations[20201016170415] = true + } +} + +// Up20201016170415 updates cvat to a new version +func Up20201016170415(tx *sql.Tx) error { + // This code is executed when the migration is applied. + return updateWorkspaceTemplateManifest("cvat_20201016170415.yaml", cvatTemplateName) +} + +// Down20201016170415 does nothing +func Down20201016170415(tx *sql.Tx) error { + // This code is executed when the migration is rolled back. + return nil +} diff --git a/db/go/db.go b/db/go/db.go index 67759069..c8683094 100644 --- a/db/go/db.go +++ b/db/go/db.go @@ -5,7 +5,10 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" v1 "github.com/onepanelio/core/pkg" + "io/ioutil" "log" + "os" + "path/filepath" "strings" ) @@ -60,6 +63,7 @@ func Initialize() { initialize20200929144301() initialize20200929153931() initialize20201001070806() + initialize20201016170415() if err := client.DB.Close(); err != nil { log.Printf("[error] closing db %v", err) @@ -128,3 +132,17 @@ func ReplaceArtifactRepositoryType(client *v1.Client, namespace *v1.Namespace, w return nil } + +// readDataFile returns the contents of a file in the db/data/{name} directory +func readDataFile(name string) (string, error) { + curDir, err := os.Getwd() + if err != nil { + return "", err + } + data, err := ioutil.ReadFile(filepath.Join(curDir, "db", "data", name)) + if err != nil { + return "", err + } + + return string(data), nil +} diff --git a/db/go/util.go b/db/go/util.go new file mode 100644 index 00000000..fa402af4 --- /dev/null +++ b/db/go/util.go @@ -0,0 +1,49 @@ +package migration + +import ( + v1 "github.com/onepanelio/core/pkg" + uid2 "github.com/onepanelio/core/pkg/util/uid" +) + +// updateWorkspaceTemplateManifest will update the workspace template given by {{templateName}} with the contents +// given by {{filename}} +// It will do so for all namespaces. +func updateWorkspaceTemplateManifest(filename, templateName string) error { + client, err := getClient() + if err != nil { + return err + } + defer client.DB.Close() + + namespaces, err := client.ListOnepanelEnabledNamespaces() + if err != nil { + return err + } + + newManifest, err := readDataFile(filename) + if err != nil { + return err + } + + uid, err := uid2.GenerateUID(templateName, 30) + if err != nil { + return err + } + + for _, namespace := range namespaces { + workspaceTemplate := &v1.WorkspaceTemplate{ + UID: uid, + Name: templateName, + Manifest: newManifest, + } + err = ReplaceArtifactRepositoryType(client, namespace, nil, workspaceTemplate) + if err != nil { + return err + } + if _, err := client.UpdateWorkspaceTemplateManifest(namespace.Name, uid, workspaceTemplate.Manifest); err != nil { + return err + } + } + + return nil +} diff --git a/main.go b/main.go index 268d6f29..77c9d870 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + migrations "github.com/onepanelio/core/db/go" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" @@ -25,7 +26,6 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/runtime" "github.com/jmoiron/sqlx" "github.com/onepanelio/core/api" - migrations "github.com/onepanelio/core/db/go" v1 "github.com/onepanelio/core/pkg" "github.com/onepanelio/core/pkg/util/env" "github.com/onepanelio/core/server"