Skip to content

Commit a47bbe9

Browse files
rushtehranirushtehrani
authored
Merge pull request #286 from onepanelio/feat/add.cvat.migration.template
feat: add cvat migration template
2 parents 183296f + 7c72a36 commit a47bbe9

File tree

6 files changed

+366
-49
lines changed

6 files changed

+366
-49
lines changed

‎Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ COPY . .
55

66
RUN go get -d -v ./...
77
RUN go install -v ./...
8-
8+
RUN go get -u github.com/pressly/goose/cmd/goose
9+
RUN go build -o /go/bin/goose ./cmd/goose.go
910

1011
FROM golang:1.13.10
1112
COPY --from=builder /go/bin/core .
1213
COPY --from=builder /go/src/db ./db
14+
COPY --from=builder /go/bin/goose .
1315

1416
EXPOSE 8888
1517
EXPOSE 8887

‎cmd/goose.go‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// This is custom goose binary to support .go migration files in ./db dir
2+
3+
package main
4+
5+
import (
6+
"flag"
7+
"fmt"
8+
"github.com/jmoiron/sqlx"
9+
_ "github.com/onepanelio/core/db"
10+
v1 "github.com/onepanelio/core/pkg"
11+
"log"
12+
"os"
13+
14+
"github.com/pressly/goose"
15+
)
16+
17+
var (
18+
flags = flag.NewFlagSet("goose", flag.ExitOnError)
19+
dir = flags.String("dir", ".", "directory with migration files")
20+
)
21+
22+
func main() {
23+
flags.Parse(os.Args[1:])
24+
args := flags.Args()
25+
26+
if len(args) < 1 {
27+
flags.Usage()
28+
return
29+
}
30+
31+
kubeConfig := v1.NewConfig()
32+
client, err := v1.NewClient(kubeConfig, nil)
33+
if err != nil {
34+
log.Fatalf("Failed to connect to Kubernetes cluster: %v", err)
35+
}
36+
config, err := client.GetSystemConfig()
37+
if err != nil {
38+
log.Fatalf("Failed to get system config: %v", err)
39+
}
40+
41+
databaseDataSourceName := fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable",
42+
config["databaseHost"], config["databaseUsername"], config["databasePassword"], config["databaseName"])
43+
44+
db := sqlx.MustConnect(config["databaseDriverName"], databaseDataSourceName)
45+
46+
command := args[0]
47+
48+
arguments := []string{}
49+
if len(args) > 2 {
50+
arguments = append(arguments, args[2:]...)
51+
}
52+
53+
if err := goose.Run(command, db.DB, *dir, arguments...); err != nil {
54+
log.Fatalf("goose %v: %v", command, err)
55+
}
56+
}

‎db/20200525160514_add_jupyter_workspace_template.go‎

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1+
// Package migration is for carrying out migrations against the database.
2+
// To support Onepanel Core operations.
13
package migration
24

35
import (
46
"database/sql"
57
v1 "github.com/onepanelio/core/pkg"
8+
uid2 "github.com/onepanelio/core/pkg/util/uid"
69
"github.com/pressly/goose"
710
"log"
811
)
@@ -48,22 +51,23 @@ routes:
4851
port:
4952
number: 80
5053
# DAG Workflow to be executed once a Workspace action completes
51-
postExecutionWorkflow:
52-
entrypoint: main
53-
templates:
54-
- name: main
55-
dag:
56-
tasks:
57-
- name: slack-notify
58-
template: slack-notify
59-
- name: slack-notify
60-
container:
61-
image: technosophos/slack-notify
62-
args:
63-
- SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
64-
command:
65-
- sh
66-
- -c`
54+
# postExecutionWorkflow:
55+
# entrypoint: main
56+
# templates:
57+
# - name: main
58+
# dag:
59+
# tasks:
60+
# - name: slack-notify
61+
# template: slack-notify
62+
# - name: slack-notify
63+
# container:
64+
# image: technosophos/slack-notify
65+
# args:
66+
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
67+
# command:
68+
# - sh
69+
# - -c
70+
`
6771

6872
const jupyterLabTemplateName = "JupyterLab"
6973

@@ -89,7 +93,7 @@ func Up20200525160514(tx *sql.Tx) error {
8993

9094
for _, namespace := range namespaces {
9195
if _, err := client.CreateWorkspaceTemplate(namespace.Name, workspaceTemplate); err != nil {
92-
log.Printf("error %v", err.Error())
96+
log.Fatalf("error %v", err.Error())
9397
}
9498
}
9599

@@ -106,10 +110,13 @@ func Down20200525160514(tx *sql.Tx) error {
106110
if err != nil {
107111
return err
108112
}
109-
113+
uid, err := uid2.GenerateUID(jupyterLabTemplateName, 30)
114+
if err != nil {
115+
return err
116+
}
110117
for _, namespace := range namespaces {
111-
if err := client.DeleteWorkspace(namespace.Name, jupyterLabTemplateName); err != nil {
112-
log.Printf("error %v", err.Error())
118+
if _, err := client.ArchiveWorkspaceTemplate(namespace.Name, uid); err != nil {
119+
log.Fatalf("error %v", err.Error())
113120
}
114121
}
115122

‎db/20200528140124_add_cvat_workspace_template.go‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
package migration
2+
3+
import (
4+
"database/sql"
5+
v1 "github.com/onepanelio/core/pkg"
6+
uid2 "github.com/onepanelio/core/pkg/util/uid"
7+
"github.com/pressly/goose"
8+
"log"
9+
)
10+
11+
const cvatWorkspaceTemplate = `# Docker containers that are part of the Workspace
12+
containers:
13+
- name: cvat-db
14+
image: postgres:10-alpine
15+
env:
16+
- name: POSTGRES_USER
17+
value: root
18+
- name: POSTGRES_DB
19+
value: cvat
20+
- name: POSTGRES_HOST_AUTH_METHOD
21+
value: trust
22+
- name: PGDATA
23+
value: /var/lib/psql/data
24+
ports:
25+
- containerPort: 5432
26+
name: tcp
27+
volumeMounts:
28+
- name: db
29+
mountPath: /var/lib/psql
30+
- name: cvat-redis
31+
image: redis:4.0-alpine
32+
ports:
33+
- containerPort: 6379
34+
name: tcp
35+
- name: cvat
36+
image: onepanel/cvat:v0.7.0
37+
env:
38+
- name: DJANGO_MODWSGI_EXTRA_ARGS
39+
value: ""
40+
- name: ALLOWED_HOSTS
41+
value: '*'
42+
- name: CVAT_REDIS_HOST
43+
value: localhost
44+
- name: CVAT_POSTGRES_HOST
45+
value: localhost
46+
- name: CVAT_SHARE_URL
47+
value: /home/django/data
48+
ports:
49+
- containerPort: 8080
50+
name: http
51+
volumeMounts:
52+
- name: data
53+
mountPath: /home/django/data
54+
- name: keys
55+
mountPath: /home/django/keys
56+
- name: logs
57+
mountPath: /home/django/logs
58+
- name: models
59+
mountPath: /home/django/models
60+
- name: cvat-ui
61+
image: onepanel/cvat-ui:v0.7.0
62+
ports:
63+
- containerPort: 80
64+
name: http
65+
ports:
66+
- name: cvat-ui
67+
port: 80
68+
protocol: TCP
69+
targetPort: 80
70+
- name: cvat
71+
port: 8080
72+
protocol: TCP
73+
targetPort: 8080
74+
routes:
75+
- match:
76+
- uri:
77+
regex: /api/.*|/git/.*|/tensorflow/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
78+
- queryParams:
79+
id:
80+
regex: \d+.*
81+
route:
82+
- destination:
83+
port:
84+
number: 8080
85+
- match:
86+
- uri:
87+
prefix: /
88+
route:
89+
- destination:
90+
port:
91+
number: 80
92+
# DAG Workflow to be executed once a Workspace action completes
93+
# postExecutionWorkflow:
94+
# entrypoint: main
95+
# templates:
96+
# - name: main
97+
# dag:
98+
# tasks:
99+
# - name: slack-notify
100+
# template: slack-notify
101+
# - name: slack-notify
102+
# container:
103+
# image: technosophos/slack-notify
104+
# args:
105+
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
106+
# command:
107+
# - sh
108+
# - -c
109+
`
110+
111+
const cvatTemplateName = "CVAT"
112+
113+
func init() {
114+
goose.AddMigration(Up20200528140124, Down20200528140124)
115+
}
116+
117+
// Up20200528140124 will insert the cvatTemplate to each user.
118+
// Each user is determined by onepanel enabled namespaces.
119+
// Any errors reported are logged as fatal.
120+
func Up20200528140124(tx *sql.Tx) error {
121+
// This code is executed when the migration is applied.
122+
123+
client, err := getClient()
124+
if err != nil {
125+
return err
126+
}
127+
128+
namespaces, err := client.ListOnepanelEnabledNamespaces()
129+
if err != nil {
130+
return err
131+
}
132+
133+
workspaceTemplate := &v1.WorkspaceTemplate{
134+
Name: cvatTemplateName,
135+
Manifest: cvatWorkspaceTemplate,
136+
}
137+
138+
for _, namespace := range namespaces {
139+
if _, err := client.CreateWorkspaceTemplate(namespace.Name, workspaceTemplate); err != nil {
140+
log.Fatalf("error %v", err.Error())
141+
}
142+
}
143+
144+
return nil
145+
}
146+
147+
// Down20200528140124 will attempt to remove cvatTemplate from each user.
148+
// Each user is determined by onepanel enabled namespaces.
149+
// DB entries are archived, K8S components are deleted.
150+
// Active workspaces with that template are terminated.
151+
// Any errors reported are logged as fatal.
152+
func Down20200528140124(tx *sql.Tx) error {
153+
// This code is executed when the migration is rolled back.
154+
155+
client, err := getClient()
156+
if err != nil {
157+
return err
158+
}
159+
160+
namespaces, err := client.ListOnepanelEnabledNamespaces()
161+
if err != nil {
162+
return err
163+
}
164+
165+
uid, err := uid2.GenerateUID(cvatTemplateName, 30)
166+
if err != nil {
167+
return err
168+
}
169+
for _, namespace := range namespaces {
170+
if _, err := client.ArchiveWorkspaceTemplate(namespace.Name, uid); err != nil {
171+
log.Fatalf("error %v", err.Error())
172+
}
173+
}
174+
175+
return nil
176+
}

‎pkg/workspace.go‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,40 @@ func (c *Client) UpdateWorkspaceStatus(namespace, uid string, status *WorkspaceS
266266
return
267267
}
268268

269+
// ListWorkspacesByTemplateID will return all the workspaces for a given workspace template id.
270+
// Sourced from database.
271+
func (c *Client) ListWorkspacesByTemplateID(namespace string, templateID uint64) (workspaces []*Workspace, err error) {
272+
sb := sb.Select(getWorkspaceColumns("w", "")...).
273+
From("workspaces w").
274+
Where(sq.And{
275+
sq.Eq{
276+
"w.namespace": namespace,
277+
"w.workspace_template_id": templateID,
278+
},
279+
sq.NotEq{
280+
"phase": WorkspaceTerminated,
281+
},
282+
})
283+
query, args, err := sb.ToSql()
284+
if err != nil {
285+
return nil, err
286+
}
287+
288+
if err := c.DB.Select(&workspaces, query, args...); err != nil {
289+
return nil, err
290+
}
291+
292+
labelMap, err := c.GetDbLabelsMapped(TypeWorkspace, WorkspacesToIds(workspaces)...)
293+
if err != nil {
294+
return nil, err
295+
}
296+
297+
for _, workspace := range workspaces {
298+
workspace.Labels = labelMap[workspace.ID]
299+
}
300+
return
301+
}
302+
269303
func (c *Client) ListWorkspaces(namespace string, paginator *pagination.PaginationRequest) (workspaces []*Workspace, err error) {
270304
sb := sb.Select(getWorkspaceColumns("w", "")...).
271305
Columns(getWorkspaceStatusColumns("w", "status")...).
@@ -328,7 +362,9 @@ func (c *Client) updateWorkspace(namespace, uid, workspaceAction, resourceAction
328362
if err != nil {
329363
return util.NewUserError(codes.NotFound, "Workspace not found.")
330364
}
331-
365+
if workspace == nil {
366+
return nil
367+
}
332368
config, err := c.GetSystemConfig()
333369
if err != nil {
334370
return
@@ -407,3 +443,9 @@ func (c *Client) ResumeWorkspace(namespace, uid string) (err error) {
407443
func (c *Client) DeleteWorkspace(namespace, uid string) (err error) {
408444
return c.updateWorkspace(namespace, uid, "delete", "delete", &WorkspaceStatus{Phase: WorkspaceTerminating})
409445
}
446+
447+
// ArchiveWorkspace archives by setting the workspace to delete or terminate.
448+
// Kicks off DB archiving and k8s cleaning.
449+
func (c *Client) ArchiveWorkspace(namespace, uid string) (err error) {
450+
return c.updateWorkspace(namespace, uid, "delete", "delete", &WorkspaceStatus{Phase: WorkspaceTerminating})
451+
}

0 commit comments

Comments
 (0)