feat: add API support to filter workflow templates by a label

api/api.swagger.json

@@ -3,7 +3,7 @@
   "info": {
     "title": "Onepanel",
     "description": "Onepanel API",
-    "version": "0.11.0",
+    "version": "0.12.0",
     "contact": {
       "name": "Onepanel project",
       "url": "https://github.com/onepanelio/core"
@@ -1364,6 +1364,12 @@
             "required": false,
             "type": "integer",
             "format": "int32"
+          },
+          {
+            "name": "labels",
+            "in": "query",
+            "required": false,
+            "type": "string"
           }
         ],
         "tags": [

api/workflow_template.proto

@@ -94,6 +94,8 @@ message ListWorkflowTemplatesRequest {
     string namespace = 1;
     int32 pageSize = 2;
     int32 page = 3;
+
+    string labels = 4;
 }
 
 message ListWorkflowTemplatesResponse {

db/go/20200728190804_update_workflow_template_labels.go

@@ -0,0 +1,70 @@
+package migration
+
+import (
+	"database/sql"
+	v1 "github.com/onepanelio/core/pkg"
+	"github.com/onepanelio/core/pkg/util/pagination"
+	"github.com/pressly/goose"
+)
+
+func initialize20200728190804() {
+	if _, ok := initializedMigrations[20200728190804]; !ok {
+		goose.AddMigration(Up20200728190804, Down20200728190804)
+		initializedMigrations[20200728190804] = true
+	}
+}
+
+// Up20200728190804 updates labels so that we keep track of WorkflowTemplate labels.
+// Before, only workflow template versions had labels, but to speed up some queries, we now cache the latest version's labels
+// for workflow templates themselves.
+func Up20200728190804(tx *sql.Tx) error {
+	// This code is executed when the migration is applied.
+	if migrationHasAlreadyBeenRun(20200728190804) {
+		return nil
+	}
+
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	for _, namespace := range namespaces {
+		paginator := pagination.Start(500)
+
+		resultCount := -1
+		for resultCount != 0 {
+			workflowTemplates, err := client.ListAllWorkflowTemplates(namespace.Name, paginator, nil)
+			if err != nil {
+				return err
+			}
+
+			for _, workflowTemplate := range workflowTemplates {
+				if err := client.ReplaceLabelsUsingKnownID(namespace.Name, v1.TypeWorkflowTemplate, workflowTemplate.ID, workflowTemplate.UID, workflowTemplate.Labels); err != nil {
+					return err
+				}
+			}
+
+			resultCount = len(workflowTemplates)
+			paginator = paginator.Advance()
+		}
+	}
+
+	return nil
+}
+
+// Down20200728190804 rolls down the migration by deleting all workflow template labels, since they did not exist before this
+func Down20200728190804(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	return client.DeleteResourceLabels(tx, v1.TypeWorkflowTemplate)
+}

db/go/db.go

@@ -4,21 +4,46 @@ import (
 	sq "github.com/Masterminds/squirrel"
 	"github.com/jmoiron/sqlx"
 	v1 "github.com/onepanelio/core/pkg"
+	"log"
 )
 
 // initializedMigrations is used to keep track of which migrations have been initialized.
-// if they are initialzed more than once, goose panics.
+// if they are initialized more than once, goose panics.
 var initializedMigrations = make(map[int]bool)
 
+// sqlRanMigrations keeps track of all the sql migrations that have been run.
+// we need to know this because in an older version some go migrations ran alongside sql.
+// So if they have already been run, we can't run them again.
+var sqlRanMigrations = make(map[uint64]bool)
+
+// migrationHasAlreadyBeenRun returns true if the migration has already been run in sql
+// see sqlRanMigrations var
+func migrationHasAlreadyBeenRun(version int) bool {
+	_, ok := sqlRanMigrations[uint64(version)]
+	return ok
+}
+
 // Initialize sets up the go migrations.
 func Initialize() {
+	client, err := getClient()
+	if err != nil {
+		log.Fatalf("unable to get client for go migrations: %v", err)
+	}
+
+	migrationsRan, err := getRanSQLMigrations(client)
+	if err != nil {
+		log.Fatalf("Unable to get already run sql migrations: %v", err)
+	}
+	sqlRanMigrations = migrationsRan
+
 	initialize20200525160514()
 	initialize20200528140124()
 	initialize20200605090509()
 	initialize20200605090535()
 	initialize20200626113635()
 	initialize20200704151301()
 	initialize20200724220450()
+  initialize20200728190804()
 }
 
 func getClient() (*v1.Client, error) {

pkg/label_types.go

@@ -1,6 +1,10 @@
 package v1
 
-import "time"
+import (
+	"fmt"
+	"strings"
+	"time"
+)
 
 // Label represents a database-backed label row.
 type Label struct {
@@ -22,3 +26,73 @@ func LabelsToMapping(labels ...*Label) map[string]string {
 
 	return result
 }
+
+// LabelsFromString parses a string into labels
+// Format: key=<key>,value=<value>&key2=<key2>,value2=<value2>
+func LabelsFromString(value string) (labels []*Label, err error) {
+	labels = make([]*Label, 0)
+
+	if value == "" {
+		return
+	}
+
+	labelParts := strings.Split(value, "&")
+	if len(labelParts) == 0 {
+		return
+	}
+
+	for _, part := range labelParts {
+		if part == "" {
+			continue
+		}
+
+		newLabel, err := LabelFromString(part)
+		if err != nil {
+			return labels, err
+		}
+		if newLabel == nil {
+			continue
+		}
+
+		labels = append(labels, newLabel)
+	}
+
+	return
+}
+
+// LabelFromString converts a parses into a label
+// Format: key=<key>,value=<value>
+func LabelFromString(value string) (label *Label, err error) {
+	parts := strings.Split(value, ",")
+	if len(parts) != 2 {
+		return nil, fmt.Errorf("label does not have two parts, key/value")
+	}
+
+	label = &Label{}
+
+	first := parts[0]
+	firstItems := strings.Split(first, "=")
+	if len(firstItems) != 2 {
+		return nil, fmt.Errorf(`incorrectly formatted label "%v"`, first)
+	}
+
+	if firstItems[0] == "key" {
+		label.Key = firstItems[1]
+	} else if firstItems[0] == "value" {
+		label.Value = firstItems[1]
+	}
+
+	second := parts[1]
+	secondItems := strings.Split(second, "=")
+	if len(secondItems) != 2 {
+		return nil, fmt.Errorf(`incorrectly formatted label "%v"`, second)
+	}
+
+	if secondItems[0] == "key" {
+		label.Key = secondItems[1]
+	} else if secondItems[0] == "value" {
+		label.Value = secondItems[1]
+	}
+
+	return label, nil
+}

pkg/label_types_test.go

@@ -0,0 +1,60 @@
+package v1
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestLabelFromString tests the LabelFromString function
+func TestLabelFromString(t *testing.T) {
+	// Blank value gives us no label
+	label, err := LabelFromString("")
+	assert.NotNil(t, err)
+	assert.Nil(t, label)
+
+	// Missing value, should give error
+	label, err = LabelFromString("key=a")
+	assert.NotNil(t, err)
+
+	// Missing value, but still have comma, should give error
+	label, err = LabelFromString("key=a,")
+	assert.NotNil(t, err)
+
+	// Missing key, should give error
+	label, err = LabelFromString("value=a")
+	assert.NotNil(t, err)
+
+	// Missing key, still have comma, should give error
+	label, err = LabelFromString("value=a,")
+	assert.NotNil(t, err)
+
+	// Correct, should not give an error
+	label, err = LabelFromString("key=a,value=b")
+	assert.Nil(t, err)
+	assert.Equal(t, label.Key, "a")
+	assert.Equal(t, label.Value, "b")
+}
+
+// TestLabelsFromString tests the LabelsFromString function
+func TestLabelsFromString(t *testing.T) {
+	// Empty should give no error and no labels
+	labels, err := LabelsFromString("")
+	assert.Nil(t, err)
+	assert.Len(t, labels, 0)
+
+	// Bad data, should give no labels
+	labels, err = LabelsFromString("&&&&")
+	assert.Nil(t, err)
+	assert.Len(t, labels, 0)
+
+	// Test just one label
+	labels, err = LabelsFromString("key=a,value=b")
+	assert.Nil(t, err)
+	assert.Len(t, labels, 1)
+
+	// Test many labels
+	labels, err = LabelsFromString("key=a,value=b&key=c,value=d&key=e,value=f")
+	assert.Nil(t, err)
+	assert.Len(t, labels, 3)
+}

pkg/labels.go

@@ -163,7 +163,8 @@ func (c *Client) ReplaceLabelsUsingKnownID(namespace, resource string, resourceI
 		Where(sq.Eq{
 			"resource":    resource,
 			"resource_id": resourceID,
-		}).RunWith(tx).
+		}).
+		RunWith(tx).
 		Exec()
 	if err != nil {
 		return err
@@ -261,6 +262,24 @@ func (c *Client) DeleteLabels(namespace, resource, uid string, keyValues map[str
 	return nil
 }
 
+// DeleteResourceLabels deletes all of the labels for a specific resource, like workflow templates.
+// NOTE: this does NOT delete k8s labels, and is only meant to be used for special cases.
+func (c *Client) DeleteResourceLabels(runner sq.BaseRunner, resource string) error {
+	tableName := TypeToTableName(resource)
+	if tableName == "" {
+		return fmt.Errorf("unknown resources '%v'", resource)
+	}
+
+	_, err := sb.Delete("labels").
+		Where(sq.Eq{
+			"resource": resource,
+		}).
+		RunWith(runner).
+		Exec()
+
+	return err
+}
+
 func (c *Client) InsertLabelsBuilder(resource string, resourceID uint64, keyValues map[string]string) sq.InsertBuilder {
 	sb := sb.Insert("labels").
 		Columns("resource", "resource_id", "key", "value")

pkg/util/pagination/pagination.go

@@ -25,6 +25,20 @@ func NewRequest(page, pageSize int32) PaginationRequest {
 	}
 }
 
+// Start creates a new PaginationRequest that starts at the first page.
+// You can provide an optional pageSize argument. If none is provided, 15 is used.
+// All arguments apart from the first one are ignored.
+func Start(pageSize ...int32) *PaginationRequest {
+	if len(pageSize) > 0 {
+		pr := NewRequest(1, pageSize[0])
+		return &pr
+	}
+
+	pr := NewRequest(1, 15)
+
+	return &pr
+}
+
 func (pr *PaginationRequest) Offset() uint64 {
 	// start at page 1.
 	return (pr.Page - 1) * pr.PageSize
@@ -44,3 +58,11 @@ func (pr *PaginationRequest) ApplyToSelect(sb *squirrel.SelectBuilder) *squirrel
 
 	return &result
 }
+
+// Advance returns a new pagination request with the page incremented by 1
+func (pr *PaginationRequest) Advance() *PaginationRequest {
+	return &PaginationRequest{
+		Page:     pr.Page + 1,
+		PageSize: pr.PageSize,
+	}
+}