Add subPath to WorkspacePipelineTaskBinding

Use case: Use two instances of a task that writes to the same workspace volume - but
they write to different path of the volume. A typical use case is two git-clone tasks
that clone two git repositories, but the files should be located in two different
directories on the workspace volume.

This commit solves this by adding the `subPath` field to the WorkspacePipelineTaskBinding.

Author: jlpettersson

docs/workspaces.md

@@ -222,6 +222,10 @@ spec:
         - use-ws-from-pipeline # important: use-ws-from-pipeline writes to the workspace first
 ```
 
+Include a `subPath` in the workspace binding to mount different parts of the same volume for different Tasks. See [a full example of this kind of Pipeline](../examples/v1beta1/pipelineruns/pipelinerun-using-different-subpaths-of-workspace.yaml) which writes data to two adjacent directories on the same Volume.
+
+The `subPath` specified in a `Pipeline` will be appended to any `subPath` specified as part of the `PipelineRun` workspace declaration. So a `PipelineRun` declaring a Workspace with `subPath` of `/foo` for a `Pipeline` who binds it to a `Task` with `subPath` of `/bar` will end up mounting the `Volume`'s `/foo/bar` directory.
+
 #### Specifying `Workspace` order in a `Pipeline`
 
 Sharing a `Workspace` between `Tasks` requires you to define the order in which those `Tasks`

examples/v1beta1/pipelineruns/pipelinerun-using-different-subpaths-of-workspace.yaml

@@ -0,0 +1,85 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: writer
+spec:
+  steps:
+    - name: write
+      image: ubuntu
+      script: echo bar > $(workspaces.task-ws.path)/foo
+  workspaces:
+    - name: task-ws
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: read-both
+spec:
+  params:
+    - name: directory1
+      type: string
+    - name: directory2
+      type: string
+  workspaces:
+    - name: local-ws
+  steps:
+    - name: read-1
+      image: ubuntu
+      script: cat $(workspaces.local-ws.path)/$(params.directory1)/foo | grep bar
+    - name: read-2
+      image: ubuntu
+      script: cat $(workspaces.local-ws.path)/$(params.directory2)/foo | grep bar
+---
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: pipeline-using-different-subpaths
+spec:
+  workspaces:
+    - name: ws
+  tasks:
+    - name: writer-1
+      taskRef:
+        name: writer
+      workspaces:
+        - name: task-ws
+          workspace: ws
+          subPath: dir-1
+    - name: writer-2
+      taskRef:
+        name: writer
+      workspaces:
+        - name: task-ws
+          workspace: ws
+          subPath: dir-2
+    - name: read-all
+      runAfter:
+        - writer-1
+        - writer-2
+      params:
+        - name: directory1
+          value: dir-1
+        - name: directory2
+          value: dir-2
+      taskRef:
+        name: read-both
+      workspaces:
+        - name: local-ws
+          workspace: ws
+---
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  generateName: pr-
+spec:
+  pipelineRef:
+    name: pipeline-using-different-subpaths
+  workspaces:
+    - name: ws
+      volumeClaimTemplate:
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 1Gi

pkg/apis/pipeline/v1alpha1/pipeline_validation_test.go

@@ -359,7 +359,7 @@ func TestPipeline_Validate(t *testing.T) {
 		p: tb.Pipeline("name", tb.PipelineSpec(
 			tb.PipelineWorkspaceDeclaration("foo"),
 			tb.PipelineTask("taskname", "taskref",
-				tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", "pipelineWorkspaceName")),
+				tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", "pipelineWorkspaceName", "")),
 		)),
 		failureExpected: true,
 	}, {

pkg/apis/pipeline/v1beta1/workspace_types.go

@@ -95,4 +95,8 @@ type WorkspacePipelineTaskBinding struct {
 	Name string `json:"name"`
 	// Workspace is the name of the workspace declared by the pipeline
 	Workspace string `json:"workspace"`
+	// SubPath is optionally a directory on the volume which should be used
+	// for this binding (i.e. the volume will be mounted at this sub directory).
+	// +optional
+	SubPath string `json:"subPath,omitempty"`
 }

pkg/reconciler/pipelinerun/pipelinerun.go

@@ -19,6 +19,7 @@ package pipelinerun
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"time"
@@ -733,9 +734,9 @@ func (c *Reconciler) createTaskRun(rprt *resources.ResolvedPipelineRunTask, pr *
 		pipelineRunWorkspaces[binding.Name] = binding
 	}
 	for _, ws := range rprt.PipelineTask.Workspaces {
-		taskWorkspaceName, pipelineWorkspaceName := ws.Name, ws.Workspace
+		taskWorkspaceName, pipelineTaskSubPath, pipelineWorkspaceName := ws.Name, ws.SubPath, ws.Workspace
 		if b, hasBinding := pipelineRunWorkspaces[pipelineWorkspaceName]; hasBinding {
-			tr.Spec.Workspaces = append(tr.Spec.Workspaces, taskWorkspaceByWorkspaceVolumeSource(b, taskWorkspaceName, pr.GetOwnerReference()))
+			tr.Spec.Workspaces = append(tr.Spec.Workspaces, taskWorkspaceByWorkspaceVolumeSource(b, taskWorkspaceName, pipelineTaskSubPath, pr.GetOwnerReference()))
 		} else {
 			return nil, fmt.Errorf("expected workspace %q to be provided by pipelinerun for pipeline task %q", pipelineWorkspaceName, rprt.PipelineTask.Name)
 		}
@@ -748,16 +749,17 @@ func (c *Reconciler) createTaskRun(rprt *resources.ResolvedPipelineRunTask, pr *
 
 // taskWorkspaceByWorkspaceVolumeSource is returning the WorkspaceBinding with the TaskRun specified name.
 // If the volume source is a volumeClaimTemplate, the template is applied and passed to TaskRun as a persistentVolumeClaim
-func taskWorkspaceByWorkspaceVolumeSource(wb v1alpha1.WorkspaceBinding, taskWorkspaceName string, owner metav1.OwnerReference) v1alpha1.WorkspaceBinding {
+func taskWorkspaceByWorkspaceVolumeSource(wb v1alpha1.WorkspaceBinding, taskWorkspaceName string, pipelineTaskSubPath string, owner metav1.OwnerReference) v1alpha1.WorkspaceBinding {
 	if wb.VolumeClaimTemplate == nil {
 		binding := *wb.DeepCopy()
 		binding.Name = taskWorkspaceName
+		binding.SubPath = combinedSubPath(wb.SubPath, pipelineTaskSubPath)
 		return binding
 	}
 
 	// apply template
 	binding := v1alpha1.WorkspaceBinding{
-		SubPath: wb.SubPath,
+		SubPath: combinedSubPath(wb.SubPath, pipelineTaskSubPath),
 		PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
 			ClaimName: volumeclaim.GetPersistentVolumeClaimName(wb.VolumeClaimTemplate, wb, owner),
 		},
@@ -766,6 +768,18 @@ func taskWorkspaceByWorkspaceVolumeSource(wb v1alpha1.WorkspaceBinding, taskWork
 	return binding
 }
 
+// combinedSubPath returns the combined value of the optional subPath from workspaceBinding and the optional
+// subPath from pipelineTask. If both is set, they are joined with a slash.
+func combinedSubPath(workspaceSubPath string, pipelineTaskSubPath string) string {
+	if workspaceSubPath == "" {
+		return pipelineTaskSubPath
+	} else if pipelineTaskSubPath == "" {
+		return workspaceSubPath
+	} else {
+		return filepath.Join(workspaceSubPath, pipelineTaskSubPath)
+	}
+}
+
 func addRetryHistory(tr *v1alpha1.TaskRun) {
 	newStatus := *tr.Status.DeepCopy()
 	newStatus.RetriesStatus = nil

pkg/reconciler/pipelinerun/pipelinerun_test.go

@@ -1718,13 +1718,13 @@ func TestReconcileWithVolumeClaimTemplateWorkspace(t *testing.T) {
 	claimName := "myclaim"
 	pipelineRunName := "test-pipeline-run"
 	ps := []*v1alpha1.Pipeline{tb.Pipeline("test-pipeline", tb.PipelineNamespace("foo"), tb.PipelineSpec(
-		tb.PipelineTask("hello-world-1", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName)),
+		tb.PipelineTask("hello-world-1", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, "")),
 		tb.PipelineTask("hello-world-2", "hello-world"),
 		tb.PipelineWorkspaceDeclaration(workspaceName),
 	))}
 
 	prs := []*v1alpha1.PipelineRun{tb.PipelineRun(pipelineRunName, tb.PipelineRunNamespace("foo"),
-		tb.PipelineRunSpec("test-pipeline", tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceName, claimName))),
+		tb.PipelineRunSpec("test-pipeline", tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceName, claimName, ""))),
 	}
 	ts := []*v1alpha1.Task{tb.Task("hello-world", tb.TaskNamespace("foo"))}
 
@@ -1792,6 +1792,120 @@ func TestReconcileWithVolumeClaimTemplateWorkspace(t *testing.T) {
 	}
 }
 
+// TestReconcileWithVolumeClaimTemplateWorkspaceUsingSubPaths tests that given a pipeline with volumeClaimTemplate workspace and
+// multiple instances of the same task, but using different subPaths in the volume - is seen as taskRuns with expected subPaths.
+func TestReconcileWithVolumeClaimTemplateWorkspaceUsingSubPaths(t *testing.T) {
+	workspaceName := "ws1"
+	workspaceNameWithSubPath := "ws2"
+	subPath1 := "customdirectory"
+	subPath2 := "otherdirecory"
+	pipelineRunWsSubPath := "mypath"
+	ps := []*v1alpha1.Pipeline{tb.Pipeline("test-pipeline", tb.PipelineNamespace("foo"), tb.PipelineSpec(
+		tb.PipelineTask("hello-world-1", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, subPath1)),
+		tb.PipelineTask("hello-world-2", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, subPath2)),
+		tb.PipelineTask("hello-world-3", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceName, "")),
+		tb.PipelineTask("hello-world-4", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceNameWithSubPath, "")),
+		tb.PipelineTask("hello-world-5", "hello-world", tb.PipelineTaskWorkspaceBinding("taskWorkspaceName", workspaceNameWithSubPath, subPath1)),
+		tb.PipelineWorkspaceDeclaration(workspaceName, workspaceNameWithSubPath),
+	))}
+
+	prs := []*v1alpha1.PipelineRun{tb.PipelineRun("test-pipeline-run", tb.PipelineRunNamespace("foo"),
+		tb.PipelineRunSpec("test-pipeline",
+			tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceName, "myclaim", ""),
+			tb.PipelineRunWorkspaceBindingVolumeClaimTemplate(workspaceNameWithSubPath, "myclaim", pipelineRunWsSubPath))),
+	}
+	ts := []*v1alpha1.Task{tb.Task("hello-world", tb.TaskNamespace("foo"))}
+
+	d := test.Data{
+		PipelineRuns: prs,
+		Pipelines:    ps,
+		Tasks:        ts,
+	}
+
+	testAssets, cancel := getPipelineRunController(t, d)
+	defer cancel()
+	c := testAssets.Controller
+	clients := testAssets.Clients
+
+	err := c.Reconciler.Reconcile(context.Background(), "foo/test-pipeline-run")
+	if err != nil {
+		t.Errorf("Did not expect to see error when reconciling PipelineRun but saw %s", err)
+	}
+
+	// Check that the PipelineRun was reconciled correctly
+	reconciledRun, err := clients.Pipeline.TektonV1alpha1().PipelineRuns("foo").Get("test-pipeline-run", metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("Somehow had error getting reconciled run out of fake client: %s", err)
+	}
+
+	taskRuns, err := clients.Pipeline.TektonV1alpha1().TaskRuns("foo").List(metav1.ListOptions{})
+	if err != nil {
+		t.Fatalf("unexpected error when listing TaskRuns: %v", err)
+	}
+
+	if len(taskRuns.Items) != 5 {
+		t.Fatalf("unexpected number of taskRuns found, expected 2, but found %d", len(taskRuns.Items))
+	}
+
+	hasSeenWorkspaceWithPipelineTaskSubPath1 := false
+	hasSeenWorkspaceWithPipelineTaskSubPath2 := false
+	hasSeenWorkspaceWithEmptyPipelineTaskSubPath := false
+	hasSeenWorkspaceWithRunSubPathAndEmptyPipelineTaskSubPath := false
+	hasSeenWorkspaceWithRunSubPathAndPipelineTaskSubPath1 := false
+	for _, tr := range taskRuns.Items {
+		for _, ws := range tr.Spec.Workspaces {
+
+			if ws.PersistentVolumeClaim == nil {
+				t.Fatalf("found taskRun workspace that is not PersistentVolumeClaim workspace. Did only expect PersistentVolumeClaims workspaces")
+			}
+
+			if ws.SubPath == subPath1 {
+				hasSeenWorkspaceWithPipelineTaskSubPath1 = true
+			}
+
+			if ws.SubPath == subPath2 {
+				hasSeenWorkspaceWithPipelineTaskSubPath2 = true
+			}
+
+			if ws.SubPath == "" {
+				hasSeenWorkspaceWithEmptyPipelineTaskSubPath = true
+			}
+
+			if ws.SubPath == pipelineRunWsSubPath {
+				hasSeenWorkspaceWithRunSubPathAndEmptyPipelineTaskSubPath = true
+			}
+
+			if ws.SubPath == fmt.Sprintf("%s/%s", pipelineRunWsSubPath, subPath1) {
+				hasSeenWorkspaceWithRunSubPathAndPipelineTaskSubPath1 = true
+			}
+		}
+	}
+
+	if !hasSeenWorkspaceWithPipelineTaskSubPath1 {
+		t.Fatalf("did not see a taskRun with a workspace using pipelineTask subPath1")
+	}
+
+	if !hasSeenWorkspaceWithPipelineTaskSubPath2 {
+		t.Fatalf("did not see a taskRun with a workspace using pipelineTask subPath2")
+	}
+
+	if !hasSeenWorkspaceWithEmptyPipelineTaskSubPath {
+		t.Fatalf("did not see a taskRun with a workspace using empty pipelineTask subPath")
+	}
+
+	if !hasSeenWorkspaceWithRunSubPathAndEmptyPipelineTaskSubPath {
+		t.Fatalf("did not see a taskRun with workspace using empty pipelineTask subPath and a subPath from pipelineRun")
+	}
+
+	if !hasSeenWorkspaceWithRunSubPathAndPipelineTaskSubPath1 {
+		t.Fatalf("did not see a taskRun with workspace using pipelineTaks subPath1 and a subPath from pipelineRun")
+	}
+
+	if !reconciledRun.Status.GetCondition(apis.ConditionSucceeded).IsUnknown() {
+		t.Errorf("Expected PipelineRun to be running, but condition status is %s", reconciledRun.Status.GetCondition(apis.ConditionSucceeded))
+	}
+}
+
 func TestReconcileWithTaskResults(t *testing.T) {
 	names.TestingSeed()
 	ps := []*v1alpha1.Pipeline{tb.Pipeline("test-pipeline", tb.PipelineNamespace("foo"), tb.PipelineSpec(

test/builder/pipeline.go

@@ -302,11 +302,12 @@ func PipelineTaskConditionResource(name, resource string, from ...string) Pipeli
 	}
 }
 
-func PipelineTaskWorkspaceBinding(name, workspace string) PipelineTaskOp {
+func PipelineTaskWorkspaceBinding(name, workspace, subPath string) PipelineTaskOp {
 	return func(pt *v1alpha1.PipelineTask) {
 		pt.Workspaces = append(pt.Workspaces, v1alpha1.WorkspacePipelineTaskBinding{
 			Name:      name,
 			Workspace: workspace,
+			SubPath:   subPath,
 		})
 	}
 }
@@ -619,10 +620,11 @@ func PipelineRunWorkspaceBindingEmptyDir(name string) PipelineRunSpecOp {
 }
 
 // PipelineRunWorkspaceBindingVolumeClaimTemplate adds an VolumeClaimTemplate Workspace to the workspaces of a pipelineRun spec.
-func PipelineRunWorkspaceBindingVolumeClaimTemplate(name string, claimName string) PipelineRunSpecOp {
+func PipelineRunWorkspaceBindingVolumeClaimTemplate(name string, claimName string, subPath string) PipelineRunSpecOp {
 	return func(spec *v1alpha1.PipelineRunSpec) {
 		spec.Workspaces = append(spec.Workspaces, v1alpha1.WorkspaceBinding{
-			Name: name,
+			Name:    name,
+			SubPath: subPath,
 			VolumeClaimTemplate: &corev1.PersistentVolumeClaim{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: claimName,

test/builder/pipeline_test.go

@@ -46,7 +46,7 @@ func TestPipeline(t *testing.T) {
 				tb.PipelineTaskConditionParam("param-name", "param-value"),
 				tb.PipelineTaskConditionResource("some-resource", "my-only-git-resource", "bar", "never-gonna"),
 			),
-			tb.PipelineTaskWorkspaceBinding("task-workspace1", "workspace1"),
+			tb.PipelineTaskWorkspaceBinding("task-workspace1", "workspace1", ""),
 		),
 		tb.PipelineTask("bar", "chocolate",
 			tb.PipelineTaskRefKind(v1alpha1.ClusterTaskKind),

test/v1alpha1/workspace_test.go

@@ -105,7 +105,7 @@ func TestWorkspacePipelineRunDuplicateWorkspaceEntriesInvalid(t *testing.T) {
 
 	pipeline := tb.Pipeline(pipelineName, tb.PipelineSpec(
 		tb.PipelineWorkspaceDeclaration("foo"),
-		tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo")),
+		tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo", "")),
 	))
 	if _, err := c.PipelineClient.Create(pipeline); err != nil {
 		t.Fatalf("Failed to create Pipeline: %s", err)
@@ -146,7 +146,7 @@ func TestWorkspacePipelineRunMissingWorkspaceInvalid(t *testing.T) {
 
 	pipeline := tb.Pipeline(pipelineName, tb.PipelineSpec(
 		tb.PipelineWorkspaceDeclaration("foo"),
-		tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo")),
+		tb.PipelineTask("task1", taskName, tb.PipelineTaskWorkspaceBinding("test", "foo", "")),
 	))
 	if _, err := c.PipelineClient.Create(pipeline); err != nil {
 		t.Fatalf("Failed to create Pipeline: %s", err)