feat: Allow users to specify image for code processor. Fixes #482 (#491)

alexec · web-flow · commit 0c5a5e822eda · 2021-11-01T11:38:31.000-07:00
Signed-off-by: Alex Collins &lt;alex_collins@intuit.com&gt;
diff --git a/README.md b/README.md
@@ -74,6 +74,7 @@ Intermediate:
 * [Events interop](docs/EVENTS_INTEROP.md)
 * [Workflow interop](docs/WORKFLOW_INTEROP.md)
 * [Meta-data](docs/META.md)
+* [Idempotence](docs/IDEMPOTENCE.md)
 
 Advanced
 
diff --git a/api/v1alpha1/code.go b/api/v1alpha1/code.go
@@ -7,13 +7,19 @@ import (
 )
 
 type Code struct {
-	Runtime Runtime `json:"runtime" protobuf:"bytes,4,opt,name=runtime,casttype=Runtime"`
-	Source  string  `json:"source" protobuf:"bytes,3,opt,name=source"`
+	Runtime Runtime `json:"runtime,omitempty" protobuf:"bytes,4,opt,name=runtime,casttype=Runtime"`
+	// Image is used in preference to Runtime.
+	Image  string `json:"image,omitempty" protobuf:"bytes,5,opt,name=image"`
+	Source string `json:"source" protobuf:"bytes,3,opt,name=source"`
 }
 
 func (in Code) getContainer(req getContainerReq) corev1.Container {
+	image := in.Image
+	if image == "" {
+		image = fmt.Sprintf(req.imageFormat, "dataflow-"+in.Runtime)
+	}
 	return containerBuilder{}.
 		init(req).
-		image(fmt.Sprintf(req.imageFormat, "dataflow-"+in.Runtime)).
+		image(image).
 		build()
 }
diff --git a/api/v1alpha1/code_test.go b/api/v1alpha1/code_test.go
@@ -7,9 +7,14 @@ import (
 )
 
 func TestCode_getContainer(t *testing.T) {
-	x := Code{
-		Runtime: "my-runtime",
-	}
-	c := x.getContainer(getContainerReq{imageFormat: "fmt-%s"})
-	assert.Equal(t, "fmt-dataflow-my-runtime", c.Image)
+	t.Run("Runtime", func(t *testing.T) {
+		x := Code{Runtime: "my-runtime"}
+		c := x.getContainer(getContainerReq{imageFormat: "fmt-%s"})
+		assert.Equal(t, "fmt-dataflow-my-runtime", c.Image)
+	})
+	t.Run("Runtime", func(t *testing.T) {
+		x := Code{Image: "my-image"}
+		c := x.getContainer(getContainerReq{})
+		assert.Equal(t, "my-image", c.Image)
+	})
 }
diff --git a/api/v1alpha1/generated.pb.go b/api/v1alpha1/generated.pb.go
diff --git a/api/v1alpha1/generated.proto b/api/v1alpha1/generated.proto
diff --git a/config/ci.yaml b/config/ci.yaml
@@ -720,6 +720,9 @@ spec:
                       type: object
                     code:
                       properties:
+                        image:
+                          description: Image is used in preference to Runtime.
+                          type: string
                         runtime:
                           enum:
                           - golang1-16
@@ -731,7 +734,6 @@ spec:
                         source:
                           type: string
                       required:
-                      - runtime
                       - source
                       type: object
                     container:
@@ -7907,6 +7909,9 @@ spec:
                 type: object
               code:
                 properties:
+                  image:
+                    description: Image is used in preference to Runtime.
+                    type: string
                   runtime:
                     enum:
                     - golang1-16
@@ -7918,7 +7923,6 @@ spec:
                   source:
                     type: string
                 required:
-                - runtime
                 - source
                 type: object
               container:
diff --git a/config/crd/bases/dataflow.argoproj.io_pipelines.yaml b/config/crd/bases/dataflow.argoproj.io_pipelines.yaml
@@ -715,6 +715,9 @@ spec:
                       type: object
                     code:
                       properties:
+                        image:
+                          description: Image is used in preference to Runtime.
+                          type: string
                         runtime:
                           enum:
                           - golang1-16
@@ -726,7 +729,6 @@ spec:
                         source:
                           type: string
                       required:
-                      - runtime
                       - source
                       type: object
                     container:
diff --git a/config/crd/bases/dataflow.argoproj.io_steps.yaml b/config/crd/bases/dataflow.argoproj.io_steps.yaml
@@ -680,6 +680,9 @@ spec:
                 type: object
               code:
                 properties:
+                  image:
+                    description: Image is used in preference to Runtime.
+                    type: string
                   runtime:
                     enum:
                     - golang1-16
@@ -691,7 +694,6 @@ spec:
                   source:
                     type: string
                 required:
-                - runtime
                 - source
                 type: object
               container:
diff --git a/config/default.yaml b/config/default.yaml
@@ -720,6 +720,9 @@ spec:
                       type: object
                     code:
                       properties:
+                        image:
+                          description: Image is used in preference to Runtime.
+                          type: string
                         runtime:
                           enum:
                           - golang1-16
@@ -731,7 +734,6 @@ spec:
                         source:
                           type: string
                       required:
-                      - runtime
                       - source
                       type: object
                     container:
@@ -7907,6 +7909,9 @@ spec:
                 type: object
               code:
                 properties:
+                  image:
+                    description: Image is used in preference to Runtime.
+                    type: string
                   runtime:
                     enum:
                     - golang1-16
@@ -7918,7 +7923,6 @@ spec:
                   source:
                     type: string
                 required:
-                - runtime
                 - source
                 type: object
               container:
diff --git a/config/dev.yaml b/config/dev.yaml
@@ -720,6 +720,9 @@ spec:
                       type: object
                     code:
                       properties:
+                        image:
+                          description: Image is used in preference to Runtime.
+                          type: string
                         runtime:
                           enum:
                           - golang1-16
@@ -731,7 +734,6 @@ spec:
                         source:
                           type: string
                       required:
-                      - runtime
                       - source
                       type: object
                     container:
@@ -7907,6 +7909,9 @@ spec:
                 type: object
               code:
                 properties:
+                  image:
+                    description: Image is used in preference to Runtime.
+                    type: string
                   runtime:
                     enum:
                     - golang1-16
@@ -7918,7 +7923,6 @@ spec:
                   source:
                     type: string
                 required:
-                - runtime
                 - source
                 type: object
               container:
diff --git a/config/quick-start.yaml b/config/quick-start.yaml
@@ -720,6 +720,9 @@ spec:
                       type: object
                     code:
                       properties:
+                        image:
+                          description: Image is used in preference to Runtime.
+                          type: string
                         runtime:
                           enum:
                           - golang1-16
@@ -731,7 +734,6 @@ spec:
                         source:
                           type: string
                       required:
-                      - runtime
                       - source
                       type: object
                     container:
@@ -7907,6 +7909,9 @@ spec:
                 type: object
               code:
                 properties:
+                  image:
+                    description: Image is used in preference to Runtime.
+                    type: string
                   runtime:
                     enum:
                     - golang1-16
@@ -7918,7 +7923,6 @@ spec:
                   source:
                     type: string
                 required:
-                - runtime
                 - source
                 type: object
               container:
diff --git a/docs/IDEMPOTENCE.md b/docs/IDEMPOTENCE.md
@@ -0,0 +1,19 @@
+# Idempotence
+
+Because network connections can always fail to either deliver a message, or receive an acknowledgement, processors and
+sinks may receive the same message twice.
+
+It maybe that you don't mind the occasional duplicate messages. But maybe that is not the case. If so, processors and
+external systems, need to be written to cater for this, to be idempotent.
+
+The safest way to do that is for each message to contain a unique identifier that is intrinsic to the message. For
+example:
+
+* If each message is the result of a horse race, the the location and time would be enough to de-duplicate messages.
+* If each message was the final year grade of a student, then the year, class, and the student's IDs would be suitable.
+
+When there is no intrinsic identifier, then you can use the identifiers that are added as [meta-data](META.md) to each
+message. You should add an identifier to you messages as soon as possible.
+
+Some sinks have inherent idempotence, e.g. when sinking to a volume, if duplicate processing results in a file being
+created with the same name, the the old file will be overwritten.
diff --git a/dsls/python/argo_dataflow/pipeline.py b/dsls/python/argo_dataflow/pipeline.py
@@ -446,7 +446,7 @@ def dump(self):
 
 
 class CodeStep(Step):
-    def __init__(self, name=None, source=None, code=None, runtime=None, sources=None, sinks=None, terminator=False):
+    def __init__(self, name=None, source=None, code=None, runtime=None, image=None, sources=None, sinks=None, terminator=False):
         super().__init__(name, sources=sources, sinks=sinks, terminator=terminator)
         if source:
             self._source = inspect.getsource(source).replace('def ' + source.__name__ + str(inspect.signature(source)),
@@ -457,13 +457,18 @@ def __init__(self, name=None, source=None, code=None, runtime=None, sources=None
             self._runtime = runtime
         else:
             self._runtime = DEFAULT_RUNTIME
+        self._image = image
 
     def dump(self):
         x = super().dump()
-        x['code'] = {
-            'runtime': self._runtime,
+        y = {
             'source': self._source,
         }
+        if self._runtime:
+            y['runtime'] = self._runtime
+        if self._image:
+            y['image'] = self._image
+        x['code'] = y
         return x
 
 
@@ -521,8 +526,8 @@ def group(self, name=None, key=None, format=None, endOfGroup=None, storage=None)
     def flatten(self, name=None):
         return FlattenStep(name, sources=[self])
 
-    def code(self, name=None, source=None, code=None, runtime=None):
-        return CodeStep(name, source=source, code=code, runtime=runtime, sources=[self])
+    def code(self, name=None, source=None, code=None, runtime=None, image=None):
+        return CodeStep(name, source=source, code=code, runtime=runtime, image=image, sources=[self])
 
     def map(self, name=None, expression=None):
         return MapStep(name, expression, sources=[self])
