Feature: When merging data and workflowData from sub-jobs, allow top-level arrays to concatenate instead of replace.

Author: jhuckaby

docs/plugins.md

@@ -339,33 +339,19 @@ To include arbitrary data output from your job, which will be automatically pass
 
 The format of the `data` object is freeform, and can contain whatever content you want.  Note that the above example is pretty-printed for display, but in practice all messages must be sent as single lines, so remember to compact your JSON when serializing it.
 
-Note that if you send multiple messages containing `data`, the top-level data object properties are shallow-merged (the latter prevails on duplicate keys).  Using this you can add data incrementally during a job run.
-
-As an advanced option, you can also incrementally *append to arrays* inside the `data` object using this syntax:
+Note that if you send multiple messages containing `data` within the same job, the top-level data object properties are shallow-merged (the latter prevails on duplicate keys).  Using this you can add data incrementally during a job run.  Additionally, if you are overwriting a top-level array with another array, it will be *concatenated* instead of replaced.  Example:
 
 ```json
-{
-	"xy": 1,
-	"push": {
-		"data.arr": [0, 1, 2]
-	}
-}
+{ "xy": 1, "data": { "arr": [0, 1, 2] } }
 ```
 
-This would append to an array named `arr` (it will be created if needed).  You can use simple `dot.path.notation` to access any array deep inside the data object.  
-
-Then, if you sent an additional message with another `push` on the same array like this:
+Then later, in the same job:
 
 ```json
-{
-	"xy": 1,
-	"push": {
-		"data.arr": [3, 4, 5]
-	}
-}
+{ "xy": 1, "data": { "arr": [3, 4, 5] } }
 ```
 
-The `arr` array would grow accordingly, and contain all 6 elements upon job completion.
+This would end up with `[0, 1, 2, 3, 4, 5]` in the final `arr` data array when the job completes.
 
 ##### Output Files
 
@@ -487,9 +473,9 @@ To update the [Workflow Data](workflows.md#sharing-data-between-all-nodes) for t
 }
 ```
 
-Note that the workflow data is shallow-merged, so you can specify a sparsely-populated object and it will only add / replace the included top-level properties.  If your job outputs multiple messages with `workflowData` they are all shallow-merged together.
+Note that the workflow data is shallow-merged, so you can specify a sparsely-populated object and it will only add / replace the included top-level properties.  If your job outputs multiple messages with `workflowData` they are all shallow-merged together.  Additionally, top-level arrays are concatenated when merging.
 
-The workflow data is only updated when the job completes.
+The workflow data is only updated in the parent workflow when the sub-job completes.
 
 ### Action Plugins
 

docs/workflows.md

@@ -206,7 +206,7 @@ xyOps offers another way to share data between nodes, including nodes that are n
 { "xy": 1, "workflowData": { "foo": "bar" } }
 ```
 
-The data is shallow-merged into the shared `workflowData` object when the sub-job completes.  Then, any subsequent jobs that launch from the same workflow are passed the updated `workflowData` object.
+The data is shallow-merged into the shared `workflowData` object when the sub-job completes (also, top-level arrays are concatenated together instead of replacing).  Then, any subsequent jobs that launch from the same workflow are passed the updated `workflowData` object.
 
 The `workflowData` object only lasts for the duration of the workflow run.  It is not persistent like [Server User Data](servers.md#user-data), but it works in the same way.
 

lib/util.js

@@ -110,6 +110,18 @@ class Util {
 		return( Tools.digestHex(JSON.stringify(a)) != Tools.digestHex(JSON.stringify(b)) );
 	}
 
+	mergeConcatInto(a, b) {
+		// like Tools.mergeHashInto, but adds array concat
+		for (var key in b) {
+			if (Array.isArray(a[key]) && Array.isArray(b[key])) {
+				a[key] = a[key].concat( b[key] );
+			}
+			else {
+				a[key] = b[key];
+			}
+		}
+	}
+	
 	getDefaultUserPrivileges() {
 		// get copy of default user privs, but remove any false keys
 		var privs = Tools.copyHash( this.config.get('default_user_privileges') || {} );

lib/workflow.js

@@ -941,7 +941,7 @@ class Workflows {
 		details.wfJobData[sub_job.id] = overrides.input.data;
 
 		// merge in workflowData from sub job
-		Tools.mergeHashInto( details.workflowData, sub_job.workflowData || {} );
+		this.mergeConcatInto( details.workflowData, sub_job.workflowData || {} );
 
 		// if job was retried, return now
 		if (sub_job.retried) return;
@@ -1143,6 +1143,7 @@ class Workflows {
 
 	tickWorkflow(job) {
 		// see if workflow has anything going on, otherwise end it
+		var self = this;
 		var workflow = job.workflow;
 		var state = workflow.state;
 		var details = this.jobDetails[job.id];
@@ -1275,7 +1276,7 @@ class Workflows {
 			(jobs || []).forEach( function(stub) {
 				if (stub.retried) return; // ignore retries when bubbling up data
 				if (stub.files && stub.files.length) details.files = details.files.concat(stub.files);
-				if (details.wfJobData[stub.id]) Tools.mergeHashInto(details.data, details.wfJobData[stub.id]);
+				if (details.wfJobData[stub.id]) self.mergeConcatInto(details.data, details.wfJobData[stub.id]);
 			} );
 		} );