Feature: Add "Batch Size" to the workflow split controller, to send multiple items to each job.

Author: jhuckaby

docs/data.md

@@ -3226,9 +3226,9 @@ For controller nodes, see the following table for details on how the `data` prop
 | `multiplex` | Will contain `stagger` (delay in seconds) for staggering jobs across servers, and `continue` (percentage) for gating success. |
 | `wait` | Will contain `wait` (delay in seconds). |
 | `repeat` | Will contain `repeat` (iteration count), and `continue` (percentage) for gating success. |
-| `split` | Will contain `split` (expression to split on), and `continue` (percentage) for gating success. |
+| `split` | Will contain `split` (expression to split on), `continue` (percentage) for gating success, an optional `filter` for filtering items, and optional `chunk` for batch size. |
 | `join` | Not used. |
-| `decision` | Will contain `label` (custom title), `icon` (custom icon), and `decision` (expression to evaluate). |
+| `decision` | Will contain `label` (custom title), `icon` (custom icon), `decision` (expression to evaluate), and `abort` (abort entire workflow on false eval). |
 
 #### WorkflowNode.x
 

docs/workflows.md

@@ -84,7 +84,7 @@ The Split controller fans out work by taking an input list and launching one sub
 
 In the UI, the split controller configuration dialog provides an "Expression Builder" button, which allows you to explore output data from recently completed jobs, and pick out a specific JSON key path to use for the expression string.
 
-Each individual sub-job will receive one item from the split data.  It will arrive in the job's [Job.input](data.md#job-input), either in `data` as a property named `item`, or as a [File](data.md#file) in the `files` array.
+Each individual sub-job will receive one item from the split data.  It will arrive in the job's [Job.input](data.md#job-input), either in `data` as a property named `item` (or `items` if multiple are batched), or as a [File](data.md#file) in the `files` array.
 
 Split requires exactly one output connection to the Event or Job node it will run per item. Concurrency and queuing are governed by the limits attached to that node. After all items complete, you can continue the flow using a `continue` wire from the controlled node. The controller includes a "continue percentage" setting so you can require that at least N% of the sub-jobs succeed before continuing.
 
@@ -102,6 +102,12 @@ Also available in context here is `index` (the 0-based index of the current item
 index % 2 == 0
 ```
 
+##### Split Batch Size
+
+By default, each split item launches one downstream job.  To process multiple items per job, increase the Split controller's Batch Size setting.  xyOps will chunk the split items in order, using one downstream job per batch, with the final batch containing any remaining items.
+
+For batched splits, each job receives an array in `data.items` (note: plural).  For `files` splits, each batched job receives multiple [File](data.md#file) objects in its `files` array.  A special case is when the Batch Size is `1`, which keeps the original legacy behavior, where each job receives a single `data.item` (note: singular) or one file.
+
 #### Join Controller
 
 The Join controller waits for multiple incoming flows to finish, then passes a combined result to the next step. You can wire multiple inputs into a Join; it initializes when the first input arrives and completes after all of its inputs have fired.

htdocs/js/pages/Workflows.class.js

@@ -2215,6 +2215,21 @@ Page.Workflows = class Workflows extends Page.Events {
 			})
 		});
 
+		// split chunk
+		html += this.getFormRow({
+			id: 'd_wfd_split_chunk',
+			content: this.getFormText({
+				id: 'fe_wfd_split_chunk',
+				type: 'number',
+				spellcheck: 'false',
+				autocomplete: 'off',
+				maxlength: 32,
+				min: 1,
+				step: 1,
+				value: node.data.chunk || '1'
+			})
+		});
+		
 		// decision expression
 		html += this.getFormRow({
 			id: 'd_wfd_if',
@@ -2300,6 +2315,7 @@ Page.Workflows = class Workflows extends Page.Events {
 					node.data.split = $('#fe_wfd_split').val().trim();
 					if (!node.data.split.length) return app.badField('#fe_wfd_split');
 					node.data.filter = $('#fe_wfd_split_filter').val().trim();
+					node.data.chunk = parseInt( $('#fe_wfd_split_chunk').val() ) || 1;
 					node.data.continue = parseInt( $('#fe_wfd_continue').val() ) || 0;
 				break;
 
@@ -2355,14 +2371,14 @@ Page.Workflows = class Workflows extends Page.Events {
 		// handle type change
 		var do_change_type = function() {
 			// show/hide sections based on type
-			$('#d_wfd_stagger, #d_wfd_wait, #d_wfd_repeat, #d_wfd_split, #d_wfd_split_filter, #d_wfd_if, #d_wfd_if_abort, #d_wfd_title, #d_wfd_icon, #d_wfd_continue').hide();
+			$('#d_wfd_stagger, #d_wfd_wait, #d_wfd_repeat, #d_wfd_split, #d_wfd_split_filter, #d_wfd_split_chunk, #d_wfd_if, #d_wfd_if_abort, #d_wfd_title, #d_wfd_icon, #d_wfd_continue').hide();
 
 			var type = $('#fe_wfd_type').val();
 			switch (type) {
 				case 'multiplex': $('#d_wfd_stagger, #d_wfd_continue').show(); break;
 				case 'wait': $('#d_wfd_wait').show(); break;
 				case 'repeat': $('#d_wfd_repeat, #d_wfd_continue').show(); break;
-				case 'split': $('#d_wfd_split, #d_wfd_split_filter, #d_wfd_continue').show(); break;
+				case 'split': $('#d_wfd_split, #d_wfd_split_filter, #d_wfd_split_chunk, #d_wfd_continue').show(); break;
 				case 'decision': $('#d_wfd_if, #d_wfd_if_abort, #d_wfd_title, #d_wfd_icon').show(); break;
 			}
 

internal/ui.json

@@ -604,12 +604,16 @@
 			},
 			"d_wfd_split": {
 				"label": "Split Expression:",
-				"caption": "Specify the path to the array for splitting, using [XYEXP](#Docs/xyexp), e.g. `data.items`.  Use the special `files` keyword to split the input files."
+				"caption": "Specify the path to the array for splitting, using [XYEXP](#Docs/xyexp), e.g. `data.myItems`.  Use the special `files` keyword to split the input files."
 			},
 			"d_wfd_split_filter": {
 				"label": "Item Filter:",
 				"caption": "Optionally enter an expression to filter items, e.g. `item.random > 0.5`.  If false, the item will be excluded from the set.  [Learn More](#Docs/workflows/split-item-filter)"
 			},
+			"d_wfd_split_chunk": {
+				"label": "Batch Size:",
+				"caption": "Optionally batch multiple items per job by increasing this value.  The default is `1`, meaning each job receives exactly one item.  [Learn More](#Docs/workflows/split-batch-size)"
+			},
 			"d_wfd_if": {
 				"label": "Expression:",
 				"caption": "Enter the expression to evaluate using [XYEXP](#Docs/xyexp), e.g. `data.random > 0.5`.  If true, control will pass onto the next node."

lib/workflow.js

@@ -721,6 +721,25 @@ class Workflows {
 			return;
 		}
 
+		// optionally chunk the list
+		var is_chunked = false;
+		if (node.data.chunk && (node.data.chunk > 1)) {
+			var chunks = [];
+			var chunk = [];
+			
+			value.forEach( function(item) {
+				chunk.push(item);
+				if (chunk.length >= node.data.chunk) {
+					chunks.push(chunk);
+					chunk = [];
+				}
+			});
+			if (chunk.length) chunks.push(chunk);
+			
+			value = chunks;
+			is_chunked = true;
+		}
+		
 		// find single node we need to control
 		var conns = Tools.findObjects( workflow.connections, { source: node.id } );
 		if (!conns.length) {
@@ -760,12 +779,13 @@ class Workflows {
 		value.forEach( function(item) {
 			var overrides = Tools.copyHash(opts.overrides, true);
 
-			if (exp == 'files') {
-				// special mode for splitting on files instead of data
-				overrides.input.files = [ item ];
+			if (is_chunked) {
+				if (exp == 'files') overrides.input.files = [ ...item ];
+				else overrides.input.data = { items: item };
 			}
 			else {
-				overrides.input.data = { item };
+				if (exp == 'files') overrides.input.files = [ item ];
+				else overrides.input.data = { item };
 			}
 
 			self.runWorkflowNode({