Feature: Workflow Split controllers now offer an optional "item filter" expression, to filter out specific items based on their properties, or array index, or other. Fixes #298.

Author: jhuckaby

docs/workflows.md

@@ -80,14 +80,28 @@ Controllers implement flow control. They generally have input and output poles a
 
 #### Split Controller
 
-The Split controller fans out work by taking an input list and launching one sub-job per item. Provide a dot-path to the list in the previous job context, such as `data.rows`. The engine resolves the path and expects an array; if the value is a string it is trimmed and split by newline.   A special case is `files`, which splits the incoming files array so that each sub-job receives exactly one file. 
+The Split controller fans out work by taking an input list and launching one sub-job per item. Provide an [expression](xyexp.md) to the list in the previous job context, such as `data.rows`. The engine resolves the path and expects an array; if the value is a string it is trimmed and split by newline.   A special case is `files`, which splits the incoming files array so that each sub-job receives exactly one file. 
 
 In the UI, the split controller configuration dialog provides an "Expression Builder" button, which allows you to explore output data from recently completed jobs, and pick out a specific JSON key path to use for the expression string.
 
 Each individual sub-job will receive one item from the split data.  It will arrive in the job's [Job.input](data.md#job-input), either in `data` as a property named `item`, or as a [File](data.md#file) in the `files` array.
 
 Split requires exactly one output connection to the Event or Job node it will run per item. Concurrency and queuing are governed by the limits attached to that node. After all items complete, you can continue the flow using a `continue` wire from the controlled node. The controller includes a "continue percentage" setting so you can require that at least N% of the sub-jobs succeed before continuing.
 
+##### Split Item Filter
+
+You can optionally filter out items from your split array, by including a special item filter expression.  In this context use the special `item` keyword to refer to the item being filtered.  If your expression evaluates to true, the item will be included in the set.  Otherwise, it will be left out.  Here is an example using a property inside the item:
+
+```js
+item.random < 0.5
+```
+
+Also available in context here is `index` (the 0-based index of the current item in the set), `workflow.params` (all workflow-level user fields), and `workflowData` (shared workflow data object).  For example, here is how to filter out all odd items, and only keep the even ones, using the modulo operator:
+
+```js
+index % 2 == 0
+```
+
 #### Join Controller
 
 The Join controller waits for multiple incoming flows to finish, then passes a combined result to the next step. You can wire multiple inputs into a Join; it initializes when the first input arrives and completes after all of its inputs have fired.

htdocs/js/pages/Workflows.class.js

@@ -2131,6 +2131,20 @@ Page.Workflows = class Workflows extends Page.Events {
 			}) + '<div class="text_field_icon mdi mdi-database-search-outline" title="' + config.ui.tooltips.wfd_exp_builder + '" onClick="$P().openExpressionBuilder(this)"></div>'
 		});
 
+		// split filter
+		html += this.getFormRow({
+			id: 'd_wfd_split_filter',
+			content: this.getFormText({
+				id: 'fe_wfd_split_filter',
+				type: 'text',
+				spellcheck: 'false',
+				autocomplete: 'off',
+				maxlength: 8192,
+				class: 'monospace',
+				value: node.data.filter || ''
+			})
+		});
+		
 		// if expression
 		html += this.getFormRow({
 			id: 'd_wfd_if',
@@ -2206,6 +2220,7 @@ Page.Workflows = class Workflows extends Page.Events {
 				case 'split':
 					node.data.split = $('#fe_wfd_split').val().trim();
 					if (!node.data.split.length) return app.badField('#fe_wfd_split');
+					node.data.filter = $('#fe_wfd_split_filter').val().trim();
 					node.data.continue = parseInt( $('#fe_wfd_continue').val() ) || 0;
 				break;
 
@@ -2260,14 +2275,14 @@ Page.Workflows = class Workflows extends Page.Events {
 		// handle type change
 		var do_change_type = function() {
 			// show/hide sections based on type
-			$('#d_wfd_stagger, #d_wfd_wait, #d_wfd_repeat, #d_wfd_split, #d_wfd_if, #d_wfd_title, #d_wfd_icon, #d_wfd_continue').hide();
+			$('#d_wfd_stagger, #d_wfd_wait, #d_wfd_repeat, #d_wfd_split, #d_wfd_split_filter, #d_wfd_if, #d_wfd_title, #d_wfd_icon, #d_wfd_continue').hide();
 
 			var type = $('#fe_wfd_type').val();
 			switch (type) {
 				case 'multiplex': $('#d_wfd_stagger, #d_wfd_continue').show(); break;
 				case 'wait': $('#d_wfd_wait').show(); break;
 				case 'repeat': $('#d_wfd_repeat, #d_wfd_continue').show(); break;
-				case 'split': $('#d_wfd_split, #d_wfd_continue').show(); break;
+				case 'split': $('#d_wfd_split, #d_wfd_split_filter, #d_wfd_continue').show(); break;
 				case 'decision': $('#d_wfd_if, #d_wfd_title, #d_wfd_icon').show(); break;
 			}
 

internal/ui.json

@@ -583,11 +583,15 @@
 			},
 			"d_wfd_split": {
 				"label": "Split Expression:",
-				"caption": "Specify the path to the array for splitting, using `dot.path.notation`.  Use the special `files` keyword to split the input files."
+				"caption": "Specify the path to the array for splitting, using [XYEXP](#Docs/xyexp), e.g. `data.items`.  Use the special `files` keyword to split the input files."
+			},
+			"d_wfd_split_filter": {
+				"label": "Item Filter:",
+				"caption": "Optionally enter an expression to filter items, e.g. `item.random > 0.5`.  If false, the item will be excluded from the set.  [Learn More](#Docs/workflows/split-item-filter)"
 			},
 			"d_wfd_if": {
 				"label": "Expression:",
-				"caption": "Enter the expression to evaluate using dot.path.notation, e.g. `data.random > 0.5`.  If true, control will pass onto the next node."
+				"caption": "Enter the expression to evaluate using [XYEXP](#Docs/xyexp), e.g. `data.random > 0.5`.  If true, control will pass onto the next node."
 			},
 			"d_wfd_title": {
 				"label": "Custom Title:",

lib/workflow.js

@@ -653,6 +653,24 @@ class Workflows {
 			return;
 		}
 
+		// allow split controller to filter items
+		if (node.data.filter) {
+			try {
+				value = value.filter( function(item, idx) {
+					context.item = item;
+					context.index = idx;
+					return jexl.evalSync( node.data.filter, context );
+				} );
+			}
+			catch (err) {
+				this.logWorkflow(job, node.id, `WARNING: Failed to evaluate filter expression: {${node.data.filter}}: ${err}`);
+				state.error = true;
+				return;
+			}
+			delete context.item;
+			delete context.index;
+		}
+		
 		// find single node we need to control
 		var conns = Tools.findObjects( workflow.connections, { source: node.id } );
 		if (!conns.length) {