Make FieldExistenceRule check for hidden fields

core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java

@@ -717,6 +717,30 @@ public boolean isTokenized(String fieldName, Set<String> ingestTypeFilter) throw
         }
     }
 
+    /**
+     * Determines whether a field has been hidden by looking for the h column in the metadata table
+     *
+     * @param fieldName
+     *            the field name
+     * @param ingestTypeFilter
+     *            the ingest type filter
+     * @return true if the field is hidden for the provided ingest types
+     * @throws TableNotFoundException
+     *             if the table does not exist
+     */
+    public boolean isHidden(String fieldName, Set<String> ingestTypeFilter) throws TableNotFoundException {
+        Preconditions.checkNotNull(fieldName);
+        Preconditions.checkNotNull(ingestTypeFilter);
+
+        Entry<String,Entry<String,Set<String>>> entry = Maps.immutableEntry(metadataTableName, Maps.immutableEntry(fieldName, ingestTypeFilter));
+
+        try {
+            return this.allFieldMetadataHelper.isIndexed(ColumnFamilyConstants.COLF_H, entry);
+        } catch (InstantiationException | ExecutionException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      * Returns a Set of all TextNormalizers in use by any type in Accumulo
      *

core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java

@@ -1,6 +1,7 @@
 package datawave.query.util;
 
 import static datawave.data.ColumnFamilyConstants.COLF_F;
+import static datawave.data.ColumnFamilyConstants.COLF_H;
 import static datawave.query.util.TestUtils.createDateFrequencyMap;
 import static org.apache.accumulo.core.iterators.LongCombiner.VAR_LEN_ENCODER;
 
@@ -104,6 +105,12 @@ private void givenMutation(String row, String columnFamily, String columnQualifi
         givenMutation(mutation);
     }
 
+    private void givenHiddenField(String row, Text colf, String datatype) {
+        Mutation mutation = new Mutation(row);
+        mutation.put(colf, new Text(datatype), new Value());
+        givenMutation(mutation);
+    }
+
     private void givenNonAggregatedFrequencyRows(String row, Text colf, String datatype, String startDate, String endDate, long count) {
         Mutation mutation = new Mutation(row);
         Value value = new Value(VAR_LEN_ENCODER.encode(count));
@@ -392,4 +399,20 @@ void testMixedEntryFormats() {
             Assertions.assertEquals(DateHelper.parse("20200103"), helper.getEarliestOccurrenceOfFieldWithType("NAME", "maze", accumuloClient, null));
         }
     }
+
+    /**
+     * Test against a table with hidden entries.
+     */
+    @Test
+    void testHiddenEntry() throws TableNotFoundException {
+        givenHiddenField("NAME", COLF_H, "csv");
+        givenHiddenField("NAME", COLF_H, "wiki");
+        givenHiddenField("EVENT_DATE", COLF_H, "maze");
+        writeMutations();
+
+        Assertions.assertTrue(helper.isHidden("NAME", Set.of("csv")));
+        Assertions.assertTrue(helper.isHidden("EVENT_DATE", Set.of()));
+        Assertions.assertFalse(helper.isHidden("NAME", Set.of("foo")));
+        Assertions.assertFalse(helper.isHidden("FOO", Set.of()));
+    }
 }

warehouse/query-core/src/main/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitor.java

@@ -74,7 +74,7 @@ public static Set<String> getNonExistentFields(MetadataHelper helper, ASTJexlScr
      *            The set of names which we have determined do not exist
      * @return the updated set of names which do not exist
      */
-    protected Object findMissingFields(JexlNode node, Object data) {
+    protected Object findMissingFields(JexlNode node, Object data) throws TableNotFoundException {
         @SuppressWarnings("unchecked")
         Set<String> nonExistentFieldNames = (null == data) ? new LinkedHashSet<>() : (Set<String>) data;
         List<ASTIdentifier> identifiers;
@@ -94,7 +94,7 @@ protected Object findMissingFields(JexlNode node, Object data) {
 
         for (ASTIdentifier identifier : identifiers) {
             String fieldName = JexlASTHelper.deconstructIdentifier(identifier);
-            if (!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName)) {
+            if ((!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName)) || helper.isHidden(fieldName, this.datatypeFilter)) {
                 nonExistentFieldNames.add(fieldName);
             }
         }
@@ -103,42 +103,74 @@ protected Object findMissingFields(JexlNode node, Object data) {
 
     @Override
     public Object visit(ASTERNode node, Object data) {
-        return findMissingFields(node, data);
+        try {
+            return findMissingFields(node, data);
+        } catch (TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
     public Object visit(ASTNRNode node, Object data) {
-        return findMissingFields(node, data);
+        try {
+            return findMissingFields(node, data);
+        } catch (TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
     public Object visit(ASTEQNode node, Object data) {
-        return findMissingFields(node, data);
+        try {
+            return findMissingFields(node, data);
+        } catch (TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
     public Object visit(ASTNENode node, Object data) {
-        return findMissingFields(node, data);
+        try {
+            return findMissingFields(node, data);
+        } catch (TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
     public Object visit(ASTGENode node, Object data) {
-        return findMissingFields(node, data);
+        try {
+            return findMissingFields(node, data);
+        } catch (TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
     public Object visit(ASTGTNode node, Object data) {
-        return findMissingFields(node, data);
+        try {
+            return findMissingFields(node, data);
+        } catch (TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
     public Object visit(ASTLENode node, Object data) {
-        return findMissingFields(node, data);
+        try {
+            return findMissingFields(node, data);
+        } catch (TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
     public Object visit(ASTLTNode node, Object data) {
-        return findMissingFields(node, data);
+        try {
+            return findMissingFields(node, data);
+        } catch (TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     @Override
@@ -152,8 +184,13 @@ public Object visit(ASTFunctionNode node, Object data) {
                 // deconstruct the identifier
                 final String testFieldName = JexlASTHelper.deconstructIdentifier(fieldName);
                 // changed to allow _ANYFIELD_ in functions
-                if (!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName)) {
-                    nonExistentFieldNames.add(testFieldName);
+                try {
+                    if ((!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName))
+                                    || this.helper.isHidden(fieldName, this.datatypeFilter)) {
+                        nonExistentFieldNames.add(testFieldName);
+                    }
+                } catch (TableNotFoundException e) {
+                    throw new RuntimeException(e);
                 }
             }
         } else if (log.isTraceEnabled()) {