Make FieldExistenceRule check for hidden fields

core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java

@@ -717,6 +717,28 @@ public boolean isTokenized(String fieldName, Set<String> ingestTypeFilter) throw
         }
     }
 
+    /**
+     * Determines whether a field has been hidden by looking for the h column in the metadata table
+     *
+     * @param fieldName
+     *            the field name
+     * @param ingestTypeFilter
+     *            the ingest type filter
+     * @return true if the field is hidden for the provided ingest types
+     */
+    public boolean isHidden(String fieldName, Set<String> ingestTypeFilter) {
+        Preconditions.checkNotNull(fieldName);
+        Preconditions.checkNotNull(ingestTypeFilter);
+
+        Entry<String,Entry<String,Set<String>>> entry = Maps.immutableEntry(metadataTableName, Maps.immutableEntry(fieldName, ingestTypeFilter));
+
+        try {
+            return this.allFieldMetadataHelper.isIndexed(ColumnFamilyConstants.COLF_H, entry);
+        } catch (InstantiationException | ExecutionException | TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      * Returns a Set of all TextNormalizers in use by any type in Accumulo
      *

core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java

@@ -104,6 +104,13 @@ private void givenMutation(String row, String columnFamily, String columnQualifi
         givenMutation(mutation);
     }
 
+    private void givenHiddenField(String row, String datatype) {
+        Text h = new Text("h");
+        Mutation mutation = new Mutation(row);
+        mutation.put(h, new Text(datatype), new Value());
+        givenMutation(mutation);
+    }
+
     private void givenNonAggregatedFrequencyRows(String row, Text colf, String datatype, String startDate, String endDate, long count) {
         Mutation mutation = new Mutation(row);
         Value value = new Value(VAR_LEN_ENCODER.encode(count));
@@ -392,4 +399,20 @@ void testMixedEntryFormats() {
             Assertions.assertEquals(DateHelper.parse("20200103"), helper.getEarliestOccurrenceOfFieldWithType("NAME", "maze", accumuloClient, null));
         }
     }
+
+    /**
+     * Test against a table with hidden entries.
+     */
+    @Test
+    void testHiddenEntry() throws TableNotFoundException {
+        givenHiddenField("NAME", "csv");
+        givenHiddenField("NAME", "wiki");
+        givenHiddenField("EVENT_DATE", "maze");
+        writeMutations();
+
+        Assertions.assertTrue(helper.isHidden("NAME", Set.of("csv")));
+        Assertions.assertTrue(helper.isHidden("EVENT_DATE", Set.of()));
+        Assertions.assertFalse(helper.isHidden("NAME", Set.of("foo")));
+        Assertions.assertFalse(helper.isHidden("FOO", Set.of()));
+    }
 }

warehouse/query-core/src/main/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitor.java

@@ -94,7 +94,7 @@ protected Object findMissingFields(JexlNode node, Object data) {
 
         for (ASTIdentifier identifier : identifiers) {
             String fieldName = JexlASTHelper.deconstructIdentifier(identifier);
-            if (!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName)) {
+            if ((!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName)) || helper.isHidden(fieldName, this.datatypeFilter)) {
                 nonExistentFieldNames.add(fieldName);
             }
         }
@@ -152,7 +152,8 @@ public Object visit(ASTFunctionNode node, Object data) {
                 // deconstruct the identifier
                 final String testFieldName = JexlASTHelper.deconstructIdentifier(fieldName);
                 // changed to allow _ANYFIELD_ in functions
-                if (!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName)) {
+                if ((!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName))
+                                || this.helper.isHidden(fieldName, this.datatypeFilter)) {
                     nonExistentFieldNames.add(testFieldName);
                 }
             }