Fix preference of tokenizer_config.json and remove doLowerCase from TokenizerConfig

Author: Soha Agarwal

extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java

@@ -34,6 +34,7 @@
 import java.nio.file.Paths;
 import java.text.Normalizer;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -61,13 +62,15 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
     private boolean cleanupTokenizationSpaces;
     private boolean stripAccents;
     private boolean addPrefixSpace;
+    private final Map<String, String> options;
 
     private HuggingFaceTokenizer(
             long handle,
             Map<String, String> options,
             TokenizerConfig config,
             PadTokenResolver.PadInfo padInfo) {
         super(handle);
+        this.options = options != null ? new HashMap<>(options) : new HashMap<>();
         truncation = TruncationStrategy.LONGEST_FIRST;
         padding = PaddingStrategy.LONGEST;
         maxLength = TokenizersLibrary.LIB.getMaxLength(handle);
@@ -100,31 +103,36 @@ private HuggingFaceTokenizer(
             modelMaxLength = 512;
         }
         if (config != null) {
-            applyConfig(config);
+            applyConfig(config, options);
         }
         updateTruncationAndPadding(padInfo);
     }
 
-    private void applyConfig(TokenizerConfig config) {
-        this.modelMaxLength = config.getModelMaxLength();
-        if (config.hasExplicitDoLowerCase() && config.isDoLowerCase()) {
-            this.doLowerCase = Locale.getDefault();
+    private void applyConfig(TokenizerConfig config, Map<String, String> options) {
+        if (options != null && !options.containsKey("modelMaxLength")) {
+            this.modelMaxLength = config.getModelMaxLength();
         }
         this.cleanupTokenizationSpaces = config.isCleanUpTokenizationSpaces();
-        if (Stream.of(
-                        config.getBosToken(),
-                        config.getClsToken(),
-                        config.getEosToken(),
-                        config.getSepToken(),
-                        config.getUnkToken(),
-                        config.getPadToken())
-                .anyMatch(token -> token != null && !token.isEmpty())) {
-            this.addSpecialTokens = true;
+        if (options != null && !options.containsKey("addSpecialTokens")) {
+
+            this.addSpecialTokens =
+                    Stream.of(
+                                    config.getBosToken(),
+                                    config.getClsToken(),
+                                    config.getEosToken(),
+                                    config.getSepToken(),
+                                    config.getUnkToken(),
+                                    config.getPadToken())
+                            .anyMatch(token -> token != null && !token.isEmpty());
         }
-        if (config.hasExplicitStripAccents()) {
+        if (options != null
+                && !options.containsKey("stripAccents")
+                && config.hasExplicitStripAccents()) {
             this.stripAccents = config.isStripAccents();
         }
-        if (config.hasExplicitAddPrefixSpace()) {
+        if (options != null
+                && !options.containsKey("addPrefixSpace")
+                && config.hasExplicitAddPrefixSpace()) {
             this.addPrefixSpace = config.isAddPrefixSpace();
         }
     }

extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/TokenizerConfig.java

@@ -41,9 +41,6 @@ public class TokenizerConfig {
     @SerializedName("model_max_length")
     private Integer modelMaxLength;
 
-    @SerializedName("do_lower_case")
-    private Boolean doLowerCase;
-
     @SerializedName("strip_accents")
     private Boolean stripAccents;
 
@@ -103,15 +100,6 @@ public int getModelMaxLength() {
         return modelMaxLength;
     }
 
-    /**
-     * Is do lower case boolean.
-     *
-     * @return the boolean
-     */
-    public boolean isDoLowerCase() {
-        return Boolean.TRUE.equals(doLowerCase);
-    }
-
     /**
      * Is strip accents boolean.
      *
@@ -202,15 +190,6 @@ public String getTokenizerClass() {
         return tokenizerClass;
     }
 
-    /**
-     * Has explicit do lower case boolean.
-     *
-     * @return the boolean
-     */
-    public boolean hasExplicitDoLowerCase() {
-        return doLowerCase != null;
-    }
-
     /**
      * Has explicit strip accents boolean.
      *

extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java

@@ -645,4 +645,49 @@ public void testConfigParameters() throws IOException {
             Assert.assertEquals(0, ids[0], "First token should have id 0 (<s>)");
         }
     }
+
+    @Test
+    public void testPreferenceWhenBothOptionsAndConfigSet() throws IOException {
+        try (HuggingFaceTokenizer tokenizer =
+                HuggingFaceTokenizer.builder()
+                        .optMaxLength(48)
+                        .optAddSpecialTokens(false)
+                        .optTokenizerPath(
+                                Paths.get("src/test/resources/fake-tokenizer-with-null-padding/"))
+                        .optTokenizerConfigPath(
+                                "src/test/resources/fake-tokenizer-with-null-padding/tokenizer_config.json")
+                        .build()) {
+            String input = "Hello World";
+            Encoding encoding = tokenizer.encode(input); // with special tokens
+            String[] tokens = encoding.getTokens();
+
+            // Verify special tokens from tokenizer.json are used
+            Assert.assertEquals(tokens[0], "▁Hello"); // bos_token/cls_token
+            Assert.assertEquals(
+                    tokens[tokens.length - 1],
+                    "▁World"); // Last actual token without special tokens
+
+            String[] testInputs = {
+                "Hello World", // Basic text
+                "Hello  World", // Multiple spaces
+                String.join(" ", Collections.nCopies(1000, "hello")), // Long text
+                "résumé café", // Accented characters
+                "Hello\nWorld", // Newlines
+                "Hello    World" // Extra spaces
+            };
+
+            for (String testInput : testInputs) {
+                encoding = tokenizer.encode(testInput);
+
+                // Verify encoding basics
+                Assert.assertNotNull(encoding);
+                Assert.assertNotNull(encoding.getIds());
+                Assert.assertNotNull(encoding.getTokens());
+
+                // Verify model_max_length constraint
+                Assert.assertTrue(
+                        encoding.getIds().length <= 48, "Encoding length should not exceed 48");
+            }
+        }
+    }
 }