Skip to content

Commit eb79fc4

Browse files
stevomitricMaxGekk
authored andcommitted
[SPARK-49204][SQL][FOLLOWUP] Fix IndexOutOfBoundsException when dealing with surrogate pairs
### What changes were proposed in this pull request? Modified the `StringLocate` ICU codepath and added a codepoint count check for the provided offset. ### Why are the changes needed? Currently, doing the following throws an `java.lang.IndexOutOfBoundsException`. ```sql select position('asd' COLLATE UNICODE, 'a', 10) ``` while the correct behavior is to return an false match result (0). ### Does this PR introduce _any_ user-facing change? Yes, fixes the `java.lang.IndexOutOfBoundsException` error. ### How was this patch tested? New test cases in this PR. ### Was this patch authored or co-authored using generative AI tooling? No Closes #47871 from stevomitric/stevomitric/fix-position-out-of-bounds. Authored-by: Stevo Mitric <stevo.mitric@databricks.com> Signed-off-by: Max Gekk <max.gekk@gmail.com>
1 parent e1b5d65 commit eb79fc4

2 files changed

Lines changed: 26 additions & 0 deletions

File tree

common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -671,6 +671,11 @@ public static int indexOf(final UTF8String target, final UTF8String pattern,
671671
// Initialize the string search with respect to the specified ICU collation.
672672
String targetStr = target.toValidString();
673673
String patternStr = pattern.toValidString();
674+
// Check if `start` is out of bounds. The provided offset `start` is given in number of
675+
// codepoints, so a simple `targetStr.length` check is not sufficient here. This check is
676+
// needed because `String.offsetByCodePoints` throws an `IndexOutOfBoundsException`
677+
// exception when the offset is out of bounds.
678+
if (targetStr.codePointCount(0, targetStr.length()) <= start) return MATCH_NOT_FOUND;
674679
StringSearch stringSearch =
675680
CollationFactory.getStringSearch(targetStr, patternStr, collationId);
676681
stringSearch.setOverlapping(true);

common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2329,6 +2329,27 @@ public void testStringLocate() throws SparkException {
23292329
assertStringLocate("b", "a🙃x🙃b", 4, "UTF8_LCASE", 5);
23302330
assertStringLocate("b", "a🙃x🙃b", 4, "UNICODE", 5);
23312331
assertStringLocate("b", "a🙃x🙃b", 4, "UNICODE_CI", 5);
2332+
// Out of bounds test cases.
2333+
assertStringLocate("a", "asd", 4, "UTF8_BINARY", 0);
2334+
assertStringLocate("a", "asd", 4, "UTF8_LCASE", 0);
2335+
assertStringLocate("a", "asd", 4, "UNICODE", 0);
2336+
assertStringLocate("a", "asd", 4, "UNICODE_CI", 0);
2337+
assertStringLocate("a", "asd", 100, "UTF8_BINARY", 0);
2338+
assertStringLocate("a", "asd", 100, "UTF8_LCASE", 0);
2339+
assertStringLocate("a", "asd", 100, "UNICODE", 0);
2340+
assertStringLocate("a", "asd", 100, "UNICODE_CI", 0);
2341+
assertStringLocate("a", "🙃🙃", 4, "UTF8_BINARY", 0);
2342+
assertStringLocate("a", "🙃🙃", 4, "UTF8_LCASE", 0);
2343+
assertStringLocate("a", "🙃🙃", 4, "UNICODE", 0);
2344+
assertStringLocate("a", "🙃🙃", 4, "UNICODE_CI", 0);
2345+
assertStringLocate("", "asd", 100, "UTF8_BINARY", 1);
2346+
assertStringLocate("", "asd", 100, "UTF8_LCASE", 1);
2347+
assertStringLocate("", "asd", 100, "UNICODE", 1);
2348+
assertStringLocate("", "asd", 100, "UNICODE_CI", 1);
2349+
assertStringLocate("asd", "", 100, "UTF8_BINARY", 0);
2350+
assertStringLocate("asd", "", 100, "UTF8_LCASE", 0);
2351+
assertStringLocate("asd", "", 100, "UNICODE", 0);
2352+
assertStringLocate("asd", "", 100, "UNICODE_CI", 0);
23322353
}
23332354

23342355
/**

0 commit comments

Comments
 (0)