Merge pull request #84 from JuliaLang/sp/is_identifier_start_char-nothrow

Non-throwing is_identifier_[start_]char

Author: pfitzseb

src/tokenize_utils.jl

@@ -4,16 +4,19 @@ const EOF_CHAR = typemax(Char)
 
 function is_identifier_char(c::Char)
     c == EOF_CHAR && return false
+    Base.ismalformed(c) && return false
     return Base.is_id_char(c)
 end
 
 function is_identifier_start_char(c::Char)
     c == EOF_CHAR && return false
+    Base.ismalformed(c) && return false
     return Base.is_id_start_char(c)
 end
 
 # Chars that we will never allow to be part of a valid non-operator identifier
 function is_never_id_char(ch::Char)
+    Base.ismalformed(ch) && return true
     cat = Unicode.category_code(ch)
     c = UInt32(ch)
     return (

test/tokenize.jl

@@ -909,4 +909,11 @@ end
     @test strtok("a .&&₁ b") ==  ["a", " ", ".&&", "₁", " ", "b", ""]
 end
 
+@testset "is_identifier[_start]_char" begin
+    malformed = first("\xe2")
+    @test Tokenize.is_identifier_char(malformed) == false
+    @test Tokenize.is_identifier_start_char(malformed) == false
+    @test Tokenize.is_never_id_char(malformed) == true
+end
+
 end