Add fl_parse() test tool to call flisp parser directly (JuliaLang/JuliaSyntax.jl#70)

This allows us to call the flisp parser directly instead of using
Meta.parse which may have been overridden with the JuliaSyntax parser.

Fix the tests to always use the flisp reference parser.

Author: c42f

JuliaSyntax/src/hooks.jl

@@ -133,11 +133,16 @@ function _core_parser_hook(code, filename, lineno, offset, options)
                offset=offset,
                code=code)
 
-        if VERSION >= v"1.8.0-DEV.1370" # https://github.com/JuliaLang/julia/pull/43876
-            return Core.Compiler.fl_parse(code, filename, lineno, offset, options)
-        else
-            return Core.Compiler.fl_parse(code, filename, offset, options)
-        end
+        _fl_parse_hook(code, filename, lineno, offset, options)
+    end
+end
+
+# Call the flisp parser
+function _fl_parse_hook(code, filename, lineno, offset, options)
+    @static if VERSION >= v"1.8.0-DEV.1370" # https://github.com/JuliaLang/julia/pull/43876
+        return Core.Compiler.fl_parse(code, filename, lineno, offset, options)
+    else
+        return Core.Compiler.fl_parse(code, filename, offset, options)
     end
 end
 

JuliaSyntax/src/utils.jl

@@ -25,15 +25,6 @@ function _printstyled(io::IO, text; fgcolor=nothing, bgcolor=nothing)
     end
 end
 
-function flisp_parse_all(code; filename="none")
-    if VERSION >= v"1.6"
-        Meta.parseall(code, filename=filename)
-    else
-        # This is approximate. It should work for well-formed code.
-        Base.parse_input_line(code, filename=filename)
-    end
-end
-
 # Really remove line numbers, even from Expr(:toplevel)
 remove_linenums!(ex) = ex
 function remove_linenums!(ex::Expr)
@@ -48,3 +39,49 @@ function remove_linenums!(ex::Expr)
     return ex
 end
 
+
+#-------------------------------------------------------------------------------
+# Copy of the Meta.parse() API, but ensuring that we call the flisp parser
+# rather than using Meta.parse() which may be using the JuliaSyntax parser.
+
+"""
+Like Meta.parse() but always call the flisp reference parser.
+"""
+function fl_parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)
+    ex, pos = fl_parse(str, 1, greedy=true, raise=raise, depwarn=depwarn)
+    if isa(ex,Expr) && ex.head === :error
+        return ex
+    end
+    if pos <= ncodeunits(str)
+        raise && throw(Meta.ParseError("extra token after end of expression"))
+        return Expr(:error, "extra token after end of expression")
+    end
+    return ex
+end
+
+function fl_parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
+                  depwarn::Bool=true)
+    ex, pos = _fl_parse_string(str, "none", 1, pos, greedy ? :statement : :atom)
+    if raise && isa(ex,Expr) && ex.head === :error
+        throw(Meta.ParseError(ex.args[1]))
+    end
+    return ex, pos
+end
+
+"""
+Like Meta.parseall() but always call the flisp reference parser.
+"""
+function fl_parseall(text::AbstractString; filename="none", lineno=1)
+    ex,_ = _fl_parse_string(text, String(filename), lineno, 1, :all)
+    return ex
+end
+
+function _fl_parse_string(text::AbstractString, filename::AbstractString,
+                          lineno::Integer, index::Integer, options)
+    if index < 1 || index > ncodeunits(text) + 1
+        throw(BoundsError(text, index))
+    end
+    ex, offset::Int = _fl_parse_hook(text, filename, lineno, index-1, options)
+    ex, offset+1
+end
+

JuliaSyntax/test/test_utils.jl

@@ -21,7 +21,8 @@ using JuliaSyntax:
     haschildren,
     children,
     child,
-    flisp_parse_all
+    fl_parseall,
+    fl_parse
 
 function remove_macro_linenums!(ex)
     if Meta.isexpr(ex, :macrocall)
@@ -63,7 +64,7 @@ function parsers_agree_on_file(filename; show_diff=false)
         # ignore this case.
         return true
     end
-    fl_ex = flisp_parse_all(text, filename=filename)
+    fl_ex = fl_parseall(text, filename=filename)
     if Meta.isexpr(fl_ex, :toplevel) && !isempty(fl_ex.args) &&
             Meta.isexpr(fl_ex.args[end], (:error, :incomplete))
         # Reference parser failed. This generally indicates a broken file not a
@@ -111,7 +112,7 @@ function equals_flisp_parse(tree)
     # Reparse with JuliaSyntax. This is a crude way to ensure we're not missing
     # some context from the parent node.
     ex,_,_ = parse(Expr, node_text)
-    fl_ex = flisp_parse_all(node_text)
+    fl_ex = fl_parseall(node_text)
     if Meta.isexpr(fl_ex, :error)
         return true  # Something went wrong in reduction; ignore these cases 😬
     end
@@ -237,7 +238,7 @@ function itest_parse(production, code; version::VersionNumber=v"1.6")
     println(stdout, "\n\n# Julia Expr:")
     show(stdout, MIME"text/plain"(), ex)
 
-    f_ex = JuliaSyntax.remove_linenums!(Meta.parse(code, raise=false))
+    f_ex = JuliaSyntax.remove_linenums!(fl_parse(code, raise=false))
     if JuliaSyntax.remove_linenums!(ex) != f_ex
         printstyled(stdout, "\n\n# flisp Julia Expr:\n", color=:red)
         show(stdout, MIME"text/plain"(), f_ex)

JuliaSyntax/test/tokenize.jl

@@ -4,6 +4,7 @@ module TokenizeTests
 using Test
 
 using JuliaSyntax:
+    JuliaSyntax,
     @K_str,
     Kind,
     kind,
@@ -693,14 +694,10 @@ for op in ops
 
     for (arity, container) in strs
         for str in container
-            expr = Meta.parse(str, raise = false)
+            expr = JuliaSyntax.fl_parse(str, raise = false)
             if VERSION < v"1.7" && str == "a .&& b"
                 expr = Expr(Symbol(".&&"), :a, :b)
             end
-            if str in (".>:b", ".<:b")
-                # HACK! See https://github.com/JuliaLang/JuliaSyntax.jl/issues/38
-                continue
-            end
             if expr isa Expr && (expr.head != :error && expr.head != :incomplete)
                 tokens = collect(tokenize(str))
                 exop = expr.head == :call ? expr.args[1] : expr.head