Merge pull request #311 from JuliaLang/c42f/fix-SourceFile-offsets

Fixes for `SourceFile` byte offsets

Author: c42f

src/diagnostics.jl

@@ -91,11 +91,6 @@ function show_diagnostics(io::IO, diagnostics::AbstractVector{Diagnostic}, text:
     show_diagnostics(io, diagnostics, SourceFile(text))
 end
 
-function emit_diagnostic(diagnostics::AbstractVector{Diagnostic},
-                         byterange::AbstractUnitRange; kws...)
-    push!(diagnostics, Diagnostic(first(byterange), last(byterange); kws...))
-end
-
 function any_error(diagnostics::AbstractVector{Diagnostic})
     any(is_error(d) for d in diagnostics)
 end

src/expr.jl

@@ -457,9 +457,8 @@ end
 
 function build_tree(::Type{Expr}, stream::ParseStream;
                     filename=nothing, first_line=1, kws...)
-    source = SourceFile(sourcetext(stream), first_index=first_byte(stream),
-                        filename=filename, first_line=first_line)
-    txtbuf = textbuf(stream)
+    source = SourceFile(stream, filename=filename, first_line=first_line)
+    txtbuf = unsafe_textbuf(stream)
     args = Any[]
     childranges = UnitRange{Int}[]
     childheads = SyntaxHead[]

src/parse_stream.jl

@@ -901,11 +901,16 @@ function emit_diagnostic(stream::ParseStream, mark::ParseStreamPosition,
     emit_diagnostic(stream, fbyte:lbyte; kws...)
 end
 
+function emit_diagnostic(diagnostics::AbstractVector{Diagnostic},
+                         byterange::AbstractUnitRange; kws...)
+    push!(diagnostics, Diagnostic(first(byterange), last(byterange); kws...))
+end
+
 #-------------------------------------------------------------------------------
 # ParseStream Post-processing
 
 function validate_tokens(stream::ParseStream)
-    txtbuf = textbuf(stream)
+    txtbuf = unsafe_textbuf(stream)
     toks = stream.tokens
     charbuf = IOBuffer()
     for i = 2:length(toks)
@@ -1103,12 +1108,19 @@ function sourcetext(stream::ParseStream; steal_textbuf=false)
     SubString(str, first_byte(stream), thisind(str, last_byte(stream)))
 end
 
+function SourceFile(stream::ParseStream; kws...)
+    return SourceFile(sourcetext(stream); first_index=first_byte(stream), kws...)
+end
+
 """
-    textbuf(stream)
+    unsafe_textbuf(stream)
 
 Return the `Vector{UInt8}` text buffer being parsed by this `ParseStream`.
+
+!!! warning
+    The caller must hold a reference to `stream` while using textbuf
 """
-textbuf(stream) = stream.textbuf
+unsafe_textbuf(stream) = stream.textbuf
 
 first_byte(stream::ParseStream) = first(stream.tokens).next_byte # Use sentinel token
 last_byte(stream::ParseStream) = _next_byte(stream)-1

src/parser.jl

@@ -125,8 +125,8 @@ function emit_diagnostic(ps::ParseState, args...; kws...)
     emit_diagnostic(ps.stream, args...; kws...)
 end
 
-function textbuf(ps::ParseState)
-    textbuf(ps.stream)
+function unsafe_textbuf(ps::ParseState)
+    unsafe_textbuf(ps.stream)
 end
 
 function first_child_position(ps::ParseState, pos::ParseStreamPosition)
@@ -3143,7 +3143,7 @@ function parse_brackets(after_parse::Function,
     return opts
 end
 
-is_indentation(b::UInt8) = (b == UInt8(' ') || b == UInt8('\t'))
+_is_indentation(b::UInt8) = (b == u8" " || b == u8"\t")
 
 # Parse a string, embedded interpolations and deindent triple quoted strings
 # by marking indentation characters as whitespace trivia.
@@ -3157,7 +3157,7 @@ function parse_string(ps::ParseState, raw::Bool)
     indent_ref_i = 0
     indent_ref_len = typemax(Int)
     indent_chunks = acquire_positions(ps.stream)
-    buf = textbuf(ps)
+    txtbuf = unsafe_textbuf(ps)
     chunk_flags = raw ? RAW_STRING_FLAG : EMPTY_FLAGS
     bump(ps, TRIVIA_FLAG)
     first_chunk = true
@@ -3212,10 +3212,10 @@ function parse_string(ps::ParseState, raw::Bool)
             if triplestr && first_chunk && span(t) <= 2 &&
                     begin
                         s = span(t)
-                        b = buf[last_byte(t)]
+                        b = txtbuf[last_byte(t)]
                         # Test whether the string is a single logical newline
-                        (s == 1 && (b == UInt8('\n') || b == UInt8('\r'))) ||
-                        (s == 2 && (buf[first_byte(t)] == UInt8('\r') && b == UInt8('\n')))
+                        (s == 1 && (b == u8"\n" || b == u8"\r")) ||
+                        (s == 2 && (txtbuf[first_byte(t)] == u8"\r" && b == u8"\n"))
                     end
                 # First line of triple string is a newline only: mark as trivia.
                 # """\nx"""    ==> (string-s "x")
@@ -3253,8 +3253,8 @@ function parse_string(ps::ParseState, raw::Bool)
                     # """\n  $a \n  $b"""   ==> (string-s a " \n" b)
                     # """\n  $a\n  $b\n"""  ==> (string-s "  " a "\n" "  " b "\n")
                     #
-                    if prev_chunk_newline && (b = buf[first_byte(t)];
-                                              b != UInt8('\n') && b != UInt8('\r'))
+                    if prev_chunk_newline && (b = txtbuf[first_byte(t)];
+                                              b != u8"\n" && b != u8"\r")
                         # Compute length of longest common prefix of mixed
                         # spaces and tabs, in bytes
                         #
@@ -3267,7 +3267,7 @@ function parse_string(ps::ParseState, raw::Bool)
                             # No indentation found yet. Find indentation we'll
                             # use as a reference
                             i = first_byte(t) - 1
-                            while i < last_byte(t) && is_indentation(buf[i+1])
+                            while i < last_byte(t) && _is_indentation(txtbuf[i+1])
                                 i += 1
                             end
                             indent_ref_i = first_byte(t)
@@ -3277,7 +3277,7 @@ function parse_string(ps::ParseState, raw::Bool)
                             # shortening length if necessary.
                             j = 0
                             while j < span(t) && j < indent_ref_len
-                                if buf[j + first_byte(t)] != buf[j + indent_ref_i]
+                                if txtbuf[j + first_byte(t)] != txtbuf[j + indent_ref_i]
                                     break
                                 end
                                 j += 1
@@ -3287,7 +3287,7 @@ function parse_string(ps::ParseState, raw::Bool)
                         # Prepare a place for indentiation trivia, if necessary
                         push!(indent_chunks, bump_invisible(ps, K"TOMBSTONE"))
                     end
-                    b = buf[last_byte(t)]
+                    b = txtbuf[last_byte(t)]
                     prev_chunk_newline = b == UInt8('\n') || b == UInt8('\r')
                 end
                 bump(ps, chunk_flags)

src/parser_api.jl

@@ -11,7 +11,7 @@ struct ParseError <: Exception
 end
 
 function ParseError(stream::ParseStream; incomplete_tag=:none, kws...)
-    source = SourceFile(sourcetext(stream); kws...)
+    source = SourceFile(stream; kws...)
     ParseError(source, stream.diagnostics, incomplete_tag)
 end
 

src/source_files.jl

@@ -76,7 +76,8 @@ function source_line_range(source::SourceFile, byte_index;
     lineidx = _source_line_index(source, byte_index)
     fbyte = source.line_starts[max(lineidx-context_lines_before, 1)]
     lbyte = source.line_starts[min(lineidx+1+context_lines_after, end)] - 1
-    fbyte,lbyte
+    return (fbyte + source.byte_offset,
+            lbyte + source.byte_offset)
 end
 
 function source_location(::Type{LineNumberNode}, source::SourceFile, byte_index)
@@ -120,7 +121,7 @@ function Base.getindex(source::SourceFile, i::Int)
 end
 
 function Base.thisind(source::SourceFile, i::Int)
-    thisind(source.code, i - source.byte_offset)
+    thisind(source.code, i - source.byte_offset) + source.byte_offset
 end
 
 Base.firstindex(source::SourceFile) = firstindex(source.code) + source.byte_offset

src/syntax_tree.jl

@@ -61,8 +61,11 @@ Base.show(io::IO, ::ErrorVal) = printstyled(io, "✘", color=:light_red)
 
 function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead};
                     keep_parens=false, position::Integer=1)
-    offset, txtbuf = _unsafe_wrap_substring(sourcetext(source))
-    _to_SyntaxNode(source, txtbuf, offset, raw, convert(Int, position), keep_parens)
+    GC.@preserve source begin
+        raw_offset, txtbuf = _unsafe_wrap_substring(source.code)
+        offset = raw_offset - source.byte_offset
+        _to_SyntaxNode(source, txtbuf, offset, raw, convert(Int, position), keep_parens)
+    end
 end
 
 function _to_SyntaxNode(source::SourceFile, txtbuf::Vector{UInt8}, offset::Int,
@@ -222,8 +225,8 @@ Base.copy(data::SyntaxData) = SyntaxData(data.source, data.raw, data.position, d
 function build_tree(::Type{SyntaxNode}, stream::ParseStream;
                     filename=nothing, first_line=1, keep_parens=false, kws...)
     green_tree = build_tree(GreenNode, stream; kws...)
-    source = SourceFile(sourcetext(stream), filename=filename, first_line=first_line)
-    SyntaxNode(source, green_tree, position=1, keep_parens=keep_parens)
+    source = SourceFile(stream, filename=filename, first_line=first_line)
+    SyntaxNode(source, green_tree, position=first_byte(stream), keep_parens=keep_parens)
 end
 
 #-------------------------------------------------------------------------------

test/hooks.jl

@@ -1,3 +1,15 @@
+function _unwrap_parse_error(core_hook_result)
+    @test Meta.isexpr(core_hook_result[1], :error, 1)
+    err = core_hook_result[1].args[1]
+    if JuliaSyntax._has_v1_10_hooks
+        @test err isa Meta.ParseError
+        return err.detail
+    else
+        @test err isa JuliaSyntax.ParseError
+        return err
+    end
+end
+
 @testset "Hooks for Core integration" begin
     @testset "whitespace parsing" begin
         @test JuliaSyntax.core_parser_hook("", "somefile", 1, 0, :statement) == Core.svec(nothing, 0)
@@ -19,26 +31,28 @@
         @test ex.args[2] == LineNumberNode(2, "otherfile")
 
         # Errors also propagate file & lineno
-        err = JuliaSyntax.core_parser_hook("[x)", "f1", 1, 0, :statement)[1].args[1]
-        if JuliaSyntax._has_v1_10_hooks
-            @test err isa Meta.ParseError
-            err = err.detail
-        else
-            @test err isa JuliaSyntax.ParseError
-        end
+        err = _unwrap_parse_error(
+            JuliaSyntax.core_parser_hook("[x)", "f1", 1, 0, :statement)
+        )
         @test err isa JuliaSyntax.ParseError
         @test err.source.filename == "f1"
         @test err.source.first_line == 1
-        err = JuliaSyntax.core_parser_hook("[x)", "f2", 2, 0, :statement)[1].args[1]
-        if JuliaSyntax._has_v1_10_hooks
-            @test err isa Meta.ParseError
-            err = err.detail
-        else
-            @test err isa JuliaSyntax.ParseError
-        end
+        err = _unwrap_parse_error(
+            JuliaSyntax.core_parser_hook("[x)", "f2", 2, 0, :statement)
+        )
         @test err isa JuliaSyntax.ParseError
         @test err.source.filename == "f2"
         @test err.source.first_line == 2
+
+        # Errors including nontrivial offset indices
+        err = _unwrap_parse_error(
+            JuliaSyntax.core_parser_hook("a\nh{x)\nb", "test.jl", 1, 2, :statement)
+        )
+        @test err isa JuliaSyntax.ParseError
+        @test err.source.first_line == 1
+        @test err.diagnostics[1].first_byte == 6
+        @test err.diagnostics[1].last_byte == 5
+        @test err.diagnostics[1].message == "Expected `}`"
     end
 
     @testset "toplevel errors" begin

test/source_files.jl

@@ -28,8 +28,14 @@
     end
 
     # byte offset
-    @test source_location(SourceFile("a\nbb\nccc\ndddd", first_index=10), 13) == (2,2)
-    @test source_line(SourceFile("a\nbb\nccc\ndddd", first_index=10), 15) == 3
+    sf = SourceFile("a\nbb\nccc\ndddd", first_index=10)
+    @test source_location(sf, 13) == (2,2)
+    @test source_line(sf, 15) == 3
+    @test source_line_range(sf, 10) == (10,11)
+    @test source_line_range(sf, 11) == (10,11)
+    @test source_line_range(sf, 12) == (12,14)
+    @test source_line_range(sf, 14) == (12,14)
+    @test source_line_range(sf, 15) == (15,18)
 
     # source_line convenience function
     @test source_line(SourceFile("a\nb\n"), 2) == 1
@@ -52,6 +58,11 @@ end
     @test sf[10:11] == "ab"
     @test view(sf, 10:11) == "ab"
 
+    @test thisind(SourceFile("xαx", first_index=10), 10) == 10
+    @test thisind(SourceFile("xαx", first_index=10), 11) == 11
+    @test thisind(SourceFile("xαx", first_index=10), 12) == 11
+    @test thisind(SourceFile("xαx", first_index=10), 13) == 13
+
     if Base.VERSION >= v"1.4"
         # Protect the `[begin` from being viewed by the parser on older Julia versions
         @test eval(Meta.parse("SourceFile(\"a\nb\n\")[begin:end]")) == "a\nb\n"

test/syntax_tree.jl

@@ -49,6 +49,12 @@
     @test length(children(node)) == 2
     node[2] = parsestmt(SyntaxNode, "y")
     @test sourcetext(child(node, 2)) == "y"
+
+    # SyntaxNode with offsets
+    t,_ = parsestmt(SyntaxNode, "begin a end\nbegin b end", 13)
+    @test t.position == 13
+    @test child(t,1).position == 19
+    @test child(t,1).val == :b
 end
 
 @testset "SyntaxNode pretty printing" begin