Skip to content

Fix vect parsing with newline before comma #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pfitzseb merged 3 commits into from
Aug 24, 2022
Merged

Fix vect parsing with newline before comma #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a401f0f
Fix vect parsing with newline before comma
pfitzseb Aug 22, 2022
cf22788
Correct fix
pfitzseb Aug 22, 2022
a7b2668
"Improve" error handling in SyntaxNode conversion
pfitzseb Aug 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions src/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ function parse_Nary(ps::ParseState, down, delimiters, closing_tokens)
break
elseif k in delimiters
# ignore empty delimited sections
# a;;;b ==> (block a b)
# a;;;b ==> (block a b)
continue
end
down(ps)
Expand Down Expand Up @@ -433,7 +433,7 @@ function parse_toplevel(ps::ParseState)
nothing
end

# Parse a newline or semicolon-delimited list of expressions.
# Parse a newline or semicolon-delimited list of expressions.
# Repeated delimiters are allowed but ignored
# a;b;c ==> (block a b c)
# a;;;b;; ==> (block a b)
Expand Down Expand Up @@ -2246,7 +2246,7 @@ function parse_imports(ps::ParseState)
bump(ps, TRIVIA_FLAG)
emark = position(ps)
initial_as = parse_import(ps, word, false)
t = peek_token(ps)
t = peek_token(ps)
k = kind(t)
has_import_prefix = false # true if we have `prefix:` in `import prefix: stuff`
has_comma = false
Expand Down Expand Up @@ -2798,7 +2798,7 @@ function parse_cat(ps::ParseState, closer, end_is_symbol)
if k == K"," || (is_closing_token(ps, k) && k != K";")
if k == K","
# [x,] ==> (vect x)
bump(ps, TRIVIA_FLAG)
bump(ps, TRIVIA_FLAG; skip_newlines = true)
Comment thread
c42f marked this conversation as resolved.
end
# [x] ==> (vect x)
# [x \n ] ==> (vect x)
Expand Down Expand Up @@ -2912,9 +2912,9 @@ end
# For example, (a=1; b=2) could be seen to parse four different ways!
#
# Function args: (kw a 1) (parameters (kw b 2))
# Tuple-like: (= a 1) (parameters (kw b 2))
# Block: (= a 1) (= b 2)
# [] vect-like: (= a 1) (parameters (= b 2))
# Tuple-like: (= a 1) (parameters (kw b 2))
# Block: (= a 1) (= b 2)
# [] vect-like: (= a 1) (parameters (= b 2))
#
# Expressions (X; Y; Z) with more semicolons are also allowed by the flisp
# parser and generally parse as nested parameters blocks. This is invalid Julia
Expand Down Expand Up @@ -3384,7 +3384,7 @@ function parse_atom(ps::ParseState, check_identifiers=true)
# 42 ==> 42
bump(ps)
elseif is_closing_token(ps, leading_kind)
# Leave closing token in place for other productions to
# Leave closing token in place for other productions to
# recover with
# ) ==> error
msg = leading_kind == K"EndMarker" ?
Expand All @@ -3395,4 +3395,3 @@ function parse_atom(ps::ParseState, check_identifiers=true)
bump(ps, error="invalid syntax atom")
end
end

7 changes: 4 additions & 3 deletions src/syntax_tree.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,10 @@ function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}, position::In
elseif is_syntax_kind(raw)
nothing
else
error("Leaf node of kind $k unknown to SyntaxNode")
val = nothing
# FIXME: this allows us to recover from trivia is_error nodes
# that we insert below
@debug "Leaf node of kind $k unknown to SyntaxNode"
ErrorVal()
end
return SyntaxNode(source, raw, position, nothing, true, val)
else
Expand Down Expand Up @@ -256,4 +258,3 @@ function highlight(code::String, node, path::Int...; color=(40,40,70))
_printstyled(stdout, code[p:q-1]; bgcolor=color)
print(stdout, code[q:end])
end

6 changes: 5 additions & 1 deletion test/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -600,8 +600,10 @@ tests = [
# parse_cat
"[]" => "(vect)"
"[x,]" => "(vect x)"
"[x\n,,]" => "(vect x (error-t ✘))"
"[x]" => "(vect x)"
"[x \n ]" => "(vect x)"
"[x \n, ]" => "(vect x)"
"[x" => "(vect x (error-t))"
"[x \n\n ]" => "(vect x)"
"[x for a in as]" => "(comprehension (generator x (= a as)))"
Expand All @@ -616,6 +618,9 @@ tests = [
# parse_vect
"[x, y]" => "(vect x y)"
"[x, y]" => "(vect x y)"
"[x,\n y]" => "(vect x y)"
"[x\n, y]" => "(vect x y)"
"[x\n,, y]" => "(vect x (error-t ✘ y))"
"[x,y ; z]" => "(vect x y (parameters z))"
"[x=1, y=2]" => "(vect (= x 1) (= y 2))"
"[x=1, ; y=2]" => "(vect (= x 1) (parameters (= y 2)))"
Expand Down Expand Up @@ -830,4 +835,3 @@ end
@test test_parse(JuliaSyntax.parse_eq, "a \u2212= b") == "(-= a b)"
@test test_parse(JuliaSyntax.parse_eq, "a .\u2212= b") == "(.-= a b)"
end