Merge pull request JuliaLang/JuliaSyntax.jl#60 from JuliaLang/sp/fix-vect-newline

Fix vect parsing with newline before comma

Author: pfitzseb

JuliaSyntax/src/parser.jl

@@ -398,7 +398,7 @@ function parse_Nary(ps::ParseState, down, delimiters, closing_tokens)
             break
         elseif k in delimiters
             # ignore empty delimited sections
-            # a;;;b  ==>  (block a b) 
+            # a;;;b  ==>  (block a b)
             continue
         end
         down(ps)
@@ -433,7 +433,7 @@ function parse_toplevel(ps::ParseState)
     nothing
 end
 
-# Parse a newline or semicolon-delimited list of expressions. 
+# Parse a newline or semicolon-delimited list of expressions.
 # Repeated delimiters are allowed but ignored
 # a;b;c     ==>  (block a b c)
 # a;;;b;;   ==>  (block a b)
@@ -2246,7 +2246,7 @@ function parse_imports(ps::ParseState)
     bump(ps, TRIVIA_FLAG)
     emark = position(ps)
     initial_as = parse_import(ps, word, false)
-    t = peek_token(ps) 
+    t = peek_token(ps)
     k = kind(t)
     has_import_prefix = false  # true if we have `prefix:` in `import prefix: stuff`
     has_comma = false
@@ -2798,7 +2798,7 @@ function parse_cat(ps::ParseState, closer, end_is_symbol)
     if k == K"," || (is_closing_token(ps, k) && k != K";")
         if k == K","
             # [x,]  ==>  (vect x)
-            bump(ps, TRIVIA_FLAG)
+            bump(ps, TRIVIA_FLAG; skip_newlines = true)
         end
         # [x]      ==>  (vect x)
         # [x \n ]  ==>  (vect x)
@@ -2912,9 +2912,9 @@ end
 # For example, (a=1; b=2) could be seen to parse four different ways!
 #
 # Function args:   (kw a 1) (parameters (kw b 2))
-# Tuple-like:      (=  a 1) (parameters (kw b 2)) 
-# Block:           (=  a 1)             (=  b 2)  
-# [] vect-like:    (=  a 1) (parameters (=  b 2)) 
+# Tuple-like:      (=  a 1) (parameters (kw b 2))
+# Block:           (=  a 1)             (=  b 2)
+# [] vect-like:    (=  a 1) (parameters (=  b 2))
 #
 # Expressions (X; Y; Z) with more semicolons are also allowed by the flisp
 # parser and generally parse as nested parameters blocks. This is invalid Julia
@@ -3384,7 +3384,7 @@ function parse_atom(ps::ParseState, check_identifiers=true)
         # 42   ==>  42
         bump(ps)
     elseif is_closing_token(ps, leading_kind)
-        # Leave closing token in place for other productions to 
+        # Leave closing token in place for other productions to
         # recover with
         # )  ==>  error
         msg = leading_kind == K"EndMarker" ?
@@ -3395,4 +3395,3 @@ function parse_atom(ps::ParseState, check_identifiers=true)
         bump(ps, error="invalid syntax atom")
     end
 end
-

JuliaSyntax/src/syntax_tree.jl

@@ -74,8 +74,10 @@ function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}, position::In
         elseif is_syntax_kind(raw)
             nothing
         else
-            error("Leaf node of kind $k unknown to SyntaxNode")
-            val = nothing
+            # FIXME: this allows us to recover from trivia is_error nodes
+            # that we insert below
+            @debug "Leaf node of kind $k unknown to SyntaxNode"
+            ErrorVal()
         end
         return SyntaxNode(source, raw, position, nothing, true, val)
     else

JuliaSyntax/test/parser.jl

@@ -600,8 +600,10 @@ tests = [
         # parse_cat
         "[]"        =>  "(vect)"
         "[x,]"      =>  "(vect x)"
+        "[x\n,,]"   =>  "(vect x (error-t ✘))"
         "[x]"       =>  "(vect x)"
         "[x \n ]"   =>  "(vect x)"
+        "[x \n, ]"  =>  "(vect x)"
         "[x"        =>  "(vect x (error-t))"
         "[x \n\n ]" =>  "(vect x)"
         "[x for a in as]"  =>  "(comprehension (generator x (= a as)))"
@@ -616,6 +618,9 @@ tests = [
         # parse_vect
         "[x, y]"        =>  "(vect x y)"
         "[x, y]"        =>  "(vect x y)"
+        "[x,\n y]"      =>  "(vect x y)"
+        "[x\n, y]"      =>  "(vect x y)"
+        "[x\n,, y]"     =>  "(vect x (error-t ✘ y))"
         "[x,y ; z]"     =>  "(vect x y (parameters z))"
         "[x=1, y=2]"    =>  "(vect (= x 1) (= y 2))"
         "[x=1, ; y=2]"  =>  "(vect (= x 1) (parameters (= y 2)))"
@@ -830,4 +835,3 @@ end
     @test test_parse(JuliaSyntax.parse_eq, "a \u2212= b") == "(-= a b)"
     @test test_parse(JuliaSyntax.parse_eq, "a .\u2212= b") == "(.-= a b)"
 end
-