Merge pull request #47 from forkeith/bug_fixes

Fix some parsing bugs

Author: trishume

src/parsing/parser.rs

@@ -312,7 +312,7 @@ impl ParseState {
                 ops.push((index, ScopeStackOp::Pop(v.len())));
             }
 
-            if initial && cur_context.clear_scopes != None {
+            if !initial && cur_context.clear_scopes != None {
                 ops.push((index, ScopeStackOp::Restore));
             }
         }
@@ -483,4 +483,96 @@ mod tests {
 
         // assert!(false);
     }
+
+    fn expect_scope_stacks(line: &str, expect: &[&str]) {
+        // check that each expected scope stack appears at least once while parsing the given test line
+        
+        //let syntax = SyntaxSet::load_syntax_file("testdata/parser_tests.sublime-syntax", true).unwrap();
+        use std::fs::File;
+        use std::io::Read;
+        let mut f = File::open("testdata/parser_tests.sublime-syntax").unwrap();
+        let mut s = String::new();
+        f.read_to_string(&mut s).unwrap();
+
+        let syntax = SyntaxDefinition::load_from_str(&s, true).unwrap();
+        
+        let mut state = ParseState::new(&syntax);
+
+        let mut ss = SyntaxSet::new();
+        ss.add_syntax(syntax);
+        ss.link_syntaxes();
+
+        let mut stack = ScopeStack::new();
+        let ops = state.parse_line(line);
+        debug_print_ops(line, &ops);
+
+        let mut criteria_met = Vec::new();
+        for &(_, ref op) in ops.iter() {
+            stack.apply(op);
+            let stack_str = format!("{:?}", stack);
+            println!("{}", stack_str);
+            for expectation in expect.iter() {
+                if stack_str.contains(expectation) {
+                    criteria_met.push(expectation);
+                }
+            }
+        }
+        if let Some(missing) = expect.iter().filter(|e| !criteria_met.contains(&e)).next() {
+            panic!("expected scope stack '{}' missing", missing);
+        }
+    }
+
+    #[test]
+    fn can_parse_non_nested_clear_scopes() {
+        let line = "'hello #simple_cleared_scopes_test world test \\n '\n";
+        let expect = [
+            "<source.test>, <example.meta-scope.after-clear-scopes.example>, <example.pushes-clear-scopes.example>",
+            "<source.test>, <example.meta-scope.after-clear-scopes.example>, <example.pops-clear-scopes.example>",
+            "<source.test>, <string.quoted.single.example>, <constant.character.escape.example>",
+        ];
+        expect_scope_stacks(&line, &expect);
+    }
+
+    #[test]
+    fn can_parse_non_nested_too_many_clear_scopes() {
+        let line = "'hello #too_many_cleared_scopes_test world test \\n '\n";
+        let expect = [
+            "<example.meta-scope.after-clear-scopes.example>, <example.pushes-clear-scopes.example>",
+            "<example.meta-scope.after-clear-scopes.example>, <example.pops-clear-scopes.example>",
+            "<source.test>, <string.quoted.single.example>, <constant.character.escape.example>",
+        ];
+        expect_scope_stacks(&line, &expect);
+    }
+
+    #[test]
+    fn can_parse_nested_clear_scopes() {
+        let line = "'hello #nested_clear_scopes_test world foo bar test \\n '\n";
+        let expect = [
+            "<source.test>, <example.meta-scope.after-clear-scopes.example>, <example.pushes-clear-scopes.example>",
+            "<source.test>, <example.meta-scope.cleared-previous-meta-scope.example>, <foo>",
+            "<source.test>, <example.meta-scope.after-clear-scopes.example>, <example.pops-clear-scopes.example>",
+            "<source.test>, <string.quoted.single.example>, <constant.character.escape.example>",
+        ];
+        expect_scope_stacks(&line, &expect);
+    }
+
+    #[test]
+    fn can_parse_infinite_loop() {
+        let line = "#infinite_loop_test 123\n";
+        let expect = [
+            "<source.test>, <constant.numeric.test>",
+        ];
+        expect_scope_stacks(&line, &expect);
+    }
+
+    #[test]
+    fn can_parse_infinite_seeming_loop() {
+        let line = "#infinite_seeming_loop_test hello\n";
+        let expect = [
+            "<source.test>, <keyword.test>",
+            "<source.test>, <test>, <string.unquoted.test>",
+            "<source.test>, <test>, <keyword.control.test>",
+        ];
+        expect_scope_stacks(&line, &expect);
+    }
 }

src/parsing/scope.rs

@@ -6,7 +6,7 @@ use std::fmt;
 use std::str::FromStr;
 use std::u64;
 use rustc_serialize::{Encodable, Encoder, Decodable, Decoder};
-use std::cmp::Ordering;
+use std::cmp::{Ordering, min};
 use std::mem;
 
 /// Multiplier on the power of 2 for MatchPower.
@@ -374,7 +374,8 @@ impl ScopeStack {
             ScopeStackOp::Clear(amount) => {
                 let cleared = match amount {
                     ClearAmount::TopN(n) => {
-                        let to_leave = self.scopes.len()-n;
+                        // don't try to clear more scopes than are on the stack 
+                        let to_leave = self.scopes.len() - min(n, self.scopes.len());
                         self.scopes.split_off(to_leave)
                     }
                     ClearAmount::All => {

testdata/parser_tests.sublime-syntax

@@ -0,0 +1,79 @@
+%YAML 1.2
+---
+name: Used by tests in src/parsing/parser.rs
+scope: source.test
+contexts:
+  main:
+    - match: '#infinite_seeming_loop_test'
+      scope: keyword.test
+      push: infinite_seeming_loop_c
+    - match: '(?=#infinite_loop_test)'
+      push: infinite_loop_test_pop_if_not_whitespace
+    - match: \'
+      scope: punctuation.definition.string.begin.example
+      push: cleared_scopes_string_test
+    - match: '\d+'
+      scope: constant.numeric.test
+
+  infinite_loop_test_pop_if_not_whitespace:
+    - match: '(?=\S)'
+      pop: true
+  infinite_seeming_loop_a:
+    - meta_content_scope: test
+    - match: 'h'
+      scope: string.unquoted.test
+    - match: 'ello'
+      scope: keyword.control.test
+  infinite_seeming_loop_b:
+    - match: ''
+      pop: true
+    - match: '(?=.)'
+      pop: true
+    - match: '(?=h)'
+      pop: true
+    - match: 'h'
+      scope: entity.name.function.test
+    - match: 'e'
+      scope: storage.type.test
+  infinite_seeming_loop_c:
+    - match: ''
+      push: [infinite_seeming_loop_a, infinite_seeming_loop_b]
+  cleared_scopes_string_test:
+    - meta_scope: string.quoted.single.example
+    - match: '#too_many_cleared_scopes_test'
+      scope: example.pushes-clear-scopes.example
+      push:
+        - clear_scopes: 10
+        - meta_scope: example.meta-scope.after-clear-scopes.example
+        - match: 'test'
+          scope: example.pops-clear-scopes.example
+          pop: true
+    - match: '#simple_cleared_scopes_test'
+      scope: example.pushes-clear-scopes.example
+      push:
+        - clear_scopes: 1
+        - meta_scope: example.meta-scope.after-clear-scopes.example
+        - match: 'test'
+          scope: example.pops-clear-scopes.example
+          pop: true
+    - match: '#nested_clear_scopes_test'
+      scope: example.pushes-clear-scopes.example
+      push:
+        - clear_scopes: 1
+        - meta_scope: example.meta-scope.after-clear-scopes.example
+        - match: 'foo'
+          scope: foo
+          push:
+            - clear_scopes: 1
+            - meta_scope: example.meta-scope.cleared-previous-meta-scope.example
+            - match: 'bar'
+              scope: bar
+              pop: true
+        - match: 'test'
+          scope: example.pops-clear-scopes.example
+          pop: true
+    - match: '\\.'
+      scope: constant.character.escape.example
+    - match: \'
+      scope: punctuation.definition.string.end.example
+      pop: true