improved error handling

Author: aaronjanse

src/parser.rs

@@ -18,6 +18,9 @@ use crate::{
 #[derive(Clone, Debug, PartialEq)]
 #[non_exhaustive]
 pub enum ParseError {
+    /// Expected is used when a token was expected but not found. Unlike UnexpectedWanted,
+    /// we continue to parse as if this token was found.
+    Expected(TextSize, Box<[SyntaxKind]>),
     /// Unexpected is used when the cause cannot be specified further
     Unexpected(TextRange),
     /// UnexpectedExtra is used when there are additional tokens to the root in the tree
@@ -37,6 +40,12 @@ pub enum ParseError {
 impl fmt::Display for ParseError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
+            ParseError::Expected(location, kinds) => write!(
+                f,
+                "wanted any of {:?} at {}",
+                kinds,
+                usize::from(*location),
+            ),
             ParseError::Unexpected(range) => {
                 write!(
                     f,
@@ -189,6 +198,41 @@ where
     fn peek(&mut self) -> Option<SyntaxKind> {
         self.peek_data().map(|&(t, _)| t)
     }
+
+    /// If the next token is in `allowed_slice`, expect_peek_any returns it.
+    /// Otherwise, it appends `self.errors` without consuming any tokens.
+    ///
+    /// This is generally preferred over expect_peek_any, but the callee must
+    /// ensure that an infinite loop won't be caused by this method not bumping.
+    fn peek_any_or_missing(&mut self, allowed_slice: &[SyntaxKind]) -> Option<SyntaxKind> {
+        let allowed = TokenSet::from_slice(allowed_slice);
+
+        match self.peek() {
+            Some(kind) if allowed.contains(kind) => Some(kind),
+            Some(_) => {
+                self.errors.push(ParseError::Expected(
+                    self.get_text_position(),
+                    allowed_slice.to_vec().into_boxed_slice(),
+                ));
+
+                None
+            }
+            None => {
+                self.errors
+                .push(ParseError::UnexpectedEOFWanted(allowed_slice.to_vec().into_boxed_slice()));
+
+                None
+            },
+        }
+    }
+
+    /// If the next token is in `allowed_slice`, expect_peek_any returns it.
+    /// Otherwise, it appends self.errors and creates a NODE_ERROR containing all tokens
+    /// until a token in `allowed_slice` is found.
+    ///
+    /// This is not always wanted. For example, in `{ foo = 1 } // { bar = 2; }`, it would
+    /// be better to recognize the semicolon as missing then move on, rather than bumping
+    /// all the way to the semicolon after `bar`. 
     fn expect_peek_any(&mut self, allowed_slice: &[SyntaxKind]) -> Option<SyntaxKind> {
         let allowed = TokenSet::from_slice(allowed_slice);
 
@@ -224,6 +268,17 @@ where
             self.bump();
         }
     }
+
+    // Expect a token or silently ignore if missing. This only bumps if the token is found,
+    // so watch out for infinite loops. This returns whether the token was found.
+    fn expect_or_missing(&mut self, expected: SyntaxKind) -> bool {
+        let found = self.peek_any_or_missing(&[expected]).is_some();
+        if found {
+            self.bump();
+        }
+        found
+    }
+
     fn expect_ident(&mut self) {
         if self.expect_peek_any(&[TOKEN_IDENT]).is_some() {
             self.start_node(NODE_IDENT);
@@ -277,7 +332,14 @@ where
     fn parse_attrpath(&mut self) {
         self.start_node(NODE_ATTRPATH);
         loop {
-            self.parse_attr();
+            match self.peek_any_or_missing(&[TOKEN_INTERPOL_START, TOKEN_STRING_START, TOKEN_IDENT]) {
+                Some(TOKEN_INTERPOL_START) => self.parse_dynamic(),
+                Some(TOKEN_STRING_START) => self.parse_string(),
+                Some(TOKEN_IDENT) => self.expect_ident(),
+                _ => {
+                    break
+                }
+            }
 
             if self.peek() == Some(TOKEN_DOT) {
                 self.bump();
@@ -399,7 +461,7 @@ where
                     self.parse_attrpath();
                     self.expect(TOKEN_ASSIGN);
                     self.parse_expr();
-                    self.expect(TOKEN_SEMICOLON);
+                    self.expect_or_missing(TOKEN_SEMICOLON);
                     self.finish_node();
                 }
             }

test_data/parser/invalid_syntax/1.expect

@@ -1,5 +1,5 @@
 error: unexpected TOKEN_SQUARE_B_CLOSE at 0..1, wanted any of [TOKEN_PAREN_OPEN, TOKEN_REC, TOKEN_CURLY_B_OPEN, TOKEN_SQUARE_B_OPEN, TOKEN_STRING_START, TOKEN_IDENT]
-error: unexpected end of file, wanted any of [TOKEN_IDENT]
+error: unexpected end of file, wanted any of [TOKEN_INTERPOL_START, TOKEN_STRING_START, TOKEN_IDENT]
 NODE_ROOT 0..2 {
   NODE_SELECT 0..2 {
     NODE_ERROR 0..1 {

test_data/parser/invalid_syntax/incomplete-attr-path.expect

@@ -0,0 +1,43 @@
+error: wanted any of [TOKEN_INTERPOL_START, TOKEN_STRING_START, TOKEN_IDENT] at 18
+NODE_ROOT 0..36 {
+  NODE_ATTR_SET 0..36 {
+    TOKEN_CURLY_B_OPEN("{") 0..1
+    TOKEN_WHITESPACE("\n    ") 1..6
+    NODE_ATTRPATH_VALUE 6..19 {
+      NODE_ATTRPATH 6..9 {
+        NODE_IDENT 6..9 {
+          TOKEN_IDENT("foo") 6..9
+        }
+      }
+      TOKEN_WHITESPACE(" ") 9..10
+      TOKEN_ASSIGN("=") 10..11
+      TOKEN_WHITESPACE(" ") 11..12
+      NODE_SELECT 12..18 {
+        NODE_IDENT 12..17 {
+          TOKEN_IDENT("hello") 12..17
+        }
+        TOKEN_DOT(".") 17..18
+        NODE_ATTRPATH 18..18 {
+        }
+      }
+      TOKEN_SEMICOLON(";") 18..19
+    }
+    TOKEN_WHITESPACE("\n    ") 19..24
+    NODE_ATTRPATH_VALUE 24..34 {
+      NODE_ATTRPATH 24..27 {
+        NODE_IDENT 24..27 {
+          TOKEN_IDENT("xyz") 24..27
+        }
+      }
+      TOKEN_WHITESPACE(" ") 27..28
+      TOKEN_ASSIGN("=") 28..29
+      TOKEN_WHITESPACE(" ") 29..30
+      NODE_LITERAL 30..33 {
+        TOKEN_INTEGER("123") 30..33
+      }
+      TOKEN_SEMICOLON(";") 33..34
+    }
+    TOKEN_WHITESPACE("\n") 34..35
+    TOKEN_CURLY_B_CLOSE("}") 35..36
+  }
+}

test_data/parser/invalid_syntax/incomplete-attr-path.nix

@@ -0,0 +1,4 @@
+{
+    foo = hello.;
+    xyz = 123;
+}

test_data/parser/invalid_syntax/missing-semicolon-path.expect

@@ -0,0 +1,53 @@
+error: wanted any of [TOKEN_INTERPOL_START, TOKEN_STRING_START, TOKEN_IDENT] at 13
+error: wanted any of [TOKEN_SEMICOLON] at 13
+NODE_ROOT 0..30 {
+  NODE_BIN_OP 0..30 {
+    NODE_ATTR_SET 0..14 {
+      TOKEN_CURLY_B_OPEN("{") 0..1
+      TOKEN_WHITESPACE(" ") 1..2
+      NODE_ATTRPATH_VALUE 2..13 {
+        NODE_ATTRPATH 2..5 {
+          NODE_IDENT 2..5 {
+            TOKEN_IDENT("foo") 2..5
+          }
+        }
+        TOKEN_WHITESPACE(" ") 5..6
+        TOKEN_ASSIGN("=") 6..7
+        TOKEN_WHITESPACE(" ") 7..8
+        NODE_SELECT 8..13 {
+          NODE_IDENT 8..11 {
+            TOKEN_IDENT("bar") 8..11
+          }
+          TOKEN_DOT(".") 11..12
+          TOKEN_WHITESPACE(" ") 12..13
+          NODE_ATTRPATH 13..13 {
+          }
+        }
+      }
+      TOKEN_CURLY_B_CLOSE("}") 13..14
+    }
+    TOKEN_WHITESPACE(" ") 14..15
+    TOKEN_UPDATE("//") 15..17
+    TOKEN_WHITESPACE(" ") 17..18
+    NODE_ATTR_SET 18..30 {
+      TOKEN_CURLY_B_OPEN("{") 18..19
+      TOKEN_WHITESPACE(" ") 19..20
+      NODE_ATTRPATH_VALUE 20..28 {
+        NODE_ATTRPATH 20..23 {
+          NODE_IDENT 20..23 {
+            TOKEN_IDENT("xyz") 20..23
+          }
+        }
+        TOKEN_WHITESPACE(" ") 23..24
+        TOKEN_ASSIGN("=") 24..25
+        TOKEN_WHITESPACE(" ") 25..26
+        NODE_LITERAL 26..27 {
+          TOKEN_INTEGER("3") 26..27
+        }
+        TOKEN_SEMICOLON(";") 27..28
+      }
+      TOKEN_WHITESPACE(" ") 28..29
+      TOKEN_CURLY_B_CLOSE("}") 29..30
+    }
+  }
+}

test_data/parser/invalid_syntax/missing-semicolon-path.nix

@@ -0,0 +1 @@
+{ foo = bar. } // { xyz = 3; }

test_data/parser/invalid_syntax/missing-semicolon.expect

@@ -0,0 +1,64 @@
+error: wanted any of [TOKEN_SEMICOLON] at 13
+error: wanted any of [TOKEN_INTERPOL_START, TOKEN_STRING_START, TOKEN_IDENT] at 14
+error: wanted any of [TOKEN_SEMICOLON] at 17
+NODE_ROOT 0..34 {
+  NODE_BIN_OP 0..34 {
+    NODE_ATTR_SET 0..18 {
+      TOKEN_CURLY_B_OPEN("{") 0..1
+      TOKEN_WHITESPACE(" ") 1..2
+      NODE_ATTRPATH_VALUE 2..13 {
+        NODE_ATTRPATH 2..5 {
+          NODE_IDENT 2..5 {
+            TOKEN_IDENT("foo") 2..5
+          }
+        }
+        TOKEN_WHITESPACE(" ") 5..6
+        TOKEN_ASSIGN("=") 6..7
+        TOKEN_WHITESPACE(" ") 7..8
+        NODE_APPLY 8..13 {
+          NODE_LITERAL 8..9 {
+            TOKEN_INTEGER("1") 8..9
+          }
+          TOKEN_WHITESPACE(" ") 9..10
+          NODE_IDENT 10..13 {
+            TOKEN_IDENT("bar") 10..13
+          }
+        }
+      }
+      TOKEN_WHITESPACE(" ") 13..14
+      NODE_ATTRPATH_VALUE 14..17 {
+        NODE_ATTRPATH 14..14 {
+        }
+        TOKEN_ASSIGN("=") 14..15
+        TOKEN_WHITESPACE(" ") 15..16
+        NODE_LITERAL 16..17 {
+          TOKEN_INTEGER("2") 16..17
+        }
+      }
+      TOKEN_CURLY_B_CLOSE("}") 17..18
+    }
+    TOKEN_WHITESPACE(" ") 18..19
+    TOKEN_UPDATE("//") 19..21
+    TOKEN_WHITESPACE(" ") 21..22
+    NODE_ATTR_SET 22..34 {
+      TOKEN_CURLY_B_OPEN("{") 22..23
+      TOKEN_WHITESPACE(" ") 23..24
+      NODE_ATTRPATH_VALUE 24..32 {
+        NODE_ATTRPATH 24..27 {
+          NODE_IDENT 24..27 {
+            TOKEN_IDENT("xyz") 24..27
+          }
+        }
+        TOKEN_WHITESPACE(" ") 27..28
+        TOKEN_ASSIGN("=") 28..29
+        TOKEN_WHITESPACE(" ") 29..30
+        NODE_LITERAL 30..31 {
+          TOKEN_INTEGER("3") 30..31
+        }
+        TOKEN_SEMICOLON(";") 31..32
+      }
+      TOKEN_WHITESPACE(" ") 32..33
+      TOKEN_CURLY_B_CLOSE("}") 33..34
+    }
+  }
+}

test_data/parser/invalid_syntax/missing-semicolon.nix

@@ -0,0 +1 @@
+{ foo = 1 bar = 2} // { xyz = 3; }