From 5c4c8bcb802361341841698b6503f78e31d5d6a8 Mon Sep 17 00:00:00 2001
From: John <j@soft.fish>
Date: Wed, 25 Oct 2023 19:28:54 -0500
Subject: [PATCH] lexer: Consume invalid characters

---
 libconlang/examples/identify_tokens.rs |  2 +-
 libconlang/src/lexer.rs                | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/libconlang/examples/identify_tokens.rs b/libconlang/examples/identify_tokens.rs
index 1fc4e65..6799df0 100644
--- a/libconlang/examples/identify_tokens.rs
+++ b/libconlang/examples/identify_tokens.rs
@@ -46,7 +46,7 @@ fn lex_tokens(file: &str, path: Option<&Path>) -> Result<(), Box<dyn Error>> {
             Ok(t) => t,
             Err(e) => {
                 println!("{e:?}");
-                break;
+                continue;
             },
         };
         if let Some(path) = path {
diff --git a/libconlang/src/lexer.rs b/libconlang/src/lexer.rs
index 477c14c..3e9ef2e 100644
--- a/libconlang/src/lexer.rs
+++ b/libconlang/src/lexer.rs
@@ -42,10 +42,10 @@ pub mod lexer_iter {
 }
 
 /// The Lexer iterates over the characters in a body of text, searching for [Tokens](Token).
-/// 
+///
 /// # Examples
 /// ```rust
-///# use conlang::lexer::Lexer;
+/// # use conlang::lexer::Lexer;
 /// // Read in your code from somewhere
 /// let some_code = "
 /// fn main () {
@@ -59,7 +59,7 @@ pub mod lexer_iter {
 /// println!("{first_token:?}");
 /// // Loop over all the rest of the tokens
 /// for token in lexer {
-///#    let token: Result<_,()> = Ok(token.unwrap());
+/// #   let token: Result<_,()> = Ok(token.unwrap());
 ///     match token {
 ///         Ok(token) => println!("{token:?}"),
 ///         Err(e) => eprintln!("{e:?}"),
@@ -123,7 +123,11 @@ impl<'t> Lexer<'t> {
             '\'' => self.consume()?.character(),
             '_' => self.identifier(),
             i if i.is_xid_start() => self.identifier(),
-            e => Err(Error::unexpected_char(e, self.line(), self.col())),
+            e => {
+                let err = Err(Error::unexpected_char(e, self.line(), self.col()));
+                let _ = self.consume();
+                err
+            }
         }
     }
     /// Returns the current line