Lexer: reduce code duplication in Rule mapping

produce_token renamed to map_rule
- Skips leading whitespace before any Rule evaluations
- Handles creation and destruction of Rule

libconlang/src/lib.rs

@@ -83,10 +83,12 @@ pub mod lexer {
                 self.cursor += len
             }
         }
-        /// Advances the cursor and produces a token
+        /// Advances the cursor and produces a token from a provided [Rule] function
-        fn produce_token(&mut self, ty: Type, len: usize) -> Option<Token> {
+        fn map_rule<F>(&mut self, rule: F, ty: Type) -> Option<Token>
+        where F: Fn(Rule) -> Rule {
+            self.skip_whitespace();
             let start = self.cursor;
-            self.cursor += len;
+            self.cursor += Rule::new(self.text()).and(rule).end()?;
             Some(Token::new(ty, start, self.cursor))
         }
         /// Gets a slice of text beginning at the cursor
@@ -126,85 +128,66 @@ pub mod lexer {
         }
         // functions for lexing individual tokens
         pub fn invalid(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.invalid(), Type::Invalid)
-            self.produce_token(Type::Invalid, Rule::new(self.text()).invalid().end()?)
         }
         // comments
         pub fn comment(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.comment(), Type::Comment)
-            self.produce_token(Type::Comment, Rule::new(self.text()).comment().end()?)
         }
         // keywords
         pub fn kw_else(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.str("else"), Type::KwElse)
-            self.produce_token(Type::KwElse, Rule::new(self.text()).str("else").end()?)
         }
         pub fn kw_for(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.str("for"), Type::KwFor)
-            self.produce_token(Type::KwFor, Rule::new(self.text()).str("for").end()?)
         }
         pub fn kw_fn(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.str("fn"), Type::KwFn)
-            self.produce_token(Type::KwFn, Rule::new(self.text()).str("fn").end()?)
         }
         pub fn kw_if(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.str("if"), Type::KwIf)
-            self.produce_token(Type::KwIf, Rule::new(self.text()).str("if").end()?)
         }
         pub fn kw_in(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.str("in"), Type::KwIn)
-            self.produce_token(Type::KwIn, Rule::new(self.text()).str("in").end()?)
         }
         pub fn kw_let(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.str("let"), Type::KwLet)
-            self.produce_token(Type::KwLet, Rule::new(self.text()).str("let").end()?)
         }
         pub fn kw_while(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.str("while"), Type::KwWhile)
-            self.produce_token(Type::KwWhile, Rule::new(self.text()).str("while").end()?)
         }
         // identifiers
         pub fn identifier(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.identifier(), Type::Identifier)
-            self.produce_token(Type::Identifier, Rule::new(self.text()).identifier().end()?)
         }
         // literals
         pub fn lit_integer(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.integer(), Type::LitInteger)
-            self.produce_token(Type::LitInteger, Rule::new(self.text()).integer().end()?)
         }
         pub fn lit_float(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.float(), Type::LitFloat)
-            self.produce_token(Type::LitFloat, Rule::new(self.text()).float().end()?)
         }
         pub fn lit_string(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.string(), Type::LitString)
-            self.produce_token(Type::LitString, Rule::new(self.text()).string().end()?)
         }
         // delimiters
         pub fn l_brack(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.char('['), Type::LBrack)
-            self.produce_token(Type::LBrack, Rule::new(self.text()).char('[').end()?)
         }
         pub fn r_brack(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.char(']'), Type::RBrack)
-            self.produce_token(Type::RBrack, Rule::new(self.text()).char(']').end()?)
         }
         pub fn l_curly(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.char('{'), Type::LCurly)
-            self.produce_token(Type::LCurly, Rule::new(self.text()).char('{').end()?)
         }
         pub fn r_curly(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.char('}'), Type::RCurly)
-            self.produce_token(Type::RCurly, Rule::new(self.text()).char('}').end()?)
         }
         pub fn l_paren(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.char('('), Type::LParen)
-            self.produce_token(Type::LParen, Rule::new(self.text()).char('(').end()?)
         }
         pub fn r_paren(&mut self) -> Option<Token> {
-            self.skip_whitespace();
+            self.map_rule(|r| r.char(')'), Type::RParen)
-            self.produce_token(Type::RParen, Rule::new(self.text()).char(')').end()?)
         }
     }