Move integer and char parsing out of the parser and back into the lexer

src/parser.rs

@@ -3,7 +3,7 @@ use crate::{
     ast::*,
     lexer::{LexError, Lexer},
     span::Span,
-    token::{TKind, Token},
+    token::{Lexeme, TKind, Token},
 };
 use std::{error::Error, fmt::Display, vec};
 
@@ -90,7 +90,7 @@ impl<'t> Parser<'t> {
     }
 
     /// Consumes the currently-peeked [Token], returning its lexeme without cloning.
-    pub fn take_lexeme(&mut self) -> Option<String> {
+    pub fn take_lexeme(&mut self) -> Option<Lexeme> {
         self.take().map(|tok| tok.lexeme)
     }
 
@@ -196,19 +196,23 @@ impl<'t> Parse<'t> for Literal {
             TKind::Character => Literal::Char(
                 p.take_lexeme()
                     .expect("should have Token")
-                    .chars()
-                    .next()
+                    .char()
                     .expect("should have one char in char literal"),
             ),
             TKind::Integer => {
                 let Token { lexeme, kind: _, span } = p.take().expect("should have Token");
                 // TODO: more complex int parsing
                 let int = lexeme
-                    .parse()
-                    .map_err(|_| ParseError::Expected(TKind::Integer, span))?;
-                Literal::Int(int)
+                    .int()
+                    .ok_or(ParseError::Expected(TKind::Integer, span))?;
+                Literal::Int(int as _)
             }
-            TKind::String => Literal::Str(p.take_lexeme().expect("should have Token")),
+            TKind::String => Literal::Str({
+                let Token { lexeme, kind: _, span } = p.take().expect("should have Token");
+                lexeme
+                    .string()
+                    .ok_or(ParseError::Expected(TKind::String, span))?
+            }),
             _ => Err(ParseError::Expected(TKind::Integer, tok.span))?,
         })
     }
@@ -246,10 +250,14 @@ impl<'t> Parse<'t> for Pat {
             TKind::True | TKind::False | TKind::Character | TKind::Integer | TKind::String => {
                 Pat::Lit(p.parse(())?)
             }
-            TKind::Identifier => match tok.lexeme.as_str() {
-                "_" => p.consume().then(Pat::Ignore),
+            TKind::Identifier => match tok.lexeme.str() {
+                Some("_") => p.consume().then(Pat::Ignore),
                 _ => {
-                    let name = p.take_lexeme().expect("should have Token");
+                    let name = p
+                        .take_lexeme()
+                        .expect("should have Token")
+                        .string()
+                        .expect("Identifier token should have String");
                     match p.peek().map(|t| t.kind)? {
                         TKind::LParen => Pat::TupStruct(name, p.parse(PPrec::Tuple)?),
                         TKind::LCurly => Pat::Struct(
@@ -262,7 +270,7 @@ impl<'t> Parse<'t> for Pat {
                     }
                 }
             },
-            TKind::Grave => Pat::MetId(p.consume().next()?.lexeme),
+            TKind::Grave => Pat::MetId(p.consume().next()?.lexeme.to_string()),
             TKind::DotDot => Pat::Rest(match p.consume().peek_if(TKind::Identifier) {
                 Some(_) => Some(p.parse(level)?),
                 None => None,
@@ -314,9 +322,14 @@ impl<'t> Parse<'t> for Ty {
         let tok = p.peek()?;
 
         let head = match tok.kind {
-            TKind::Identifier => match tok.lexeme.as_str() {
-                "_" => p.consume().then(Ty::Infer),
-                _ => Ty::Named(p.take_lexeme().expect("should have Token")),
+            TKind::Identifier => match tok.lexeme.str() {
+                Some("_") => p.consume().then(Ty::Infer),
+                _ => Ty::Named(
+                    p.take_lexeme()
+                        .expect("should have Token")
+                        .string()
+                        .expect("Identifier token should have String"),
+                ),
             },
             TKind::LBrack => {
                 let ty = p.consume().parse(level)?;
@@ -550,7 +563,7 @@ impl<'t> Parse<'t> for Fn {
     fn parse(p: &mut Parser<'t>, _level: Self::Prec) -> PResult<Self> {
         match p.consume().next_if(TKind::Identifier) {
             Ok(Token { lexeme, .. }) => Ok(Self(
-                Some(lexeme),
+                lexeme.string(),
                 p.parse(PPrec::Typed)?,
                 p.parse(Prec::Body.next())?,
             )),
@@ -603,12 +616,18 @@ impl<'t> Parse<'t> for MatchArm {
 impl<'t> Parse<'t> for MakeArm {
     type Prec = ();
     fn parse(p: &mut Parser<'t>, _level: ()) -> PResult<Self> {
-        Ok(MakeArm(p.next_if(TKind::Identifier)?.lexeme, {
-            p.next_if(TKind::Colon)
-                .ok()
-                .map(|_| p.parse(Prec::Body.value()))
-                .transpose()?
-        }))
+        Ok(MakeArm(
+            p.next_if(TKind::Identifier)?
+                .lexeme
+                .string()
+                .expect("Identifier should have String"),
+            {
+                p.next_if(TKind::Colon)
+                    .ok()
+                    .map(|_| p.parse(Prec::Body.value()))
+                    .transpose()?
+            },
+        ))
     }
 }
 
@@ -639,8 +658,8 @@ impl<'t> Parse<'t> for Expr {
             Ps::End if level == prec.next() => Expr::Op(Op::Tuple, vec![]),
             Ps::End => Err(ParseError::NotPrefix(tok.kind, span))?,
 
-            Ps::Id => Expr::Id(p.take_lexeme().expect("should have ident")),
-            Ps::Mid => Expr::MetId(p.consume().next()?.lexeme),
+            Ps::Id => Expr::Id(p.take_lexeme().expect("should have ident").to_string()),
+            Ps::Mid => Expr::MetId(p.consume().next()?.lexeme.to_string()),
             Ps::Lit => Expr::Lit(p.parse(())?),
             Ps::Let => Expr::Let(p.parse(())?),
             Ps::Const => Expr::Const(p.parse(())?),