j/Doughlang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move integer and char parsing out of the parser and back into the lexer

Browse Source
This commit is contained in:
John
2025-10-10 14:45:08 -04:00
parent 0cbb800c19
commit b5d552376e
4 changed files with 114 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/lexer.rs
View File

@@ -75,15 +75,20 @@ impl<'t> Lexer<'t> {
LexError { pos: self.head, res }
}
pub fn as_str(&self) -> (&'t str, Span) {
let span = Span(self.head, self.tail);
(&self.text[Range::from(span)], span)
}
/// Produces a Token
pub fn produce(&mut self, kind: TKind) -> Token {
self.advance_tail();
let span = Span(self.head, self.tail);
let (lexeme, span) = self.as_str();
self.head = self.tail;
Token { lexeme: self.text[Range::from(span)].to_owned(), kind, span }
Token { lexeme: Lexeme::String(lexeme.to_owned()), kind, span }
}
pub fn produce_with_lexeme(&mut self, kind: TKind, lexeme: String) -> Token {
pub fn produce_with_lexeme(&mut self, kind: TKind, lexeme: Lexeme) -> Token {
self.advance_tail();
let span = Span(self.head, self.tail);
self.head = self.tail;
@@ -226,9 +231,10 @@ impl<'t> Lexer<'t> {
pub fn identifier(&mut self) -> Result<Token, LexError> {
while self.consume().peek().is_some_and(is_xid_continue) {}
let (lexeme, _span) = self.as_str();
let token = self.produce(TKind::Identifier);
Ok(Token {
kind: match token.lexeme.as_str() {
kind: match lexeme {
"as" => TKind::As,
"break" => TKind::Break,
"const" => TKind::Const,
@@ -236,6 +242,7 @@ impl<'t> Lexer<'t> {
"else" => TKind::Else,
"false" => TKind::False,
"fn" => TKind::Fn,
"for" => TKind::For,
"if" => TKind::If,
"let" => TKind::Let,
"loop" => TKind::Loop,
@@ -261,7 +268,7 @@ impl<'t> Lexer<'t> {
None => '\0',
};
if self.take().is_some_and(|c| c == '\'') {
Ok(self.produce_with_lexeme(TKind::Character, c.into()))
Ok(self.produce_with_lexeme(TKind::Character, Lexeme::Char(c)))
} else {
Err(self.error("Unterminated character"))
}
@@ -279,7 +286,7 @@ impl<'t> Lexer<'t> {
})
}
lexeme.shrink_to_fit();
Ok(self.produce_with_lexeme(TKind::String, lexeme))
Ok(self.produce_with_lexeme(TKind::String, Lexeme::String(lexeme)))
}
pub fn escape(&mut self) -> Result<char, LexError> {
@@ -318,10 +325,17 @@ impl<'t> Lexer<'t> {
}
pub fn digits<const BASE: u32>(&mut self) -> Result<Token, LexError> {
while self.peek().is_some_and(|c| c.is_digit(BASE)) {
let mut int: u128 = 0;
while let Some(c) = self.peek() {
int = match c.to_digit(BASE).ok_or(c) {
Err('_') => int,
Ok(c) => int.wrapping_mul(BASE as _).wrapping_add(c as _),
_ => break,
};
self.consume();
}
Ok(self.produce(TKind::Integer))
Ok(self.produce_with_lexeme(TKind::Integer, Lexeme::Integer(int, BASE)))
}
pub fn digit<const BASE: u32>(&mut self) -> Result<u32, LexError> {