j/Conlang
1
1
Fork 0
Code Issues 5 Pull Requests Packages Projects 2 Releases Wiki Activity

cl-token: Rename Type to TokenKind, Data to TokenData to match the project's general style

Browse Source
This commit is contained in:
John 2024-04-12 14:36:26 -05:00
parent 902494e95a
commit 2091cce570
8 changed files with 440 additions and 430 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
cl-lexer/src/lib.rs
View File

@ -2,7 +2,7 @@
#![warn(clippy::all)] #![warn(clippy::all)]
#![feature(decl_macro)] #![feature(decl_macro)]
use cl_structures::span::Loc; use cl_structures::span::Loc;
use cl_token::*; use cl_token::{TokenKind as Kind, *};
use std::{ use std::{
iter::Peekable, iter::Peekable,
str::{Chars, FromStr}, str::{Chars, FromStr},
@ -97,33 +97,33 @@ impl<'t> Lexer<'t> {
/// Scans through the text, searching for the next [Token] /// Scans through the text, searching for the next [Token]
pub fn scan(&mut self) -> LResult<Token> { pub fn scan(&mut self) -> LResult<Token> {
match self.skip_whitespace().peek()? { match self.skip_whitespace().peek()? {
'{' => self.consume()?.produce(Type::LCurly, ()), '{' => self.consume()?.produce(Kind::LCurly, ()),
'}' => self.consume()?.produce(Type::RCurly, ()), '}' => self.consume()?.produce(Kind::RCurly, ()),
'[' => self.consume()?.produce(Type::LBrack, ()), '[' => self.consume()?.produce(Kind::LBrack, ()),
']' => self.consume()?.produce(Type::RBrack, ()), ']' => self.consume()?.produce(Kind::RBrack, ()),
'(' => self.consume()?.produce(Type::LParen, ()), '(' => self.consume()?.produce(Kind::LParen, ()),
')' => self.consume()?.produce(Type::RParen, ()), ')' => self.consume()?.produce(Kind::RParen, ()),
'&' => self.consume()?.amp(), '&' => self.consume()?.amp(),
'@' => self.consume()?.produce(Type::At, ()), '@' => self.consume()?.produce(Kind::At, ()),
'\\' => self.consume()?.produce(Type::Backslash, ()), '\\' => self.consume()?.produce(Kind::Backslash, ()),
'!' => self.consume()?.bang(), '!' => self.consume()?.bang(),
'|' => self.consume()?.bar(), '|' => self.consume()?.bar(),
':' => self.consume()?.colon(), ':' => self.consume()?.colon(),
',' => self.consume()?.produce(Type::Comma, ()), ',' => self.consume()?.produce(Kind::Comma, ()),
'.' => self.consume()?.dot(), '.' => self.consume()?.dot(),
'=' => self.consume()?.equal(), '=' => self.consume()?.equal(),
'`' => self.consume()?.produce(Type::Grave, ()), '`' => self.consume()?.produce(Kind::Grave, ()),
'>' => self.consume()?.greater(), '>' => self.consume()?.greater(),
'#' => self.consume()?.hash(), '#' => self.consume()?.hash(),
'<' => self.consume()?.less(), '<' => self.consume()?.less(),
'-' => self.consume()?.minus(), '-' => self.consume()?.minus(),
'+' => self.consume()?.plus(), '+' => self.consume()?.plus(),
'?' => self.consume()?.produce(Type::Question, ()), '?' => self.consume()?.produce(Kind::Question, ()),
'%' => self.consume()?.rem(), '%' => self.consume()?.rem(),
';' => self.consume()?.produce(Type::Semi, ()), ';' => self.consume()?.produce(Kind::Semi, ()),
'/' => self.consume()?.slash(), '/' => self.consume()?.slash(),
'*' => self.consume()?.star(), '*' => self.consume()?.star(),
'~' => self.consume()?.produce(Type::Tilde, ()), '~' => self.consume()?.produce(Kind::Tilde, ()),
'^' => self.consume()?.xor(), '^' => self.consume()?.xor(),
'0' => self.consume()?.int_with_base(), '0' => self.consume()?.int_with_base(),
'1'..='9' => self.digits::<10>(), '1'..='9' => self.digits::<10>(),
@ -157,7 +157,7 @@ impl<'t> Lexer<'t> {
.copied() .copied()
.ok_or(Error::end_of_file(self.line(), self.col())) .ok_or(Error::end_of_file(self.line(), self.col()))
} }
fn produce(&mut self, ty: Type, data: impl Into<Data>) -> LResult<Token> { fn produce(&mut self, ty: TokenKind, data: impl Into<TokenData>) -> LResult<Token> {
let loc = self.start_loc; let loc = self.start_loc;
self.start_loc = self.current_loc; self.start_loc = self.current_loc;
self.start = self.current; self.start = self.current;
@ -192,120 +192,120 @@ impl<'t> Lexer<'t> {
impl<'t> Lexer<'t> { impl<'t> Lexer<'t> {
fn amp(&mut self) -> LResult<Token> { fn amp(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('&') => self.consume()?.produce(Type::AmpAmp, ()), Ok('&') => self.consume()?.produce(Kind::AmpAmp, ()),
Ok('=') => self.consume()?.produce(Type::AmpEq, ()), Ok('=') => self.consume()?.produce(Kind::AmpEq, ()),
_ => self.produce(Type::Amp, ()), _ => self.produce(Kind::Amp, ()),
} }
} }
fn bang(&mut self) -> LResult<Token> { fn bang(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('!') => self.consume()?.produce(Type::BangBang, ()), Ok('!') => self.consume()?.produce(Kind::BangBang, ()),
Ok('=') => self.consume()?.produce(Type::BangEq, ()), Ok('=') => self.consume()?.produce(Kind::BangEq, ()),
_ => self.produce(Type::Bang, ()), _ => self.produce(Kind::Bang, ()),
} }
} }
fn bar(&mut self) -> LResult<Token> { fn bar(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('|') => self.consume()?.produce(Type::BarBar, ()), Ok('|') => self.consume()?.produce(Kind::BarBar, ()),
Ok('=') => self.consume()?.produce(Type::BarEq, ()), Ok('=') => self.consume()?.produce(Kind::BarEq, ()),
_ => self.produce(Type::Bar, ()), _ => self.produce(Kind::Bar, ()),
} }
} }
fn colon(&mut self) -> LResult<Token> { fn colon(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok(':') => self.consume()?.produce(Type::ColonColon, ()), Ok(':') => self.consume()?.produce(Kind::ColonColon, ()),
_ => self.produce(Type::Colon, ()), _ => self.produce(Kind::Colon, ()),
} }
} }
fn dot(&mut self) -> LResult<Token> { fn dot(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('.') => { Ok('.') => {
if let Ok('=') = self.consume()?.peek() { if let Ok('=') = self.consume()?.peek() {
self.consume()?.produce(Type::DotDotEq, ()) self.consume()?.produce(Kind::DotDotEq, ())
} else { } else {
self.produce(Type::DotDot, ()) self.produce(Kind::DotDot, ())
} }
} }
_ => self.produce(Type::Dot, ()), _ => self.produce(Kind::Dot, ()),
} }
} }
fn equal(&mut self) -> LResult<Token> { fn equal(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::EqEq, ()), Ok('=') => self.consume()?.produce(Kind::EqEq, ()),
Ok('>') => self.consume()?.produce(Type::FatArrow, ()), Ok('>') => self.consume()?.produce(Kind::FatArrow, ()),
_ => self.produce(Type::Eq, ()), _ => self.produce(Kind::Eq, ()),
} }
} }
fn greater(&mut self) -> LResult<Token> { fn greater(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::GtEq, ()), Ok('=') => self.consume()?.produce(Kind::GtEq, ()),
Ok('>') => { Ok('>') => {
if let Ok('=') = self.consume()?.peek() { if let Ok('=') = self.consume()?.peek() {
self.consume()?.produce(Type::GtGtEq, ()) self.consume()?.produce(Kind::GtGtEq, ())
} else { } else {
self.produce(Type::GtGt, ()) self.produce(Kind::GtGt, ())
} }
} }
_ => self.produce(Type::Gt, ()), _ => self.produce(Kind::Gt, ()),
} }
} }
fn hash(&mut self) -> LResult<Token> { fn hash(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('!') => self.consume()?.produce(Type::HashBang, ()), Ok('!') => self.consume()?.produce(Kind::HashBang, ()),
_ => self.produce(Type::Hash, ()), _ => self.produce(Kind::Hash, ()),
} }
} }
fn less(&mut self) -> LResult<Token> { fn less(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::LtEq, ()), Ok('=') => self.consume()?.produce(Kind::LtEq, ()),
Ok('<') => { Ok('<') => {
if let Ok('=') = self.consume()?.peek() { if let Ok('=') = self.consume()?.peek() {
self.consume()?.produce(Type::LtLtEq, ()) self.consume()?.produce(Kind::LtLtEq, ())
} else { } else {
self.produce(Type::LtLt, ()) self.produce(Kind::LtLt, ())
} }
} }
_ => self.produce(Type::Lt, ()), _ => self.produce(Kind::Lt, ()),
} }
} }
fn minus(&mut self) -> LResult<Token> { fn minus(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::MinusEq, ()), Ok('=') => self.consume()?.produce(Kind::MinusEq, ()),
Ok('>') => self.consume()?.produce(Type::Arrow, ()), Ok('>') => self.consume()?.produce(Kind::Arrow, ()),
_ => self.produce(Type::Minus, ()), _ => self.produce(Kind::Minus, ()),
} }
} }
fn plus(&mut self) -> LResult<Token> { fn plus(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::PlusEq, ()), Ok('=') => self.consume()?.produce(Kind::PlusEq, ()),
_ => self.produce(Type::Plus, ()), _ => self.produce(Kind::Plus, ()),
} }
} }
fn rem(&mut self) -> LResult<Token> { fn rem(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::RemEq, ()), Ok('=') => self.consume()?.produce(Kind::RemEq, ()),
_ => self.produce(Type::Rem, ()), _ => self.produce(Kind::Rem, ()),
} }
} }
fn slash(&mut self) -> LResult<Token> { fn slash(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::SlashEq, ()), Ok('=') => self.consume()?.produce(Kind::SlashEq, ()),
Ok('/') => self.consume()?.line_comment(), Ok('/') => self.consume()?.line_comment(),
Ok('*') => self.consume()?.block_comment(), Ok('*') => self.consume()?.block_comment(),
_ => self.produce(Type::Slash, ()), _ => self.produce(Kind::Slash, ()),
} }
} }
fn star(&mut self) -> LResult<Token> { fn star(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::StarEq, ()), Ok('=') => self.consume()?.produce(Kind::StarEq, ()),
_ => self.produce(Type::Star, ()), _ => self.produce(Kind::Star, ()),
} }
} }
fn xor(&mut self) -> LResult<Token> { fn xor(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::XorEq, ()), Ok('=') => self.consume()?.produce(Kind::XorEq, ()),
Ok('^') => self.consume()?.produce(Type::XorXor, ()), Ok('^') => self.consume()?.produce(Kind::XorXor, ()),
_ => self.produce(Type::Xor, ()), _ => self.produce(Kind::Xor, ()),
} }
} }
} }
@ -315,7 +315,7 @@ impl<'t> Lexer<'t> {
while Ok('\n') != self.peek() { while Ok('\n') != self.peek() {
self.consume()?; self.consume()?;
} }
self.produce(Type::Comment, ()) self.produce(Kind::Comment, ())
} }
fn block_comment(&mut self) -> LResult<Token> { fn block_comment(&mut self) -> LResult<Token> {
while let Ok(c) = self.next() { while let Ok(c) = self.next() {
@ -323,7 +323,7 @@ impl<'t> Lexer<'t> {
break; break;
} }
} }
self.produce(Type::Comment, ()) self.produce(Kind::Comment, ())
} }
} }
/// Identifiers /// Identifiers
@ -333,10 +333,10 @@ impl<'t> Lexer<'t> {
while let Ok(c) = self.xid_continue() { while let Ok(c) = self.xid_continue() {
out.push(c) out.push(c)
} }
if let Ok(keyword) = Type::from_str(&out) { if let Ok(keyword) = Kind::from_str(&out) {
self.produce(keyword, ()) self.produce(keyword, ())
} else { } else {
self.produce(Type::Identifier, Data::Identifier(out.into())) self.produce(Kind::Identifier, TokenData::Identifier(out.into()))
} }
} }
fn xid_start(&mut self) -> LResult<char> { fn xid_start(&mut self) -> LResult<char> {
@ -367,7 +367,7 @@ impl<'t> Lexer<'t> {
Ok('o') => self.consume()?.digits::<8>(), Ok('o') => self.consume()?.digits::<8>(),
Ok('b') => self.consume()?.digits::<2>(), Ok('b') => self.consume()?.digits::<2>(),
Ok('0'..='9') => self.digits::<10>(), Ok('0'..='9') => self.digits::<10>(),
_ => self.produce(Type::Integer, 0), _ => self.produce(Kind::Integer, 0),
} }
} }
fn digits<const B: u32>(&mut self) -> LResult<Token> { fn digits<const B: u32>(&mut self) -> LResult<Token> {
@ -375,7 +375,7 @@ impl<'t> Lexer<'t> {
while let Ok(true) = self.peek().as_ref().map(char::is_ascii_alphanumeric) { while let Ok(true) = self.peek().as_ref().map(char::is_ascii_alphanumeric) {
value = value * B as u128 + self.digit::<B>()? as u128; value = value * B as u128 + self.digit::<B>()? as u128;
} }
self.produce(Type::Integer, value) self.produce(Kind::Integer, value)
} }
fn digit<const B: u32>(&mut self) -> LResult<u32> { fn digit<const B: u32>(&mut self) -> LResult<u32> {
let digit = self.peek()?; let digit = self.peek()?;
@ -396,12 +396,12 @@ impl<'t> Lexer<'t> {
{ {
value.push(self.unescape()?) value.push(self.unescape()?)
} }
self.consume()?.produce(Type::String, value) self.consume()?.produce(Kind::String, value)
} }
fn character(&mut self) -> LResult<Token> { fn character(&mut self) -> LResult<Token> {
let out = self.unescape()?; let out = self.unescape()?;
match self.peek()? { match self.peek()? {
'\'' => self.consume()?.produce(Type::Character, out), '\'' => self.consume()?.produce(Kind::Character, out),
_ => Err(Error::unmatched_delimiters('\'', self.line(), self.col())), _ => Err(Error::unmatched_delimiters('\'', self.line(), self.col())),
} }
} }
@ -477,7 +477,7 @@ pub mod error {
pub enum Reason { pub enum Reason {
/// Found an opening delimiter of type [char], but not the expected closing delimiter /// Found an opening delimiter of type [char], but not the expected closing delimiter
UnmatchedDelimiters(char), UnmatchedDelimiters(char),
/// Found a character that doesn't belong to any [Type](cl_token::token_type::Type) /// Found a character that doesn't belong to any [TokenKind](cl_token::TokenKind)
UnexpectedChar(char), UnexpectedChar(char),
/// Found a character that's not valid in identifiers while looking for an identifier /// Found a character that's not valid in identifiers while looking for an identifier
NotIdentifier(char), NotIdentifier(char),

108
cl-lexer/src/tests.rs
View File

@ -35,7 +35,7 @@ macro td ($($id:expr),*) {
mod ident { mod ident {
use super::*; use super::*;
macro ident ($($id:literal),*) { macro ident ($($id:literal),*) {
[$(Data::Identifier($id.into())),*] [$(TokenData::Identifier($id.into())),*]
} }
test_lexer_data_type! { test_lexer_data_type! {
underscore { "_ _" => ident!["_", "_"] } underscore { "_ _" => ident!["_", "_"] }
@ -47,7 +47,7 @@ mod ident {
mod keyword { mod keyword {
use super::*; use super::*;
macro kw($($k:ident),*) { macro kw($($k:ident),*) {
[ $(Type::$k,)* ] [ $(TokenKind::$k,)* ]
} }
test_lexer_output_type! { test_lexer_output_type! {
kw_break { "break break" => kw![Break, Break] } kw_break { "break break" => kw![Break, Break] }
@ -111,57 +111,57 @@ mod string {
mod punct { mod punct {
use super::*; use super::*;
test_lexer_output_type! { test_lexer_output_type! {
l_curly { "{ {" => [ Type::LCurly, Type::LCurly ] } l_curly { "{ {" => [ TokenKind::LCurly, TokenKind::LCurly ] }
r_curly { "} }" => [ Type::RCurly, Type::RCurly ] } r_curly { "} }" => [ TokenKind::RCurly, TokenKind::RCurly ] }
l_brack { "[ [" => [ Type::LBrack, Type::LBrack ] } l_brack { "[ [" => [ TokenKind::LBrack, TokenKind::LBrack ] }
r_brack { "] ]" => [ Type::RBrack, Type::RBrack ] } r_brack { "] ]" => [ TokenKind::RBrack, TokenKind::RBrack ] }
l_paren { "( (" => [ Type::LParen, Type::LParen ] } l_paren { "( (" => [ TokenKind::LParen, TokenKind::LParen ] }
r_paren { ") )" => [ Type::RParen, Type::RParen ] } r_paren { ") )" => [ TokenKind::RParen, TokenKind::RParen ] }
amp { "& &" => [ Type::Amp, Type::Amp ] } amp { "& &" => [ TokenKind::Amp, TokenKind::Amp ] }
amp_amp { "&& &&" => [ Type::AmpAmp, Type::AmpAmp ] } amp_amp { "&& &&" => [ TokenKind::AmpAmp, TokenKind::AmpAmp ] }
amp_eq { "&= &=" => [ Type::AmpEq, Type::AmpEq ] } amp_eq { "&= &=" => [ TokenKind::AmpEq, TokenKind::AmpEq ] }
arrow { "-> ->" => [ Type::Arrow, Type::Arrow] } arrow { "-> ->" => [ TokenKind::Arrow, TokenKind::Arrow] }
at { "@ @" => [ Type::At, Type::At] } at { "@ @" => [ TokenKind::At, TokenKind::At] }
backslash { "\\ \\" => [ Type::Backslash, Type::Backslash] } backslash { "\\ \\" => [ TokenKind::Backslash, TokenKind::Backslash] }
bang { "! !" => [ Type::Bang, Type::Bang] } bang { "! !" => [ TokenKind::Bang, TokenKind::Bang] }
bangbang { "!! !!" => [ Type::BangBang, Type::BangBang] } bangbang { "!! !!" => [ TokenKind::BangBang, TokenKind::BangBang] }
bangeq { "!= !=" => [ Type::BangEq, Type::BangEq] } bangeq { "!= !=" => [ TokenKind::BangEq, TokenKind::BangEq] }
bar { "| |" => [ Type::Bar, Type::Bar] } bar { "| |" => [ TokenKind::Bar, TokenKind::Bar] }
barbar { "|| ||" => [ Type::BarBar, Type::BarBar] } barbar { "|| ||" => [ TokenKind::BarBar, TokenKind::BarBar] }
bareq { "|= |=" => [ Type::BarEq, Type::BarEq] } bareq { "|= |=" => [ TokenKind::BarEq, TokenKind::BarEq] }
colon { ": :" => [ Type::Colon, Type::Colon] } colon { ": :" => [ TokenKind::Colon, TokenKind::Colon] }
comma { ", ," => [ Type::Comma, Type::Comma] } comma { ", ," => [ TokenKind::Comma, TokenKind::Comma] }
dot { ". ." => [ Type::Dot, Type::Dot] } dot { ". ." => [ TokenKind::Dot, TokenKind::Dot] }
dotdot { ".. .." => [ Type::DotDot, Type::DotDot] } dotdot { ".. .." => [ TokenKind::DotDot, TokenKind::DotDot] }
dotdoteq { "..= ..=" => [ Type::DotDotEq, Type::DotDotEq] } dotdoteq { "..= ..=" => [ TokenKind::DotDotEq, TokenKind::DotDotEq] }
eq { "= =" => [ Type::Eq, Type::Eq] } eq { "= =" => [ TokenKind::Eq, TokenKind::Eq] }
eqeq { "== ==" => [ Type::EqEq, Type::EqEq] } eqeq { "== ==" => [ TokenKind::EqEq, TokenKind::EqEq] }
fatarrow { "=> =>" => [ Type::FatArrow, Type::FatArrow] } fatarrow { "=> =>" => [ TokenKind::FatArrow, TokenKind::FatArrow] }
grave { "` `" => [ Type::Grave, Type::Grave] } grave { "` `" => [ TokenKind::Grave, TokenKind::Grave] }
gt { "> >" => [ Type::Gt, Type::Gt] } gt { "> >" => [ TokenKind::Gt, TokenKind::Gt] }
gteq { ">= >=" => [ Type::GtEq, Type::GtEq] } gteq { ">= >=" => [ TokenKind::GtEq, TokenKind::GtEq] }
gtgt { ">> >>" => [ Type::GtGt, Type::GtGt] } gtgt { ">> >>" => [ TokenKind::GtGt, TokenKind::GtGt] }
gtgteq { ">>= >>=" => [ Type::GtGtEq, Type::GtGtEq] } gtgteq { ">>= >>=" => [ TokenKind::GtGtEq, TokenKind::GtGtEq] }
hash { "# #" => [ Type::Hash, Type::Hash] } hash { "# #" => [ TokenKind::Hash, TokenKind::Hash] }
lt { "< <" => [ Type::Lt, Type::Lt] } lt { "< <" => [ TokenKind::Lt, TokenKind::Lt] }
lteq { "<= <=" => [ Type::LtEq, Type::LtEq] } lteq { "<= <=" => [ TokenKind::LtEq, TokenKind::LtEq] }
ltlt { "<< <<" => [ Type::LtLt, Type::LtLt] } ltlt { "<< <<" => [ TokenKind::LtLt, TokenKind::LtLt] }
ltlteq { "<<= <<=" => [ Type::LtLtEq, Type::LtLtEq] } ltlteq { "<<= <<=" => [ TokenKind::LtLtEq, TokenKind::LtLtEq] }
minus { "- -" => [ Type::Minus, Type::Minus] } minus { "- -" => [ TokenKind::Minus, TokenKind::Minus] }
minuseq { "-= -=" => [ Type::MinusEq, Type::MinusEq] } minuseq { "-= -=" => [ TokenKind::MinusEq, TokenKind::MinusEq] }
plus { "+ +" => [ Type::Plus, Type::Plus] } plus { "+ +" => [ TokenKind::Plus, TokenKind::Plus] }
pluseq { "+= +=" => [ Type::PlusEq, Type::PlusEq] } pluseq { "+= +=" => [ TokenKind::PlusEq, TokenKind::PlusEq] }
question { "? ?" => [ Type::Question, Type::Question] } question { "? ?" => [ TokenKind::Question, TokenKind::Question] }
rem { "% %" => [ Type::Rem, Type::Rem] } rem { "% %" => [ TokenKind::Rem, TokenKind::Rem] }
remeq { "%= %=" => [ Type::RemEq, Type::RemEq] } remeq { "%= %=" => [ TokenKind::RemEq, TokenKind::RemEq] }
semi { "; ;" => [ Type::Semi, Type::Semi] } semi { "; ;" => [ TokenKind::Semi, TokenKind::Semi] }
slash { "/ /" => [ Type::Slash, Type::Slash] } slash { "/ /" => [ TokenKind::Slash, TokenKind::Slash] }
slasheq { "/= /=" => [ Type::SlashEq, Type::SlashEq] } slasheq { "/= /=" => [ TokenKind::SlashEq, TokenKind::SlashEq] }
star { "* *" => [ Type::Star, Type::Star] } star { "* *" => [ TokenKind::Star, TokenKind::Star] }
stareq { "*= *=" => [ Type::StarEq, Type::StarEq] } stareq { "*= *=" => [ TokenKind::StarEq, TokenKind::StarEq] }
tilde { "~ ~" => [ Type::Tilde, Type::Tilde] } tilde { "~ ~" => [ TokenKind::Tilde, TokenKind::Tilde] }
xor { "^ ^" => [ Type::Xor, Type::Xor] } xor { "^ ^" => [ TokenKind::Xor, TokenKind::Xor] }
xoreq { "^= ^=" => [ Type::XorEq, Type::XorEq] } xoreq { "^= ^=" => [ TokenKind::XorEq, TokenKind::XorEq] }
xorxor { "^^ ^^" => [ Type::XorXor, Type::XorXor] } xorxor { "^^ ^^" => [ TokenKind::XorXor, TokenKind::XorXor] }
} }
} }

6
cl-parser/src/error.rs
View File

@ -21,10 +21,10 @@ pub enum ErrorKind {
UnmatchedParentheses, UnmatchedParentheses,
UnmatchedCurlyBraces, UnmatchedCurlyBraces,
UnmatchedSquareBrackets, UnmatchedSquareBrackets,
Unexpected(Type), Unexpected(TokenKind),
Expected { Expected {
want: Type, want: TokenKind,
got: Type, got: TokenKind,
}, },
/// No rules matched /// No rules matched
Nothing, Nothing,

390
cl-parser/src/parser.rs
View File

@ -39,8 +39,8 @@ impl<'t> Parser<'t> {
.scan() .scan()
.map_err(|e| self.error(e.into(), while_parsing))? .map_err(|e| self.error(e.into(), while_parsing))?
{ {
t if t.ty() == Type::Invalid => continue, t if t.ty() == TokenKind::Invalid => continue,
t if t.ty() == Type::Comment => continue, t if t.ty() == TokenKind::Comment => continue,
t => break Ok(t), t => break Ok(t),
} }
} }
@ -63,8 +63,8 @@ impl<'t> Parser<'t> {
self.loc = Loc::from(&self.lexer); self.loc = Loc::from(&self.lexer);
self.next.take() self.next.take()
} }
/// Looks ahead at the next [Token]'s [Type] /// Looks ahead at the next [Token]'s [TokenKind]
pub fn peek_type(&mut self, while_parsing: Parsing) -> PResult<Type> { pub fn peek_kind(&mut self, while_parsing: Parsing) -> PResult<TokenKind> {
self.peek(while_parsing).map(|t| t.ty()) self.peek(while_parsing).map(|t| t.ty())
} }
/// Consumes one [Token] /// Consumes one [Token]
@ -75,9 +75,9 @@ impl<'t> Parser<'t> {
None => self.consume_from_lexer(while_parsing), None => self.consume_from_lexer(while_parsing),
} }
} }
/// Consumes the next [Token] if it matches the pattern [Type] /// Consumes the next [Token] if it matches the pattern [TokenKind]
pub fn match_type(&mut self, want: Type, while_parsing: Parsing) -> PResult<Token> { pub fn match_type(&mut self, want: TokenKind, while_parsing: Parsing) -> PResult<Token> {
let got = self.peek_type(while_parsing)?; let got = self.peek_kind(while_parsing)?;
if got == want { if got == want {
Ok(self.consume_peeked().expect("should not fail after peek")) Ok(self.consume_peeked().expect("should not fail after peek"))
} else { } else {
@ -88,16 +88,16 @@ impl<'t> Parser<'t> {
// the three matched delimiter pairs // the three matched delimiter pairs
/// Square brackets: `[` `]` /// Square brackets: `[` `]`
const BRACKETS: (Type, Type) = (Type::LBrack, Type::RBrack); const BRACKETS: (TokenKind, TokenKind) = (TokenKind::LBrack, TokenKind::RBrack);
/// Curly braces: `{` `}` /// Curly braces: `{` `}`
const CURLIES: (Type, Type) = (Type::LCurly, Type::RCurly); const CURLIES: (TokenKind, TokenKind) = (TokenKind::LCurly, TokenKind::RCurly);
/// Parentheses: `(` `)` /// Parentheses: `(` `)`
const PARENS: (Type, Type) = (Type::LParen, Type::RParen); const PARENS: (TokenKind, TokenKind) = (TokenKind::LParen, TokenKind::RParen);
/// Parses constructions of the form `delim.0 f delim.1` (i.e. `(` `foobar` `)`) /// Parses constructions of the form `delim.0 f delim.1` (i.e. `(` `foobar` `)`)
const fn delim<'t, T>( const fn delim<'t, T>(
f: impl Fn(&mut Parser<'t>) -> PResult<T>, f: impl Fn(&mut Parser<'t>) -> PResult<T>,
delim: (Type, Type), delim: (TokenKind, TokenKind),
while_parsing: Parsing, while_parsing: Parsing,
) -> impl Fn(&mut Parser<'t>) -> PResult<T> { ) -> impl Fn(&mut Parser<'t>) -> PResult<T> {
move |parser| { move |parser| {
@ -113,15 +113,15 @@ const fn delim<'t, T>(
/// where `~until` is a negative lookahead assertion /// where `~until` is a negative lookahead assertion
const fn sep<'t, T>( const fn sep<'t, T>(
f: impl Fn(&mut Parser<'t>) -> PResult<T>, f: impl Fn(&mut Parser<'t>) -> PResult<T>,
sep: Type, sep: TokenKind,
until: Type, until: TokenKind,
while_parsing: Parsing, while_parsing: Parsing,
) -> impl Fn(&mut Parser<'t>) -> PResult<Vec<T>> { ) -> impl Fn(&mut Parser<'t>) -> PResult<Vec<T>> {
move |parser| { move |parser| {
let mut args = vec![]; let mut args = vec![];
while until != parser.peek_type(while_parsing)? { while until != parser.peek_kind(while_parsing)? {
args.push(f(parser)?); args.push(f(parser)?);
if sep != parser.peek_type(while_parsing)? { if sep != parser.peek_kind(while_parsing)? {
break; break;
} }
parser.consume_peeked(); parser.consume_peeked();
@ -136,30 +136,30 @@ const fn sep<'t, T>(
#[allow(dead_code)] #[allow(dead_code)]
const fn rep<'t, T>( const fn rep<'t, T>(
f: impl Fn(&mut Parser<'t>) -> PResult<T>, f: impl Fn(&mut Parser<'t>) -> PResult<T>,
until: Type, until: TokenKind,
while_parsing: Parsing, while_parsing: Parsing,
) -> impl Fn(&mut Parser<'t>) -> PResult<Vec<T>> { ) -> impl Fn(&mut Parser<'t>) -> PResult<Vec<T>> {
move |parser| { move |parser| {
let mut out = vec![]; let mut out = vec![];
while until != parser.peek_type(while_parsing)? { while until != parser.peek_kind(while_parsing)? {
out.push(f(parser)?) out.push(f(parser)?)
} }
Ok(out) Ok(out)
} }
} }
/// Expands to a pattern which matches item-like [Token] [Type]s /// Expands to a pattern which matches item-like [Token] [TokenKind]s
macro item_like() { macro item_like() {
Type::Hash TokenKind::Hash
| Type::Pub | TokenKind::Pub
| Type::Type | TokenKind::Type
| Type::Const | TokenKind::Const
| Type::Static | TokenKind::Static
| Type::Mod | TokenKind::Mod
| Type::Fn | TokenKind::Fn
| Type::Struct | TokenKind::Struct
| Type::Enum | TokenKind::Enum
| Type::Impl | TokenKind::Impl
} }
/// Top level parsing /// Top level parsing
@ -167,8 +167,8 @@ impl<'t> Parser<'t> {
/// Parses a [File] /// Parses a [File]
pub fn file(&mut self) -> PResult<File> { pub fn file(&mut self) -> PResult<File> {
let mut items = vec![]; let mut items = vec![];
while match self.peek_type(Parsing::File) { while match self.peek_kind(Parsing::File) {
Ok(Type::RCurly) | Err(Error { reason: EndOfInput, .. }) => false, Ok(TokenKind::RCurly) | Err(Error { reason: EndOfInput, .. }) => false,
Ok(_) => true, Ok(_) => true,
Err(e) => Err(e)?, Err(e) => Err(e)?,
} { } {
@ -203,13 +203,13 @@ impl<'t> Parser<'t> {
/// See also: [Parser::path_part], [Parser::identifier] /// See also: [Parser::path_part], [Parser::identifier]
pub fn path(&mut self) -> PResult<Path> { pub fn path(&mut self) -> PResult<Path> {
const PARSING: Parsing = Parsing::PathExpr; const PARSING: Parsing = Parsing::PathExpr;
let absolute = matches!(self.peek_type(PARSING)?, Type::ColonColon); let absolute = matches!(self.peek_kind(PARSING)?, TokenKind::ColonColon);
if absolute { if absolute {
self.consume_peeked(); self.consume_peeked();
} }
let mut parts = vec![self.path_part()?]; let mut parts = vec![self.path_part()?];
while let Ok(Type::ColonColon) = self.peek_type(PARSING) { while let Ok(TokenKind::ColonColon) = self.peek_kind(PARSING) {
self.consume_peeked(); self.consume_peeked();
parts.push(self.path_part()?); parts.push(self.path_part()?);
} }
@ -224,8 +224,8 @@ impl<'t> Parser<'t> {
let start = self.loc(); let start = self.loc();
Ok(Stmt { Ok(Stmt {
kind: self.stmtkind()?, kind: self.stmtkind()?,
semi: match self.peek_type(PARSING) { semi: match self.peek_kind(PARSING) {
Ok(Type::Semi) => { Ok(TokenKind::Semi) => {
self.consume_peeked(); self.consume_peeked();
Semi::Terminated Semi::Terminated
} }
@ -247,11 +247,11 @@ impl<'t> Parser<'t> {
impl<'t> Parser<'t> { impl<'t> Parser<'t> {
/// Parses an [attribute set](Attrs) /// Parses an [attribute set](Attrs)
pub fn attributes(&mut self) -> PResult<Attrs> { pub fn attributes(&mut self) -> PResult<Attrs> {
if self.match_type(Type::Hash, Parsing::Attrs).is_err() { if self.match_type(TokenKind::Hash, Parsing::Attrs).is_err() {
return Ok(Attrs { meta: vec![] }); return Ok(Attrs { meta: vec![] });
} }
let meta = delim( let meta = delim(
sep(Self::meta, Type::Comma, BRACKETS.1, Parsing::Attrs), sep(Self::meta, TokenKind::Comma, BRACKETS.1, Parsing::Attrs),
BRACKETS, BRACKETS,
Parsing::Attrs, Parsing::Attrs,
); );
@ -263,16 +263,16 @@ impl<'t> Parser<'t> {
pub fn meta_kind(&mut self) -> PResult<MetaKind> { pub fn meta_kind(&mut self) -> PResult<MetaKind> {
const PARSING: Parsing = Parsing::Meta; const PARSING: Parsing = Parsing::Meta;
let lit_tuple = delim( let lit_tuple = delim(
sep(Self::literal, Type::Comma, PARENS.1, PARSING), sep(Self::literal, TokenKind::Comma, PARENS.1, PARSING),
PARENS, PARENS,
PARSING, PARSING,
); );
Ok(match self.peek_type(PARSING) { Ok(match self.peek_kind(PARSING) {
Ok(Type::Eq) => { Ok(TokenKind::Eq) => {
self.consume_peeked(); self.consume_peeked();
MetaKind::Equals(self.literal()?) MetaKind::Equals(self.literal()?)
} }
Ok(Type::LParen) => MetaKind::Func(lit_tuple(self)?), Ok(TokenKind::LParen) => MetaKind::Func(lit_tuple(self)?),
_ => MetaKind::Plain, _ => MetaKind::Plain,
}) })
} }
@ -284,104 +284,104 @@ impl<'t> Parser<'t> {
/// ///
/// See also: [Parser::item] /// See also: [Parser::item]
pub fn itemkind(&mut self) -> PResult<ItemKind> { pub fn itemkind(&mut self) -> PResult<ItemKind> {
Ok(match self.peek_type(Parsing::Item)? { Ok(match self.peek_kind(Parsing::Item)? {
Type::Type => self.parse_alias()?.into(), TokenKind::Type => self.parse_alias()?.into(),
Type::Const => self.parse_const()?.into(), TokenKind::Const => self.parse_const()?.into(),
Type::Static => self.parse_static()?.into(), TokenKind::Static => self.parse_static()?.into(),
Type::Mod => self.parse_module()?.into(), TokenKind::Mod => self.parse_module()?.into(),
Type::Fn => self.parse_function()?.into(), TokenKind::Fn => self.parse_function()?.into(),
Type::Struct => self.parse_struct()?.into(), TokenKind::Struct => self.parse_struct()?.into(),
Type::Enum => self.parse_enum()?.into(), TokenKind::Enum => self.parse_enum()?.into(),
Type::Impl => self.parse_impl()?.into(), TokenKind::Impl => self.parse_impl()?.into(),
t => Err(self.error(Unexpected(t), Parsing::Item))?, t => Err(self.error(Unexpected(t), Parsing::Item))?,
}) })
} }
pub fn parse_alias(&mut self) -> PResult<Alias> { pub fn parse_alias(&mut self) -> PResult<Alias> {
const PARSING: Parsing = Parsing::Alias; const PARSING: Parsing = Parsing::Alias;
self.match_type(Type::Type, PARSING)?; self.match_type(TokenKind::Type, PARSING)?;
let out = Ok(Alias { let out = Ok(Alias {
to: self.identifier()?, to: self.identifier()?,
from: if self.match_type(Type::Eq, PARSING).is_ok() { from: if self.match_type(TokenKind::Eq, PARSING).is_ok() {
Some(self.ty()?.into()) Some(self.ty()?.into())
} else { } else {
None None
}, },
}); });
self.match_type(Type::Semi, PARSING)?; self.match_type(TokenKind::Semi, PARSING)?;
out out
} }
pub fn parse_const(&mut self) -> PResult<Const> { pub fn parse_const(&mut self) -> PResult<Const> {
const PARSING: Parsing = Parsing::Const; const PARSING: Parsing = Parsing::Const;
self.match_type(Type::Const, PARSING)?; self.match_type(TokenKind::Const, PARSING)?;
let out = Ok(Const { let out = Ok(Const {
name: self.identifier()?, name: self.identifier()?,
ty: { ty: {
self.match_type(Type::Colon, PARSING)?; self.match_type(TokenKind::Colon, PARSING)?;
self.ty()?.into() self.ty()?.into()
}, },
init: { init: {
self.match_type(Type::Eq, PARSING)?; self.match_type(TokenKind::Eq, PARSING)?;
self.expr()?.into() self.expr()?.into()
}, },
}); });
self.match_type(Type::Semi, PARSING)?; self.match_type(TokenKind::Semi, PARSING)?;
out out
} }
pub fn parse_static(&mut self) -> PResult<Static> { pub fn parse_static(&mut self) -> PResult<Static> {
const PARSING: Parsing = Parsing::Static; const PARSING: Parsing = Parsing::Static;
self.match_type(Type::Static, PARSING)?; self.match_type(TokenKind::Static, PARSING)?;
let out = Ok(Static { let out = Ok(Static {
mutable: self.mutability()?, mutable: self.mutability()?,
name: self.identifier()?, name: self.identifier()?,
ty: { ty: {
self.match_type(Type::Colon, PARSING)?; self.match_type(TokenKind::Colon, PARSING)?;
self.ty()?.into() self.ty()?.into()
}, },
init: { init: {
self.match_type(Type::Eq, PARSING)?; self.match_type(TokenKind::Eq, PARSING)?;
self.expr()?.into() self.expr()?.into()
}, },
}); });
self.match_type(Type::Semi, PARSING)?; self.match_type(TokenKind::Semi, PARSING)?;
out out
} }
pub fn parse_module(&mut self) -> PResult<Module> { pub fn parse_module(&mut self) -> PResult<Module> {
const PARSING: Parsing = Parsing::Module; const PARSING: Parsing = Parsing::Module;
self.match_type(Type::Mod, PARSING)?; self.match_type(TokenKind::Mod, PARSING)?;
Ok(Module { name: self.identifier()?, kind: self.modulekind()? }) Ok(Module { name: self.identifier()?, kind: self.modulekind()? })
} }
pub fn modulekind(&mut self) -> PResult<ModuleKind> { pub fn modulekind(&mut self) -> PResult<ModuleKind> {
const PARSING: Parsing = Parsing::ModuleKind; const PARSING: Parsing = Parsing::ModuleKind;
let inline = delim(Self::file, CURLIES, PARSING); let inline = delim(Self::file, CURLIES, PARSING);
match self.peek_type(PARSING)? { match self.peek_kind(PARSING)? {
Type::LCurly => Ok(ModuleKind::Inline(inline(self)?)), TokenKind::LCurly => Ok(ModuleKind::Inline(inline(self)?)),
Type::Semi => { TokenKind::Semi => {
self.consume_peeked(); self.consume_peeked();
Ok(ModuleKind::Outline) Ok(ModuleKind::Outline)
} }
got => Err(self.error(Expected { want: Type::Semi, got }, PARSING)), got => Err(self.error(Expected { want: TokenKind::Semi, got }, PARSING)),
} }
} }
pub fn parse_function(&mut self) -> PResult<Function> { pub fn parse_function(&mut self) -> PResult<Function> {
const PARSING: Parsing = Parsing::Function; const PARSING: Parsing = Parsing::Function;
self.match_type(Type::Fn, PARSING)?; self.match_type(TokenKind::Fn, PARSING)?;
Ok(Function { Ok(Function {
name: self.identifier()?, name: self.identifier()?,
args: self.parse_params()?, args: self.parse_params()?,
rety: match self.peek_type(PARSING)? { rety: match self.peek_kind(PARSING)? {
Type::LCurly | Type::Semi => None, TokenKind::LCurly | TokenKind::Semi => None,
Type::Arrow => { TokenKind::Arrow => {
self.consume_peeked(); self.consume_peeked();
Some(self.ty()?.into()) Some(self.ty()?.into())
} }
got => Err(self.error(Expected { want: Type::Arrow, got }, PARSING))?, got => Err(self.error(Expected { want: TokenKind::Arrow, got }, PARSING))?,
}, },
body: match self.peek_type(PARSING)? { body: match self.peek_kind(PARSING)? {
Type::LCurly => Some(self.block()?), TokenKind::LCurly => Some(self.block()?),
Type::Semi => { TokenKind::Semi => {
self.consume_peeked(); self.consume_peeked();
None None
} }
@ -392,7 +392,7 @@ impl<'t> Parser<'t> {
pub fn parse_params(&mut self) -> PResult<Vec<Param>> { pub fn parse_params(&mut self) -> PResult<Vec<Param>> {
const PARSING: Parsing = Parsing::Function; const PARSING: Parsing = Parsing::Function;
delim( delim(
sep(Self::parse_param, Type::Comma, PARENS.1, PARSING), sep(Self::parse_param, TokenKind::Comma, PARENS.1, PARSING),
PARENS, PARENS,
PARSING, PARSING,
)(self) )(self)
@ -402,24 +402,24 @@ impl<'t> Parser<'t> {
mutability: self.mutability()?, mutability: self.mutability()?,
name: self.identifier()?, name: self.identifier()?,
ty: { ty: {
self.match_type(Type::Colon, Parsing::Param)?; self.match_type(TokenKind::Colon, Parsing::Param)?;
self.ty()?.into() self.ty()?.into()
}, },
}) })
} }
pub fn parse_struct(&mut self) -> PResult<Struct> { pub fn parse_struct(&mut self) -> PResult<Struct> {
const PARSING: Parsing = Parsing::Struct; const PARSING: Parsing = Parsing::Struct;
self.match_type(Type::Struct, PARSING)?; self.match_type(TokenKind::Struct, PARSING)?;
Ok(Struct { Ok(Struct {
name: self.identifier()?, name: self.identifier()?,
kind: match self.peek_type(PARSING)? { kind: match self.peek_kind(PARSING)? {
Type::LParen => self.structkind_tuple()?, TokenKind::LParen => self.structkind_tuple()?,
Type::LCurly => self.structkind_struct()?, TokenKind::LCurly => self.structkind_struct()?,
Type::Semi => { TokenKind::Semi => {
self.consume_peeked(); self.consume_peeked();
StructKind::Empty StructKind::Empty
} }
got => Err(self.error(Expected { want: Type::Semi, got }, PARSING))?, got => Err(self.error(Expected { want: TokenKind::Semi, got }, PARSING))?,
}, },
}) })
} }
@ -427,7 +427,7 @@ impl<'t> Parser<'t> {
const PARSING: Parsing = Parsing::StructKind; const PARSING: Parsing = Parsing::StructKind;
Ok(StructKind::Tuple(delim( Ok(StructKind::Tuple(delim(
sep(Self::ty, Type::Comma, PARENS.1, PARSING), sep(Self::ty, TokenKind::Comma, PARENS.1, PARSING),
PARENS, PARENS,
PARSING, PARSING,
)(self)?)) )(self)?))
@ -435,7 +435,7 @@ impl<'t> Parser<'t> {
pub fn structkind_struct(&mut self) -> PResult<StructKind> { pub fn structkind_struct(&mut self) -> PResult<StructKind> {
const PARSING: Parsing = Parsing::StructKind; const PARSING: Parsing = Parsing::StructKind;
Ok(StructKind::Struct(delim( Ok(StructKind::Struct(delim(
sep(Self::struct_member, Type::Comma, CURLIES.1, PARSING), sep(Self::struct_member, TokenKind::Comma, CURLIES.1, PARSING),
CURLIES, CURLIES,
PARSING, PARSING,
)(self)?)) )(self)?))
@ -446,7 +446,7 @@ impl<'t> Parser<'t> {
vis: self.visibility()?, vis: self.visibility()?,
name: self.identifier()?, name: self.identifier()?,
ty: { ty: {
self.match_type(Type::Colon, PARSING)?; self.match_type(TokenKind::Colon, PARSING)?;
self.ty()? self.ty()?
}, },
}) })
@ -454,16 +454,21 @@ impl<'t> Parser<'t> {
pub fn parse_enum(&mut self) -> PResult<Enum> { pub fn parse_enum(&mut self) -> PResult<Enum> {
// Enum = "enum" Identifier '{' (Variant ',')* Variant? '}' ; // Enum = "enum" Identifier '{' (Variant ',')* Variant? '}' ;
const PARSING: Parsing = Parsing::Enum; const PARSING: Parsing = Parsing::Enum;
self.match_type(Type::Enum, PARSING)?; self.match_type(TokenKind::Enum, PARSING)?;
Ok(Enum { Ok(Enum {
name: self.identifier()?, name: self.identifier()?,
kind: match self.peek_type(PARSING)? { kind: match self.peek_kind(PARSING)? {
Type::LCurly => EnumKind::Variants(delim( TokenKind::LCurly => EnumKind::Variants(delim(
sep(Self::enum_variant, Type::Comma, Type::RCurly, PARSING), sep(
Self::enum_variant,
TokenKind::Comma,
TokenKind::RCurly,
PARSING,
),
CURLIES, CURLIES,
PARSING, PARSING,
)(self)?), )(self)?),
Type::Semi => { TokenKind::Semi => {
self.consume_peeked(); self.consume_peeked();
EnumKind::NoVariants EnumKind::NoVariants
} }
@ -476,27 +481,32 @@ impl<'t> Parser<'t> {
const PARSING: Parsing = Parsing::Variant; const PARSING: Parsing = Parsing::Variant;
Ok(Variant { Ok(Variant {
name: self.identifier()?, name: self.identifier()?,
kind: match self.peek_type(PARSING)? { kind: match self.peek_kind(PARSING)? {
Type::Eq => self.variantkind_clike()?, TokenKind::Eq => self.variantkind_clike()?,
Type::LCurly => self.variantkind_struct()?, TokenKind::LCurly => self.variantkind_struct()?,
Type::LParen => self.variantkind_tuple()?, TokenKind::LParen => self.variantkind_tuple()?,
_ => VariantKind::Plain, _ => VariantKind::Plain,
}, },
}) })
} }
pub fn variantkind_clike(&mut self) -> PResult<VariantKind> { pub fn variantkind_clike(&mut self) -> PResult<VariantKind> {
const PARSING: Parsing = Parsing::VariantKind; const PARSING: Parsing = Parsing::VariantKind;
self.match_type(Type::Eq, PARSING)?; self.match_type(TokenKind::Eq, PARSING)?;
let tok = self.match_type(Type::Integer, PARSING)?; let tok = self.match_type(TokenKind::Integer, PARSING)?;
Ok(VariantKind::CLike(match tok.data() { Ok(VariantKind::CLike(match tok.data() {
Data::Integer(i) => *i, TokenData::Integer(i) => *i,
_ => panic!("Expected token data for {tok:?} while parsing {PARSING}"), _ => panic!("Expected token data for {tok:?} while parsing {PARSING}"),
})) }))
} }
pub fn variantkind_struct(&mut self) -> PResult<VariantKind> { pub fn variantkind_struct(&mut self) -> PResult<VariantKind> {
const PARSING: Parsing = Parsing::VariantKind; const PARSING: Parsing = Parsing::VariantKind;
Ok(VariantKind::Struct(delim( Ok(VariantKind::Struct(delim(
sep(Self::struct_member, Type::Comma, Type::RCurly, PARSING), sep(
Self::struct_member,
TokenKind::Comma,
TokenKind::RCurly,
PARSING,
),
CURLIES, CURLIES,
PARSING, PARSING,
)(self)?)) )(self)?))
@ -504,7 +514,7 @@ impl<'t> Parser<'t> {
pub fn variantkind_tuple(&mut self) -> PResult<VariantKind> { pub fn variantkind_tuple(&mut self) -> PResult<VariantKind> {
const PARSING: Parsing = Parsing::VariantKind; const PARSING: Parsing = Parsing::VariantKind;
Ok(VariantKind::Tuple(delim( Ok(VariantKind::Tuple(delim(
sep(Self::ty, Type::Comma, Type::RParen, PARSING), sep(Self::ty, TokenKind::Comma, TokenKind::RParen, PARSING),
PARENS, PARENS,
PARSING, PARSING,
)(self)?)) )(self)?))
@ -512,19 +522,19 @@ impl<'t> Parser<'t> {
pub fn parse_impl(&mut self) -> PResult<Impl> { pub fn parse_impl(&mut self) -> PResult<Impl> {
const PARSING: Parsing = Parsing::Impl; const PARSING: Parsing = Parsing::Impl;
self.match_type(Type::Impl, PARSING)?; self.match_type(TokenKind::Impl, PARSING)?;
Err(self.error(Todo, PARSING)) Err(self.error(Todo, PARSING))
} }
pub fn visibility(&mut self) -> PResult<Visibility> { pub fn visibility(&mut self) -> PResult<Visibility> {
if let Type::Pub = self.peek_type(Parsing::Visibility)? { if let TokenKind::Pub = self.peek_kind(Parsing::Visibility)? {
self.consume_peeked(); self.consume_peeked();
return Ok(Visibility::Public); return Ok(Visibility::Public);
}; };
Ok(Visibility::Private) Ok(Visibility::Private)
} }
pub fn mutability(&mut self) -> PResult<Mutability> { pub fn mutability(&mut self) -> PResult<Mutability> {
if let Type::Mut = self.peek_type(Parsing::Mutability)? { if let TokenKind::Mut = self.peek_kind(Parsing::Mutability)? {
self.consume_peeked(); self.consume_peeked();
return Ok(Mutability::Mut); return Ok(Mutability::Mut);
}; };
@ -539,18 +549,18 @@ impl<'t> Parser<'t> {
/// See also: [Parser::ty] /// See also: [Parser::ty]
pub fn tykind(&mut self) -> PResult<TyKind> { pub fn tykind(&mut self) -> PResult<TyKind> {
const PARSING: Parsing = Parsing::TyKind; const PARSING: Parsing = Parsing::TyKind;
let out = match self.peek_type(PARSING)? { let out = match self.peek_kind(PARSING)? {
Type::Bang => { TokenKind::Bang => {
self.consume_peeked(); self.consume_peeked();
TyKind::Never TyKind::Never
} }
Type::SelfTy => { TokenKind::SelfTy => {
self.consume_peeked(); self.consume_peeked();
TyKind::SelfTy TyKind::SelfTy
} }
Type::Amp | Type::AmpAmp => self.tyref()?.into(), TokenKind::Amp | TokenKind::AmpAmp => self.tyref()?.into(),
Type::LParen => self.tytuple()?.into(), TokenKind::LParen => self.tytuple()?.into(),
Type::Fn => self.tyfn()?.into(), TokenKind::Fn => self.tyfn()?.into(),
path_like!() => self.path()?.into(), path_like!() => self.path()?.into(),
t => Err(self.error(Unexpected(t), PARSING))?, t => Err(self.error(Unexpected(t), PARSING))?,
}; };
@ -562,7 +572,7 @@ impl<'t> Parser<'t> {
const PARSING: Parsing = Parsing::TyTuple; const PARSING: Parsing = Parsing::TyTuple;
Ok(TyTuple { Ok(TyTuple {
types: delim( types: delim(
sep(Self::ty, Type::Comma, PARENS.1, PARSING), sep(Self::ty, TokenKind::Comma, PARENS.1, PARSING),
PARENS, PARENS,
PARSING, PARSING,
)(self)?, )(self)?,
@ -573,9 +583,9 @@ impl<'t> Parser<'t> {
const PARSING: Parsing = Parsing::TyRef; const PARSING: Parsing = Parsing::TyRef;
let mut count = 0; let mut count = 0;
loop { loop {
match self.peek_type(PARSING)? { match self.peek_kind(PARSING)? {
Type::Amp => count += 1, TokenKind::Amp => count += 1,
Type::AmpAmp => count += 2, TokenKind::AmpAmp => count += 2,
_ => break, _ => break,
} }
self.consume_peeked(); self.consume_peeked();
@ -585,12 +595,12 @@ impl<'t> Parser<'t> {
/// [TyFn] = `fn` [TyTuple] (-> [Ty])? /// [TyFn] = `fn` [TyTuple] (-> [Ty])?
pub fn tyfn(&mut self) -> PResult<TyFn> { pub fn tyfn(&mut self) -> PResult<TyFn> {
const PARSING: Parsing = Parsing::TyFn; const PARSING: Parsing = Parsing::TyFn;
self.match_type(Type::Fn, PARSING)?; self.match_type(TokenKind::Fn, PARSING)?;
Ok(TyFn { Ok(TyFn {
args: self.tytuple()?, args: self.tytuple()?,
rety: { rety: {
match self.peek_type(PARSING)? { match self.peek_kind(PARSING)? {
Type::Arrow => { TokenKind::Arrow => {
self.consume_peeked(); self.consume_peeked();
Some(self.ty()?.into()) Some(self.ty()?.into())
} }
@ -601,33 +611,33 @@ impl<'t> Parser<'t> {
} }
} }
/// Expands to a pattern which matches literal-like [Type]s /// Expands to a pattern which matches literal-like [TokenKind]s
macro literal_like() { macro literal_like() {
Type::True | Type::False | Type::String | Type::Character | Type::Integer | Type::Float TokenKind::True | TokenKind::False | TokenKind::String | TokenKind::Character | TokenKind::Integer | TokenKind::Float
} }
/// Expands to a pattern which matches path-like [token Types](Type) /// Expands to a pattern which matches path-like [token Types](Type)
macro path_like() { macro path_like() {
Type::Super | Type::SelfKw | Type::Identifier | Type::ColonColon TokenKind::Super | TokenKind::SelfKw | TokenKind::Identifier | TokenKind::ColonColon
} }
/// # Path parsing /// # Path parsing
impl<'t> Parser<'t> { impl<'t> Parser<'t> {
/// [PathPart] = `super` | `self` | [Identifier] /// [PathPart] = `super` | `self` | [Identifier]
pub fn path_part(&mut self) -> PResult<PathPart> { pub fn path_part(&mut self) -> PResult<PathPart> {
const PARSING: Parsing = Parsing::PathPart; const PARSING: Parsing = Parsing::PathPart;
let out = match self.peek_type(PARSING)? { let out = match self.peek_kind(PARSING)? {
Type::Super => PathPart::SuperKw, TokenKind::Super => PathPart::SuperKw,
Type::SelfKw => PathPart::SelfKw, TokenKind::SelfKw => PathPart::SelfKw,
Type::Identifier => PathPart::Ident(self.identifier()?), TokenKind::Identifier => PathPart::Ident(self.identifier()?),
t => return Err(self.error(Unexpected(t), PARSING)), t => return Err(self.error(Unexpected(t), PARSING)),
}; };
self.consume_peeked(); self.consume_peeked();
Ok(out) Ok(out)
} }
/// [Identifier] = [`Identifier`](Type::Identifier) /// [Identifier] = [`Identifier`](TokenKind::Identifier)
pub fn identifier(&mut self) -> PResult<Identifier> { pub fn identifier(&mut self) -> PResult<Identifier> {
let tok = self.match_type(Type::Identifier, Parsing::Identifier)?; let tok = self.match_type(TokenKind::Identifier, Parsing::Identifier)?;
match tok.data() { match tok.data() {
Data::Identifier(ident) => Ok(ident.into()), TokenData::Identifier(ident) => Ok(ident.into()),
_ => panic!("Expected token data for {tok:?}"), _ => panic!("Expected token data for {tok:?}"),
} }
} }
@ -639,26 +649,26 @@ impl<'t> Parser<'t> {
/// ///
/// See also: [Parser::stmt] /// See also: [Parser::stmt]
pub fn stmtkind(&mut self) -> PResult<StmtKind> { pub fn stmtkind(&mut self) -> PResult<StmtKind> {
Ok(match self.peek_type(Parsing::StmtKind)? { Ok(match self.peek_kind(Parsing::StmtKind)? {
Type::Semi => StmtKind::Empty, TokenKind::Semi => StmtKind::Empty,
Type::Let => self.parse_let()?.into(), TokenKind::Let => self.parse_let()?.into(),
item_like!() => self.item()?.into(), item_like!() => self.item()?.into(),
_ => self.expr()?.into(), _ => self.expr()?.into(),
}) })
} }
pub fn parse_let(&mut self) -> PResult<Let> { pub fn parse_let(&mut self) -> PResult<Let> {
self.match_type(Type::Let, Parsing::Let)?; self.match_type(TokenKind::Let, Parsing::Let)?;
Ok(Let { Ok(Let {
mutable: self.mutability()?, mutable: self.mutability()?,
name: self.identifier()?, name: self.identifier()?,
ty: if Ok(Type::Colon) == self.peek_type(Parsing::Let) { ty: if Ok(TokenKind::Colon) == self.peek_kind(Parsing::Let) {
self.consume_peeked(); self.consume_peeked();
Some(self.ty()?.into()) Some(self.ty()?.into())
} else { } else {
None None
}, },
init: if Ok(Type::Eq) == self.peek_type(Parsing::Let) { init: if Ok(TokenKind::Eq) == self.peek_kind(Parsing::Let) {
self.consume_peeked(); self.consume_peeked();
Some(self.expr()?.into()) Some(self.expr()?.into())
} else { } else {
@ -772,10 +782,10 @@ impl<'t> Parser<'t> {
const PARSING: Parsing = Parsing::Call; const PARSING: Parsing = Parsing::Call;
let callee = self.expr_from(Self::exprkind_index)?; let callee = self.expr_from(Self::exprkind_index)?;
let mut args = vec![]; let mut args = vec![];
while Ok(Type::LParen) == self.peek_type(PARSING) { while Ok(TokenKind::LParen) == self.peek_kind(PARSING) {
self.consume_peeked(); self.consume_peeked();
args.push(self.tuple()?); args.push(self.tuple()?);
self.match_type(Type::RParen, PARSING)?; self.match_type(TokenKind::RParen, PARSING)?;
} }
if args.is_empty() { if args.is_empty() {
Ok(callee.kind) Ok(callee.kind)
@ -787,31 +797,31 @@ impl<'t> Parser<'t> {
pub fn exprkind_index(&mut self) -> PResult<ExprKind> { pub fn exprkind_index(&mut self) -> PResult<ExprKind> {
const PARSING: Parsing = Parsing::Index; const PARSING: Parsing = Parsing::Index;
let head = self.expr_from(Self::exprkind_primary)?; let head = self.expr_from(Self::exprkind_primary)?;
if Ok(Type::LBrack) != self.peek_type(PARSING) { if Ok(TokenKind::LBrack) != self.peek_kind(PARSING) {
return Ok(head.kind); return Ok(head.kind);
} }
let mut indices = vec![]; let mut indices = vec![];
while Ok(Type::LBrack) == self.peek_type(PARSING) { while Ok(TokenKind::LBrack) == self.peek_kind(PARSING) {
indices.push(delim(Self::tuple, BRACKETS, PARSING)(self)?.into()); indices.push(delim(Self::tuple, BRACKETS, PARSING)(self)?.into());
} }
Ok(Index { head: head.into(), indices }.into()) Ok(Index { head: head.into(), indices }.into())
} }
/// Delegates to the set of highest-priority rules based on unambiguous pattern matching /// Delegates to the set of highest-priority rules based on unambiguous pattern matching
pub fn exprkind_primary(&mut self) -> PResult<ExprKind> { pub fn exprkind_primary(&mut self) -> PResult<ExprKind> {
match self.peek_type(Parsing::Expr)? { match self.peek_kind(Parsing::Expr)? {
Type::Amp | Type::AmpAmp => self.exprkind_addrof(), TokenKind::Amp | TokenKind::AmpAmp => self.exprkind_addrof(),
Type::LCurly => self.exprkind_block(), TokenKind::LCurly => self.exprkind_block(),
Type::LBrack => self.exprkind_array(), TokenKind::LBrack => self.exprkind_array(),
Type::LParen => self.exprkind_empty_group_or_tuple(), TokenKind::LParen => self.exprkind_empty_group_or_tuple(),
literal_like!() => Ok(self.literal()?.into()), literal_like!() => Ok(self.literal()?.into()),
path_like!() => Ok(self.path()?.into()), path_like!() => Ok(self.path()?.into()),
Type::If => Ok(self.parse_if()?.into()), TokenKind::If => Ok(self.parse_if()?.into()),
Type::For => Ok(self.parse_for()?.into()), TokenKind::For => Ok(self.parse_for()?.into()),
Type::While => Ok(self.parse_while()?.into()), TokenKind::While => Ok(self.parse_while()?.into()),
Type::Break => Ok(self.parse_break()?.into()), TokenKind::Break => Ok(self.parse_break()?.into()),
Type::Return => Ok(self.parse_return()?.into()), TokenKind::Return => Ok(self.parse_return()?.into()),
Type::Continue => Ok(self.parse_continue()?.into()), TokenKind::Continue => Ok(self.parse_continue()?.into()),
_ => Err(self.error(Nothing, Parsing::Expr)), _ => Err(self.error(Nothing, Parsing::Expr)),
} }
} }
@ -821,10 +831,10 @@ impl<'t> Parser<'t> {
/// so they can't be independent subexpressions /// so they can't be independent subexpressions
pub fn exprkind_array(&mut self) -> PResult<ExprKind> { pub fn exprkind_array(&mut self) -> PResult<ExprKind> {
const PARSING: Parsing = Parsing::Array; const PARSING: Parsing = Parsing::Array;
const START: Type = Type::LBrack; const START: TokenKind = TokenKind::LBrack;
const END: Type = Type::RBrack; const END: TokenKind = TokenKind::RBrack;
self.match_type(START, PARSING)?; self.match_type(START, PARSING)?;
match self.peek_type(PARSING)? { match self.peek_kind(PARSING)? {
END => { END => {
self.consume_peeked(); self.consume_peeked();
Ok(Array { values: vec![] }.into()) Ok(Array { values: vec![] }.into())
@ -835,10 +845,10 @@ impl<'t> Parser<'t> {
/// [ArrayRep] = `[` [Expr] `;` [Expr] `]` /// [ArrayRep] = `[` [Expr] `;` [Expr] `]`
pub fn exprkind_array_rep(&mut self) -> PResult<ExprKind> { pub fn exprkind_array_rep(&mut self) -> PResult<ExprKind> {
const PARSING: Parsing = Parsing::Array; const PARSING: Parsing = Parsing::Array;
const END: Type = Type::RBrack; const END: TokenKind = TokenKind::RBrack;
let first = self.expr()?; let first = self.expr()?;
let out: ExprKind = match self.peek_type(PARSING)? { let out: ExprKind = match self.peek_kind(PARSING)? {
Type::Semi => ArrayRep { TokenKind::Semi => ArrayRep {
value: first.into(), value: first.into(),
repeat: { repeat: {
self.consume_peeked(); self.consume_peeked();
@ -846,12 +856,12 @@ impl<'t> Parser<'t> {
}, },
} }
.into(), .into(),
Type::RBrack => Array { values: vec![first] }.into(), TokenKind::RBrack => Array { values: vec![first] }.into(),
Type::Comma => Array { TokenKind::Comma => Array {
values: { values: {
self.consume_peeked(); self.consume_peeked();
let mut out = vec![first]; let mut out = vec![first];
out.extend(sep(Self::expr, Type::Comma, END, PARSING)(self)?); out.extend(sep(Self::expr, TokenKind::Comma, END, PARSING)(self)?);
out out
}, },
} }
@ -867,9 +877,9 @@ impl<'t> Parser<'t> {
const PARSING: Parsing = Parsing::AddrOf; const PARSING: Parsing = Parsing::AddrOf;
let mut count = 0; let mut count = 0;
loop { loop {
match self.peek_type(PARSING)? { match self.peek_kind(PARSING)? {
Type::Amp => count += 1, TokenKind::Amp => count += 1,
Type::AmpAmp => count += 2, TokenKind::AmpAmp => count += 2,
_ => break, _ => break,
} }
self.consume_peeked(); self.consume_peeked();
@ -884,13 +894,13 @@ impl<'t> Parser<'t> {
/// ///
/// [ExprKind::Empty] and [Group] are special cases of [Tuple] /// [ExprKind::Empty] and [Group] are special cases of [Tuple]
pub fn exprkind_empty_group_or_tuple(&mut self) -> PResult<ExprKind> { pub fn exprkind_empty_group_or_tuple(&mut self) -> PResult<ExprKind> {
self.match_type(Type::LParen, Parsing::Group)?; self.match_type(TokenKind::LParen, Parsing::Group)?;
let out = match self.peek_type(Parsing::Group)? { let out = match self.peek_kind(Parsing::Group)? {
Type::RParen => Ok(ExprKind::Empty), TokenKind::RParen => Ok(ExprKind::Empty),
_ => self.exprkind_group(), _ => self.exprkind_group(),
}; };
match self.peek_type(Parsing::Group) { match self.peek_kind(Parsing::Group) {
Ok(Type::RParen) => self.consume_peeked(), Ok(TokenKind::RParen) => self.consume_peeked(),
_ => Err(self.error(UnmatchedParentheses, Parsing::Group))?, _ => Err(self.error(UnmatchedParentheses, Parsing::Group))?,
}; };
out out
@ -898,14 +908,14 @@ impl<'t> Parser<'t> {
/// [Group] = `(`([Empty](ExprKind::Empty)|[Expr]|[Tuple])`)` /// [Group] = `(`([Empty](ExprKind::Empty)|[Expr]|[Tuple])`)`
pub fn exprkind_group(&mut self) -> PResult<ExprKind> { pub fn exprkind_group(&mut self) -> PResult<ExprKind> {
let first = self.expr()?; let first = self.expr()?;
match self.peek_type(Parsing::Group)? { match self.peek_kind(Parsing::Group)? {
Type::Comma => { TokenKind::Comma => {
let mut exprs = vec![first]; let mut exprs = vec![first];
self.consume_peeked(); self.consume_peeked();
while Type::RParen != self.peek_type(Parsing::Tuple)? { while TokenKind::RParen != self.peek_kind(Parsing::Tuple)? {
exprs.push(self.expr()?); exprs.push(self.expr()?);
match self.peek_type(Parsing::Tuple)? { match self.peek_kind(Parsing::Tuple)? {
Type::Comma => self.consume_peeked(), TokenKind::Comma => self.consume_peeked(),
_ => break, _ => break,
}; };
} }
@ -918,22 +928,22 @@ impl<'t> Parser<'t> {
/// ## Subexpressions /// ## Subexpressions
impl<'t> Parser<'t> { impl<'t> Parser<'t> {
/// [Literal] = [String](Type::String) | [Character](Type::Character) /// [Literal] = [String](TokenKind::String) | [Character](TokenKind::Character)
/// | [Float](Type::Float) (TODO) | [Integer](Type::Integer) | `true` | `false` /// | [Float](TokenKind::Float) (TODO) | [Integer](TokenKind::Integer) | `true` | `false`
pub fn literal(&mut self) -> PResult<Literal> { pub fn literal(&mut self) -> PResult<Literal> {
let tok = self.consume(Parsing::Literal)?; let tok = self.consume(Parsing::Literal)?;
// keyword literals true and false // keyword literals true and false
match tok.ty() { match tok.ty() {
Type::True => return Ok(Literal::Bool(true)), TokenKind::True => return Ok(Literal::Bool(true)),
Type::False => return Ok(Literal::Bool(false)), TokenKind::False => return Ok(Literal::Bool(false)),
Type::String | Type::Character | Type::Integer | Type::Float => (), TokenKind::String | TokenKind::Character | TokenKind::Integer | TokenKind::Float => (),
t => return Err(self.error(Unexpected(t), Parsing::Literal)), t => return Err(self.error(Unexpected(t), Parsing::Literal)),
} }
Ok(match tok.data() { Ok(match tok.data() {
Data::String(v) => Literal::from(v.as_str()), TokenData::String(v) => Literal::from(v.as_str()),
Data::Character(v) => Literal::from(*v), TokenData::Character(v) => Literal::from(*v),
Data::Integer(v) => Literal::from(*v), TokenData::Integer(v) => Literal::from(*v),
Data::Float(v) => todo!("Literal::Float({v})"), TokenData::Float(v) => todo!("Literal::Float({v})"),
_ => panic!("Expected token data for {tok:?}"), _ => panic!("Expected token data for {tok:?}"),
}) })
} }
@ -946,8 +956,8 @@ impl<'t> Parser<'t> {
Err(e) => return Err(e), Err(e) => return Err(e),
} { } {
exprs.push(expr); exprs.push(expr);
match self.peek_type(Parsing::Tuple)? { match self.peek_kind(Parsing::Tuple)? {
Type::Comma => self.consume_peeked(), TokenKind::Comma => self.consume_peeked(),
_ => break, _ => break,
}; };
} }
@ -963,22 +973,22 @@ impl<'t> Parser<'t> {
impl<'t> Parser<'t> { impl<'t> Parser<'t> {
/// [Break] = `break` [Expr]? /// [Break] = `break` [Expr]?
pub fn parse_break(&mut self) -> PResult<Break> { pub fn parse_break(&mut self) -> PResult<Break> {
self.match_type(Type::Break, Parsing::Break)?; self.match_type(TokenKind::Break, Parsing::Break)?;
Ok(Break { body: self.optional_expr()?.map(Into::into) }) Ok(Break { body: self.optional_expr()?.map(Into::into) })
} }
/// [Return] = `return` [Expr]? /// [Return] = `return` [Expr]?
pub fn parse_return(&mut self) -> PResult<Return> { pub fn parse_return(&mut self) -> PResult<Return> {
self.match_type(Type::Return, Parsing::Return)?; self.match_type(TokenKind::Return, Parsing::Return)?;
Ok(Return { body: self.optional_expr()?.map(Into::into) }) Ok(Return { body: self.optional_expr()?.map(Into::into) })
} }
/// [Continue] = `continue` /// [Continue] = `continue`
pub fn parse_continue(&mut self) -> PResult<Continue> { pub fn parse_continue(&mut self) -> PResult<Continue> {
self.match_type(Type::Continue, Parsing::Continue)?; self.match_type(TokenKind::Continue, Parsing::Continue)?;
Ok(Continue) Ok(Continue)
} }
/// [While] = `while` [Expr] [Block] [Else]? /// [While] = `while` [Expr] [Block] [Else]?
pub fn parse_while(&mut self) -> PResult<While> { pub fn parse_while(&mut self) -> PResult<While> {
self.match_type(Type::While, Parsing::While)?; self.match_type(TokenKind::While, Parsing::While)?;
Ok(While { Ok(While {
cond: self.expr()?.into(), cond: self.expr()?.into(),
pass: self.block()?.into(), pass: self.block()?.into(),
@ -988,7 +998,7 @@ impl<'t> Parser<'t> {
/// [If] = <code>`if` [Expr] [Block] [Else]?</code> /// [If] = <code>`if` [Expr] [Block] [Else]?</code>
#[rustfmt::skip] // second line is barely not long enough #[rustfmt::skip] // second line is barely not long enough
pub fn parse_if(&mut self) -> PResult<If> { pub fn parse_if(&mut self) -> PResult<If> {
self.match_type(Type::If, Parsing::If)?; self.match_type(TokenKind::If, Parsing::If)?;
Ok(If { Ok(If {
cond: self.expr()?.into(), cond: self.expr()?.into(),
pass: self.block()?.into(), pass: self.block()?.into(),
@ -997,9 +1007,9 @@ impl<'t> Parser<'t> {
} }
/// [For]: `for` Pattern (TODO) `in` [Expr] [Block] [Else]? /// [For]: `for` Pattern (TODO) `in` [Expr] [Block] [Else]?
pub fn parse_for(&mut self) -> PResult<For> { pub fn parse_for(&mut self) -> PResult<For> {
self.match_type(Type::For, Parsing::For)?; self.match_type(TokenKind::For, Parsing::For)?;
let bind = self.identifier()?; let bind = self.identifier()?;
self.match_type(Type::In, Parsing::For)?; self.match_type(TokenKind::In, Parsing::For)?;
Ok(For { Ok(For {
bind, bind,
cond: self.expr()?.into(), cond: self.expr()?.into(),
@ -1009,8 +1019,8 @@ impl<'t> Parser<'t> {
} }
/// [Else]: (`else` [Block])? /// [Else]: (`else` [Block])?
pub fn parse_else(&mut self) -> PResult<Else> { pub fn parse_else(&mut self) -> PResult<Else> {
match self.peek_type(Parsing::Else) { match self.peek_kind(Parsing::Else) {
Ok(Type::Else) => { Ok(TokenKind::Else) => {
self.consume_peeked(); self.consume_peeked();
Ok(self.expr()?.into()) Ok(self.expr()?.into())
} }
@ -1023,8 +1033,8 @@ impl<'t> Parser<'t> {
macro operator($($name:ident ($returns:ident) {$($t:ident => $p:ident),*$(,)?};)*) {$( macro operator($($name:ident ($returns:ident) {$($t:ident => $p:ident),*$(,)?};)*) {$(
pub fn $name (&mut self) -> PResult<$returns> { pub fn $name (&mut self) -> PResult<$returns> {
const PARSING: Parsing = Parsing::$returns; const PARSING: Parsing = Parsing::$returns;
let out = Ok(match self.peek_type(PARSING) { let out = Ok(match self.peek_kind(PARSING) {
$(Ok(Type::$t) => $returns::$p,)* $(Ok(TokenKind::$t) => $returns::$p,)*
Err(e) => Err(e)?, Err(e) => Err(e)?,
Ok(t) => Err(self.error(Unexpected(t), PARSING))?, Ok(t) => Err(self.error(Unexpected(t), PARSING))?,
}); });
@ -1095,7 +1105,7 @@ impl<'t> Parser<'t> {
pub fn member_op(&mut self) -> PResult<()> { pub fn member_op(&mut self) -> PResult<()> {
const PARSING: Parsing = Parsing::Member; const PARSING: Parsing = Parsing::Member;
match self.peek(PARSING)?.ty() { match self.peek(PARSING)?.ty() {
Type::Dot => {} TokenKind::Dot => {}
t => Err(self.error(Unexpected(t), PARSING))?, t => Err(self.error(Unexpected(t), PARSING))?,
} }
self.consume_peeked(); self.consume_peeked();

6
cl-token/src/lib.rs
View File

@ -1,6 +1,6 @@
//! # Token //! # Token
//! //!
//! Stores a component of a file as a [Type], some [Data], and a line and column number //! Stores a component of a file as a [TokenKind], some [TokenData], and a line and column number
#![warn(clippy::all)] #![warn(clippy::all)]
#![feature(decl_macro)] #![feature(decl_macro)]
@ -9,5 +9,5 @@ pub mod token_data;
pub mod token_type; pub mod token_type;
pub use token::Token; pub use token::Token;
pub use token_data::Data; pub use token_data::TokenData;
pub use token_type::Type; pub use token_type::TokenKind;

30
cl-token/src/token.rs
View File

@ -1,34 +1,34 @@
//! A [Token] contains a single unit of lexical information, and an optional bit of [Data] //! A [Token] contains a single unit of lexical information, and an optional bit of [TokenData]
use super::{Data, Type}; use super::{TokenData, TokenKind};
/// Contains a single unit of lexical information, /// Contains a single unit of lexical information,
/// and an optional bit of [Data] /// and an optional bit of [TokenData]
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct Token { pub struct Token {
ty: Type, ty: TokenKind,
data: Data, data: TokenData,
line: u32, line: u32,
col: u32, col: u32,
} }
impl Token { impl Token {
/// Creates a new [Token] out of a [Type], [Data], line, and column. /// Creates a new [Token] out of a [TokenKind], [TokenData], line, and column.
pub fn new(ty: Type, data: impl Into<Data>, line: u32, col: u32) -> Self { pub fn new(ty: TokenKind, data: impl Into<TokenData>, line: u32, col: u32) -> Self {
Self { ty, data: data.into(), line, col } Self { ty, data: data.into(), line, col }
} }
/// Casts this token to a new [Type] /// Casts this token to a new [TokenKind]
pub fn cast(self, ty: Type) -> Self { pub fn cast(self, ty: TokenKind) -> Self {
Self { ty, ..self } Self { ty, ..self }
} }
/// Returns the [Type] of this token /// Returns the [TokenKind] of this token
pub fn ty(&self) -> Type { pub fn ty(&self) -> TokenKind {
self.ty self.ty
} }
/// Returns a reference to this token's [Data] /// Returns a reference to this token's [TokenData]
pub fn data(&self) -> &Data { pub fn data(&self) -> &TokenData {
&self.data &self.data
} }
/// Converts this token into its inner [Data] /// Converts this token into its inner [TokenData]
pub fn into_data(self) -> Data { pub fn into_data(self) -> TokenData {
self.data self.data
} }
/// Returns the line where this token originated /// Returns the line where this token originated

22
cl-token/src/token_data.rs
View File

@ -1,9 +1,9 @@
//! Additional data stored within a [Token](super::Token), //! Additional data stored within a [Token](super::Token),
//! external to its [Type](super::token_type::Type) //! external to its [TokenKind](super::token_type::TokenKind)
/// Additional data stored within a [Token](super::Token), /// Additional data stored within a [Token](super::Token),
/// external to its [Type](super::token_type::Type) /// external to its [TokenKind](super::token_type::TokenKind)
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub enum Data { pub enum TokenData {
/// [Token](super::Token) contains an [identifier](str) /// [Token](super::Token) contains an [identifier](str)
Identifier(Box<str>), Identifier(Box<str>),
/// [Token](super::Token) contains a [String] /// [Token](super::Token) contains a [String]
@ -27,19 +27,19 @@ from! {
} }
/// Implements [From] for an enum /// Implements [From] for an enum
macro from($($value:ident: $src:ty => $dst:expr),*$(,)?) { macro from($($value:ident: $src:ty => $dst:expr),*$(,)?) {
$(impl From<$src> for Data { $(impl From<$src> for TokenData {
fn from($value: $src) -> Self { $dst } fn from($value: $src) -> Self { $dst }
})* })*
} }
impl std::fmt::Display for Data { impl std::fmt::Display for TokenData {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
Data::Identifier(v) => v.fmt(f), TokenData::Identifier(v) => v.fmt(f),
Data::String(v) => write!(f, "\"{v}\""), TokenData::String(v) => write!(f, "\"{v}\""),
Data::Character(v) => write!(f, "'{v}'"), TokenData::Character(v) => write!(f, "'{v}'"),
Data::Integer(v) => v.fmt(f), TokenData::Integer(v) => v.fmt(f),
Data::Float(v) => v.fmt(f), TokenData::Float(v) => v.fmt(f),
Data::None => "None".fmt(f), TokenData::None => "None".fmt(f),
} }
} }
} }

178
cl-token/src/token_type.rs
View File

@ -3,7 +3,7 @@ use std::{fmt::Display, str::FromStr};
/// Stores a [Token's](super::Token) lexical information /// Stores a [Token's](super::Token) lexical information
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Type { pub enum TokenKind {
// Invalid syntax // Invalid syntax
Invalid, Invalid,
// Any kind of comment // Any kind of comment
@ -98,102 +98,102 @@ pub enum Type {
XorXor, // ^^ XorXor, // ^^
} }
impl Display for Type { impl Display for TokenKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
Type::Invalid => "invalid".fmt(f), TokenKind::Invalid => "invalid".fmt(f),
Type::Comment => "comment".fmt(f), TokenKind::Comment => "comment".fmt(f),
Type::Identifier => "identifier".fmt(f), TokenKind::Identifier => "identifier".fmt(f),
Type::Break => "break".fmt(f), TokenKind::Break => "break".fmt(f),
Type::Cl => "cl".fmt(f), TokenKind::Cl => "cl".fmt(f),
Type::Const => "const".fmt(f), TokenKind::Const => "const".fmt(f),
Type::Continue => "continue".fmt(f), TokenKind::Continue => "continue".fmt(f),
Type::Else => "else".fmt(f), TokenKind::Else => "else".fmt(f),
Type::Enum => "enum".fmt(f), TokenKind::Enum => "enum".fmt(f),
Type::False => "false".fmt(f), TokenKind::False => "false".fmt(f),
Type::For => "for".fmt(f), TokenKind::For => "for".fmt(f),
Type::Fn => "fn".fmt(f), TokenKind::Fn => "fn".fmt(f),
Type::If => "if".fmt(f), TokenKind::If => "if".fmt(f),
Type::Impl => "impl".fmt(f), TokenKind::Impl => "impl".fmt(f),
Type::In => "in".fmt(f), TokenKind::In => "in".fmt(f),
Type::Let => "let".fmt(f), TokenKind::Let => "let".fmt(f),
Type::Mod => "mod".fmt(f), TokenKind::Mod => "mod".fmt(f),
Type::Mut => "mut".fmt(f), TokenKind::Mut => "mut".fmt(f),
Type::Pub => "pub".fmt(f), TokenKind::Pub => "pub".fmt(f),
Type::Return => "return".fmt(f), TokenKind::Return => "return".fmt(f),
Type::SelfKw => "self".fmt(f), TokenKind::SelfKw => "self".fmt(f),
Type::SelfTy => "Self".fmt(f), TokenKind::SelfTy => "Self".fmt(f),
Type::Static => "static".fmt(f), TokenKind::Static => "static".fmt(f),
Type::Struct => "struct".fmt(f), TokenKind::Struct => "struct".fmt(f),
Type::Super => "super".fmt(f), TokenKind::Super => "super".fmt(f),
Type::True => "true".fmt(f), TokenKind::True => "true".fmt(f),
Type::Type => "type".fmt(f), TokenKind::Type => "type".fmt(f),
Type::While => "while".fmt(f), TokenKind::While => "while".fmt(f),
Type::Integer => "integer literal".fmt(f), TokenKind::Integer => "integer literal".fmt(f),
Type::Float => "float literal".fmt(f), TokenKind::Float => "float literal".fmt(f),
Type::String => "string literal".fmt(f), TokenKind::String => "string literal".fmt(f),
Type::Character => "char literal".fmt(f), TokenKind::Character => "char literal".fmt(f),
Type::LCurly => "left curly".fmt(f), TokenKind::LCurly => "left curly".fmt(f),
Type::RCurly => "right curly".fmt(f), TokenKind::RCurly => "right curly".fmt(f),
Type::LBrack => "left brack".fmt(f), TokenKind::LBrack => "left brack".fmt(f),
Type::RBrack => "right brack".fmt(f), TokenKind::RBrack => "right brack".fmt(f),
Type::LParen => "left paren".fmt(f), TokenKind::LParen => "left paren".fmt(f),
Type::RParen => "right paren".fmt(f), TokenKind::RParen => "right paren".fmt(f),
Type::Amp => "and".fmt(f), TokenKind::Amp => "and".fmt(f),
Type::AmpAmp => "and-and".fmt(f), TokenKind::AmpAmp => "and-and".fmt(f),
Type::AmpEq => "and-assign".fmt(f), TokenKind::AmpEq => "and-assign".fmt(f),
Type::Arrow => "arrow".fmt(f), TokenKind::Arrow => "arrow".fmt(f),
Type::At => "at".fmt(f), TokenKind::At => "at".fmt(f),
Type::Backslash => "backslash".fmt(f), TokenKind::Backslash => "backslash".fmt(f),
Type::Bang => "bang".fmt(f), TokenKind::Bang => "bang".fmt(f),
Type::BangBang => "not-not".fmt(f), TokenKind::BangBang => "not-not".fmt(f),
Type::BangEq => "not equal to".fmt(f), TokenKind::BangEq => "not equal to".fmt(f),
Type::Bar => "or".fmt(f), TokenKind::Bar => "or".fmt(f),
Type::BarBar => "or-or".fmt(f), TokenKind::BarBar => "or-or".fmt(f),
Type::BarEq => "or-assign".fmt(f), TokenKind::BarEq => "or-assign".fmt(f),
Type::Colon => "colon".fmt(f), TokenKind::Colon => "colon".fmt(f),
Type::ColonColon => "path separator".fmt(f), TokenKind::ColonColon => "path separator".fmt(f),
Type::Comma => "comma".fmt(f), TokenKind::Comma => "comma".fmt(f),
Type::Dot => "dot".fmt(f), TokenKind::Dot => "dot".fmt(f),
Type::DotDot => "exclusive range".fmt(f), TokenKind::DotDot => "exclusive range".fmt(f),
Type::DotDotEq => "inclusive range".fmt(f), TokenKind::DotDotEq => "inclusive range".fmt(f),
Type::Eq => "assign".fmt(f), TokenKind::Eq => "assign".fmt(f),
Type::EqEq => "equal to".fmt(f), TokenKind::EqEq => "equal to".fmt(f),
Type::FatArrow => "fat arrow".fmt(f), TokenKind::FatArrow => "fat arrow".fmt(f),
Type::Grave => "grave".fmt(f), TokenKind::Grave => "grave".fmt(f),
Type::Gt => "greater than".fmt(f), TokenKind::Gt => "greater than".fmt(f),
Type::GtEq => "greater than or equal to".fmt(f), TokenKind::GtEq => "greater than or equal to".fmt(f),
Type::GtGt => "shift right".fmt(f), TokenKind::GtGt => "shift right".fmt(f),
Type::GtGtEq => "shift right-assign".fmt(f), TokenKind::GtGtEq => "shift right-assign".fmt(f),
Type::Hash => "hash".fmt(f), TokenKind::Hash => "hash".fmt(f),
Type::HashBang => "shebang".fmt(f), TokenKind::HashBang => "shebang".fmt(f),
Type::Lt => "less than".fmt(f), TokenKind::Lt => "less than".fmt(f),
Type::LtEq => "less than or equal to".fmt(f), TokenKind::LtEq => "less than or equal to".fmt(f),
Type::LtLt => "shift left".fmt(f), TokenKind::LtLt => "shift left".fmt(f),
Type::LtLtEq => "shift left-assign".fmt(f), TokenKind::LtLtEq => "shift left-assign".fmt(f),
Type::Minus => "sub".fmt(f), TokenKind::Minus => "sub".fmt(f),
Type::MinusEq => "sub-assign".fmt(f), TokenKind::MinusEq => "sub-assign".fmt(f),
Type::Plus => "add".fmt(f), TokenKind::Plus => "add".fmt(f),
Type::PlusEq => "add-assign".fmt(f), TokenKind::PlusEq => "add-assign".fmt(f),
Type::Question => "huh?".fmt(f), TokenKind::Question => "huh?".fmt(f),
Type::Rem => "rem".fmt(f), TokenKind::Rem => "rem".fmt(f),
Type::RemEq => "rem-assign".fmt(f), TokenKind::RemEq => "rem-assign".fmt(f),
Type::Semi => "ignore".fmt(f), TokenKind::Semi => "ignore".fmt(f),
Type::Slash => "div".fmt(f), TokenKind::Slash => "div".fmt(f),
Type::SlashEq => "div-assign".fmt(f), TokenKind::SlashEq => "div-assign".fmt(f),
Type::Star => "star".fmt(f), TokenKind::Star => "star".fmt(f),
Type::StarEq => "star-assign".fmt(f), TokenKind::StarEq => "star-assign".fmt(f),
Type::Tilde => "tilde".fmt(f), TokenKind::Tilde => "tilde".fmt(f),
Type::Xor => "xor".fmt(f), TokenKind::Xor => "xor".fmt(f),
Type::XorEq => "xor-assign".fmt(f), TokenKind::XorEq => "xor-assign".fmt(f),
Type::XorXor => "cat-ears".fmt(f), TokenKind::XorXor => "cat-ears".fmt(f),
} }
} }
} }
impl FromStr for Type { impl FromStr for TokenKind {
/// [FromStr] can only fail when an identifier isn't a keyword /// [FromStr] can only fail when an identifier isn't a keyword
type Err = (); type Err = ();
/// Parses a string s to return a Keyword /// Parses a string s to return a Keyword