diff --git a/cl-lexer/src/lib.rs b/cl-lexer/src/lib.rs index f25bb8b..3c12b73 100644 --- a/cl-lexer/src/lib.rs +++ b/cl-lexer/src/lib.rs @@ -2,7 +2,7 @@ #![warn(clippy::all)] #![feature(decl_macro)] use cl_structures::span::Loc; -use cl_token::*; +use cl_token::{TokenKind as Kind, *}; use std::{ iter::Peekable, str::{Chars, FromStr}, @@ -97,33 +97,33 @@ impl<'t> Lexer<'t> { /// Scans through the text, searching for the next [Token] pub fn scan(&mut self) -> LResult { match self.skip_whitespace().peek()? { - '{' => self.consume()?.produce(Type::LCurly, ()), - '}' => self.consume()?.produce(Type::RCurly, ()), - '[' => self.consume()?.produce(Type::LBrack, ()), - ']' => self.consume()?.produce(Type::RBrack, ()), - '(' => self.consume()?.produce(Type::LParen, ()), - ')' => self.consume()?.produce(Type::RParen, ()), + '{' => self.consume()?.produce(Kind::LCurly, ()), + '}' => self.consume()?.produce(Kind::RCurly, ()), + '[' => self.consume()?.produce(Kind::LBrack, ()), + ']' => self.consume()?.produce(Kind::RBrack, ()), + '(' => self.consume()?.produce(Kind::LParen, ()), + ')' => self.consume()?.produce(Kind::RParen, ()), '&' => self.consume()?.amp(), - '@' => self.consume()?.produce(Type::At, ()), - '\\' => self.consume()?.produce(Type::Backslash, ()), + '@' => self.consume()?.produce(Kind::At, ()), + '\\' => self.consume()?.produce(Kind::Backslash, ()), '!' => self.consume()?.bang(), '|' => self.consume()?.bar(), ':' => self.consume()?.colon(), - ',' => self.consume()?.produce(Type::Comma, ()), + ',' => self.consume()?.produce(Kind::Comma, ()), '.' => self.consume()?.dot(), '=' => self.consume()?.equal(), - '`' => self.consume()?.produce(Type::Grave, ()), + '`' => self.consume()?.produce(Kind::Grave, ()), '>' => self.consume()?.greater(), '#' => self.consume()?.hash(), '<' => self.consume()?.less(), '-' => self.consume()?.minus(), '+' => self.consume()?.plus(), - '?' => self.consume()?.produce(Type::Question, ()), + '?' => self.consume()?.produce(Kind::Question, ()), '%' => self.consume()?.rem(), - ';' => self.consume()?.produce(Type::Semi, ()), + ';' => self.consume()?.produce(Kind::Semi, ()), '/' => self.consume()?.slash(), '*' => self.consume()?.star(), - '~' => self.consume()?.produce(Type::Tilde, ()), + '~' => self.consume()?.produce(Kind::Tilde, ()), '^' => self.consume()?.xor(), '0' => self.consume()?.int_with_base(), '1'..='9' => self.digits::<10>(), @@ -157,7 +157,7 @@ impl<'t> Lexer<'t> { .copied() .ok_or(Error::end_of_file(self.line(), self.col())) } - fn produce(&mut self, ty: Type, data: impl Into) -> LResult { + fn produce(&mut self, ty: TokenKind, data: impl Into) -> LResult { let loc = self.start_loc; self.start_loc = self.current_loc; self.start = self.current; @@ -192,120 +192,120 @@ impl<'t> Lexer<'t> { impl<'t> Lexer<'t> { fn amp(&mut self) -> LResult { match self.peek() { - Ok('&') => self.consume()?.produce(Type::AmpAmp, ()), - Ok('=') => self.consume()?.produce(Type::AmpEq, ()), - _ => self.produce(Type::Amp, ()), + Ok('&') => self.consume()?.produce(Kind::AmpAmp, ()), + Ok('=') => self.consume()?.produce(Kind::AmpEq, ()), + _ => self.produce(Kind::Amp, ()), } } fn bang(&mut self) -> LResult { match self.peek() { - Ok('!') => self.consume()?.produce(Type::BangBang, ()), - Ok('=') => self.consume()?.produce(Type::BangEq, ()), - _ => self.produce(Type::Bang, ()), + Ok('!') => self.consume()?.produce(Kind::BangBang, ()), + Ok('=') => self.consume()?.produce(Kind::BangEq, ()), + _ => self.produce(Kind::Bang, ()), } } fn bar(&mut self) -> LResult { match self.peek() { - Ok('|') => self.consume()?.produce(Type::BarBar, ()), - Ok('=') => self.consume()?.produce(Type::BarEq, ()), - _ => self.produce(Type::Bar, ()), + Ok('|') => self.consume()?.produce(Kind::BarBar, ()), + Ok('=') => self.consume()?.produce(Kind::BarEq, ()), + _ => self.produce(Kind::Bar, ()), } } fn colon(&mut self) -> LResult { match self.peek() { - Ok(':') => self.consume()?.produce(Type::ColonColon, ()), - _ => self.produce(Type::Colon, ()), + Ok(':') => self.consume()?.produce(Kind::ColonColon, ()), + _ => self.produce(Kind::Colon, ()), } } fn dot(&mut self) -> LResult { match self.peek() { Ok('.') => { if let Ok('=') = self.consume()?.peek() { - self.consume()?.produce(Type::DotDotEq, ()) + self.consume()?.produce(Kind::DotDotEq, ()) } else { - self.produce(Type::DotDot, ()) + self.produce(Kind::DotDot, ()) } } - _ => self.produce(Type::Dot, ()), + _ => self.produce(Kind::Dot, ()), } } fn equal(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce(Type::EqEq, ()), - Ok('>') => self.consume()?.produce(Type::FatArrow, ()), - _ => self.produce(Type::Eq, ()), + Ok('=') => self.consume()?.produce(Kind::EqEq, ()), + Ok('>') => self.consume()?.produce(Kind::FatArrow, ()), + _ => self.produce(Kind::Eq, ()), } } fn greater(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce(Type::GtEq, ()), + Ok('=') => self.consume()?.produce(Kind::GtEq, ()), Ok('>') => { if let Ok('=') = self.consume()?.peek() { - self.consume()?.produce(Type::GtGtEq, ()) + self.consume()?.produce(Kind::GtGtEq, ()) } else { - self.produce(Type::GtGt, ()) + self.produce(Kind::GtGt, ()) } } - _ => self.produce(Type::Gt, ()), + _ => self.produce(Kind::Gt, ()), } } fn hash(&mut self) -> LResult { match self.peek() { - Ok('!') => self.consume()?.produce(Type::HashBang, ()), - _ => self.produce(Type::Hash, ()), + Ok('!') => self.consume()?.produce(Kind::HashBang, ()), + _ => self.produce(Kind::Hash, ()), } } fn less(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce(Type::LtEq, ()), + Ok('=') => self.consume()?.produce(Kind::LtEq, ()), Ok('<') => { if let Ok('=') = self.consume()?.peek() { - self.consume()?.produce(Type::LtLtEq, ()) + self.consume()?.produce(Kind::LtLtEq, ()) } else { - self.produce(Type::LtLt, ()) + self.produce(Kind::LtLt, ()) } } - _ => self.produce(Type::Lt, ()), + _ => self.produce(Kind::Lt, ()), } } fn minus(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce(Type::MinusEq, ()), - Ok('>') => self.consume()?.produce(Type::Arrow, ()), - _ => self.produce(Type::Minus, ()), + Ok('=') => self.consume()?.produce(Kind::MinusEq, ()), + Ok('>') => self.consume()?.produce(Kind::Arrow, ()), + _ => self.produce(Kind::Minus, ()), } } fn plus(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce(Type::PlusEq, ()), - _ => self.produce(Type::Plus, ()), + Ok('=') => self.consume()?.produce(Kind::PlusEq, ()), + _ => self.produce(Kind::Plus, ()), } } fn rem(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce(Type::RemEq, ()), - _ => self.produce(Type::Rem, ()), + Ok('=') => self.consume()?.produce(Kind::RemEq, ()), + _ => self.produce(Kind::Rem, ()), } } fn slash(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce(Type::SlashEq, ()), + Ok('=') => self.consume()?.produce(Kind::SlashEq, ()), Ok('/') => self.consume()?.line_comment(), Ok('*') => self.consume()?.block_comment(), - _ => self.produce(Type::Slash, ()), + _ => self.produce(Kind::Slash, ()), } } fn star(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce(Type::StarEq, ()), - _ => self.produce(Type::Star, ()), + Ok('=') => self.consume()?.produce(Kind::StarEq, ()), + _ => self.produce(Kind::Star, ()), } } fn xor(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce(Type::XorEq, ()), - Ok('^') => self.consume()?.produce(Type::XorXor, ()), - _ => self.produce(Type::Xor, ()), + Ok('=') => self.consume()?.produce(Kind::XorEq, ()), + Ok('^') => self.consume()?.produce(Kind::XorXor, ()), + _ => self.produce(Kind::Xor, ()), } } } @@ -315,7 +315,7 @@ impl<'t> Lexer<'t> { while Ok('\n') != self.peek() { self.consume()?; } - self.produce(Type::Comment, ()) + self.produce(Kind::Comment, ()) } fn block_comment(&mut self) -> LResult { while let Ok(c) = self.next() { @@ -323,7 +323,7 @@ impl<'t> Lexer<'t> { break; } } - self.produce(Type::Comment, ()) + self.produce(Kind::Comment, ()) } } /// Identifiers @@ -333,10 +333,10 @@ impl<'t> Lexer<'t> { while let Ok(c) = self.xid_continue() { out.push(c) } - if let Ok(keyword) = Type::from_str(&out) { + if let Ok(keyword) = Kind::from_str(&out) { self.produce(keyword, ()) } else { - self.produce(Type::Identifier, Data::Identifier(out.into())) + self.produce(Kind::Identifier, TokenData::Identifier(out.into())) } } fn xid_start(&mut self) -> LResult { @@ -367,7 +367,7 @@ impl<'t> Lexer<'t> { Ok('o') => self.consume()?.digits::<8>(), Ok('b') => self.consume()?.digits::<2>(), Ok('0'..='9') => self.digits::<10>(), - _ => self.produce(Type::Integer, 0), + _ => self.produce(Kind::Integer, 0), } } fn digits(&mut self) -> LResult { @@ -375,7 +375,7 @@ impl<'t> Lexer<'t> { while let Ok(true) = self.peek().as_ref().map(char::is_ascii_alphanumeric) { value = value * B as u128 + self.digit::()? as u128; } - self.produce(Type::Integer, value) + self.produce(Kind::Integer, value) } fn digit(&mut self) -> LResult { let digit = self.peek()?; @@ -396,12 +396,12 @@ impl<'t> Lexer<'t> { { value.push(self.unescape()?) } - self.consume()?.produce(Type::String, value) + self.consume()?.produce(Kind::String, value) } fn character(&mut self) -> LResult { let out = self.unescape()?; match self.peek()? { - '\'' => self.consume()?.produce(Type::Character, out), + '\'' => self.consume()?.produce(Kind::Character, out), _ => Err(Error::unmatched_delimiters('\'', self.line(), self.col())), } } @@ -477,7 +477,7 @@ pub mod error { pub enum Reason { /// Found an opening delimiter of type [char], but not the expected closing delimiter UnmatchedDelimiters(char), - /// Found a character that doesn't belong to any [Type](cl_token::token_type::Type) + /// Found a character that doesn't belong to any [TokenKind](cl_token::TokenKind) UnexpectedChar(char), /// Found a character that's not valid in identifiers while looking for an identifier NotIdentifier(char), diff --git a/cl-lexer/src/tests.rs b/cl-lexer/src/tests.rs index c395126..199ab35 100644 --- a/cl-lexer/src/tests.rs +++ b/cl-lexer/src/tests.rs @@ -35,7 +35,7 @@ macro td ($($id:expr),*) { mod ident { use super::*; macro ident ($($id:literal),*) { - [$(Data::Identifier($id.into())),*] + [$(TokenData::Identifier($id.into())),*] } test_lexer_data_type! { underscore { "_ _" => ident!["_", "_"] } @@ -47,7 +47,7 @@ mod ident { mod keyword { use super::*; macro kw($($k:ident),*) { - [ $(Type::$k,)* ] + [ $(TokenKind::$k,)* ] } test_lexer_output_type! { kw_break { "break break" => kw![Break, Break] } @@ -111,57 +111,57 @@ mod string { mod punct { use super::*; test_lexer_output_type! { - l_curly { "{ {" => [ Type::LCurly, Type::LCurly ] } - r_curly { "} }" => [ Type::RCurly, Type::RCurly ] } - l_brack { "[ [" => [ Type::LBrack, Type::LBrack ] } - r_brack { "] ]" => [ Type::RBrack, Type::RBrack ] } - l_paren { "( (" => [ Type::LParen, Type::LParen ] } - r_paren { ") )" => [ Type::RParen, Type::RParen ] } - amp { "& &" => [ Type::Amp, Type::Amp ] } - amp_amp { "&& &&" => [ Type::AmpAmp, Type::AmpAmp ] } - amp_eq { "&= &=" => [ Type::AmpEq, Type::AmpEq ] } - arrow { "-> ->" => [ Type::Arrow, Type::Arrow] } - at { "@ @" => [ Type::At, Type::At] } - backslash { "\\ \\" => [ Type::Backslash, Type::Backslash] } - bang { "! !" => [ Type::Bang, Type::Bang] } - bangbang { "!! !!" => [ Type::BangBang, Type::BangBang] } - bangeq { "!= !=" => [ Type::BangEq, Type::BangEq] } - bar { "| |" => [ Type::Bar, Type::Bar] } - barbar { "|| ||" => [ Type::BarBar, Type::BarBar] } - bareq { "|= |=" => [ Type::BarEq, Type::BarEq] } - colon { ": :" => [ Type::Colon, Type::Colon] } - comma { ", ," => [ Type::Comma, Type::Comma] } - dot { ". ." => [ Type::Dot, Type::Dot] } - dotdot { ".. .." => [ Type::DotDot, Type::DotDot] } - dotdoteq { "..= ..=" => [ Type::DotDotEq, Type::DotDotEq] } - eq { "= =" => [ Type::Eq, Type::Eq] } - eqeq { "== ==" => [ Type::EqEq, Type::EqEq] } - fatarrow { "=> =>" => [ Type::FatArrow, Type::FatArrow] } - grave { "` `" => [ Type::Grave, Type::Grave] } - gt { "> >" => [ Type::Gt, Type::Gt] } - gteq { ">= >=" => [ Type::GtEq, Type::GtEq] } - gtgt { ">> >>" => [ Type::GtGt, Type::GtGt] } - gtgteq { ">>= >>=" => [ Type::GtGtEq, Type::GtGtEq] } - hash { "# #" => [ Type::Hash, Type::Hash] } - lt { "< <" => [ Type::Lt, Type::Lt] } - lteq { "<= <=" => [ Type::LtEq, Type::LtEq] } - ltlt { "<< <<" => [ Type::LtLt, Type::LtLt] } - ltlteq { "<<= <<=" => [ Type::LtLtEq, Type::LtLtEq] } - minus { "- -" => [ Type::Minus, Type::Minus] } - minuseq { "-= -=" => [ Type::MinusEq, Type::MinusEq] } - plus { "+ +" => [ Type::Plus, Type::Plus] } - pluseq { "+= +=" => [ Type::PlusEq, Type::PlusEq] } - question { "? ?" => [ Type::Question, Type::Question] } - rem { "% %" => [ Type::Rem, Type::Rem] } - remeq { "%= %=" => [ Type::RemEq, Type::RemEq] } - semi { "; ;" => [ Type::Semi, Type::Semi] } - slash { "/ /" => [ Type::Slash, Type::Slash] } - slasheq { "/= /=" => [ Type::SlashEq, Type::SlashEq] } - star { "* *" => [ Type::Star, Type::Star] } - stareq { "*= *=" => [ Type::StarEq, Type::StarEq] } - tilde { "~ ~" => [ Type::Tilde, Type::Tilde] } - xor { "^ ^" => [ Type::Xor, Type::Xor] } - xoreq { "^= ^=" => [ Type::XorEq, Type::XorEq] } - xorxor { "^^ ^^" => [ Type::XorXor, Type::XorXor] } + l_curly { "{ {" => [ TokenKind::LCurly, TokenKind::LCurly ] } + r_curly { "} }" => [ TokenKind::RCurly, TokenKind::RCurly ] } + l_brack { "[ [" => [ TokenKind::LBrack, TokenKind::LBrack ] } + r_brack { "] ]" => [ TokenKind::RBrack, TokenKind::RBrack ] } + l_paren { "( (" => [ TokenKind::LParen, TokenKind::LParen ] } + r_paren { ") )" => [ TokenKind::RParen, TokenKind::RParen ] } + amp { "& &" => [ TokenKind::Amp, TokenKind::Amp ] } + amp_amp { "&& &&" => [ TokenKind::AmpAmp, TokenKind::AmpAmp ] } + amp_eq { "&= &=" => [ TokenKind::AmpEq, TokenKind::AmpEq ] } + arrow { "-> ->" => [ TokenKind::Arrow, TokenKind::Arrow] } + at { "@ @" => [ TokenKind::At, TokenKind::At] } + backslash { "\\ \\" => [ TokenKind::Backslash, TokenKind::Backslash] } + bang { "! !" => [ TokenKind::Bang, TokenKind::Bang] } + bangbang { "!! !!" => [ TokenKind::BangBang, TokenKind::BangBang] } + bangeq { "!= !=" => [ TokenKind::BangEq, TokenKind::BangEq] } + bar { "| |" => [ TokenKind::Bar, TokenKind::Bar] } + barbar { "|| ||" => [ TokenKind::BarBar, TokenKind::BarBar] } + bareq { "|= |=" => [ TokenKind::BarEq, TokenKind::BarEq] } + colon { ": :" => [ TokenKind::Colon, TokenKind::Colon] } + comma { ", ," => [ TokenKind::Comma, TokenKind::Comma] } + dot { ". ." => [ TokenKind::Dot, TokenKind::Dot] } + dotdot { ".. .." => [ TokenKind::DotDot, TokenKind::DotDot] } + dotdoteq { "..= ..=" => [ TokenKind::DotDotEq, TokenKind::DotDotEq] } + eq { "= =" => [ TokenKind::Eq, TokenKind::Eq] } + eqeq { "== ==" => [ TokenKind::EqEq, TokenKind::EqEq] } + fatarrow { "=> =>" => [ TokenKind::FatArrow, TokenKind::FatArrow] } + grave { "` `" => [ TokenKind::Grave, TokenKind::Grave] } + gt { "> >" => [ TokenKind::Gt, TokenKind::Gt] } + gteq { ">= >=" => [ TokenKind::GtEq, TokenKind::GtEq] } + gtgt { ">> >>" => [ TokenKind::GtGt, TokenKind::GtGt] } + gtgteq { ">>= >>=" => [ TokenKind::GtGtEq, TokenKind::GtGtEq] } + hash { "# #" => [ TokenKind::Hash, TokenKind::Hash] } + lt { "< <" => [ TokenKind::Lt, TokenKind::Lt] } + lteq { "<= <=" => [ TokenKind::LtEq, TokenKind::LtEq] } + ltlt { "<< <<" => [ TokenKind::LtLt, TokenKind::LtLt] } + ltlteq { "<<= <<=" => [ TokenKind::LtLtEq, TokenKind::LtLtEq] } + minus { "- -" => [ TokenKind::Minus, TokenKind::Minus] } + minuseq { "-= -=" => [ TokenKind::MinusEq, TokenKind::MinusEq] } + plus { "+ +" => [ TokenKind::Plus, TokenKind::Plus] } + pluseq { "+= +=" => [ TokenKind::PlusEq, TokenKind::PlusEq] } + question { "? ?" => [ TokenKind::Question, TokenKind::Question] } + rem { "% %" => [ TokenKind::Rem, TokenKind::Rem] } + remeq { "%= %=" => [ TokenKind::RemEq, TokenKind::RemEq] } + semi { "; ;" => [ TokenKind::Semi, TokenKind::Semi] } + slash { "/ /" => [ TokenKind::Slash, TokenKind::Slash] } + slasheq { "/= /=" => [ TokenKind::SlashEq, TokenKind::SlashEq] } + star { "* *" => [ TokenKind::Star, TokenKind::Star] } + stareq { "*= *=" => [ TokenKind::StarEq, TokenKind::StarEq] } + tilde { "~ ~" => [ TokenKind::Tilde, TokenKind::Tilde] } + xor { "^ ^" => [ TokenKind::Xor, TokenKind::Xor] } + xoreq { "^= ^=" => [ TokenKind::XorEq, TokenKind::XorEq] } + xorxor { "^^ ^^" => [ TokenKind::XorXor, TokenKind::XorXor] } } } diff --git a/cl-parser/src/error.rs b/cl-parser/src/error.rs index 7db443f..18ece96 100644 --- a/cl-parser/src/error.rs +++ b/cl-parser/src/error.rs @@ -21,10 +21,10 @@ pub enum ErrorKind { UnmatchedParentheses, UnmatchedCurlyBraces, UnmatchedSquareBrackets, - Unexpected(Type), + Unexpected(TokenKind), Expected { - want: Type, - got: Type, + want: TokenKind, + got: TokenKind, }, /// No rules matched Nothing, diff --git a/cl-parser/src/parser.rs b/cl-parser/src/parser.rs index ea25521..8a6b9d8 100644 --- a/cl-parser/src/parser.rs +++ b/cl-parser/src/parser.rs @@ -39,8 +39,8 @@ impl<'t> Parser<'t> { .scan() .map_err(|e| self.error(e.into(), while_parsing))? { - t if t.ty() == Type::Invalid => continue, - t if t.ty() == Type::Comment => continue, + t if t.ty() == TokenKind::Invalid => continue, + t if t.ty() == TokenKind::Comment => continue, t => break Ok(t), } } @@ -63,8 +63,8 @@ impl<'t> Parser<'t> { self.loc = Loc::from(&self.lexer); self.next.take() } - /// Looks ahead at the next [Token]'s [Type] - pub fn peek_type(&mut self, while_parsing: Parsing) -> PResult { + /// Looks ahead at the next [Token]'s [TokenKind] + pub fn peek_kind(&mut self, while_parsing: Parsing) -> PResult { self.peek(while_parsing).map(|t| t.ty()) } /// Consumes one [Token] @@ -75,9 +75,9 @@ impl<'t> Parser<'t> { None => self.consume_from_lexer(while_parsing), } } - /// Consumes the next [Token] if it matches the pattern [Type] - pub fn match_type(&mut self, want: Type, while_parsing: Parsing) -> PResult { - let got = self.peek_type(while_parsing)?; + /// Consumes the next [Token] if it matches the pattern [TokenKind] + pub fn match_type(&mut self, want: TokenKind, while_parsing: Parsing) -> PResult { + let got = self.peek_kind(while_parsing)?; if got == want { Ok(self.consume_peeked().expect("should not fail after peek")) } else { @@ -88,16 +88,16 @@ impl<'t> Parser<'t> { // the three matched delimiter pairs /// Square brackets: `[` `]` -const BRACKETS: (Type, Type) = (Type::LBrack, Type::RBrack); +const BRACKETS: (TokenKind, TokenKind) = (TokenKind::LBrack, TokenKind::RBrack); /// Curly braces: `{` `}` -const CURLIES: (Type, Type) = (Type::LCurly, Type::RCurly); +const CURLIES: (TokenKind, TokenKind) = (TokenKind::LCurly, TokenKind::RCurly); /// Parentheses: `(` `)` -const PARENS: (Type, Type) = (Type::LParen, Type::RParen); +const PARENS: (TokenKind, TokenKind) = (TokenKind::LParen, TokenKind::RParen); /// Parses constructions of the form `delim.0 f delim.1` (i.e. `(` `foobar` `)`) const fn delim<'t, T>( f: impl Fn(&mut Parser<'t>) -> PResult, - delim: (Type, Type), + delim: (TokenKind, TokenKind), while_parsing: Parsing, ) -> impl Fn(&mut Parser<'t>) -> PResult { move |parser| { @@ -113,15 +113,15 @@ const fn delim<'t, T>( /// where `~until` is a negative lookahead assertion const fn sep<'t, T>( f: impl Fn(&mut Parser<'t>) -> PResult, - sep: Type, - until: Type, + sep: TokenKind, + until: TokenKind, while_parsing: Parsing, ) -> impl Fn(&mut Parser<'t>) -> PResult> { move |parser| { let mut args = vec![]; - while until != parser.peek_type(while_parsing)? { + while until != parser.peek_kind(while_parsing)? { args.push(f(parser)?); - if sep != parser.peek_type(while_parsing)? { + if sep != parser.peek_kind(while_parsing)? { break; } parser.consume_peeked(); @@ -136,30 +136,30 @@ const fn sep<'t, T>( #[allow(dead_code)] const fn rep<'t, T>( f: impl Fn(&mut Parser<'t>) -> PResult, - until: Type, + until: TokenKind, while_parsing: Parsing, ) -> impl Fn(&mut Parser<'t>) -> PResult> { move |parser| { let mut out = vec![]; - while until != parser.peek_type(while_parsing)? { + while until != parser.peek_kind(while_parsing)? { out.push(f(parser)?) } Ok(out) } } -/// Expands to a pattern which matches item-like [Token] [Type]s +/// Expands to a pattern which matches item-like [Token] [TokenKind]s macro item_like() { - Type::Hash - | Type::Pub - | Type::Type - | Type::Const - | Type::Static - | Type::Mod - | Type::Fn - | Type::Struct - | Type::Enum - | Type::Impl + TokenKind::Hash + | TokenKind::Pub + | TokenKind::Type + | TokenKind::Const + | TokenKind::Static + | TokenKind::Mod + | TokenKind::Fn + | TokenKind::Struct + | TokenKind::Enum + | TokenKind::Impl } /// Top level parsing @@ -167,8 +167,8 @@ impl<'t> Parser<'t> { /// Parses a [File] pub fn file(&mut self) -> PResult { let mut items = vec![]; - while match self.peek_type(Parsing::File) { - Ok(Type::RCurly) | Err(Error { reason: EndOfInput, .. }) => false, + while match self.peek_kind(Parsing::File) { + Ok(TokenKind::RCurly) | Err(Error { reason: EndOfInput, .. }) => false, Ok(_) => true, Err(e) => Err(e)?, } { @@ -203,13 +203,13 @@ impl<'t> Parser<'t> { /// See also: [Parser::path_part], [Parser::identifier] pub fn path(&mut self) -> PResult { const PARSING: Parsing = Parsing::PathExpr; - let absolute = matches!(self.peek_type(PARSING)?, Type::ColonColon); + let absolute = matches!(self.peek_kind(PARSING)?, TokenKind::ColonColon); if absolute { self.consume_peeked(); } let mut parts = vec![self.path_part()?]; - while let Ok(Type::ColonColon) = self.peek_type(PARSING) { + while let Ok(TokenKind::ColonColon) = self.peek_kind(PARSING) { self.consume_peeked(); parts.push(self.path_part()?); } @@ -224,8 +224,8 @@ impl<'t> Parser<'t> { let start = self.loc(); Ok(Stmt { kind: self.stmtkind()?, - semi: match self.peek_type(PARSING) { - Ok(Type::Semi) => { + semi: match self.peek_kind(PARSING) { + Ok(TokenKind::Semi) => { self.consume_peeked(); Semi::Terminated } @@ -247,11 +247,11 @@ impl<'t> Parser<'t> { impl<'t> Parser<'t> { /// Parses an [attribute set](Attrs) pub fn attributes(&mut self) -> PResult { - if self.match_type(Type::Hash, Parsing::Attrs).is_err() { + if self.match_type(TokenKind::Hash, Parsing::Attrs).is_err() { return Ok(Attrs { meta: vec![] }); } let meta = delim( - sep(Self::meta, Type::Comma, BRACKETS.1, Parsing::Attrs), + sep(Self::meta, TokenKind::Comma, BRACKETS.1, Parsing::Attrs), BRACKETS, Parsing::Attrs, ); @@ -263,16 +263,16 @@ impl<'t> Parser<'t> { pub fn meta_kind(&mut self) -> PResult { const PARSING: Parsing = Parsing::Meta; let lit_tuple = delim( - sep(Self::literal, Type::Comma, PARENS.1, PARSING), + sep(Self::literal, TokenKind::Comma, PARENS.1, PARSING), PARENS, PARSING, ); - Ok(match self.peek_type(PARSING) { - Ok(Type::Eq) => { + Ok(match self.peek_kind(PARSING) { + Ok(TokenKind::Eq) => { self.consume_peeked(); MetaKind::Equals(self.literal()?) } - Ok(Type::LParen) => MetaKind::Func(lit_tuple(self)?), + Ok(TokenKind::LParen) => MetaKind::Func(lit_tuple(self)?), _ => MetaKind::Plain, }) } @@ -284,104 +284,104 @@ impl<'t> Parser<'t> { /// /// See also: [Parser::item] pub fn itemkind(&mut self) -> PResult { - Ok(match self.peek_type(Parsing::Item)? { - Type::Type => self.parse_alias()?.into(), - Type::Const => self.parse_const()?.into(), - Type::Static => self.parse_static()?.into(), - Type::Mod => self.parse_module()?.into(), - Type::Fn => self.parse_function()?.into(), - Type::Struct => self.parse_struct()?.into(), - Type::Enum => self.parse_enum()?.into(), - Type::Impl => self.parse_impl()?.into(), + Ok(match self.peek_kind(Parsing::Item)? { + TokenKind::Type => self.parse_alias()?.into(), + TokenKind::Const => self.parse_const()?.into(), + TokenKind::Static => self.parse_static()?.into(), + TokenKind::Mod => self.parse_module()?.into(), + TokenKind::Fn => self.parse_function()?.into(), + TokenKind::Struct => self.parse_struct()?.into(), + TokenKind::Enum => self.parse_enum()?.into(), + TokenKind::Impl => self.parse_impl()?.into(), t => Err(self.error(Unexpected(t), Parsing::Item))?, }) } pub fn parse_alias(&mut self) -> PResult { const PARSING: Parsing = Parsing::Alias; - self.match_type(Type::Type, PARSING)?; + self.match_type(TokenKind::Type, PARSING)?; let out = Ok(Alias { to: self.identifier()?, - from: if self.match_type(Type::Eq, PARSING).is_ok() { + from: if self.match_type(TokenKind::Eq, PARSING).is_ok() { Some(self.ty()?.into()) } else { None }, }); - self.match_type(Type::Semi, PARSING)?; + self.match_type(TokenKind::Semi, PARSING)?; out } pub fn parse_const(&mut self) -> PResult { const PARSING: Parsing = Parsing::Const; - self.match_type(Type::Const, PARSING)?; + self.match_type(TokenKind::Const, PARSING)?; let out = Ok(Const { name: self.identifier()?, ty: { - self.match_type(Type::Colon, PARSING)?; + self.match_type(TokenKind::Colon, PARSING)?; self.ty()?.into() }, init: { - self.match_type(Type::Eq, PARSING)?; + self.match_type(TokenKind::Eq, PARSING)?; self.expr()?.into() }, }); - self.match_type(Type::Semi, PARSING)?; + self.match_type(TokenKind::Semi, PARSING)?; out } pub fn parse_static(&mut self) -> PResult { const PARSING: Parsing = Parsing::Static; - self.match_type(Type::Static, PARSING)?; + self.match_type(TokenKind::Static, PARSING)?; let out = Ok(Static { mutable: self.mutability()?, name: self.identifier()?, ty: { - self.match_type(Type::Colon, PARSING)?; + self.match_type(TokenKind::Colon, PARSING)?; self.ty()?.into() }, init: { - self.match_type(Type::Eq, PARSING)?; + self.match_type(TokenKind::Eq, PARSING)?; self.expr()?.into() }, }); - self.match_type(Type::Semi, PARSING)?; + self.match_type(TokenKind::Semi, PARSING)?; out } pub fn parse_module(&mut self) -> PResult { const PARSING: Parsing = Parsing::Module; - self.match_type(Type::Mod, PARSING)?; + self.match_type(TokenKind::Mod, PARSING)?; Ok(Module { name: self.identifier()?, kind: self.modulekind()? }) } pub fn modulekind(&mut self) -> PResult { const PARSING: Parsing = Parsing::ModuleKind; let inline = delim(Self::file, CURLIES, PARSING); - match self.peek_type(PARSING)? { - Type::LCurly => Ok(ModuleKind::Inline(inline(self)?)), - Type::Semi => { + match self.peek_kind(PARSING)? { + TokenKind::LCurly => Ok(ModuleKind::Inline(inline(self)?)), + TokenKind::Semi => { self.consume_peeked(); Ok(ModuleKind::Outline) } - got => Err(self.error(Expected { want: Type::Semi, got }, PARSING)), + got => Err(self.error(Expected { want: TokenKind::Semi, got }, PARSING)), } } pub fn parse_function(&mut self) -> PResult { const PARSING: Parsing = Parsing::Function; - self.match_type(Type::Fn, PARSING)?; + self.match_type(TokenKind::Fn, PARSING)?; Ok(Function { name: self.identifier()?, args: self.parse_params()?, - rety: match self.peek_type(PARSING)? { - Type::LCurly | Type::Semi => None, - Type::Arrow => { + rety: match self.peek_kind(PARSING)? { + TokenKind::LCurly | TokenKind::Semi => None, + TokenKind::Arrow => { self.consume_peeked(); Some(self.ty()?.into()) } - got => Err(self.error(Expected { want: Type::Arrow, got }, PARSING))?, + got => Err(self.error(Expected { want: TokenKind::Arrow, got }, PARSING))?, }, - body: match self.peek_type(PARSING)? { - Type::LCurly => Some(self.block()?), - Type::Semi => { + body: match self.peek_kind(PARSING)? { + TokenKind::LCurly => Some(self.block()?), + TokenKind::Semi => { self.consume_peeked(); None } @@ -392,7 +392,7 @@ impl<'t> Parser<'t> { pub fn parse_params(&mut self) -> PResult> { const PARSING: Parsing = Parsing::Function; delim( - sep(Self::parse_param, Type::Comma, PARENS.1, PARSING), + sep(Self::parse_param, TokenKind::Comma, PARENS.1, PARSING), PARENS, PARSING, )(self) @@ -402,24 +402,24 @@ impl<'t> Parser<'t> { mutability: self.mutability()?, name: self.identifier()?, ty: { - self.match_type(Type::Colon, Parsing::Param)?; + self.match_type(TokenKind::Colon, Parsing::Param)?; self.ty()?.into() }, }) } pub fn parse_struct(&mut self) -> PResult { const PARSING: Parsing = Parsing::Struct; - self.match_type(Type::Struct, PARSING)?; + self.match_type(TokenKind::Struct, PARSING)?; Ok(Struct { name: self.identifier()?, - kind: match self.peek_type(PARSING)? { - Type::LParen => self.structkind_tuple()?, - Type::LCurly => self.structkind_struct()?, - Type::Semi => { + kind: match self.peek_kind(PARSING)? { + TokenKind::LParen => self.structkind_tuple()?, + TokenKind::LCurly => self.structkind_struct()?, + TokenKind::Semi => { self.consume_peeked(); StructKind::Empty } - got => Err(self.error(Expected { want: Type::Semi, got }, PARSING))?, + got => Err(self.error(Expected { want: TokenKind::Semi, got }, PARSING))?, }, }) } @@ -427,7 +427,7 @@ impl<'t> Parser<'t> { const PARSING: Parsing = Parsing::StructKind; Ok(StructKind::Tuple(delim( - sep(Self::ty, Type::Comma, PARENS.1, PARSING), + sep(Self::ty, TokenKind::Comma, PARENS.1, PARSING), PARENS, PARSING, )(self)?)) @@ -435,7 +435,7 @@ impl<'t> Parser<'t> { pub fn structkind_struct(&mut self) -> PResult { const PARSING: Parsing = Parsing::StructKind; Ok(StructKind::Struct(delim( - sep(Self::struct_member, Type::Comma, CURLIES.1, PARSING), + sep(Self::struct_member, TokenKind::Comma, CURLIES.1, PARSING), CURLIES, PARSING, )(self)?)) @@ -446,7 +446,7 @@ impl<'t> Parser<'t> { vis: self.visibility()?, name: self.identifier()?, ty: { - self.match_type(Type::Colon, PARSING)?; + self.match_type(TokenKind::Colon, PARSING)?; self.ty()? }, }) @@ -454,16 +454,21 @@ impl<'t> Parser<'t> { pub fn parse_enum(&mut self) -> PResult { // Enum = "enum" Identifier '{' (Variant ',')* Variant? '}' ; const PARSING: Parsing = Parsing::Enum; - self.match_type(Type::Enum, PARSING)?; + self.match_type(TokenKind::Enum, PARSING)?; Ok(Enum { name: self.identifier()?, - kind: match self.peek_type(PARSING)? { - Type::LCurly => EnumKind::Variants(delim( - sep(Self::enum_variant, Type::Comma, Type::RCurly, PARSING), + kind: match self.peek_kind(PARSING)? { + TokenKind::LCurly => EnumKind::Variants(delim( + sep( + Self::enum_variant, + TokenKind::Comma, + TokenKind::RCurly, + PARSING, + ), CURLIES, PARSING, )(self)?), - Type::Semi => { + TokenKind::Semi => { self.consume_peeked(); EnumKind::NoVariants } @@ -476,27 +481,32 @@ impl<'t> Parser<'t> { const PARSING: Parsing = Parsing::Variant; Ok(Variant { name: self.identifier()?, - kind: match self.peek_type(PARSING)? { - Type::Eq => self.variantkind_clike()?, - Type::LCurly => self.variantkind_struct()?, - Type::LParen => self.variantkind_tuple()?, + kind: match self.peek_kind(PARSING)? { + TokenKind::Eq => self.variantkind_clike()?, + TokenKind::LCurly => self.variantkind_struct()?, + TokenKind::LParen => self.variantkind_tuple()?, _ => VariantKind::Plain, }, }) } pub fn variantkind_clike(&mut self) -> PResult { const PARSING: Parsing = Parsing::VariantKind; - self.match_type(Type::Eq, PARSING)?; - let tok = self.match_type(Type::Integer, PARSING)?; + self.match_type(TokenKind::Eq, PARSING)?; + let tok = self.match_type(TokenKind::Integer, PARSING)?; Ok(VariantKind::CLike(match tok.data() { - Data::Integer(i) => *i, + TokenData::Integer(i) => *i, _ => panic!("Expected token data for {tok:?} while parsing {PARSING}"), })) } pub fn variantkind_struct(&mut self) -> PResult { const PARSING: Parsing = Parsing::VariantKind; Ok(VariantKind::Struct(delim( - sep(Self::struct_member, Type::Comma, Type::RCurly, PARSING), + sep( + Self::struct_member, + TokenKind::Comma, + TokenKind::RCurly, + PARSING, + ), CURLIES, PARSING, )(self)?)) @@ -504,7 +514,7 @@ impl<'t> Parser<'t> { pub fn variantkind_tuple(&mut self) -> PResult { const PARSING: Parsing = Parsing::VariantKind; Ok(VariantKind::Tuple(delim( - sep(Self::ty, Type::Comma, Type::RParen, PARSING), + sep(Self::ty, TokenKind::Comma, TokenKind::RParen, PARSING), PARENS, PARSING, )(self)?)) @@ -512,19 +522,19 @@ impl<'t> Parser<'t> { pub fn parse_impl(&mut self) -> PResult { const PARSING: Parsing = Parsing::Impl; - self.match_type(Type::Impl, PARSING)?; + self.match_type(TokenKind::Impl, PARSING)?; Err(self.error(Todo, PARSING)) } pub fn visibility(&mut self) -> PResult { - if let Type::Pub = self.peek_type(Parsing::Visibility)? { + if let TokenKind::Pub = self.peek_kind(Parsing::Visibility)? { self.consume_peeked(); return Ok(Visibility::Public); }; Ok(Visibility::Private) } pub fn mutability(&mut self) -> PResult { - if let Type::Mut = self.peek_type(Parsing::Mutability)? { + if let TokenKind::Mut = self.peek_kind(Parsing::Mutability)? { self.consume_peeked(); return Ok(Mutability::Mut); }; @@ -539,18 +549,18 @@ impl<'t> Parser<'t> { /// See also: [Parser::ty] pub fn tykind(&mut self) -> PResult { const PARSING: Parsing = Parsing::TyKind; - let out = match self.peek_type(PARSING)? { - Type::Bang => { + let out = match self.peek_kind(PARSING)? { + TokenKind::Bang => { self.consume_peeked(); TyKind::Never } - Type::SelfTy => { + TokenKind::SelfTy => { self.consume_peeked(); TyKind::SelfTy } - Type::Amp | Type::AmpAmp => self.tyref()?.into(), - Type::LParen => self.tytuple()?.into(), - Type::Fn => self.tyfn()?.into(), + TokenKind::Amp | TokenKind::AmpAmp => self.tyref()?.into(), + TokenKind::LParen => self.tytuple()?.into(), + TokenKind::Fn => self.tyfn()?.into(), path_like!() => self.path()?.into(), t => Err(self.error(Unexpected(t), PARSING))?, }; @@ -562,7 +572,7 @@ impl<'t> Parser<'t> { const PARSING: Parsing = Parsing::TyTuple; Ok(TyTuple { types: delim( - sep(Self::ty, Type::Comma, PARENS.1, PARSING), + sep(Self::ty, TokenKind::Comma, PARENS.1, PARSING), PARENS, PARSING, )(self)?, @@ -573,9 +583,9 @@ impl<'t> Parser<'t> { const PARSING: Parsing = Parsing::TyRef; let mut count = 0; loop { - match self.peek_type(PARSING)? { - Type::Amp => count += 1, - Type::AmpAmp => count += 2, + match self.peek_kind(PARSING)? { + TokenKind::Amp => count += 1, + TokenKind::AmpAmp => count += 2, _ => break, } self.consume_peeked(); @@ -585,12 +595,12 @@ impl<'t> Parser<'t> { /// [TyFn] = `fn` [TyTuple] (-> [Ty])? pub fn tyfn(&mut self) -> PResult { const PARSING: Parsing = Parsing::TyFn; - self.match_type(Type::Fn, PARSING)?; + self.match_type(TokenKind::Fn, PARSING)?; Ok(TyFn { args: self.tytuple()?, rety: { - match self.peek_type(PARSING)? { - Type::Arrow => { + match self.peek_kind(PARSING)? { + TokenKind::Arrow => { self.consume_peeked(); Some(self.ty()?.into()) } @@ -601,33 +611,33 @@ impl<'t> Parser<'t> { } } -/// Expands to a pattern which matches literal-like [Type]s +/// Expands to a pattern which matches literal-like [TokenKind]s macro literal_like() { - Type::True | Type::False | Type::String | Type::Character | Type::Integer | Type::Float + TokenKind::True | TokenKind::False | TokenKind::String | TokenKind::Character | TokenKind::Integer | TokenKind::Float } /// Expands to a pattern which matches path-like [token Types](Type) macro path_like() { - Type::Super | Type::SelfKw | Type::Identifier | Type::ColonColon + TokenKind::Super | TokenKind::SelfKw | TokenKind::Identifier | TokenKind::ColonColon } /// # Path parsing impl<'t> Parser<'t> { /// [PathPart] = `super` | `self` | [Identifier] pub fn path_part(&mut self) -> PResult { const PARSING: Parsing = Parsing::PathPart; - let out = match self.peek_type(PARSING)? { - Type::Super => PathPart::SuperKw, - Type::SelfKw => PathPart::SelfKw, - Type::Identifier => PathPart::Ident(self.identifier()?), + let out = match self.peek_kind(PARSING)? { + TokenKind::Super => PathPart::SuperKw, + TokenKind::SelfKw => PathPart::SelfKw, + TokenKind::Identifier => PathPart::Ident(self.identifier()?), t => return Err(self.error(Unexpected(t), PARSING)), }; self.consume_peeked(); Ok(out) } - /// [Identifier] = [`Identifier`](Type::Identifier) + /// [Identifier] = [`Identifier`](TokenKind::Identifier) pub fn identifier(&mut self) -> PResult { - let tok = self.match_type(Type::Identifier, Parsing::Identifier)?; + let tok = self.match_type(TokenKind::Identifier, Parsing::Identifier)?; match tok.data() { - Data::Identifier(ident) => Ok(ident.into()), + TokenData::Identifier(ident) => Ok(ident.into()), _ => panic!("Expected token data for {tok:?}"), } } @@ -639,26 +649,26 @@ impl<'t> Parser<'t> { /// /// See also: [Parser::stmt] pub fn stmtkind(&mut self) -> PResult { - Ok(match self.peek_type(Parsing::StmtKind)? { - Type::Semi => StmtKind::Empty, - Type::Let => self.parse_let()?.into(), + Ok(match self.peek_kind(Parsing::StmtKind)? { + TokenKind::Semi => StmtKind::Empty, + TokenKind::Let => self.parse_let()?.into(), item_like!() => self.item()?.into(), _ => self.expr()?.into(), }) } pub fn parse_let(&mut self) -> PResult { - self.match_type(Type::Let, Parsing::Let)?; + self.match_type(TokenKind::Let, Parsing::Let)?; Ok(Let { mutable: self.mutability()?, name: self.identifier()?, - ty: if Ok(Type::Colon) == self.peek_type(Parsing::Let) { + ty: if Ok(TokenKind::Colon) == self.peek_kind(Parsing::Let) { self.consume_peeked(); Some(self.ty()?.into()) } else { None }, - init: if Ok(Type::Eq) == self.peek_type(Parsing::Let) { + init: if Ok(TokenKind::Eq) == self.peek_kind(Parsing::Let) { self.consume_peeked(); Some(self.expr()?.into()) } else { @@ -772,10 +782,10 @@ impl<'t> Parser<'t> { const PARSING: Parsing = Parsing::Call; let callee = self.expr_from(Self::exprkind_index)?; let mut args = vec![]; - while Ok(Type::LParen) == self.peek_type(PARSING) { + while Ok(TokenKind::LParen) == self.peek_kind(PARSING) { self.consume_peeked(); args.push(self.tuple()?); - self.match_type(Type::RParen, PARSING)?; + self.match_type(TokenKind::RParen, PARSING)?; } if args.is_empty() { Ok(callee.kind) @@ -787,31 +797,31 @@ impl<'t> Parser<'t> { pub fn exprkind_index(&mut self) -> PResult { const PARSING: Parsing = Parsing::Index; let head = self.expr_from(Self::exprkind_primary)?; - if Ok(Type::LBrack) != self.peek_type(PARSING) { + if Ok(TokenKind::LBrack) != self.peek_kind(PARSING) { return Ok(head.kind); } let mut indices = vec![]; - while Ok(Type::LBrack) == self.peek_type(PARSING) { + while Ok(TokenKind::LBrack) == self.peek_kind(PARSING) { indices.push(delim(Self::tuple, BRACKETS, PARSING)(self)?.into()); } Ok(Index { head: head.into(), indices }.into()) } /// Delegates to the set of highest-priority rules based on unambiguous pattern matching pub fn exprkind_primary(&mut self) -> PResult { - match self.peek_type(Parsing::Expr)? { - Type::Amp | Type::AmpAmp => self.exprkind_addrof(), - Type::LCurly => self.exprkind_block(), - Type::LBrack => self.exprkind_array(), - Type::LParen => self.exprkind_empty_group_or_tuple(), + match self.peek_kind(Parsing::Expr)? { + TokenKind::Amp | TokenKind::AmpAmp => self.exprkind_addrof(), + TokenKind::LCurly => self.exprkind_block(), + TokenKind::LBrack => self.exprkind_array(), + TokenKind::LParen => self.exprkind_empty_group_or_tuple(), literal_like!() => Ok(self.literal()?.into()), path_like!() => Ok(self.path()?.into()), - Type::If => Ok(self.parse_if()?.into()), - Type::For => Ok(self.parse_for()?.into()), - Type::While => Ok(self.parse_while()?.into()), - Type::Break => Ok(self.parse_break()?.into()), - Type::Return => Ok(self.parse_return()?.into()), - Type::Continue => Ok(self.parse_continue()?.into()), + TokenKind::If => Ok(self.parse_if()?.into()), + TokenKind::For => Ok(self.parse_for()?.into()), + TokenKind::While => Ok(self.parse_while()?.into()), + TokenKind::Break => Ok(self.parse_break()?.into()), + TokenKind::Return => Ok(self.parse_return()?.into()), + TokenKind::Continue => Ok(self.parse_continue()?.into()), _ => Err(self.error(Nothing, Parsing::Expr)), } } @@ -821,10 +831,10 @@ impl<'t> Parser<'t> { /// so they can't be independent subexpressions pub fn exprkind_array(&mut self) -> PResult { const PARSING: Parsing = Parsing::Array; - const START: Type = Type::LBrack; - const END: Type = Type::RBrack; + const START: TokenKind = TokenKind::LBrack; + const END: TokenKind = TokenKind::RBrack; self.match_type(START, PARSING)?; - match self.peek_type(PARSING)? { + match self.peek_kind(PARSING)? { END => { self.consume_peeked(); Ok(Array { values: vec![] }.into()) @@ -835,10 +845,10 @@ impl<'t> Parser<'t> { /// [ArrayRep] = `[` [Expr] `;` [Expr] `]` pub fn exprkind_array_rep(&mut self) -> PResult { const PARSING: Parsing = Parsing::Array; - const END: Type = Type::RBrack; + const END: TokenKind = TokenKind::RBrack; let first = self.expr()?; - let out: ExprKind = match self.peek_type(PARSING)? { - Type::Semi => ArrayRep { + let out: ExprKind = match self.peek_kind(PARSING)? { + TokenKind::Semi => ArrayRep { value: first.into(), repeat: { self.consume_peeked(); @@ -846,12 +856,12 @@ impl<'t> Parser<'t> { }, } .into(), - Type::RBrack => Array { values: vec![first] }.into(), - Type::Comma => Array { + TokenKind::RBrack => Array { values: vec![first] }.into(), + TokenKind::Comma => Array { values: { self.consume_peeked(); let mut out = vec![first]; - out.extend(sep(Self::expr, Type::Comma, END, PARSING)(self)?); + out.extend(sep(Self::expr, TokenKind::Comma, END, PARSING)(self)?); out }, } @@ -867,9 +877,9 @@ impl<'t> Parser<'t> { const PARSING: Parsing = Parsing::AddrOf; let mut count = 0; loop { - match self.peek_type(PARSING)? { - Type::Amp => count += 1, - Type::AmpAmp => count += 2, + match self.peek_kind(PARSING)? { + TokenKind::Amp => count += 1, + TokenKind::AmpAmp => count += 2, _ => break, } self.consume_peeked(); @@ -884,13 +894,13 @@ impl<'t> Parser<'t> { /// /// [ExprKind::Empty] and [Group] are special cases of [Tuple] pub fn exprkind_empty_group_or_tuple(&mut self) -> PResult { - self.match_type(Type::LParen, Parsing::Group)?; - let out = match self.peek_type(Parsing::Group)? { - Type::RParen => Ok(ExprKind::Empty), + self.match_type(TokenKind::LParen, Parsing::Group)?; + let out = match self.peek_kind(Parsing::Group)? { + TokenKind::RParen => Ok(ExprKind::Empty), _ => self.exprkind_group(), }; - match self.peek_type(Parsing::Group) { - Ok(Type::RParen) => self.consume_peeked(), + match self.peek_kind(Parsing::Group) { + Ok(TokenKind::RParen) => self.consume_peeked(), _ => Err(self.error(UnmatchedParentheses, Parsing::Group))?, }; out @@ -898,14 +908,14 @@ impl<'t> Parser<'t> { /// [Group] = `(`([Empty](ExprKind::Empty)|[Expr]|[Tuple])`)` pub fn exprkind_group(&mut self) -> PResult { let first = self.expr()?; - match self.peek_type(Parsing::Group)? { - Type::Comma => { + match self.peek_kind(Parsing::Group)? { + TokenKind::Comma => { let mut exprs = vec![first]; self.consume_peeked(); - while Type::RParen != self.peek_type(Parsing::Tuple)? { + while TokenKind::RParen != self.peek_kind(Parsing::Tuple)? { exprs.push(self.expr()?); - match self.peek_type(Parsing::Tuple)? { - Type::Comma => self.consume_peeked(), + match self.peek_kind(Parsing::Tuple)? { + TokenKind::Comma => self.consume_peeked(), _ => break, }; } @@ -918,22 +928,22 @@ impl<'t> Parser<'t> { /// ## Subexpressions impl<'t> Parser<'t> { - /// [Literal] = [String](Type::String) | [Character](Type::Character) - /// | [Float](Type::Float) (TODO) | [Integer](Type::Integer) | `true` | `false` + /// [Literal] = [String](TokenKind::String) | [Character](TokenKind::Character) + /// | [Float](TokenKind::Float) (TODO) | [Integer](TokenKind::Integer) | `true` | `false` pub fn literal(&mut self) -> PResult { let tok = self.consume(Parsing::Literal)?; // keyword literals true and false match tok.ty() { - Type::True => return Ok(Literal::Bool(true)), - Type::False => return Ok(Literal::Bool(false)), - Type::String | Type::Character | Type::Integer | Type::Float => (), + TokenKind::True => return Ok(Literal::Bool(true)), + TokenKind::False => return Ok(Literal::Bool(false)), + TokenKind::String | TokenKind::Character | TokenKind::Integer | TokenKind::Float => (), t => return Err(self.error(Unexpected(t), Parsing::Literal)), } Ok(match tok.data() { - Data::String(v) => Literal::from(v.as_str()), - Data::Character(v) => Literal::from(*v), - Data::Integer(v) => Literal::from(*v), - Data::Float(v) => todo!("Literal::Float({v})"), + TokenData::String(v) => Literal::from(v.as_str()), + TokenData::Character(v) => Literal::from(*v), + TokenData::Integer(v) => Literal::from(*v), + TokenData::Float(v) => todo!("Literal::Float({v})"), _ => panic!("Expected token data for {tok:?}"), }) } @@ -946,8 +956,8 @@ impl<'t> Parser<'t> { Err(e) => return Err(e), } { exprs.push(expr); - match self.peek_type(Parsing::Tuple)? { - Type::Comma => self.consume_peeked(), + match self.peek_kind(Parsing::Tuple)? { + TokenKind::Comma => self.consume_peeked(), _ => break, }; } @@ -963,22 +973,22 @@ impl<'t> Parser<'t> { impl<'t> Parser<'t> { /// [Break] = `break` [Expr]? pub fn parse_break(&mut self) -> PResult { - self.match_type(Type::Break, Parsing::Break)?; + self.match_type(TokenKind::Break, Parsing::Break)?; Ok(Break { body: self.optional_expr()?.map(Into::into) }) } /// [Return] = `return` [Expr]? pub fn parse_return(&mut self) -> PResult { - self.match_type(Type::Return, Parsing::Return)?; + self.match_type(TokenKind::Return, Parsing::Return)?; Ok(Return { body: self.optional_expr()?.map(Into::into) }) } /// [Continue] = `continue` pub fn parse_continue(&mut self) -> PResult { - self.match_type(Type::Continue, Parsing::Continue)?; + self.match_type(TokenKind::Continue, Parsing::Continue)?; Ok(Continue) } /// [While] = `while` [Expr] [Block] [Else]? pub fn parse_while(&mut self) -> PResult { - self.match_type(Type::While, Parsing::While)?; + self.match_type(TokenKind::While, Parsing::While)?; Ok(While { cond: self.expr()?.into(), pass: self.block()?.into(), @@ -988,7 +998,7 @@ impl<'t> Parser<'t> { /// [If] = `if` [Expr] [Block] [Else]? #[rustfmt::skip] // second line is barely not long enough pub fn parse_if(&mut self) -> PResult { - self.match_type(Type::If, Parsing::If)?; + self.match_type(TokenKind::If, Parsing::If)?; Ok(If { cond: self.expr()?.into(), pass: self.block()?.into(), @@ -997,9 +1007,9 @@ impl<'t> Parser<'t> { } /// [For]: `for` Pattern (TODO) `in` [Expr] [Block] [Else]? pub fn parse_for(&mut self) -> PResult { - self.match_type(Type::For, Parsing::For)?; + self.match_type(TokenKind::For, Parsing::For)?; let bind = self.identifier()?; - self.match_type(Type::In, Parsing::For)?; + self.match_type(TokenKind::In, Parsing::For)?; Ok(For { bind, cond: self.expr()?.into(), @@ -1009,8 +1019,8 @@ impl<'t> Parser<'t> { } /// [Else]: (`else` [Block])? pub fn parse_else(&mut self) -> PResult { - match self.peek_type(Parsing::Else) { - Ok(Type::Else) => { + match self.peek_kind(Parsing::Else) { + Ok(TokenKind::Else) => { self.consume_peeked(); Ok(self.expr()?.into()) } @@ -1023,8 +1033,8 @@ impl<'t> Parser<'t> { macro operator($($name:ident ($returns:ident) {$($t:ident => $p:ident),*$(,)?};)*) {$( pub fn $name (&mut self) -> PResult<$returns> { const PARSING: Parsing = Parsing::$returns; - let out = Ok(match self.peek_type(PARSING) { - $(Ok(Type::$t) => $returns::$p,)* + let out = Ok(match self.peek_kind(PARSING) { + $(Ok(TokenKind::$t) => $returns::$p,)* Err(e) => Err(e)?, Ok(t) => Err(self.error(Unexpected(t), PARSING))?, }); @@ -1095,7 +1105,7 @@ impl<'t> Parser<'t> { pub fn member_op(&mut self) -> PResult<()> { const PARSING: Parsing = Parsing::Member; match self.peek(PARSING)?.ty() { - Type::Dot => {} + TokenKind::Dot => {} t => Err(self.error(Unexpected(t), PARSING))?, } self.consume_peeked(); diff --git a/cl-token/src/lib.rs b/cl-token/src/lib.rs index 6296c75..b248953 100644 --- a/cl-token/src/lib.rs +++ b/cl-token/src/lib.rs @@ -1,6 +1,6 @@ //! # Token //! -//! Stores a component of a file as a [Type], some [Data], and a line and column number +//! Stores a component of a file as a [TokenKind], some [TokenData], and a line and column number #![warn(clippy::all)] #![feature(decl_macro)] @@ -9,5 +9,5 @@ pub mod token_data; pub mod token_type; pub use token::Token; -pub use token_data::Data; -pub use token_type::Type; +pub use token_data::TokenData; +pub use token_type::TokenKind; diff --git a/cl-token/src/token.rs b/cl-token/src/token.rs index 5675688..ed69b71 100644 --- a/cl-token/src/token.rs +++ b/cl-token/src/token.rs @@ -1,34 +1,34 @@ -//! A [Token] contains a single unit of lexical information, and an optional bit of [Data] -use super::{Data, Type}; +//! A [Token] contains a single unit of lexical information, and an optional bit of [TokenData] +use super::{TokenData, TokenKind}; /// Contains a single unit of lexical information, -/// and an optional bit of [Data] +/// and an optional bit of [TokenData] #[derive(Clone, Debug, PartialEq)] pub struct Token { - ty: Type, - data: Data, + ty: TokenKind, + data: TokenData, line: u32, col: u32, } impl Token { - /// Creates a new [Token] out of a [Type], [Data], line, and column. - pub fn new(ty: Type, data: impl Into, line: u32, col: u32) -> Self { + /// Creates a new [Token] out of a [TokenKind], [TokenData], line, and column. + pub fn new(ty: TokenKind, data: impl Into, line: u32, col: u32) -> Self { Self { ty, data: data.into(), line, col } } - /// Casts this token to a new [Type] - pub fn cast(self, ty: Type) -> Self { + /// Casts this token to a new [TokenKind] + pub fn cast(self, ty: TokenKind) -> Self { Self { ty, ..self } } - /// Returns the [Type] of this token - pub fn ty(&self) -> Type { + /// Returns the [TokenKind] of this token + pub fn ty(&self) -> TokenKind { self.ty } - /// Returns a reference to this token's [Data] - pub fn data(&self) -> &Data { + /// Returns a reference to this token's [TokenData] + pub fn data(&self) -> &TokenData { &self.data } - /// Converts this token into its inner [Data] - pub fn into_data(self) -> Data { + /// Converts this token into its inner [TokenData] + pub fn into_data(self) -> TokenData { self.data } /// Returns the line where this token originated diff --git a/cl-token/src/token_data.rs b/cl-token/src/token_data.rs index 5f54d63..da47282 100644 --- a/cl-token/src/token_data.rs +++ b/cl-token/src/token_data.rs @@ -1,9 +1,9 @@ //! Additional data stored within a [Token](super::Token), -//! external to its [Type](super::token_type::Type) +//! external to its [TokenKind](super::token_type::TokenKind) /// Additional data stored within a [Token](super::Token), -/// external to its [Type](super::token_type::Type) +/// external to its [TokenKind](super::token_type::TokenKind) #[derive(Clone, Debug, PartialEq)] -pub enum Data { +pub enum TokenData { /// [Token](super::Token) contains an [identifier](str) Identifier(Box), /// [Token](super::Token) contains a [String] @@ -27,19 +27,19 @@ from! { } /// Implements [From] for an enum macro from($($value:ident: $src:ty => $dst:expr),*$(,)?) { - $(impl From<$src> for Data { + $(impl From<$src> for TokenData { fn from($value: $src) -> Self { $dst } })* } -impl std::fmt::Display for Data { +impl std::fmt::Display for TokenData { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Data::Identifier(v) => v.fmt(f), - Data::String(v) => write!(f, "\"{v}\""), - Data::Character(v) => write!(f, "'{v}'"), - Data::Integer(v) => v.fmt(f), - Data::Float(v) => v.fmt(f), - Data::None => "None".fmt(f), + TokenData::Identifier(v) => v.fmt(f), + TokenData::String(v) => write!(f, "\"{v}\""), + TokenData::Character(v) => write!(f, "'{v}'"), + TokenData::Integer(v) => v.fmt(f), + TokenData::Float(v) => v.fmt(f), + TokenData::None => "None".fmt(f), } } } diff --git a/cl-token/src/token_type.rs b/cl-token/src/token_type.rs index 2694e0b..d499f21 100644 --- a/cl-token/src/token_type.rs +++ b/cl-token/src/token_type.rs @@ -3,7 +3,7 @@ use std::{fmt::Display, str::FromStr}; /// Stores a [Token's](super::Token) lexical information #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum Type { +pub enum TokenKind { // Invalid syntax Invalid, // Any kind of comment @@ -98,102 +98,102 @@ pub enum Type { XorXor, // ^^ } -impl Display for Type { +impl Display for TokenKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Type::Invalid => "invalid".fmt(f), - Type::Comment => "comment".fmt(f), - Type::Identifier => "identifier".fmt(f), + TokenKind::Invalid => "invalid".fmt(f), + TokenKind::Comment => "comment".fmt(f), + TokenKind::Identifier => "identifier".fmt(f), - Type::Break => "break".fmt(f), - Type::Cl => "cl".fmt(f), - Type::Const => "const".fmt(f), - Type::Continue => "continue".fmt(f), - Type::Else => "else".fmt(f), - Type::Enum => "enum".fmt(f), - Type::False => "false".fmt(f), - Type::For => "for".fmt(f), - Type::Fn => "fn".fmt(f), - Type::If => "if".fmt(f), - Type::Impl => "impl".fmt(f), - Type::In => "in".fmt(f), - Type::Let => "let".fmt(f), - Type::Mod => "mod".fmt(f), - Type::Mut => "mut".fmt(f), - Type::Pub => "pub".fmt(f), - Type::Return => "return".fmt(f), - Type::SelfKw => "self".fmt(f), - Type::SelfTy => "Self".fmt(f), - Type::Static => "static".fmt(f), - Type::Struct => "struct".fmt(f), - Type::Super => "super".fmt(f), - Type::True => "true".fmt(f), - Type::Type => "type".fmt(f), - Type::While => "while".fmt(f), + TokenKind::Break => "break".fmt(f), + TokenKind::Cl => "cl".fmt(f), + TokenKind::Const => "const".fmt(f), + TokenKind::Continue => "continue".fmt(f), + TokenKind::Else => "else".fmt(f), + TokenKind::Enum => "enum".fmt(f), + TokenKind::False => "false".fmt(f), + TokenKind::For => "for".fmt(f), + TokenKind::Fn => "fn".fmt(f), + TokenKind::If => "if".fmt(f), + TokenKind::Impl => "impl".fmt(f), + TokenKind::In => "in".fmt(f), + TokenKind::Let => "let".fmt(f), + TokenKind::Mod => "mod".fmt(f), + TokenKind::Mut => "mut".fmt(f), + TokenKind::Pub => "pub".fmt(f), + TokenKind::Return => "return".fmt(f), + TokenKind::SelfKw => "self".fmt(f), + TokenKind::SelfTy => "Self".fmt(f), + TokenKind::Static => "static".fmt(f), + TokenKind::Struct => "struct".fmt(f), + TokenKind::Super => "super".fmt(f), + TokenKind::True => "true".fmt(f), + TokenKind::Type => "type".fmt(f), + TokenKind::While => "while".fmt(f), - Type::Integer => "integer literal".fmt(f), - Type::Float => "float literal".fmt(f), - Type::String => "string literal".fmt(f), - Type::Character => "char literal".fmt(f), + TokenKind::Integer => "integer literal".fmt(f), + TokenKind::Float => "float literal".fmt(f), + TokenKind::String => "string literal".fmt(f), + TokenKind::Character => "char literal".fmt(f), - Type::LCurly => "left curly".fmt(f), - Type::RCurly => "right curly".fmt(f), - Type::LBrack => "left brack".fmt(f), - Type::RBrack => "right brack".fmt(f), - Type::LParen => "left paren".fmt(f), - Type::RParen => "right paren".fmt(f), - Type::Amp => "and".fmt(f), - Type::AmpAmp => "and-and".fmt(f), - Type::AmpEq => "and-assign".fmt(f), - Type::Arrow => "arrow".fmt(f), - Type::At => "at".fmt(f), - Type::Backslash => "backslash".fmt(f), - Type::Bang => "bang".fmt(f), - Type::BangBang => "not-not".fmt(f), - Type::BangEq => "not equal to".fmt(f), - Type::Bar => "or".fmt(f), - Type::BarBar => "or-or".fmt(f), - Type::BarEq => "or-assign".fmt(f), - Type::Colon => "colon".fmt(f), - Type::ColonColon => "path separator".fmt(f), - Type::Comma => "comma".fmt(f), - Type::Dot => "dot".fmt(f), - Type::DotDot => "exclusive range".fmt(f), - Type::DotDotEq => "inclusive range".fmt(f), - Type::Eq => "assign".fmt(f), - Type::EqEq => "equal to".fmt(f), - Type::FatArrow => "fat arrow".fmt(f), - Type::Grave => "grave".fmt(f), - Type::Gt => "greater than".fmt(f), - Type::GtEq => "greater than or equal to".fmt(f), - Type::GtGt => "shift right".fmt(f), - Type::GtGtEq => "shift right-assign".fmt(f), - Type::Hash => "hash".fmt(f), - Type::HashBang => "shebang".fmt(f), - Type::Lt => "less than".fmt(f), - Type::LtEq => "less than or equal to".fmt(f), - Type::LtLt => "shift left".fmt(f), - Type::LtLtEq => "shift left-assign".fmt(f), - Type::Minus => "sub".fmt(f), - Type::MinusEq => "sub-assign".fmt(f), - Type::Plus => "add".fmt(f), - Type::PlusEq => "add-assign".fmt(f), - Type::Question => "huh?".fmt(f), - Type::Rem => "rem".fmt(f), - Type::RemEq => "rem-assign".fmt(f), - Type::Semi => "ignore".fmt(f), - Type::Slash => "div".fmt(f), - Type::SlashEq => "div-assign".fmt(f), - Type::Star => "star".fmt(f), - Type::StarEq => "star-assign".fmt(f), - Type::Tilde => "tilde".fmt(f), - Type::Xor => "xor".fmt(f), - Type::XorEq => "xor-assign".fmt(f), - Type::XorXor => "cat-ears".fmt(f), + TokenKind::LCurly => "left curly".fmt(f), + TokenKind::RCurly => "right curly".fmt(f), + TokenKind::LBrack => "left brack".fmt(f), + TokenKind::RBrack => "right brack".fmt(f), + TokenKind::LParen => "left paren".fmt(f), + TokenKind::RParen => "right paren".fmt(f), + TokenKind::Amp => "and".fmt(f), + TokenKind::AmpAmp => "and-and".fmt(f), + TokenKind::AmpEq => "and-assign".fmt(f), + TokenKind::Arrow => "arrow".fmt(f), + TokenKind::At => "at".fmt(f), + TokenKind::Backslash => "backslash".fmt(f), + TokenKind::Bang => "bang".fmt(f), + TokenKind::BangBang => "not-not".fmt(f), + TokenKind::BangEq => "not equal to".fmt(f), + TokenKind::Bar => "or".fmt(f), + TokenKind::BarBar => "or-or".fmt(f), + TokenKind::BarEq => "or-assign".fmt(f), + TokenKind::Colon => "colon".fmt(f), + TokenKind::ColonColon => "path separator".fmt(f), + TokenKind::Comma => "comma".fmt(f), + TokenKind::Dot => "dot".fmt(f), + TokenKind::DotDot => "exclusive range".fmt(f), + TokenKind::DotDotEq => "inclusive range".fmt(f), + TokenKind::Eq => "assign".fmt(f), + TokenKind::EqEq => "equal to".fmt(f), + TokenKind::FatArrow => "fat arrow".fmt(f), + TokenKind::Grave => "grave".fmt(f), + TokenKind::Gt => "greater than".fmt(f), + TokenKind::GtEq => "greater than or equal to".fmt(f), + TokenKind::GtGt => "shift right".fmt(f), + TokenKind::GtGtEq => "shift right-assign".fmt(f), + TokenKind::Hash => "hash".fmt(f), + TokenKind::HashBang => "shebang".fmt(f), + TokenKind::Lt => "less than".fmt(f), + TokenKind::LtEq => "less than or equal to".fmt(f), + TokenKind::LtLt => "shift left".fmt(f), + TokenKind::LtLtEq => "shift left-assign".fmt(f), + TokenKind::Minus => "sub".fmt(f), + TokenKind::MinusEq => "sub-assign".fmt(f), + TokenKind::Plus => "add".fmt(f), + TokenKind::PlusEq => "add-assign".fmt(f), + TokenKind::Question => "huh?".fmt(f), + TokenKind::Rem => "rem".fmt(f), + TokenKind::RemEq => "rem-assign".fmt(f), + TokenKind::Semi => "ignore".fmt(f), + TokenKind::Slash => "div".fmt(f), + TokenKind::SlashEq => "div-assign".fmt(f), + TokenKind::Star => "star".fmt(f), + TokenKind::StarEq => "star-assign".fmt(f), + TokenKind::Tilde => "tilde".fmt(f), + TokenKind::Xor => "xor".fmt(f), + TokenKind::XorEq => "xor-assign".fmt(f), + TokenKind::XorXor => "cat-ears".fmt(f), } } } -impl FromStr for Type { +impl FromStr for TokenKind { /// [FromStr] can only fail when an identifier isn't a keyword type Err = (); /// Parses a string s to return a Keyword