From fc3cbbf4505bc3db7364242da6cd6e03549dd6ca Mon Sep 17 00:00:00 2001 From: John Date: Sat, 13 Apr 2024 03:33:26 -0500 Subject: [PATCH] Conlang v0.0.5: Pratternization cl-token: - Minimize data redundancy by consolidating TokenKind::Literal; TokenData::{String, Identifier} - Rename Op to Punct cl-ast: - Remove ExprKind::{Member, Call} in favor of making them 'binary' operators - Consolidate boxes (TODO: consolidate more boxes) - Remove repetition vecs in favor of boxes (this may come with performance tradeoffs!) cl-lexer: - Reflect changes from cl-token cl-interpret, cl-repl/src/examples: - Reflect changes from cl-ast cl-parser: - Switch to Pratt parsing for expressions - TODO: Code cleanup - TODO: Use total ordering for Precedence instead of binding powers (that's what the binding powers are there for anyway) - Switch functional parsers to take Punct instead of TokenKind - It's not like we need a `for`-separated list - Remove `binary` macro. No longer needed with precedence climbing. - Repurpose `operator` macro to produce both the operator and the respective Precedence - Remove several of the smaller parser functions, since they've been consolidated into the larger `exprkind` --- Cargo.toml | 2 +- cl-ast/src/ast_impl.rs | 75 +--- cl-ast/src/lib.rs | 53 +-- cl-interpret/src/interpret.rs | 172 ++++---- cl-lexer/src/lib.rs | 124 +++--- cl-lexer/src/tests.rs | 5 +- cl-parser/src/parser.rs | 782 ++++++++++++++++------------------ cl-repl/examples/yaml.rs | 58 +-- cl-token/src/lib.rs | 2 +- cl-token/src/token_data.rs | 4 - cl-token/src/token_type.rs | 137 +++--- 11 files changed, 636 insertions(+), 778 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ff79b6d..6359944 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ resolver = "2" [workspace.package] repository = "https://git.soft.fish/j/Conlang" -version = "0.0.4" +version = "0.0.5" authors = ["John Breaux "] edition = "2021" license = "MIT" diff --git a/cl-ast/src/ast_impl.rs b/cl-ast/src/ast_impl.rs index de31185..bb2a0f6 100644 --- a/cl-ast/src/ast_impl.rs +++ b/cl-ast/src/ast_impl.rs @@ -5,7 +5,10 @@ mod display { //! Implements [Display] for [AST](super::super) Types use super::*; pub use delimiters::*; - use std::fmt::{Display, Write}; + use std::{ + borrow::Borrow, + fmt::{Display, Write}, + }; mod delimiters { #![allow(dead_code)] #[derive(Clone, Copy, Debug)] @@ -307,13 +310,16 @@ mod display { impl Display for Expr { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self.kind { + self.kind.fmt(f) + } + } + impl Display for ExprKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { ExprKind::Assign(v) => v.fmt(f), ExprKind::Binary(v) => v.fmt(f), ExprKind::Unary(v) => v.fmt(f), ExprKind::Index(v) => v.fmt(f), - ExprKind::Call(v) => v.fmt(f), - ExprKind::Member(v) => v.fmt(f), ExprKind::Path(v) => v.fmt(f), ExprKind::Literal(v) => v.fmt(f), ExprKind::Array(v) => v.fmt(f), @@ -334,8 +340,8 @@ mod display { } impl Display for Assign { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let Self { head, op, tail } = self; - write!(f, "{head} {op} {tail}") + let Self { kind, parts } = self; + write!(f, "{} {kind} {}", parts.0, parts.1) } } impl Display for AssignKind { @@ -358,12 +364,13 @@ mod display { } impl Display for Binary { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let Self { head, tail } = self; - write!(f, "{head}")?; - for (kind, expr) in tail { - write!(f, " {kind} {expr}")?; + let Self { kind, parts } = self; + let (head, tail) = parts.borrow(); + match kind { + BinaryKind::Dot => write!(f, "{head}{kind}{tail}"), + BinaryKind::Call => write!(f, "{head}{tail}"), + _ => write!(f, "{head} {kind} {tail}"), } - Ok(()) } } impl Display for BinaryKind { @@ -391,17 +398,15 @@ mod display { BinaryKind::Div => "/", BinaryKind::Rem => "%", BinaryKind::Dot => ".", + BinaryKind::Call => "()", } .fmt(f) } } impl Display for Unary { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let Self { ops: kinds, tail } = self; - for kind in kinds { - kind.fmt(f)? - } - tail.fmt(f) + let Self { kind, tail } = self; + write!(f, "{kind}{tail}") } } impl Display for UnaryKind { @@ -416,29 +421,11 @@ mod display { .fmt(f) } } - impl Display for Call { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let Self { callee, args } = self; - callee.fmt(f)?; - for args in args { - args.fmt(f)?; - } - Ok(()) - } - } impl Display for Tuple { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { delimit(separate(&self.exprs, ", "), INLINE_PARENS)(f) } } - impl Display for Member { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let Self { head: parent, tail: children } = self; - write!(f, "{parent}.")?; - separate(children, ".")(f)?; - Ok(()) - } - } impl Display for Index { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let Self { head, indices } = self; @@ -449,11 +436,6 @@ mod display { Ok(()) } } - impl Display for Indices { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - delimit(separate(&self.exprs, ", "), INLINE_SQUARE)(f) - } - } impl Display for Path { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let Self { absolute, parts } = self; @@ -624,8 +606,6 @@ mod convert { Assign => ExprKind::Assign, Binary => ExprKind::Binary, Unary => ExprKind::Unary, - Call => ExprKind::Call, - Member => ExprKind::Member, Index => ExprKind::Index, Path => ExprKind::Path, Literal => ExprKind::Literal, @@ -646,18 +626,7 @@ mod convert { bool => Literal::Bool, char => Literal::Char, u128 => Literal::Int, - &str => Literal::String, - } - } - - impl From for Indices { - fn from(value: Tuple) -> Self { - Self { exprs: value.exprs } - } - } - impl From for Tuple { - fn from(value: Indices) -> Self { - Self { exprs: value.exprs } + String => Literal::String, } } diff --git a/cl-ast/src/lib.rs b/cl-ast/src/lib.rs index af890ac..be9ac3f 100644 --- a/cl-ast/src/lib.rs +++ b/cl-ast/src/lib.rs @@ -324,10 +324,6 @@ pub enum ExprKind { Binary(Binary), /// A [Unary] expression: [`UnaryKind`]\* [`Expr`] Unary(Unary), - /// A [Member] access expression: [`Expr`] (`.` [`Expr`])+ - Member(Member), - /// A [Call] expression, with arguments: a(foo, bar) - Call(Call), /// An Array [Index] expression: a[10, 20, 30] Index(Index), /// A [path expression](Path): `::`? [PathPart] (`::` [PathPart])* @@ -366,9 +362,8 @@ pub enum ExprKind { /// An [Assign]ment expression: [`Expr`] ([`AssignKind`] [`Expr`])\+ #[derive(Clone, Debug, PartialEq, Eq)] pub struct Assign { - pub head: Box, - pub op: AssignKind, - pub tail: Box, + pub kind: AssignKind, + pub parts: Box<(ExprKind, ExprKind)>, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -390,8 +385,8 @@ pub enum AssignKind { /// A [Binary] expression: [`Expr`] ([`BinaryKind`] [`Expr`])\+ #[derive(Clone, Debug, PartialEq, Eq)] pub struct Binary { - pub head: Box, - pub tail: Vec<(BinaryKind, Expr)>, + pub kind: BinaryKind, + pub parts: Box<(ExprKind, ExprKind)>, } /// A [Binary] operator @@ -419,17 +414,18 @@ pub enum BinaryKind { Div, Rem, Dot, + Call, } /// A [Unary] expression: [`UnaryKind`]\* [`Expr`] #[derive(Clone, Debug, PartialEq, Eq)] pub struct Unary { - pub ops: Vec, - pub tail: Box, + pub kind: UnaryKind, + pub tail: Box, } /// A [Unary] operator -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum UnaryKind { Deref, Neg, @@ -439,32 +435,11 @@ pub enum UnaryKind { /// Unused Tilde, } - -/// A [Member] access expression: [`Expr`] (`.` [`Expr`])+ -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Member { - pub head: Box, - pub tail: Vec, -} - -/// A [Call] expression, with arguments: a(foo, bar) -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Call { - pub callee: Box, - pub args: Vec, -} - /// A repeated [Index] expression: a[10, 20, 30][40, 50, 60] #[derive(Clone, Debug, PartialEq, Eq)] pub struct Index { - pub head: Box, - pub indices: Vec, -} - -/// A single [Index] expression: a[10, 20, 30] -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Indices { - pub exprs: Vec, + pub head: Box, + pub indices: Vec, } /// A [Literal]: 0x42, 1e123, 2.4, "Hello" @@ -486,8 +461,8 @@ pub struct Array { /// `[` [Expr] `;` [Literal] `]` #[derive(Clone, Debug, PartialEq, Eq)] pub struct ArrayRep { - pub value: Box, - pub repeat: Box, + pub value: Box, + pub repeat: Box, } /// An address-of expression: `&` `mut`? [`Expr`] @@ -495,7 +470,7 @@ pub struct ArrayRep { pub struct AddrOf { pub count: usize, pub mutable: Mutability, - pub expr: Box, + pub expr: Box, } /// A [Block] expression: `{` [`Stmt`]\* [`Expr`]? `}` @@ -507,7 +482,7 @@ pub struct Block { /// A [Grouping](Group) expression `(` [`Expr`] `)` #[derive(Clone, Debug, PartialEq, Eq)] pub struct Group { - pub expr: Box, + pub expr: Box, } /// A [Tuple] expression: `(` [`Expr`] (`,` [`Expr`])+ `)` diff --git a/cl-interpret/src/interpret.rs b/cl-interpret/src/interpret.rs index 1afb039..4bac82b 100644 --- a/cl-interpret/src/interpret.rs +++ b/cl-interpret/src/interpret.rs @@ -5,6 +5,8 @@ //! meaningless to get a pointer to one, and would be undefined behavior to dereference a pointer to //! one in any situation. +use std::borrow::Borrow; + use super::*; use cl_ast::*; /// A work-in-progress tree walk interpreter for Conlang @@ -107,14 +109,18 @@ impl Interpret for Let { } } impl Interpret for Expr { + #[inline] fn interpret(&self, env: &mut Environment) -> IResult { let Self { extents: _, kind } = self; - match kind { + kind.interpret(env) + } +} +impl Interpret for ExprKind { + fn interpret(&self, env: &mut Environment) -> IResult { + match self { ExprKind::Assign(v) => v.interpret(env), ExprKind::Binary(v) => v.interpret(env), ExprKind::Unary(v) => v.interpret(env), - ExprKind::Member(v) => v.interpret(env), - ExprKind::Call(v) => v.interpret(env), ExprKind::Index(v) => v.interpret(env), ExprKind::Path(v) => v.interpret(env), ExprKind::Literal(v) => v.interpret(env), @@ -136,9 +142,10 @@ impl Interpret for Expr { } impl Interpret for Assign { fn interpret(&self, env: &mut Environment) -> IResult { - let Assign { head, op, tail } = self; + let Assign { kind: op, parts } = self; + let (head, tail) = parts.borrow(); // Resolve the head pattern - let head = match &head.kind { + let head = match &head { ExprKind::Path(Path { parts, .. }) if parts.len() == 1 => { match parts.last().expect("parts should not be empty") { PathPart::SuperKw => Err(Error::NotAssignable)?, @@ -146,8 +153,6 @@ impl Interpret for Assign { PathPart::Ident(Identifier(s)) => s, } } - ExprKind::Member(_) => todo!("Member access assignment"), - ExprKind::Call(_) => todo!("Assignment to the result of a function call?"), ExprKind::Index(_) => todo!("Assignment to an index operation"), ExprKind::Path(_) => todo!("Path expression resolution (IMPORTANT)"), ExprKind::Empty | ExprKind::Group(_) | ExprKind::Tuple(_) => { @@ -193,106 +198,87 @@ impl Interpret for Assign { } impl Interpret for Binary { fn interpret(&self, env: &mut Environment) -> IResult { - let Binary { head, tail } = self; - let mut head = head.interpret(env)?; + let Binary { kind, parts } = self; + let (head, tail) = parts.borrow(); + + let head = head.interpret(env)?; + // Short-circuiting ops - for (op, tail) in tail { - match op { - BinaryKind::LogAnd => { - if head.truthy()? { - head = tail.interpret(env)?; - continue; - } - return Ok(head); // Short circuiting - } - BinaryKind::LogOr => { - if !head.truthy()? { - head = tail.interpret(env)?; - continue; - } - return Ok(head); // Short circuiting - } - BinaryKind::LogXor => { - head = ConValue::Bool(head.truthy()? ^ tail.interpret(env)?.truthy()?); - continue; - } - _ => {} + match kind { + BinaryKind::LogAnd => { + return if head.truthy()? { + tail.interpret(env) + } else { + Ok(head) + }; // Short circuiting } - let tail = tail.interpret(env)?; - head = match op { - BinaryKind::Mul => env.call("mul", &[head, tail]), - BinaryKind::Div => env.call("div", &[head, tail]), - BinaryKind::Rem => env.call("rem", &[head, tail]), - BinaryKind::Add => env.call("add", &[head, tail]), - BinaryKind::Sub => env.call("sub", &[head, tail]), - BinaryKind::Shl => env.call("shl", &[head, tail]), - BinaryKind::Shr => env.call("shr", &[head, tail]), - BinaryKind::BitAnd => env.call("and", &[head, tail]), - BinaryKind::BitOr => env.call("or", &[head, tail]), - BinaryKind::BitXor => env.call("xor", &[head, tail]), - BinaryKind::RangeExc => env.call("range_exc", &[head, tail]), - BinaryKind::RangeInc => env.call("range_inc", &[head, tail]), - BinaryKind::Lt => env.call("lt", &[head, tail]), - BinaryKind::LtEq => env.call("lt_eq", &[head, tail]), - BinaryKind::Equal => env.call("eq", &[head, tail]), - BinaryKind::NotEq => env.call("neq", &[head, tail]), - BinaryKind::GtEq => env.call("gt_eq", &[head, tail]), - BinaryKind::Gt => env.call("gt", &[head, tail]), - BinaryKind::Dot => todo!("search within a type's namespace!"), - _ => Ok(head), - }?; + BinaryKind::LogOr => { + return if !head.truthy()? { + tail.interpret(env) + } else { + Ok(head) + }; // Short circuiting + } + BinaryKind::LogXor => { + return Ok(ConValue::Bool( + head.truthy()? ^ tail.interpret(env)?.truthy()?, + )); + } + _ => {} + } + let tail = tail.interpret(env)?; + match kind { + BinaryKind::Mul => env.call("mul", &[head, tail]), + BinaryKind::Div => env.call("div", &[head, tail]), + BinaryKind::Rem => env.call("rem", &[head, tail]), + BinaryKind::Add => env.call("add", &[head, tail]), + BinaryKind::Sub => env.call("sub", &[head, tail]), + BinaryKind::Shl => env.call("shl", &[head, tail]), + BinaryKind::Shr => env.call("shr", &[head, tail]), + BinaryKind::BitAnd => env.call("and", &[head, tail]), + BinaryKind::BitOr => env.call("or", &[head, tail]), + BinaryKind::BitXor => env.call("xor", &[head, tail]), + BinaryKind::RangeExc => env.call("range_exc", &[head, tail]), + BinaryKind::RangeInc => env.call("range_inc", &[head, tail]), + BinaryKind::Lt => env.call("lt", &[head, tail]), + BinaryKind::LtEq => env.call("lt_eq", &[head, tail]), + BinaryKind::Equal => env.call("eq", &[head, tail]), + BinaryKind::NotEq => env.call("neq", &[head, tail]), + BinaryKind::GtEq => env.call("gt_eq", &[head, tail]), + BinaryKind::Gt => env.call("gt", &[head, tail]), + BinaryKind::Dot => todo!("search within a type's namespace!"), + BinaryKind::Call => match tail { + ConValue::Empty => head.call(env, &[]), + ConValue::Tuple(args) => head.call(env, &args), + _ => Err(Error::TypeError), + }, + _ => Ok(head), } - Ok(head) } } + impl Interpret for Unary { fn interpret(&self, env: &mut Environment) -> IResult { - let Unary { tail, ops } = self; - let mut operand = tail.interpret(env)?; - - for op in ops.iter().rev() { - operand = match op { - UnaryKind::Deref => env.call("deref", &[operand])?, - UnaryKind::Neg => env.call("neg", &[operand])?, - UnaryKind::Not => env.call("not", &[operand])?, - UnaryKind::At => { - println!("{operand}"); - operand - } - UnaryKind::Tilde => unimplemented!("Tilde operator"), - }; + let Unary { kind, tail } = self; + let operand = tail.interpret(env)?; + match kind { + UnaryKind::Deref => env.call("deref", &[operand]), + UnaryKind::Neg => env.call("neg", &[operand]), + UnaryKind::Not => env.call("not", &[operand]), + UnaryKind::At => { + println!("{operand}"); + Ok(operand) + } + UnaryKind::Tilde => unimplemented!("Tilde operator"), } - Ok(operand) - } -} -impl Interpret for Member { - fn interpret(&self, env: &mut Environment) -> IResult { - todo!("Interpret member accesses in {env}") - } -} -impl Interpret for Call { - fn interpret(&self, env: &mut Environment) -> IResult { - let Self { callee, args } = self; - // evaluate the callee - let mut callee = callee.interpret(env)?; - for args in args { - let ConValue::Tuple(args) = args.interpret(env)? else { - Err(Error::TypeError)? - }; - callee = callee.call(env, &args)?; - } - Ok(callee) } } impl Interpret for Index { fn interpret(&self, env: &mut Environment) -> IResult { let Self { head, indices } = self; let mut head = head.interpret(env)?; - for indices in indices { - let Indices { exprs } = indices; - for index in exprs { - head = head.index(&index.interpret(env)?)?; - } + for index in indices { + head = head.index(&index.interpret(env)?)?; } Ok(head) } diff --git a/cl-lexer/src/lib.rs b/cl-lexer/src/lib.rs index afb1287..399ac4f 100644 --- a/cl-lexer/src/lib.rs +++ b/cl-lexer/src/lib.rs @@ -2,7 +2,7 @@ #![warn(clippy::all)] #![feature(decl_macro)] use cl_structures::span::Loc; -use cl_token::{token_type::Op, TokenKind as Kind, *}; +use cl_token::{TokenKind as Kind, *}; use std::{ iter::Peekable, str::{Chars, FromStr}, @@ -97,33 +97,33 @@ impl<'t> Lexer<'t> { /// Scans through the text, searching for the next [Token] pub fn scan(&mut self) -> LResult { match self.skip_whitespace().peek()? { - '{' => self.consume()?.produce_op(Op::LCurly), - '}' => self.consume()?.produce_op(Op::RCurly), - '[' => self.consume()?.produce_op(Op::LBrack), - ']' => self.consume()?.produce_op(Op::RBrack), - '(' => self.consume()?.produce_op(Op::LParen), - ')' => self.consume()?.produce_op(Op::RParen), + '{' => self.consume()?.produce_op(Punct::LCurly), + '}' => self.consume()?.produce_op(Punct::RCurly), + '[' => self.consume()?.produce_op(Punct::LBrack), + ']' => self.consume()?.produce_op(Punct::RBrack), + '(' => self.consume()?.produce_op(Punct::LParen), + ')' => self.consume()?.produce_op(Punct::RParen), '&' => self.consume()?.amp(), - '@' => self.consume()?.produce_op(Op::At), - '\\' => self.consume()?.produce_op(Op::Backslash), + '@' => self.consume()?.produce_op(Punct::At), + '\\' => self.consume()?.produce_op(Punct::Backslash), '!' => self.consume()?.bang(), '|' => self.consume()?.bar(), ':' => self.consume()?.colon(), - ',' => self.consume()?.produce_op(Op::Comma), + ',' => self.consume()?.produce_op(Punct::Comma), '.' => self.consume()?.dot(), '=' => self.consume()?.equal(), - '`' => self.consume()?.produce_op(Op::Grave), + '`' => self.consume()?.produce_op(Punct::Grave), '>' => self.consume()?.greater(), '#' => self.consume()?.hash(), '<' => self.consume()?.less(), '-' => self.consume()?.minus(), '+' => self.consume()?.plus(), - '?' => self.consume()?.produce_op(Op::Question), + '?' => self.consume()?.produce_op(Punct::Question), '%' => self.consume()?.rem(), - ';' => self.consume()?.produce_op(Op::Semi), + ';' => self.consume()?.produce_op(Punct::Semi), '/' => self.consume()?.slash(), '*' => self.consume()?.star(), - '~' => self.consume()?.produce_op(Op::Tilde), + '~' => self.consume()?.produce_op(Punct::Tilde), '^' => self.consume()?.xor(), '0' => self.consume()?.int_with_base(), '1'..='9' => self.digits::<10>(), @@ -163,8 +163,8 @@ impl<'t> Lexer<'t> { self.start = self.current; Ok(Token::new(kind, data, loc.0, loc.1)) } - fn produce_op(&mut self, kind: Op) -> LResult { - self.produce(TokenKind::Op(kind), ()) + fn produce_op(&mut self, kind: Punct) -> LResult { + self.produce(TokenKind::Punct(kind), ()) } fn skip_whitespace(&mut self) -> &mut Self { while let Ok(c) = self.peek() { @@ -195,120 +195,120 @@ impl<'t> Lexer<'t> { impl<'t> Lexer<'t> { fn amp(&mut self) -> LResult { match self.peek() { - Ok('&') => self.consume()?.produce_op(Op::AmpAmp), - Ok('=') => self.consume()?.produce_op(Op::AmpEq), - _ => self.produce_op(Op::Amp), + Ok('&') => self.consume()?.produce_op(Punct::AmpAmp), + Ok('=') => self.consume()?.produce_op(Punct::AmpEq), + _ => self.produce_op(Punct::Amp), } } fn bang(&mut self) -> LResult { match self.peek() { - Ok('!') => self.consume()?.produce_op(Op::BangBang), - Ok('=') => self.consume()?.produce_op(Op::BangEq), - _ => self.produce_op(Op::Bang), + Ok('!') => self.consume()?.produce_op(Punct::BangBang), + Ok('=') => self.consume()?.produce_op(Punct::BangEq), + _ => self.produce_op(Punct::Bang), } } fn bar(&mut self) -> LResult { match self.peek() { - Ok('|') => self.consume()?.produce_op(Op::BarBar), - Ok('=') => self.consume()?.produce_op(Op::BarEq), - _ => self.produce_op(Op::Bar), + Ok('|') => self.consume()?.produce_op(Punct::BarBar), + Ok('=') => self.consume()?.produce_op(Punct::BarEq), + _ => self.produce_op(Punct::Bar), } } fn colon(&mut self) -> LResult { match self.peek() { - Ok(':') => self.consume()?.produce_op(Op::ColonColon), - _ => self.produce_op(Op::Colon), + Ok(':') => self.consume()?.produce_op(Punct::ColonColon), + _ => self.produce_op(Punct::Colon), } } fn dot(&mut self) -> LResult { match self.peek() { Ok('.') => { if let Ok('=') = self.consume()?.peek() { - self.consume()?.produce_op(Op::DotDotEq) + self.consume()?.produce_op(Punct::DotDotEq) } else { - self.produce_op(Op::DotDot) + self.produce_op(Punct::DotDot) } } - _ => self.produce_op(Op::Dot), + _ => self.produce_op(Punct::Dot), } } fn equal(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce_op(Op::EqEq), - Ok('>') => self.consume()?.produce_op(Op::FatArrow), - _ => self.produce_op(Op::Eq), + Ok('=') => self.consume()?.produce_op(Punct::EqEq), + Ok('>') => self.consume()?.produce_op(Punct::FatArrow), + _ => self.produce_op(Punct::Eq), } } fn greater(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce_op(Op::GtEq), + Ok('=') => self.consume()?.produce_op(Punct::GtEq), Ok('>') => { if let Ok('=') = self.consume()?.peek() { - self.consume()?.produce_op(Op::GtGtEq) + self.consume()?.produce_op(Punct::GtGtEq) } else { - self.produce_op(Op::GtGt) + self.produce_op(Punct::GtGt) } } - _ => self.produce_op(Op::Gt), + _ => self.produce_op(Punct::Gt), } } fn hash(&mut self) -> LResult { match self.peek() { - Ok('!') => self.consume()?.produce_op(Op::HashBang), - _ => self.produce_op(Op::Hash), + Ok('!') => self.consume()?.produce_op(Punct::HashBang), + _ => self.produce_op(Punct::Hash), } } fn less(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce_op(Op::LtEq), + Ok('=') => self.consume()?.produce_op(Punct::LtEq), Ok('<') => { if let Ok('=') = self.consume()?.peek() { - self.consume()?.produce_op(Op::LtLtEq) + self.consume()?.produce_op(Punct::LtLtEq) } else { - self.produce_op(Op::LtLt) + self.produce_op(Punct::LtLt) } } - _ => self.produce_op(Op::Lt), + _ => self.produce_op(Punct::Lt), } } fn minus(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce_op(Op::MinusEq), - Ok('>') => self.consume()?.produce_op(Op::Arrow), - _ => self.produce_op(Op::Minus), + Ok('=') => self.consume()?.produce_op(Punct::MinusEq), + Ok('>') => self.consume()?.produce_op(Punct::Arrow), + _ => self.produce_op(Punct::Minus), } } fn plus(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce_op(Op::PlusEq), - _ => self.produce_op(Op::Plus), + Ok('=') => self.consume()?.produce_op(Punct::PlusEq), + _ => self.produce_op(Punct::Plus), } } fn rem(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce_op(Op::RemEq), - _ => self.produce_op(Op::Rem), + Ok('=') => self.consume()?.produce_op(Punct::RemEq), + _ => self.produce_op(Punct::Rem), } } fn slash(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce_op(Op::SlashEq), + Ok('=') => self.consume()?.produce_op(Punct::SlashEq), Ok('/') => self.consume()?.line_comment(), Ok('*') => self.consume()?.block_comment(), - _ => self.produce_op(Op::Slash), + _ => self.produce_op(Punct::Slash), } } fn star(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce_op(Op::StarEq), - _ => self.produce_op(Op::Star), + Ok('=') => self.consume()?.produce_op(Punct::StarEq), + _ => self.produce_op(Punct::Star), } } fn xor(&mut self) -> LResult { match self.peek() { - Ok('=') => self.consume()?.produce_op(Op::XorEq), - Ok('^') => self.consume()?.produce_op(Op::XorXor), - _ => self.produce_op(Op::Xor), + Ok('=') => self.consume()?.produce_op(Punct::XorEq), + Ok('^') => self.consume()?.produce_op(Punct::XorXor), + _ => self.produce_op(Punct::Xor), } } } @@ -339,7 +339,7 @@ impl<'t> Lexer<'t> { if let Ok(keyword) = Kind::from_str(&out) { self.produce(keyword, ()) } else { - self.produce(Kind::Identifier, TokenData::Identifier(out.into())) + self.produce(Kind::Identifier, TokenData::String(out)) } } fn xid_start(&mut self) -> LResult { @@ -370,7 +370,7 @@ impl<'t> Lexer<'t> { Ok('o') => self.consume()?.digits::<8>(), Ok('b') => self.consume()?.digits::<2>(), Ok('0'..='9') => self.digits::<10>(), - _ => self.produce(Kind::Integer, 0), + _ => self.produce(Kind::Literal, 0), } } fn digits(&mut self) -> LResult { @@ -378,7 +378,7 @@ impl<'t> Lexer<'t> { while let Ok(true) = self.peek().as_ref().map(char::is_ascii_alphanumeric) { value = value * B as u128 + self.digit::()? as u128; } - self.produce(Kind::Integer, value) + self.produce(Kind::Literal, value) } fn digit(&mut self) -> LResult { let digit = self.peek()?; @@ -399,12 +399,12 @@ impl<'t> Lexer<'t> { { value.push(self.unescape()?) } - self.consume()?.produce(Kind::String, value) + self.consume()?.produce(Kind::Literal, value) } fn character(&mut self) -> LResult { let out = self.unescape()?; match self.peek()? { - '\'' => self.consume()?.produce(Kind::Character, out), + '\'' => self.consume()?.produce(Kind::Literal, out), _ => Err(Error::unmatched_delimiters('\'', self.line(), self.col())), } } diff --git a/cl-lexer/src/tests.rs b/cl-lexer/src/tests.rs index 465fb24..cb82cd6 100644 --- a/cl-lexer/src/tests.rs +++ b/cl-lexer/src/tests.rs @@ -35,7 +35,7 @@ macro td ($($id:expr),*) { mod ident { use super::*; macro ident ($($id:literal),*) { - [$(TokenData::Identifier($id.into())),*] + [$(TokenData::String($id.into())),*] } test_lexer_data_type! { underscore { "_ _" => ident!["_", "_"] } @@ -109,9 +109,8 @@ mod string { } } mod punct { - use cl_token::token_type::Op; macro op($op:ident) { - TokenKind::Op(Op::$op) + TokenKind::Punct(Punct::$op) } use super::*; diff --git a/cl-parser/src/parser.rs b/cl-parser/src/parser.rs index 9cb0ccb..862ed5f 100644 --- a/cl-parser/src/parser.rs +++ b/cl-parser/src/parser.rs @@ -6,7 +6,6 @@ use crate::error::{ }; use cl_ast::*; use cl_lexer::Lexer; -use cl_token::token_type::Op; /// Parses a sequence of [Tokens](Token) into an [AST](cl_ast) #[derive(Debug)] @@ -85,29 +84,29 @@ impl<'t> Parser<'t> { } } #[inline] - pub fn match_op(&mut self, want: Op, while_parsing: Parsing) -> PResult { - self.match_type(TokenKind::Op(want), while_parsing) + pub fn match_op(&mut self, want: Punct, while_parsing: Parsing) -> PResult { + self.match_type(TokenKind::Punct(want), while_parsing) } } // the three matched delimiter pairs /// Square brackets: `[` `]` -const BRACKETS: (TokenKind, TokenKind) = (TokenKind::Op(Op::LBrack), TokenKind::Op(Op::RBrack)); +const BRACKETS: (Punct, Punct) = (Punct::LBrack, Punct::RBrack); /// Curly braces: `{` `}` -const CURLIES: (TokenKind, TokenKind) = (TokenKind::Op(Op::LCurly), TokenKind::Op(Op::RCurly)); +const CURLIES: (Punct, Punct) = (Punct::LCurly, Punct::RCurly); /// Parentheses: `(` `)` -const PARENS: (TokenKind, TokenKind) = (TokenKind::Op(Op::LParen), TokenKind::Op(Op::RParen)); +const PARENS: (Punct, Punct) = (Punct::LParen, Punct::RParen); /// Parses constructions of the form `delim.0 f delim.1` (i.e. `(` `foobar` `)`) const fn delim<'t, T>( f: impl Fn(&mut Parser<'t>) -> PResult, - delim: (TokenKind, TokenKind), + delim: (Punct, Punct), while_parsing: Parsing, ) -> impl Fn(&mut Parser<'t>) -> PResult { move |parser| { - parser.match_type(delim.0, while_parsing)?; + parser.match_op(delim.0, while_parsing)?; let out = f(parser)?; - parser.match_type(delim.1, while_parsing)?; + parser.match_op(delim.1, while_parsing)?; Ok(out) } } @@ -117,15 +116,15 @@ const fn delim<'t, T>( /// where `~until` is a negative lookahead assertion const fn sep<'t, T>( f: impl Fn(&mut Parser<'t>) -> PResult, - sep: TokenKind, - until: TokenKind, + sep: Punct, + until: Punct, while_parsing: Parsing, ) -> impl Fn(&mut Parser<'t>) -> PResult> { move |parser| { let mut args = vec![]; - while until != parser.peek_kind(while_parsing)? { + while TokenKind::Punct(until) != parser.peek_kind(while_parsing)? { args.push(f(parser)?); - if sep != parser.peek_kind(while_parsing)? { + if TokenKind::Punct(sep) != parser.peek_kind(while_parsing)? { break; } parser.consume_peeked(); @@ -140,12 +139,12 @@ const fn sep<'t, T>( #[allow(dead_code)] const fn rep<'t, T>( f: impl Fn(&mut Parser<'t>) -> PResult, - until: TokenKind, + until: Punct, while_parsing: Parsing, ) -> impl Fn(&mut Parser<'t>) -> PResult> { move |parser| { let mut out = vec![]; - while until != parser.peek_kind(while_parsing)? { + while TokenKind::Punct(until) != parser.peek_kind(while_parsing)? { out.push(f(parser)?) } Ok(out) @@ -154,7 +153,7 @@ const fn rep<'t, T>( /// Expands to a pattern which matches item-like [Token] [TokenKind]s macro item_like() { - TokenKind::Op(Op::Hash) + TokenKind::Punct(Punct::Hash) | TokenKind::Pub | TokenKind::Type | TokenKind::Const @@ -172,7 +171,7 @@ impl<'t> Parser<'t> { pub fn file(&mut self) -> PResult { let mut items = vec![]; while match self.peek_kind(Parsing::File) { - Ok(TokenKind::Op(Op::RCurly)) | Err(Error { reason: EndOfInput, .. }) => false, + Ok(TokenKind::Punct(Punct::RCurly)) | Err(Error { reason: EndOfInput, .. }) => false, Ok(_) => true, Err(e) => Err(e)?, } { @@ -207,13 +206,16 @@ impl<'t> Parser<'t> { /// See also: [Parser::path_part], [Parser::identifier] pub fn path(&mut self) -> PResult { const PARSING: Parsing = Parsing::PathExpr; - let absolute = matches!(self.peek_kind(PARSING)?, TokenKind::Op(Op::ColonColon)); + let absolute = matches!( + self.peek_kind(PARSING)?, + TokenKind::Punct(Punct::ColonColon) + ); if absolute { self.consume_peeked(); } let mut parts = vec![self.path_part()?]; - while let Ok(TokenKind::Op(Op::ColonColon)) = self.peek_kind(PARSING) { + while let Ok(TokenKind::Punct(Punct::ColonColon)) = self.peek_kind(PARSING) { self.consume_peeked(); parts.push(self.path_part()?); } @@ -229,7 +231,7 @@ impl<'t> Parser<'t> { Ok(Stmt { kind: self.stmtkind()?, semi: match self.peek_kind(PARSING) { - Ok(TokenKind::Op(Op::Semi)) => { + Ok(TokenKind::Punct(Punct::Semi)) => { self.consume_peeked(); Semi::Terminated } @@ -243,7 +245,7 @@ impl<'t> Parser<'t> { /// /// See also: [Parser::exprkind] pub fn expr(&mut self) -> PResult { - self.expr_from(Self::exprkind) + self.expr_from(|this| this.exprkind(0)) } } @@ -251,16 +253,11 @@ impl<'t> Parser<'t> { impl<'t> Parser<'t> { /// Parses an [attribute set](Attrs) pub fn attributes(&mut self) -> PResult { - if self.match_op(Op::Hash, Parsing::Attrs).is_err() { + if self.match_op(Punct::Hash, Parsing::Attrs).is_err() { return Ok(Attrs { meta: vec![] }); } let meta = delim( - sep( - Self::meta, - TokenKind::Op(Op::Comma), - BRACKETS.1, - Parsing::Attrs, - ), + sep(Self::meta, Punct::Comma, BRACKETS.1, Parsing::Attrs), BRACKETS, Parsing::Attrs, ); @@ -272,16 +269,16 @@ impl<'t> Parser<'t> { pub fn meta_kind(&mut self) -> PResult { const PARSING: Parsing = Parsing::Meta; let lit_tuple = delim( - sep(Self::literal, TokenKind::Op(Op::Comma), PARENS.1, PARSING), + sep(Self::literal, Punct::Comma, PARENS.1, PARSING), PARENS, PARSING, ); Ok(match self.peek_kind(PARSING) { - Ok(TokenKind::Op(Op::Eq)) => { + Ok(TokenKind::Punct(Punct::Eq)) => { self.consume_peeked(); MetaKind::Equals(self.literal()?) } - Ok(TokenKind::Op(Op::LParen)) => MetaKind::Func(lit_tuple(self)?), + Ok(TokenKind::Punct(Punct::LParen)) => MetaKind::Func(lit_tuple(self)?), _ => MetaKind::Plain, }) } @@ -311,13 +308,13 @@ impl<'t> Parser<'t> { self.match_type(TokenKind::Type, PARSING)?; let out = Ok(Alias { to: self.identifier()?, - from: if self.match_op(Op::Eq, PARSING).is_ok() { + from: if self.match_op(Punct::Eq, PARSING).is_ok() { Some(self.ty()?.into()) } else { None }, }); - self.match_op(Op::Semi, PARSING)?; + self.match_op(Punct::Semi, PARSING)?; out } @@ -327,15 +324,15 @@ impl<'t> Parser<'t> { let out = Ok(Const { name: self.identifier()?, ty: { - self.match_op(Op::Colon, PARSING)?; + self.match_op(Punct::Colon, PARSING)?; self.ty()?.into() }, init: { - self.match_op(Op::Eq, PARSING)?; + self.match_op(Punct::Eq, PARSING)?; self.expr()?.into() }, }); - self.match_op(Op::Semi, PARSING)?; + self.match_op(Punct::Semi, PARSING)?; out } pub fn parse_static(&mut self) -> PResult { @@ -345,15 +342,15 @@ impl<'t> Parser<'t> { mutable: self.mutability()?, name: self.identifier()?, ty: { - self.match_op(Op::Colon, PARSING)?; + self.match_op(Punct::Colon, PARSING)?; self.ty()?.into() }, init: { - self.match_op(Op::Eq, PARSING)?; + self.match_op(Punct::Eq, PARSING)?; self.expr()?.into() }, }); - self.match_op(Op::Semi, PARSING)?; + self.match_op(Punct::Semi, PARSING)?; out } pub fn parse_module(&mut self) -> PResult { @@ -366,12 +363,15 @@ impl<'t> Parser<'t> { let inline = delim(Self::file, CURLIES, PARSING); match self.peek_kind(PARSING)? { - TokenKind::Op(Op::LCurly) => Ok(ModuleKind::Inline(inline(self)?)), - TokenKind::Op(Op::Semi) => { + TokenKind::Punct(Punct::LCurly) => Ok(ModuleKind::Inline(inline(self)?)), + TokenKind::Punct(Punct::Semi) => { self.consume_peeked(); Ok(ModuleKind::Outline) } - got => Err(self.error(Expected { want: TokenKind::Op(Op::Semi), got }, PARSING)), + got => Err(self.error( + Expected { want: TokenKind::Punct(Punct::Semi), got }, + PARSING, + )), } } pub fn parse_function(&mut self) -> PResult { @@ -381,16 +381,19 @@ impl<'t> Parser<'t> { name: self.identifier()?, args: self.parse_params()?, rety: match self.peek_kind(PARSING)? { - TokenKind::Op(Op::LCurly) | TokenKind::Op(Op::Semi) => None, - TokenKind::Op(Op::Arrow) => { + TokenKind::Punct(Punct::LCurly) | TokenKind::Punct(Punct::Semi) => None, + TokenKind::Punct(Punct::Arrow) => { self.consume_peeked(); Some(self.ty()?.into()) } - got => Err(self.error(Expected { want: TokenKind::Op(Op::Arrow), got }, PARSING))?, + got => Err(self.error( + Expected { want: TokenKind::Punct(Punct::Arrow), got }, + PARSING, + ))?, }, body: match self.peek_kind(PARSING)? { - TokenKind::Op(Op::LCurly) => Some(self.block()?), - TokenKind::Op(Op::Semi) => { + TokenKind::Punct(Punct::LCurly) => Some(self.block()?), + TokenKind::Punct(Punct::Semi) => { self.consume_peeked(); None } @@ -401,12 +404,7 @@ impl<'t> Parser<'t> { pub fn parse_params(&mut self) -> PResult> { const PARSING: Parsing = Parsing::Function; delim( - sep( - Self::parse_param, - TokenKind::Op(Op::Comma), - PARENS.1, - PARSING, - ), + sep(Self::parse_param, Punct::Comma, PARENS.1, PARSING), PARENS, PARSING, )(self) @@ -416,7 +414,7 @@ impl<'t> Parser<'t> { mutability: self.mutability()?, name: self.identifier()?, ty: { - self.match_op(Op::Colon, Parsing::Param)?; + self.match_op(Punct::Colon, Parsing::Param)?; self.ty()?.into() }, }) @@ -427,13 +425,16 @@ impl<'t> Parser<'t> { Ok(Struct { name: self.identifier()?, kind: match self.peek_kind(PARSING)? { - TokenKind::Op(Op::LParen) => self.structkind_tuple()?, - TokenKind::Op(Op::LCurly) => self.structkind_struct()?, - TokenKind::Op(Op::Semi) => { + TokenKind::Punct(Punct::LParen) => self.structkind_tuple()?, + TokenKind::Punct(Punct::LCurly) => self.structkind_struct()?, + TokenKind::Punct(Punct::Semi) => { self.consume_peeked(); StructKind::Empty } - got => Err(self.error(Expected { want: TokenKind::Op(Op::Semi), got }, PARSING))?, + got => Err(self.error( + Expected { want: TokenKind::Punct(Punct::Semi), got }, + PARSING, + ))?, }, }) } @@ -441,7 +442,7 @@ impl<'t> Parser<'t> { const PARSING: Parsing = Parsing::StructKind; Ok(StructKind::Tuple(delim( - sep(Self::ty, TokenKind::Op(Op::Comma), PARENS.1, PARSING), + sep(Self::ty, Punct::Comma, PARENS.1, PARSING), PARENS, PARSING, )(self)?)) @@ -449,12 +450,7 @@ impl<'t> Parser<'t> { pub fn structkind_struct(&mut self) -> PResult { const PARSING: Parsing = Parsing::StructKind; Ok(StructKind::Struct(delim( - sep( - Self::struct_member, - TokenKind::Op(Op::Comma), - CURLIES.1, - PARSING, - ), + sep(Self::struct_member, Punct::Comma, CURLIES.1, PARSING), CURLIES, PARSING, )(self)?)) @@ -465,7 +461,7 @@ impl<'t> Parser<'t> { vis: self.visibility()?, name: self.identifier()?, ty: { - self.match_op(Op::Colon, PARSING)?; + self.match_op(Punct::Colon, PARSING)?; self.ty()? }, }) @@ -477,17 +473,12 @@ impl<'t> Parser<'t> { Ok(Enum { name: self.identifier()?, kind: match self.peek_kind(PARSING)? { - TokenKind::Op(Op::LCurly) => EnumKind::Variants(delim( - sep( - Self::enum_variant, - TokenKind::Op(Op::Comma), - TokenKind::Op(Op::RCurly), - PARSING, - ), + TokenKind::Punct(Punct::LCurly) => EnumKind::Variants(delim( + sep(Self::enum_variant, Punct::Comma, Punct::RCurly, PARSING), CURLIES, PARSING, )(self)?), - TokenKind::Op(Op::Semi) => { + TokenKind::Punct(Punct::Semi) => { self.consume_peeked(); EnumKind::NoVariants } @@ -501,17 +492,17 @@ impl<'t> Parser<'t> { Ok(Variant { name: self.identifier()?, kind: match self.peek_kind(PARSING)? { - TokenKind::Op(Op::Eq) => self.variantkind_clike()?, - TokenKind::Op(Op::LCurly) => self.variantkind_struct()?, - TokenKind::Op(Op::LParen) => self.variantkind_tuple()?, + TokenKind::Punct(Punct::Eq) => self.variantkind_clike()?, + TokenKind::Punct(Punct::LCurly) => self.variantkind_struct()?, + TokenKind::Punct(Punct::LParen) => self.variantkind_tuple()?, _ => VariantKind::Plain, }, }) } pub fn variantkind_clike(&mut self) -> PResult { const PARSING: Parsing = Parsing::VariantKind; - self.match_op(Op::Eq, PARSING)?; - let tok = self.match_type(TokenKind::Integer, PARSING)?; + self.match_op(Punct::Eq, PARSING)?; + let tok = self.match_type(TokenKind::Literal, PARSING)?; Ok(VariantKind::CLike(match tok.data() { TokenData::Integer(i) => *i, _ => panic!("Expected token data for {tok:?} while parsing {PARSING}"), @@ -520,12 +511,7 @@ impl<'t> Parser<'t> { pub fn variantkind_struct(&mut self) -> PResult { const PARSING: Parsing = Parsing::VariantKind; Ok(VariantKind::Struct(delim( - sep( - Self::struct_member, - TokenKind::Op(Op::Comma), - TokenKind::Op(Op::RCurly), - PARSING, - ), + sep(Self::struct_member, Punct::Comma, Punct::RCurly, PARSING), CURLIES, PARSING, )(self)?)) @@ -533,12 +519,7 @@ impl<'t> Parser<'t> { pub fn variantkind_tuple(&mut self) -> PResult { const PARSING: Parsing = Parsing::VariantKind; Ok(VariantKind::Tuple(delim( - sep( - Self::ty, - TokenKind::Op(Op::Comma), - TokenKind::Op(Op::RParen), - PARSING, - ), + sep(Self::ty, Punct::Comma, Punct::RParen, PARSING), PARENS, PARSING, )(self)?)) @@ -574,7 +555,7 @@ impl<'t> Parser<'t> { pub fn tykind(&mut self) -> PResult { const PARSING: Parsing = Parsing::TyKind; let out = match self.peek_kind(PARSING)? { - TokenKind::Op(Op::Bang) => { + TokenKind::Punct(Punct::Bang) => { self.consume_peeked(); TyKind::Never } @@ -582,8 +563,8 @@ impl<'t> Parser<'t> { self.consume_peeked(); TyKind::SelfTy } - TokenKind::Op(Op::Amp) | TokenKind::Op(Op::AmpAmp) => self.tyref()?.into(), - TokenKind::Op(Op::LParen) => self.tytuple()?.into(), + TokenKind::Punct(Punct::Amp) | TokenKind::Punct(Punct::AmpAmp) => self.tyref()?.into(), + TokenKind::Punct(Punct::LParen) => self.tytuple()?.into(), TokenKind::Fn => self.tyfn()?.into(), path_like!() => self.path()?.into(), t => Err(self.error(Unexpected(t), PARSING))?, @@ -596,7 +577,7 @@ impl<'t> Parser<'t> { const PARSING: Parsing = Parsing::TyTuple; Ok(TyTuple { types: delim( - sep(Self::ty, TokenKind::Op(Op::Comma), PARENS.1, PARSING), + sep(Self::ty, Punct::Comma, PARENS.1, PARSING), PARENS, PARSING, )(self)?, @@ -608,8 +589,8 @@ impl<'t> Parser<'t> { let mut count = 0; loop { match self.peek_kind(PARSING)? { - TokenKind::Op(Op::Amp) => count += 1, - TokenKind::Op(Op::AmpAmp) => count += 2, + TokenKind::Punct(Punct::Amp) => count += 1, + TokenKind::Punct(Punct::AmpAmp) => count += 2, _ => break, } self.consume_peeked(); @@ -624,7 +605,7 @@ impl<'t> Parser<'t> { args: self.tytuple()?, rety: { match self.peek_kind(PARSING)? { - TokenKind::Op(Op::Arrow) => { + TokenKind::Punct(Punct::Arrow) => { self.consume_peeked(); Some(self.ty()?.into()) } @@ -637,16 +618,14 @@ impl<'t> Parser<'t> { /// Expands to a pattern which matches literal-like [TokenKind]s macro literal_like() { - TokenKind::True - | TokenKind::False - | TokenKind::String - | TokenKind::Character - | TokenKind::Integer - | TokenKind::Float + TokenKind::True | TokenKind::False | TokenKind::Literal } /// Expands to a pattern which matches path-like [token Types](Type) macro path_like() { - TokenKind::Super | TokenKind::SelfKw | TokenKind::Identifier | TokenKind::Op(Op::ColonColon) + TokenKind::Super + | TokenKind::SelfKw + | TokenKind::Identifier + | TokenKind::Punct(Punct::ColonColon) } /// # Path parsing impl<'t> Parser<'t> { @@ -666,7 +645,7 @@ impl<'t> Parser<'t> { pub fn identifier(&mut self) -> PResult { let tok = self.match_type(TokenKind::Identifier, Parsing::Identifier)?; match tok.data() { - TokenData::Identifier(ident) => Ok(ident.into()), + TokenData::String(ident) => Ok(ident.into()), _ => panic!("Expected token data for {tok:?}"), } } @@ -679,7 +658,7 @@ impl<'t> Parser<'t> { /// See also: [Parser::stmt] pub fn stmtkind(&mut self) -> PResult { Ok(match self.peek_kind(Parsing::StmtKind)? { - TokenKind::Op(Op::Semi) => StmtKind::Empty, + TokenKind::Punct(Punct::Semi) => StmtKind::Empty, TokenKind::Let => self.parse_let()?.into(), item_like!() => self.item()?.into(), _ => self.expr()?.into(), @@ -691,13 +670,13 @@ impl<'t> Parser<'t> { Ok(Let { mutable: self.mutability()?, name: self.identifier()?, - ty: if Ok(TokenKind::Op(Op::Colon)) == self.peek_kind(Parsing::Let) { + ty: if Ok(TokenKind::Punct(Punct::Colon)) == self.peek_kind(Parsing::Let) { self.consume_peeked(); Some(self.ty()?.into()) } else { None }, - init: if Ok(TokenKind::Op(Op::Eq)) == self.peek_kind(Parsing::Let) { + init: if Ok(TokenKind::Punct(Punct::Eq)) == self.peek_kind(Parsing::Let) { self.consume_peeked(); Some(self.expr()?.into()) } else { @@ -707,31 +686,8 @@ impl<'t> Parser<'t> { } } -macro binary($($name:ident {$lower:ident, $op:ident})*) { -$(pub fn $name(&mut self) -> PResult { - let head = self.expr_from(Self::$lower)?; - let mut tail = vec![]; - loop { - match self.$op() { - Ok(op) => tail.push((op, self.expr_from(Self::$lower)?)), - Err(Error { reason: Unexpected(_) | EndOfInput, ..}) => break, - Err(e) => Err(e)?, - } - } - if tail.is_empty() { - return Ok(head.kind); - } - Ok(Binary { head: head.into(), tail }.into()) -})* -} /// # Expression parsing impl<'t> Parser<'t> { - /// Parses an [ExprKind] - /// - /// See also: [Parser::expr], [Parser::exprkind_primary] - pub fn exprkind(&mut self) -> PResult { - self.exprkind_assign() - } /// Creates an [Expr] with the given [ExprKind]-parser pub fn expr_from(&mut self, f: impl Fn(&mut Self) -> PResult) -> PResult { let start = self.loc(); @@ -745,278 +701,227 @@ impl<'t> Parser<'t> { } } - /// [Assign] = [Path] ([AssignKind] [Assign]) | [Compare](Binary) - pub fn exprkind_assign(&mut self) -> PResult { - let head = self.expr_from(Self::exprkind_compare)?; - // TODO: Formalize the concept of a "place expression" - if !matches!( - head.kind, - ExprKind::Path(_) | ExprKind::Call(_) | ExprKind::Member(_) | ExprKind::Index(_) - ) { - return Ok(head.kind); - } - let Ok(op) = self.assign_op() else { - return Ok(head.kind); + /// Parses an [ExprKind] + /// + /// See also: [Parser::expr] + pub fn exprkind(&mut self, power: u8) -> PResult { + let parsing = Parsing::ExprKind; + // + let mut head = match self.peek_kind(Parsing::Unary)? { + literal_like!() => self.literal()?.into(), + path_like!() => self.path()?.into(), + TokenKind::Punct(Punct::Amp | Punct::AmpAmp) => self.addrof()?.into(), + TokenKind::Punct(Punct::LCurly) => self.block()?.into(), + TokenKind::Punct(Punct::LBrack) => self.exprkind_arraylike()?, + TokenKind::Punct(Punct::LParen) => self.exprkind_tuplelike()?, + TokenKind::Punct(op) => { + let (kind, prec) = from_prefix(op) + .ok_or_else(|| self.error(Unexpected(TokenKind::Punct(op)), parsing))?; + let ((), after) = prec.prefix().expect("should have a precedence"); + self.consume_peeked(); + Unary { kind, tail: self.exprkind(after)?.into() }.into() + } + TokenKind::While => ExprKind::While(self.parse_while()?), + TokenKind::If => ExprKind::If(self.parse_if()?), + TokenKind::For => ExprKind::For(self.parse_for()?), + TokenKind::Break => { + self.consume_peeked(); + Break { body: self.optional_expr()?.map(Into::into) }.into() + } + TokenKind::Return => { + self.consume_peeked(); + Return { body: self.optional_expr()?.map(Into::into) }.into() + } + TokenKind::Continue => { + self.consume_peeked(); + Continue.into() + } + t => Err(self.error(Unexpected(t), Parsing::Unary))?, }; - Ok( - Assign { - head: Box::new(head), - op, - tail: self.expr_from(Self::exprkind_assign)?.into(), - } - .into(), - ) - } - // TODO: use a pratt parser for binary expressions, to simplify this - binary! { - exprkind_compare {exprkind_range, compare_op} - exprkind_range {exprkind_logic, range_op} - exprkind_logic {exprkind_bitwise, logic_op} - exprkind_bitwise {exprkind_shift, bitwise_op} - exprkind_shift {exprkind_factor, shift_op} - exprkind_factor {exprkind_term, factor_op} - exprkind_term {exprkind_unary, term_op} - } - /// [Unary] = [UnaryKind]* [Member] - pub fn exprkind_unary(&mut self) -> PResult { - let mut ops = vec![]; - loop { - match self.unary_op() { - Ok(v) => ops.push(v), - Err(Error { reason: Unexpected(_), .. }) => break, - Err(e) => Err(e)?, - } - } - let tail = self.expr_from(Self::exprkind_member)?; - if ops.is_empty() { - return Ok(tail.kind); - } - Ok(Unary { ops, tail: Box::new(tail) }.into()) - } - /// [Member] = [Call] `.` [Call] - pub fn exprkind_member(&mut self) -> PResult { - let head = self.expr_from(Self::exprkind_call)?; - let mut tail = vec![]; - while self.member_op().is_ok() { - tail.push(self.expr_from(Self::exprkind_call)?) - } - if tail.is_empty() { - Ok(head.kind) - } else { - Ok(Member { head: head.into(), tail }.into()) - } - } - /// Call = [Index] (`(` [Tuple]? `)`)* - pub fn exprkind_call(&mut self) -> PResult { - const PARSING: Parsing = Parsing::Call; - let callee = self.expr_from(Self::exprkind_index)?; - let mut args = vec![]; - while Ok(TokenKind::Op(Op::LParen)) == self.peek_kind(PARSING) { - self.consume_peeked(); - args.push(self.tuple()?); - self.match_op(Op::RParen, PARSING)?; - } - if args.is_empty() { - Ok(callee.kind) - } else { - Ok(Call { callee: callee.into(), args }.into()) - } - } - /// [Index] = [Primary](Parser::exprkind_primary) (`[` [Indices] `]`)* - pub fn exprkind_index(&mut self) -> PResult { - const PARSING: Parsing = Parsing::Index; - let head = self.expr_from(Self::exprkind_primary)?; - if Ok(TokenKind::Op(Op::LBrack)) != self.peek_kind(PARSING) { - return Ok(head.kind); + + fn from_postfix(op: Punct) -> Option { + Some(match op { + Punct::LBrack => Precedence::Index, + Punct::LParen => Precedence::Postfix, + _ => None?, + }) } - let mut indices = vec![]; - while Ok(TokenKind::Op(Op::LBrack)) == self.peek_kind(PARSING) { - indices.push(delim(Self::tuple, BRACKETS, PARSING)(self)?.into()); - } - Ok(Index { head: head.into(), indices }.into()) - } - /// Delegates to the set of highest-priority rules based on unambiguous pattern matching - pub fn exprkind_primary(&mut self) -> PResult { - match self.peek_kind(Parsing::Expr)? { - TokenKind::Op(Op::Amp) | TokenKind::Op(Op::AmpAmp) => self.exprkind_addrof(), - TokenKind::Op(Op::LCurly) => self.exprkind_block(), - TokenKind::Op(Op::LBrack) => self.exprkind_array(), - TokenKind::Op(Op::LParen) => self.exprkind_empty_group_or_tuple(), - literal_like!() => Ok(self.literal()?.into()), - path_like!() => Ok(self.path()?.into()), - TokenKind::If => Ok(self.parse_if()?.into()), - TokenKind::For => Ok(self.parse_for()?.into()), - TokenKind::While => Ok(self.parse_while()?.into()), - TokenKind::Break => Ok(self.parse_break()?.into()), - TokenKind::Return => Ok(self.parse_return()?.into()), - TokenKind::Continue => Ok(self.parse_continue()?.into()), - _ => Err(self.error(Nothing, Parsing::Expr)), + while let Ok(TokenKind::Punct(op)) = self.peek_kind(parsing) { + if let Some((before, ())) = from_postfix(op).and_then(Precedence::postfix) { + if before < power { + break; + } + self.consume_peeked(); + + head = match op { + Punct::LBrack => { + let indices = sep(Self::expr, Punct::Comma, Punct::RBrack, parsing)(self)?; + self.match_op(Punct::RBrack, parsing)?; + ExprKind::Index(Index { head: head.into(), indices }) + } + Punct::LParen => { + let exprs = sep(Self::expr, Punct::Comma, Punct::RParen, parsing)(self)?; + self.match_op(Punct::RParen, parsing)?; + Binary { + kind: BinaryKind::Call, + parts: (head, Tuple { exprs }.into()).into(), + } + .into() + } + _ => Err(self.error(Unexpected(TokenKind::Punct(op)), parsing))?, + }; + continue; + } + // infix expressions + if let Some((kind, prec)) = from_infix(op) { + let (before, after) = prec.infix().expect("should have a precedence"); + if before < power { + break; + } + self.consume_peeked(); + + let tail = self.exprkind(after)?; + head = Binary { kind, parts: (head, tail).into() }.into(); + continue; + } + + if let Some((kind, prec)) = from_assign(op) { + let (before, after) = prec.infix().expect("should have a precedence"); + if before < power { + break; + } + self.consume_peeked(); + + let tail = self.exprkind(after)?; + head = Assign { kind, parts: (head, tail).into() }.into(); + continue; + } + break; } + Ok(head) } + /// [Array] = '[' ([Expr] ',')* [Expr]? ']' /// /// Array and ArrayRef are ambiguous until the second token, /// so they can't be independent subexpressions - pub fn exprkind_array(&mut self) -> PResult { + pub fn exprkind_arraylike(&mut self) -> PResult { const PARSING: Parsing = Parsing::Array; - const START: TokenKind = TokenKind::Op(Op::LBrack); - const END: TokenKind = TokenKind::Op(Op::RBrack); - self.match_type(START, PARSING)?; - match self.peek_kind(PARSING)? { - END => { - self.consume_peeked(); - Ok(Array { values: vec![] }.into()) - } - _ => self.exprkind_array_rep(), - } + const START: Punct = Punct::LBrack; + const END: Punct = Punct::RBrack; + + self.match_op(START, PARSING)?; + let out = match self.peek_kind(PARSING)? { + TokenKind::Punct(END) => Array { values: vec![] }.into(), + _ => self.exprkind_array_rep()?, + }; + self.match_op(END, PARSING)?; + Ok(out) } + /// [ArrayRep] = `[` [Expr] `;` [Expr] `]` pub fn exprkind_array_rep(&mut self) -> PResult { const PARSING: Parsing = Parsing::Array; - const END: TokenKind = TokenKind::Op(Op::RBrack); + const END: Punct = Punct::RBrack; + let first = self.expr()?; - let out: ExprKind = match self.peek_kind(PARSING)? { - TokenKind::Op(Op::Semi) => ArrayRep { - value: first.into(), + Ok(match self.peek_kind(PARSING)? { + TokenKind::Punct(Punct::Semi) => ArrayRep { + value: first.kind.into(), repeat: { self.consume_peeked(); - Box::new(self.expr()?) + Box::new(self.exprkind(0)?) }, } .into(), - TokenKind::Op(Op::RBrack) => Array { values: vec![first] }.into(), - TokenKind::Op(Op::Comma) => Array { + TokenKind::Punct(Punct::RBrack) => Array { values: vec![first] }.into(), + TokenKind::Punct(Punct::Comma) => Array { values: { self.consume_peeked(); let mut out = vec![first]; - out.extend(sep(Self::expr, TokenKind::Op(Op::Comma), END, PARSING)( - self, - )?); + out.extend(sep(Self::expr, Punct::Comma, END, PARSING)(self)?); out }, } .into(), ty => Err(self.error(Unexpected(ty), PARSING))?, - }; - self.match_type(END, PARSING)?; - Ok(out) - } - - /// [AddrOf] = (`&`|`&&`)* [Expr] - pub fn exprkind_addrof(&mut self) -> PResult { - const PARSING: Parsing = Parsing::AddrOf; - let mut count = 0; - loop { - match self.peek_kind(PARSING)? { - TokenKind::Op(Op::Amp) => count += 1, - TokenKind::Op(Op::AmpAmp) => count += 2, - _ => break, - } - self.consume_peeked(); - } - Ok(AddrOf { count, mutable: self.mutability()?, expr: self.expr()?.into() }.into()) - } - /// [Block] = `{` [Stmt]* `}` - pub fn exprkind_block(&mut self) -> PResult { - self.block().map(Into::into) + }) } /// [Group] = `(`([Empty](ExprKind::Empty)|[Expr]|[Tuple])`)` /// /// [ExprKind::Empty] and [Group] are special cases of [Tuple] - pub fn exprkind_empty_group_or_tuple(&mut self) -> PResult { - self.match_op(Op::LParen, Parsing::Group)?; + pub fn exprkind_tuplelike(&mut self) -> PResult { + self.match_op(Punct::LParen, Parsing::Group)?; let out = match self.peek_kind(Parsing::Group)? { - TokenKind::Op(Op::RParen) => Ok(ExprKind::Empty), + TokenKind::Punct(Punct::RParen) => Ok(ExprKind::Empty), _ => self.exprkind_group(), }; - match self.peek_kind(Parsing::Group) { - Ok(TokenKind::Op(Op::RParen)) => self.consume_peeked(), - _ => Err(self.error(UnmatchedParentheses, Parsing::Group))?, - }; + self.match_op(Punct::RParen, Parsing::Group)?; out } /// [Group] = `(`([Empty](ExprKind::Empty)|[Expr]|[Tuple])`)` pub fn exprkind_group(&mut self) -> PResult { let first = self.expr()?; match self.peek_kind(Parsing::Group)? { - TokenKind::Op(Op::Comma) => { + TokenKind::Punct(Punct::Comma) => { let mut exprs = vec![first]; self.consume_peeked(); - while TokenKind::Op(Op::RParen) != self.peek_kind(Parsing::Tuple)? { + while TokenKind::Punct(Punct::RParen) != self.peek_kind(Parsing::Tuple)? { exprs.push(self.expr()?); match self.peek_kind(Parsing::Tuple)? { - TokenKind::Op(Op::Comma) => self.consume_peeked(), + TokenKind::Punct(Punct::Comma) => self.consume_peeked(), _ => break, }; } Ok(Tuple { exprs }.into()) } - _ => Ok(Group { expr: first.into() }.into()), + _ => Ok(Group { expr: first.kind.into() }.into()), } } } /// ## Subexpressions impl<'t> Parser<'t> { - /// [Literal] = [String](TokenKind::String) | [Character](TokenKind::Character) - /// | [Float](TokenKind::Float) (TODO) | [Integer](TokenKind::Integer) | `true` | `false` - pub fn literal(&mut self) -> PResult { - let tok = self.consume(Parsing::Literal)?; - // keyword literals true and false - match tok.ty() { - TokenKind::True => return Ok(Literal::Bool(true)), - TokenKind::False => return Ok(Literal::Bool(false)), - TokenKind::String | TokenKind::Character | TokenKind::Integer | TokenKind::Float => (), - t => return Err(self.error(Unexpected(t), Parsing::Literal)), - } - Ok(match tok.data() { - TokenData::String(v) => Literal::from(v.as_str()), - TokenData::Character(v) => Literal::from(*v), - TokenData::Integer(v) => Literal::from(*v), - TokenData::Float(v) => todo!("Literal::Float({v})"), - _ => panic!("Expected token data for {tok:?}"), - }) - } - /// [Tuple] = ([Expr] `,`)* [Expr]? - pub fn tuple(&mut self) -> PResult { - let mut exprs = vec![]; - while let Some(expr) = match self.expr() { - Ok(v) => Some(v), - Err(Error { reason: Nothing, .. }) => None, - Err(e) => return Err(e), - } { - exprs.push(expr); - match self.peek_kind(Parsing::Tuple)? { - TokenKind::Op(Op::Comma) => self.consume_peeked(), + /// [AddrOf] = (`&`|`&&`)* [Expr] + pub fn addrof(&mut self) -> PResult { + const PARSING: Parsing = Parsing::AddrOf; + let mut count = 0; + loop { + count += match self.peek_kind(PARSING)? { + TokenKind::Punct(Punct::Amp) => 1, + TokenKind::Punct(Punct::AmpAmp) => 2, _ => break, }; + self.consume_peeked(); } - Ok(Tuple { exprs }) + Ok(AddrOf { count, mutable: self.mutability()?, expr: self.exprkind(0)?.into() }) + } + /// [Literal] = [LITERAL](TokenKind::Literal) | `true` | `false` + pub fn literal(&mut self) -> PResult { + let Token { ty, data, .. } = self.consume(Parsing::Literal)?; + match ty { + TokenKind::True => return Ok(Literal::Bool(true)), + TokenKind::False => return Ok(Literal::Bool(false)), + TokenKind::Literal => (), + t => return Err(self.error(Unexpected(t), Parsing::Literal)), + } + Ok(match data { + TokenData::String(v) => Literal::String(v), + TokenData::Character(v) => Literal::Char(v), + TokenData::Integer(v) => Literal::Int(v), + TokenData::Float(v) => todo!("Literal::Float({v})"), + _ => panic!("Expected token data for {ty:?}"), + }) } /// [Block] = `{` [Stmt]* `}` pub fn block(&mut self) -> PResult { - const PARSING: Parsing = Parsing::Block; - Ok(Block { stmts: delim(rep(Self::stmt, CURLIES.1, PARSING), CURLIES, PARSING)(self)? }) + const A_BLOCK: Parsing = Parsing::Block; + Ok(Block { stmts: delim(rep(Self::stmt, CURLIES.1, A_BLOCK), CURLIES, A_BLOCK)(self)? }) } } /// ## Control flow subexpressions impl<'t> Parser<'t> { - /// [Break] = `break` [Expr]? - pub fn parse_break(&mut self) -> PResult { - self.match_type(TokenKind::Break, Parsing::Break)?; - Ok(Break { body: self.optional_expr()?.map(Into::into) }) - } - /// [Return] = `return` [Expr]? - pub fn parse_return(&mut self) -> PResult { - self.match_type(TokenKind::Return, Parsing::Return)?; - Ok(Return { body: self.optional_expr()?.map(Into::into) }) - } - /// [Continue] = `continue` - pub fn parse_continue(&mut self) -> PResult { - self.match_type(TokenKind::Continue, Parsing::Continue)?; - Ok(Continue) - } /// [While] = `while` [Expr] [Block] [Else]? pub fn parse_while(&mut self) -> PResult { self.match_type(TokenKind::While, Parsing::While)?; @@ -1061,85 +966,134 @@ impl<'t> Parser<'t> { } } -macro operator($($name:ident ($returns:ident) {$($t:ident => $p:ident),*$(,)?};)*) {$( -pub fn $name (&mut self) -> PResult<$returns> { - const PARSING: Parsing = Parsing::$returns; - let out = Ok(match self.peek_kind(PARSING) { - $(Ok(TokenKind::Op(Op::$t)) => $returns::$p,)* - Err(e) => Err(e)?, - Ok(t) => Err(self.error(Unexpected(t), PARSING))?, - }); - self.consume_peeked(); - out +/// Precedence provides a total ordering among operators +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Precedence { + Assign, + Compare, + Range, + Index, + Logic, + Bitwise, + Shift, + Factor, + Term, + Unary, + Postfix, + Member, // left-associative } -)*} -/// ## Operator Kinds -impl<'t> Parser<'t> { - operator! { - assign_op (AssignKind) { - Eq => Plain, // = - AmpEq => And, // &= - BarEq => Or, // |= - XorEq => Xor, // ^= - LtLtEq => Shl, // <<= - GtGtEq => Shr, // >>= - PlusEq => Add, // += - MinusEq => Sub, // -= - StarEq => Mul, // *= - SlashEq => Div, // /= - RemEq => Rem, // %= - }; - compare_op (BinaryKind) { - Lt => Lt, // < - LtEq => LtEq, // <= - EqEq => Equal, // == - BangEq => NotEq,// != - GtEq => GtEq, // >= - Gt => Gt, // > - }; - range_op (BinaryKind) { - DotDot => RangeExc, // .. - DotDotEq => RangeInc,// ..= - }; - logic_op (BinaryKind) { - AmpAmp => LogAnd, // && - BarBar => LogOr, // || - XorXor => LogXor, // ^^ - }; - bitwise_op (BinaryKind) { - Amp => BitAnd, // & - Bar => BitOr, // | - Xor => BitXor, // ^ - }; - shift_op (BinaryKind) { - LtLt => Shl, // << - GtGt => Shr, // >> - }; - factor_op (BinaryKind) { - Plus => Add, // + - Minus => Sub, // - - }; - term_op (BinaryKind) { - Star => Mul, // * - Slash => Div, // / - Rem => Rem, // % - }; - unary_op (UnaryKind) { - Star => Deref, // * - Minus => Neg, // - - Bang => Not, // ! - At => At, // @ - Tilde => Tilde, // ~ - }; +impl Precedence { + #[inline] + pub fn level(self) -> u8 { + (self as u8) << 1 } - pub fn member_op(&mut self) -> PResult<()> { - const PARSING: Parsing = Parsing::Member; - match self.peek(PARSING)?.ty() { - TokenKind::Op(Op::Dot) => {} - t => Err(self.error(Unexpected(t), PARSING))?, + pub fn prefix(self) -> Option<((), u8)> { + match self { + Self::Unary => Some(((), self.level())), + _ => None, + } + } + pub fn infix(self) -> Option<(u8, u8)> { + let level = self.level(); + match self { + Self::Unary => None, + Self::Member | Self::Assign => Some((level + 1, level)), + _ => Some((level, level + 1)), + } + } + pub fn postfix(self) -> Option<(u8, ())> { + match self { + Self::Index | Self::Postfix => Some((self.level(), ())), + _ => None, } - self.consume_peeked(); - Ok(()) } } +impl From for Precedence { + fn from(_value: AssignKind) -> Self { + Precedence::Assign + } +} + +impl From for Precedence { + fn from(value: BinaryKind) -> Self { + use BinaryKind as Op; + match value { + Op::Call => Precedence::Postfix, + Op::Dot => Precedence::Member, + Op::Mul | Op::Div | Op::Rem => Precedence::Term, + Op::Add | Op::Sub => Precedence::Factor, + Op::Shl | Op::Shr => Precedence::Shift, + Op::BitAnd | Op::BitOr | Op::BitXor => Precedence::Bitwise, + Op::LogAnd | Op::LogOr | Op::LogXor => Precedence::Logic, + Op::RangeExc | Op::RangeInc => Precedence::Range, + Op::Lt | Op::LtEq | Op::Equal | Op::NotEq | Op::GtEq | Op::Gt => Precedence::Compare, + } + } +} +impl From for Precedence { + fn from(value: UnaryKind) -> Self { + use UnaryKind as Op; + match value { + Op::Deref | Op::Neg | Op::Not | Op::At | Op::Tilde => Precedence::Unary, + } + } +} + +/// Creates helper functions for +macro operator($($name:ident ($takes:ident => $returns:ident) {$($t:ident => $p:ident),*$(,)?};)*) {$( + pub fn $name (value: $takes) -> Option<($returns, Precedence)> { + match value { + $($takes::$t => Some(($returns::$p, Precedence::from($returns::$p))),)* + _ => None?, + } + })* +} + +operator! { + from_prefix (Punct => UnaryKind) { + Star => Deref, + Minus => Neg, + Bang => Not, + At => At, + Tilde => Tilde, + }; + from_assign(Punct => AssignKind) { + Eq => Plain, + AmpEq => And, + BarEq => Or, + XorEq => Xor, + LtLtEq => Shl, + GtGtEq => Shr, + PlusEq => Add, + MinusEq => Sub, + StarEq => Mul, + SlashEq => Div, + RemEq => Rem, + }; + from_infix (Punct => BinaryKind) { + + Lt => Lt, + LtEq => LtEq, + EqEq => Equal, + BangEq => NotEq, + GtEq => GtEq, + Gt => Gt, + DotDot => RangeExc, + DotDotEq => RangeInc, + AmpAmp => LogAnd, + BarBar => LogOr, + XorXor => LogXor, + Amp => BitAnd, + Bar => BitOr, + Xor => BitXor, + LtLt => Shl, + GtGt => Shr, + Plus => Add, + Minus => Sub, + Star => Mul, + Slash => Div, + Rem => Rem, + Dot => Dot, + }; +} diff --git a/cl-repl/examples/yaml.rs b/cl-repl/examples/yaml.rs index 7625393..9027468 100644 --- a/cl-repl/examples/yaml.rs +++ b/cl-repl/examples/yaml.rs @@ -363,8 +363,6 @@ pub mod yamlify { ExprKind::Assign(k) => k.yaml(y), ExprKind::Binary(k) => k.yaml(y), ExprKind::Unary(k) => k.yaml(y), - ExprKind::Member(k) => k.yaml(y), - ExprKind::Call(k) => k.yaml(y), ExprKind::Index(k) => k.yaml(y), ExprKind::Path(k) => k.yaml(y), ExprKind::Literal(k) => k.yaml(y), @@ -386,18 +384,25 @@ pub mod yamlify { } impl Yamlify for Assign { fn yaml(&self, y: &mut Yamler) { - let Self { head, op: _, tail } = self; - y.key("Assign").pair("head", head).pair("tail", tail); + let Self { kind, parts } = self; + y.key("Assign") + .pair("kind", kind) + .pair("head", &parts.0) + .pair("tail", &parts.1); + } + } + impl Yamlify for AssignKind { + fn yaml(&self, y: &mut Yamler) { + y.value(self); } } impl Yamlify for Binary { fn yaml(&self, y: &mut Yamler) { - let Self { head, tail } = self; - let mut y = y.key("Binary"); - y.pair("head", head); - for (op, expr) in tail { - y.key("tail").pair("op", op).pair("expr", expr); - } + let Self { kind, parts } = self; + y.key("Binary") + .pair("kind", kind) + .pair("head", &parts.0) + .pair("tail", &parts.1); } } impl Yamlify for BinaryKind { @@ -407,12 +412,8 @@ pub mod yamlify { } impl Yamlify for Unary { fn yaml(&self, y: &mut Yamler) { - let Self { ops, tail } = self; - let mut y = y.key("Unary"); - for op in ops { - y.pair("op", op); - } - y.pair("tail", tail); + let Self { kind, tail } = self; + y.key("Unary").pair("kind", kind).pair("tail", tail); } } impl Yamlify for UnaryKind { @@ -420,18 +421,6 @@ pub mod yamlify { y.value(self); } } - impl Yamlify for Member { - fn yaml(&self, y: &mut Yamler) { - let Self { head, tail } = self; - y.key("Member").pair("head", head).pair("tail", tail); - } - } - impl Yamlify for Call { - fn yaml(&self, y: &mut Yamler) { - let Self { callee, args } = self; - y.key("Call").pair("callee", callee).pair("args", args); - } - } impl Yamlify for Tuple { fn yaml(&self, y: &mut Yamler) { let Self { exprs } = self; @@ -444,12 +433,6 @@ pub mod yamlify { y.key("Index").pair("head", head).list(indices); } } - impl Yamlify for Indices { - fn yaml(&self, y: &mut Yamler) { - let Self { exprs } = self; - y.key("Indices").list(exprs); - } - } impl Yamlify for Array { fn yaml(&self, y: &mut Yamler) { let Self { values } = self; @@ -466,8 +449,11 @@ pub mod yamlify { } impl Yamlify for AddrOf { fn yaml(&self, y: &mut Yamler) { - let Self { count: _, mutable, expr } = self; - y.key("Addr").yaml(mutable).pair("expr", expr); + let Self { count, mutable, expr } = self; + y.key("AddrOf") + .yaml(mutable) + .pair("count", count) + .pair("expr", expr); } } impl Yamlify for Group { diff --git a/cl-token/src/lib.rs b/cl-token/src/lib.rs index b248953..eb35c6d 100644 --- a/cl-token/src/lib.rs +++ b/cl-token/src/lib.rs @@ -10,4 +10,4 @@ pub mod token_type; pub use token::Token; pub use token_data::TokenData; -pub use token_type::TokenKind; +pub use token_type::{Punct, TokenKind}; diff --git a/cl-token/src/token_data.rs b/cl-token/src/token_data.rs index da47282..f772cdc 100644 --- a/cl-token/src/token_data.rs +++ b/cl-token/src/token_data.rs @@ -4,8 +4,6 @@ /// external to its [TokenKind](super::token_type::TokenKind) #[derive(Clone, Debug, PartialEq)] pub enum TokenData { - /// [Token](super::Token) contains an [identifier](str) - Identifier(Box), /// [Token](super::Token) contains a [String] String(String), /// [Token](super::Token) contains a [character](char) @@ -18,7 +16,6 @@ pub enum TokenData { None, } from! { - value: &str => Self::Identifier(value.into()), value: String => Self::String(value), value: u128 => Self::Integer(value), value: f64 => Self::Float(value), @@ -34,7 +31,6 @@ macro from($($value:ident: $src:ty => $dst:expr),*$(,)?) { impl std::fmt::Display for TokenData { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - TokenData::Identifier(v) => v.fmt(f), TokenData::String(v) => write!(f, "\"{v}\""), TokenData::Character(v) => write!(f, "'{v}'"), TokenData::Integer(v) => v.fmt(f), diff --git a/cl-token/src/token_type.rs b/cl-token/src/token_type.rs index 49bbc27..5563c4b 100644 --- a/cl-token/src/token_type.rs +++ b/cl-token/src/token_type.rs @@ -4,11 +4,13 @@ use std::{fmt::Display, str::FromStr}; /// Stores a [Token's](super::Token) lexical information #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum TokenKind { - // Invalid syntax + /// Invalid sequence Invalid, - // Any kind of comment + /// Any kind of comment Comment, - // A non-keyword identifier + /// Any tokenizable literal (See [TokenData](super::TokenData)) + Literal, + /// A non-keyword identifier Identifier, // A keyword Break, @@ -36,18 +38,13 @@ pub enum TokenKind { True, Type, While, - // Literals - Integer, - Float, - String, - Character, - // Delimiters and punctuation - Op(Op), + /// Delimiter or punctuation + Punct(Punct), } /// An operator character (delimiter, punctuation) #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum Op { +pub enum Punct { LCurly, // { RCurly, // } LBrack, // [ @@ -109,6 +106,7 @@ impl Display for TokenKind { match self { TokenKind::Invalid => "invalid".fmt(f), TokenKind::Comment => "comment".fmt(f), + TokenKind::Literal => "literal".fmt(f), TokenKind::Identifier => "identifier".fmt(f), TokenKind::Break => "break".fmt(f), @@ -137,12 +135,7 @@ impl Display for TokenKind { TokenKind::Type => "type".fmt(f), TokenKind::While => "while".fmt(f), - TokenKind::Integer => "integer literal".fmt(f), - TokenKind::Float => "float literal".fmt(f), - TokenKind::String => "string literal".fmt(f), - TokenKind::Character => "char literal".fmt(f), - - TokenKind::Op(op) => op.fmt(f), + TokenKind::Punct(op) => op.fmt(f), } } } @@ -182,63 +175,63 @@ impl FromStr for TokenKind { } } -impl Display for Op { +impl Display for Punct { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Op::LCurly => "left curly".fmt(f), - Op::RCurly => "right curly".fmt(f), - Op::LBrack => "left brack".fmt(f), - Op::RBrack => "right brack".fmt(f), - Op::LParen => "left paren".fmt(f), - Op::RParen => "right paren".fmt(f), - Op::Amp => "and".fmt(f), - Op::AmpAmp => "and-and".fmt(f), - Op::AmpEq => "and-assign".fmt(f), - Op::Arrow => "arrow".fmt(f), - Op::At => "at".fmt(f), - Op::Backslash => "backslash".fmt(f), - Op::Bang => "bang".fmt(f), - Op::BangBang => "not-not".fmt(f), - Op::BangEq => "not equal to".fmt(f), - Op::Bar => "or".fmt(f), - Op::BarBar => "or-or".fmt(f), - Op::BarEq => "or-assign".fmt(f), - Op::Colon => "colon".fmt(f), - Op::ColonColon => "path separator".fmt(f), - Op::Comma => "comma".fmt(f), - Op::Dot => "dot".fmt(f), - Op::DotDot => "exclusive range".fmt(f), - Op::DotDotEq => "inclusive range".fmt(f), - Op::Eq => "assign".fmt(f), - Op::EqEq => "equal to".fmt(f), - Op::FatArrow => "fat arrow".fmt(f), - Op::Grave => "grave".fmt(f), - Op::Gt => "greater than".fmt(f), - Op::GtEq => "greater than or equal to".fmt(f), - Op::GtGt => "shift right".fmt(f), - Op::GtGtEq => "shift right-assign".fmt(f), - Op::Hash => "hash".fmt(f), - Op::HashBang => "shebang".fmt(f), - Op::Lt => "less than".fmt(f), - Op::LtEq => "less than or equal to".fmt(f), - Op::LtLt => "shift left".fmt(f), - Op::LtLtEq => "shift left-assign".fmt(f), - Op::Minus => "sub".fmt(f), - Op::MinusEq => "sub-assign".fmt(f), - Op::Plus => "add".fmt(f), - Op::PlusEq => "add-assign".fmt(f), - Op::Question => "huh?".fmt(f), - Op::Rem => "rem".fmt(f), - Op::RemEq => "rem-assign".fmt(f), - Op::Semi => "ignore".fmt(f), - Op::Slash => "div".fmt(f), - Op::SlashEq => "div-assign".fmt(f), - Op::Star => "star".fmt(f), - Op::StarEq => "star-assign".fmt(f), - Op::Tilde => "tilde".fmt(f), - Op::Xor => "xor".fmt(f), - Op::XorEq => "xor-assign".fmt(f), - Op::XorXor => "cat-ears".fmt(f), + Punct::LCurly => "{".fmt(f), + Punct::RCurly => "}".fmt(f), + Punct::LBrack => "[".fmt(f), + Punct::RBrack => "]".fmt(f), + Punct::LParen => "(".fmt(f), + Punct::RParen => ")".fmt(f), + Punct::Amp => "&".fmt(f), + Punct::AmpAmp => "&&".fmt(f), + Punct::AmpEq => "&=".fmt(f), + Punct::Arrow => "->".fmt(f), + Punct::At => "@".fmt(f), + Punct::Backslash => "\\".fmt(f), + Punct::Bang => "!".fmt(f), + Punct::BangBang => "!!".fmt(f), + Punct::BangEq => "!=".fmt(f), + Punct::Bar => "|".fmt(f), + Punct::BarBar => "||".fmt(f), + Punct::BarEq => "|=".fmt(f), + Punct::Colon => ":".fmt(f), + Punct::ColonColon => "::".fmt(f), + Punct::Comma => ",".fmt(f), + Punct::Dot => ".".fmt(f), + Punct::DotDot => "..".fmt(f), + Punct::DotDotEq => "..=".fmt(f), + Punct::Eq => "=".fmt(f), + Punct::EqEq => "==".fmt(f), + Punct::FatArrow => "=>".fmt(f), + Punct::Grave => "`".fmt(f), + Punct::Gt => ">".fmt(f), + Punct::GtEq => ">=".fmt(f), + Punct::GtGt => ">>".fmt(f), + Punct::GtGtEq => ">>=".fmt(f), + Punct::Hash => "#".fmt(f), + Punct::HashBang => "#!".fmt(f), + Punct::Lt => "<".fmt(f), + Punct::LtEq => "<=".fmt(f), + Punct::LtLt => "<<".fmt(f), + Punct::LtLtEq => "<<=".fmt(f), + Punct::Minus => "-".fmt(f), + Punct::MinusEq => "-=".fmt(f), + Punct::Plus => "+".fmt(f), + Punct::PlusEq => "+=".fmt(f), + Punct::Question => "?".fmt(f), + Punct::Rem => "%".fmt(f), + Punct::RemEq => "%=".fmt(f), + Punct::Semi => ";".fmt(f), + Punct::Slash => "/".fmt(f), + Punct::SlashEq => "/=".fmt(f), + Punct::Star => "*".fmt(f), + Punct::StarEq => "*=".fmt(f), + Punct::Tilde => "~".fmt(f), + Punct::Xor => "^".fmt(f), + Punct::XorEq => "^=".fmt(f), + Punct::XorXor => "^^".fmt(f), } } }