From e4c008bd4b165aa5047a18eda9e3fb1a0a39ce32 Mon Sep 17 00:00:00 2001 From: John Date: Mon, 5 Jan 2026 15:17:22 -0500 Subject: [PATCH] doughlang: Expressions in patterns ast: - Document operators - Parameterize Pat with Annotation - Consolidate Path and Lit into "Value" (Expr) - Consolidate NamedRecord/Namedtuple into PatOp::TypePrefixed - Allow Pat patterns - Additional helper functions on Expr and Pat lexer: - Support inner-doc comment syntax `//!` - Cleans up `///` or `//!` prefix parser: - Make Parse::Prec `Default` - Allow expression elision after `..`/`..=` - Fix Parser::consume not updating elide_do state - Add token splitting, to make `&&Expr` and `&&Pat` easier - error: Embed Pat precedence in ParseError --- examples/7fold.rs | 2 +- examples/to-lisp.rs | 21 +--- examples/weight.rs | 140 ++++++++++++++++++++++++ samples/elf.cl | 12 +++ samples/usable.do | 24 ++--- src/ast.rs | 225 ++++++++++++++++++++++++++------------- src/ast/display.rs | 36 ++++--- src/ast/fold.rs | 13 +-- src/ast/macro_matcher.rs | 21 ++-- src/ast/visit.rs | 11 +- src/lexer.rs | 12 ++- src/parser.rs | 26 ++++- src/parser/error.rs | 11 +- src/parser/expr.rs | 58 +++++----- src/parser/pat.rs | 186 +++++++++++++++++--------------- src/token.rs | 224 ++++++++++++++++++++++++++++---------- 16 files changed, 700 insertions(+), 322 deletions(-) create mode 100644 examples/weight.rs create mode 100644 samples/elf.cl diff --git a/examples/7fold.rs b/examples/7fold.rs index 8e9c66f..fbb2872 100644 --- a/examples/7fold.rs +++ b/examples/7fold.rs @@ -130,7 +130,7 @@ impl<'a> Visit<'a> for CountNodes { item.children(self) } - fn visit_pat(&mut self, item: &'a Pat) -> Result<(), Self::Error> { + fn visit_pat(&mut self, item: &'a Pat) -> Result<(), Self::Error> { self.patterns += 1; item.children(self) } diff --git a/examples/to-lisp.rs b/examples/to-lisp.rs index a804d9d..95fa4fe 100644 --- a/examples/to-lisp.rs +++ b/examples/to-lisp.rs @@ -54,6 +54,8 @@ impl ToLisp { PatOp::Slice => "slice", PatOp::ArRep => "ar-rep", PatOp::Typed => "typed", + PatOp::TypePrefixed => "type-prefixed", + PatOp::Generic => "generic-in", PatOp::Fn => "fn", PatOp::Alt => "alt", } @@ -196,28 +198,13 @@ impl<'a> Visit<'a> for ToLisp { Ok(()) } - fn visit_pat(&mut self, item: &'a Pat) -> Result<(), Self::Error> { + fn visit_pat(&mut self, item: &'a Pat) -> Result<(), Self::Error> { match item { Pat::Ignore => print!("(ignore)"), Pat::Never => print!("(never)"), Pat::MetId(id) => print!("(replace {id})"), Pat::Name(name) => print!("{name}"), - Pat::Path(path) => path.visit_in(self)?, - Pat::NamedRecord(path, pat) => { - print!("(struct-pat "); - path.visit_in(self)?; - print!(" "); - pat.visit_in(self)?; - print!(")") - } - Pat::NamedTuple(path, pat) => { - print!("(named "); - path.visit_in(self)?; - print!(" "); - pat.visit_in(self)?; - print!(")") - } - Pat::Lit(literal) => literal.visit_in(self)?, + Pat::Value(literal) => literal.visit_in(self)?, Pat::Op(pat_op, pats) => { print!("({}", self.pat_op(*pat_op)); for pat in pats { diff --git a/examples/weight.rs b/examples/weight.rs new file mode 100644 index 0000000..f75fba8 --- /dev/null +++ b/examples/weight.rs @@ -0,0 +1,140 @@ +//! Demonstrates the visitor pattern + +use std::{convert::Infallible, error::Error}; + +use doughlang::{ + ast::{ + visit::{Visit, Walk}, + *, + }, + lexer::Lexer, + parser::{Parser, expr::Prec}, +}; + +const CODE: &str = r#" +let x: [i32; 4] = [1, 2, 3, 4]; + +use foo::bar::baz::qux::{a, b, c, d, e}; + +struct Point {x: T, y: T} + +let v = Point { + x, y: 20 + x +} + +fn main() -> (&str, bool, i128) { + // An if expression is like the ternary conditional operator in C + let y = if 10 < 50 { + "\u{1f988}" + } else { + "x" + } + + // A `while` expression is like the while-else construct in Python, + // but it returns a value via the `break` keyword + let z = while false { + // do a thing repeatedly + break true + } else { + // If `while` does not `break`, fall through to the `else` expression + false + } + + // The same is true of `for` expressions! + let w = for idx in 0..100 { + if idx > 2 * 2 { + break idx + } + } else 12345; // semicolon operator required here + + + // desugars to + let w = match ((0..100).into_iter()) { + __iter => loop match (__iter.next()) { + None => break 12345, + Some (idx) => { + if (idx > (2 * 2)) { + break idx + } + }, + }, + }; + + // A block evaluates to its last expression, + // or Empty if there is none + // (🦈, false, 5) + (y, z, w) +} + +"#; + +fn main() -> Result<(), Box> { + let ast: Anno = Parser::new(Lexer::new(CODE)).parse(Prec::MIN)?; + println!("{ast}"); + let mut counters = Weight::new(); + counters.visit(&ast); + println!("{counters:#?}"); + println!("{}", CODE.len()); + + Ok(()) +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct Weight { + size: usize, +} + +impl Weight { + fn new() -> Self { + Self::default() + } +} + +impl<'a> Visit<'a> for Weight { + type Error = Infallible; + + fn visit_expr(&mut self, item: &'a Expr) -> Result<(), Self::Error> { + self.size += size_of_val(item); + item.children(self) + } + + fn visit_ident(&mut self, item: &'a str) -> Result<(), Self::Error> { + self.size += size_of_val(item); + item.children(self) + } + + fn visit_path(&mut self, item: &'a Path) -> Result<(), Self::Error> { + self.size += size_of_val(item) + size_of_val(item.parts.as_slice()); + item.children(self) + } + + fn visit_literal(&mut self, item: &'a Literal) -> Result<(), Self::Error> { + self.size += size_of_val(item); + item.children(self) + } + + fn visit_use(&mut self, item: &'a Use) -> Result<(), Self::Error> { + self.size += size_of_val(item); + item.children(self) + } + + fn visit_pat(&mut self, item: &'a Pat) -> Result<(), Self::Error> { + self.size += size_of_val(item); + item.children(self) + } + + fn visit_bind(&mut self, item: &'a Bind) -> Result<(), Self::Error> { + self.size += size_of_val(item); + item.children(self) + } + + fn visit_make(&mut self, item: &'a Make) -> Result<(), Self::Error> { + self.size += size_of_val(item); + item.children(self) + } + + fn visit_makearm(&mut self, item: &'a MakeArm) -> Result<(), Self::Error> { + self.size += size_of_val(item); + item.children(self) + } +} diff --git a/samples/elf.cl b/samples/elf.cl new file mode 100644 index 0000000..ad4ed78 --- /dev/null +++ b/samples/elf.cl @@ -0,0 +1,12 @@ +use num::{ + i32, u32, u16, u8 +} + +type Elf32_Addr, Elf32_Half, Elf32_Off, Elf32_Sword, Elf32_Word, UChar = u32, u16, u32, i32, u32, u8; + +let Option = type Some(T) | None; +// is equivalent to +enum Option { + Some(T), + None +}; diff --git a/samples/usable.do b/samples/usable.do index bd7bdb1..7c02a31 100644 --- a/samples/usable.do +++ b/samples/usable.do @@ -7,15 +7,15 @@ enum Expr { fn execute(expr: Expr) -> f64 { match expr { - ExprAtom(value) => value, - ExprOp('*', [lhs, rhs]) => execute(lhs) * execute(rhs), - ExprOp('/', [lhs, rhs]) => execute(lhs) / execute(rhs), - ExprOp('%', [lhs, rhs]) => execute(lhs) % execute(rhs), - ExprOp('+', [lhs, rhs]) => execute(lhs) + execute(rhs), - ExprOp('-', [lhs, rhs]) => execute(lhs) - execute(rhs), - // ExprOp('>', [lhs, rhs]) => (execute(lhs) as u64 >> execute(rhs) as u64) as f64, - // ExprOp('<', [lhs, rhs]) => (execute(lhs) as u64 << execute(rhs) as u64) as f64, - ExprOp('-', [lhs]) => - execute(lhs), + Expr::Atom(value) => value, + Expr::Op('*', [lhs, rhs]) => execute(lhs) * execute(rhs), + Expr::Op('/', [lhs, rhs]) => execute(lhs) / execute(rhs), + Expr::Op('%', [lhs, rhs]) => execute(lhs) % execute(rhs), + Expr::Op('+', [lhs, rhs]) => execute(lhs) + execute(rhs), + Expr::Op('-', [lhs, rhs]) => execute(lhs) - execute(rhs), + // Expr::Op('>', [lhs, rhs]) => (execute(lhs) as u64 >> execute(rhs) as u64) as f64, + // Expr::Op('<', [lhs, rhs]) => (execute(lhs) as u64 << execute(rhs) as u64) as f64, + Expr::Op('-', [lhs]) => - execute(lhs), other => { panic("Unknown operation: " + fmt(other)) } @@ -25,9 +25,9 @@ fn execute(expr: Expr) -> f64 { /// Formats an expression to a string fn fmt_expr(expr: Expr) -> str { match expr { - ExprAtom(value) => fmt(value), - ExprOp(operator, [lhs, rhs]) => fmt('(', fmt_expr(lhs), ' ', operator, ' ', fmt_expr(rhs), ')'), - ExprOp(operator, [rhs]) => fmt(operator, fmt_expr(rhs)), + Expr::Atom(value) => fmt(value), + Expr::Op(operator, [lhs, rhs]) => fmt('(', fmt_expr(lhs), ' ', operator, ' ', fmt_expr(rhs), ')'), + Expr::Op(operator, [rhs]) => fmt(operator, fmt_expr(rhs)), _ => println("Unexpected expr: ", expr), } } diff --git a/src/ast.rs b/src/ast.rs index 450dc05..d0b90a0 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -65,78 +65,137 @@ pub enum Expr { /// - Traditional binary and unary operators (add, sub, neg, assign) #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Op { - // -- true operators - Do, // Expr ; Expr - As, // Expr as Expr - Macro, // macro { (Pat => Expr)* } - Block, // { Expr } - Array, // [ Expr,* ] - ArRep, // [ Expr ; Expr ] - Group, // ( Expr ,?) - Tuple, // Expr (, Expr)* - Meta, // #[ Expr ] + /// `Expr (; Expr)*` + Do, + /// `Expr as Expr` + As, + /// `macro { (Pat => Expr)* }` + Macro, + /// `{ Expr }` + Block, + /// `[ Expr,* ]` + Array, + /// `[ Expr ; Expr ]` + ArRep, + /// `( Expr )` + Group, + /// `Expr (, Expr)*` + Tuple, + /// `#[ Expr ]` + Meta, - Try, // Expr '?' - Index, // Expr [ Expr,* ] - Call, // Expr ( Expr,* ) + /// `Expr '?'` + Try, + /// `Expr [ Expr,* ]` + Index, + /// `Expr ( Expr,* )` + Call, - Pub, // pub Expr - Const, // const Expr - Static, // static Expr - Loop, // loop Expr - Match, // match Expr { ,* } - If, // if Expr Expr (else Expr)? - While, // while Expr Expr (else Expr)? - Break, // break Expr - Return, // return Expr - Continue, // continue + /// `pub Expr` + Pub, + /// `const Expr` + Const, + /// `static Expr` + Static, + /// `loop Expr` + Loop, + /// `match Expr { ,* }` + Match, + /// `if Expr Expr (else Expr)?` + If, + /// `while Expr Expr (else Expr)?` + While, + /// `break Expr` + Break, + /// `return Expr` + Return, + /// `continue` + Continue, - Dot, // Expr . Expr + /// `Expr . Expr` + Dot, - RangeEx, // Expr? ..Expr - RangeIn, // Expr? ..=Expr - Neg, // -Expr - Not, // !Expr - Identity, // !!Expr - Refer, // &Expr - Deref, // *Expr + /// `Expr? ..Expr` + RangeEx, + /// `Expr? ..=Expr` + RangeIn, + /// `-Expr` + Neg, + /// `!Expr` + Not, + /// `!!Expr` + Identity, + /// `&Expr` + Refer, + /// `*Expr` + Deref, - Mul, // Expr * Expr - Div, // Expr / Expr - Rem, // Expr % Expr + /// `Expr * Expr` + Mul, + /// `Expr / Expr` + Div, + /// `Expr % Expr` + Rem, - Add, // Expr + Expr - Sub, // Expr - Expr + /// `Expr + Expr` + Add, + /// `Expr - Expr` + Sub, - Shl, // Expr << Expr - Shr, // Expr >> Expr + /// `Expr << Expr` + Shl, + /// `Expr >> Expr` + Shr, - And, // Expr & Expr - Xor, // Expr ^ Expr - Or, // Expr | Expr + /// `Expr & Expr` + And, + /// `Expr ^ Expr` + Xor, + /// `Expr | Expr` + Or, - Lt, // Expr < Expr - Leq, // Expr <= Expr - Eq, // Expr == Expr - Neq, // Expr != Expr - Geq, // Expr >= Expr - Gt, // Expr > Expr + /// `Expr < Expr` + Lt, + /// `Expr <= Expr` + Leq, + /// `Expr == Expr` + Eq, + /// `Expr != Expr` + Neq, + /// `Expr >= Expr` + Geq, + /// `Expr > Expr` + Gt, - LogAnd, // Expr && Expr - LogXor, // Expr ^^ Expr - LogOr, // Expr || Expr + /// `Expr && Expr` + LogAnd, + /// `Expr ^^ Expr` + LogXor, + /// `Expr || Expr` + LogOr, - Set, // Expr = Expr - MulSet, // Expr *= Expr - DivSet, // Expr /= Expr - RemSet, // Expr %= Expr - AddSet, // Expr += Expr - SubSet, // Expr -= Expr - ShlSet, // Expr <<= Expr - ShrSet, // Expr >>= Expr - AndSet, // Expr &= Expr - XorSet, // Expr ^= Expr - OrSet, // Expr |= Expr + /// `Expr = Expr` + Set, + /// `Expr *= Expr` + MulSet, + /// `Expr /= Expr` + DivSet, + /// `Expr %= Expr` + RemSet, + /// `Expr += Expr` + AddSet, + /// `Expr -= Expr` + SubSet, + /// `Expr <<= Expr` + ShlSet, + /// `Expr >>= Expr` + ShrSet, + /// `Expr &= Expr` + AndSet, + /// `Expr ^= Expr` + XorSet, + /// `Expr |= Expr` + OrSet, } /// A qualified identifier @@ -181,11 +240,11 @@ pub enum Use { /// ```ignore /// let Pat (= Expr (else Expr)?)? /// type Pat (= Expr)? -/// struct Pat -/// enum Pat /// fn Pat Expr /// mod Pat Expr /// impl Pat Expr +/// struct Pat +/// enum Pat /// for Pat in Expr Expr (else Expr)? /// Pat => Expr // in match /// ``` @@ -193,7 +252,7 @@ pub enum Use { pub struct Bind( pub BindOp, pub Vec, - pub Pat, + pub Pat, pub Vec, A>>, ); @@ -237,7 +296,7 @@ pub struct MakeArm(pub String, pub Option, A> /// /// This covers both patterns in Match expressions, and type annotations. #[derive(Clone, Debug, PartialEq, Eq)] -pub enum Pat { +pub enum Pat { /// Matches anything without binding Ignore, /// Matches nothing, ever @@ -246,16 +305,10 @@ pub enum Pat { MetId(String), /// Matches anything, and binds it to a name Name(String), - /// Matches against a named const value - Path(Path), - /// Matches a Struct Expression `Ident { Pat }` - NamedRecord(Path, Box), - /// Matches a Tuple Struct Expression `Ident ( Pat )` - NamedTuple(Path, Box), - /// Matches a literal value by equality comparison - Lit(Literal), + /// Matches a value by equality comparison + Value(Box, A>>), /// Matches a compound pattern - Op(PatOp, Vec), + Op(PatOp, Vec>), } /// Operators on lists of patterns @@ -285,6 +338,10 @@ pub enum PatOp { ArRep, /// Matches a type annotation or struct member Typed, + /// Matches a prefix-type-annotated structure + TypePrefixed, + /// Matches a generic specialization annotation + Generic, /// Changes the binding mode to "function-body" Fn, /// Matches one of a list of alternatives @@ -321,6 +378,13 @@ impl Expr { Self::Op(Op::Do, exprs) } + pub fn to_tuple(self, annotation: A) -> Self { + match self { + Self::Op(Op::Tuple, _) => self, + _ => Self::Op(Op::Tuple, vec![self.anno(annotation)]), + } + } + pub const fn is_place(&self) -> bool { matches!( self, @@ -328,6 +392,10 @@ impl Expr { ) } + pub const fn is_value(&self) -> bool { + !self.is_place() + } + #[allow(clippy::type_complexity)] pub const fn as_slice(&self) -> Option<(Op, &[Anno, A>])> { match self { @@ -337,6 +405,15 @@ impl Expr { } } +impl Pat { + pub fn to_tuple(self) -> Self { + match self { + Self::Op(PatOp::Tuple, _) => self, + _ => Self::Op(PatOp::Tuple, vec![self]), + } + } +} + impl From<&str> for Path { fn from(value: &str) -> Self { Self { parts: vec![value.to_owned()] } diff --git a/src/ast/display.rs b/src/ast/display.rs index ad07170..37e9b63 100644 --- a/src/ast/display.rs +++ b/src/ast/display.rs @@ -46,7 +46,10 @@ impl Display for Expr { }, Self::Op(op @ Op::Call, exprs) => match exprs.as_slice() { - [callee, args @ ..] => f.delimit(fmt!("{callee}("), ")").list(args, ", "), + [callee, Anno(Expr::Op(Op::Tuple, args), _)] => { + f.delimit(fmt!("{callee}("), ")").list(args, ", ") + } + [callee, args @ ..] => f.delimit(fmt!("{callee}(?"), "?)").list(args, ", "), [] => write!(f, "{op}"), }, Self::Op(op @ Op::Index, exprs) => match exprs.as_slice() { @@ -191,11 +194,14 @@ impl Display for Bind { } } BindOp::Struct | BindOp::Enum => match pat { - Pat::NamedRecord(name, bind) => match bind.as_ref() { - Pat::Op(PatOp::Tuple, parts) => f + Pat::Op(PatOp::TypePrefixed, bind) => match bind.as_slice() { + [name, Pat::Op(PatOp::Record, parts)] => f .delimit_indented(fmt!("{name} {{"), "}") .list_wrap("\n", parts, ",\n", ",\n"), - other => write!(f, "{name} {{ {other} }}"), + [name, Pat::Op(PatOp::Tuple, parts)] => { + f.delimit(fmt!("{name}("), ")").list(parts, ", ") + } + _ => pat.fmt(f), }, _ => pat.fmt(f), }, @@ -248,22 +254,14 @@ impl Display for MakeArm { } } -impl Display for Pat { +impl Display for Pat { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Ignore => "_".fmt(f), Self::Never => "!".fmt(f), - Self::Lit(literal) => literal.fmt(f), + Self::Value(literal) => literal.fmt(f), Self::MetId(name) => write!(f, "`{name}"), Self::Name(name) => name.fmt(f), - Self::Path(path) => path.fmt(f), - Self::NamedRecord(name, bind) => match bind.as_ref() { - Pat::Op(PatOp::Tuple, parts) => { - f.delimit(fmt!("{name} {{ "), " }").list(parts, ", ") - } - other => write!(f, "{name} {{ {other} }}"), - }, - Self::NamedTuple(name, bind) => write!(f, "{name} {bind}"), Self::Op(PatOp::Record, pats) => f.delimit("{ ", " }").list(pats, ", "), Self::Op(PatOp::Tuple, pats) => f.delimit("(", ")").list(pats, ", "), Self::Op(PatOp::Slice, pats) => f.delimit("[", "]").list(pats, ", "), @@ -273,6 +271,14 @@ impl Display for Pat { pats => f.list(pats, op), }, Self::Op(op @ PatOp::Alt, pats) => f.list(pats, op), + Self::Op(op @ PatOp::Generic, pats) => match pats.as_slice() { + [] => op.fmt(f), + [first, rest @ ..] => f.delimit(fmt!("{first}<"), ">").list(rest, ", "), + }, + Self::Op(op @ PatOp::TypePrefixed, pats) => match pats.as_slice() { + [] => op.fmt(f), + [first, rest @ ..] => f.delimit(fmt!("{first} "), "").list(rest, ",? "), + }, Self::Op(op, pats) => match pats.as_slice() { [] => op.fmt(f), [rest] => write!(f, "{op}{rest}"), @@ -297,6 +303,8 @@ impl Display for PatOp { Self::Slice => ", ", Self::ArRep => "[;]", Self::Typed => ": ", + Self::Generic => "T<>", + Self::TypePrefixed => "T()", Self::Fn => " -> ", Self::Alt => " | ", }) diff --git a/src/ast/fold.rs b/src/ast/fold.rs index c00860c..6486dd9 100644 --- a/src/ast/fold.rs +++ b/src/ast/fold.rs @@ -45,7 +45,7 @@ pub trait Fold { } /// Consumes a [`Pat`], possibly transforms it, and produces a replacement [`Pat`] - fn fold_pat(&mut self, pat: Pat) -> Result { + fn fold_pat(&mut self, pat: Pat) -> Result, Self::Error> { pat.children(self) } @@ -138,7 +138,7 @@ impl Foldable for Use { } } -impl Foldable for Pat { +impl Foldable for Pat { fn fold_in + ?Sized>(self, folder: &mut F) -> Result { folder.fold_pat(self) } @@ -149,14 +149,7 @@ impl Foldable for Pat { Self::Never => Self::Never, Self::MetId(name) => Self::MetId(name.fold_in(folder)?), Self::Name(name) => Self::Name(name.fold_in(folder)?), - Self::Path(path) => Self::Path(path.fold_in(folder)?), - Self::NamedRecord(path, pat) => { - Self::NamedRecord(path.fold_in(folder)?, pat.fold_in(folder)?) - } - Self::NamedTuple(path, pat) => { - Self::NamedTuple(path.fold_in(folder)?, pat.fold_in(folder)?) - } - Self::Lit(literal) => Self::Lit(literal.fold_in(folder)?), + Self::Value(expr) => Self::Value(expr.fold_in(folder)?), Self::Op(op, pats) => Self::Op(op, pats.fold_in(folder)?), }) } diff --git a/src/ast/macro_matcher.rs b/src/ast/macro_matcher.rs index ef63e4c..66bf5d1 100644 --- a/src/ast/macro_matcher.rs +++ b/src/ast/macro_matcher.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; #[derive(Clone, Debug)] pub struct Subst { pub exp: HashMap>, - pub pat: HashMap, + pub pat: HashMap>, } impl Default for Subst { @@ -16,7 +16,7 @@ impl Default for Subst { } } impl Subst { - fn add_pat(&mut self, name: String, pat: &Pat) -> bool { + fn add_pat(&mut self, name: String, pat: &Pat) -> bool { if self.exp.contains_key(&name) { return false; } @@ -161,7 +161,7 @@ impl Match for MakeArm { } } -impl Match for Pat { +impl Match for Pat { fn recurse(sub: &mut Subst, pat: &Self, expr: &Self) -> bool { match (pat, expr) { (Pat::MetId(name), _) if name == "_" => true, @@ -172,14 +172,8 @@ impl Match for Pat { (Pat::Never, _) => false, (Pat::Name(pat), Pat::Name(expr)) => pat == expr, (Pat::Name(_), _) => false, - (Pat::Path(_), Pat::Path(_)) => true, - (Pat::Path(_), _) => false, - (Pat::NamedRecord(_, pat), Pat::NamedRecord(_, expr)) => Match::recurse(sub, pat, expr), - (Pat::NamedRecord(..), _) => false, - (Pat::NamedTuple(_, pat), Pat::NamedTuple(_, expr)) => Match::recurse(sub, pat, expr), - (Pat::NamedTuple(..), _) => false, - (Pat::Lit(pat), Pat::Lit(expr)) => pat == expr, - (Pat::Lit(_), _) => false, + (Pat::Value(pat), Pat::Value(expr)) => pat == expr, + (Pat::Value(_), _) => false, (Pat::Op(_, pat), Pat::Op(_, expr)) => Match::recurse(sub, pat, expr), (Pat::Op(..), _) => false, } @@ -187,14 +181,13 @@ impl Match for Pat { fn apply(&mut self, sub: &Subst) { match self { - Pat::Ignore | Pat::Never | Pat::Name(_) | Pat::Path(_) | Pat::Lit(_) => {} + Pat::Ignore | Pat::Never | Pat::Name(_) => {} + Pat::Value(expr) => expr.apply(sub), Pat::MetId(id) => { if let Some(expr) = sub.pat.get(id) { *self = expr.clone(); } } - Pat::NamedRecord(_, expr) => expr.apply(sub), - Pat::NamedTuple(_, expr) => expr.apply(sub), Pat::Op(_, pats) => pats.apply(sub), } } diff --git a/src/ast/visit.rs b/src/ast/visit.rs index 9f1b98c..9bac91e 100644 --- a/src/ast/visit.rs +++ b/src/ast/visit.rs @@ -24,7 +24,7 @@ pub trait Visit<'a> { fn visit_use(&mut self, item: &'a Use) -> Result<(), Self::Error> { item.children(self) } - fn visit_pat(&mut self, item: &'a Pat) -> Result<(), Self::Error> { + fn visit_pat(&mut self, item: &'a Pat) -> Result<(), Self::Error> { item.children(self) } fn visit_bind(&mut self, item: &'a Bind) -> Result<(), Self::Error> { @@ -107,18 +107,13 @@ impl<'a> Walk<'a> for Use { } } -impl<'a> Walk<'a> for Pat { +impl<'a, A: Annotation> Walk<'a> for Pat { fn children + ?Sized>(&'a self, v: &mut V) -> Result<(), V::Error> { match self { Self::Ignore | Self::Never => Ok(()), Self::MetId(id) => id.visit_in(v), Self::Name(name) => name.visit_in(v), - Self::Path(path) => path.visit_in(v), - Self::NamedRecord(path, pat) | Self::NamedTuple(path, pat) => { - path.visit_in(v)?; - pat.visit_in(v) - } - Self::Lit(literal) => literal.visit_in(v), + Self::Value(literal) => literal.visit_in(v), Self::Op(_, pats) => pats.visit_in(v), } } diff --git a/src/lexer.rs b/src/lexer.rs index ba9e9af..ace8b36 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -257,11 +257,19 @@ impl<'t> Lexer<'t> { /// Consumes characters until the lexer reaches a newline `'\n'` pub fn line_comment(&mut self) -> Result { let kind = match self.consume().peek() { - Some('!' | '/') => TKind::Doc, + Some('/') => TKind::OutDoc, + Some('!') => TKind::InDoc, _ => TKind::Comment, }; while self.consume().peek().is_some_and(|c| c != '\n') {} - Ok(self.produce(kind)) + let (lexeme, _) = self.as_str(); + let lexeme = lexeme + .strip_prefix("///") + .or_else(|| lexeme.strip_prefix("//!")) + .map(|lexeme| lexeme.strip_prefix(" ").unwrap_or(lexeme)) + .unwrap_or(lexeme); + + Ok(self.produce_with_lexeme(kind, Lexeme::String(lexeme.into()))) } /// Consumes characters until the lexer reaches the end of a *nested* block comment. diff --git a/src/parser.rs b/src/parser.rs index bc8d059..c119211 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -8,7 +8,7 @@ use crate::{ }; pub trait Parse<'t> { - type Prec: Copy; + type Prec: Copy + Default; fn parse(p: &mut Parser<'t>, _level: Self::Prec) -> PResult where Self: Sized; @@ -85,7 +85,10 @@ impl<'t> Parser<'t> { if let Ok(tok) = &tok { self.last_loc = tok.span; - self.elide_do = matches!(tok.kind, TKind::RCurly | TKind::Semi) + self.elide_do = matches!( + tok.kind, + TKind::RCurly | TKind::Semi | TKind::DotDot | TKind::DotDotEq + ) } tok @@ -183,9 +186,26 @@ impl<'t> Parser<'t> { /// Consumes the currently peeked token without returning it. pub fn consume(&mut self) -> &mut Self { - self.next_tok = None; + if self.next_tok.as_ref().is_some_and(|tok| tok.is_ok()) { + let _ = self.take(); + } self } + + /// Consumes the next token, and attempts to split it into multiple. + /// + /// If the next token cannot be split, it will be returned. + pub fn split(&mut self) -> PResult { + let Token { lexeme, kind, span } = self.next()?; + let kind = match kind.split() { + Err(_) => kind, + Ok((out, next)) => { + self.next_tok = Some(Ok(Token { lexeme: lexeme.clone(), kind: next, span })); + out + } + }; + Ok(Token { lexeme, kind, span }) + } } impl<'t> Parse<'t> for Path { diff --git a/src/parser/error.rs b/src/parser/error.rs index 7326fd2..dde48e5 100644 --- a/src/parser/error.rs +++ b/src/parser/error.rs @@ -1,3 +1,4 @@ +use super::pat::Prec as PatPrec; use crate::{ast::BindOp, lexer::LexError, span::Span, token::TKind}; use std::{error::Error, fmt::Display}; @@ -11,7 +12,7 @@ pub enum ParseError { Expected(TKind, TKind, Span), NotLiteral(TKind, Span), NotUse(TKind, Span), - NotPattern(TKind, Span), + NotPattern(TKind, PatPrec, Span), NotMatch(BindOp, BindOp, Span), NotPrefix(TKind, Span), NotInfix(TKind, Span), @@ -30,7 +31,9 @@ impl Display for ParseError { Self::Expected(e, tk, loc) => write!(f, "{loc}: Expected {e:?}, got {tk:?}."), Self::NotLiteral(tk, loc) => write!(f, "{loc}: {tk:?} is not valid in a literal."), Self::NotUse(tk, loc) => write!(f, "{loc}: {tk:?} is no use!"), - Self::NotPattern(tk, loc) => write!(f, "{loc}: {tk:?} is not valid in a pattern."), + Self::NotPattern(tk, prec, loc) => { + write!(f, "{loc}: {tk:?} is not valid in a {prec:?} pattern.") + } Self::NotMatch(bk, ex, loc) => { write!(f, "{loc}: {bk:?} is not valid in a {ex:?} expression.") } @@ -46,6 +49,7 @@ pub type PResult = Result; pub trait PResultExt { fn no_eof(self) -> PResult; fn allow_eof(self) -> PResult>; + fn is_eof(&self) -> bool; } impl PResultExt for PResult { @@ -62,6 +66,9 @@ impl PResultExt for PResult { Err(e) => Err(e), } } + fn is_eof(&self) -> bool { + matches!(self, Err(ParseError::EOF(_))) + } } /// Opens a scope where [`ParseError::EOF`] is unexpected (See [`PResultExt::no_eof`]) diff --git a/src/parser/expr.rs b/src/parser/expr.rs index f40b5e9..a785db8 100644 --- a/src/parser/expr.rs +++ b/src/parser/expr.rs @@ -76,6 +76,7 @@ pub enum Ps { Lit, // Literal Use, // use Use Def, // any definition (let, const, static, struct, enum, fn, ...) + Doc, // Documentation Comment For, // for Pat in Expr Expr else Expr Lambda0, // || Expr Lambda, // | Pat,* | Expr @@ -90,6 +91,8 @@ pub enum Ps { /// and its [precedence level](Prec) fn from_prefix(token: &Token) -> PResult<(Ps, Prec)> { Ok(match token.kind { + TKind::OutDoc => (Ps::Doc, Prec::Max), + TKind::InDoc => (Ps::Doc, Prec::Max), TKind::Do => (Ps::Op(Op::Do), Prec::Do), TKind::Semi => (Ps::End, Prec::Body), @@ -207,12 +210,9 @@ impl<'t> Parse<'t> for Expr { fn parse(p: &mut Parser<'t>, level: usize) -> PResult { const MIN: usize = Prec::MIN; - // TODO: in-tree doc comments - while p.next_if(TKind::Doc)?.is_ok() {} - // Prefix let tok @ &Token { kind, span, .. } = p.peek()?; - let ((op, prec), span) = (from_prefix(tok)?, span); + let (op, prec) = from_prefix(tok)?; no_eof(move || { let mut head = match op { // "End" is produced when an "empty" expression is syntactically required. @@ -227,11 +227,19 @@ impl<'t> Parse<'t> for Expr { Ps::Lit => Expr::Lit(p.parse(())?), Ps::Use => Expr::Use(p.consume().parse(())?), Ps::Def => Expr::Bind(p.parse(None)?), + Ps::Doc => { + let comment = Literal::Str(p.take_lexeme()?.string().unwrap()); + let comment = Expr::Lit(comment).anno(span); + let next = p.parse(level)?; + Expr::Op(Op::Meta, vec![comment, next]) + } + Ps::For => parse_for(p, ())?, Ps::Lambda | Ps::Lambda0 => { p.consume(); let args = if op == Ps::Lambda { p.opt(PPrec::Tuple, TKind::Bar)? + .map(Pat::to_tuple) .unwrap_or(Pat::Op(PatOp::Tuple, vec![])) } else { Pat::Op(PatOp::Tuple, vec![]) @@ -246,7 +254,6 @@ impl<'t> Parse<'t> for Expr { vec![p.parse(Prec::Body.next())?], ))) } - Ps::For => parse_for(p, ())?, Ps::Op(Op::Match) => parse_match(p)?, Ps::Op(Op::Meta) => Expr::Op( Op::Meta, @@ -260,10 +267,10 @@ impl<'t> Parse<'t> for Expr { ), Ps::Op(Op::Block) => Expr::Op( Op::Block, - p.consume().opt(MIN, TKind::RCurly)?.into_iter().collect(), + p.consume().opt(MIN, kind.flip())?.into_iter().collect(), ), Ps::Op(Op::Array) => parse_array(p)?, - Ps::Op(Op::Group) => match p.consume().opt(MIN, TKind::RParen)? { + Ps::Op(Op::Group) => match p.consume().opt(MIN, kind.flip())? { Some(value) => Expr::Op(Op::Group, vec![value]), None => Expr::Op(Op::Tuple, vec![]), }, @@ -295,12 +302,11 @@ impl<'t> Parse<'t> for Expr { }; // Infix and Postfix - while let Ok(Some(tok)) = p.peek().allow_eof() + while let Ok(Some(tok @ &Token { kind, .. })) = p.peek().allow_eof() && let Ok((op, prec)) = from_infix(tok) && level <= prec.prev() && op != Ps::End { - let kind = tok.kind; let span = span.merge(p.span()); head = match op { @@ -320,7 +326,7 @@ impl<'t> Parse<'t> for Expr { span, match p.consume().peek().allow_eof()? { Some(_) => p.parse(prec.next())?, - None => Anno(Default::default(), span), + None => Anno(Expr::Omitted, span), }, ), Ps::Op(Op::Index) => Expr::Op( @@ -328,15 +334,14 @@ impl<'t> Parse<'t> for Expr { p.consume() .list(vec![head.anno(span)], 0, TKind::Comma, TKind::RBrack)?, ), - Ps::Op(Op::Call) => Expr::Op( - Op::Call, - vec![ - head.anno(span), - p.consume() - .opt(0, TKind::RParen)? - .unwrap_or_else(|| Expr::Op(Op::Tuple, vec![]).anno(span)), - ], - ), + Ps::Op(Op::Call) => { + let head = head.anno(span); + let args = match p.consume().opt::>(0, TKind::RParen)? { + None => Expr::Op(Op::Tuple, vec![]).anno(span), + Some(Anno(expr, span)) => expr.to_tuple(span).anno(span), + }; + Expr::Op(Op::Call, vec![head, args]) + } Ps::Op(op @ (Op::Tuple | Op::Dot | Op::LogAnd | Op::LogOr)) => Expr::Op( op, p.consume() @@ -445,7 +450,7 @@ fn from_bind(p: &mut Parser<'_>) -> PResult<(BindOp, PPrec, Option, Optio TKind::Enum => (BindOp::Enum, PPrec::Tuple, None, None, None), TKind::Fn => (BindOp::Fn, PPrec::Fn, None, Some(Prec::Body), None), TKind::Mod => (BindOp::Mod, PPrec::Max, None, Some(Prec::Body), None), - TKind::Impl => (BindOp::Impl, PPrec::Max, None, Some(Prec::Body), None), + TKind::Impl => (BindOp::Impl, PPrec::Fn, None, Some(Prec::Body), None), TKind::Bar => (BindOp::Match, PPrec::Alt, Some(TKind::FatArrow), Some(Prec::Body), None), // no consume! _ => return Ok((BindOp::Match, PPrec::Alt, Some(TKind::FatArrow), Some(Prec::Body), None)), @@ -481,11 +486,14 @@ impl<'t> Parse<'t> for Bind { return Ok(Self(level, generics, pat, vec![])); }; - // `=>` for match, `=`? for everything else - if let Some(arrow) = arrow - && p.next_if(arrow).allow_eof()?.is_none_or(|v| v.is_err()) - { - return Ok(Self(level, generics, pat, vec![])); + // `=>` for match, `=` for `let`, `type` + if let Some(arrow) = arrow { + if p.next_if(arrow).allow_eof()?.is_none_or(|v| v.is_err()) { + return Ok(Self(level, generics, pat, vec![])); + } + } else { + // Allow prefix `=`? for the rest of them + p.next_if(TKind::Eq).allow_eof()?; } // `=` Expr diff --git a/src/parser/pat.rs b/src/parser/pat.rs index d8826e2..28bd515 100644 --- a/src/parser/pat.rs +++ b/src/parser/pat.rs @@ -1,4 +1,4 @@ -use super::{PResult, PResultExt, Parse, ParseError, Parser}; +use super::{PResult, PResultExt, Parse, ParseError, Parser, expr::Prec as ExPrec}; use crate::{ ast::*, token::{TKind, Token}, @@ -8,9 +8,10 @@ use crate::{ /// /// Lower (toward [Prec::Min]) precedence levels can contain /// all higher (toward [Prec::Max]) precedence levels. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] pub enum Prec { /// The lowest precedence + #[default] Min, /// "Alternate" pattern: `Pat | Pat` Alt, @@ -18,7 +19,7 @@ pub enum Prec { Tuple, /// Type annotation: `Pat : Pat` Typed, - /// Function pattern: `Pat -> Pat` + /// Function signature: `foo(bar: baz, ..) -> qux` Fn, /// Range pattern: `Pat .. Pat`, `Pat ..= Pat` Range, @@ -51,8 +52,52 @@ impl Prec { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Prefix { + /// Consume (and disregard) this prefix token + Consume, + Underscore, + Never, + MetId, + Id, + Array, + Constant, + Op(PatOp), + Split(PatOp), +} + +fn from_prefix(token: &Token) -> PResult<(Prefix, Prec)> { + Ok(match token.kind { + TKind::True + | TKind::False + | TKind::Character + | TKind::Integer + | TKind::String + | TKind::Minus + | TKind::Const => (Prefix::Constant, Prec::Max), + TKind::Identifier if token.lexeme.str() == Some("_") => (Prefix::Underscore, Prec::Max), + TKind::ColonColon | TKind::Identifier => (Prefix::Id, Prec::Max), + TKind::Bang => (Prefix::Never, Prec::Max), + TKind::Amp => (Prefix::Op(PatOp::Ref), Prec::Max), + TKind::AmpAmp => (Prefix::Split(PatOp::Ref), Prec::Max), + TKind::Star => (Prefix::Op(PatOp::Ptr), Prec::Max), + TKind::Mut => (Prefix::Op(PatOp::Mut), Prec::Max), + TKind::Pub => (Prefix::Op(PatOp::Pub), Prec::Max), + TKind::Grave => (Prefix::MetId, Prec::Max), + + TKind::Fn => (Prefix::Op(PatOp::Fn), Prec::Fn), + TKind::Bar => (Prefix::Consume, Prec::Alt), + TKind::DotDot => (Prefix::Op(PatOp::Rest), Prec::Max), + TKind::DotDotEq => (Prefix::Op(PatOp::RangeIn), Prec::Max), + TKind::LCurly => (Prefix::Op(PatOp::Record), Prec::Typed), + TKind::LParen => (Prefix::Op(PatOp::Tuple), Prec::Fn), + TKind::LBrack => (Prefix::Array, Prec::Max), + kind => Err(ParseError::NotPrefix(kind, token.span))?, + }) +} + /// Tries to map the incoming Token to a [pattern operator](PatOp) -/// and its [precedence level](Prec) +/// and its following [precedence level](Prec) fn from_infix(token: &Token) -> Option<(PatOp, Prec)> { Some(match token.kind { TKind::Arrow => (PatOp::Fn, Prec::Fn), @@ -61,6 +106,10 @@ fn from_infix(token: &Token) -> Option<(PatOp, Prec)> { TKind::Comma => (PatOp::Tuple, Prec::Tuple), TKind::DotDot => (PatOp::RangeEx, Prec::Range), TKind::DotDotEq => (PatOp::RangeIn, Prec::Range), + TKind::Lt => (PatOp::Generic, Prec::Fn), + TKind::LCurly => (PatOp::TypePrefixed, Prec::Typed), + // LParen is used in function signatures + TKind::LParen => (PatOp::TypePrefixed, Prec::Fn), _ => None?, }) } @@ -69,99 +118,70 @@ impl<'t> Parse<'t> for Pat { type Prec = Prec; fn parse(p: &mut Parser<'t>, level: Prec) -> PResult { - let tok = p.peek()?; + let tok @ &Token { kind, span, .. } = p.peek()?; + let (op, prec) = from_prefix(tok)?; + if level > prec { + return Err(ParseError::NotPattern(kind, level, span)); + } - // Prefix - let mut head = match tok.kind { - TKind::True | TKind::False | TKind::Character | TKind::Integer | TKind::String => { - Pat::Lit(p.parse(())?) - } - TKind::Bar => p.consume().parse(level)?, - TKind::Bang => p.consume().then(Pat::Never), - TKind::Fn => Pat::Op(PatOp::Fn, vec![p.consume().parse(Prec::Typed)?]), - TKind::Amp => Pat::Op(PatOp::Ref, vec![p.consume().parse(Prec::Max)?]), - TKind::Star => Pat::Op(PatOp::Ptr, vec![p.consume().parse(Prec::Max)?]), - TKind::Mut => Pat::Op(PatOp::Mut, vec![p.consume().parse(Prec::Max)?]), - TKind::Pub => Pat::Op(PatOp::Pub, vec![p.consume().parse(Prec::Max)?]), - TKind::AmpAmp => Pat::Op( - PatOp::Ref, - vec![Pat::Op(PatOp::Ref, vec![p.consume().parse(Prec::Max)?])], - ), - TKind::Identifier => match tok.lexeme.str() { - Some("_") => p.consume().then(Pat::Ignore), - _ => { - let mut path: Path = p.parse(())?; - // TODO: make these postfix. - match p.peek().map(Token::kind) { - Ok(TKind::LParen) => Pat::NamedTuple(path, p.parse(Prec::Typed)?), - Ok(TKind::LCurly) if level <= Prec::Tuple.next() => Pat::NamedRecord( - path, - p.consume() - .opt(Prec::Tuple, TKind::RCurly)? - .unwrap_or_else(|| Box::new(Pat::Op(PatOp::Tuple, vec![]))), - ), - Ok(_) | Err(ParseError::EOF(_)) => match path.parts.len() { - 1 => Self::Name(path.parts.pop().expect("name has 1 part")), - _ => Self::Path(path), - }, - Err(e) => Err(e)?, - } + let mut head = match op { + Prefix::Consume => p.consume().parse(level)?, + Prefix::Underscore => p.consume().then(Pat::Ignore), + Prefix::Never => p.consume().then(Pat::Never), + Prefix::MetId => Pat::MetId(p.consume().next()?.lexeme.to_string()), + Prefix::Constant => Pat::Value(p.parse(ExPrec::Unary.value())?), + Prefix::Array => parse_array_pat(p)?, + Prefix::Id => { + let Anno(mut path, span): Anno = p.parse(())?; + // TODO: make these postfix. + match path.parts.len() { + 1 => Pat::Name(path.parts.pop().expect("name has 1 part")), + _ => Pat::Value(Box::new(Anno(Expr::Id(path), span))), } - }, - TKind::Grave => Pat::MetId(p.consume().next()?.lexeme.to_string()), - TKind::DotDot => Pat::Op( - PatOp::Rest, - // Identifier in Rest position always becomes binder - match p.consume().peek().allow_eof()?.map(Token::kind) { - Some(TKind::Identifier) => vec![Pat::Name( - p.take_lexeme() - .expect("should have lexeme") - .string() - .expect("should be string"), - )], - Some(TKind::Grave | TKind::Integer | TKind::Character) => vec![p.parse(level)?], - _ => vec![], - }, - ), - TKind::DotDotEq => Pat::Op( - PatOp::RangeIn, - match p.consume().peek().allow_eof()?.map(Token::kind) { - Some(TKind::Grave | TKind::Integer | TKind::Character) => vec![p.parse(level)?], - _ => vec![], - }, - ), - TKind::LCurly => Pat::Op( - PatOp::Record, + } + Prefix::Op(op @ (PatOp::Record | PatOp::Tuple)) => Pat::Op( + op, p.consume() - .list(vec![], Prec::Typed, TKind::Comma, TKind::RCurly)?, + .list(vec![], Prec::Tuple.next(), TKind::Comma, kind.flip())?, ), - TKind::LParen => Pat::Op( - PatOp::Tuple, - p.consume() - .list(vec![], Prec::Typed, TKind::Comma, TKind::RParen)?, - ), - TKind::LBrack => parse_array_pat(p)?, - _ => Err(ParseError::NotPattern(tok.kind, tok.span))?, + Prefix::Op(op @ (PatOp::Rest | PatOp::RangeEx | PatOp::RangeIn)) => { + // next token must continue a pattern + match p.consume().peek().allow_eof()? { + Some(tok) if from_prefix(tok).is_ok() => Pat::Op(op, vec![p.parse(prec)?]), + _ => Pat::Op(op, vec![]), + } + } + Prefix::Op(op) => Pat::Op(op, vec![p.consume().parse(prec)?]), + Prefix::Split(op) => { + p.split()?; + Pat::Op(op, vec![p.parse(prec)?]) + } }; - while let Ok(Some(tok)) = p.peek().allow_eof() + while let Ok(Some(tok @ &Token { kind, .. })) = p.peek().allow_eof() && let Some((op, prec)) = from_infix(tok) && level <= prec { - let kind = tok.kind; head = match op { - PatOp::Typed => Pat::Op(op, vec![head, p.consume().parse(prec.next())?]), - PatOp::Fn => Pat::Op(op, vec![head, p.consume().parse(Prec::Fn)?]), PatOp::RangeEx | PatOp::RangeIn => Pat::Op( op, - match p.consume().peek().map(Token::kind) { - Ok(TKind::Integer | TKind::Character | TKind::Identifier) => { - vec![head, p.parse(prec.next())?] - } - _ => vec![head], + if let Some(tok) = p.consume().peek().allow_eof()? + && from_prefix(tok).is_ok() + { + vec![head, p.parse(prec)?] + } else { + vec![head] }, ), - _ => Pat::Op(op, p.consume().list_bare(vec![head], prec.next(), kind)?), + PatOp::Generic => Pat::Op( + op, + p.consume() + .list(vec![head], prec, TKind::Comma, kind.flip())?, + ), + PatOp::TypePrefixed => Pat::Op(op, vec![head, p.parse(prec)?]), + PatOp::Tuple => Pat::Op(op, p.consume().list_bare(vec![head], prec.next(), kind)?), + PatOp::Fn => Pat::Op(op, vec![head, p.consume().parse(prec)?]), + _ => Pat::Op(op, vec![head, p.consume().parse(prec.next())?]), } } Ok(head) diff --git a/src/token.rs b/src/token.rs index 2364832..dd220b3 100644 --- a/src/token.rs +++ b/src/token.rs @@ -66,8 +66,12 @@ impl std::fmt::Display for Lexeme { /// The lexical classification of a [Token]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum TKind { - Comment, // Line or block comment - Doc, // Doc comment + /// Line or block comment + Comment, + /// Outer doc comment ///.* + OutDoc, + /// Inner doc comment: //!.* + InDoc, As, Break, @@ -100,59 +104,165 @@ pub enum TKind { Identifier, // or Keyword Character, String, - Integer, // 0(x[0-9A-Fa-f]* | d[0-9]* | o[0-7]* | b[0-1]*) | [1-9][0-9]* - LCurly, // { - RCurly, // } - LBrack, // [ - RBrack, // ] - LParen, // ( - RParen, // ) - Amp, // & - AmpAmp, // && - AmpEq, // &= - Arrow, // -> - At, // @ - Backslash, // \ - Bang, // ! - BangBang, // !! - BangEq, // != - Bar, // | - BarBar, // || - BarEq, // |= - Colon, // : - ColonColon, // :: - Comma, // , - Dot, // . - DotDot, // .. - DotDotEq, // ..= - Eq, // = - EqEq, // == - FatArrow, // => - Grave, // ` - Gt, // > - GtEq, // >= - GtGt, // >> - GtGtEq, // >>= - Hash, // # - HashBang, // #! - Lt, // < - LtEq, // <= - LtLt, // << - LtLtEq, // <<= - Minus, // - - MinusEq, // -= - Plus, // + - PlusEq, // += - Question, // ? - Rem, // % - RemEq, // %= - Semi, // ; - Slash, // / - SlashEq, // /= - Star, // * - StarEq, // *= - Tilde, // ~ - Xor, // ^ - XorEq, // ^= - XorXor, // ^^ + /// `0(x[0-9A-Fa-f]* | d[0-9]* | o[0-7]* | b[0-1]*) | [1-9][0-9]*` + Integer, + /// { + LCurly, + /// } + RCurly, + /// [ + LBrack, + /// ] + RBrack, + /// ( + LParen, + /// ) + RParen, + /// & + Amp, + /// && + AmpAmp, + /// &= + AmpEq, + /// -> + Arrow, + /// @ + At, + /// \ + Backslash, + /// ! + Bang, + /// !! + BangBang, + /// != + BangEq, + /// | + Bar, + /// || + BarBar, + /// |= + BarEq, + /// : + Colon, + /// :: + ColonColon, + /// , + Comma, + /// . + Dot, + /// .. + DotDot, + /// ..= + DotDotEq, + /// = + Eq, + /// == + EqEq, + /// => + FatArrow, + /// \` + Grave, + /// > + Gt, + /// >= + GtEq, + /// >> + GtGt, + /// >>= + GtGtEq, + /// # + Hash, + /// #! + HashBang, + /// < + Lt, + /// <= + LtEq, + /// << + LtLt, + /// <<= + LtLtEq, + /// - + Minus, + /// -= + MinusEq, + /// + + Plus, + /// += + PlusEq, + /// ? + Question, + /// % + Rem, + /// %= + RemEq, + /// ; + Semi, + /// / + Slash, + /// /= + SlashEq, + /// * + Star, + /// *= + StarEq, + /// ~ + Tilde, + /// ^ + Xor, + /// ^= + XorEq, + /// ^^ + XorXor, +} + +impl TKind { + pub const fn flip(self) -> Self { + match self { + Self::LCurly => Self::RCurly, + Self::RCurly => Self::LCurly, + Self::LBrack => Self::RBrack, + Self::RBrack => Self::LBrack, + Self::LParen => Self::RParen, + Self::RParen => Self::LParen, + Self::Gt => Self::Lt, + Self::GtGt => Self::LtLt, + Self::LtLt => Self::GtGt, + Self::Lt => Self::Gt, + _ => self, + } + } + + /// Splits a single [TKind] into two, if possible, or returns the original. + pub const fn split(self) -> Result<(Self, Self), Self> { + Ok(match self { + Self::AmpAmp => (Self::Amp, Self::Amp), + Self::AmpEq => (Self::Amp, Self::Eq), + Self::Arrow => (Self::Minus, Self::Gt), + Self::BangBang => (Self::Bang, Self::Bang), + Self::BangEq => (Self::Bang, Self::Eq), + Self::BarBar => (Self::Bar, Self::Bar), + Self::BarEq => (Self::Bar, Self::Eq), + Self::ColonColon => (Self::Colon, Self::Colon), + Self::DotDot => (Self::Dot, Self::Dot), + Self::DotDotEq => (Self::DotDot, Self::Eq), + Self::EqEq => (Self::Eq, Self::Eq), + Self::FatArrow => (Self::Eq, Self::Gt), + Self::GtEq => (Self::Gt, Self::Eq), + Self::GtGt => (Self::Gt, Self::Gt), + Self::GtGtEq => (Self::Gt, Self::GtEq), + Self::HashBang => (Self::Hash, Self::Bang), + Self::LtEq => (Self::Lt, Self::Eq), + Self::LtLt => (Self::Lt, Self::Lt), + Self::LtLtEq => (Self::Lt, Self::LtEq), + Self::MinusEq => (Self::Minus, Self::Eq), + Self::PlusEq => (Self::Plus, Self::Eq), + Self::RemEq => (Self::Rem, Self::Eq), + Self::SlashEq => (Self::Slash, Self::Eq), + Self::StarEq => (Self::Star, Self::Eq), + Self::XorEq => (Self::Xor, Self::Eq), + Self::XorXor => (Self::Xor, Self::Xor), + _ => return Err(self), + }) + } }