j/Conlang
1
1
Fork 0
Code Issues 5 Pull Requests Packages Projects 2 Releases Wiki Activity

Token: Add range operators, rename for consistency

Browse Source
This commit is contained in:
John 2023-10-20 17:09:14 -05:00
parent 34525ec77a
commit 1e5f7149d9
6 changed files with 283 additions and 252 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
grammar.ebnf
View File

@ -8,8 +8,8 @@ Identifier = IDENTIFIER ;
(* # Expressions *) (* # Expressions *)
(* expression *) (* expression *)
Block = '{' Expr? '}' ;
Expr = Ignore ; Expr = Ignore ;
Block = '{' Stmt* Expr? '}' ;
Group = '(' Expr? ')' ; Group = '(' Expr? ')' ;
Primary = Item | Identifier | Literal Primary = Item | Identifier | Literal
| Block | Group | Branch ; | Block | Group | Branch ;
@ -30,6 +30,7 @@ IgnoreOp = ';' ;
AssignOp = '=' | "+=" | "-=" | "*=" | "/=" | AssignOp = '=' | "+=" | "-=" | "*=" | "/=" |
"&=" | "|=" | "^=" |"<<=" |">>=" ; "&=" | "|=" | "^=" |"<<=" |">>=" ;
CompareOp = '<' | "<=" | "==" | "!=" | ">=" | '>' ; CompareOp = '<' | "<=" | "==" | "!=" | ">=" | '>' ;
RangeOp = ".." | "..=" ;
LogicOp = "&&" | "||" | "^^" ; LogicOp = "&&" | "||" | "^^" ;
BitwiseOp = '&' | '|' | '^' ; BitwiseOp = '&' | '|' | '^' ;

24
libconlang/src/ast.rs
View File

@ -609,7 +609,7 @@ pub mod expression {
/// (`*`, `/`, `%`) /// (`*`, `/`, `%`)
Factor { Factor {
Mul := Type::Star, Mul := Type::Star,
Div := Type::Div, Div := Type::Slash,
Rem := Type::Rem, Rem := Type::Rem,
} }
/// (`+`, `-`) /// (`+`, `-`)
@ -619,8 +619,8 @@ pub mod expression {
} }
/// (`<<`, `>>`) /// (`<<`, `>>`)
Shift { Shift {
Lsh := Type::Lsh, Lsh := Type::LtLt,
Rsh := Type::Rsh, Rsh := Type::GtGt,
} }
/// (`&`, `|`, `^`) /// (`&`, `|`, `^`)
Bitwise { Bitwise {
@ -632,14 +632,14 @@ pub mod expression {
Logic { Logic {
LogAnd := Type::AmpAmp, LogAnd := Type::AmpAmp,
LogOr := Type::BarBar, LogOr := Type::BarBar,
LogXor := Type::CatEar, LogXor := Type::XorXor,
} }
/// (`<`, `<=`, `==`, `!=`, `>=`, `>`) /// (`<`, `<=`, `==`, `!=`, `>=`, `>`)
Compare { Compare {
Less := Type::Lt, Less := Type::Lt,
LessEq := Type::LtEq, LessEq := Type::LtEq,
Equal := Type::EqEq, Equal := Type::EqEq,
NotEq := Type::NotEq, NotEq := Type::BangEq,
GreaterEq := Type::GtEq, GreaterEq := Type::GtEq,
Greater := Type::Gt, Greater := Type::Gt,
} }
@ -647,15 +647,15 @@ pub mod expression {
/// `&=`, `|=`, `^=`, `<<=`, `>>=`) /// `&=`, `|=`, `^=`, `<<=`, `>>=`)
Assign { Assign {
Assign := Type::Eq, Assign := Type::Eq,
AddAssign := Type::AddEq, AddAssign := Type::PlusEq,
SubAssign := Type::SubEq, SubAssign := Type::MinusEq,
MulAssign := Type::StarEq, MulAssign := Type::StarEq,
DivAssign := Type::DivEq, DivAssign := Type::SlashEq,
BitAndAssign := Type::AndEq, BitAndAssign := Type::AmpEq,
BitOrAssign := Type::OrEq, BitOrAssign := Type::BarEq,
BitXorAssign := Type::XorEq, BitXorAssign := Type::XorEq,
ShlAssign := Type::LshEq, ShlAssign := Type::LtLtEq,
ShrAssign := Type::RshEq, ShrAssign := Type::GtGtEq,
} }
/// (`;`) /// (`;`)
Ignore { Ignore {

152
libconlang/src/lexer.rs
View File

@ -104,53 +104,55 @@ impl<'t> Lexer<'t> {
} }
/// Evaluates punctuation rules /// Evaluates punctuation rules
pub fn punctuation(&mut self) -> Option<Token> { pub fn punctuation(&mut self) -> Option<Token> {
None.or_else(|| self.amp_amp()) None.or_else(|| self.amp_amp()) // &&
.or_else(|| self.bar_bar()) .or_else(|| self.amp_eq()) // &=
.or_else(|| self.not_not()) .or_else(|| self.amp()) // &
.or_else(|| self.cat_ear()) .or_else(|| self.at()) // @
.or_else(|| self.eq_eq()) .or_else(|| self.backslash()) // \
.or_else(|| self.gt_eq()) .or_else(|| self.bang_bang()) // !!
.or_else(|| self.lt_eq()) .or_else(|| self.bang_eq()) // !=
.or_else(|| self.not_eq()) .or_else(|| self.bang()) // !
.or_else(|| self.lsh_eq()) .or_else(|| self.bar_bar()) // ||
.or_else(|| self.rsh_eq()) .or_else(|| self.bar_eq()) // |=
.or_else(|| self.star_eq()) .or_else(|| self.bar()) // |
.or_else(|| self.div_eq()) .or_else(|| self.colon()) // :
.or_else(|| self.rem_eq()) .or_else(|| self.comma()) // ,
.or_else(|| self.add_eq()) .or_else(|| self.dot_dot_eq()) // ..=
.or_else(|| self.sub_eq()) .or_else(|| self.dot_dot()) // ..
.or_else(|| self.and_eq()) .or_else(|| self.dot()) // .
.or_else(|| self.or_eq()) .or_else(|| self.eq_eq()) // ==
.or_else(|| self.xor_eq()) .or_else(|| self.fatarrow()) // =>
.or_else(|| self.lsh()) .or_else(|| self.eq()) // =
.or_else(|| self.rsh()) .or_else(|| self.grave()) // `
.or_else(|| self.arrow()) .or_else(|| self.gt_eq()) // >=
.or_else(|| self.fatarrow()) .or_else(|| self.gt_gt_eq()) // >>=
.or_else(|| self.semi()) .or_else(|| self.gt_gt()) // >>
.or_else(|| self.dot()) .or_else(|| self.gt()) // >
.or_else(|| self.star()) .or_else(|| self.hash()) // #
.or_else(|| self.div()) .or_else(|| self.lt_eq()) // <=
.or_else(|| self.plus()) .or_else(|| self.lt_lt_eq()) // <<=
.or_else(|| self.sub()) .or_else(|| self.lt_lt()) // <<
.or_else(|| self.rem()) .or_else(|| self.lt()) // <
.or_else(|| self.bang()) .or_else(|| self.minus_eq()) // -=
.or_else(|| self.eq()) .or_else(|| self.arrow()) // ->
.or_else(|| self.lt()) .or_else(|| self.minus()) // -
.or_else(|| self.gt()) .or_else(|| self.plus_eq()) // +=
.or_else(|| self.amp()) .or_else(|| self.plus()) // +
.or_else(|| self.bar()) .or_else(|| self.question()) // ?
.or_else(|| self.xor()) .or_else(|| self.rem_eq()) // %=
.or_else(|| self.hash()) .or_else(|| self.rem()) // %
.or_else(|| self.at()) .or_else(|| self.semi()) // ;
.or_else(|| self.colon()) .or_else(|| self.slash_eq()) // /=
.or_else(|| self.backslash()) .or_else(|| self.slash()) // /
.or_else(|| self.question()) .or_else(|| self.star_eq()) // *=
.or_else(|| self.comma()) .or_else(|| self.star()) // *
.or_else(|| self.tilde()) .or_else(|| self.tilde()) // ~
.or_else(|| self.grave()) .or_else(|| self.xor_eq()) // ^=
.or_else(|| self.xor_xor()) // ^^
.or_else(|| self.xor()) // ^
} }
pub fn unary_op(&mut self) -> Option<Token> { pub fn unary_op(&mut self) -> Option<Token> {
self.bang().or_else(|| self.sub()) self.bang().or_else(|| self.minus())
} }
// functions for lexing individual tokens // functions for lexing individual tokens
pub fn invalid(&mut self) -> Option<Token> { pub fn invalid(&mut self) -> Option<Token> {
@ -204,11 +206,11 @@ impl<'t> Lexer<'t> {
self.map_rule(|r| r.char(')'), Type::RParen) self.map_rule(|r| r.char(')'), Type::RParen)
} }
// compound punctuation // compound punctuation
pub fn lsh(&mut self) -> Option<Token> { pub fn lt_lt(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("<<"), Type::Lsh) self.map_rule(|r| r.str("<<"), Type::LtLt)
} }
pub fn rsh(&mut self) -> Option<Token> { pub fn gt_gt(&mut self) -> Option<Token> {
self.map_rule(|r| r.str(">>"), Type::Rsh) self.map_rule(|r| r.str(">>"), Type::GtGt)
} }
pub fn amp_amp(&mut self) -> Option<Token> { pub fn amp_amp(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("&&"), Type::AmpAmp) self.map_rule(|r| r.str("&&"), Type::AmpAmp)
@ -216,11 +218,11 @@ impl<'t> Lexer<'t> {
pub fn bar_bar(&mut self) -> Option<Token> { pub fn bar_bar(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("||"), Type::BarBar) self.map_rule(|r| r.str("||"), Type::BarBar)
} }
pub fn not_not(&mut self) -> Option<Token> { pub fn bang_bang(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("!!"), Type::NotNot) self.map_rule(|r| r.str("!!"), Type::BangBang)
} }
pub fn cat_ear(&mut self) -> Option<Token> { pub fn xor_xor(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("^^"), Type::CatEar) self.map_rule(|r| r.str("^^"), Type::XorXor)
} }
pub fn eq_eq(&mut self) -> Option<Token> { pub fn eq_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("=="), Type::EqEq) self.map_rule(|r| r.str("=="), Type::EqEq)
@ -231,38 +233,44 @@ impl<'t> Lexer<'t> {
pub fn lt_eq(&mut self) -> Option<Token> { pub fn lt_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("<="), Type::LtEq) self.map_rule(|r| r.str("<="), Type::LtEq)
} }
pub fn not_eq(&mut self) -> Option<Token> { pub fn bang_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("!="), Type::NotEq) self.map_rule(|r| r.str("!="), Type::BangEq)
} }
pub fn star_eq(&mut self) -> Option<Token> { pub fn star_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("*="), Type::StarEq) self.map_rule(|r| r.str("*="), Type::StarEq)
} }
pub fn div_eq(&mut self) -> Option<Token> { pub fn slash_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("/="), Type::DivEq) self.map_rule(|r| r.str("/="), Type::SlashEq)
} }
pub fn rem_eq(&mut self) -> Option<Token> { pub fn rem_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("%="), Type::RemEq) self.map_rule(|r| r.str("%="), Type::RemEq)
} }
pub fn add_eq(&mut self) -> Option<Token> { pub fn plus_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("+="), Type::AddEq) self.map_rule(|r| r.str("+="), Type::PlusEq)
} }
pub fn sub_eq(&mut self) -> Option<Token> { pub fn minus_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("-="), Type::SubEq) self.map_rule(|r| r.str("-="), Type::MinusEq)
} }
pub fn and_eq(&mut self) -> Option<Token> { pub fn amp_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("&="), Type::AndEq) self.map_rule(|r| r.str("&="), Type::AmpEq)
} }
pub fn or_eq(&mut self) -> Option<Token> { pub fn bar_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("|="), Type::OrEq) self.map_rule(|r| r.str("|="), Type::BarEq)
} }
pub fn xor_eq(&mut self) -> Option<Token> { pub fn xor_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("^="), Type::XorEq) self.map_rule(|r| r.str("^="), Type::XorEq)
} }
pub fn lsh_eq(&mut self) -> Option<Token> { pub fn lt_lt_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("<<="), Type::LshEq) self.map_rule(|r| r.str("<<="), Type::LtLtEq)
} }
pub fn rsh_eq(&mut self) -> Option<Token> { pub fn gt_gt_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str(">>="), Type::RshEq) self.map_rule(|r| r.str(">>="), Type::GtGtEq)
}
pub fn dot_dot_eq(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("..="), Type::DotDotEq)
}
pub fn dot_dot(&mut self) -> Option<Token> {
self.map_rule(|r| r.str(".."), Type::DotDot)
} }
pub fn arrow(&mut self) -> Option<Token> { pub fn arrow(&mut self) -> Option<Token> {
self.map_rule(|r| r.str("->"), Type::Arrow) self.map_rule(|r| r.str("->"), Type::Arrow)
@ -280,13 +288,13 @@ impl<'t> Lexer<'t> {
pub fn star(&mut self) -> Option<Token> { pub fn star(&mut self) -> Option<Token> {
self.map_rule(|r| r.char('*'), Type::Star) self.map_rule(|r| r.char('*'), Type::Star)
} }
pub fn div(&mut self) -> Option<Token> { pub fn slash(&mut self) -> Option<Token> {
self.map_rule(|r| r.char('/'), Type::Div) self.map_rule(|r| r.char('/'), Type::Slash)
} }
pub fn plus(&mut self) -> Option<Token> { pub fn plus(&mut self) -> Option<Token> {
self.map_rule(|r| r.char('+'), Type::Plus) self.map_rule(|r| r.char('+'), Type::Plus)
} }
pub fn sub(&mut self) -> Option<Token> { pub fn minus(&mut self) -> Option<Token> {
self.map_rule(|r| r.char('-'), Type::Minus) self.map_rule(|r| r.char('-'), Type::Minus)
} }
pub fn rem(&mut self) -> Option<Token> { pub fn rem(&mut self) -> Option<Token> {

106
libconlang/src/tests.rs
View File

@ -90,11 +90,7 @@ mod lexer {
#[test] #[test]
fn identifier() { fn identifier() {
assert_whole_input_is_token( assert_whole_input_is_token("valid_identifier", Lexer::identifier, Type::Identifier);
"valid_identifier",
Lexer::identifier,
Type::Identifier,
);
assert_whole_input_is_token("_0", Lexer::identifier, Type::Identifier); assert_whole_input_is_token("_0", Lexer::identifier, Type::Identifier);
assert_whole_input_is_token("_", Lexer::identifier, Type::Identifier); assert_whole_input_is_token("_", Lexer::identifier, Type::Identifier);
} }
@ -129,12 +125,7 @@ mod lexer {
#[test] #[test]
fn base16() { fn base16() {
assert_has_type_and_range("0x1234", Lexer::integer, Type::Integer, 0..6); assert_has_type_and_range("0x1234", Lexer::integer, Type::Integer, 0..6);
assert_has_type_and_range( assert_has_type_and_range("0x1234 \"hello\"", Lexer::integer, Type::Integer, 0..6);
"0x1234 \"hello\"",
Lexer::integer,
Type::Integer,
0..6,
);
} }
#[test] #[test]
fn base10() { fn base10() {
@ -186,7 +177,7 @@ mod lexer {
"\" \\\"This is a quote\\\" \"", "\" \\\"This is a quote\\\" \"",
Lexer::string, Lexer::string,
Type::String, Type::String,
1..22 1..22,
); );
} }
} }
@ -254,14 +245,21 @@ mod lexer {
use super::*; use super::*;
mod compound { mod compound {
use super::*; use super::*;
#[test] #[test]
fn lsh() { fn dot_dot() {
assert_whole_input_is_token("<<", Lexer::lsh, Type::Lsh) assert_whole_input_is_token("..", Lexer::dot_dot, Type::DotDot)
} }
#[test] #[test]
fn rsh() { fn dot_dot_eq() {
assert_whole_input_is_token(">>", Lexer::rsh, Type::Rsh) assert_whole_input_is_token("..=", Lexer::dot_dot_eq, Type::DotDotEq)
}
#[test]
fn lt_lt() {
assert_whole_input_is_token("<<", Lexer::lt_lt, Type::LtLt)
}
#[test]
fn gt_gt() {
assert_whole_input_is_token(">>", Lexer::gt_gt, Type::GtGt)
} }
#[test] #[test]
fn amp_amp() { fn amp_amp() {
@ -272,12 +270,12 @@ mod lexer {
assert_whole_input_is_token("||", Lexer::bar_bar, Type::BarBar) assert_whole_input_is_token("||", Lexer::bar_bar, Type::BarBar)
} }
#[test] #[test]
fn not_not() { fn bang_bang() {
assert_whole_input_is_token("!!", Lexer::not_not, Type::NotNot) assert_whole_input_is_token("!!", Lexer::bang_bang, Type::BangBang)
} }
#[test] #[test]
fn cat_ear() { fn xor_xor() {
assert_whole_input_is_token("^^", Lexer::cat_ear, Type::CatEar) assert_whole_input_is_token("^^", Lexer::xor_xor, Type::XorXor)
} }
#[test] #[test]
fn eq_eq() { fn eq_eq() {
@ -292,44 +290,44 @@ mod lexer {
assert_whole_input_is_token("<=", Lexer::lt_eq, Type::LtEq) assert_whole_input_is_token("<=", Lexer::lt_eq, Type::LtEq)
} }
#[test] #[test]
fn not_eq() { fn bang_eq() {
assert_whole_input_is_token("!=", Lexer::not_eq, Type::NotEq) assert_whole_input_is_token("!=", Lexer::bang_eq, Type::BangEq)
} }
#[test] #[test]
fn star_eq() { fn star_eq() {
assert_whole_input_is_token("*=", Lexer::star_eq, Type::StarEq) assert_whole_input_is_token("*=", Lexer::star_eq, Type::StarEq)
} }
#[test] #[test]
fn div_eq() { fn slash_eq() {
assert_whole_input_is_token("/=", Lexer::div_eq, Type::DivEq) assert_whole_input_is_token("/=", Lexer::slash_eq, Type::SlashEq)
} }
#[test] #[test]
fn add_eq() { fn plus_eq() {
assert_whole_input_is_token("+=", Lexer::add_eq, Type::AddEq) assert_whole_input_is_token("+=", Lexer::plus_eq, Type::PlusEq)
} }
#[test] #[test]
fn sub_eq() { fn minus_eq() {
assert_whole_input_is_token("-=", Lexer::sub_eq, Type::SubEq) assert_whole_input_is_token("-=", Lexer::minus_eq, Type::MinusEq)
} }
#[test] #[test]
fn and_eq() { fn amp_eq() {
assert_whole_input_is_token("&=", Lexer::and_eq, Type::AndEq) assert_whole_input_is_token("&=", Lexer::amp_eq, Type::AmpEq)
} }
#[test] #[test]
fn or_eq() { fn bar_eq() {
assert_whole_input_is_token("|=", Lexer::or_eq, Type::OrEq) assert_whole_input_is_token("|=", Lexer::bar_eq, Type::BarEq)
} }
#[test] #[test]
fn xor_eq() { fn xor_eq() {
assert_whole_input_is_token("^=", Lexer::xor_eq, Type::XorEq) assert_whole_input_is_token("^=", Lexer::xor_eq, Type::XorEq)
} }
#[test] #[test]
fn lsh_eq() { fn lt_lt_eq() {
assert_whole_input_is_token("<<=", Lexer::lsh_eq, Type::LshEq) assert_whole_input_is_token("<<=", Lexer::lt_lt_eq, Type::LtLtEq)
} }
#[test] #[test]
fn rsh_eq() { fn gt_gt_eq() {
assert_whole_input_is_token(">>=", Lexer::rsh_eq, Type::RshEq) assert_whole_input_is_token(">>=", Lexer::gt_gt_eq, Type::GtGtEq)
} }
} }
@ -337,10 +335,11 @@ mod lexer {
use super::*; use super::*;
#[test] #[test]
fn punctuation_class() { fn punctuation_class() {
// go from least to most specific
assert_whole_input_is_token(";", Lexer::punctuation, Type::Semi); assert_whole_input_is_token(";", Lexer::punctuation, Type::Semi);
assert_whole_input_is_token(".", Lexer::punctuation, Type::Dot); assert_whole_input_is_token(".", Lexer::punctuation, Type::Dot);
assert_whole_input_is_token("*", Lexer::punctuation, Type::Star); assert_whole_input_is_token("*", Lexer::punctuation, Type::Star);
assert_whole_input_is_token("/", Lexer::punctuation, Type::Div); assert_whole_input_is_token("/", Lexer::punctuation, Type::Slash);
assert_whole_input_is_token("+", Lexer::punctuation, Type::Plus); assert_whole_input_is_token("+", Lexer::punctuation, Type::Plus);
assert_whole_input_is_token("-", Lexer::punctuation, Type::Minus); assert_whole_input_is_token("-", Lexer::punctuation, Type::Minus);
assert_whole_input_is_token("%", Lexer::punctuation, Type::Rem); assert_whole_input_is_token("%", Lexer::punctuation, Type::Rem);
@ -359,6 +358,27 @@ mod lexer {
assert_whole_input_is_token("~", Lexer::punctuation, Type::Tilde); assert_whole_input_is_token("~", Lexer::punctuation, Type::Tilde);
assert_whole_input_is_token("`", Lexer::punctuation, Type::Grave); assert_whole_input_is_token("`", Lexer::punctuation, Type::Grave);
assert_whole_input_is_token("\\", Lexer::punctuation, Type::Backslash); assert_whole_input_is_token("\\", Lexer::punctuation, Type::Backslash);
assert_whole_input_is_token("<<", Lexer::punctuation, Type::LtLt);
assert_whole_input_is_token(">>", Lexer::punctuation, Type::GtGt);
assert_whole_input_is_token("&&", Lexer::punctuation, Type::AmpAmp);
assert_whole_input_is_token("||", Lexer::punctuation, Type::BarBar);
assert_whole_input_is_token("!!", Lexer::punctuation, Type::BangBang);
assert_whole_input_is_token("^^", Lexer::punctuation, Type::XorXor);
assert_whole_input_is_token("==", Lexer::punctuation, Type::EqEq);
assert_whole_input_is_token(">=", Lexer::punctuation, Type::GtEq);
assert_whole_input_is_token("<=", Lexer::punctuation, Type::LtEq);
assert_whole_input_is_token("!=", Lexer::punctuation, Type::BangEq);
assert_whole_input_is_token("*=", Lexer::punctuation, Type::StarEq);
assert_whole_input_is_token("/=", Lexer::punctuation, Type::SlashEq);
assert_whole_input_is_token("+=", Lexer::punctuation, Type::PlusEq);
assert_whole_input_is_token("-=", Lexer::punctuation, Type::MinusEq);
assert_whole_input_is_token("&=", Lexer::punctuation, Type::AmpEq);
assert_whole_input_is_token("|=", Lexer::punctuation, Type::BarEq);
assert_whole_input_is_token("^=", Lexer::punctuation, Type::XorEq);
assert_whole_input_is_token("..", Lexer::punctuation, Type::DotDot);
assert_whole_input_is_token("..=", Lexer::punctuation, Type::DotDotEq);
assert_whole_input_is_token("<<=", Lexer::punctuation, Type::LtLtEq);
assert_whole_input_is_token(">>=", Lexer::punctuation, Type::GtGtEq);
} }
// individual functions below // individual functions below
#[test] #[test]
@ -374,8 +394,8 @@ mod lexer {
assert_whole_input_is_token("*", Lexer::star, Type::Star) assert_whole_input_is_token("*", Lexer::star, Type::Star)
} }
#[test] #[test]
fn div() { fn slash() {
assert_whole_input_is_token("/", Lexer::div, Type::Div) assert_whole_input_is_token("/", Lexer::slash, Type::Slash)
} }
#[test] #[test]
fn plus() { fn plus() {
@ -383,7 +403,7 @@ mod lexer {
} }
#[test] #[test]
fn minus() { fn minus() {
assert_whole_input_is_token("-", Lexer::sub, Type::Minus) assert_whole_input_is_token("-", Lexer::minus, Type::Minus)
} }
#[test] #[test]
fn rem() { fn rem() {
@ -406,11 +426,11 @@ mod lexer {
assert_whole_input_is_token(">", Lexer::gt, Type::Gt) assert_whole_input_is_token(">", Lexer::gt, Type::Gt)
} }
#[test] #[test]
fn and() { fn amp() {
assert_whole_input_is_token("&", Lexer::amp, Type::Amp) assert_whole_input_is_token("&", Lexer::amp, Type::Amp)
} }
#[test] #[test]
fn or() { fn bar() {
assert_whole_input_is_token("|", Lexer::bar, Type::Bar) assert_whole_input_is_token("|", Lexer::bar, Type::Bar)
} }
#[test] #[test]

106
libconlang/src/token.rs
View File

@ -16,59 +16,59 @@ pub enum Type {
Float, Float,
String, String,
Character, Character,
// Delimiters // Delimiters and punctuation
LCurly, LCurly, // {
RCurly, RCurly, // }
LBrack, LBrack, // [
RBrack, RBrack, // ]
LParen, LParen, // (
RParen, RParen, // )
// Compound punctuation Amp, // &
Lsh, AmpAmp, // &&
Rsh, AmpEq, // &=
AmpAmp, Arrow, // ->
BarBar, At, // @
NotNot, Backslash, // \
CatEar, Bang, // !
EqEq, BangBang, // !!
GtEq, BangEq, // !=
LtEq, Bar, // |
NotEq, BarBar, // ||
StarEq, BarEq, // |=
DivEq, Colon, // :
RemEq, Comma, // ,
AddEq, Dot, // .
SubEq, DotDot, // ..
AndEq, DotDotEq, // ..=
OrEq, Eq, // =
XorEq, EqEq, // ==
LshEq, FatArrow, // =>
RshEq, Grave, // `
Arrow, Gt, // >
FatArrow, GtEq, // >=
// Simple punctuation GtGt, // >>
Semi, GtGtEq, // >>=
Dot, Hash, // #
Star, Lt, // <
Div, LtEq, // <=
Plus, LtLt, // <<
Minus, LtLtEq, // <<=
Rem, Minus, // -
Bang, MinusEq, // -=
Eq, Plus, // +
Lt, PlusEq, // +=
Gt, Question, // ?
Amp, Rem, // %
Bar, RemEq, // %=
Xor, Semi, // ;
Hash, Slash, // /
At, SlashEq, // /=
Colon, Star, // *
Backslash, StarEq, // *=
Question, Tilde, // ~
Comma, Xor, // ^
Tilde, XorEq, // ^=
Grave, XorXor, // ^^
} }
/// Represents a reserved word. /// Represents a reserved word.

144
libconlang/src/token/token_type.rs
View File

@ -5,64 +5,66 @@ use std::fmt::Display;
impl Display for Type { impl Display for Type {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
Type::Invalid => Display::fmt("invalid", f), Type::Invalid => "invalid".fmt(f),
Type::Comment => Display::fmt("comment", f), Type::Comment => "comment".fmt(f),
Type::Identifier => Display::fmt("identifier", f), Type::Identifier => "identifier".fmt(f),
Type::Keyword(k) => Display::fmt(k, f), Type::Keyword(k) => k.fmt(f),
Type::Integer => Display::fmt("integer literal", f), Type::Integer => "integer literal".fmt(f),
Type::Float => Display::fmt("float literal", f), Type::Float => "float literal".fmt(f),
Type::String => Display::fmt("string literal", f), Type::String => "string literal".fmt(f),
Type::Character => Display::fmt("char literal", f), Type::Character => "char literal".fmt(f),
Type::LCurly => Display::fmt("left curly", f), Type::LCurly => "left curly".fmt(f),
Type::RCurly => Display::fmt("right curly", f), Type::RCurly => "right curly".fmt(f),
Type::LBrack => Display::fmt("left brack", f), Type::LBrack => "left brack".fmt(f),
Type::RBrack => Display::fmt("right brack", f), Type::RBrack => "right brack".fmt(f),
Type::LParen => Display::fmt("left paren", f), Type::LParen => "left paren".fmt(f),
Type::RParen => Display::fmt("right paren", f), Type::RParen => "right paren".fmt(f),
Type::Lsh => Display::fmt("shift left", f), Type::Amp => "and".fmt(f),
Type::Rsh => Display::fmt("shift right", f), Type::AmpAmp => "and-and".fmt(f),
Type::AmpAmp => Display::fmt("and-and", f), Type::AmpEq => "and-assign".fmt(f),
Type::BarBar => Display::fmt("or-or", f), Type::Arrow => "arrow".fmt(f),
Type::NotNot => Display::fmt("not-not", f), Type::At => "at".fmt(f),
Type::CatEar => Display::fmt("cat-ears", f), Type::Backslash => "backslash".fmt(f),
Type::EqEq => Display::fmt("equal to", f), Type::Bang => "bang".fmt(f),
Type::GtEq => Display::fmt("greater than or equal to", f), Type::BangBang => "not-not".fmt(f),
Type::LtEq => Display::fmt("less than or equal to", f), Type::BangEq => "not equal to".fmt(f),
Type::NotEq => Display::fmt("not equal to", f), Type::Bar => "or".fmt(f),
Type::StarEq => Display::fmt("star-assign", f), Type::BarBar => "or-or".fmt(f),
Type::DivEq => Display::fmt("div-assign", f), Type::BarEq => "or-assign".fmt(f),
Type::RemEq => Display::fmt("rem-assign", f), Type::Colon => "colon".fmt(f),
Type::AddEq => Display::fmt("add-assign", f), Type::Comma => "comma".fmt(f),
Type::SubEq => Display::fmt("sub-assign", f), Type::Dot => "dot".fmt(f),
Type::AndEq => Display::fmt("and-assign", f), Type::DotDot => "exclusive range".fmt(f),
Type::OrEq => Display::fmt("or-assign", f), Type::DotDotEq => "inclusive range".fmt(f),
Type::XorEq => Display::fmt("xor-assign", f), Type::Eq => "assign".fmt(f),
Type::LshEq => Display::fmt("shift left-assign", f), Type::EqEq => "equal to".fmt(f),
Type::RshEq => Display::fmt("shift right-assign", f), Type::FatArrow => "fat arrow".fmt(f),
Type::Arrow => Display::fmt("arrow", f), Type::Grave => "grave".fmt(f),
Type::FatArrow => Display::fmt("fat arrow", f), Type::Gt => "greater than".fmt(f),
Type::Semi => Display::fmt("ignore", f), Type::GtEq => "greater than or equal to".fmt(f),
Type::Dot => Display::fmt("dot", f), Type::GtGt => "shift right".fmt(f),
Type::Star => Display::fmt("star", f), Type::GtGtEq => "shift right-assign".fmt(f),
Type::Div => Display::fmt("div", f), Type::Hash => "hash".fmt(f),
Type::Plus => Display::fmt("add", f), Type::Lt => "less than".fmt(f),
Type::Minus => Display::fmt("sub", f), Type::LtEq => "less than or equal to".fmt(f),
Type::Rem => Display::fmt("rem", f), Type::LtLt => "shift left".fmt(f),
Type::Bang => Display::fmt("bang", f), Type::LtLtEq => "shift left-assign".fmt(f),
Type::Eq => Display::fmt("assign", f), Type::Minus => "sub".fmt(f),
Type::Lt => Display::fmt("less than", f), Type::MinusEq => "sub-assign".fmt(f),
Type::Gt => Display::fmt("greater than", f), Type::Plus => "add".fmt(f),
Type::Amp => Display::fmt("and", f), Type::PlusEq => "add-assign".fmt(f),
Type::Bar => Display::fmt("or", f), Type::Question => "huh?".fmt(f),
Type::Xor => Display::fmt("xor", f), Type::Rem => "rem".fmt(f),
Type::Hash => Display::fmt("hash", f), Type::RemEq => "rem-assign".fmt(f),
Type::At => Display::fmt("at", f), Type::Semi => "ignore".fmt(f),
Type::Colon => Display::fmt("colon", f), Type::Slash => "div".fmt(f),
Type::Backslash => Display::fmt("backslash", f), Type::SlashEq => "div-assign".fmt(f),
Type::Question => Display::fmt("huh?", f), Type::Star => "star".fmt(f),
Type::Comma => Display::fmt("comma", f), Type::StarEq => "star-assign".fmt(f),
Type::Tilde => Display::fmt("tilde", f), Type::Tilde => "tilde".fmt(f),
Type::Grave => Display::fmt("grave", f), Type::Xor => "xor".fmt(f),
Type::XorEq => "xor-assign".fmt(f),
Type::XorXor => "cat-ears".fmt(f),
} }
} }
} }
@ -70,23 +72,23 @@ impl Display for Type {
impl Display for Keyword { impl Display for Keyword {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
Self::Break => Display::fmt("break", f), Self::Break => "break".fmt(f),
Self::Continue => Display::fmt("continue", f), Self::Continue => "continue".fmt(f),
Self::Else => Display::fmt("else", f), Self::Else => "else".fmt(f),
Self::False => Display::fmt("false", f), Self::False => "false".fmt(f),
Self::For => Display::fmt("for", f), Self::For => "for".fmt(f),
Self::Fn => Display::fmt("fn", f), Self::Fn => "fn".fmt(f),
Self::If => Display::fmt("if", f), Self::If => "if".fmt(f),
Self::In => Display::fmt("in", f), Self::In => "in".fmt(f),
Self::Let => Display::fmt("let", f), Self::Let => "let".fmt(f),
Self::Return => Display::fmt("return", f), Self::Return => "return".fmt(f),
Self::True => Display::fmt("true", f), Self::True => "true".fmt(f),
Self::While => Display::fmt("while", f), Self::While => "while".fmt(f),
} }
} }
} }
impl std::str::FromStr for Keyword { impl std::str::FromStr for Keyword {
type Err = (); type Err = (); // If an identifier isn't a keyword, that's okay.
fn from_str(s: &str) -> Result<Self, Self::Err> { fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(match s { Ok(match s {
"break" => Self::Break, "break" => Self::Break,