j/Conlang
1
1
Fork 0
Code Issues 5 Pull Requests Packages Projects 2 Releases Wiki Activity

lexer: Add :: and #! tokens

Browse Source
This commit is contained in:
John 2023-10-30 19:17:03 -05:00
parent c3e02d21ad
commit 9b7cf9c017
2 changed files with 70 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
libconlang/src/lexer.rs
View File

@ -100,13 +100,13 @@ impl<'t> Lexer<'t> {
'\\' => self.consume()?.produce(Type::Backslash, ()), '\\' => self.consume()?.produce(Type::Backslash, ()),
'!' => self.consume()?.bang(), '!' => self.consume()?.bang(),
'|' => self.consume()?.bar(), '|' => self.consume()?.bar(),
':' => self.consume()?.produce(Type::Colon, ()), ':' => self.consume()?.colon(),
',' => self.consume()?.produce(Type::Comma, ()), ',' => self.consume()?.produce(Type::Comma, ()),
'.' => self.consume()?.dot(), '.' => self.consume()?.dot(),
'=' => self.consume()?.equal(), '=' => self.consume()?.equal(),
'`' => self.consume()?.produce(Type::Grave, ()), '`' => self.consume()?.produce(Type::Grave, ()),
'>' => self.consume()?.greater(), '>' => self.consume()?.greater(),
'#' => self.consume()?.produce(Type::Hash, ()), '#' => self.consume()?.hash(),
'<' => self.consume()?.less(), '<' => self.consume()?.less(),
'-' => self.consume()?.minus(), '-' => self.consume()?.minus(),
'+' => self.consume()?.plus(), '+' => self.consume()?.plus(),
@ -203,6 +203,12 @@ impl<'t> Lexer<'t> {
_ => self.produce(Type::Bar, ()), _ => self.produce(Type::Bar, ()),
} }
} }
fn colon(&mut self) -> LResult<Token> {
match self.peek() {
Ok(':') => self.consume()?.produce(Type::ColonColon, ()),
_ => self.produce(Type::Colon, ()),
}
}
fn dot(&mut self) -> LResult<Token> { fn dot(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('.') => { Ok('.') => {
@ -235,6 +241,12 @@ impl<'t> Lexer<'t> {
_ => self.produce(Type::Gt, ()), _ => self.produce(Type::Gt, ()),
} }
} }
fn hash(&mut self) -> LResult<Token> {
match self.peek() {
Ok('!') => self.consume()?.produce(Type::HashBang, ()),
_ => self.produce(Type::Hash, ()),
}
}
fn less(&mut self) -> LResult<Token> { fn less(&mut self) -> LResult<Token> {
match self.peek() { match self.peek() {
Ok('=') => self.consume()?.produce(Type::LtEq, ()), Ok('=') => self.consume()?.produce(Type::LtEq, ()),

108
libconlang/src/token/token_type.rs
View File

@ -17,58 +17,60 @@ pub enum Type {
String, String,
Character, Character,
// Delimiters and punctuation // Delimiters and punctuation
LCurly, // { LCurly, // {
RCurly, // } RCurly, // }
LBrack, // [ LBrack, // [
RBrack, // ] RBrack, // ]
LParen, // ( LParen, // (
RParen, // ) RParen, // )
Amp, // & Amp, // &
AmpAmp, // && AmpAmp, // &&
AmpEq, // &= AmpEq, // &=
Arrow, // -> Arrow, // ->
At, // @ At, // @
Backslash, // \ Backslash, // \
Bang, // ! Bang, // !
BangBang, // !! BangBang, // !!
BangEq, // != BangEq, // !=
Bar, // | Bar, // |
BarBar, // || BarBar, // ||
BarEq, // |= BarEq, // |=
Colon, // : Colon, // :
Comma, // , ColonColon, // ::
Dot, // . Comma, // ,
DotDot, // .. Dot, // .
DotDotEq, // ..= DotDot, // ..
Eq, // = DotDotEq, // ..=
EqEq, // == Eq, // =
FatArrow, // => EqEq, // ==
Grave, // ` FatArrow, // =>
Gt, // > Grave, // `
GtEq, // >= Gt, // >
GtGt, // >> GtEq, // >=
GtGtEq, // >>= GtGt, // >>
Hash, // # GtGtEq, // >>=
Lt, // < Hash, // #
LtEq, // <= HashBang, // #!
LtLt, // << Lt, // <
LtLtEq, // <<= LtEq, // <=
Minus, // - LtLt, // <<
MinusEq, // -= LtLtEq, // <<=
Plus, // + Minus, // -
PlusEq, // += MinusEq, // -=
Question, // ? Plus, // +
Rem, // % PlusEq, // +=
RemEq, // %= Question, // ?
Semi, // ; Rem, // %
Slash, // / RemEq, // %=
SlashEq, // /= Semi, // ;
Star, // * Slash, // /
StarEq, // *= SlashEq, // /=
Tilde, // ~ Star, // *
Xor, // ^ StarEq, // *=
XorEq, // ^= Tilde, // ~
XorXor, // ^^ Xor, // ^
XorEq, // ^=
XorXor, // ^^
} }
/// Represents a reserved word. /// Represents a reserved word.
@ -119,6 +121,7 @@ impl Display for Type {
Type::BarBar => "or-or".fmt(f), Type::BarBar => "or-or".fmt(f),
Type::BarEq => "or-assign".fmt(f), Type::BarEq => "or-assign".fmt(f),
Type::Colon => "colon".fmt(f), Type::Colon => "colon".fmt(f),
Type::ColonColon => "path separator".fmt(f),
Type::Comma => "comma".fmt(f), Type::Comma => "comma".fmt(f),
Type::Dot => "dot".fmt(f), Type::Dot => "dot".fmt(f),
Type::DotDot => "exclusive range".fmt(f), Type::DotDot => "exclusive range".fmt(f),
@ -132,6 +135,7 @@ impl Display for Type {
Type::GtGt => "shift right".fmt(f), Type::GtGt => "shift right".fmt(f),
Type::GtGtEq => "shift right-assign".fmt(f), Type::GtGtEq => "shift right-assign".fmt(f),
Type::Hash => "hash".fmt(f), Type::Hash => "hash".fmt(f),
Type::HashBang => "shebang".fmt(f),
Type::Lt => "less than".fmt(f), Type::Lt => "less than".fmt(f),
Type::LtEq => "less than or equal to".fmt(f), Type::LtEq => "less than or equal to".fmt(f),
Type::LtLt => "shift left".fmt(f), Type::LtLt => "shift left".fmt(f),