From 5d9c45c26a2b1f7641a9f1ba4467dcc0d3f93825 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 17 Oct 2023 13:23:34 -0500 Subject: [PATCH] token: Move module into file --- libconlang/src/lib.rs | 154 +---------------------------- libconlang/src/token.rs | 134 +++++++++++++++++++++++++ libconlang/src/token/token_type.rs | 20 ++++ 3 files changed, 155 insertions(+), 153 deletions(-) create mode 100644 libconlang/src/token.rs diff --git a/libconlang/src/lib.rs b/libconlang/src/lib.rs index 7fd54ac..5cc1329 100644 --- a/libconlang/src/lib.rs +++ b/libconlang/src/lib.rs @@ -1,159 +1,7 @@ //! Conlang is an expression-based programming language with similarities to Rust #![warn(clippy::all)] #![feature(decl_macro)] -pub mod token { - //! Stores a component of a file as a type and span - use std::ops::Range; - - mod token_type; - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - pub enum Type { - // Invalid syntax - Invalid, - // Any kind of comment - Comment, - // Any identifier - Identifier, - Keyword(Keyword), - // Literals - Integer, - Float, - String, - Character, - // Delimiters - LCurly, - RCurly, - LBrack, - RBrack, - LParen, - RParen, - // Compound punctuation - Lsh, - Rsh, - AmpAmp, - BarBar, - NotNot, - CatEar, - EqEq, - GtEq, - LtEq, - NotEq, - StarEq, - DivEq, - RemEq, - AddEq, - SubEq, - AndEq, - OrEq, - XorEq, - LshEq, - RshEq, - Arrow, - FatArrow, - // Simple punctuation - Semi, - Dot, - Star, - Div, - Plus, - Minus, - Rem, - Bang, - Eq, - Lt, - Gt, - Amp, - Bar, - Xor, - Hash, - At, - Colon, - Backslash, - Question, - Comma, - Tilde, - Grave, - } - - /// Represents a reserved word. - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - pub enum Keyword { - Break, - Continue, - Else, - False, - For, - Fn, - If, - In, - Let, - Return, - True, - While, - } - impl std::str::FromStr for Keyword { - type Err = (); - fn from_str(s: &str) -> Result { - Ok(match s { - "break" => Self::Break, - "continue" => Self::Continue, - "else" => Self::Else, - "false" => Self::False, - "for" => Self::For, - "fn" => Self::Fn, - "if" => Self::If, - "in" => Self::In, - "let" => Self::Let, - "return" => Self::Return, - "true" => Self::True, - "while" => Self::While, - _ => Err(())?, - }) - } - } - - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - pub struct Token { - ty: Type, - pub head: usize, - pub tail: usize, - line: usize, - col: usize, - } - impl Token { - pub fn new(ty: Type, head: usize, tail: usize, line: usize, col: usize) -> Self { - Self { ty, head, tail, line, col } - } - pub fn cast(self, ty: Type) -> Self { - Self { ty, ..self } - } - // Hack to work around - pub fn rebound(self, head: usize, tail: usize) -> Self { - Self { head, tail, ..self } - } - pub fn line(&self) -> usize { - self.line - } - pub fn col(&self) -> usize { - self.col - } - pub fn is_empty(&self) -> bool { - self.tail == self.head - } - /// Gets the length of the token, in bytes - pub fn len(&self) -> usize { - self.tail - self.head - } - /// Gets the [Type] of the token - pub fn ty(&self) -> Type { - self.ty - } - /// Gets the exclusive range of the token - pub fn range(&self) -> Range { - self.head..self.tail - } - } -} +pub mod token; pub mod ast { //! # The Abstract Syntax Tree diff --git a/libconlang/src/token.rs b/libconlang/src/token.rs new file mode 100644 index 0000000..c8d1ffb --- /dev/null +++ b/libconlang/src/token.rs @@ -0,0 +1,134 @@ +//! Stores a component of a file as a type and span +use std::ops::Range; + +mod token_type; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Type { + // Invalid syntax + Invalid, + // Any kind of comment + Comment, + // Any identifier + Identifier, + Keyword(Keyword), + // Literals + Integer, + Float, + String, + Character, + // Delimiters + LCurly, + RCurly, + LBrack, + RBrack, + LParen, + RParen, + // Compound punctuation + Lsh, + Rsh, + AmpAmp, + BarBar, + NotNot, + CatEar, + EqEq, + GtEq, + LtEq, + NotEq, + StarEq, + DivEq, + RemEq, + AddEq, + SubEq, + AndEq, + OrEq, + XorEq, + LshEq, + RshEq, + Arrow, + FatArrow, + // Simple punctuation + Semi, + Dot, + Star, + Div, + Plus, + Minus, + Rem, + Bang, + Eq, + Lt, + Gt, + Amp, + Bar, + Xor, + Hash, + At, + Colon, + Backslash, + Question, + Comma, + Tilde, + Grave, +} + +/// Represents a reserved word. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Keyword { + Break, + Continue, + Else, + False, + For, + Fn, + If, + In, + Let, + Return, + True, + While, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Token { + ty: Type, + pub head: usize, + pub tail: usize, + line: usize, + col: usize, +} +impl Token { + pub fn new(ty: Type, head: usize, tail: usize, line: usize, col: usize) -> Self { + Self { ty, head, tail, line, col } + } + /// Cast this [Token] to a new [Type] + pub fn cast(self, ty: Type) -> Self { + Self { ty, ..self } + } + /// Hack to work around the current [lexer's design limitations](crate::lexer) + pub fn rebound(self, head: usize, tail: usize) -> Self { + Self { head, tail, ..self } + } + /// Gets the line from this token + pub fn line(&self) -> usize { + self.line + } + /// Gets the column from this token + pub fn col(&self) -> usize { + self.col + } + pub fn is_empty(&self) -> bool { + self.tail == self.head + } + /// Gets the length of the token, in bytes + pub fn len(&self) -> usize { + self.tail - self.head + } + /// Gets the [Type] of the token + pub fn ty(&self) -> Type { + self.ty + } + /// Gets the exclusive range of the token + pub fn range(&self) -> Range { + self.head..self.tail + } +} diff --git a/libconlang/src/token/token_type.rs b/libconlang/src/token/token_type.rs index 4a83227..bb1b98c 100644 --- a/libconlang/src/token/token_type.rs +++ b/libconlang/src/token/token_type.rs @@ -85,3 +85,23 @@ impl Display for Keyword { } } } +impl std::str::FromStr for Keyword { + type Err = (); + fn from_str(s: &str) -> Result { + Ok(match s { + "break" => Self::Break, + "continue" => Self::Continue, + "else" => Self::Else, + "false" => Self::False, + "for" => Self::For, + "fn" => Self::Fn, + "if" => Self::If, + "in" => Self::In, + "let" => Self::Let, + "return" => Self::Return, + "true" => Self::True, + "while" => Self::While, + _ => Err(())?, + }) + } +}