cl-token: Move token definition into its own crate
This commit is contained in:
@@ -14,3 +14,4 @@ repository.workspace = true
|
||||
[dependencies]
|
||||
unicode-xid = "0.2.4"
|
||||
cl-structures = { path = "../cl-structures" }
|
||||
cl-token = { path = "../cl-token" }
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Converts a text file into tokens
|
||||
use crate::token::preamble::*;
|
||||
use cl_token::*;
|
||||
use cl_structures::span::Loc;
|
||||
use std::{
|
||||
iter::Peekable,
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
#![warn(clippy::all)]
|
||||
#![feature(decl_macro)]
|
||||
|
||||
pub mod token;
|
||||
|
||||
pub mod ast;
|
||||
|
||||
pub mod lexer;
|
||||
|
||||
@@ -12,13 +12,9 @@ use self::error::{
|
||||
use crate::{
|
||||
ast::*,
|
||||
lexer::{error::Error as LexError, Lexer},
|
||||
token::{
|
||||
token_data::Data,
|
||||
token_type::{Keyword, Type},
|
||||
Token,
|
||||
},
|
||||
};
|
||||
use cl_structures::span::*;
|
||||
use cl_token::*;
|
||||
|
||||
pub mod error {
|
||||
use std::fmt::Display;
|
||||
|
||||
@@ -5,8 +5,8 @@ mod ast {
|
||||
// TODO
|
||||
}
|
||||
mod lexer {
|
||||
#[allow(unused_imports)]
|
||||
use crate::{lexer::Lexer, token::preamble::*};
|
||||
use crate::lexer::Lexer;
|
||||
use cl_token::*;
|
||||
|
||||
macro test_lexer_output_type ($($f:ident {$($test:expr => $expect:expr),*$(,)?})*) {$(
|
||||
#[test]
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
//! # Token
|
||||
//!
|
||||
//! Stores a component of a file as a [Type], some [Data], and a line and column number
|
||||
|
||||
pub mod token_data;
|
||||
pub mod token_type;
|
||||
pub mod preamble {
|
||||
//! Common imports for working with [tokens](super)
|
||||
pub use super::{
|
||||
token_data::Data,
|
||||
token_type::{Keyword, Type},
|
||||
Token,
|
||||
};
|
||||
}
|
||||
|
||||
use token_data::Data;
|
||||
use token_type::Type;
|
||||
|
||||
/// Contains a single unit of lexical information,
|
||||
/// and an optional bit of [Data]
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Token {
|
||||
ty: Type,
|
||||
data: Data,
|
||||
line: u32,
|
||||
col: u32,
|
||||
}
|
||||
impl Token {
|
||||
/// Creates a new [Token] out of a [Type], [Data], line, and column.
|
||||
pub fn new(ty: Type, data: impl Into<Data>, line: u32, col: u32) -> Self {
|
||||
Self { ty, data: data.into(), line, col }
|
||||
}
|
||||
/// Casts this token to a new [Type]
|
||||
pub fn cast(self, ty: Type) -> Self {
|
||||
Self { ty, ..self }
|
||||
}
|
||||
/// Returns the [Type] of this token
|
||||
pub fn ty(&self) -> Type {
|
||||
self.ty
|
||||
}
|
||||
/// Returns a reference to this token's [Data]
|
||||
pub fn data(&self) -> &Data {
|
||||
&self.data
|
||||
}
|
||||
/// Converts this token into its inner [Data]
|
||||
pub fn into_data(self) -> Data {
|
||||
self.data
|
||||
}
|
||||
/// Returns the line where this token originated
|
||||
pub fn line(&self) -> u32 {
|
||||
self.line
|
||||
}
|
||||
/// Returns the column where this token originated
|
||||
pub fn col(&self) -> u32 {
|
||||
self.col
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
//! Additional data stored within a [Token](super::Token),
|
||||
//! external to its [Type](super::token_type::Type)
|
||||
/// Additional data stored within a [Token](super::Token),
|
||||
/// external to its [Type](super::token_type::Type)
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Data {
|
||||
/// [Token](super::Token) contains an [identifier](str)
|
||||
Identifier(Box<str>),
|
||||
/// [Token](super::Token) contains a [String]
|
||||
String(String),
|
||||
/// [Token](super::Token) contains a [character](char)
|
||||
Character(char),
|
||||
/// [Token](super::Token) contains an [integer](u128)
|
||||
Integer(u128),
|
||||
/// [Token](super::Token) contains a [float](f64)
|
||||
Float(f64),
|
||||
/// [Token](super::Token) contains no additional data
|
||||
None,
|
||||
}
|
||||
from! {
|
||||
value: &str => Self::Identifier(value.into()),
|
||||
value: String => Self::String(value),
|
||||
value: u128 => Self::Integer(value),
|
||||
value: f64 => Self::Float(value),
|
||||
value: char => Self::Character(value),
|
||||
_v: () => Self::None,
|
||||
}
|
||||
/// Implements [From] for an enum
|
||||
macro from($($value:ident: $src:ty => $dst:expr),*$(,)?) {
|
||||
$(impl From<$src> for Data {
|
||||
fn from($value: $src) -> Self { $dst }
|
||||
})*
|
||||
}
|
||||
impl std::fmt::Display for Data {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Data::Identifier(v) => v.fmt(f),
|
||||
Data::String(v) => write!(f, "\"{v}\""),
|
||||
Data::Character(v) => write!(f, "'{v}'"),
|
||||
Data::Integer(v) => v.fmt(f),
|
||||
Data::Float(v) => v.fmt(f),
|
||||
Data::None => "None".fmt(f),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,239 +0,0 @@
|
||||
//! Stores a [Token's](super::Token) lexical information
|
||||
use std::{fmt::Display, str::FromStr};
|
||||
|
||||
/// Stores a [Token's](super::Token) lexical information
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum Type {
|
||||
// Invalid syntax
|
||||
Invalid,
|
||||
// Any kind of comment
|
||||
Comment,
|
||||
// Any identifier
|
||||
Identifier,
|
||||
Keyword(Keyword),
|
||||
// Literals
|
||||
Integer,
|
||||
Float,
|
||||
String,
|
||||
Character,
|
||||
// Delimiters and punctuation
|
||||
LCurly, // {
|
||||
RCurly, // }
|
||||
LBrack, // [
|
||||
RBrack, // ]
|
||||
LParen, // (
|
||||
RParen, // )
|
||||
Amp, // &
|
||||
AmpAmp, // &&
|
||||
AmpEq, // &=
|
||||
Arrow, // ->
|
||||
At, // @
|
||||
Backslash, // \
|
||||
Bang, // !
|
||||
BangBang, // !!
|
||||
BangEq, // !=
|
||||
Bar, // |
|
||||
BarBar, // ||
|
||||
BarEq, // |=
|
||||
Colon, // :
|
||||
ColonColon, // ::
|
||||
Comma, // ,
|
||||
Dot, // .
|
||||
DotDot, // ..
|
||||
DotDotEq, // ..=
|
||||
Eq, // =
|
||||
EqEq, // ==
|
||||
FatArrow, // =>
|
||||
Grave, // `
|
||||
Gt, // >
|
||||
GtEq, // >=
|
||||
GtGt, // >>
|
||||
GtGtEq, // >>=
|
||||
Hash, // #
|
||||
HashBang, // #!
|
||||
Lt, // <
|
||||
LtEq, // <=
|
||||
LtLt, // <<
|
||||
LtLtEq, // <<=
|
||||
Minus, // -
|
||||
MinusEq, // -=
|
||||
Plus, // +
|
||||
PlusEq, // +=
|
||||
Question, // ?
|
||||
Rem, // %
|
||||
RemEq, // %=
|
||||
Semi, // ;
|
||||
Slash, // /
|
||||
SlashEq, // /=
|
||||
Star, // *
|
||||
StarEq, // *=
|
||||
Tilde, // ~
|
||||
Xor, // ^
|
||||
XorEq, // ^=
|
||||
XorXor, // ^^
|
||||
}
|
||||
|
||||
/// Represents a reserved word.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum Keyword {
|
||||
Break,
|
||||
Cl,
|
||||
Const,
|
||||
Continue,
|
||||
Else,
|
||||
Enum,
|
||||
False,
|
||||
For,
|
||||
Fn,
|
||||
If,
|
||||
Impl,
|
||||
In,
|
||||
Let,
|
||||
Mod,
|
||||
Mut,
|
||||
Pub,
|
||||
Return,
|
||||
SelfKw,
|
||||
SelfTy,
|
||||
Static,
|
||||
Struct,
|
||||
Super,
|
||||
True,
|
||||
Type,
|
||||
While,
|
||||
}
|
||||
|
||||
impl Display for Type {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Type::Invalid => "invalid".fmt(f),
|
||||
Type::Comment => "comment".fmt(f),
|
||||
Type::Identifier => "identifier".fmt(f),
|
||||
Type::Keyword(k) => k.fmt(f),
|
||||
Type::Integer => "integer literal".fmt(f),
|
||||
Type::Float => "float literal".fmt(f),
|
||||
Type::String => "string literal".fmt(f),
|
||||
Type::Character => "char literal".fmt(f),
|
||||
Type::LCurly => "left curly".fmt(f),
|
||||
Type::RCurly => "right curly".fmt(f),
|
||||
Type::LBrack => "left brack".fmt(f),
|
||||
Type::RBrack => "right brack".fmt(f),
|
||||
Type::LParen => "left paren".fmt(f),
|
||||
Type::RParen => "right paren".fmt(f),
|
||||
Type::Amp => "and".fmt(f),
|
||||
Type::AmpAmp => "and-and".fmt(f),
|
||||
Type::AmpEq => "and-assign".fmt(f),
|
||||
Type::Arrow => "arrow".fmt(f),
|
||||
Type::At => "at".fmt(f),
|
||||
Type::Backslash => "backslash".fmt(f),
|
||||
Type::Bang => "bang".fmt(f),
|
||||
Type::BangBang => "not-not".fmt(f),
|
||||
Type::BangEq => "not equal to".fmt(f),
|
||||
Type::Bar => "or".fmt(f),
|
||||
Type::BarBar => "or-or".fmt(f),
|
||||
Type::BarEq => "or-assign".fmt(f),
|
||||
Type::Colon => "colon".fmt(f),
|
||||
Type::ColonColon => "path separator".fmt(f),
|
||||
Type::Comma => "comma".fmt(f),
|
||||
Type::Dot => "dot".fmt(f),
|
||||
Type::DotDot => "exclusive range".fmt(f),
|
||||
Type::DotDotEq => "inclusive range".fmt(f),
|
||||
Type::Eq => "assign".fmt(f),
|
||||
Type::EqEq => "equal to".fmt(f),
|
||||
Type::FatArrow => "fat arrow".fmt(f),
|
||||
Type::Grave => "grave".fmt(f),
|
||||
Type::Gt => "greater than".fmt(f),
|
||||
Type::GtEq => "greater than or equal to".fmt(f),
|
||||
Type::GtGt => "shift right".fmt(f),
|
||||
Type::GtGtEq => "shift right-assign".fmt(f),
|
||||
Type::Hash => "hash".fmt(f),
|
||||
Type::HashBang => "shebang".fmt(f),
|
||||
Type::Lt => "less than".fmt(f),
|
||||
Type::LtEq => "less than or equal to".fmt(f),
|
||||
Type::LtLt => "shift left".fmt(f),
|
||||
Type::LtLtEq => "shift left-assign".fmt(f),
|
||||
Type::Minus => "sub".fmt(f),
|
||||
Type::MinusEq => "sub-assign".fmt(f),
|
||||
Type::Plus => "add".fmt(f),
|
||||
Type::PlusEq => "add-assign".fmt(f),
|
||||
Type::Question => "huh?".fmt(f),
|
||||
Type::Rem => "rem".fmt(f),
|
||||
Type::RemEq => "rem-assign".fmt(f),
|
||||
Type::Semi => "ignore".fmt(f),
|
||||
Type::Slash => "div".fmt(f),
|
||||
Type::SlashEq => "div-assign".fmt(f),
|
||||
Type::Star => "star".fmt(f),
|
||||
Type::StarEq => "star-assign".fmt(f),
|
||||
Type::Tilde => "tilde".fmt(f),
|
||||
Type::Xor => "xor".fmt(f),
|
||||
Type::XorEq => "xor-assign".fmt(f),
|
||||
Type::XorXor => "cat-ears".fmt(f),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for Keyword {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Break => "break".fmt(f),
|
||||
Self::Cl => "cl".fmt(f),
|
||||
Self::Const => "const".fmt(f),
|
||||
Self::Continue => "continue".fmt(f),
|
||||
Self::Else => "else".fmt(f),
|
||||
Self::Enum => "enum".fmt(f),
|
||||
Self::False => "false".fmt(f),
|
||||
Self::For => "for".fmt(f),
|
||||
Self::Fn => "fn".fmt(f),
|
||||
Self::If => "if".fmt(f),
|
||||
Self::Impl => "impl".fmt(f),
|
||||
Self::In => "in".fmt(f),
|
||||
Self::Let => "let".fmt(f),
|
||||
Self::Mod => "mod".fmt(f),
|
||||
Self::Mut => "mut".fmt(f),
|
||||
Self::Pub => "pub".fmt(f),
|
||||
Self::Return => "return".fmt(f),
|
||||
Self::SelfKw => "self".fmt(f),
|
||||
Self::SelfTy => "Self".fmt(f),
|
||||
Self::Static => "static".fmt(f),
|
||||
Self::Struct => "struct".fmt(f),
|
||||
Self::Super => "super".fmt(f),
|
||||
Self::True => "true".fmt(f),
|
||||
Self::Type => "type".fmt(f),
|
||||
Self::While => "while".fmt(f),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl FromStr for Keyword {
|
||||
/// [FromStr] can only fail when an identifier isn't a keyword
|
||||
type Err = ();
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
Ok(match s {
|
||||
"break" => Self::Break,
|
||||
"cl" => Self::Cl,
|
||||
"const" => Self::Const,
|
||||
"continue" => Self::Continue,
|
||||
"else" => Self::Else,
|
||||
"enum" => Self::Enum,
|
||||
"false" => Self::False,
|
||||
"for" => Self::For,
|
||||
"fn" => Self::Fn,
|
||||
"if" => Self::If,
|
||||
"impl" => Self::Impl,
|
||||
"in" => Self::In,
|
||||
"let" => Self::Let,
|
||||
"mod" => Self::Mod,
|
||||
"mut" => Self::Mut,
|
||||
"pub" => Self::Pub,
|
||||
"return" => Self::Return,
|
||||
"self" => Self::SelfKw,
|
||||
"Self" => Self::SelfTy,
|
||||
"static" => Self::Static,
|
||||
"struct" => Self::Struct,
|
||||
"super" => Self::Super,
|
||||
"true" => Self::True,
|
||||
"type" => Self::Type,
|
||||
"while" => Self::While,
|
||||
_ => Err(())?,
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user