token::token_data::Data: Renamed from token::TokenData

& Moved into its own module
+ Added token::preamble; common imports when working with Token

libconlang/src/token/token_data.rs

@@ -0,0 +1,45 @@
+//! Additional data stored within a [Token](super::Token),
+//! external to its [Type](super::token_type::Type)
+/// Additional data stored within a [Token](super::Token),
+/// external to its [Type](super::token_type::Type)
+#[derive(Clone, Debug, PartialEq)]
+pub enum Data {
+    /// [Token](super::Token) contains an [identifier](str)
+    Identifier(Box<str>),
+    /// [Token](super::Token) contains a [String]
+    String(String),
+    /// [Token](super::Token) contains a [character](char)
+    Character(char),
+    /// [Token](super::Token) contains an [integer](u128)
+    Integer(u128),
+    /// [Token](super::Token) contains a [float](f64)
+    Float(f64),
+    /// [Token](super::Token) contains no additional data
+    None,
+}
+from! {
+    value: &str => Self::Identifier(value.into()),
+    value: String => Self::String(value),
+    value: u128 => Self::Integer(value),
+    value: f64 => Self::Float(value),
+    value: char => Self::Character(value),
+    _v:    () => Self::None,
+}
+/// Implements [From] for an enum
+macro from($($value:ident: $src:ty => $dst:expr),*$(,)?) {
+    $(impl From<$src> for Data {
+        fn from($value: $src) -> Self { $dst }
+    })*
+}
+impl std::fmt::Display for Data {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Data::Identifier(v) => v.fmt(f),
+            Data::String(v) => write!(f, "\"{v}\""),
+            Data::Character(v) => write!(f, "'{v}'"),
+            Data::Integer(v) => v.fmt(f),
+            Data::Float(v) => v.fmt(f),
+            Data::None => "None".fmt(f),
+        }
+    }
+}

libconlang/src/token/token_type.rs

@@ -1,6 +1,92 @@
-//! Trait impls and helper functions for [Type] and [Keyword]
-use super::{Keyword, Type};
-use std::fmt::Display;
+//! Stores a [Token's](super::Token) lexical information
+use std::{fmt::Display, str::FromStr};
+
+/// Stores a [Token's](super::Token) lexical information
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum Type {
+    // Invalid syntax
+    Invalid,
+    // Any kind of comment
+    Comment,
+    // Any identifier
+    Identifier,
+    Keyword(Keyword),
+    // Literals
+    Integer,
+    Float,
+    String,
+    Character,
+    // Delimiters and punctuation
+    LCurly,    // {
+    RCurly,    // }
+    LBrack,    // [
+    RBrack,    // ]
+    LParen,    // (
+    RParen,    // )
+    Amp,       // &
+    AmpAmp,    // &&
+    AmpEq,     // &=
+    Arrow,     // ->
+    At,        // @
+    Backslash, // \
+    Bang,      // !
+    BangBang,  // !!
+    BangEq,    // !=
+    Bar,       // |
+    BarBar,    // ||
+    BarEq,     // |=
+    Colon,     // :
+    Comma,     // ,
+    Dot,       // .
+    DotDot,    // ..
+    DotDotEq,  // ..=
+    Eq,        // =
+    EqEq,      // ==
+    FatArrow,  // =>
+    Grave,     // `
+    Gt,        // >
+    GtEq,      // >=
+    GtGt,      // >>
+    GtGtEq,    // >>=
+    Hash,      // #
+    Lt,        // <
+    LtEq,      // <=
+    LtLt,      // <<
+    LtLtEq,    // <<=
+    Minus,     // -
+    MinusEq,   // -=
+    Plus,      // +
+    PlusEq,    // +=
+    Question,  // ?
+    Rem,       // %
+    RemEq,     // %=
+    Semi,      // ;
+    Slash,     // /
+    SlashEq,   // /=
+    Star,      // *
+    StarEq,    // *=
+    Tilde,     // ~
+    Xor,       // ^
+    XorEq,     // ^=
+    XorXor,    // ^^
+}
+
+/// Represents a reserved word.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum Keyword {
+    Break,
+    Continue,
+    Else,
+    False,
+    For,
+    Fn,
+    If,
+    In,
+    Let,
+    Return,
+    True,
+    While,
+}
 
 impl Display for Type {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -87,8 +173,9 @@ impl Display for Keyword {
         }
     }
 }
-impl std::str::FromStr for Keyword {
-    type Err = (); // If an identifier isn't a keyword, that's okay.
+impl FromStr for Keyword {
+    /// [FromStr] can only fail when an identifier isn't a keyword
+    type Err = ();
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(match s {
             "break" => Self::Break,