token: Move module into file

libconlang/src/lib.rs

@@ -1,159 +1,7 @@
 //! Conlang is an expression-based programming language with similarities to Rust
 #![warn(clippy::all)]
 #![feature(decl_macro)]
-pub mod token {
+pub mod token;
-    //! Stores a component of a file as a type and span
-    use std::ops::Range;
-
-    mod token_type;
-    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
-    pub enum Type {
-        // Invalid syntax
-        Invalid,
-        // Any kind of comment
-        Comment,
-        // Any identifier
-        Identifier,
-        Keyword(Keyword),
-        // Literals
-        Integer,
-        Float,
-        String,
-        Character,
-        // Delimiters
-        LCurly,
-        RCurly,
-        LBrack,
-        RBrack,
-        LParen,
-        RParen,
-        // Compound punctuation
-        Lsh,
-        Rsh,
-        AmpAmp,
-        BarBar,
-        NotNot,
-        CatEar,
-        EqEq,
-        GtEq,
-        LtEq,
-        NotEq,
-        StarEq,
-        DivEq,
-        RemEq,
-        AddEq,
-        SubEq,
-        AndEq,
-        OrEq,
-        XorEq,
-        LshEq,
-        RshEq,
-        Arrow,
-        FatArrow,
-        // Simple punctuation
-        Semi,
-        Dot,
-        Star,
-        Div,
-        Plus,
-        Minus,
-        Rem,
-        Bang,
-        Eq,
-        Lt,
-        Gt,
-        Amp,
-        Bar,
-        Xor,
-        Hash,
-        At,
-        Colon,
-        Backslash,
-        Question,
-        Comma,
-        Tilde,
-        Grave,
-    }
-
-    /// Represents a reserved word.
-    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
-    pub enum Keyword {
-        Break,
-        Continue,
-        Else,
-        False,
-        For,
-        Fn,
-        If,
-        In,
-        Let,
-        Return,
-        True,
-        While,
-    }
-    impl std::str::FromStr for Keyword {
-        type Err = ();
-        fn from_str(s: &str) -> Result<Self, Self::Err> {
-            Ok(match s {
-                "break" => Self::Break,
-                "continue" => Self::Continue,
-                "else" => Self::Else,
-                "false" => Self::False,
-                "for" => Self::For,
-                "fn" => Self::Fn,
-                "if" => Self::If,
-                "in" => Self::In,
-                "let" => Self::Let,
-                "return" => Self::Return,
-                "true" => Self::True,
-                "while" => Self::While,
-                _ => Err(())?,
-            })
-        }
-    }
-
-    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
-    pub struct Token {
-        ty: Type,
-        pub head: usize,
-        pub tail: usize,
-        line: usize,
-        col: usize,
-    }
-    impl Token {
-        pub fn new(ty: Type, head: usize, tail: usize, line: usize, col: usize) -> Self {
-            Self { ty, head, tail, line, col }
-        }
-        pub fn cast(self, ty: Type) -> Self {
-            Self { ty, ..self }
-        }
-        // Hack to work around
-        pub fn rebound(self, head: usize, tail: usize) -> Self {
-            Self { head, tail, ..self }
-        }
-        pub fn line(&self) -> usize {
-            self.line
-        }
-        pub fn col(&self) -> usize {
-            self.col
-        }
-        pub fn is_empty(&self) -> bool {
-            self.tail == self.head
-        }
-        /// Gets the length of the token, in bytes
-        pub fn len(&self) -> usize {
-            self.tail - self.head
-        }
-        /// Gets the [Type] of the token
-        pub fn ty(&self) -> Type {
-            self.ty
-        }
-        /// Gets the exclusive range of the token
-        pub fn range(&self) -> Range<usize> {
-            self.head..self.tail
-        }
-    }
-}
 
 pub mod ast {
     //! # The Abstract Syntax Tree

libconlang/src/token.rs

@@ -0,0 +1,134 @@
+//! Stores a component of a file as a type and span
+use std::ops::Range;
+
+mod token_type;
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Type {
+    // Invalid syntax
+    Invalid,
+    // Any kind of comment
+    Comment,
+    // Any identifier
+    Identifier,
+    Keyword(Keyword),
+    // Literals
+    Integer,
+    Float,
+    String,
+    Character,
+    // Delimiters
+    LCurly,
+    RCurly,
+    LBrack,
+    RBrack,
+    LParen,
+    RParen,
+    // Compound punctuation
+    Lsh,
+    Rsh,
+    AmpAmp,
+    BarBar,
+    NotNot,
+    CatEar,
+    EqEq,
+    GtEq,
+    LtEq,
+    NotEq,
+    StarEq,
+    DivEq,
+    RemEq,
+    AddEq,
+    SubEq,
+    AndEq,
+    OrEq,
+    XorEq,
+    LshEq,
+    RshEq,
+    Arrow,
+    FatArrow,
+    // Simple punctuation
+    Semi,
+    Dot,
+    Star,
+    Div,
+    Plus,
+    Minus,
+    Rem,
+    Bang,
+    Eq,
+    Lt,
+    Gt,
+    Amp,
+    Bar,
+    Xor,
+    Hash,
+    At,
+    Colon,
+    Backslash,
+    Question,
+    Comma,
+    Tilde,
+    Grave,
+}
+
+/// Represents a reserved word.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Keyword {
+    Break,
+    Continue,
+    Else,
+    False,
+    For,
+    Fn,
+    If,
+    In,
+    Let,
+    Return,
+    True,
+    While,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Token {
+    ty: Type,
+    pub head: usize,
+    pub tail: usize,
+    line: usize,
+    col: usize,
+}
+impl Token {
+    pub fn new(ty: Type, head: usize, tail: usize, line: usize, col: usize) -> Self {
+        Self { ty, head, tail, line, col }
+    }
+    /// Cast this [Token] to a new [Type]
+    pub fn cast(self, ty: Type) -> Self {
+        Self { ty, ..self }
+    }
+    /// Hack to work around the current [lexer's design limitations](crate::lexer)
+    pub fn rebound(self, head: usize, tail: usize) -> Self {
+        Self { head, tail, ..self }
+    }
+    /// Gets the line from this token
+    pub fn line(&self) -> usize {
+        self.line
+    }
+    /// Gets the column from this token
+    pub fn col(&self) -> usize {
+        self.col
+    }
+    pub fn is_empty(&self) -> bool {
+        self.tail == self.head
+    }
+    /// Gets the length of the token, in bytes
+    pub fn len(&self) -> usize {
+        self.tail - self.head
+    }
+    /// Gets the [Type] of the token
+    pub fn ty(&self) -> Type {
+        self.ty
+    }
+    /// Gets the exclusive range of the token
+    pub fn range(&self) -> Range<usize> {
+        self.head..self.tail
+    }
+}

libconlang/src/token/token_type.rs

@@ -85,3 +85,23 @@ impl Display for Keyword {
         }
     }
 }
+impl std::str::FromStr for Keyword {
+    type Err = ();
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "break" => Self::Break,
+            "continue" => Self::Continue,
+            "else" => Self::Else,
+            "false" => Self::False,
+            "for" => Self::For,
+            "fn" => Self::Fn,
+            "if" => Self::If,
+            "in" => Self::In,
+            "let" => Self::Let,
+            "return" => Self::Return,
+            "true" => Self::True,
+            "while" => Self::While,
+            _ => Err(())?,
+        })
+    }
+}