From 6e1d5af134164e26f22b5abf003c7f8415aae09a Mon Sep 17 00:00:00 2001 From: John Date: Thu, 29 Feb 2024 19:36:06 -0600 Subject: [PATCH] cl-token: Move token definition into its own crate --- Cargo.toml | 8 +++++++- cl-repl/Cargo.toml | 1 + cl-repl/examples/identify_tokens.rs | 3 ++- cl-repl/src/lib.rs | 3 ++- cl-token/Cargo.toml | 10 ++++++++++ cl-token/src/lib.rs | 12 ++++++++++++ {libconlang => cl-token}/src/token.rs | 19 ++----------------- .../src/token => cl-token/src}/token_data.rs | 0 .../src/token => cl-token/src}/token_type.rs | 0 libconlang/Cargo.toml | 1 + libconlang/src/lexer.rs | 2 +- libconlang/src/lib.rs | 2 -- libconlang/src/parser.rs | 6 +----- libconlang/src/tests.rs | 4 ++-- 14 files changed, 41 insertions(+), 30 deletions(-) create mode 100644 cl-token/Cargo.toml create mode 100644 cl-token/src/lib.rs rename {libconlang => cl-token}/src/token.rs (74%) rename {libconlang/src/token => cl-token/src}/token_data.rs (100%) rename {libconlang/src/token => cl-token/src}/token_type.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 62902bf..9168de3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,11 @@ [workspace] -members = ["libconlang", "cl-repl", "cl-interpret", "cl-structures"] +members = [ + "libconlang", + "cl-repl", + "cl-interpret", + "cl-structures", + "cl-token", +] resolver = "2" [workspace.package] diff --git a/cl-repl/Cargo.toml b/cl-repl/Cargo.toml index 940b145..ca36a13 100644 --- a/cl-repl/Cargo.toml +++ b/cl-repl/Cargo.toml @@ -12,6 +12,7 @@ publish.workspace = true [dependencies] conlang = { path = "../libconlang" } cl-interpret = { path = "../cl-interpret" } +cl-token = { path = "../cl-token" } crossterm = "0.27.0" [dev-dependencies] diff --git a/cl-repl/examples/identify_tokens.rs b/cl-repl/examples/identify_tokens.rs index cbd0b61..a7b255f 100644 --- a/cl-repl/examples/identify_tokens.rs +++ b/cl-repl/examples/identify_tokens.rs @@ -1,5 +1,6 @@ //! This example grabs input from stdin, lexes it, and prints which lexer rules matched #![allow(unused_imports)] +use cl_token::Token; use conlang::lexer::Lexer; use std::{ error::Error, @@ -57,7 +58,7 @@ fn lex_tokens(file: &str, path: Option<&Path>) -> Result<(), Box> { Ok(()) } -fn print_token(t: conlang::token::Token) { +fn print_token(t: Token) { println!( "{:02}:{:02}: {:#19} │{}│", t.line(), diff --git a/cl-repl/src/lib.rs b/cl-repl/src/lib.rs index 362ea3b..bb46eb7 100644 --- a/cl-repl/src/lib.rs +++ b/cl-repl/src/lib.rs @@ -194,7 +194,8 @@ pub mod cli { program::{Parsable, Parsed, Program}, }; use cl_interpret::env::Environment; - use conlang::{resolver::Resolver, token::Token}; + use cl_token::Token; + use conlang::resolver::Resolver; use std::{ convert::Infallible, error::Error, diff --git a/cl-token/Cargo.toml b/cl-token/Cargo.toml new file mode 100644 index 0000000..95193f1 --- /dev/null +++ b/cl-token/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "cl-token" +repository.workspace = true +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] diff --git a/cl-token/src/lib.rs b/cl-token/src/lib.rs new file mode 100644 index 0000000..fd5f812 --- /dev/null +++ b/cl-token/src/lib.rs @@ -0,0 +1,12 @@ +//! # Token +//! +//! Stores a component of a file as a [Type], some [Data], and a line and column number +#![feature(decl_macro)] + +pub mod token; +pub mod token_data; +pub mod token_type; + +pub use token::Token; +pub use token_data::Data; +pub use token_type::{Keyword, Type}; diff --git a/libconlang/src/token.rs b/cl-token/src/token.rs similarity index 74% rename from libconlang/src/token.rs rename to cl-token/src/token.rs index 6335e51..5675688 100644 --- a/libconlang/src/token.rs +++ b/cl-token/src/token.rs @@ -1,20 +1,5 @@ -//! # Token -//! -//! Stores a component of a file as a [Type], some [Data], and a line and column number - -pub mod token_data; -pub mod token_type; -pub mod preamble { - //! Common imports for working with [tokens](super) - pub use super::{ - token_data::Data, - token_type::{Keyword, Type}, - Token, - }; -} - -use token_data::Data; -use token_type::Type; +//! A [Token] contains a single unit of lexical information, and an optional bit of [Data] +use super::{Data, Type}; /// Contains a single unit of lexical information, /// and an optional bit of [Data] diff --git a/libconlang/src/token/token_data.rs b/cl-token/src/token_data.rs similarity index 100% rename from libconlang/src/token/token_data.rs rename to cl-token/src/token_data.rs diff --git a/libconlang/src/token/token_type.rs b/cl-token/src/token_type.rs similarity index 100% rename from libconlang/src/token/token_type.rs rename to cl-token/src/token_type.rs diff --git a/libconlang/Cargo.toml b/libconlang/Cargo.toml index ea0bf6b..de72233 100644 --- a/libconlang/Cargo.toml +++ b/libconlang/Cargo.toml @@ -14,3 +14,4 @@ repository.workspace = true [dependencies] unicode-xid = "0.2.4" cl-structures = { path = "../cl-structures" } +cl-token = { path = "../cl-token" } diff --git a/libconlang/src/lexer.rs b/libconlang/src/lexer.rs index a668ecf..5b947e0 100644 --- a/libconlang/src/lexer.rs +++ b/libconlang/src/lexer.rs @@ -1,5 +1,5 @@ //! Converts a text file into tokens -use crate::token::preamble::*; +use cl_token::*; use cl_structures::span::Loc; use std::{ iter::Peekable, diff --git a/libconlang/src/lib.rs b/libconlang/src/lib.rs index cea409a..5ce6951 100644 --- a/libconlang/src/lib.rs +++ b/libconlang/src/lib.rs @@ -2,8 +2,6 @@ #![warn(clippy::all)] #![feature(decl_macro)] -pub mod token; - pub mod ast; pub mod lexer; diff --git a/libconlang/src/parser.rs b/libconlang/src/parser.rs index 3f1925b..bb67afa 100644 --- a/libconlang/src/parser.rs +++ b/libconlang/src/parser.rs @@ -12,13 +12,9 @@ use self::error::{ use crate::{ ast::*, lexer::{error::Error as LexError, Lexer}, - token::{ - token_data::Data, - token_type::{Keyword, Type}, - Token, - }, }; use cl_structures::span::*; +use cl_token::*; pub mod error { use std::fmt::Display; diff --git a/libconlang/src/tests.rs b/libconlang/src/tests.rs index 855623c..e45f773 100644 --- a/libconlang/src/tests.rs +++ b/libconlang/src/tests.rs @@ -5,8 +5,8 @@ mod ast { // TODO } mod lexer { - #[allow(unused_imports)] - use crate::{lexer::Lexer, token::preamble::*}; + use crate::lexer::Lexer; + use cl_token::*; macro test_lexer_output_type ($($f:ident {$($test:expr => $expect:expr),*$(,)?})*) {$( #[test]