John
b5abd2bff1
- Scan the input string *linearly*, without backtracking - Peek at most one character (unicode code-point) ahead - Store data (unescaped string literals and chars, identifiers, integers, floats) inside Token - This unfortunately makes tokens non-Copy - Refactor Parser to accommodate these changes - On the bright side, Parser no longer needs a reference to the text! - Write a new set of lexer tests - TODO: write a new set of token tests using tokendata Every day, we get closer to parsing `dummy.cl`!
69 lines
1.5 KiB
Rust
69 lines
1.5 KiB
Rust
//! This example grabs input from stdin, lexes it, and prints which lexer rules matched
|
|
#![allow(unused_imports)]
|
|
use conlang::lexer::Lexer;
|
|
use std::{
|
|
error::Error,
|
|
io::{stdin, IsTerminal, Read},
|
|
path::{Path, PathBuf},
|
|
};
|
|
|
|
fn main() -> Result<(), Box<dyn Error>> {
|
|
let conf = Config::new();
|
|
if conf.paths.is_empty() {
|
|
take_stdin()?;
|
|
} else {
|
|
for path in conf.paths.iter().map(PathBuf::as_path) {
|
|
lex_tokens(&std::fs::read_to_string(path)?, Some(path))?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
struct Config {
|
|
paths: Vec<PathBuf>,
|
|
}
|
|
|
|
impl Config {
|
|
fn new() -> Self {
|
|
Config { paths: std::env::args().skip(1).map(PathBuf::from).collect() }
|
|
}
|
|
}
|
|
|
|
fn take_stdin() -> Result<(), Box<dyn Error>> {
|
|
if stdin().is_terminal() {
|
|
for line in stdin().lines() {
|
|
lex_tokens(&line?, None)?
|
|
}
|
|
} else {
|
|
lex_tokens(&std::io::read_to_string(stdin())?, None)?
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn lex_tokens(file: &str, path: Option<&Path>) -> Result<(), Box<dyn Error>> {
|
|
for token in Lexer::new(file) {
|
|
let token = match token {
|
|
Ok(t) => t,
|
|
Err(e) => {
|
|
println!("{e:?}");
|
|
break;
|
|
},
|
|
};
|
|
if let Some(path) = path {
|
|
print!("{path:?}:")
|
|
}
|
|
print_token(token);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn print_token(t: conlang::token::Token) {
|
|
println!(
|
|
"{:02}:{:02}: {:#19} │{:?}│",
|
|
t.line(),
|
|
t.col(),
|
|
t.ty(),
|
|
t.data(),
|
|
)
|
|
}
|