From 46e72e4889842b940ae8cb7503e4060053474834 Mon Sep 17 00:00:00 2001 From: John Date: Mon, 25 Sep 2023 18:06:03 -0500 Subject: [PATCH] conlang: add toy program to interact with the tokenizer --- libconlang/examples/identify_tokens.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 libconlang/examples/identify_tokens.rs diff --git a/libconlang/examples/identify_tokens.rs b/libconlang/examples/identify_tokens.rs new file mode 100644 index 0000000..c2ac4a8 --- /dev/null +++ b/libconlang/examples/identify_tokens.rs @@ -0,0 +1,18 @@ +//! This example grabs input from stdin, lexes it, and prints which lexer rules matched +#![allow(unused_imports)] +use conlang::lexer::Lexer; +use std::{io::stdin, error::Error}; + +fn main() -> Result<(), Box>{ + // get input from stdin + for line in stdin().lines() { + let line = line?; + // lex the line + for func in [Lexer::line_comment, Lexer::block_comment, Lexer::shebang_comment, Lexer::identifier, Lexer::integer, Lexer::float, Lexer::string] { + if let Some(token) = func(&mut Lexer::new(&line)) { + println!("{:?}: {}", token, &line[token.range()]) + } + } + } + Ok(()) +} \ No newline at end of file