83 lines
2.6 KiB
Rust
83 lines
2.6 KiB
Rust
// © 2023 John Breaux
|
|
//! Iterates over &[str], producing [Token]s
|
|
|
|
// Things we need:
|
|
// ✔ 1. Lexer/Tokenizer
|
|
// ✔ 1. Instructions
|
|
// ✔ 1. Instruction mnemonics /ad.../
|
|
// ✔ 2. Byte/Word Mode Marker /(.\[bw\])?/
|
|
// ✔ 2. Operands
|
|
// ✔ 1. Registers /(r1[0-5]|r[0-9])/
|
|
// ✔ 2. Immediate Values /#/
|
|
// ✔ 3. Absolute addresses /&/
|
|
// ✔ 4. Numbers /[0-9A-Fa-f]+
|
|
// ✔ 5. Jump Offsets: basically numbers /$?([+-]?[0-9A-Fa-f]{1,4})/
|
|
// ✔ 3. Label definitions /(^.*):/
|
|
// ✔ 4. Comments (may be useful for debugging)
|
|
|
|
pub mod context;
|
|
pub mod ignore;
|
|
pub mod preprocessed;
|
|
pub mod token;
|
|
pub mod token_stream;
|
|
|
|
use crate::Error;
|
|
use context::Context;
|
|
use token::{Token, Type};
|
|
use token_stream::TokenStream;
|
|
|
|
/// Iterates over &[str], producing [Token]s
|
|
#[must_use = "iterators are lazy and do nothing unless consumed"]
|
|
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
|
pub struct Tokenizer<'t> {
|
|
text: &'t str,
|
|
idx: usize,
|
|
context: Context,
|
|
}
|
|
|
|
impl<'t> Tokenizer<'t> {
|
|
/// Produces a new [Tokenizer] from a [str]ing slice
|
|
pub fn new<T>(text: &'t T) -> Self
|
|
where T: AsRef<str> + ?Sized {
|
|
Tokenizer { text: text.as_ref(), idx: 0, context: Default::default() }
|
|
}
|
|
|
|
fn count(&mut self, token: &Token) {
|
|
// update the context
|
|
self.context.count(token);
|
|
// advance the index
|
|
self.idx += token.len();
|
|
}
|
|
}
|
|
|
|
impl<'text> Iterator for Tokenizer<'text> {
|
|
type Item = Token<'text>;
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
if self.idx >= self.text.len() {
|
|
return None;
|
|
}
|
|
let token = Token::from(&self.text[self.idx..]);
|
|
// Process [Type::Directive]s
|
|
// Count the token
|
|
self.count(&token);
|
|
Some(token)
|
|
}
|
|
}
|
|
|
|
impl<'text> TokenStream<'text> for Tokenizer<'text> {
|
|
fn context(&self) -> Context { self.context }
|
|
// Tokenizer has access to the source buffer, and can implement expect and peek without cloning
|
|
// itself. This can go wrong, of course, if an [Identifier] is expected, since all instructions and
|
|
// registers are valid identifiers.
|
|
fn expect(&mut self, expected: Type) -> Result<Self::Item, Error> {
|
|
let token = Token::expect(&self.text[self.idx..], expected).map_err(|e| e.context(self.context()))?;
|
|
self.count(&token);
|
|
Ok(token)
|
|
}
|
|
fn peek(&mut self) -> Self::Item { Token::from(&self.text[self.idx..]) }
|
|
fn peek_expect(&mut self, expected: Type) -> Result<Self::Item, Error> {
|
|
Token::expect(&self.text[self.idx..], expected)
|
|
}
|
|
}
|