diff --git a/libconlang/src/lib.rs b/libconlang/src/lib.rs index 5f528b8..a6e8d40 100644 --- a/libconlang/src/lib.rs +++ b/libconlang/src/lib.rs @@ -991,7 +991,24 @@ pub mod lexer { use crate::token::{Token, Type}; use lerox::Combinator; - #[allow(dead_code)] + pub struct IntoIter<'t> { + lexer: Lexer<'t>, + } + impl<'t> Iterator for IntoIter<'t> { + type Item = Token; + fn next(&mut self) -> Option { + self.lexer.any() + } + } + impl<'t> IntoIterator for Lexer<'t> { + type Item = Token; + type IntoIter = IntoIter<'t>; + fn into_iter(self) -> Self::IntoIter { + IntoIter { lexer: self } + } + } + + #[derive(Clone, Debug)] pub struct Lexer<'t> { text: &'t str, cursor: usize, @@ -1003,6 +1020,12 @@ pub mod lexer { pub fn new(text: &'t str) -> Self { Self { text, cursor: 0, line: 1, col: 1 } } + /// Consumes the entire [`Lexer`], producing a [`Vec`] + /// and returning the original string + pub fn consume(self) -> (Vec, &'t str) { + let text = self.text; + (self.into_iter().collect(), text) + } /// Counts some length #[inline] fn count_len(&mut self, len: usize) -> &mut Self {