j/Conlang
1
1
Fork 0
Code Issues 5 Pull Requests Packages Projects 2 Releases Wiki Activity

lexer: Only allow lines up to 4GB in size. Saves 8 bytes per token

Browse Source
This commit is contained in:
John 2023-10-20 15:34:54 -05:00
parent a26a01fc17
commit 34525ec77a
2 changed files with 12 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libconlang/src/lexer.rs
View File

@ -23,8 +23,8 @@ impl<'t> IntoIterator for Lexer<'t> {
pub struct Lexer<'t> { pub struct Lexer<'t> {
text: &'t str, text: &'t str,
cursor: usize, cursor: usize,
line: usize, line: u32,
col: usize, col: u32,
} }
/// Implements the non-terminals of a language /// Implements the non-terminals of a language
impl<'t> Lexer<'t> { impl<'t> Lexer<'t> {
@ -41,12 +41,12 @@ impl<'t> Lexer<'t> {
#[inline] #[inline]
fn count_len(&mut self, len: usize) -> &mut Self { fn count_len(&mut self, len: usize) -> &mut Self {
self.cursor += len; self.cursor += len;
self.col += len; self.col += len as u32;
self self
} }
/// Counts a line /// Counts a line
#[inline] #[inline]
fn count_line(&mut self, lines: usize) -> &mut Self { fn count_line(&mut self, lines: u32) -> &mut Self {
self.line += lines; self.line += lines;
self.col = 1; self.col = 1;
self self

16
libconlang/src/token.rs
View File

@ -2,7 +2,7 @@
use std::ops::Range; use std::ops::Range;
mod token_type; mod token_type;
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Type { pub enum Type {
// Invalid syntax // Invalid syntax
Invalid, Invalid,
@ -72,7 +72,7 @@ pub enum Type {
} }
/// Represents a reserved word. /// Represents a reserved word.
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Keyword { pub enum Keyword {
Break, Break,
Continue, Continue,
@ -88,16 +88,16 @@ pub enum Keyword {
While, While,
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Token { pub struct Token {
ty: Type, ty: Type,
pub head: usize, pub head: usize,
pub tail: usize, pub tail: usize,
line: usize, line: u32,
col: usize, col: u32,
} }
impl Token { impl Token {
pub fn new(ty: Type, head: usize, tail: usize, line: usize, col: usize) -> Self { pub fn new(ty: Type, head: usize, tail: usize, line: u32, col: u32) -> Self {
Self { ty, head, tail, line, col } Self { ty, head, tail, line, col }
} }
/// Cast this [Token] to a new [Type] /// Cast this [Token] to a new [Type]
@ -109,11 +109,11 @@ impl Token {
Self { head, tail, ..self } Self { head, tail, ..self }
} }
/// Gets the line from this token /// Gets the line from this token
pub fn line(&self) -> usize { pub fn line(&self) -> u32 {
self.line self.line
} }
/// Gets the column from this token /// Gets the column from this token
pub fn col(&self) -> usize { pub fn col(&self) -> u32 {
self.col self.col
} }
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {