j/msp430-repl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

token.rs: Lex numbers of arbitrary length, for better errors

Browse Source
This commit is contained in:
John
2023-08-23 00:00:28 -05:00
parent b3af047d3a
commit 8230d73495
1 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/tokenizer/token.rs
View File

@@ -4,6 +4,7 @@
//! A [Token] represents all valid sequences of characters, //! A [Token] represents all valid sequences of characters,
//! sorted by meaning //! sorted by meaning
use crate::Error;
use regex::Regex; use regex::Regex;
use std::{ use std::{
fmt::{Debug, Display}, fmt::{Debug, Display},
@@ -56,8 +57,6 @@ impl<$t> From<&$t str> for $type {
}; };
} }
use crate::Error;
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Token<'text> { pub struct Token<'text> {
/// The type of this token /// The type of this token
@@ -194,7 +193,7 @@ regex_impl! {<'text> Token<'text> {
regex!(Type::RadixMarkerBin = r"(?i)^0b") regex!(Type::RadixMarkerBin = r"(?i)^0b")
} }
pub fn expect_number(text: &str) -> Option<Self> { pub fn expect_number(text: &str) -> Option<Self> {
regex!(Type::Number = r"^+?[[:xdigit:]]{1,5}") regex!(Type::Number = r"^+?[[:xdigit:]]+")
} }
pub fn expect_minus(text: &str) -> Option<Self> { pub fn expect_minus(text: &str) -> Option<Self> {
regex!(Type::Minus = r"^-") regex!(Type::Minus = r"^-")