diff --git a/libconlang/src/ast.rs b/libconlang/src/ast.rs index 0b572bc..0921c25 100644 --- a/libconlang/src/ast.rs +++ b/libconlang/src/ast.rs @@ -285,7 +285,7 @@ pub struct Start(pub expression::Expr); /// An Identifier stores the name of an item /// # Syntax -/// [`Identifier`] := [`IDENTIFIER`](crate::token::Type::Identifier) +/// [`Identifier`] := [`IDENTIFIER`](crate::token::token_type::Type::Identifier) #[derive(Clone, Debug, Hash)] pub struct Identifier(pub String); @@ -354,31 +354,31 @@ pub mod literal { pub enum Literal { /// Represents a literal string value /// # Syntax - /// [`Literal::String`] := [`STRING`](crate::token::Type::String) + /// [`Literal::String`] := [`STRING`](crate::token::token_type::Type::String) String(String), /// Represents a literal [char] value /// # Syntax - /// [`Literal::Char`] := [`CHARACTER`](crate::token::Type::Character) + /// [`Literal::Char`] := [`CHARACTER`](crate::token::token_type::Type::Character) Char(char), /// Represents a literal [bool] value /// # Syntax /// [`Literal::Bool`] := - /// [`TRUE`](crate::token::Keyword::True) - /// | [`FALSE`](crate::token::Keyword::False) + /// [`TRUE`](crate::token::token_type::Keyword::True) + /// | [`FALSE`](crate::token::token_type::Keyword::False) Bool(bool), /// Represents a literal float value /// # Syntax - /// [`Float`] := [`FLOAT`](crate::token::Type::Float) + /// [`Float`] := [`FLOAT`](crate::token::token_type::Type::Float) Float(Float), /// Represents a literal integer value /// # Syntax - /// [`u128`] := [`INTEGER`](crate::token::Type::Integer) + /// [`u128`] := [`INTEGER`](crate::token::token_type::Type::Integer) Int(u128), } /// Represents a literal float value /// # Syntax - /// [`Float`] := [`FLOAT`](crate::token::Type::Float) + /// [`Float`] := [`FLOAT`](crate::token::token_type::Type::Float) #[derive(Clone, Debug)] pub struct Float { pub sign: bool, diff --git a/libconlang/src/lexer.rs b/libconlang/src/lexer.rs index f3059e3..477c14c 100644 --- a/libconlang/src/lexer.rs +++ b/libconlang/src/lexer.rs @@ -1,5 +1,5 @@ //! Converts a text file into tokens -use crate::token::{Keyword, Token, TokenData, Type}; +use crate::token::preamble::*; use std::{ iter::Peekable, str::{Chars, FromStr}, @@ -7,12 +7,13 @@ use std::{ use unicode_xid::UnicodeXID; pub mod lexer_iter { + //! Iterator over a [`Lexer`], returning [`LResult`]s use super::{ error::{LResult, Reason}, Lexer, Token, }; - /// Fallible iterator over a [Lexer], returning optional [LResult]s + /// Iterator over a [`Lexer`], returning [`LResult`]s pub struct LexerIter<'t> { lexer: Lexer<'t>, } @@ -40,6 +41,31 @@ pub mod lexer_iter { } } +/// The Lexer iterates over the characters in a body of text, searching for [Tokens](Token). +/// +/// # Examples +/// ```rust +///# use conlang::lexer::Lexer; +/// // Read in your code from somewhere +/// let some_code = " +/// fn main () { +/// // TODO: code goes here! +/// } +/// "; +/// // Create a lexer over your code +/// let mut lexer = Lexer::new(some_code); +/// // Scan for a single token +/// let first_token = lexer.scan().unwrap(); +/// println!("{first_token:?}"); +/// // Loop over all the rest of the tokens +/// for token in lexer { +///# let token: Result<_,()> = Ok(token.unwrap()); +/// match token { +/// Ok(token) => println!("{token:?}"), +/// Err(e) => eprintln!("{e:?}"), +/// } +/// } +/// ``` #[derive(Clone, Debug)] pub struct Lexer<'t> { iter: Peekable>, @@ -50,6 +76,7 @@ pub struct Lexer<'t> { } impl<'t> Lexer<'t> { + /// Creates a new [Lexer] over a [str] pub fn new(text: &'t str) -> Self { Self { iter: text.chars().peekable(), @@ -59,6 +86,7 @@ impl<'t> Lexer<'t> { current_loc: (1, 1), } } + /// Scans through the text, searching for the next [Token] pub fn scan(&mut self) -> LResult { match self.skip_whitespace().peek()? { '{' => self.consume()?.produce(Type::LCurly, ()), @@ -98,11 +126,11 @@ impl<'t> Lexer<'t> { e => Err(Error::unexpected_char(e, self.line(), self.col())), } } - /// Gets the line of the next token + /// Returns the current line pub fn line(&self) -> u32 { self.start_loc.0 } - /// Gets the column of the next token + /// Returns the current column pub fn col(&self) -> u32 { self.start_loc.1 } @@ -403,8 +431,10 @@ impl<'t> Lexer<'t> { use error::{Error, LResult, Reason}; pub mod error { + //! [Error] type for the [Lexer](super::Lexer) use std::fmt::Display; + /// Result type with [Err] = [Error] pub type LResult = Result; #[derive(Clone, Debug, PartialEq, Eq)] pub struct Error { @@ -415,14 +445,25 @@ pub mod error { /// The reason for the [Error] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Reason { + /// Found an opening delimiter of type [char], but not the expected closing delimiter UnmatchedDelimiters(char), + /// Found a character that doesn't belong to any [Type](crate::token::token_type::Type) UnexpectedChar(char), + /// Found a character that's not valid in identifiers while looking for an identifier NotIdentifier(char), + /// Found a character that's not valid in an escape sequence while looking for an escape + /// sequence UnknownEscape(char), + /// Escape sequence contains invalid hexadecimal digit or unmatched braces InvalidEscape(char), + /// Character is not a valid digit in the requested base InvalidDigit(char), + /// Base conversion requested, but the base character was not in the set of known + /// characters UnknownBase(char), + /// Unicode escape does not map to a valid unicode code-point BadUnicode(u32), + /// Reached end of input EndOfFile, } error_impl! { @@ -441,7 +482,11 @@ pub mod error { pub(super) fn mask_reason(self, reason: Reason) -> Self { Self { reason, ..self } } - /// Gets the (line, col) where the error happened + /// Returns the [Reason] for this error + pub fn reason(&self) -> &Reason { + &self.reason + } + /// Returns the (line, col) where the error happened pub fn location(&self) -> (u32, u32) { (self.line, self.col) } diff --git a/libconlang/src/pretty_printer.rs b/libconlang/src/pretty_printer.rs index 4e99198..84baaf5 100644 --- a/libconlang/src/pretty_printer.rs +++ b/libconlang/src/pretty_printer.rs @@ -1,10 +1,14 @@ +//! A [Printer] pretty-prints a Conlang [syntax tree](crate::ast) use super::ast::preamble::*; use std::{ fmt::Display, io::{stdout, Result as IOResult, StdoutLock, Write}, }; +/// Prettily prints this node pub trait PrettyPrintable { + /// Prettily prints this node fn print(&self); + /// Prettily writes this node into the given [Writer](Write) fn write(&self, into: impl Write) -> IOResult<()>; } impl PrettyPrintable for Start { @@ -16,6 +20,7 @@ impl PrettyPrintable for Start { } } +/// Prints a Conlang [syntax tree](crate::ast) into a [Writer](Write) #[derive(Debug)] pub struct Printer { level: u32, diff --git a/libconlang/src/token.rs b/libconlang/src/token.rs index 79e22b9..9389962 100644 --- a/libconlang/src/token.rs +++ b/libconlang/src/token.rs @@ -17,7 +17,7 @@ use token_data::Data; use token_type::Type; /// Contains a single unit of lexical information, -/// and an optional bit of [data](TokenData) +/// and an optional bit of [Data] #[derive(Clone, Debug, PartialEq)] pub struct Token { ty: Type,