//! Tokenization and parsing of source code. use std::fmt::{self, Display, Formatter}; use crate::func::Function; use crate::size::Size; mod tokens; mod parsing; pub use tokens::{tokenize, Tokens}; pub use parsing::{parse, ParseContext, ParseError, ParseResult}; /// A logical unit of the incoming text stream. #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum Token<'s> { /// One or more whitespace (non-newline) codepoints. Space, /// A line feed (`\n`, `\r\n` and some more as defined by the Unicode standard). Newline, /// A left bracket: `[`. LeftBracket, /// A right bracket: `]`. RightBracket, /// A colon (`:`) indicating the beginning of function arguments (Function /// header only). /// /// If a colon occurs outside of a function header, it will be tokenized as /// [Text](Token::Text), just like the other tokens annotated with /// _Header only_. Colon, /// An equals (`=`) sign assigning a function argument a value (Header only). Equals, /// A comma (`,`) separating two function arguments (Header only). Comma, /// Quoted text as a string value (Header only). Quoted(&'s str), /// An underscore, indicating text in italics (Body only). Underscore, /// A star, indicating bold text (Body only). Star, /// A backtick, indicating monospace text (Body only). Backtick, /// A line comment. LineComment(&'s str), /// A block comment. BlockComment(&'s str), /// A star followed by a slash unexpectedly ending a block comment /// (the comment was not started before, otherwise a /// [BlockComment](Token::BlockComment) would be returned). StarSlash, /// Any consecutive string which does not contain markup. Text(&'s str), } /// A tree representation of source code. #[derive(Debug, PartialEq)] pub struct SyntaxTree { pub nodes: Vec>, } impl SyntaxTree { /// Create an empty syntax tree. pub fn new() -> SyntaxTree { SyntaxTree { nodes: vec![] } } } /// A node in the syntax tree. #[derive(Debug, PartialEq)] pub enum Node { /// Whitespace. Space, /// A line feed. Newline, /// Indicates that italics were enabled / disabled. ToggleItalics, /// Indicates that boldface was enabled / disabled. ToggleBold, /// Indicates that monospace was enabled / disabled. ToggleMonospace, /// Literal text. Text(String), /// A function invocation. Func(FuncCall), } /// A function invocation, consisting of header and a dynamically parsed body. #[derive(Debug)] pub struct FuncCall { pub header: Spanned, pub body: Spanned>, } /// Contains header information of a function invocation. #[derive(Debug, Clone, PartialEq)] pub struct FuncHeader { pub name: Spanned, pub args: FuncArgs, } /// The arguments passed to a function. #[derive(Debug, Clone, PartialEq)] pub struct FuncArgs { pub positional: Vec>, pub keyword: Vec, Spanned)>> } impl FuncArgs { /// Create an empty collection of arguments. fn new() -> FuncArgs { FuncArgs { positional: vec![], keyword: vec![], } } } /// An argument or return value. #[derive(Debug, Clone, PartialEq)] pub enum Expression { Ident(String), Str(String), Number(f64), Size(Size), Bool(bool), } impl PartialEq for FuncCall { fn eq(&self, other: &FuncCall) -> bool { (self.header == other.header) && (&self.body == &other.body) } } impl Display for Expression { fn fmt(&self, f: &mut Formatter) -> fmt::Result { use Expression::*; match self { Ident(s) => write!(f, "{}", s), Str(s) => write!(f, "{:?}", s), Number(n) => write!(f, "{}", n), Size(s) => write!(f, "{}", s), Bool(b) => write!(f, "{}", b), } } } /// Annotates a value with the part of the source code it corresponds to. #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub struct Spanned { pub val: T, pub span: Span, } impl Spanned { pub fn new(val: T, span: Span) -> Spanned { Spanned { val, span } } pub fn value(&self) -> &T { &self.val } pub fn span_map(self, f: F) -> Spanned where F: FnOnce(T) -> U { Spanned::new(f(self.val), self.span) } } /// Describes a slice of source code. #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub struct Span { pub start: usize, pub end: usize, } impl Span { pub fn new(start: usize, end: usize) -> Span { Span { start, end } } pub fn at(index: usize) -> Span { Span { start: index, end: index + 1 } } pub fn pair(&self) -> (usize, usize) { (self.start, self.end) } pub fn expand(&mut self, other: Span) { self.start = self.start.min(other.start); self.end = self.end.max(other.end); } }