use crate::geom::Unit; /// A minimal semantic entity of source code. #[derive(Debug, Copy, Clone, PartialEq)] pub enum Token<'s> { /// One or more whitespace characters. /// /// The contained `usize` denotes the number of newlines that were contained /// in the whitespace. Space(usize), /// A consecutive non-markup string. Text(&'s str), /// A line comment with inner string contents `//\n`. LineComment(&'s str), /// A block comment with inner string contents `/**/`. /// /// The comment can contain nested block comments. BlockComment(&'s str), /// An end of a block comment that was not started. StarSlash, /// A star: `*`. Star, /// An underscore: `_`. Underscore, /// A tilde: `~`. Tilde, /// A backslash followed by whitespace: `\`. Backslash, /// A hashtag indicating a section heading: `#`. Hashtag, /// A raw block: `` `...` ``. Raw(TokenRaw<'s>), /// A unicode escape sequence: `\u{1F5FA}`. UnicodeEscape(TokenUnicodeEscape<'s>), /// A left bracket: `[`. LeftBracket, /// A right bracket: `]`. RightBracket, /// A left brace: `{`. LeftBrace, /// A right brace: `}`. RightBrace, /// A left parenthesis: `(`. LeftParen, /// A right parenthesis: `)`. RightParen, /// A colon: `:`. Colon, /// A comma: `,`. Comma, /// A pipe: `|`. Pipe, /// A plus: `+`. Plus, /// A hyphen: `-`. Hyphen, /// A slash: `/`. Slash, /// An identifier: `center`. Ident(&'s str), /// A boolean: `true`, `false`. Bool(bool), /// An integer: `120`. Int(i64), /// A floating-point number: `1.2`, `10e-4`. Float(f64), /// A length: `12pt`, `3cm`. Length(f64, Unit), /// A percentage: `50%`. /// /// _Note_: `50%` is stored as `50.0` here, as in the corresponding /// [literal](super::Lit::Percent). Percent(f64), /// A hex value: `#20d82a`. Hex(&'s str), /// A quoted string: `"..."`. Str(TokenStr<'s>), /// Things that are not valid tokens. Invalid(&'s str), } /// A quoted string: `"..."`. #[derive(Debug, Copy, Clone, PartialEq)] pub struct TokenStr<'s> { /// The string inside the quotes. /// /// _Note_: If the string contains escape sequences these are not yet /// applied to be able to just store a string slice here instead of /// a `String`. The resolving is done later in the parser. pub string: &'s str, /// Whether the closing quote was present. pub terminated: bool, } /// A unicode escape sequence: `\u{1F5FA}`. #[derive(Debug, Copy, Clone, PartialEq)] pub struct TokenUnicodeEscape<'s> { /// The escape sequence between two braces. pub sequence: &'s str, /// Whether the closing brace was present. pub terminated: bool, } /// A raw block: `` `...` ``. #[derive(Debug, Copy, Clone, PartialEq)] pub struct TokenRaw<'s> { /// The raw text between the backticks. pub text: &'s str, /// The number of opening backticks. pub backticks: usize, /// Whether all closing backticks were present. pub terminated: bool, } impl<'s> Token<'s> { /// The natural-language name of this token for use in error messages. pub fn name(self) -> &'static str { match self { Self::Space(_) => "space", Self::Text(_) => "text", Self::LineComment(_) => "line comment", Self::BlockComment(_) => "block comment", Self::StarSlash => "end of block comment", Self::Star => "star", Self::Underscore => "underscore", Self::Backslash => "backslash", Self::Hashtag => "hashtag", Self::Tilde => "tidle", Self::Raw { .. } => "raw block", Self::UnicodeEscape { .. } => "unicode escape sequence", Self::LeftBracket => "opening bracket", Self::RightBracket => "closing bracket", Self::LeftBrace => "opening brace", Self::RightBrace => "closing brace", Self::LeftParen => "opening paren", Self::RightParen => "closing paren", Self::Colon => "colon", Self::Comma => "comma", Self::Pipe => "pipe", Self::Plus => "plus sign", Self::Hyphen => "minus sign", Self::Slash => "slash", Self::Ident(_) => "identifier", Self::Bool(_) => "bool", Self::Int(_) => "integer", Self::Float(_) => "float", Self::Length(..) => "length", Self::Percent(_) => "percentage", Self::Hex(_) => "hex value", Self::Str { .. } => "string", Self::Invalid(_) => "invalid token", } } }