use enumset::EnumSet; use logos::Logos; pub fn lex(src: &str) -> Vec<(SyntaxKind, &str)> { let mut lex = SyntaxKind::lexer(src); let mut r = Vec::new(); while let Some(tok_res) = lex.next() { r.push((tok_res.unwrap_or(SyntaxKind::LEX_ERR), lex.slice())) } r } #[derive(enumset::EnumSetType, Logos, Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)] #[repr(u16)] #[enumset(no_super_impls)] #[allow(non_camel_case_types)] pub enum SyntaxKind { #[token("def")] DEF_KW = 0, #[token("let")] LET_KW, #[token("in")] IN_KW, #[regex("[\\d]+")] INT_NUM, #[regex("[+-]?([\\d]+\\.[\\d]*|[\\d]*\\.[\\d]+)")] FLOAT_NUM, #[regex(r#""([^"\\]|\\["\\bnfrt]|u[a-fA-F0-9]{4})*""#)] STRING, MATRIX, MAT_ROW, VEC, LIST, // either of a vec, a matrix or a list COLLECTION_ITEM, DECL, PARENTHESIZED_EXPR, EXPR, LITERAL, #[token("(")] L_PAREN, #[token(")")] R_PAREN, #[token("{")] L_BRACE, #[token("}")] R_BRACE, #[token("[")] L_BRACK, #[token("]")] R_BRACK, #[token("<")] L_ANGLE, #[token(">")] R_ANGLE, #[token("+")] PLUS, #[token("-")] MINUS, #[token("*")] STAR, #[token("/")] SLASH, #[token("%")] PERCENT, #[token("^")] CARET, INSTR, INSTR_NAME, INSTR_PARAMS, ATTR_SET, ATTR, ATTR_NAME, ATTR_VALUE, #[regex("[a-zA-Z_]+[a-zA-Z_\\-\\d]*")] IDENT, #[regex("\\$[a-zA-Z0-9_\\-]+")] VAR, #[regex("\\@[a-zA-Z0-9_\\-]+")] INPUT_VAR, #[token("$")] DOLLAR, #[token("@")] AT, #[token(",")] COMMA, #[token("|")] PIPE, #[token("@|")] MAPPING_PIPE, #[token("!|")] NULL_PIPE, PIPELINE, #[token("=")] EQ, #[token(":")] COLON, #[token(";")] SEMICOLON, #[token(".")] DOT, #[token("!")] BANG, #[regex("[ \\t\\f]+")] WHITESPACE, #[token("\n")] NEWLINE, PARSE_ERR, LEX_ERR, ROOT, EOF, TOMBSTONE, ERROR, } pub type TokenSet = EnumSet; impl From for rowan::SyntaxKind { fn from(kind: SyntaxKind) -> Self { Self(kind as u16) } } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Lang {} impl rowan::Language for Lang { type Kind = SyntaxKind; #[allow(unsafe_code)] fn kind_from_raw(raw: rowan::SyntaxKind) -> Self::Kind { assert!(raw.0 <= SyntaxKind::ROOT as u16); unsafe { std::mem::transmute::(raw.0) } } fn kind_to_raw(kind: Self::Kind) -> rowan::SyntaxKind { kind.into() } }