use enumset::EnumSet; use logos::Logos; pub fn lex(src: &str) -> Vec<(SyntaxKind, &str)> { let mut lex = SyntaxKind::lexer(src); let mut r = Vec::new(); while let Some(tok_res) = lex.next() { r.push((tok_res.unwrap_or(SyntaxKind::LEX_ERR), lex.slice())) } r } #[derive(enumset::EnumSetType, Logos, Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)] #[repr(u16)] #[enumset(no_super_impls)] #[allow(non_camel_case_types)] pub enum SyntaxKind { #[token("def")] DEF_KW = 0, DEF, DEF_NAME, DEF_BODY, #[token("let")] LET_KW, #[token("in")] IN_KW, LET_IN, #[token("::")] PATH_SEP, #[token("mod")] MOD_KW, MODULE, MODULE_NAME, MODULE_BODY, USE, #[token("use")] USE_KW, USE_PAT, PAT_ITEM, PAT_GLOB, PAT_GROUP, #[regex("[\\d]+")] INT_NUM, #[regex("[+-]?([\\d]+\\.[\\d]*|[\\d]*\\.[\\d]+)")] FLOAT_NUM, #[regex(r#""([^"\\]|\\["\\bnfrt]|u[a-fA-F0-9]{4})*""#)] STRING, MATRIX, MAT_ROW, VEC, LIST, // either of a vec, a matrix or a list COLLECTION_ITEM, PARENTHESIZED_EXPR, EXPR, LITERAL, #[token("(")] L_PAREN, #[token(")")] R_PAREN, #[token("{")] L_BRACE, #[token("}")] R_BRACE, #[token("[")] L_BRACK, #[token("]")] R_BRACK, #[token("<")] L_ANGLE, #[token(">")] R_ANGLE, #[token("+")] PLUS, #[token("-")] MINUS, #[token("*")] STAR, #[token("/")] SLASH, #[token("%")] PERCENT, #[token("^")] CARET, INSTR, INSTR_NAME, INSTR_PARAMS, ATTR_SET, ATTR, ATTR_NAME, ATTR_VALUE, #[regex("[a-zA-Z_]+[a-zA-Z_\\-\\d]*")] IDENT, #[regex("\\$[a-zA-Z0-9_\\-]+")] VAR, #[regex("\\@[a-zA-Z0-9_\\-]+")] INPUT_VAR, #[token("$")] DOLLAR, #[token("@")] AT, #[token(",")] COMMA, #[token("|")] PIPE, #[token("@|")] MAPPING_PIPE, #[token("!|")] NULL_PIPE, PIPELINE, #[token("=")] EQ, #[token(":")] COLON, #[token(";")] SEMICOLON, #[token(".")] DOT, #[token("!")] BANG, #[regex("[ \\t\\f]+")] WHITESPACE, #[token("\n")] NEWLINE, PARSE_ERR, LEX_ERR, ROOT, EOF, TOMBSTONE, } pub type TokenSet = EnumSet; impl From for rowan::SyntaxKind { fn from(kind: SyntaxKind) -> Self { Self(kind as u16) } }