forked from katzen-cafe/iowo
140 lines
2.4 KiB
Rust
140 lines
2.4 KiB
Rust
use enumset::EnumSet;
|
|
use logos::Logos;
|
|
|
|
pub fn lex(src: &str) -> Vec<(SyntaxKind, &str)> {
|
|
let mut lex = SyntaxKind::lexer(src);
|
|
let mut r = Vec::new();
|
|
|
|
while let Some(tok_res) = lex.next() {
|
|
r.push((tok_res.unwrap_or(SyntaxKind::LEX_ERR), lex.slice()))
|
|
}
|
|
|
|
r
|
|
}
|
|
|
|
#[derive(enumset::EnumSetType, Logos, Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)]
|
|
#[repr(u16)]
|
|
#[enumset(no_super_impls)]
|
|
#[allow(non_camel_case_types)]
|
|
pub enum SyntaxKind {
|
|
#[token("def")]
|
|
DEF_KW = 0,
|
|
DEF,
|
|
DEF_NAME,
|
|
DEF_BODY,
|
|
#[token("let")]
|
|
LET_KW,
|
|
#[token("in")]
|
|
IN_KW,
|
|
LET_IN,
|
|
#[token("::")]
|
|
PATH_SEP,
|
|
#[token("mod")]
|
|
MOD_KW,
|
|
MODULE,
|
|
MODULE_NAME,
|
|
MODULE_BODY,
|
|
USE,
|
|
#[token("use")]
|
|
USE_KW,
|
|
USE_PAT,
|
|
PAT_ITEM,
|
|
PAT_GLOB,
|
|
PAT_GROUP,
|
|
#[regex("[\\d]+")]
|
|
INT_NUM,
|
|
#[regex("[+-]?([\\d]+\\.[\\d]*|[\\d]*\\.[\\d]+)")]
|
|
FLOAT_NUM,
|
|
#[regex(r#""([^"\\]|\\["\\bnfrt]|u[a-fA-F0-9]{4})*""#)]
|
|
STRING,
|
|
MATRIX,
|
|
MAT_ROW,
|
|
VEC,
|
|
LIST,
|
|
// either of a vec, a matrix or a list
|
|
COLLECTION_ITEM,
|
|
PARENTHESIZED_EXPR,
|
|
EXPR,
|
|
LITERAL,
|
|
#[token("(")]
|
|
L_PAREN,
|
|
#[token(")")]
|
|
R_PAREN,
|
|
#[token("{")]
|
|
L_BRACE,
|
|
#[token("}")]
|
|
R_BRACE,
|
|
#[token("[")]
|
|
L_BRACK,
|
|
#[token("]")]
|
|
R_BRACK,
|
|
#[token("<")]
|
|
L_ANGLE,
|
|
#[token(">")]
|
|
R_ANGLE,
|
|
#[token("+")]
|
|
PLUS,
|
|
#[token("-")]
|
|
MINUS,
|
|
#[token("*")]
|
|
STAR,
|
|
#[token("/")]
|
|
SLASH,
|
|
#[token("%")]
|
|
PERCENT,
|
|
#[token("^")]
|
|
CARET,
|
|
INSTR,
|
|
INSTR_NAME,
|
|
INSTR_PARAMS,
|
|
ATTR_SET,
|
|
ATTR,
|
|
ATTR_NAME,
|
|
ATTR_VALUE,
|
|
#[regex("[a-zA-Z_]+[a-zA-Z_\\-\\d]*")]
|
|
IDENT,
|
|
#[regex("\\$[a-zA-Z0-9_\\-]+")]
|
|
VAR,
|
|
#[regex("\\@[a-zA-Z0-9_\\-]+")]
|
|
INPUT_VAR,
|
|
#[token("$")]
|
|
DOLLAR,
|
|
#[token("@")]
|
|
AT,
|
|
#[token(",")]
|
|
COMMA,
|
|
#[token("|")]
|
|
PIPE,
|
|
#[token("@|")]
|
|
MAPPING_PIPE,
|
|
#[token("!|")]
|
|
NULL_PIPE,
|
|
PIPELINE,
|
|
#[token("=")]
|
|
EQ,
|
|
#[token(":")]
|
|
COLON,
|
|
#[token(";")]
|
|
SEMICOLON,
|
|
#[token(".")]
|
|
DOT,
|
|
#[token("!")]
|
|
BANG,
|
|
#[regex("[ \\t\\f]+")]
|
|
WHITESPACE,
|
|
#[token("\n")]
|
|
NEWLINE,
|
|
PARSE_ERR,
|
|
LEX_ERR,
|
|
ROOT,
|
|
EOF,
|
|
TOMBSTONE,
|
|
}
|
|
|
|
pub type TokenSet = EnumSet<SyntaxKind>;
|
|
|
|
impl From<SyntaxKind> for rowan::SyntaxKind {
|
|
fn from(kind: SyntaxKind) -> Self {
|
|
Self(kind as u16)
|
|
}
|
|
}
|