lang: lexer

crates/lang/src/tokens/tests.rs

@@ -0,0 +1,107 @@
+use logos::Logos;
+
+use super::Token;
+
+/// generates tests for the lexer to avoid writing boilerplate
+macro_rules! lexer_test {
+    ($name:ident, $input:literal, $out:expr) => {
+        #[test]
+        fn $name() {
+            let lex = Token::lexer($input);
+            let toks = lex.map(|tok| tok.unwrap()).collect::<Vec<_>>();
+            assert_eq!(toks, $out);
+        }
+    };
+}
+
+lexer_test! {
+    test_lex_simple_pipeline,
+    "streamer | processor | sink",
+    [
+        Token::Word("streamer"),
+        Token::Pipe,
+        Token::Word("processor"),
+        Token::Pipe,
+        Token::Word("sink")
+    ]
+}
+
+lexer_test! {
+    test_lex_var_ident,
+    "$identifier",
+    [ Token::VarIdent("identifier") ]
+}
+
+lexer_test! {
+    test_lex_subgroup,
+    "subgroup(first, second) = a | b { 1: $first } | c { 1: $second }",
+    [
+        Token::Word("subgroup"),
+        Token::ParenOpen,
+        Token::Word("first"),
+        Token::Comma,
+        Token::Word("second"),
+        Token::ParenClose,
+        Token::Equals,
+        Token::Word("a"),
+        Token::Pipe,
+        Token::Word("b"),
+        Token::BraceOpen,
+        Token::Word("1"),
+        Token::Colon,
+        Token::VarIdent("first"),
+        Token::BraceClose,
+        Token::Pipe,
+        Token::Word("c"),
+        Token::BraceOpen,
+        Token::Word("1"),
+        Token::Colon,
+        Token::VarIdent("second"),
+        Token::BraceClose
+    ]
+}
+
+lexer_test! {
+    text_lex_crossing_pipeline_reordering,
+    "a >first, second|second, first> c",
+    [
+        Token::Word("a"),
+        Token::GreaterThan,
+        Token::Word("first"),
+        Token::Comma,
+        Token::Word("second"),
+        Token::Pipe,
+        Token::Word("second"),
+        Token::Comma,
+        Token::Word("first"),
+        Token::GreaterThan,
+        Token::Word("c")
+    ]
+}
+
+lexer_test! {
+    test_lex_crossing_input_args,
+    "a >second| c { second: @first }",
+    [
+        Token::Word("a"),
+        Token::GreaterThan,
+        Token::Word("second"),
+        Token::Pipe,
+        Token::Word("c"),
+        Token::BraceOpen,
+        Token::Word("second"),
+        Token::Colon,
+        Token::InputIdent("first"),
+        Token::BraceClose
+    ]
+}
+
+lexer_test! {
+    test_lex_map_io_named,
+    "a @| c",
+    [
+        Token::Word("a"),
+        Token::MappingPipe,
+        Token::Word("c")
+    ]
+}