katzen-cafe/iowo
2
4
Fork 1
Code Issues 4 Pull requests 2 Projects Releases Packages Wiki Activity

lang: rework ast structure

Browse source
This commit is contained in:
Schrottkatze 2024-04-07 00:55:12 +02:00
parent ace69b0094
commit b6e304fa78
Signed by: schrottkatze
SSH key fingerprint: SHA256:hXb3t1vINBFCiDCmhRABHX5ocdbLiKyCdKI4HK2Rbbc
5 changed files with 209 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
crates/lang/src/err_reporting.rs
View file

@ -64,6 +64,7 @@ impl<'filename, 'tokens, 'src> ErrorCollector<'filename, 'tokens, 'src> {
for ((file, stage), errs) in raw_errors.into_iter() { for ((file, stage), errs) in raw_errors.into_iter() {
for err in errs { for err in errs {
eprintln!("e: {err:?}");
Report::build(ariadne::ReportKind::Error, file, err.span().start) Report::build(ariadne::ReportKind::Error, file, err.span().start)
.with_message(format!("error at stage {stage:?}, {:?}", err.reason())) .with_message(format!("error at stage {stage:?}, {:?}", err.reason()))
.with_label( .with_label(

47
crates/lang/src/parser.rs
View file

@ -2,14 +2,12 @@ use std::ops::Range;
use chumsky::{ use chumsky::{
error::Rich, error::Rich,
extra,
input::{Stream, ValueInput}, input::{Stream, ValueInput},
prelude::*, prelude::*,
primitive::just, primitive::just,
recursive::recursive, recursive::recursive,
select,
span::SimpleSpan, span::SimpleSpan,
IterParser, Parser, IterParser,
}; };
use indexmap::IndexMap; use indexmap::IndexMap;
use logos::{Logos, Source}; use logos::{Logos, Source};
@ -19,7 +17,7 @@ use crate::tokens::Token;
pub mod ast; pub mod ast;
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
use self::ast::{Expr, File}; use self::ast::{Expr, Expression, File};
pub type Span = SimpleSpan; pub type Span = SimpleSpan;
pub type Spanned<T> = (T, Span); pub type Spanned<T> = (T, Span);
@ -31,11 +29,13 @@ pub fn parse<'src>(src: &'src str) -> ParseResult<File<'_>, Rich<'_, Token<'_>>>
.map(|(t, s)| (t.expect("TODO: add lexer error(s)"), Span::from(s))) .map(|(t, s)| (t.expect("TODO: add lexer error(s)"), Span::from(s)))
.collect(); .collect();
let tok_stream = Stream::from_iter(toks).spanned((src.len()..src.len()).into()); let tok_stream = Stream::from_iter(toks).spanned((src.len()..src.len()).into());
expr_parser().parse(tok_stream) parser().parse(tok_stream)
} }
pub(crate) fn parser<
fn expr_parser<'tokens, 'src: 'tokens, I: ValueInput<'tokens, Token = Token<'src>, Span = Span>>( 'tokens,
) -> impl Parser<'tokens, I, File<'src>, extra::Err<Rich<'tokens, Token<'src>, Span>>> { 'src: 'tokens,
I: ValueInput<'tokens, Token = Token<'src>, Span = Span>,
>() -> impl Parser<'tokens, I, File<'src>, extra::Err<Rich<'tokens, Token<'src>, Span>>> {
let word = select! { Token::Word(word) => word }; let word = select! { Token::Word(word) => word };
let expr = recursive(|expr| { let expr = recursive(|expr| {
@ -43,7 +43,7 @@ fn expr_parser<'tokens, 'src: 'tokens, I: ValueInput<'tokens, Token = Token<'src
Token::VarIdent(name) => (Expr::Var as fn(_) -> _, name), Token::VarIdent(name) => (Expr::Var as fn(_) -> _, name),
Token::InputIdent(name) => (Expr::InputVar as fn(_) -> _, name) Token::InputIdent(name) => (Expr::InputVar as fn(_) -> _, name)
} }
.map_with(|(item_type, name), extra| item_type((name, extra.span()))) .map_with(|(item_type, name), extra| Expression::new(item_type(name), extra.span()))
.labelled("variable"); .labelled("variable");
let attrset = word let attrset = word
@ -62,11 +62,15 @@ fn expr_parser<'tokens, 'src: 'tokens, I: ValueInput<'tokens, Token = Token<'src
let node = word let node = word
.map_with(|v, e| (v, e.span())) .map_with(|v, e| (v, e.span()))
.then(attrset.clone().or_not()) .then(attrset.clone().or_not())
.map(|(name, params)| Expr::Node(name, params)) .map_with(|(name, params), extra| {
Expression::new(Expr::Node(name, params), extra.span())
})
.or(var) .or(var)
.or(attrset.map(Expr::AttrSet)) .or(attrset
.map_with(|attrset, extra| Expression::new(Expr::AttrSet(attrset), extra.span())))
.labelled("node"); .labelled("node");
#[allow(clippy::let_and_return)]
let pipeline = node let pipeline = node
.clone() .clone()
.then(choice(( .then(choice((
@ -75,8 +79,8 @@ fn expr_parser<'tokens, 'src: 'tokens, I: ValueInput<'tokens, Token = Token<'src
just(Token::NullPipe).to(Expr::NullPipe as fn(_, _) -> _), just(Token::NullPipe).to(Expr::NullPipe as fn(_, _) -> _),
))) )))
.repeated() .repeated()
.foldr(node, |(curr, pipe), next| { .foldr_with(node, |(curr, pipe), next, extra| {
pipe(Box::new(curr), Box::new(next)) Expression::new(pipe(Box::new(curr), Box::new(next)), extra.span())
}); });
pipeline pipeline
@ -85,14 +89,25 @@ fn expr_parser<'tokens, 'src: 'tokens, I: ValueInput<'tokens, Token = Token<'src
let decl = just(Token::Def).ignore_then( let decl = just(Token::Def).ignore_then(
word.map_with(|n, e| (n, e.span())) word.map_with(|n, e| (n, e.span()))
.then_ignore(just(Token::Equals)) .then_ignore(just(Token::Equals))
.then(expr.clone().map_with(|expr, extra| (expr, extra.span()))) .then(expr.clone().map(|expr| expr))
.then_ignore(just(Token::SemiColon)), .then_ignore(just(Token::SemiColon)),
); );
expr.map_with(|expr, extra| File { expr.map(|expr| File {
decls: IndexMap::from_iter([(("main", (0..0).into()), (expr, extra.span()))]), decls: IndexMap::from_iter([(("main", (0..0).into()), expr)]),
}) })
.or(decl.repeated().collect::<Vec<_>>().map(|decls| File { .or(decl.repeated().collect::<Vec<_>>().map(|decls| File {
decls: IndexMap::from_iter(decls), decls: IndexMap::from_iter(decls),
})) }))
} }
pub mod asg {
use std::collections::BTreeMap;
use petgraph::{adj::NodeIndex, data::Build, graph::DiGraph};
use super::{ast::Expr, Spanned};
pub struct Asg<'src> {
ast: Expr<'src>,
}
}

40
crates/lang/src/parser/ast.rs
View file

@ -2,35 +2,47 @@ use std::collections::{BTreeMap, HashMap};
use indexmap::IndexMap; use indexmap::IndexMap;
use super::Spanned; use super::{Span, Spanned};
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct File<'src> { pub struct File<'src> {
pub decls: IndexMap<Spanned<&'src str>, Spanned<Expr<'src>>>, pub decls: IndexMap<Spanned<&'src str>, Expression<'src>>,
}
#[derive(Debug, PartialEq, Eq)]
pub struct Expression<'src> {
pub expr: Expr<'src>,
pub span: Span,
}
impl<'src> Expression<'src> {
pub fn new(expr: Expr<'src>, span: Span) -> Self {
Self { expr, span }
}
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub enum Expr<'src> { pub enum Expr<'src> {
Node( Node(
Spanned<&'src str>, Spanned<&'src str>,
Option<Spanned<IndexMap<Spanned<&'src str>, Expr<'src>>>>, Option<Spanned<IndexMap<Spanned<&'src str>, Expression<'src>>>>,
), ),
SimplePipe(Box<Expr<'src>>, Box<Expr<'src>>), SimplePipe(Box<Expression<'src>>, Box<Expression<'src>>),
NamingPipe( NamingPipe(
Box<Expr<'src>>, Box<Expression<'src>>,
(Vec<Spanned<&'src str>>, Vec<Spanned<&'src str>>), (Vec<Spanned<&'src str>>, Vec<Spanned<&'src str>>),
Box<Expr<'src>>, Box<Expression<'src>>,
), ),
MappingPipe(Box<Expr<'src>>, Box<Expr<'src>>), MappingPipe(Box<Expression<'src>>, Box<Expression<'src>>),
NullPipe(Box<Expr<'src>>, Box<Expr<'src>>), NullPipe(Box<Expression<'src>>, Box<Expression<'src>>),
MultiPipe(IndexMap<Spanned<&'src str>, Expr<'src>>), MultiPipe(IndexMap<Spanned<&'src str>, Expression<'src>>),
LetIn( LetIn(
IndexMap<Spanned<&'src str>, Box<Expr<'src>>>, IndexMap<Spanned<&'src str>, Box<Expression<'src>>>,
Box<Expr<'src>>, Box<Expression<'src>>,
), ),
// $ // $
Var(Spanned<&'src str>), Var(&'src str),
// @ // @
InputVar(Spanned<&'src str>), InputVar(&'src str),
AttrSet(Spanned<IndexMap<Spanned<&'src str>, Expr<'src>>>), AttrSet(Spanned<IndexMap<Spanned<&'src str>, Expression<'src>>>),
} }

266
crates/lang/src/parser/tests.rs
View file

@ -6,138 +6,138 @@ use chumsky::prelude::*;
use indexmap::IndexMap; use indexmap::IndexMap;
use logos::Logos; use logos::Logos;
#[test] // #[test]
fn test_parse_node_with_params() { // fn test_parse_node_with_params() {
const INPUT: &str = "meow [ hello: $foo, world: @bar]"; // const INPUT: &str = "meow [ hello: $foo, world: @bar]";
assert_eq!( // assert_eq!(
parse(INPUT).unwrap(), // parse(INPUT).unwrap(),
File { // File {
decls: IndexMap::from_iter([( // decls: IndexMap::from_iter([(
("main", (0..0).into()), // ("main", (0..0).into()),
( // (
Expr::Node( // Expr::Node(
("meow", (0..4).into()), // ("meow", (0..4).into()),
Some(( // Some((
IndexMap::from_iter([ // IndexMap::from_iter([
( // (
("hello", (7..12).into()), // ("hello", (7..12).into()),
Expr::Var(("foo", (14..18).into())) // Expr::Var(("foo", (14..18).into()))
), // ),
( // (
("world", (20..25).into()), // ("world", (20..25).into()),
Expr::InputVar(("bar", (27..31).into())) // Expr::InputVar(("bar", (27..31).into()))
) // )
]), // ]),
(5..32).into() // (5..32).into()
)) // ))
), // ),
(0..32).into() // (0..32).into()
) // )
)]) // )])
} // }
); // );
} // }
fn test_parse_multiple_top_level_complex() { // fn test_parse_multiple_top_level_complex() {
const INPUT: &str = r"def main = meow // const INPUT: &str = r"def main = meow
| uwu // | uwu
[ foo: @bar // [ foo: @bar
, hello: world @| test [ more: params ] | yay // , hello: world @| test [ more: params ] | yay
] // ]
!| awa // !| awa
@| nya // @| nya
| rawr; // | rawr;
def test = meow // def test = meow
[ hello: $foo // [ hello: $foo
, world: @bar // , world: @bar
]; // ];
"; // ";
assert_eq!( // assert_eq!(
parse(INPUT).unwrap(), // parse(INPUT).unwrap(),
File { // File {
decls: IndexMap::from_iter([ // decls: IndexMap::from_iter([
( // (
("main", (4..8).into()), // ("main", (4..8).into()),
( // (
Expr::SimplePipe( // Expr::SimplePipe(
Box::new(Expr::Node(("meow", (11..15).into()), None)), // Box::new(Expr::Node(("meow", (11..15).into()), None)),
Box::new(Expr::NullPipe( // Box::new(Expr::NullPipe(
Box::new(Expr::Node( // Box::new(Expr::Node(
("uwu", (20..23).into()), // ("uwu", (20..23).into()),
Some(( // Some((
IndexMap::from_iter([ // IndexMap::from_iter([
( // (
("foo", (29..32).into()), // ("foo", (29..32).into()),
Expr::InputVar(("bar", (34..38).into())) // Expr::InputVar(("bar", (34..38).into()))
), // ),
( // (
("hello", (44..49).into()), // ("hello", (44..49).into()),
Expr::MappingPipe( // Expr::MappingPipe(
Box::new(Expr::Node( // Box::new(Expr::Node(
("world", (51..56).into()), // ("world", (51..56).into()),
None // None
)), // )),
Box::new(Expr::SimplePipe( // Box::new(Expr::SimplePipe(
Box::new(Expr::Node( // Box::new(Expr::Node(
("test", (60..64).into()), // ("test", (60..64).into()),
Some(( // Some((
IndexMap::from_iter([( // IndexMap::from_iter([(
("more", (67..71).into()), // ("more", (67..71).into()),
Expr::Node( // Expr::Node(
("params", (73..79).into()), // ("params", (73..79).into()),
None // None
) // )
)]), // )]),
(65..81).into() // (65..81).into()
)) // ))
)), // )),
Box::new(Expr::Node( // Box::new(Expr::Node(
("yay", (84..87).into()), // ("yay", (84..87).into()),
None // None
)) // ))
)) // ))
) // )
) // )
]), // ]),
(27..92).into() // (27..92).into()
)) // ))
)), // )),
Box::new(Expr::MappingPipe( // Box::new(Expr::MappingPipe(
Box::new(Expr::Node(("awa", (97..100).into()), None)), // Box::new(Expr::Node(("awa", (97..100).into()), None)),
Box::new(Expr::SimplePipe( // Box::new(Expr::SimplePipe(
Box::new(Expr::Node(("nya", (106..109).into()), None)), // Box::new(Expr::Node(("nya", (106..109).into()), None)),
Box::new(Expr::Node(("rawr", (114..118).into()), None)) // Box::new(Expr::Node(("rawr", (114..118).into()), None))
)) // ))
)) // ))
)) // ))
), // ),
(11..118).into() // (11..118).into()
), // ),
), // ),
( // (
("test", (125..129).into()), // ("test", (125..129).into()),
( // (
Expr::Node( // Expr::Node(
("meow", (132..136).into()), // ("meow", (132..136).into()),
Some(( // Some((
IndexMap::from_iter([ // IndexMap::from_iter([
( // (
("hello", (141..146).into()), // ("hello", (141..146).into()),
Expr::Var(("foo", (148..152).into())) // Expr::Var(("foo", (148..152).into()))
), // ),
( // (
("world", (156..161).into()), // ("world", (156..161).into()),
Expr::InputVar(("bar", (163..167).into())) // Expr::InputVar(("bar", (163..167).into()))
) // )
]), // ]),
(139..171).into() // (139..171).into()
)) // ))
), // ),
(132..171).into() // (132..171).into()
) // )
) // )
]) // ])
} // }
); // );
} // }

19
testfiles/test.owo
View file

@ -1 +1,18 @@
meow gay | uwu def main = meow
| uwu
[ foo: @bar
, hello: world @| test [ more: params ] | yay
]
!| awa
@| nya
| rawr;
def test = meow
[ hello: $foo
, world: @bar
];
def blah = [
pipe1: meow | uwu,
pipe2: lorem | ipsum
] | flkjdsafkjl;