Compare commits
2 commits
01de2f385a
...
a56df7c41c
Author | SHA1 | Date | |
---|---|---|---|
a56df7c41c | |||
fdfff2c33a |
17 changed files with 68 additions and 35 deletions
4
Cargo.lock
generated
4
Cargo.lock
generated
|
@ -132,6 +132,10 @@ dependencies = [
|
||||||
"logos-codegen",
|
"logos-codegen",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lopal"
|
||||||
|
version = "0.1.0"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lopal_core"
|
name = "lopal_core"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
[workspace]
|
[workspace]
|
||||||
members = [
|
members = [
|
||||||
|
"crates/lopal",
|
||||||
"crates/lopal_core",
|
"crates/lopal_core",
|
||||||
"crates/lopal_json"
|
"crates/lopal_json"
|
||||||
]
|
]
|
||||||
|
@ -22,7 +23,7 @@ if_then_some_else_none = "warn"
|
||||||
integer_division = "warn"
|
integer_division = "warn"
|
||||||
let_underscore_must_use = "warn"
|
let_underscore_must_use = "warn"
|
||||||
manual_clamp = "warn"
|
manual_clamp = "warn"
|
||||||
pedantic = "warn"
|
pedantic = { level = "warn", priority = -1 }
|
||||||
str_to_string = "warn"
|
str_to_string = "warn"
|
||||||
unneeded_field_pattern = "warn"
|
unneeded_field_pattern = "warn"
|
||||||
unnested_or_patterns = "warn"
|
unnested_or_patterns = "warn"
|
||||||
|
@ -31,7 +32,8 @@ allow_attributes_without_reason = "deny"
|
||||||
cast_lossless = "deny"
|
cast_lossless = "deny"
|
||||||
fallible_impl_from = "deny"
|
fallible_impl_from = "deny"
|
||||||
unnecessary_cast = "deny"
|
unnecessary_cast = "deny"
|
||||||
unwrap_used = "deny"
|
# TODO: reenable
|
||||||
|
# unwrap_used = "deny"
|
||||||
# allowed, since you can give reasons
|
# allowed, since you can give reasons
|
||||||
expect_used = "allow"
|
expect_used = "allow"
|
||||||
|
|
||||||
|
|
9
crates/lopal/Cargo.toml
Normal file
9
crates/lopal/Cargo.toml
Normal file
|
@ -0,0 +1,9 @@
|
||||||
|
[package]
|
||||||
|
name = "lopal"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
5
crates/lopal/README.md
Normal file
5
crates/lopal/README.md
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
# LOssless PArser Library
|
||||||
|
|
||||||
|
Yeah :3
|
||||||
|
|
||||||
|
Reserved for the main crate, work is currently happening over in [lopal_core]
|
3
crates/lopal/src/main.rs
Normal file
3
crates/lopal/src/main.rs
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
fn main() {
|
||||||
|
println!("Hello, world!");
|
||||||
|
}
|
3
crates/lopal_core/README.md
Normal file
3
crates/lopal_core/README.md
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
# LoPaL Core
|
||||||
|
|
||||||
|
Currently most development is happening here, until i get to a lot of rewriting and stuff.
|
1
crates/lopal_core/src/GreenNodeBuilder;
Normal file
1
crates/lopal_core/src/GreenNodeBuilder;
Normal file
|
@ -0,0 +1 @@
|
||||||
|
|
|
@ -1,4 +1,10 @@
|
||||||
|
//! Heavily in-dev parser library.
|
||||||
|
//!
|
||||||
|
//! Inspired by rust-analyzer.
|
||||||
|
//!
|
||||||
|
//! See [_Modern Parser Generator_](https://matklad.github.io/2018/06/06/modern-parser-generator.html) by Matklad
|
||||||
#![feature(iter_collect_into)]
|
#![feature(iter_collect_into)]
|
||||||
|
#![allow(dead_code, reason = "This is an unfinished library.")]
|
||||||
pub mod parser;
|
pub mod parser;
|
||||||
|
|
||||||
pub use parser::{
|
pub use parser::{
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
use std::{cell::Cell, fmt, marker::PhantomData, mem};
|
use std::{cell::Cell, fmt, marker::PhantomData, mem};
|
||||||
|
|
||||||
use enumset::{EnumSet, EnumSetType};
|
use enumset::{EnumSet, EnumSetType};
|
||||||
use rowan::{GreenNode, GreenNodeBuilder};
|
use rowan::GreenNodeBuilder;
|
||||||
|
|
||||||
use crate::parser::event::NodeKind;
|
use crate::parser::event::NodeKind;
|
||||||
|
|
||||||
|
@ -14,7 +14,8 @@ mod input;
|
||||||
pub mod marker;
|
pub mod marker;
|
||||||
pub mod output;
|
pub mod output;
|
||||||
|
|
||||||
/// this is used to define some required SyntaxKinds like an EOF token or an error token
|
/// this is used to define some required `SyntaxKinds` like an EOF token or an error token
|
||||||
|
/// These should be different, idk what happens if they're the same.
|
||||||
pub trait SyntaxElement
|
pub trait SyntaxElement
|
||||||
where
|
where
|
||||||
Self: EnumSetType
|
Self: EnumSetType
|
||||||
|
@ -32,6 +33,10 @@ where
|
||||||
const SYNTAX_ROOT: Self;
|
const SYNTAX_ROOT: Self;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The core parser that you interact with.
|
||||||
|
/// Should be passed to all parser functions as first argument.
|
||||||
|
/// Currently very minimal/bare-bones.
|
||||||
|
/// See the [`lopal_json`](https://forge.katzen.cafe/schrottkatze/lopal/src/branch/main/crates/lopal_json) source code for reference.
|
||||||
pub struct Parser<'src, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
pub struct Parser<'src, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
||||||
input: Input<'src, SyntaxKind>,
|
input: Input<'src, SyntaxKind>,
|
||||||
pos: usize,
|
pos: usize,
|
||||||
|
@ -40,9 +45,7 @@ pub struct Parser<'src, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
||||||
steps: Cell<u32>,
|
steps: Cell<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
|
impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> Parser<'_, SyntaxKind, SyntaxErr> {
|
||||||
Parser<'src, SyntaxKind, SyntaxErr>
|
|
||||||
{
|
|
||||||
/// eat all meaningless tokens at the end of the file.
|
/// eat all meaningless tokens at the end of the file.
|
||||||
pub fn eat_succeeding_meaningless(&mut self) {
|
pub fn eat_succeeding_meaningless(&mut self) {
|
||||||
self.push_ev(Event::Eat {
|
self.push_ev(Event::Eat {
|
||||||
|
@ -107,11 +110,7 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
|
||||||
|
|
||||||
pub fn finish(self) -> ParserOutput<SyntaxKind, SyntaxErr> {
|
pub fn finish(self) -> ParserOutput<SyntaxKind, SyntaxErr> {
|
||||||
let Self {
|
let Self {
|
||||||
input,
|
input, mut events, ..
|
||||||
pos,
|
|
||||||
mut events,
|
|
||||||
step_limit,
|
|
||||||
steps,
|
|
||||||
} = self;
|
} = self;
|
||||||
let (mut raw_toks, meaningless_tokens) = input.dissolve();
|
let (mut raw_toks, meaningless_tokens) = input.dissolve();
|
||||||
let mut builder = GreenNodeBuilder::new();
|
let mut builder = GreenNodeBuilder::new();
|
||||||
|
@ -139,7 +138,7 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
|
||||||
let mut idx = i;
|
let mut idx = i;
|
||||||
let mut fp = forward_parent;
|
let mut fp = forward_parent;
|
||||||
while let Some(fwd) = fp {
|
while let Some(fwd) = fp {
|
||||||
idx += fwd as usize;
|
idx += fwd;
|
||||||
fp = match mem::replace(&mut events[idx], Event::tombstone()) {
|
fp = match mem::replace(&mut events[idx], Event::tombstone()) {
|
||||||
Event::Start {
|
Event::Start {
|
||||||
kind,
|
kind,
|
||||||
|
@ -174,9 +173,9 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
|
||||||
NodeKind::Syntax(kind) => builder.start_node(kind.into()),
|
NodeKind::Syntax(kind) => builder.start_node(kind.into()),
|
||||||
NodeKind::Error(err) => {
|
NodeKind::Error(err) => {
|
||||||
errors.push(err);
|
errors.push(err);
|
||||||
builder.start_node(SyntaxKind::SYNTAX_ERROR.into())
|
builder.start_node(SyntaxKind::SYNTAX_ERROR.into());
|
||||||
}
|
}
|
||||||
_ => {}
|
NodeKind::Tombstone => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -210,6 +209,7 @@ pub struct ParserBuilder<
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'src, SyntaxKind: SyntaxElement> ParserBuilder<'src, SyntaxKind> {
|
impl<'src, SyntaxKind: SyntaxElement> ParserBuilder<'src, SyntaxKind> {
|
||||||
|
#[must_use]
|
||||||
pub fn new(raw_toks: Vec<(SyntaxKind, &'src str)>) -> Self {
|
pub fn new(raw_toks: Vec<(SyntaxKind, &'src str)>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
raw_toks,
|
raw_toks,
|
||||||
|
@ -220,16 +220,19 @@ impl<'src, SyntaxKind: SyntaxElement> ParserBuilder<'src, SyntaxKind> {
|
||||||
|
|
||||||
/// Sets the parser step limit.
|
/// Sets the parser step limit.
|
||||||
/// Defaults to 4096
|
/// Defaults to 4096
|
||||||
|
#[must_use]
|
||||||
pub fn step_limit(mut self, new: u32) -> Self {
|
pub fn step_limit(mut self, new: u32) -> Self {
|
||||||
self.step_limit = new;
|
self.step_limit = new;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn add_meaningless(mut self, kind: SyntaxKind) -> Self {
|
pub fn add_meaningless(mut self, kind: SyntaxKind) -> Self {
|
||||||
self.meaningless_token_kinds.insert(kind);
|
self.meaningless_token_kinds.insert(kind);
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
pub fn add_meaningless_many(mut self, kind: Vec<SyntaxKind>) -> Self {
|
pub fn add_meaningless_many(mut self, kind: Vec<SyntaxKind>) -> Self {
|
||||||
self.meaningless_token_kinds
|
self.meaningless_token_kinds
|
||||||
.insert_all(kind.into_iter().collect());
|
.insert_all(kind.into_iter().collect());
|
||||||
|
|
|
@ -1,7 +1,10 @@
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
|
||||||
/// A marker trait... for now!
|
/// A marker trait... for now!
|
||||||
// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
|
///
|
||||||
|
/// Marks a type as, well, an error type.
|
||||||
|
///
|
||||||
|
/// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
|
||||||
pub trait SyntaxError
|
pub trait SyntaxError
|
||||||
where
|
where
|
||||||
Self: fmt::Debug + Clone + PartialEq + Eq,
|
Self: fmt::Debug + Clone + PartialEq + Eq,
|
||||||
|
|
|
@ -1,8 +1,6 @@
|
||||||
use enumset::EnumSetType;
|
|
||||||
|
|
||||||
use super::{error::SyntaxError, SyntaxElement};
|
use super::{error::SyntaxError, SyntaxElement};
|
||||||
|
|
||||||
pub enum Event<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
pub(crate) enum Event<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
||||||
Start {
|
Start {
|
||||||
kind: NodeKind<SyntaxKind, SyntaxErr>,
|
kind: NodeKind<SyntaxKind, SyntaxErr>,
|
||||||
forward_parent: Option<usize>,
|
forward_parent: Option<usize>,
|
||||||
|
@ -23,7 +21,7 @@ impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> Event<SyntaxKind, Syntax
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, PartialEq, Eq)]
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
pub enum NodeKind<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
pub(crate) enum NodeKind<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
||||||
Tombstone,
|
Tombstone,
|
||||||
Syntax(SyntaxKind),
|
Syntax(SyntaxKind),
|
||||||
Error(SyntaxErr),
|
Error(SyntaxErr),
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
use enumset::{EnumSet, EnumSetType};
|
use enumset::EnumSet;
|
||||||
|
|
||||||
use super::SyntaxElement;
|
use super::SyntaxElement;
|
||||||
|
|
||||||
|
@ -18,7 +18,7 @@ impl<'src, SyntaxKind: SyntaxElement> Input<'src, SyntaxKind> {
|
||||||
let mut meaningful_toks = Vec::new();
|
let mut meaningful_toks = Vec::new();
|
||||||
|
|
||||||
if let Some(meaningless) = meaningless {
|
if let Some(meaningless) = meaningless {
|
||||||
let meaningful_toks = raw_toks
|
raw_toks
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i))
|
.filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i))
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
use drop_bomb::DropBomb;
|
use drop_bomb::DropBomb;
|
||||||
use rowan::SyntaxKind;
|
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
error::SyntaxError,
|
error::SyntaxError,
|
||||||
|
@ -87,7 +86,7 @@ impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> CompletedMarker<SyntaxKi
|
||||||
Event::Start { forward_parent, .. } => {
|
Event::Start { forward_parent, .. } => {
|
||||||
// point forward parent of the node this marker completed to the new node
|
// point forward parent of the node this marker completed to the new node
|
||||||
// will later be used to make the new node a parent of the current node.
|
// will later be used to make the new node a parent of the current node.
|
||||||
*forward_parent = Some(new_pos.pos - self.pos)
|
*forward_parent = Some(new_pos.pos - self.pos);
|
||||||
}
|
}
|
||||||
_ => unreachable!(),
|
_ => unreachable!(),
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
use std::{fmt, marker::PhantomData};
|
use std::marker::PhantomData;
|
||||||
|
|
||||||
use rowan::{GreenNode, GreenNodeData, GreenTokenData, NodeOrToken};
|
use rowan::{GreenNode, GreenNodeData, GreenTokenData, NodeOrToken};
|
||||||
|
|
||||||
|
@ -41,14 +41,14 @@ fn debug_print_output<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>(
|
||||||
match node {
|
match node {
|
||||||
NodeOrToken::Node(n) => {
|
NodeOrToken::Node(n) => {
|
||||||
let kind: SyntaxKind = node.kind().into();
|
let kind: SyntaxKind = node.kind().into();
|
||||||
if kind != SyntaxKind::SYNTAX_ERROR {
|
if kind == SyntaxKind::SYNTAX_ERROR {
|
||||||
write!(f, "{:?} {{{maybe_newline}", kind)?;
|
|
||||||
} else {
|
|
||||||
let err = errs
|
let err = errs
|
||||||
.pop()
|
.pop()
|
||||||
.expect("all error syntax nodes should correspond to an error");
|
.expect("all error syntax nodes should correspond to an error");
|
||||||
|
|
||||||
write!(f, "{:?}: {err:?} {{{maybe_newline}", kind)?;
|
write!(f, "{kind:?}: {err:?} {{{maybe_newline}")?;
|
||||||
|
} else {
|
||||||
|
write!(f, "{kind:?} {{{maybe_newline}")?;
|
||||||
}
|
}
|
||||||
for c in n.children() {
|
for c in n.children() {
|
||||||
debug_print_output::<SyntaxKind, SyntaxErr>(c, f, lvl + 1, errs)?;
|
debug_print_output::<SyntaxKind, SyntaxErr>(c, f, lvl + 1, errs)?;
|
||||||
|
|
|
@ -1,11 +1,7 @@
|
||||||
use array::array;
|
use array::array;
|
||||||
use enumset::{enum_set, EnumSet};
|
use enumset::{enum_set, EnumSet};
|
||||||
use lopal_core::parser::ParserBuilder;
|
|
||||||
|
|
||||||
use crate::{
|
use crate::{syntax_error::SyntaxError, syntax_kind::SyntaxKind};
|
||||||
syntax_error::SyntaxError,
|
|
||||||
syntax_kind::{lex, SyntaxKind},
|
|
||||||
};
|
|
||||||
|
|
||||||
use self::object::object;
|
use self::object::object;
|
||||||
|
|
||||||
|
|
|
@ -14,7 +14,7 @@ pub fn lex(src: &str) -> Vec<(SyntaxKind, &str)> {
|
||||||
#[derive(enumset::EnumSetType, Debug, Logos, PartialEq, Eq, Clone, Copy, Hash)]
|
#[derive(enumset::EnumSetType, Debug, Logos, PartialEq, Eq, Clone, Copy, Hash)]
|
||||||
#[repr(u16)]
|
#[repr(u16)]
|
||||||
#[enumset(no_super_impls)]
|
#[enumset(no_super_impls)]
|
||||||
#[allow(non_camel_case_types)]
|
#[allow(non_camel_case_types, reason = "convention")]
|
||||||
pub enum SyntaxKind {
|
pub enum SyntaxKind {
|
||||||
OBJECT,
|
OBJECT,
|
||||||
MEMBER,
|
MEMBER,
|
||||||
|
|
|
@ -30,6 +30,7 @@
|
||||||
{
|
{
|
||||||
default = pkgs.mkShell rec {
|
default = pkgs.mkShell rec {
|
||||||
buildInputs = with pkgs; [
|
buildInputs = with pkgs; [
|
||||||
|
cargo-watch
|
||||||
toolchain
|
toolchain
|
||||||
];
|
];
|
||||||
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs;
|
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs;
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue