Compare commits
2 commits
01de2f385a
...
a56df7c41c
Author | SHA1 | Date | |
---|---|---|---|
a56df7c41c | |||
fdfff2c33a |
17 changed files with 68 additions and 35 deletions
4
Cargo.lock
generated
4
Cargo.lock
generated
|
@ -132,6 +132,10 @@ dependencies = [
|
|||
"logos-codegen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lopal"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "lopal_core"
|
||||
version = "0.1.0"
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
[workspace]
|
||||
members = [
|
||||
"crates/lopal",
|
||||
"crates/lopal_core",
|
||||
"crates/lopal_json"
|
||||
]
|
||||
|
@ -22,7 +23,7 @@ if_then_some_else_none = "warn"
|
|||
integer_division = "warn"
|
||||
let_underscore_must_use = "warn"
|
||||
manual_clamp = "warn"
|
||||
pedantic = "warn"
|
||||
pedantic = { level = "warn", priority = -1 }
|
||||
str_to_string = "warn"
|
||||
unneeded_field_pattern = "warn"
|
||||
unnested_or_patterns = "warn"
|
||||
|
@ -31,7 +32,8 @@ allow_attributes_without_reason = "deny"
|
|||
cast_lossless = "deny"
|
||||
fallible_impl_from = "deny"
|
||||
unnecessary_cast = "deny"
|
||||
unwrap_used = "deny"
|
||||
# TODO: reenable
|
||||
# unwrap_used = "deny"
|
||||
# allowed, since you can give reasons
|
||||
expect_used = "allow"
|
||||
|
||||
|
|
9
crates/lopal/Cargo.toml
Normal file
9
crates/lopal/Cargo.toml
Normal file
|
@ -0,0 +1,9 @@
|
|||
[package]
|
||||
name = "lopal"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
5
crates/lopal/README.md
Normal file
5
crates/lopal/README.md
Normal file
|
@ -0,0 +1,5 @@
|
|||
# LOssless PArser Library
|
||||
|
||||
Yeah :3
|
||||
|
||||
Reserved for the main crate, work is currently happening over in [lopal_core]
|
3
crates/lopal/src/main.rs
Normal file
3
crates/lopal/src/main.rs
Normal file
|
@ -0,0 +1,3 @@
|
|||
fn main() {
|
||||
println!("Hello, world!");
|
||||
}
|
3
crates/lopal_core/README.md
Normal file
3
crates/lopal_core/README.md
Normal file
|
@ -0,0 +1,3 @@
|
|||
# LoPaL Core
|
||||
|
||||
Currently most development is happening here, until i get to a lot of rewriting and stuff.
|
1
crates/lopal_core/src/GreenNodeBuilder;
Normal file
1
crates/lopal_core/src/GreenNodeBuilder;
Normal file
|
@ -0,0 +1 @@
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
//! Heavily in-dev parser library.
|
||||
//!
|
||||
//! Inspired by rust-analyzer.
|
||||
//!
|
||||
//! See [_Modern Parser Generator_](https://matklad.github.io/2018/06/06/modern-parser-generator.html) by Matklad
|
||||
#![feature(iter_collect_into)]
|
||||
#![allow(dead_code, reason = "This is an unfinished library.")]
|
||||
pub mod parser;
|
||||
|
||||
pub use parser::{
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
use std::{cell::Cell, fmt, marker::PhantomData, mem};
|
||||
|
||||
use enumset::{EnumSet, EnumSetType};
|
||||
use rowan::{GreenNode, GreenNodeBuilder};
|
||||
use rowan::GreenNodeBuilder;
|
||||
|
||||
use crate::parser::event::NodeKind;
|
||||
|
||||
|
@ -14,7 +14,8 @@ mod input;
|
|||
pub mod marker;
|
||||
pub mod output;
|
||||
|
||||
/// this is used to define some required SyntaxKinds like an EOF token or an error token
|
||||
/// this is used to define some required `SyntaxKinds` like an EOF token or an error token
|
||||
/// These should be different, idk what happens if they're the same.
|
||||
pub trait SyntaxElement
|
||||
where
|
||||
Self: EnumSetType
|
||||
|
@ -32,6 +33,10 @@ where
|
|||
const SYNTAX_ROOT: Self;
|
||||
}
|
||||
|
||||
/// The core parser that you interact with.
|
||||
/// Should be passed to all parser functions as first argument.
|
||||
/// Currently very minimal/bare-bones.
|
||||
/// See the [`lopal_json`](https://forge.katzen.cafe/schrottkatze/lopal/src/branch/main/crates/lopal_json) source code for reference.
|
||||
pub struct Parser<'src, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
||||
input: Input<'src, SyntaxKind>,
|
||||
pos: usize,
|
||||
|
@ -40,9 +45,7 @@ pub struct Parser<'src, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
|||
steps: Cell<u32>,
|
||||
}
|
||||
|
||||
impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
|
||||
Parser<'src, SyntaxKind, SyntaxErr>
|
||||
{
|
||||
impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> Parser<'_, SyntaxKind, SyntaxErr> {
|
||||
/// eat all meaningless tokens at the end of the file.
|
||||
pub fn eat_succeeding_meaningless(&mut self) {
|
||||
self.push_ev(Event::Eat {
|
||||
|
@ -107,11 +110,7 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
|
|||
|
||||
pub fn finish(self) -> ParserOutput<SyntaxKind, SyntaxErr> {
|
||||
let Self {
|
||||
input,
|
||||
pos,
|
||||
mut events,
|
||||
step_limit,
|
||||
steps,
|
||||
input, mut events, ..
|
||||
} = self;
|
||||
let (mut raw_toks, meaningless_tokens) = input.dissolve();
|
||||
let mut builder = GreenNodeBuilder::new();
|
||||
|
@ -139,7 +138,7 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
|
|||
let mut idx = i;
|
||||
let mut fp = forward_parent;
|
||||
while let Some(fwd) = fp {
|
||||
idx += fwd as usize;
|
||||
idx += fwd;
|
||||
fp = match mem::replace(&mut events[idx], Event::tombstone()) {
|
||||
Event::Start {
|
||||
kind,
|
||||
|
@ -174,9 +173,9 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
|
|||
NodeKind::Syntax(kind) => builder.start_node(kind.into()),
|
||||
NodeKind::Error(err) => {
|
||||
errors.push(err);
|
||||
builder.start_node(SyntaxKind::SYNTAX_ERROR.into())
|
||||
builder.start_node(SyntaxKind::SYNTAX_ERROR.into());
|
||||
}
|
||||
_ => {}
|
||||
NodeKind::Tombstone => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -210,6 +209,7 @@ pub struct ParserBuilder<
|
|||
}
|
||||
|
||||
impl<'src, SyntaxKind: SyntaxElement> ParserBuilder<'src, SyntaxKind> {
|
||||
#[must_use]
|
||||
pub fn new(raw_toks: Vec<(SyntaxKind, &'src str)>) -> Self {
|
||||
Self {
|
||||
raw_toks,
|
||||
|
@ -220,16 +220,19 @@ impl<'src, SyntaxKind: SyntaxElement> ParserBuilder<'src, SyntaxKind> {
|
|||
|
||||
/// Sets the parser step limit.
|
||||
/// Defaults to 4096
|
||||
#[must_use]
|
||||
pub fn step_limit(mut self, new: u32) -> Self {
|
||||
self.step_limit = new;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn add_meaningless(mut self, kind: SyntaxKind) -> Self {
|
||||
self.meaningless_token_kinds.insert(kind);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn add_meaningless_many(mut self, kind: Vec<SyntaxKind>) -> Self {
|
||||
self.meaningless_token_kinds
|
||||
.insert_all(kind.into_iter().collect());
|
||||
|
|
|
@ -1,7 +1,10 @@
|
|||
use std::fmt;
|
||||
|
||||
/// A marker trait... for now!
|
||||
// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
|
||||
///
|
||||
/// Marks a type as, well, an error type.
|
||||
///
|
||||
/// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
|
||||
pub trait SyntaxError
|
||||
where
|
||||
Self: fmt::Debug + Clone + PartialEq + Eq,
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
use enumset::EnumSetType;
|
||||
|
||||
use super::{error::SyntaxError, SyntaxElement};
|
||||
|
||||
pub enum Event<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
||||
pub(crate) enum Event<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
||||
Start {
|
||||
kind: NodeKind<SyntaxKind, SyntaxErr>,
|
||||
forward_parent: Option<usize>,
|
||||
|
@ -23,7 +21,7 @@ impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> Event<SyntaxKind, Syntax
|
|||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub enum NodeKind<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
||||
pub(crate) enum NodeKind<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
|
||||
Tombstone,
|
||||
Syntax(SyntaxKind),
|
||||
Error(SyntaxErr),
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
use enumset::{EnumSet, EnumSetType};
|
||||
use enumset::EnumSet;
|
||||
|
||||
use super::SyntaxElement;
|
||||
|
||||
|
@ -18,7 +18,7 @@ impl<'src, SyntaxKind: SyntaxElement> Input<'src, SyntaxKind> {
|
|||
let mut meaningful_toks = Vec::new();
|
||||
|
||||
if let Some(meaningless) = meaningless {
|
||||
let meaningful_toks = raw_toks
|
||||
raw_toks
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i))
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
use drop_bomb::DropBomb;
|
||||
use rowan::SyntaxKind;
|
||||
|
||||
use super::{
|
||||
error::SyntaxError,
|
||||
|
@ -87,7 +86,7 @@ impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> CompletedMarker<SyntaxKi
|
|||
Event::Start { forward_parent, .. } => {
|
||||
// point forward parent of the node this marker completed to the new node
|
||||
// will later be used to make the new node a parent of the current node.
|
||||
*forward_parent = Some(new_pos.pos - self.pos)
|
||||
*forward_parent = Some(new_pos.pos - self.pos);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
use std::{fmt, marker::PhantomData};
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use rowan::{GreenNode, GreenNodeData, GreenTokenData, NodeOrToken};
|
||||
|
||||
|
@ -41,14 +41,14 @@ fn debug_print_output<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>(
|
|||
match node {
|
||||
NodeOrToken::Node(n) => {
|
||||
let kind: SyntaxKind = node.kind().into();
|
||||
if kind != SyntaxKind::SYNTAX_ERROR {
|
||||
write!(f, "{:?} {{{maybe_newline}", kind)?;
|
||||
} else {
|
||||
if kind == SyntaxKind::SYNTAX_ERROR {
|
||||
let err = errs
|
||||
.pop()
|
||||
.expect("all error syntax nodes should correspond to an error");
|
||||
|
||||
write!(f, "{:?}: {err:?} {{{maybe_newline}", kind)?;
|
||||
write!(f, "{kind:?}: {err:?} {{{maybe_newline}")?;
|
||||
} else {
|
||||
write!(f, "{kind:?} {{{maybe_newline}")?;
|
||||
}
|
||||
for c in n.children() {
|
||||
debug_print_output::<SyntaxKind, SyntaxErr>(c, f, lvl + 1, errs)?;
|
||||
|
|
|
@ -1,11 +1,7 @@
|
|||
use array::array;
|
||||
use enumset::{enum_set, EnumSet};
|
||||
use lopal_core::parser::ParserBuilder;
|
||||
|
||||
use crate::{
|
||||
syntax_error::SyntaxError,
|
||||
syntax_kind::{lex, SyntaxKind},
|
||||
};
|
||||
use crate::{syntax_error::SyntaxError, syntax_kind::SyntaxKind};
|
||||
|
||||
use self::object::object;
|
||||
|
||||
|
|
|
@ -14,7 +14,7 @@ pub fn lex(src: &str) -> Vec<(SyntaxKind, &str)> {
|
|||
#[derive(enumset::EnumSetType, Debug, Logos, PartialEq, Eq, Clone, Copy, Hash)]
|
||||
#[repr(u16)]
|
||||
#[enumset(no_super_impls)]
|
||||
#[allow(non_camel_case_types)]
|
||||
#[allow(non_camel_case_types, reason = "convention")]
|
||||
pub enum SyntaxKind {
|
||||
OBJECT,
|
||||
MEMBER,
|
||||
|
|
|
@ -30,6 +30,7 @@
|
|||
{
|
||||
default = pkgs.mkShell rec {
|
||||
buildInputs = with pkgs; [
|
||||
cargo-watch
|
||||
toolchain
|
||||
];
|
||||
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs;
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue