Compare commits

...

2 commits

Author SHA1 Message Date
a56df7c41c
fix some clippy stuff 2025-05-13 10:05:59 +02:00
fdfff2c33a
get ready for early publish 2025-05-13 09:49:20 +02:00
17 changed files with 68 additions and 35 deletions

4
Cargo.lock generated
View file

@ -132,6 +132,10 @@ dependencies = [
"logos-codegen", "logos-codegen",
] ]
[[package]]
name = "lopal"
version = "0.1.0"
[[package]] [[package]]
name = "lopal_core" name = "lopal_core"
version = "0.1.0" version = "0.1.0"

View file

@ -1,5 +1,6 @@
[workspace] [workspace]
members = [ members = [
"crates/lopal",
"crates/lopal_core", "crates/lopal_core",
"crates/lopal_json" "crates/lopal_json"
] ]
@ -22,7 +23,7 @@ if_then_some_else_none = "warn"
integer_division = "warn" integer_division = "warn"
let_underscore_must_use = "warn" let_underscore_must_use = "warn"
manual_clamp = "warn" manual_clamp = "warn"
pedantic = "warn" pedantic = { level = "warn", priority = -1 }
str_to_string = "warn" str_to_string = "warn"
unneeded_field_pattern = "warn" unneeded_field_pattern = "warn"
unnested_or_patterns = "warn" unnested_or_patterns = "warn"
@ -31,7 +32,8 @@ allow_attributes_without_reason = "deny"
cast_lossless = "deny" cast_lossless = "deny"
fallible_impl_from = "deny" fallible_impl_from = "deny"
unnecessary_cast = "deny" unnecessary_cast = "deny"
unwrap_used = "deny" # TODO: reenable
# unwrap_used = "deny"
# allowed, since you can give reasons # allowed, since you can give reasons
expect_used = "allow" expect_used = "allow"

9
crates/lopal/Cargo.toml Normal file
View file

@ -0,0 +1,9 @@
[package]
name = "lopal"
version = "0.1.0"
edition = "2024"
[dependencies]
[lints]
workspace = true

5
crates/lopal/README.md Normal file
View file

@ -0,0 +1,5 @@
# LOssless PArser Library
Yeah :3
Reserved for the main crate, work is currently happening over in [lopal_core]

3
crates/lopal/src/main.rs Normal file
View file

@ -0,0 +1,3 @@
fn main() {
println!("Hello, world!");
}

View file

@ -0,0 +1,3 @@
# LoPaL Core
Currently most development is happening here, until i get to a lot of rewriting and stuff.

View file

@ -0,0 +1 @@

View file

@ -1,4 +1,10 @@
//! Heavily in-dev parser library.
//!
//! Inspired by rust-analyzer.
//!
//! See [_Modern Parser Generator_](https://matklad.github.io/2018/06/06/modern-parser-generator.html) by Matklad
#![feature(iter_collect_into)] #![feature(iter_collect_into)]
#![allow(dead_code, reason = "This is an unfinished library.")]
pub mod parser; pub mod parser;
pub use parser::{ pub use parser::{

View file

@ -1,7 +1,7 @@
use std::{cell::Cell, fmt, marker::PhantomData, mem}; use std::{cell::Cell, fmt, marker::PhantomData, mem};
use enumset::{EnumSet, EnumSetType}; use enumset::{EnumSet, EnumSetType};
use rowan::{GreenNode, GreenNodeBuilder}; use rowan::GreenNodeBuilder;
use crate::parser::event::NodeKind; use crate::parser::event::NodeKind;
@ -14,7 +14,8 @@ mod input;
pub mod marker; pub mod marker;
pub mod output; pub mod output;
/// this is used to define some required SyntaxKinds like an EOF token or an error token /// this is used to define some required `SyntaxKinds` like an EOF token or an error token
/// These should be different, idk what happens if they're the same.
pub trait SyntaxElement pub trait SyntaxElement
where where
Self: EnumSetType Self: EnumSetType
@ -32,6 +33,10 @@ where
const SYNTAX_ROOT: Self; const SYNTAX_ROOT: Self;
} }
/// The core parser that you interact with.
/// Should be passed to all parser functions as first argument.
/// Currently very minimal/bare-bones.
/// See the [`lopal_json`](https://forge.katzen.cafe/schrottkatze/lopal/src/branch/main/crates/lopal_json) source code for reference.
pub struct Parser<'src, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> { pub struct Parser<'src, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
input: Input<'src, SyntaxKind>, input: Input<'src, SyntaxKind>,
pos: usize, pos: usize,
@ -40,9 +45,7 @@ pub struct Parser<'src, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
steps: Cell<u32>, steps: Cell<u32>,
} }
impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> Parser<'_, SyntaxKind, SyntaxErr> {
Parser<'src, SyntaxKind, SyntaxErr>
{
/// eat all meaningless tokens at the end of the file. /// eat all meaningless tokens at the end of the file.
pub fn eat_succeeding_meaningless(&mut self) { pub fn eat_succeeding_meaningless(&mut self) {
self.push_ev(Event::Eat { self.push_ev(Event::Eat {
@ -107,11 +110,7 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
pub fn finish(self) -> ParserOutput<SyntaxKind, SyntaxErr> { pub fn finish(self) -> ParserOutput<SyntaxKind, SyntaxErr> {
let Self { let Self {
input, input, mut events, ..
pos,
mut events,
step_limit,
steps,
} = self; } = self;
let (mut raw_toks, meaningless_tokens) = input.dissolve(); let (mut raw_toks, meaningless_tokens) = input.dissolve();
let mut builder = GreenNodeBuilder::new(); let mut builder = GreenNodeBuilder::new();
@ -139,7 +138,7 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
let mut idx = i; let mut idx = i;
let mut fp = forward_parent; let mut fp = forward_parent;
while let Some(fwd) = fp { while let Some(fwd) = fp {
idx += fwd as usize; idx += fwd;
fp = match mem::replace(&mut events[idx], Event::tombstone()) { fp = match mem::replace(&mut events[idx], Event::tombstone()) {
Event::Start { Event::Start {
kind, kind,
@ -174,9 +173,9 @@ impl<'src, 'toks, SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>
NodeKind::Syntax(kind) => builder.start_node(kind.into()), NodeKind::Syntax(kind) => builder.start_node(kind.into()),
NodeKind::Error(err) => { NodeKind::Error(err) => {
errors.push(err); errors.push(err);
builder.start_node(SyntaxKind::SYNTAX_ERROR.into()) builder.start_node(SyntaxKind::SYNTAX_ERROR.into());
} }
_ => {} NodeKind::Tombstone => {}
} }
} }
} }
@ -210,6 +209,7 @@ pub struct ParserBuilder<
} }
impl<'src, SyntaxKind: SyntaxElement> ParserBuilder<'src, SyntaxKind> { impl<'src, SyntaxKind: SyntaxElement> ParserBuilder<'src, SyntaxKind> {
#[must_use]
pub fn new(raw_toks: Vec<(SyntaxKind, &'src str)>) -> Self { pub fn new(raw_toks: Vec<(SyntaxKind, &'src str)>) -> Self {
Self { Self {
raw_toks, raw_toks,
@ -220,16 +220,19 @@ impl<'src, SyntaxKind: SyntaxElement> ParserBuilder<'src, SyntaxKind> {
/// Sets the parser step limit. /// Sets the parser step limit.
/// Defaults to 4096 /// Defaults to 4096
#[must_use]
pub fn step_limit(mut self, new: u32) -> Self { pub fn step_limit(mut self, new: u32) -> Self {
self.step_limit = new; self.step_limit = new;
self self
} }
#[must_use]
pub fn add_meaningless(mut self, kind: SyntaxKind) -> Self { pub fn add_meaningless(mut self, kind: SyntaxKind) -> Self {
self.meaningless_token_kinds.insert(kind); self.meaningless_token_kinds.insert(kind);
self self
} }
#[must_use]
pub fn add_meaningless_many(mut self, kind: Vec<SyntaxKind>) -> Self { pub fn add_meaningless_many(mut self, kind: Vec<SyntaxKind>) -> Self {
self.meaningless_token_kinds self.meaningless_token_kinds
.insert_all(kind.into_iter().collect()); .insert_all(kind.into_iter().collect());

View file

@ -1,7 +1,10 @@
use std::fmt; use std::fmt;
/// A marker trait... for now! /// A marker trait... for now!
// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible ///
/// Marks a type as, well, an error type.
///
/// TODO: constrain that conversion to `NodeKind::Error` is enforced to be possible
pub trait SyntaxError pub trait SyntaxError
where where
Self: fmt::Debug + Clone + PartialEq + Eq, Self: fmt::Debug + Clone + PartialEq + Eq,

View file

@ -1,8 +1,6 @@
use enumset::EnumSetType;
use super::{error::SyntaxError, SyntaxElement}; use super::{error::SyntaxError, SyntaxElement};
pub enum Event<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> { pub(crate) enum Event<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
Start { Start {
kind: NodeKind<SyntaxKind, SyntaxErr>, kind: NodeKind<SyntaxKind, SyntaxErr>,
forward_parent: Option<usize>, forward_parent: Option<usize>,
@ -23,7 +21,7 @@ impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> Event<SyntaxKind, Syntax
} }
#[derive(Clone, PartialEq, Eq)] #[derive(Clone, PartialEq, Eq)]
pub enum NodeKind<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> { pub(crate) enum NodeKind<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> {
Tombstone, Tombstone,
Syntax(SyntaxKind), Syntax(SyntaxKind),
Error(SyntaxErr), Error(SyntaxErr),

View file

@ -1,4 +1,4 @@
use enumset::{EnumSet, EnumSetType}; use enumset::EnumSet;
use super::SyntaxElement; use super::SyntaxElement;
@ -18,7 +18,7 @@ impl<'src, SyntaxKind: SyntaxElement> Input<'src, SyntaxKind> {
let mut meaningful_toks = Vec::new(); let mut meaningful_toks = Vec::new();
if let Some(meaningless) = meaningless { if let Some(meaningless) = meaningless {
let meaningful_toks = raw_toks raw_toks
.iter() .iter()
.enumerate() .enumerate()
.filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i)) .filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i))

View file

@ -1,5 +1,4 @@
use drop_bomb::DropBomb; use drop_bomb::DropBomb;
use rowan::SyntaxKind;
use super::{ use super::{
error::SyntaxError, error::SyntaxError,
@ -87,7 +86,7 @@ impl<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError> CompletedMarker<SyntaxKi
Event::Start { forward_parent, .. } => { Event::Start { forward_parent, .. } => {
// point forward parent of the node this marker completed to the new node // point forward parent of the node this marker completed to the new node
// will later be used to make the new node a parent of the current node. // will later be used to make the new node a parent of the current node.
*forward_parent = Some(new_pos.pos - self.pos) *forward_parent = Some(new_pos.pos - self.pos);
} }
_ => unreachable!(), _ => unreachable!(),
} }

View file

@ -1,4 +1,4 @@
use std::{fmt, marker::PhantomData}; use std::marker::PhantomData;
use rowan::{GreenNode, GreenNodeData, GreenTokenData, NodeOrToken}; use rowan::{GreenNode, GreenNodeData, GreenTokenData, NodeOrToken};
@ -41,14 +41,14 @@ fn debug_print_output<SyntaxKind: SyntaxElement, SyntaxErr: SyntaxError>(
match node { match node {
NodeOrToken::Node(n) => { NodeOrToken::Node(n) => {
let kind: SyntaxKind = node.kind().into(); let kind: SyntaxKind = node.kind().into();
if kind != SyntaxKind::SYNTAX_ERROR { if kind == SyntaxKind::SYNTAX_ERROR {
write!(f, "{:?} {{{maybe_newline}", kind)?;
} else {
let err = errs let err = errs
.pop() .pop()
.expect("all error syntax nodes should correspond to an error"); .expect("all error syntax nodes should correspond to an error");
write!(f, "{:?}: {err:?} {{{maybe_newline}", kind)?; write!(f, "{kind:?}: {err:?} {{{maybe_newline}")?;
} else {
write!(f, "{kind:?} {{{maybe_newline}")?;
} }
for c in n.children() { for c in n.children() {
debug_print_output::<SyntaxKind, SyntaxErr>(c, f, lvl + 1, errs)?; debug_print_output::<SyntaxKind, SyntaxErr>(c, f, lvl + 1, errs)?;

View file

@ -1,11 +1,7 @@
use array::array; use array::array;
use enumset::{enum_set, EnumSet}; use enumset::{enum_set, EnumSet};
use lopal_core::parser::ParserBuilder;
use crate::{ use crate::{syntax_error::SyntaxError, syntax_kind::SyntaxKind};
syntax_error::SyntaxError,
syntax_kind::{lex, SyntaxKind},
};
use self::object::object; use self::object::object;

View file

@ -14,7 +14,7 @@ pub fn lex(src: &str) -> Vec<(SyntaxKind, &str)> {
#[derive(enumset::EnumSetType, Debug, Logos, PartialEq, Eq, Clone, Copy, Hash)] #[derive(enumset::EnumSetType, Debug, Logos, PartialEq, Eq, Clone, Copy, Hash)]
#[repr(u16)] #[repr(u16)]
#[enumset(no_super_impls)] #[enumset(no_super_impls)]
#[allow(non_camel_case_types)] #[allow(non_camel_case_types, reason = "convention")]
pub enum SyntaxKind { pub enum SyntaxKind {
OBJECT, OBJECT,
MEMBER, MEMBER,

View file

@ -30,6 +30,7 @@
{ {
default = pkgs.mkShell rec { default = pkgs.mkShell rec {
buildInputs = with pkgs; [ buildInputs = with pkgs; [
cargo-watch
toolchain toolchain
]; ];
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs; LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs;