lang: current state idk try to parallelize

This commit is contained in:
Schrottkatze 2024-06-10 09:47:48 +02:00
parent 3e2c5946c8
commit 37651a83bc
Signed by: schrottkatze
SSH key fingerprint: SHA256:hXb3t1vINBFCiDCmhRABHX5ocdbLiKyCdKI4HK2Rbbc
11 changed files with 522 additions and 15 deletions

91
Cargo.lock generated
View file

@ -206,6 +206,28 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "crossbeam"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-queue",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
dependencies = [
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-deque" name = "crossbeam-deque"
version = "0.8.4" version = "0.8.4"
@ -228,6 +250,15 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "crossbeam-queue"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
dependencies = [
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.18" version = "0.8.18"
@ -286,6 +317,19 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "dashmap"
version = "5.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
dependencies = [
"cfg-if",
"hashbrown",
"lock_api",
"once_cell",
"parking_lot_core",
]
[[package]] [[package]]
name = "deranged" name = "deranged"
version = "0.3.11" version = "0.3.11"
@ -563,12 +607,15 @@ version = "0.1.0"
dependencies = [ dependencies = [
"ariadne", "ariadne",
"clap", "clap",
"crossbeam",
"dashmap",
"drop_bomb", "drop_bomb",
"ego-tree", "ego-tree",
"enumset", "enumset",
"indexmap", "indexmap",
"indoc", "indoc",
"logos", "logos",
"num_cpus",
"owo-colors", "owo-colors",
"petgraph", "petgraph",
"rowan", "rowan",
@ -588,9 +635,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.152" version = "0.2.155"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
[[package]] [[package]]
name = "libredox" name = "libredox"
@ -600,7 +647,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
dependencies = [ dependencies = [
"bitflags 2.4.1", "bitflags 2.4.1",
"libc", "libc",
"redox_syscall", "redox_syscall 0.4.1",
] ]
[[package]] [[package]]
@ -701,6 +748,16 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi",
"libc",
]
[[package]] [[package]]
name = "num_threads" name = "num_threads"
version = "0.1.6" version = "0.1.6"
@ -710,6 +767,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "once_cell"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]] [[package]]
name = "option-ext" name = "option-ext"
version = "0.2.0" version = "0.2.0"
@ -725,6 +788,19 @@ dependencies = [
"supports-color", "supports-color",
] ]
[[package]]
name = "parking_lot_core"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [
"cfg-if",
"libc",
"redox_syscall 0.5.1",
"smallvec",
"windows-targets 0.52.0",
]
[[package]] [[package]]
name = "petgraph" name = "petgraph"
version = "0.6.4" version = "0.6.4"
@ -877,6 +953,15 @@ dependencies = [
"bitflags 1.3.2", "bitflags 1.3.2",
] ]
[[package]]
name = "redox_syscall"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e"
dependencies = [
"bitflags 2.4.1",
]
[[package]] [[package]]
name = "redox_users" name = "redox_users"
version = "0.4.4" version = "0.4.4"

View file

@ -16,6 +16,9 @@ rowan = "0.15.15"
drop_bomb = "0.1.5" drop_bomb = "0.1.5"
enumset = "1.1.3" enumset = "1.1.3"
indoc = "2" indoc = "2"
num_cpus = "1.16"
dashmap = "5.5.3"
crossbeam = "0.8.4"
owo-colors = {version = "4", features = ["supports-colors"]} owo-colors = {version = "4", features = ["supports-colors"]}
[lints] [lints]

View file

@ -2,6 +2,10 @@
use crate::lst_parser::syntax_kind::SyntaxKind; use crate::lst_parser::syntax_kind::SyntaxKind;
pub mod ast;
pub mod lst_parser;
pub mod world;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Lang {} pub enum Lang {}
impl rowan::Language for Lang { impl rowan::Language for Lang {
@ -19,6 +23,3 @@ impl rowan::Language for Lang {
pub type SyntaxNode = rowan::SyntaxNode<Lang>; pub type SyntaxNode = rowan::SyntaxNode<Lang>;
pub type SyntaxToken = rowan::SyntaxNode<Lang>; pub type SyntaxToken = rowan::SyntaxNode<Lang>;
pub type SyntaxElement = rowan::NodeOrToken<SyntaxNode, SyntaxToken>; pub type SyntaxElement = rowan::NodeOrToken<SyntaxNode, SyntaxToken>;
pub mod ast;
pub mod lst_parser;

View file

@ -15,18 +15,18 @@ struct Args {
fn main() { fn main() {
let args = Args::parse(); let args = Args::parse();
let n = args.file.clone(); let n = args.file.clone();
// let f = fs::read_to_string(n.clone()).expect("failed to read file"); let f = fs::read_to_string(n.clone()).expect("failed to read file");
// let toks = dbg!(syntax_kind::lex(&f)); let toks = dbg!(syntax_kind::lex(&f));
// let input = input::Input::new(&toks); let input = input::Input::new(&toks);
// let mut parser = lst_parser::Parser::new(input); let mut parser = lst_parser::Parser::new(input);
// grammar::source_file(&mut parser); grammar::source_file(&mut parser);
// let p_out = dbg!(parser.finish()); let p_out = dbg!(parser.finish());
// let o = Output::from_parser_output(toks, p_out); let o = Output::from_parser_output(toks, p_out);
// println!("{}", o.debug_colored()); println!("{}", o.debug_colored());
World::new(n); // World::new(n);
} }

47
crates/lang/src/world.rs Normal file
View file

@ -0,0 +1,47 @@
use std::{
fmt::Display,
path::{Path, PathBuf},
};
use crate::world::world_creation_pool::WorldCreationPool;
use self::{files::Files, registry::Registry};
pub mod error;
pub mod files;
pub mod modules;
pub mod nodes;
pub mod registry;
mod world_creation_pool;
pub struct World {}
impl World {
pub fn new(entry_point: &Path) -> Result<Self, WorldCreationError> {
let files = Files::new();
let reg = Registry::new();
let (entry_point_id, errors) = files.add_and_parse(entry_point).map_err(|e| {
WorldCreationError::FailedToOpenEntryPoint(entry_point.to_path_buf(), e)
})?;
let pool = WorldCreationPool::new(files, reg);
todo!()
}
}
#[derive(Debug)]
pub enum WorldCreationError {
FailedToOpenEntryPoint(PathBuf, std::io::Error),
}
impl Display for WorldCreationError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
WorldCreationError::FailedToOpenEntryPoint(entry_path, e) => {
write!(f, "failed to open entry_point '{entry_path:?}': {e}")
}
}
}
}

View file

@ -0,0 +1,17 @@
use std::path::PathBuf;
use crate::lst_parser::error::SyntaxError;
use super::files::FileId;
#[derive(Debug)]
pub enum Error {
Syntax(FileId, SyntaxError),
FailedToOpenFileMod(PathBuf, std::io::Error),
}
impl Error {
pub fn from_syntax(file: FileId, e: SyntaxError) -> Self {
Self::Syntax(file, e)
}
}

View file

@ -0,0 +1,116 @@
use std::{
clone, fs,
path::{Path, PathBuf},
sync::{Arc, Mutex, RwLock},
};
use dashmap::DashMap;
use rowan::ast::{AstNode, AstPtr};
use crate::lst_parser::{
error::SyntaxError, grammar::source_file, input::Input, output::Output, syntax_kind, Parser,
};
use super::{error::Error, modules::Module, nodes};
pub struct Loc<N: AstNode> {
file: FileId,
syntax: AstPtr<N>,
}
#[derive(Clone)]
pub struct Files(Arc<FilesInner>);
impl Files {
pub fn new() -> Self {
Self(Arc::new(FilesInner::new()))
}
pub fn add_and_parse(&self, file: &Path) -> Result<(FileId, Vec<Error>), std::io::Error> {
self.0.add_and_parse(file)
}
pub fn get(&self, id: FileId) -> Arc<SourceFile> {
self.0.get(id)
}
}
/// global file store
/// contains all known files etc.
struct FilesInner {
paths: Mutex<Vec<PathBuf>>,
store: DashMap<PathBuf, Arc<SourceFile>>,
}
impl FilesInner {
fn new() -> Self {
Self {
paths: Mutex::new(Vec::new()),
store: DashMap::new(),
}
}
fn add_and_parse(&self, path: &Path) -> Result<(FileId, Vec<Error>), std::io::Error> {
let (file, errors) = SourceFile::read_and_parse(&path)?;
// add file to paths and unlock again
let id = {
let path = path.clone();
let mut paths = self.paths.lock().unwrap();
let r = paths.len();
paths.push(path.to_path_buf());
FileId(r)
};
let _ = self.store.insert(path.to_path_buf(), Arc::new(file));
Ok((
id,
errors
.into_iter()
.map(|e| Error::from_syntax(id, e))
.collect(),
))
}
fn get(&self, id: FileId) -> Arc<SourceFile> {
let path = {
let paths = self.paths.lock().unwrap();
paths[id.0].clone()
};
self.store.get(&path).unwrap().clone()
}
}
pub struct SourceFile {
pub lst: Mutex<nodes::Root>,
root_module: Option<Arc<Module>>,
}
impl SourceFile {
fn read_and_parse(path: &Path) -> Result<(Self, Vec<SyntaxError>), std::io::Error> {
let source_text = fs::read_to_string(&path)?;
let toks = syntax_kind::lex(&source_text);
let input = Input::new(&toks);
let mut parser = Parser::new(input);
source_file(&mut parser);
let events = parser.finish();
let out = Output::from_parser_output(toks, events);
let lst = out.syntax();
Ok((
Self {
lst: Mutex::new(nodes::Root::cast(lst).unwrap()),
root_module: None,
},
out.errors(),
))
}
}
#[derive(Debug, Clone, Copy)]
pub struct FileId(pub usize);

View file

@ -0,0 +1,38 @@
use std::sync::Arc;
use dashmap::{DashMap, DashSet};
use rowan::ast::AstNode;
use super::{
files::{FileId, Loc},
nodes,
registry::ItemPath,
world_creation_pool::WorkerCtx,
// world_creation_pool::WorkerCtx,
};
pub struct Module {
/// is none if this is the root module of the entry point file
decl: Option<Loc<nodes::Mod>>,
body: ModuleBody,
own_path: ItemPath,
child_modules: Arc<DashMap<String, Arc<Module>>>,
child_defs: Arc<DashMap<String, Def>>,
}
impl Module {
pub fn parse_file(ctx: WorkerCtx, file: FileId, decl: Option<Loc<nodes::Mod>>) {
let f = ctx.files.get(file);
let tree = f.lst.lock().unwrap();
let children = (&*tree).syntax().children();
}
}
struct Def;
enum ModuleBody {
InLine(Loc<nodes::ModBody>),
File(FileId),
}

View file

@ -0,0 +1,77 @@
use crate::lst_parser::syntax_kind::SyntaxKind::*;
use crate::SyntaxNode;
use rowan::Language;
macro_rules! ast_nodes {
($($ast:ident, $kind:ident);+) => {
$(
#[derive(Debug, PartialEq, Eq, Hash)]
#[repr(transparent)]
pub struct $ast(SyntaxNode);
impl rowan::ast::AstNode for $ast {
type Language = crate::Lang;
fn can_cast(kind: <Self::Language as Language>::Kind) -> bool {
kind == $kind
}
fn cast(node: SyntaxNode) -> Option<Self> {
if node.kind() == $kind {
Some(Self(node))
} else {
None
}
}
fn syntax(&self) -> &SyntaxNode {
&self.0
}
}
)+
};
}
ast_nodes!(
Def, DEF;
DefName, DEF_NAME;
DefBody, DEF_BODY;
Mod, MODULE;
ModName, MODULE_NAME;
ModBody, MODULE_BODY;
Use, USE;
UsePat, USE_PAT;
PatItem, PAT_ITEM;
PatGlob, PAT_GLOB;
PatGroup, PAT_GROUP;
Literal, LITERAL;
IntLit, INT_NUM;
FloatLit, FLOAT_NUM;
StringLit, STRING;
Matrix, MATRIX;
MatrixRow, MAT_ROW;
Vector, VEC;
List, LIST;
CollectionItem, COLLECTION_ITEM;
ParenthesizedExpr, PARENTHESIZED_EXPR;
Expression, EXPR;
Pipeline, PIPELINE;
Instruction, INSTR;
InstructionName, INSTR_NAME;
InstructionParams, INSTR_PARAMS;
AttributeSet, ATTR_SET;
Attribute, ATTR;
AttributeName, ATTR_NAME;
AttributeValue, ATTR_VALUE;
ParseError, PARSE_ERR;
LexError, LEX_ERR;
Root, ROOT;
Eof, EOF
);

View file

@ -0,0 +1,41 @@
use std::sync::{Arc, Mutex, RwLock};
use dashmap::{DashMap, DashSet};
use super::modules::Module;
pub type ItemPath = Vec<String>;
#[derive(Clone)]
pub struct Registry(Arc<RegistryInner>);
impl Registry {
pub fn new() -> Self {
Self(Arc::new(RegistryInner::new()))
}
}
struct RegistryInner {
/// ids will refer to paths of modules
paths: RwLock<Vec<ItemPath>>,
modules: DashMap<ItemPath, Arc<Module>>,
names: DashMap<String, DashSet<RegistryId>>,
}
impl RegistryInner {
fn new() -> Self {
Self {
paths: RwLock::new(Vec::new()),
modules: DashMap::new(),
names: DashMap::new(),
}
}
}
pub enum RegistryId {
/// refer to the modlue item itself, the name must be the same as the last item of the path
Module(usize),
/// refer to a def in a module
/// name must refer to name of child of module
Def(usize),
}

View file

@ -0,0 +1,82 @@
use std::{path::PathBuf, thread};
use crossbeam::channel::{Receiver, Sender};
use super::{
error::Error,
files::{FileId, Files, Loc},
nodes,
registry::Registry,
};
pub(super) struct WorldCreationPool {
workers: Vec<Worker>,
tx: Sender<Job>,
}
impl WorldCreationPool {
pub fn new(files: Files, reg: Registry) -> Self {
let mut workers = Vec::new();
let (tx, rx) = crossbeam::channel::unbounded();
for i in 0..num_cpus::get() {
workers.push(Worker::new(
i,
files.clone(),
reg.clone(),
JobSender(tx.clone()),
rx.clone(),
))
}
Self { workers, tx }
}
}
pub struct JobSender(Sender<Job>);
enum Job {
ParseFileMod {
file: FileId,
decl: Option<Loc<nodes::Mod>>,
},
OpenFile(PathBuf),
}
struct Worker {
id: usize,
// thead: thread::JoinHandle<Vec<Error>>,
thread: thread::JoinHandle<()>,
}
pub struct WorkerCtx {
errors: Vec<Error>,
pub files: Files,
pub reg: Registry,
pub tx: JobSender,
}
impl Worker {
fn new(id: usize, files: Files, reg: Registry, sender: JobSender, rx: Receiver<Job>) -> Self {
let ctx = WorkerCtx {
errors: Vec::new(),
files,
reg,
tx: sender,
};
let thread_handle = thread::spawn(move || {
for msg in rx {
match msg {
Job::ParseFileMod { file, decl } => todo!(),
Job::OpenFile(path) => todo!(),
}
}
});
Self {
id,
thread: thread_handle,
}
}
}