lang: current state idk try to parallelize

Cargo.lock

@@ -206,6 +206,28 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-epoch",
+ "crossbeam-queue",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-deque"
 version = "0.8.4"
@@ -228,6 +250,15 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "crossbeam-queue"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.18"
@@ -286,6 +317,19 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "dashmap"
+version = "5.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
+dependencies = [
+ "cfg-if",
+ "hashbrown",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core",
+]
+
 [[package]]
 name = "deranged"
 version = "0.3.11"
@@ -563,12 +607,15 @@ version = "0.1.0"
 dependencies = [
  "ariadne",
  "clap",
+ "crossbeam",
+ "dashmap",
  "drop_bomb",
  "ego-tree",
  "enumset",
  "indexmap",
  "indoc",
  "logos",
+ "num_cpus",
  "owo-colors",
  "petgraph",
  "rowan",
@@ -588,9 +635,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
 
 [[package]]
 name = "libc"
-version = "0.2.152"
+version = "0.2.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
 
 [[package]]
 name = "libredox"
@@ -600,7 +647,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
 dependencies = [
  "bitflags 2.4.1",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.4.1",
 ]
 
 [[package]]
@@ -701,6 +748,16 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "num_threads"
 version = "0.1.6"
@@ -710,6 +767,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "once_cell"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+
 [[package]]
 name = "option-ext"
 version = "0.2.0"
@@ -725,6 +788,19 @@ dependencies = [
  "supports-color",
 ]
 
+[[package]]
+name = "parking_lot_core"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall 0.5.1",
+ "smallvec",
+ "windows-targets 0.52.0",
+]
+
 [[package]]
 name = "petgraph"
 version = "0.6.4"
@@ -877,6 +953,15 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e"
+dependencies = [
+ "bitflags 2.4.1",
+]
+
 [[package]]
 name = "redox_users"
 version = "0.4.4"

crates/lang/Cargo.toml

@@ -16,6 +16,9 @@ rowan = "0.15.15"
 drop_bomb = "0.1.5"
 enumset = "1.1.3"
 indoc = "2"
+num_cpus = "1.16"
+dashmap = "5.5.3"
+crossbeam = "0.8.4"
 owo-colors = {version = "4", features = ["supports-colors"]}
 
 [lints]

crates/lang/src/lib.rs

@@ -2,6 +2,10 @@
 
 use crate::lst_parser::syntax_kind::SyntaxKind;
 
+pub mod ast;
+pub mod lst_parser;
+pub mod world;
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Lang {}
 impl rowan::Language for Lang {
@@ -19,6 +23,3 @@ impl rowan::Language for Lang {
 pub type SyntaxNode = rowan::SyntaxNode<Lang>;
 pub type SyntaxToken = rowan::SyntaxNode<Lang>;
 pub type SyntaxElement = rowan::NodeOrToken<SyntaxNode, SyntaxToken>;
-
-pub mod ast;
-pub mod lst_parser;

crates/lang/src/main.rs

@@ -15,18 +15,18 @@ struct Args {
 fn main() {
     let args = Args::parse();
     let n = args.file.clone();
-    // let f = fs::read_to_string(n.clone()).expect("failed to read file");
+    let f = fs::read_to_string(n.clone()).expect("failed to read file");
 
-    // let toks = dbg!(syntax_kind::lex(&f));
+    let toks = dbg!(syntax_kind::lex(&f));
-    // let input = input::Input::new(&toks);
+    let input = input::Input::new(&toks);
-    // let mut parser = lst_parser::Parser::new(input);
+    let mut parser = lst_parser::Parser::new(input);
 
-    // grammar::source_file(&mut parser);
+    grammar::source_file(&mut parser);
 
-    // let p_out = dbg!(parser.finish());
+    let p_out = dbg!(parser.finish());
-    // let o = Output::from_parser_output(toks, p_out);
+    let o = Output::from_parser_output(toks, p_out);
 
-    // println!("{}", o.debug_colored());
+    println!("{}", o.debug_colored());
 
-    World::new(n);
+    // World::new(n);
 }

crates/lang/src/world.rs

@@ -0,0 +1,47 @@
+use std::{
+    fmt::Display,
+    path::{Path, PathBuf},
+};
+
+use crate::world::world_creation_pool::WorldCreationPool;
+
+use self::{files::Files, registry::Registry};
+
+pub mod error;
+pub mod files;
+pub mod modules;
+pub mod nodes;
+pub mod registry;
+mod world_creation_pool;
+
+pub struct World {}
+
+impl World {
+    pub fn new(entry_point: &Path) -> Result<Self, WorldCreationError> {
+        let files = Files::new();
+        let reg = Registry::new();
+
+        let (entry_point_id, errors) = files.add_and_parse(entry_point).map_err(|e| {
+            WorldCreationError::FailedToOpenEntryPoint(entry_point.to_path_buf(), e)
+        })?;
+
+        let pool = WorldCreationPool::new(files, reg);
+
+        todo!()
+    }
+}
+
+#[derive(Debug)]
+pub enum WorldCreationError {
+    FailedToOpenEntryPoint(PathBuf, std::io::Error),
+}
+
+impl Display for WorldCreationError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            WorldCreationError::FailedToOpenEntryPoint(entry_path, e) => {
+                write!(f, "failed to open entry_point '{entry_path:?}': {e}")
+            }
+        }
+    }
+}

crates/lang/src/world/error.rs

@@ -0,0 +1,17 @@
+use std::path::PathBuf;
+
+use crate::lst_parser::error::SyntaxError;
+
+use super::files::FileId;
+
+#[derive(Debug)]
+pub enum Error {
+    Syntax(FileId, SyntaxError),
+    FailedToOpenFileMod(PathBuf, std::io::Error),
+}
+
+impl Error {
+    pub fn from_syntax(file: FileId, e: SyntaxError) -> Self {
+        Self::Syntax(file, e)
+    }
+}

crates/lang/src/world/files.rs

@@ -0,0 +1,116 @@
+use std::{
+    clone, fs,
+    path::{Path, PathBuf},
+    sync::{Arc, Mutex, RwLock},
+};
+
+use dashmap::DashMap;
+use rowan::ast::{AstNode, AstPtr};
+
+use crate::lst_parser::{
+    error::SyntaxError, grammar::source_file, input::Input, output::Output, syntax_kind, Parser,
+};
+
+use super::{error::Error, modules::Module, nodes};
+
+pub struct Loc<N: AstNode> {
+    file: FileId,
+    syntax: AstPtr<N>,
+}
+
+#[derive(Clone)]
+pub struct Files(Arc<FilesInner>);
+
+impl Files {
+    pub fn new() -> Self {
+        Self(Arc::new(FilesInner::new()))
+    }
+
+    pub fn add_and_parse(&self, file: &Path) -> Result<(FileId, Vec<Error>), std::io::Error> {
+        self.0.add_and_parse(file)
+    }
+
+    pub fn get(&self, id: FileId) -> Arc<SourceFile> {
+        self.0.get(id)
+    }
+}
+
+/// global file store
+/// contains all known files etc.
+struct FilesInner {
+    paths: Mutex<Vec<PathBuf>>,
+    store: DashMap<PathBuf, Arc<SourceFile>>,
+}
+
+impl FilesInner {
+    fn new() -> Self {
+        Self {
+            paths: Mutex::new(Vec::new()),
+            store: DashMap::new(),
+        }
+    }
+
+    fn add_and_parse(&self, path: &Path) -> Result<(FileId, Vec<Error>), std::io::Error> {
+        let (file, errors) = SourceFile::read_and_parse(&path)?;
+
+        // add file to paths and unlock again
+        let id = {
+            let path = path.clone();
+            let mut paths = self.paths.lock().unwrap();
+            let r = paths.len();
+            paths.push(path.to_path_buf());
+            FileId(r)
+        };
+
+        let _ = self.store.insert(path.to_path_buf(), Arc::new(file));
+
+        Ok((
+            id,
+            errors
+                .into_iter()
+                .map(|e| Error::from_syntax(id, e))
+                .collect(),
+        ))
+    }
+
+    fn get(&self, id: FileId) -> Arc<SourceFile> {
+        let path = {
+            let paths = self.paths.lock().unwrap();
+            paths[id.0].clone()
+        };
+
+        self.store.get(&path).unwrap().clone()
+    }
+}
+
+pub struct SourceFile {
+    pub lst: Mutex<nodes::Root>,
+    root_module: Option<Arc<Module>>,
+}
+
+impl SourceFile {
+    fn read_and_parse(path: &Path) -> Result<(Self, Vec<SyntaxError>), std::io::Error> {
+        let source_text = fs::read_to_string(&path)?;
+
+        let toks = syntax_kind::lex(&source_text);
+        let input = Input::new(&toks);
+        let mut parser = Parser::new(input);
+
+        source_file(&mut parser);
+
+        let events = parser.finish();
+        let out = Output::from_parser_output(toks, events);
+        let lst = out.syntax();
+
+        Ok((
+            Self {
+                lst: Mutex::new(nodes::Root::cast(lst).unwrap()),
+                root_module: None,
+            },
+            out.errors(),
+        ))
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct FileId(pub usize);

crates/lang/src/world/modules.rs

@@ -0,0 +1,38 @@
+use std::sync::Arc;
+
+use dashmap::{DashMap, DashSet};
+use rowan::ast::AstNode;
+
+use super::{
+    files::{FileId, Loc},
+    nodes,
+    registry::ItemPath,
+    world_creation_pool::WorkerCtx,
+    // world_creation_pool::WorkerCtx,
+};
+
+pub struct Module {
+    /// is none if this is the root module of the entry point file
+    decl: Option<Loc<nodes::Mod>>,
+    body: ModuleBody,
+    own_path: ItemPath,
+    child_modules: Arc<DashMap<String, Arc<Module>>>,
+    child_defs: Arc<DashMap<String, Def>>,
+}
+
+impl Module {
+    pub fn parse_file(ctx: WorkerCtx, file: FileId, decl: Option<Loc<nodes::Mod>>) {
+        let f = ctx.files.get(file);
+
+        let tree = f.lst.lock().unwrap();
+
+        let children = (&*tree).syntax().children();
+    }
+}
+
+struct Def;
+
+enum ModuleBody {
+    InLine(Loc<nodes::ModBody>),
+    File(FileId),
+}

crates/lang/src/world/nodes.rs

@@ -0,0 +1,77 @@
+use crate::lst_parser::syntax_kind::SyntaxKind::*;
+use crate::SyntaxNode;
+use rowan::Language;
+macro_rules! ast_nodes {
+($($ast:ident, $kind:ident);+) => {
+    $(
+        #[derive(Debug, PartialEq, Eq, Hash)]
+        #[repr(transparent)]
+        pub struct $ast(SyntaxNode);
+        impl rowan::ast::AstNode for $ast {
+            type Language = crate::Lang;
+
+            fn can_cast(kind: <Self::Language as Language>::Kind) -> bool {
+                kind == $kind
+            }
+
+            fn cast(node: SyntaxNode) -> Option<Self> {
+                if node.kind() == $kind {
+                    Some(Self(node))
+                } else {
+                    None
+                }
+            }
+
+            fn syntax(&self) -> &SyntaxNode {
+                &self.0
+            }
+        }
+    )+
+};
+}
+ast_nodes!(
+    Def, DEF;
+    DefName, DEF_NAME;
+    DefBody, DEF_BODY;
+
+    Mod, MODULE;
+    ModName, MODULE_NAME;
+    ModBody, MODULE_BODY;
+
+    Use, USE;
+    UsePat, USE_PAT;
+    PatItem, PAT_ITEM;
+    PatGlob, PAT_GLOB;
+    PatGroup, PAT_GROUP;
+
+    Literal, LITERAL;
+    IntLit, INT_NUM;
+    FloatLit, FLOAT_NUM;
+    StringLit, STRING;
+
+    Matrix, MATRIX;
+    MatrixRow, MAT_ROW;
+    Vector, VEC;
+    List, LIST;
+    CollectionItem, COLLECTION_ITEM;
+
+    ParenthesizedExpr, PARENTHESIZED_EXPR;
+    Expression, EXPR;
+
+    Pipeline, PIPELINE;
+
+    Instruction, INSTR;
+    InstructionName, INSTR_NAME;
+    InstructionParams, INSTR_PARAMS;
+
+    AttributeSet, ATTR_SET;
+    Attribute, ATTR;
+    AttributeName, ATTR_NAME;
+    AttributeValue, ATTR_VALUE;
+
+    ParseError, PARSE_ERR;
+    LexError, LEX_ERR;
+
+    Root, ROOT;
+    Eof, EOF
+);

crates/lang/src/world/registry.rs

@@ -0,0 +1,41 @@
+use std::sync::{Arc, Mutex, RwLock};
+
+use dashmap::{DashMap, DashSet};
+
+use super::modules::Module;
+
+pub type ItemPath = Vec<String>;
+
+#[derive(Clone)]
+pub struct Registry(Arc<RegistryInner>);
+
+impl Registry {
+    pub fn new() -> Self {
+        Self(Arc::new(RegistryInner::new()))
+    }
+}
+
+struct RegistryInner {
+    /// ids will refer to paths of modules
+    paths: RwLock<Vec<ItemPath>>,
+    modules: DashMap<ItemPath, Arc<Module>>,
+    names: DashMap<String, DashSet<RegistryId>>,
+}
+
+impl RegistryInner {
+    fn new() -> Self {
+        Self {
+            paths: RwLock::new(Vec::new()),
+            modules: DashMap::new(),
+            names: DashMap::new(),
+        }
+    }
+}
+
+pub enum RegistryId {
+    /// refer to the modlue item itself, the name must be the same as the last item of the path
+    Module(usize),
+    /// refer to a def in a module
+    /// name must refer to name of child of module
+    Def(usize),
+}

crates/lang/src/world/world_creation_pool.rs

@@ -0,0 +1,82 @@
+use std::{path::PathBuf, thread};
+
+use crossbeam::channel::{Receiver, Sender};
+
+use super::{
+    error::Error,
+    files::{FileId, Files, Loc},
+    nodes,
+    registry::Registry,
+};
+
+pub(super) struct WorldCreationPool {
+    workers: Vec<Worker>,
+    tx: Sender<Job>,
+}
+
+impl WorldCreationPool {
+    pub fn new(files: Files, reg: Registry) -> Self {
+        let mut workers = Vec::new();
+        let (tx, rx) = crossbeam::channel::unbounded();
+
+        for i in 0..num_cpus::get() {
+            workers.push(Worker::new(
+                i,
+                files.clone(),
+                reg.clone(),
+                JobSender(tx.clone()),
+                rx.clone(),
+            ))
+        }
+
+        Self { workers, tx }
+    }
+}
+
+pub struct JobSender(Sender<Job>);
+
+enum Job {
+    ParseFileMod {
+        file: FileId,
+        decl: Option<Loc<nodes::Mod>>,
+    },
+    OpenFile(PathBuf),
+}
+
+struct Worker {
+    id: usize,
+    // thead: thread::JoinHandle<Vec<Error>>,
+    thread: thread::JoinHandle<()>,
+}
+
+pub struct WorkerCtx {
+    errors: Vec<Error>,
+    pub files: Files,
+    pub reg: Registry,
+    pub tx: JobSender,
+}
+
+impl Worker {
+    fn new(id: usize, files: Files, reg: Registry, sender: JobSender, rx: Receiver<Job>) -> Self {
+        let ctx = WorkerCtx {
+            errors: Vec::new(),
+            files,
+            reg,
+            tx: sender,
+        };
+
+        let thread_handle = thread::spawn(move || {
+            for msg in rx {
+                match msg {
+                    Job::ParseFileMod { file, decl } => todo!(),
+                    Job::OpenFile(path) => todo!(),
+                }
+            }
+        });
+
+        Self {
+            id,
+            thread: thread_handle,
+        }
+    }
+}