feat: graph-ir (continueing #6) #10

Merged
schrottkatze merged 37 commits from schrottkatze/iowo:graph-ir into main 2024-01-23 12:55:01 +00:00
16 changed files with 229 additions and 115 deletions
Showing only changes of commit 3c529c3a1a - Show all commits

3
Cargo.lock generated
View file

@ -63,6 +63,8 @@ dependencies = [
"ariadne", "ariadne",
"clap", "clap",
"dirs", "dirs",
"eval",
"ir",
"owo-colors", "owo-colors",
"ron", "ron",
"serde", "serde",
@ -272,6 +274,7 @@ dependencies = [
"clap", "clap",
"image", "image",
"ir", "ir",
"serde",
] ]
[[package]] [[package]]

View file

@ -6,14 +6,16 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
clap = { workspace = true, features = [ "derive", "env" ] }
serde = { workspace = true, features = [ "derive" ] }
ron = "0.8"
serde_json = "1.0"
ariadne = "0.4" ariadne = "0.4"
time = { version = "0.3", features = [ "local-offset" ] } clap = { workspace = true, features = [ "derive", "env" ] }
dirs = "5" dirs = "5"
eval = { path = "../eval" }
ir = { path = "../ir" }
owo-colors = "4" owo-colors = "4"
ron = "0.8"
serde = { workspace = true, features = [ "derive" ] }
serde_json = "1.0"
time = { version = "0.3", features = [ "local-offset" ] }
[lints] [lints]
workspace = true workspace = true

View file

@ -1,3 +1,5 @@
use std::path::PathBuf;
use clap::Parser; use clap::Parser;
use self::{ use self::{
@ -10,6 +12,9 @@ mod config_file;
/// this struct may hold all configuration /// this struct may hold all configuration
pub struct Config { pub struct Config {
pub source: PathBuf,
pub evaluator: eval::Available,
pub startup_msg: bool, pub startup_msg: bool,
} }
@ -17,37 +22,37 @@ impl Config {
/// Get the configs from all possible places (args, file, env...) /// Get the configs from all possible places (args, file, env...)
pub fn read() -> Self { pub fn read() -> Self {
let args = Args::parse(); let args = Args::parse();
let config_path = if let Some(config_path) = args.config_path { let config = if let Some(config) = args.config_path {
Ok(config_path) Ok(config)
} else { } else {
find_config_file() find_config_file()
}; };
// try to read a maybe existing config file // try to read a maybe existing config file
let file_config = if let Ok(config_path) = config_path { let config = config.ok().and_then(|path| {
let file_config = Configs::read(config_path); Configs::read(path).map_or_else(
|e| {
match file_config {
Ok(c) => Some(c),
Err(e) => {
eprintln!("Config error: {e:?}"); eprintln!("Config error: {e:?}");
eprintln!("Proceeding with defaults or cli args..."); eprintln!("Proceeding with defaults or cli args...");
None None
} },
} Some,
} else { )
None });
};
if let Some(file_config) = file_config { if let Some(file) = config {
Self { Self {
source: args.source,
evaluator: args.evaluator.and(file.evaluator).unwrap_or_default(),
// this is negated because to an outward api, the negative is more intuitive, // this is negated because to an outward api, the negative is more intuitive,
// while in the source the other way around is more intuitive // while in the source the other way around is more intuitive
startup_msg: !(args.no_startup_message || file_config.no_startup_message), startup_msg: !(args.no_startup_message || file.no_startup_message),
} }
} else { } else {
Self { Self {
source: args.source,
startup_msg: !args.no_startup_message, startup_msg: !args.no_startup_message,
evaluator: args.evaluator.unwrap_or_default(),
} }
} }
} }
@ -56,7 +61,7 @@ impl Config {
pub mod error { pub mod error {
/// Errors that can occur when reading configs /// Errors that can occur when reading configs
#[derive(Debug)] #[derive(Debug)]
pub enum ConfigError { pub enum Config {
/// The config dir doesn't exist /// The config dir doesn't exist
NoConfigDir, NoConfigDir,
/// We didn't find a config file in the config dir /// We didn't find a config file in the config dir
@ -73,19 +78,19 @@ pub mod error {
SerdeRonError(ron::error::SpannedError), SerdeRonError(ron::error::SpannedError),
} }
impl From<std::io::Error> for ConfigError { impl From<std::io::Error> for Config {
fn from(value: std::io::Error) -> Self { fn from(value: std::io::Error) -> Self {
Self::IoError(value) Self::IoError(value)
} }
} }
impl From<serde_json::Error> for ConfigError { impl From<serde_json::Error> for Config {
fn from(value: serde_json::Error) -> Self { fn from(value: serde_json::Error) -> Self {
Self::SerdeJsonError(value) Self::SerdeJsonError(value)
} }
} }
impl From<ron::error::SpannedError> for ConfigError { impl From<ron::error::SpannedError> for Config {
fn from(value: ron::error::SpannedError) -> Self { fn from(value: ron::error::SpannedError) -> Self {
Self::SerdeRonError(value) Self::SerdeRonError(value)
} }

View file

@ -4,6 +4,14 @@ use clap::{builder::BoolishValueParser, ArgAction, Parser};
#[derive(Parser)] #[derive(Parser)]
pub(crate) struct Args { pub(crate) struct Args {
/// What file contains the pipeline to evaluate.
pub source: PathBuf,
/// How to actually run the pipeline.
/// Overrides the config file. Defaults to the debug evaluator.
#[arg(short, long)]
pub evaluator: Option<eval::Available>,
/// Read this config file. /// Read this config file.
#[arg(short, long)] #[arg(short, long)]
pub config_path: Option<PathBuf>, pub config_path: Option<PathBuf>,

View file

@ -5,7 +5,7 @@ use std::{
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use super::error::ConfigError; use super::error::Config;
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
pub struct Configs { pub struct Configs {
@ -13,6 +13,7 @@ pub struct Configs {
pub example_value: i32, pub example_value: i32,
#[serde(default)] #[serde(default)]
pub no_startup_message: bool, pub no_startup_message: bool,
pub evaluator: Option<eval::Available>,
} }
/// what the fuck serde why do i need this /// what the fuck serde why do i need this
@ -21,9 +22,9 @@ fn default_example_value() -> i32 {
} }
/// Find the location of a config file and check if there is, in fact, a file /// Find the location of a config file and check if there is, in fact, a file
pub(super) fn find_config_file() -> Result<PathBuf, ConfigError> { pub(super) fn find_config_file() -> Result<PathBuf, Config> {
let Some(config_path) = dirs::config_dir() else { let Some(config_path) = dirs::config_dir() else {
return Err(ConfigError::NoConfigDir); return Err(Config::NoConfigDir);
}; };
let ron_path = config_path.with_file_name("config.ron"); let ron_path = config_path.with_file_name("config.ron");
@ -34,19 +35,19 @@ pub(super) fn find_config_file() -> Result<PathBuf, ConfigError> {
} else if Path::new(&json_path).exists() { } else if Path::new(&json_path).exists() {
Ok(json_path) Ok(json_path)
} else { } else {
Err(ConfigError::NoConfigFileFound) Err(Config::NoConfigFileFound)
} }
} }
impl Configs { impl Configs {
pub fn read(p: PathBuf) -> Result<Self, ConfigError> { pub fn read(p: PathBuf) -> Result<Self, Config> {
match p match p
.extension() .extension()
.map(|v| v.to_str().expect("config path to be UTF-8")) .map(|v| v.to_str().expect("config path to be UTF-8"))
{ {
Some("ron") => Ok(serde_json::from_str(&fs::read_to_string(p)?)?), Some("ron") => Ok(serde_json::from_str(&fs::read_to_string(p)?)?),
Some("json") => Ok(ron::from_str(&fs::read_to_string(p)?)?), Some("json") => Ok(ron::from_str(&fs::read_to_string(p)?)?),
e => Err(ConfigError::UnknownExtension(e.map(str::to_string))), e => Err(Config::UnknownExtension(e.map(str::to_string))),
} }
} }
} }

View file

@ -1,3 +1,5 @@
use std::fs;
use config::Config; use config::Config;
use welcome_msg::print_startup_msg; use welcome_msg::print_startup_msg;
@ -14,4 +16,14 @@ fn main() {
if cfg.startup_msg { if cfg.startup_msg {
print_startup_msg(); print_startup_msg();
} }
let source =
fs::read_to_string(cfg.source).expect("can't find source file lol handle me better please");
let ir =
ir::from_ron(&source).expect("aww failed to parse source to graph ir handle me better");
let mut machine = cfg.evaluator.pick();
machine.feed(ir);
machine.eval_full();
} }

View file

@ -9,6 +9,7 @@ edition = "2021"
clap = { workspace = true, features = [ "derive" ] } clap = { workspace = true, features = [ "derive" ] }
image = "0.24" image = "0.24"
ir = { path = "../ir" } ir = { path = "../ir" }
serde = { workspace = true }
[lints] [lints]
workspace = true workspace = true

View file

@ -1 +0,0 @@
pub(crate) struct CpuExecutor;

View file

@ -1,37 +0,0 @@
use ir::instruction::{Filter, Kind};
use crate::value::Dynamic;
mod instructions;
pub struct Executor;
impl crate::Executor for Executor {
fn execute(instruction: Kind, input: Option<Dynamic>) -> Option<Dynamic> {
match instruction {
Kind::Read(read_instruction) => {
Some(Dynamic::Image(instructions::read::read(read_instruction)))
}
Kind::Write(write_instruction) => {
instructions::write::write(
write_instruction,
match input {
Some(Dynamic::Image(ref img)) => img,
_ => panic!("awawwawwa"),
},
);
None
}
Kind::Math(_) => todo!(),
Kind::Blend(_) => todo!(),
Kind::Noise(_) => todo!(),
Kind::Filter(filter_instruction) => match filter_instruction {
Filter::Invert => Some(Dynamic::Image(instructions::filters::invert::invert(
match input {
Some(Dynamic::Image(img)) => img,
_ => panic!("invalid value type for invert"),
},
))),
},
}
}
}

View file

@ -0,0 +1,57 @@
use std::mem;
use ir::{
instruction::{Filter, Kind},
GraphIr, Instruction, InstructionRef,
};
use crate::value::Dynamic;
mod instr;
#[derive(Debug, Default)]
pub struct Evaluator {
ir: GraphIr,
}
impl crate::Evaluator for Evaluator {
fn feed(&mut self, ir: GraphIr) {
// TODO: should add instead of replace, see note in Evaluator trait above this method
self.ir = ir;
}
fn eval_full(&mut self) {
let queue: Vec<Instruction> = self
.ir
.topological_sort()
.into_iter()
.map(Into::into)
.collect();
for instr in queue {
self.step(instr);
}
}
}
impl Evaluator {
#[allow(clippy::needless_pass_by_value)]
fn step(&mut self, instr: Instruction) {
let _ = match instr.kind {
Kind::Read(details) => Some(Dynamic::Image(instr::read::read(details))),
Kind::Write(details) => {
instr::write::write(details, todo!());
None
}
Kind::Math(_) => todo!(),
Kind::Blend(_) => todo!(),
Kind::Noise(_) => todo!(),
Kind::Filter(filter_instruction) => match filter_instruction {
Filter::Invert => Some(Dynamic::Image(instr::filters::invert::invert(
match todo!() {
Some(Dynamic::Image(img)) => img,
_ => panic!("invalid value type for invert"),
},
))),
},
};
}
}

View file

@ -0,0 +1 @@
pub mod debug;

View file

@ -1,34 +1,43 @@
use ir::instruction::Kind; use ir::GraphIr;
use value::Dynamic;
mod debug; mod kind;
mod value; mod value;
/// The available executors /// Can collapse a [`GraphIr`] in meaningful ways and do interesting work on it.
/// unused in early dev. ///
#[derive(Debug, Clone, Copy, clap::ValueEnum)] /// It's surprisingly difficult to find a fitting description for this.
pub enum RegisteredExecutor { pub trait Evaluator {
/// the debug executor is single threaded and really, *really* slow. And unstable. Don't use. Unless you're a dev working on this. /// Take some [`GraphIr`] which will then be processed later.
/// May be called multiple times, in which the [`GraphIr`]s should add up.
// TODO: atm they definitely don't add up -- add some functionality to GraphIr to
// make it combine two graphs into one
fn feed(&mut self, ir: GraphIr);
/// Walk through the _whole_ [`GraphIr`] and run through each instruction.
fn eval_full(&mut self);
// TODO: for an LSP or the like, eval_single which starts at a given instr
}
/// The available [`Evaluator`]s.
///
/// Checklist for adding new ones:
///
/// 1. Create a new module under the [`kind`] module.
/// 2. Add a struct and implement [`Evaluator`] for it.
#[derive(Clone, Copy, Debug, Default, clap::ValueEnum, serde::Deserialize, serde::Serialize)]
pub enum Available {
/// Runs fully on the CPU. Single-threaded, debug-friendly and quick to implement.
#[default]
Debug, Debug,
} }
trait Executor { impl Available {
fn execute(instruction: Kind, input: Option<Dynamic>) -> Option<Dynamic>; /// Selects the [`Evaluator`] corresponding to this label.
} #[must_use]
pub fn pick(&self) -> Box<dyn Evaluator> {
pub fn execute_all(instructions: Vec<Kind>) { match self {
let mut tmp = None; Self::Debug => Box::new(kind::debug::Evaluator::default()),
}
for instruction in instructions {
tmp = debug::Executor::execute(instruction, tmp);
} }
} }
// scratchpad lol:
// execution structure:
// 1. take in rpl
// 2. analyse/validate structure against allowed executors
// 3. assign executors to instructions
// 4. optimize
// 5. prepare memory management patterns
// 6. run

View file

@ -56,7 +56,7 @@ pub fn from_ron(source: &str) -> ron::error::SpannedResult<GraphIr> {
/// So the vertices of the DAG are the **sockets** /// So the vertices of the DAG are the **sockets**
/// (which are either [`id::Input`] or [`id::Output`] depending on the direction), /// (which are either [`id::Input`] or [`id::Output`] depending on the direction),
/// and each **socket** in turn belongs to an **instruction**. /// and each **socket** in turn belongs to an **instruction**.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
pub struct GraphIr { pub struct GraphIr {
/// "Backbone" storage of all **instruction** IDs to /// "Backbone" storage of all **instruction** IDs to
/// what **kind of instruction** they are. /// what **kind of instruction** they are.
@ -123,13 +123,13 @@ impl GraphIr {
/// to actually have multiple [`GraphIr`]s at one point in time. /// to actually have multiple [`GraphIr`]s at one point in time.
/// Open an issue if that poses a problem for you. /// Open an issue if that poses a problem for you.
#[must_use] #[must_use]
pub fn resolve<'ir>(&'ir self, subject: &id::Instruction) -> Option<Instruction<'ir>> { pub fn resolve<'ir>(&'ir self, subject: &id::Instruction) -> Option<InstructionRef<'ir>> {
let (id, kind) = self.instructions.get_key_value(subject)?; let (id, kind) = self.instructions.get_key_value(subject)?;
let input_sources = self.input_sources(subject)?.collect(); let input_sources = self.input_sources(subject)?.collect();
let output_targets = self.output_targets(subject)?.collect(); let output_targets = self.output_targets(subject)?.collect();
Some(Instruction { Some(InstructionRef {
id, id,
kind, kind,
input_sources, input_sources,
@ -141,7 +141,7 @@ impl GraphIr {
/// ///
/// The same caveats as for [`GraphIr::resolve`] apply. /// The same caveats as for [`GraphIr::resolve`] apply.
#[must_use] #[must_use]
pub fn owner_of_input<'ir>(&'ir self, input: &id::Input) -> Option<Instruction<'ir>> { pub fn owner_of_input<'ir>(&'ir self, input: &id::Input) -> Option<InstructionRef<'ir>> {
self.resolve(&input.socket().belongs_to) self.resolve(&input.socket().belongs_to)
} }
@ -149,7 +149,7 @@ impl GraphIr {
/// ///
/// The same caveats as for [`GraphIr::resolve`] apply. /// The same caveats as for [`GraphIr::resolve`] apply.
#[must_use] #[must_use]
pub fn owner_of_output<'ir>(&'ir self, output: &id::Output) -> Option<Instruction<'ir>> { pub fn owner_of_output<'ir>(&'ir self, output: &id::Output) -> Option<InstructionRef<'ir>> {
self.resolve(&output.socket().belongs_to) self.resolve(&output.socket().belongs_to)
} }
@ -163,7 +163,7 @@ impl GraphIr {
#[must_use] #[must_use]
// yes, this function could probably return an iterator and be lazy // yes, this function could probably return an iterator and be lazy
// no, not today // no, not today
pub fn topological_sort(&self) -> Vec<Instruction> { pub fn topological_sort(&self) -> Vec<InstructionRef> {
// count how many incoming edges each vertex has // count how many incoming edges each vertex has
let mut nonzero_input_counts: Map<_, NonZeroUsize> = let mut nonzero_input_counts: Map<_, NonZeroUsize> =
self.rev_edges self.rev_edges
@ -240,18 +240,58 @@ impl GraphIr {
} }
} }
/// A full instruction in context, with its inputs and outputs. /// Constructs an [`id::Socket`] a bit more tersely.
fn socket(id: &id::Instruction, idx: u16) -> id::Socket {
id::Socket {
belongs_to: id.clone(),
idx: id::SocketIdx(idx),
}
}
/// A full instruction bundeled in context, with its inputs and outputs.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct Instruction<'ir> { pub struct Instruction {
pub id: id::Instruction,
pub kind: instruction::Kind,
// can't have these two public since then a user might corrupt their length
input_sources: Vec<Option<id::Output>>,
output_targets: Vec<Set<id::Input>>,
}
impl Instruction {
/// Where this instruction gets its inputs from.
///
/// [`None`] means that this input is unfilled,
/// and must be filled before the instruction can be ran.
#[must_use]
pub fn input_sources(&self) -> &[Option<id::Output>] {
&self.input_sources
}
/// To whom outputs are sent.
#[must_use]
pub fn output_targets(&self) -> &[Set<id::Input>] {
&self.output_targets
}
}
/// [`Instruction`], but every single field is borrowed instead.
/// See its docs.
///
/// Use the [`From`] impl to handily convert into an [`Instruction`].
/// The other way around is unlikely to be wanted — since you already have an [`Instruction`],
/// chances are you just want to take a reference (`&`) of it.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct InstructionRef<'ir> {
pub id: &'ir id::Instruction, pub id: &'ir id::Instruction,
pub kind: &'ir instruction::Kind, pub kind: &'ir instruction::Kind,
// can't have these two public since then a user might corrupt their length
input_sources: Vec<Option<&'ir id::Output>>, input_sources: Vec<Option<&'ir id::Output>>,
output_targets: Vec<Option<&'ir Set<id::Input>>>, output_targets: Vec<Option<&'ir Set<id::Input>>>,
} }
impl<'ir> Instruction<'ir> { impl<'ir> InstructionRef<'ir> {
/// Where this instruction gets its inputs from. /// Where this instruction gets its inputs from.
/// ///
/// [`None`] means that this input is unfilled, /// [`None`] means that this input is unfilled,
@ -268,10 +308,23 @@ impl<'ir> Instruction<'ir> {
} }
} }
/// Constructs an [`id::Socket`] a bit more tersely. // would love to use ToOwned but Rust has no specialization yet
fn socket(id: &id::Instruction, idx: u16) -> id::Socket { // and it'd hurt a blanket impl of ToOwned otherwise
id::Socket { impl From<InstructionRef<'_>> for Instruction {
belongs_to: id.clone(), fn from(source: InstructionRef<'_>) -> Self {
idx: id::SocketIdx(idx), Self {
id: source.id.clone(),
kind: source.kind.clone(),
input_sources: source
.input_sources
.into_iter()
.map(Option::<&_>::cloned)
.collect(),
output_targets: source
.output_targets
.into_iter()
.map(|outputs| outputs.map(Clone::clone).unwrap_or_default())
.collect(),
}
} }
} }

View file

@ -7,7 +7,7 @@
subtitle: [don't worry, we're just dreaming], subtitle: [don't worry, we're just dreaming],
) )
= Evaluation stages = Processing stages
#graphics.stages-overview #graphics.stages-overview
@ -18,7 +18,7 @@ This has a number of benefits and implications:
- Bugs are easier to trace down to one stage. - Bugs are easier to trace down to one stage.
- Stages are also replacable, pluggable and usable somewhere else. - Stages are also replacable, pluggable and usable somewhere else.
- For example, - For example,
one could write a Just-In-Time compiler as a new executor to replace the runtime stage, one could write a Just-In-Time compiler as a new evaluator to replace the runtime stage,
while preserving the source #arrow() graph IR step. while preserving the source #arrow() graph IR step.
However, this also makes the architecture somewhat more complicated. So here we try our best to describe how each stage looks like. If you have any feedback, feel free to drop it on #link("https://forge.katzen.cafe/katzen-cafe/iowo/issues")[the issues in the repository]! However, this also makes the architecture somewhat more complicated. So here we try our best to describe how each stage looks like. If you have any feedback, feel free to drop it on #link("https://forge.katzen.cafe/katzen-cafe/iowo/issues")[the issues in the repository]!
@ -155,12 +155,12 @@ Runs through all functions in the graph IR.
It does not have any significantly other representation, It does not have any significantly other representation,
and despite its name there's _no_ bytecode involved. and despite its name there's _no_ bytecode involved.
Different runtimes are called executors. Different runtimes are called evaluators.
Executors operate on instructions instead of functions. Evaluators operate on instructions instead of functions.
=== Scheduler === Scheduler
Looks at the graph IR and decides when the VM should execute what. Looks at the graph IR and decides when the VM should evaluate what.
=== VM <vm> === VM <vm>

View file

@ -37,7 +37,7 @@
"AST", "AST",
"graph IR", "graph IR",
"runtime", "runtime",
"executor", "evaluator",
"optimizer", "optimizer",
"scheduler", "scheduler",