use crate::{
blean::{self, Blean},
error::{corrupt, Result},
ndjson::{self, Ndjson},
record::{Counts, Record, Sink, Table},
Env, Exporter,
};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use serde_json::Value;
use std::{
cell::OnceCell,
fs::File,
io::{BufWriter, Write},
path::{Path, PathBuf},
};
use xxhash_rust::xxh3::xxh3_128;
pub fn meta(bytes: &[u8]) -> Result<Value> {
let json = if blean::sniff(bytes) {
match blean::entries(bytes)?.next() {
Some(Ok((Record::Meta(m), _))) => m.into_owned(),
Some(Err(e)) => return Err(e),
_ => return Err(corrupt("export does not start with meta")),
}
} else {
let line = bytes.split(|&b| b == b'\n').next().unwrap_or_default();
let v: Value = serde_json::from_slice(line)?;
v.get("meta")
.ok_or_else(|| corrupt("export does not start with meta"))?
.to_string()
};
Ok(serde_json::from_str(&json)?)
}
struct Module {
name: String,
imports: Vec<(String, String)>,
interface: Option<String>,
public: bool,
inside: Vec<String>,
}
fn module(meta: &Value) -> Result<Module> {
let m = meta
.get("module")
.ok_or_else(|| corrupt("not a per-module export"))?;
let s = |v: &Value| v.as_str().map(str::to_owned);
let bad = || corrupt("bad module in meta");
Ok(Module {
name: m.get("name").and_then(s).ok_or_else(bad)?,
imports: m
.get("imports")
.and_then(Value::as_array)
.ok_or_else(bad)?
.iter()
.map(|i| Some((s(i.get("name")?)?, s(i.get("blake3")?)?)))
.collect::<Option<_>>()
.ok_or_else(bad)?,
interface: m.get("interface").and_then(s),
public: m.get("public").and_then(Value::as_bool) == Some(true),
inside: (m
.get("inside")
.and_then(Value::as_array)
.into_iter()
.flatten())
.filter_map(s)
.collect(),
})
}
#[must_use]
pub fn hash(bytes: &[u8]) -> String {
blake3::hash(bytes).to_hex().to_string()
}
pub fn order(files: &[&[u8]]) -> Result<Vec<usize>> {
plan(files, None)
}
pub fn view(files: &[&[u8]], root: &str) -> Result<Vec<usize>> {
plan(files, Some(root))
}
fn plan(files: &[&[u8]], root: Option<&str>) -> Result<Vec<usize>> {
let mods = files
.par_iter()
.map(|b| module(&meta(b)?))
.collect::<Result<Vec<_>>>()?;
let mut by_name: FxHashMap<&str, Vec<usize>> = FxHashMap::default();
for (i, m) in mods.iter().enumerate() {
let given = by_name.entry(m.name.as_str()).or_default();
if root.is_none() && !given.is_empty() {
return Err(corrupt(format!("module {} given twice", m.name)));
}
given.push(i);
}
let hashes: Vec<OnceCell<String>> = vec![OnceCell::new(); files.len()];
let hashed = |j: usize| hashes[j].get_or_init(|| hash(files[j]));
let start = root
.map(|r| {
(by_name.get(r).into_iter().flatten().copied())
.find(|&j| !mods[j].public)
.ok_or_else(|| corrupt(format!("module {r} is not given")))
})
.transpose()?;
let whole = start.is_none_or(|j| mods[j].interface.is_none());
let imports = |m: &Module| -> Result<Vec<usize>> {
let mut d = Vec::new();
for (i, h) in &m.imports {
let given = by_name
.get(i.as_str())
.ok_or_else(|| corrupt(format!("{} imports {i}, which is not given", m.name)))?;
let of = |j: usize| mods[j].interface.as_ref() == Some(h);
let found = (given.iter().copied())
.filter(|&j| hashed(j) == h || whole && of(j))
.min_by_key(|&j| mods[j].public);
d.push(found.ok_or_else(|| {
corrupt(if given.iter().any(|&j| of(j)) {
format!(
"{} imports the interface of {i}, which is not given",
m.name
)
} else {
format!("{} was exported against another {i}", m.name)
})
})?);
}
Ok(d)
};
let mut deps = vec![Vec::new(); files.len()];
let mut seen = vec![false; files.len()];
let mut stack = start.map_or_else(|| (0..files.len()).collect(), |s| vec![s]);
while let Some(i) = stack.pop() {
if !std::mem::replace(&mut seen[i], true) {
deps[i] = imports(&mods[i])?;
stack.extend(&deps[i]);
}
}
if start.is_some() {
let reached: FxHashMap<&str, usize> = (0..files.len())
.filter(|&i| seen[i] && !mods[i].public)
.map(|i| (mods[i].name.as_str(), i))
.collect();
for d in deps.iter_mut().flatten() {
*d = *reached.get(mods[*d].name.as_str()).unwrap_or(d);
}
for (i, m) in mods.iter().enumerate() {
deps[i].extend(m.inside.iter().filter_map(|o| reached.get(o.as_str())));
}
}
let starts = start.map_or_else(|| (0..files.len()).collect(), |s| vec![s]);
let mut out = Vec::with_capacity(files.len());
let mut state = vec![0u8; files.len()];
for root in starts {
let mut stack = vec![(root, 0)];
while let Some((i, k)) = stack.pop() {
if k == 0 {
match state[i] {
2 => continue,
1 => return Err(corrupt(format!("import cycle at {}", mods[i].name))),
_ => state[i] = 1,
}
}
if let Some(&j) = deps[i].get(k) {
stack.push((i, k + 1));
if state[j] != 2 {
stack.push((j, 0));
}
} else {
state[i] = 2;
out.push(i);
}
}
}
Ok(out)
}
struct Shift<'s, S> {
sink: &'s mut S,
base: Counts,
next: Counts,
names: &'s mut Vec<u128>,
declared: &'s mut FxHashSet<u128>,
}
impl<S: Sink> Shift<'_, S> {
fn name(&mut self, pre: u32, tag: u8, last: &[u8]) {
self.names.push(named(self.names[pre as usize], tag, last));
}
const fn id(&self, t: Table, i: u32) -> u32 {
match t {
Table::Names if i > 0 => self.base.names + i - 1,
Table::Levels if i > 0 => self.base.levels + i - 1,
Table::Exprs => self.base.exprs + i,
_ => i,
}
}
}
impl<S: Sink> Sink for &mut Shift<'_, S> {
type Output = ();
fn record(&mut self, id: Option<u32>, r: &Record<'_>) -> Result<()> {
match r {
Record::Meta(_) => Ok(()),
Record::End(c) => {
self.next = Counts {
names: self.base.names + c.names - 1,
levels: self.base.levels + c.levels - 1,
exprs: self.base.exprs + c.exprs,
};
Ok(())
}
r => {
let mut r = r.clone();
r.map_ids(&|t, i| self.id(t, i));
let id = match (id, r.table()) {
(Some(i), Some(t)) => Some(self.id(t, i)),
(id, _) => id,
};
match &r {
Record::NameStr { pre, str } => self.name(*pre, 0, str.as_bytes()),
Record::NameNum { pre, i } => self.name(*pre, 1, &i.to_le_bytes()),
Record::Decl(d) => {
let n = d.name().and_then(|n| self.names.get(n as usize));
if n.is_some_and(|&n| !self.declared.insert(n)) {
return Ok(());
}
}
_ => {}
}
self.sink.record(id, &r)
}
}
}
fn finish(self) -> Result<()> {
Ok(())
}
}
pub fn join<S: Sink>(files: &[&[u8]], sink: S) -> Result<S::Output> {
join_in(files, &order(files)?, None, sink)
}
pub fn join_view<S: Sink>(files: &[&[u8]], root: &str, sink: S) -> Result<S::Output> {
join_in(files, &view(files, root)?, Some(root), sink)
}
fn named(pre: u128, tag: u8, last: &[u8]) -> u128 {
let mut b = pre.to_le_bytes().to_vec();
b.push(tag);
b.extend_from_slice(last);
xxh3_128(&b)
}
struct Declared {
names: Vec<u128>,
decls: Vec<Option<u128>>,
}
impl Sink for Declared {
type Output = Vec<Option<u128>>;
fn record(&mut self, _: Option<u32>, r: &Record<'_>) -> Result<()> {
match r {
Record::NameStr { pre, str } => {
let h = named(self.names[*pre as usize], 0, str.as_bytes());
self.names.push(h);
}
Record::NameNum { pre, i } => {
let h = named(self.names[*pre as usize], 1, &i.to_le_bytes());
self.names.push(h);
}
Record::Decl(d) => {
let n = d.name().and_then(|n| self.names.get(n as usize));
self.decls.push(n.copied());
}
_ => {}
}
Ok(())
}
fn finish(self) -> Result<Self::Output> {
Ok(self.decls)
}
}
fn count(files: &[&[u8]], order: &[usize]) -> Result<u64> {
let declared = order
.par_iter()
.map(|&i| {
let names = vec![0];
read(
files[i],
Declared {
names,
decls: Vec::new(),
},
)
})
.collect::<Result<Vec<_>>>()?;
let mut seen = FxHashSet::default();
Ok(declared
.iter()
.flatten()
.filter(|n| n.is_none_or(|n| seen.insert(n)))
.count() as u64)
}
fn read<S: Sink>(bytes: &[u8], sink: S) -> Result<S::Output> {
if blean::sniff(bytes) {
blean::read(bytes, sink)
} else {
ndjson::read(bytes, sink)
}
}
fn pour<S: Sink>(files: &[&[u8]], order: &[usize], sink: &mut S) -> Result<Counts> {
let mut next = Counts::default();
let mut names = vec![0];
let mut declared = FxHashSet::default();
for &i in order {
let mut shift = Shift {
sink: &mut *sink,
base: next,
next,
names: &mut names,
declared: &mut declared,
};
read(files[i], &mut shift)?;
next = shift.next;
}
Ok(next)
}
pub fn join_in<S: Sink>(
files: &[&[u8]],
order: &[usize],
root: Option<&str>,
mut sink: S,
) -> Result<S::Output> {
let Some((_, imports)) = order.split_last() else {
return Err(corrupt("nothing to join"));
};
let mut m = meta(files[order[0]])?;
if let Some(o) = m.as_object_mut() {
o.remove("module");
if let Some(root) = root {
let n = count(files, imports)?;
o.insert(
"view".into(),
serde_json::json!({"module": root, "imported": n}),
);
}
}
sink.record(None, &Record::Meta(m.to_string().into()))?;
let next = pour(files, order, &mut sink)?;
sink.record(None, &Record::End(next))?;
sink.finish()
}
struct Hashing<W> {
inner: W,
hash: blake3::Hasher,
}
impl<W: Write> Write for Hashing<W> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let n = self.inner.write(buf)?;
self.hash.update(&buf[..n]);
Ok(n)
}
fn flush(&mut self) -> std::io::Result<()> {
self.inner.flush()
}
}
type Out = Hashing<BufWriter<File>>;
pub fn split(env: &Env, dir: &Path, blean: bool) -> Result<Vec<PathBuf>> {
split_each(env, dir, blean, false, &|| {})
}
pub fn split_interfaces(env: &Env, dir: &Path, blean: bool) -> Result<Vec<PathBuf>> {
split_each(env, dir, blean, true, &|| {})
}
pub fn split_each(
env: &Env,
dir: &Path,
blean: bool,
interfaces: bool,
each: &dyn Fn(),
) -> Result<Vec<PathBuf>> {
if blean {
split_with(env, dir, "blean", interfaces, Blean::new, each)
} else {
split_with(env, dir, "ndjson", interfaces, Ndjson::new, each)
}
}
fn split_with<S: Sink<Output = Out>>(
env: &Env,
dir: &Path,
ext: &str,
interfaces: bool,
sink: impl Fn(Out) -> S,
each: &dyn Fn(),
) -> Result<Vec<PathBuf>> {
let public = dir.join("public");
std::fs::create_dir_all(if interfaces { &public } else { dir })?;
let paths: Vec<PathBuf> = env
.modules
.iter()
.map(|m| dir.join(format!("{m}.{ext}")))
.collect();
let files: Vec<(usize, bool, PathBuf)> = (0..paths.len())
.flat_map(|m| {
let shown = (interfaces && env.exported[m].is_some())
.then(|| (m, true, public.join(format!("{}.{ext}", env.modules[m]))));
shown.into_iter().chain([(m, false, paths[m].clone())])
})
.collect();
let open = |p: &Path| -> Result<S> {
Ok(sink(Hashing {
inner: BufWriter::with_capacity(1 << 20, File::create(p)?),
hash: blake3::Hasher::new(),
}))
};
let Some((_, _, first)) = files.first() else {
return Ok(paths);
};
let index: FxHashMap<&str, usize> = env
.modules
.iter()
.enumerate()
.map(|(i, m)| (m.as_str(), i))
.collect();
let mut whole: Vec<String> = Vec::with_capacity(paths.len());
let mut shown: Vec<Option<String>> = vec![None; paths.len()];
let mut ex = if interfaces {
Exporter::restricted(env, open(first)?)
} else {
Exporter::per_module(env, open(first)?)
};
for (k, (m, interface, _)) in files.iter().enumerate() {
let (m, name) = (*m, &env.modules[*m]);
let module = interfaces && env.exported[m].is_some();
let imports = env.imports[m]
.iter()
.filter(|i| !interface || i.exported)
.map(|i| {
let j = (index.get(i.module.as_str()).copied())
.filter(|&j| j < whole.len())
.ok_or_else(|| corrupt(format!("{name} imports unloaded {}", i.module)))?;
let seen = shown[j]
.as_ref()
.filter(|_| module && (*interface || !i.all));
Ok(serde_json::json!({
"name": i.module,
"blake3": seen.unwrap_or(&whole[j]),
"all": i.all,
"public": i.exported,
"meta": i.meta,
}))
})
.collect::<Result<Vec<_>>>()?;
let mut meta = serde_json::json!({"name": name, "imports": imports});
if *interface {
meta["public"] = true.into();
meta["inside"] = ex.inside(m).map(|o| env.modules[o].as_str()).collect();
ex.meta_with(Some(meta))?;
ex.interface(m)?;
} else {
if let Some(h) = &shown[m] {
meta["interface"] = h.as_str().into();
}
ex.meta_with(Some(meta))?;
ex.module(m)?;
}
each();
let Some((_, _, next)) = files.get(k + 1) else {
break;
};
let mut out = ex.split(open(next)?)?;
out.flush()?;
let h = out.hash.finalize().to_hex().to_string();
if *interface {
shown[m] = Some(h);
} else {
whole.push(h);
}
}
ex.finish()?.flush()?;
Ok(paths)
}