#[cfg(test)]
use crate::storage::FactRead;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fmt::Write as _;
#[cfg(test)]
use std::fs;
use std::path::PathBuf;
use std::sync::OnceLock;
use crate::clock;
use crate::collection_names::open_configured_acquiring;
#[cfg(test)]
use crate::collection_names::open_configured;
#[cfg(feature = "local-embed")]
use crate::schemas::embeddings::{self, Embedding768};
use crate::schemas::files::DEFAULT_SCOPE_ID as FILES_SCOPE_ID;
use crate::schemas::wiki::{self as schema, extract_link_targets};
use crate::storage::{read, AcquiringReader, FactArchive, FacultySnapshot, FacultyStore, Storage};
use crate::wiki::{
self as wiki_model, EntryRecord, FrontierModel, LinkClass, LinkReference, RevisionDraft,
RevisionRecord,
};
use anyhow::{anyhow, bail, Context, Result};
#[cfg(test)]
use hifitime::Epoch;
use triblespace::core::blob::encodings::succinctarchive::{
Rank9AcceleratedSuccinctArchiveBlob, SuccinctArchiveBlob,
};
use triblespace::core::collection::latest::LatestIndex;
use triblespace::core::collection::{CollectionCommit, CollectionSnapshotExt, CollectionStoreExt};
use triblespace::core::metadata;
use triblespace::core::query::TriblePattern;
use triblespace::core::repo::pile::PileSnapshot;
use triblespace::core::repo::SnapshotSource;
use triblespace::prelude::*;
#[cfg(feature = "local-embed")]
const EMBEDDINGS_SCOPE_ID: Id = triblespace::macros::id_hex!("F6BE4C16A56001FEA03A5927C6ED3814");
use crate::out::Out;
#[derive(Clone, Debug)]
pub struct Wiki {
storage: Storage,
}
#[derive(Clone, Debug)]
pub struct Export {
pub revision: Id,
pub bytes: anybytes::Bytes,
}
impl Export {
pub fn uri(&self) -> String {
format!("wiki:{:x}", self.revision)
}
}
#[derive(Clone, Debug, Default)]
pub struct ListOptions {
pub tags: Vec<String>,
pub with_backlink_tag: Vec<String>,
pub without_backlink_tag: Vec<String>,
pub with_backlink_type: Vec<String>,
pub without_backlink_type: Vec<String>,
pub all: bool,
}
#[derive(Clone, Debug)]
pub struct ImportDocument {
pub title: String,
pub content: String,
}
impl Wiki {
pub fn new(pile: PathBuf, key: Option<PathBuf>) -> Self {
Self::with_storage(Storage::new(pile, key))
}
pub fn with_storage(storage: Storage) -> Self {
Self { storage }
}
fn storage(&self) -> WikiStorage<'_> {
WikiStorage {
storage: &self.storage,
}
}
pub fn create(&self, title: &str, content: &str, tags: &[String], force: bool) -> Result<Id> {
cmd_create(
self.storage(),
title.into(),
content.into(),
tags.to_vec(),
force,
)
}
pub fn edit(
&self,
id: &str,
content: Option<&str>,
title: Option<&str>,
tags: &[String],
force: bool,
) -> Result<Id> {
cmd_edit(
self.storage(),
id.into(),
content.map(str::to_owned),
title.map(str::to_owned),
tags.to_vec(),
force,
)
}
pub fn show(&self, id: &str, exact: bool) -> Result<String> {
cmd_show(self.storage(), id.into(), exact)
}
pub fn export(&self, id: &str, exact: bool) -> Result<Export> {
cmd_export(self.storage(), id.into(), exact)
}
pub fn diff(&self, id: &str, from: Option<usize>, to: Option<usize>) -> Result<String> {
cmd_diff(self.storage(), id.into(), from, to)
}
pub fn archive(&self, id: &str) -> Result<Option<Id>> {
self.tag(id, "archived", true)
}
pub fn restore(&self, id: &str) -> Result<Option<Id>> {
self.tag(id, "archived", false)
}
pub fn revert(&self, id: &str, to: usize) -> Result<Id> {
cmd_revert(self.storage(), id.into(), to)
}
pub fn links(
&self,
id: Option<&str>,
top: usize,
strict: bool,
out: &mut Out<'_>,
) -> Result<()> {
cmd_links(self.storage(), id.map(str::to_owned), top, strict, out)
}
pub fn list(&self, options: &ListOptions) -> Result<String> {
cmd_list(
self.storage(),
options.tags.clone(),
options.with_backlink_tag.clone(),
options.without_backlink_tag.clone(),
options.with_backlink_type.clone(),
options.without_backlink_type.clone(),
options.all,
)
}
pub fn history(&self, id: &str) -> Result<String> {
cmd_history(self.storage(), id.into())
}
pub fn tag(&self, id: &str, name: &str, add: bool) -> Result<Option<Id>> {
mutate_tags(self.storage(), id.into(), name, add)
}
pub fn tags(&self, out: &mut Out<'_>) -> Result<()> {
cmd_tag_list(self.storage(), out)
}
pub fn mint_tag(&self, name: &str) -> Result<Id> {
cmd_tag_mint(self.storage(), name.into())
}
pub fn search(&self, query: &str, context: bool, all: bool) -> Result<String> {
cmd_search(self.storage(), query.into(), context, all)
}
pub fn embed(&self, out: &mut Out<'_>) -> Result<()> {
cmd_embed(self.storage(), out)
}
pub fn similar(&self, query: &str) -> Result<String> {
cmd_similar(self.storage(), query.into())
}
pub fn check(&self, compile: bool, out: &mut Out<'_>) -> Result<()> {
cmd_check(self.storage(), compile, out)
}
pub fn fix_truncated(&self, input: &str, out: &mut Out<'_>) -> Result<()> {
cmd_fix_truncated(self.storage(), input.into(), out)
}
pub fn lint(&self, fix: bool, check: bool, out: &mut Out<'_>) -> Result<()> {
cmd_lint(self.storage(), fix, check, out)
}
pub fn import_texts(&self, documents: Vec<ImportDocument>, tags: &[String]) -> Result<Vec<Id>> {
cmd_import(self.storage(), documents, tags.to_vec())
}
pub fn export_all(&self) -> Result<Vec<(Id, String)>> {
cmd_batch_export(self.storage())
}
pub fn import_revisions(&self, imports: Vec<(Id, String)>) -> Result<()> {
cmd_batch_import(self.storage(), imports)
}
}
#[derive(Clone, Copy)]
struct WikiStorage<'a> {
storage: &'a Storage,
}
#[derive(Clone)]
struct WikiView {
facts: FactArchive,
reader: FacultySnapshot,
latest: LatestIndex,
}
impl WikiStorage<'_> {
fn with_pile<T>(
&self,
f: impl FnOnce(
&mut FacultyStore,
&ed25519_dalek::SigningKey,
&std::sync::Arc<tokio::runtime::Runtime>,
) -> Result<T>,
) -> Result<T> {
self.storage
.with_store(|pile, signer, runtime| f(pile, signer, runtime))
}
fn views<T>(
&self,
scopes: &[(Id, &str)],
prepare: impl FnMut(&WikiView, &[FactArchive]) -> Result<T>,
) -> Result<T> {
self.with_pile(|pile, signer, runtime| {
let source = open_configured_acquiring(
pile, schema::DEFAULT_SCOPE_ID, signer.verifying_key(), runtime,
)?;
let mut auxiliaries = Vec::with_capacity(scopes.len());
for &(scope, label) in scopes {
let collection = open_configured_acquiring(
pile, scope, signer.verifying_key(), runtime,
)?;
auxiliaries.push((collection, label));
}
views_in(pile, source, signer, &auxiliaries, runtime, prepare)
})
}
fn view<T>(&self, mut prepare: impl FnMut(&WikiView) -> Result<T>) -> Result<T> {
self.views(&[], |wiki, _| prepare(wiki))
}
fn view_with_scope<T>(
&self,
scope: Id,
label: &str,
mut prepare: impl FnMut(&WikiView, &FactArchive) -> Result<T>,
) -> Result<T> {
self.views(&[(scope, label)], |wiki, facts| prepare(wiki, &facts[0]))
}
#[cfg(feature = "local-embed")]
fn publish_scope(&self, scope: Id, fragment: Fragment) -> Result<CollectionCommit> {
self.with_pile(|pile, signer, runtime| {
let collection = open_configured_acquiring(
pile, scope, signer.verifying_key(), runtime,
)?;
let commit = pile
.commit(collection, signer, fragment)
.context("publish native collection fragment")?;
runtime
.block_on(crate::storage::ensure_downstream(pile, collection, signer))
.context(
"Wiki auxiliary fragment was committed, but ensuring its derived views failed",
)?;
Ok(commit)
})
}
fn publish(&self, fragment: Fragment) -> Result<CollectionCommit> {
self.with_pile(|pile, signer, runtime| {
let collection = open_configured_acquiring(
pile,
schema::DEFAULT_SCOPE_ID,
signer.verifying_key(),
runtime,
)?;
crate::collection_names::require_command_write_admission_acquiring(
pile, collection, signer, "Wiki", "wiki show", runtime,
)?;
let commit = pile
.commit(collection, signer, fragment)
.context("publish Wiki fragment")?;
runtime
.block_on(async {
let latest = wiki_model::latest_for_source(pile, collection)?;
crate::storage::seed_attached(pile, latest, signer).await?;
crate::storage::ensure_downstream(pile, collection, signer).await?;
Ok::<_, anyhow::Error>(())
})
.context("Wiki fragment was committed, but ensuring its derived views failed")?;
Ok(commit)
})
}
fn author_fragment(&self) -> Result<(Fragment, Id)> {
self.storage
.with_store(|_, signer, _| Ok(wiki_model::author_record(&signer.verifying_key())))
}
}
fn views_in<T>(
pile: &mut FacultyStore,
wiki_source: Collection<blobencodings::SimpleArchive>,
_signer: &ed25519_dalek::SigningKey,
scopes: &[(Collection<blobencodings::SimpleArchive>, &str)],
runtime: &std::sync::Arc<tokio::runtime::Runtime>,
mut prepare: impl FnMut(&WikiView, &[FactArchive]) -> Result<T>,
) -> Result<T> {
let wiki_succinct = pile
.attach::<SuccinctArchiveBlob>(wiki_source, ())
.context("register Wiki Succinct collection")?;
let wiki_rank9 = pile
.attach::<Rank9AcceleratedSuccinctArchiveBlob>(wiki_source, wiki_succinct)
.context("register Wiki Rank9 collection")?;
let latest = wiki_model::latest_for_source(pile, wiki_source)?;
let mut auxiliaries = Vec::with_capacity(scopes.len());
for &(source, label) in scopes {
let succinct = pile
.attach::<SuccinctArchiveBlob>(source, ())
.with_context(|| format!("register {label} Succinct collection"))?;
let rank9 = pile
.attach::<Rank9AcceleratedSuccinctArchiveBlob>(source, succinct)
.with_context(|| format!("register {label} Rank9 collection"))?;
auxiliaries.push((rank9, label));
}
let reader = pile
.snapshot()
.context("freeze Wiki and auxiliary snapshot")?;
let acquiring = AcquiringReader::new(reader.clone(), runtime.clone());
let observed_facts = acquiring
.attached_acquiring(wiki_rank9)
.context("observe Wiki fact collection")?;
let observed_latest = acquiring
.attached_acquiring(latest)
.context("observe Wiki supersession index")?;
let facts = crate::storage::acquire_attached_facts(&observed_facts)
.context("read Wiki fact collection")?;
let latest = crate::storage::require_complete_attached_read(
observed_latest
.read_acquiring::<LatestIndex>()
.context("read Wiki supersession index")?,
)?;
let mut auxiliary_facts = Vec::with_capacity(auxiliaries.len());
for (rank9, label) in &auxiliaries {
auxiliary_facts.push(
crate::storage::acquire_facts(&acquiring, *rank9)
.with_context(|| format!("read {label} fact collection"))?,
);
}
let mut view = WikiView {
facts,
reader,
latest,
};
let snapshot = view.reader.clone();
runtime.block_on(read(pile, &snapshot, |reader| {
view.reader = reader.clone();
prepare(&view, &auxiliary_facts)
}))
}
fn now_interval() -> Result<Inline<inlineencodings::NsTAIInterval>> {
clock::point_now()
}
fn entry_label(entry: &EntryRecord) -> String {
entry
.roots
.first()
.map(|id| format!("{id:x}"))
.unwrap_or_else(|| "<empty>".to_owned())
}
fn all_selectors<P: TriblePattern>(facts: &P) -> BTreeSet<Id> {
wiki_model::revision_ids(facts)
}
fn resolve_prefix<P: TriblePattern>(facts: &P, raw: &str) -> Result<Id> {
let clean = raw.trim().to_ascii_lowercase();
if clean.is_empty() || clean.len() > 32 || !clean.bytes().all(|byte| byte.is_ascii_hexdigit()) {
bail!("invalid Wiki selector '{raw}'");
}
let candidates = all_selectors(facts);
let matches: Vec<Id> = candidates
.into_iter()
.filter(|id| format!("{id:x}").starts_with(&clean))
.collect();
match matches.as_slice() {
[] => bail!("no Wiki id matches '{raw}'"),
[only] => Ok(*only),
many => bail!("ambiguous Wiki id '{raw}' ({} matches)", many.len()),
}
}
fn selector_revisions(
view: &WikiView,
selector: Id,
follow_frontier: bool,
) -> Result<Vec<RevisionRecord>> {
let revisions = wiki_model::revision_records(&view.facts, selector);
if revisions.is_empty() {
bail!("unknown Wiki selector {selector:x}");
}
if follow_frontier {
Ok(wiki_model::entry(&view.facts, &view.latest, selector)
.expect("queryable revision belongs to one entry")
.frontier)
} else {
Ok(revisions)
}
}
fn mutation_entry(view: &WikiView, raw: &str) -> Result<EntryRecord> {
let selector = resolve_prefix(&view.facts, raw)?;
wiki_model::entry(&view.facts, &view.latest, selector)
.ok_or_else(|| anyhow!("unknown Wiki selector {selector:x}"))
}
fn read_string(reader: &PileSnapshot, handle: schema::TextHandle) -> Result<String> {
wiki_model::read_text(reader, handle)
}
fn tag_name<P: TriblePattern>(facts: &P, reader: &PileSnapshot, id: Id) -> Result<String> {
let mut names = BTreeSet::new();
for handle in find!(
handle: schema::TextHandle,
pattern!(facts, [{ id @ metadata::name: ?handle }])
) {
names.insert(read_string(reader, handle)?);
}
Ok(if names.is_empty() {
schema::TAG_SPECS
.iter()
.find_map(|(known, label)| (*known == id).then_some((*label).to_owned()))
.unwrap_or_else(|| format!("{id:x}"))
} else {
names.into_iter().collect::<Vec<_>>().join(" / ")
})
}
fn tag_ids_named<P: TriblePattern>(
facts: &P,
reader: &PileSnapshot,
wanted: &str,
) -> Result<BTreeSet<Id>> {
let wanted = wanted.trim();
let mut ids = BTreeSet::new();
for (id, handle) in find!(
(id: Id, handle: schema::TextHandle),
pattern!(facts, [{ ?id @ metadata::name: ?handle }])
) {
if read_string(reader, handle)?.eq_ignore_ascii_case(wanted) {
ids.insert(id);
}
}
Ok(ids)
}
fn format_tags<P: TriblePattern>(
facts: &P,
reader: &PileSnapshot,
tags: &BTreeSet<Id>,
) -> Result<String> {
let mut names = Vec::new();
for tag in tags {
names.push(tag_name(facts, reader, *tag)?);
}
Ok(if names.is_empty() {
String::new()
} else {
format!(" [{}]", names.join(", "))
})
}
fn resolve_tags<P: TriblePattern>(
facts: &P,
reader: &PileSnapshot,
names: &[String],
fragment: &mut Fragment,
) -> Result<BTreeSet<Id>> {
if names.is_empty() {
return Ok(BTreeSet::new());
}
let mut by_name: BTreeMap<String, BTreeSet<Id>> = BTreeMap::new();
for (id, handle) in find!(
(id: Id, handle: schema::TextHandle),
pattern!(facts, [{ ?id @ metadata::name: ?handle }])
) {
by_name
.entry(read_string(reader, handle)?.to_ascii_lowercase())
.or_default()
.insert(id);
}
let mut out = BTreeSet::new();
for raw in names {
let name = raw.trim().to_ascii_lowercase();
if name.is_empty() {
continue;
}
if let Some(ids) = by_name.get(&name) {
out.extend(ids.iter().copied());
} else {
let (record, id, _) = wiki_model::tag_record(&name)?;
*fragment += record;
by_name.insert(name, BTreeSet::from([id]));
out.insert(id);
}
}
Ok(out)
}
fn agreed<T: Clone + Eq>(
entry: &EntryRecord,
field: impl Fn(&RevisionRecord) -> T,
name: &str,
) -> Result<T> {
let first = entry.frontier.first().expect("entry frontier is non-empty");
let value = field(first);
if entry
.frontier
.iter()
.skip(1)
.any(|head| field(head) != value)
{
bail!("entry frontier disagrees on {name}; supply a complete resolution explicitly");
}
Ok(value)
}
fn stage_revision(
storage: WikiStorage<'_>,
fragment: &mut Fragment,
entry: Option<&EntryRecord>,
title: String,
content: String,
tags: BTreeSet<Id>,
) -> Result<Id> {
let (author_fragment, author) = storage.author_fragment()?;
*fragment += author_fragment;
let predecessors = entry
.map(|entry| entry.frontier.iter().map(|head| head.id).collect())
.unwrap_or_default();
let (record, revision) = wiki_model::revision_record(RevisionDraft {
title,
content,
tags,
predecessors,
author,
authored_at: now_interval()?,
})?;
*fragment += record;
Ok(revision)
}
fn known_link_ids<P: TriblePattern>(facts: &P) -> BTreeSet<Id> {
all_selectors(facts)
}
fn validate_links<P: TriblePattern>(content: &str, facts: &P, allow_dangling: bool) -> Result<()> {
let known = known_link_ids(facts);
let mut failures = Vec::new();
let re = regex::Regex::new(r"wiki:(?:[A-Za-z_][A-Za-z0-9_]*:)?([0-9A-Fa-f]+)").unwrap();
for captures in re.captures_iter(content) {
let token = &captures[1];
if token.len() != 32 {
failures.push(format!("truncated link wiki:{token}"));
} else if let Some(id) = Id::from_hex(token) {
if !known.contains(&id) && !allow_dangling {
failures.push(format!("broken link wiki:{token}"));
}
}
}
if failures.is_empty() {
Ok(())
} else {
bail!("Wiki link validation failed:\n {}", failures.join("\n "))
}
}
struct ReferenceResolver<'a, P: TriblePattern> {
wiki: &'a P,
files: Option<&'a P>,
}
impl<P: TriblePattern> Copy for ReferenceResolver<'_, P> {}
impl<P: TriblePattern> Clone for ReferenceResolver<'_, P> {
fn clone(&self) -> Self {
*self
}
}
impl<P: TriblePattern> ReferenceResolver<'_, P> {
fn expand(&self, scheme: &str, rest: &str) -> Result<String> {
match scheme {
"wiki" => {
let (kind, hex) = split_typed(rest);
Ok(format!("{kind}{:x}", resolve_prefix(self.wiki, hex)?))
}
"files" => {
if rest.contains(':') {
bail!("files references do not have typed targets");
}
let clean = rest.trim().to_ascii_lowercase();
let reference = match self.files {
Some(files) => crate::files::resolve_reference(files, &clean),
None if clean.len() == 32 || clean.len() == 64 => {
crate::files::resolve_reference(&TribleSet::new(), &clean)
}
None => {
bail!("cannot resolve short files selector without the Files collection")
}
}?;
Ok(reference.hex())
}
_ => bail!("unknown reference scheme '{scheme}'"),
}
}
}
fn split_typed(rest: &str) -> (String, &str) {
if let Some((kind, hex)) = rest.split_once(':') {
if !kind.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return (format!("{kind}:"), hex);
}
}
(String::new(), rest)
}
fn lint_fix<P: TriblePattern>(content: &str, resolver: ReferenceResolver<'_, P>) -> String {
let mut output = String::with_capacity(content.len());
let mut fenced = false;
for line in content.lines() {
if line.trim_start().starts_with("```") {
fenced = !fenced;
}
let line = if fenced {
line.to_owned()
} else {
lint_line(line, resolver)
};
output.push_str(&line);
output.push('\n');
}
if !content.ends_with('\n') {
output.pop();
}
output
}
fn regexes() -> &'static LintPatterns {
static PATTERNS: OnceLock<LintPatterns> = OnceLock::new();
PATTERNS.get_or_init(|| LintPatterns {
bold: regex::Regex::new(r"\*\*([^*]+)\*\*").unwrap(),
markdown_links: regex::Regex::new(
r"\[([^\]]+)\]\((wiki|files):((?:[A-Za-z_][A-Za-z0-9_]*:)?[0-9A-Fa-f]+)\)",
)
.unwrap(),
web_links: regex::Regex::new(r"\[([^\]]+)\]\((https?://[^)]+)\)").unwrap(),
wiki_references: regex::Regex::new(r"wiki:((?:[A-Za-z_][A-Za-z0-9_]*:)?[0-9A-Fa-f]+)\b")
.unwrap(),
})
}
struct LintPatterns {
bold: regex::Regex,
markdown_links: regex::Regex,
web_links: regex::Regex,
wiki_references: regex::Regex,
}
fn lint_line<P: TriblePattern>(line: &str, resolver: ReferenceResolver<'_, P>) -> String {
let patterns = regexes();
let line = if let Some(rest) = line.strip_prefix("### ") {
format!("=== {rest}")
} else if let Some(rest) = line.strip_prefix("## ") {
format!("== {rest}")
} else if let Some(rest) = line.strip_prefix("# ") {
format!("= {rest}")
} else {
line.to_owned()
};
let line = patterns.bold.replace_all(&line, "*$1*").to_string();
let line = patterns
.markdown_links
.replace_all(&line, |captures: ®ex::Captures| {
let scheme = &captures[2];
let rest = &captures[3];
let resolved = resolver
.expand(scheme, rest)
.unwrap_or_else(|_| rest.to_ascii_lowercase());
format!("#link(\"{scheme}:{resolved}\")[{}]", &captures[1])
})
.to_string();
let line = patterns
.web_links
.replace_all(&line, "#link(\"$2\")[$1]")
.to_string();
let line = patterns
.wiki_references
.replace_all(&line, |captures: ®ex::Captures| {
let rest = &captures[1];
match resolver.expand("wiki", rest) {
Ok(resolved) if !resolved.eq_ignore_ascii_case(rest) => format!("wiki:{resolved}"),
_ => captures[0].to_owned(),
}
})
.to_string();
if matches!(line.trim(), "---" | "***" | "___") {
String::new()
} else {
line
}
}
fn validate_typst(content: &str) -> Result<()> {
let world = typst_validate::ValidateWorld::new(content);
world
.validate()
.map_err(|errors| anyhow!("typst compilation failed:\n{}", errors.join("\n")))
}
fn prepare_content(
raw: &str,
wiki: &FactArchive,
files: Option<&FactArchive>,
allow_dangling: bool,
) -> Result<String> {
let content = lint_fix(raw, ReferenceResolver { wiki, files });
validate_typst(&content)?;
validate_links(&content, wiki, allow_dangling)?;
Ok(content)
}
fn revision_title(reader: &PileSnapshot, revision: &RevisionRecord) -> Result<String> {
read_string(reader, revision.title)
}
fn revision_content(reader: &PileSnapshot, revision: &RevisionRecord) -> Result<String> {
read_string(reader, revision.content)
}
fn cmd_create(
storage: WikiStorage<'_>,
title: String,
content: String,
tags: Vec<String>,
force: bool,
) -> Result<Id> {
let raw = content;
let (content, tags, mut fragment) =
storage.view_with_scope(FILES_SCOPE_ID, "Files", |view, files| {
let content = prepare_content(&raw, &view.facts, Some(files), force)?;
let mut fragment = Fragment::empty();
let tags = resolve_tags(&view.facts, &view.reader, &tags, &mut fragment)?;
Ok((content, tags, fragment))
})?;
let revision = stage_revision(storage, &mut fragment, None, title, content, tags)?;
storage.publish(fragment)?;
Ok(revision)
}
fn cmd_edit(
storage: WikiStorage<'_>,
id: String,
content: Option<String>,
title: Option<String>,
tag_names: Vec<String>,
force: bool,
) -> Result<Id> {
let scopes = if content.is_some() {
vec![(FILES_SCOPE_ID, "Files")]
} else {
Vec::new()
};
let (entry, title, content, tags, mut fragment) = storage.views(&scopes, |view, files| {
let entry = mutation_entry(view, &id)?;
if content.is_none() && title.is_none() && tag_names.is_empty() && entry.frontier.len() == 1
{
bail!("nothing to change");
}
let title = match &title {
Some(value) => value.clone(),
None => read_string(&view.reader, agreed(&entry, |head| head.title, "title")?)?,
};
let content = match &content {
Some(raw) => prepare_content(raw, &view.facts, files.first(), force)?,
None => read_string(
&view.reader,
agreed(&entry, |head| head.content, "content")?,
)?,
};
let mut fragment = Fragment::empty();
let tags = if tag_names.is_empty() {
agreed(&entry, |head| head.tags.clone(), "tags")?
} else {
resolve_tags(&view.facts, &view.reader, &tag_names, &mut fragment)?
};
Ok((entry, title, content, tags, fragment))
})?;
let revision = stage_revision(storage, &mut fragment, Some(&entry), title, content, tags)?;
storage.publish(fragment)?;
Ok(revision)
}
fn render_revision(
facts: &FactArchive,
reader: &PileSnapshot,
revision: &RevisionRecord,
) -> Result<String> {
let mut report = String::new();
let title = revision_title(reader, revision)?;
writeln!(report, "# {title}").unwrap();
writeln!(report, "revision: {:x}", revision.id).unwrap();
if !revision.supersedes.is_empty() {
writeln!(
report,
"supersedes: {}",
revision
.supersedes
.iter()
.map(|id| format!("{id:x}"))
.collect::<Vec<_>>()
.join(", ")
)
.unwrap();
}
let tags = format_tags(facts, reader, &revision.tags)?;
if !tags.is_empty() {
writeln!(report, "tags:{tags}").unwrap();
}
report.push('\n');
report.push_str(&revision_content(reader, revision)?);
Ok(report)
}
fn cmd_show(storage: WikiStorage<'_>, id: String, exact: bool) -> Result<String> {
let report = storage.view(|view| {
let selector = resolve_prefix(&view.facts, &id)?;
let revisions = selector_revisions(view, selector, !exact)?;
let mut report = String::new();
if revisions.len() > 1 {
writeln!(
report,
"fork: {} current revisions ({}); all shown, --exact pins one",
revisions.len(),
revisions
.iter()
.map(|revision| format!("{:x}", revision.id))
.collect::<Vec<_>>()
.join(", ")
)
.unwrap();
}
for (index, revision) in revisions.iter().enumerate() {
if index > 0 {
report.push_str("\n---\n\n");
}
report.push_str(&render_revision(&view.facts, &view.reader, revision)?);
}
Ok(report)
})?;
Ok(report)
}
fn cmd_export(storage: WikiStorage<'_>, id: String, exact: bool) -> Result<Export> {
let content = storage.view(|view| {
let selector = resolve_prefix(&view.facts, &id)?;
let revisions = selector_revisions(view, selector, !exact)?;
let [revision] = revisions.as_slice() else {
bail!(
"selector resolves to a fork ({}); choose one with --exact",
revisions
.iter()
.map(|revision| format!("{:x}", revision.id))
.collect::<Vec<_>>()
.join(", ")
)
};
Ok(Export {
revision: revision.id,
bytes: anybytes::Bytes::from(revision_content(&view.reader, revision)?.into_bytes()),
})
})?;
Ok(content)
}
fn unified_diff(old: &str, new: &str) -> Vec<String> {
let old: Vec<&str> = old.lines().collect();
let new: Vec<&str> = new.lines().collect();
let mut out = Vec::new();
let count = old.len().max(new.len());
for index in 0..count {
match (old.get(index), new.get(index)) {
(Some(left), Some(right)) if left == right => out.push(format!(" {left}")),
(Some(left), Some(right)) => {
out.push(format!("-{left}"));
out.push(format!("+{right}"));
}
(Some(left), None) => out.push(format!("-{left}")),
(None, Some(right)) => out.push(format!("+{right}")),
(None, None) => {}
}
}
out
}
fn cmd_diff(
storage: WikiStorage<'_>,
id: String,
from: Option<usize>,
to: Option<usize>,
) -> Result<String> {
let report = storage.view(|view| {
let entry = mutation_entry(view, &id)?;
let rows = wiki_model::entry_history(&view.facts, &entry);
if rows.len() < 2 {
bail!("entry has only {} revision(s)", rows.len());
}
let left = from.unwrap_or(rows.len() - 1).saturating_sub(1);
let right = to.unwrap_or(rows.len()).saturating_sub(1);
let Some(old) = rows.get(left) else {
bail!("--from is out of range")
};
let Some(new) = rows.get(right) else {
bail!("--to is out of range")
};
let mut report = String::new();
writeln!(
report,
"--- {} {}",
old.id,
revision_title(&view.reader, old)?
)
.unwrap();
writeln!(
report,
"+++ {} {}",
new.id,
revision_title(&view.reader, new)?
)
.unwrap();
for line in unified_diff(
&revision_content(&view.reader, old)?,
&revision_content(&view.reader, new)?,
) {
writeln!(report, "{line}").unwrap();
}
Ok(report)
})?;
Ok(report)
}
fn mutate_tags(storage: WikiStorage<'_>, id: String, name: &str, add: bool) -> Result<Option<Id>> {
let normalized = name.trim().to_ascii_lowercase();
if normalized.is_empty() {
bail!("tag name cannot be empty");
}
let prepared = storage.views(&[], |view, _| {
let entry = mutation_entry(view, &id)?;
let mut fragment = Fragment::empty();
let mut tags: BTreeSet<Id> = agreed(&entry, |head| head.tags.clone(), "tags")?
.into_iter()
.collect();
let desired = if add {
resolve_tags(
&view.facts,
&view.reader,
std::slice::from_ref(&normalized),
&mut fragment,
)?
} else {
let ids = tag_ids_named(&view.facts, &view.reader, &normalized)?;
if ids.is_empty() {
bail!("unknown tag '{normalized}'");
}
ids
};
let changed = if add {
let before = tags.len();
tags.extend(desired);
tags.len() != before
} else {
let before = tags.len();
for id in desired {
tags.remove(&id);
}
tags.len() != before
};
if !changed {
return Ok(None);
}
let title = read_string(&view.reader, agreed(&entry, |head| head.title, "title")?)?;
let content = read_string(
&view.reader,
agreed(&entry, |head| head.content, "content")?,
)?;
Ok(Some((entry, title, content, tags, fragment)))
})?;
let Some((entry, title, content, tags, mut fragment)) = prepared else {
return Ok(None);
};
let revision = stage_revision(storage, &mut fragment, Some(&entry), title, content, tags)?;
storage.publish(fragment)?;
Ok(Some(revision))
}
fn cmd_revert(storage: WikiStorage<'_>, id: String, to: usize) -> Result<Id> {
let (entry, title, content, tags) = storage.views(&[], |view, _| {
let entry = mutation_entry(view, &id)?;
let rows = wiki_model::entry_history(&view.facts, &entry);
let Some(chosen) = rows.get(to.saturating_sub(1)) else {
bail!("revision index out of range")
};
let title = revision_title(&view.reader, chosen)?;
let content = revision_content(&view.reader, chosen)?;
let tags = chosen.tags.iter().copied().collect();
Ok((entry, title, content, tags))
})?;
let mut fragment = Fragment::empty();
let revision = stage_revision(storage, &mut fragment, Some(&entry), title, content, tags)?;
storage.publish(fragment)?;
Ok(revision)
}
fn revision_links(reader: &PileSnapshot, revision: &RevisionRecord) -> Result<BTreeSet<Id>> {
let mut out = BTreeSet::new();
for raw in extract_link_targets(&revision_content(reader, revision)?) {
if let Some(id) = Id::from_hex(&raw) {
out.insert(id);
}
}
Ok(out)
}
fn derived_links(reader: &PileSnapshot, entry: &EntryRecord) -> Result<BTreeSet<Id>> {
let mut out = BTreeSet::new();
for head in &entry.frontier {
out.extend(revision_links(reader, head)?);
}
Ok(out)
}
#[derive(Default)]
struct BacklinkSummary {
tags: BTreeSet<Id>,
types: BTreeSet<String>,
}
fn backlink_summaries(
reader: &PileSnapshot,
facts: &FactArchive,
) -> Result<BTreeMap<Id, BacklinkSummary>> {
let expression = regex::Regex::new(
r#"#link\("wiki:(?:(?P<kind>[A-Za-z_][A-Za-z0-9_]*):)?(?P<id>[0-9A-Fa-f]{32})"\)"#,
)
.expect("static Wiki link expression");
let mut summaries = BTreeMap::<Id, BacklinkSummary>::new();
for id in wiki_model::revision_ids(facts) {
for source in wiki_model::revision_records(facts, id) {
let content = revision_content(reader, &source)?;
for captures in expression.captures_iter(&content) {
let target = Id::from_hex(&captures["id"]).expect("expression matched a full id");
let summary = summaries.entry(target).or_default();
summary.tags.extend(source.tags.iter().copied());
if let Some(kind) = captures.name("kind") {
summary.types.insert(kind.as_str().to_ascii_lowercase());
}
}
}
}
Ok(summaries)
}
fn cmd_links(
storage: WikiStorage<'_>,
id: Option<String>,
top: usize,
strict: bool,
out: &mut Out<'_>,
) -> Result<()> {
let Some(id) = id else {
let model =
storage.view(|view| FrontierModel::load(&view.reader, &view.facts, &view.latest))?;
return cmd_link_audit(&model, top, strict, out);
};
let (outgoing, incoming) = storage.view(|view| {
let entry = match mutation_entry(view, &id) {
Ok(entry) => entry,
Err(error) => return Err(explain_selector(view, &id, error)?),
};
Ok((
derived_links(&view.reader, &entry)?,
incoming_revisions(view, &entry)?,
))
})?;
out.line(format!("outgoing:"))?;
for target in outgoing {
out.line(format!(" {target:x}"))?;
}
out.line(format!("incoming:"))?;
for source in incoming {
out.line(format!(" {source:x}"))?;
}
Ok(())
}
fn explain_selector(view: &WikiView, raw: &str, error: anyhow::Error) -> Result<anyhow::Error> {
let Some(target) = Id::from_hex(raw.trim()) else {
return Ok(error);
};
let model = FrontierModel::load(&view.reader, &view.facts, &view.latest)?;
let entry = |index: usize| {
let entry = &model.entries[index];
format!("{} [wiki:{:x}]", short(&entry.title(), 55), entry.label)
};
let diagnosis = match model.classify(target) {
LinkClass::Legacy { entries, retired } => format!(
"it is a LEGACY FRAGMENT ANCHOR for {}{}. Anchors stopped being \
selectors on 2026-08-18 -- an id names a revision or it names \
nothing -- so cite a revision of that entry instead.",
entries
.iter()
.map(|index| entry(*index))
.collect::<Vec<_>>()
.join(" | "),
if retired { " (archived)" } else { "" }
),
LinkClass::Unwritten(Some(kind)) => format!(
"it names a {}, not a page. Nothing in the wiki is addressable by it.",
kind.label()
),
LinkClass::Unwritten(None) => {
"no fragment has ever had it, at any revision. If it came from another \
pile, it is that pile's id and does not travel."
.to_owned()
}
LinkClass::Live(index) | LinkClass::Retired(index) => {
format!("it resolves to {}", entry(index))
}
LinkClass::Ambiguous(candidates) => format!(
"it names several disconnected entries: {}",
candidates
.iter()
.map(|index| entry(*index))
.collect::<Vec<_>>()
.join(" | ")
),
};
Ok(error.context(diagnosis))
}
fn describe_target(model: &FrontierModel, reference: &LinkReference) -> String {
let entry = |index: usize| {
let entry = &model.entries[index];
format!("{} [wiki:{:x}]", short(&entry.title(), 55), entry.label)
};
match &reference.class {
LinkClass::Live(index) | LinkClass::Retired(index) => entry(*index),
LinkClass::Ambiguous(candidates) => candidates
.iter()
.map(|index| entry(*index))
.collect::<Vec<_>>()
.join(" | "),
LinkClass::Legacy { entries, retired } => format!(
"{}{}",
entries
.iter()
.map(|index| entry(*index))
.collect::<Vec<_>>()
.join(" | "),
if *retired { " (archived)" } else { "" }
),
LinkClass::Unwritten(Some(kind)) => format!("names a {}, not a page", kind.label()),
LinkClass::Unwritten(None) => "nothing, at any revision".to_owned(),
}
}
fn print_class(
model: &FrontierModel,
heading: &str,
rows: &[LinkReference],
top: usize,
out: &mut Out<'_>,
) -> Result<()> {
if rows.is_empty() {
return Ok(());
}
out.line(format!("\n--- {heading} ({}) ---", rows.len()))?;
for reference in rows.iter().take(top) {
out.line(format!(
"{} [wiki:{:x}]",
short(&reference.source_title, 60),
reference.source
))?;
out.line(format!(
" -> wiki:{:x} {}",
reference.target,
describe_target(model, reference)
))?;
}
if rows.len() > top {
out.line(format!(" … {} more", rows.len() - top))?;
}
Ok(())
}
fn cmd_link_audit(
model: &FrontierModel,
top: usize,
strict: bool,
out: &mut Out<'_>,
) -> Result<()> {
let audit = model.audit();
let unreferenced = model.unreferenced(&audit);
out.line(format!("=== WIKI: Frontier Link Audit ===\n"))?;
out.line(format!(
"Live entries: {} ({} current states)",
model.active_count(),
audit.states
))?;
out.line(format!("Outgoing citations: {}", audit.total))?;
out.line(format!(" resolve live: {}", audit.live))?;
out.line(format!(
" ambiguous selector: {} (a fork is evidence, not breakage)",
audit.ambiguous.len()
))?;
out.line(format!(
" archived target: {} <- BROKEN: the frontier dropped it",
audit.retired.len()
))?;
out.line(format!(
" legacy anchor only: {} <- migration signal, still reachable",
audit.legacy.len()
))?;
out.line(format!(
" never written: {} <- forward references, a TODO list",
audit.unwritten.len()
))?;
out.line(format!(
"Legacy anchors indexed: {} (a zero above means none is CITED, not\n\
\x20 that none exists)",
model.anchor_count()
))?;
print_class(&model, "ARCHIVED TARGETS", &audit.retired, top, out)?;
print_class(&model, "LEGACY ANCHORS", &audit.legacy, top, out)?;
print_class(&model, "NEVER WRITTEN", &audit.unwritten, top, out)?;
print_class(&model, "AMBIGUOUS SELECTORS", &audit.ambiguous, top, out)?;
out.line(format!(
"\n--- UNREFERENCED LIVE ENTRIES ({} of {}) ---",
unreferenced.len(),
model.active_count()
))?;
for index in unreferenced.iter().take(top) {
let entry = &model.entries[*index];
out.line(format!(
"{} [wiki:{:x}]",
short(&entry.title(), 60),
entry.label
))?;
}
if unreferenced.len() > top {
out.line(format!(" … {} more", unreferenced.len() - top))?;
}
if strict && audit.breakage() > 0 {
bail!(
"{} frontier citation(s) point into an archived entry",
audit.breakage()
);
}
Ok(())
}
fn short(value: &str, chars: usize) -> String {
value.chars().take(chars).collect()
}
fn incoming_revisions(view: &WikiView, entry: &EntryRecord) -> Result<Vec<Id>> {
let target_ids: BTreeSet<Id> = entry.members.iter().copied().collect();
let mut out = Vec::new();
for id in wiki_model::revision_ids(&view.facts) {
for source in wiki_model::revision_records(&view.facts, id) {
if revision_links(&view.reader, &source)?
.iter()
.any(|id| target_ids.contains(id))
{
out.push(source.id);
}
}
}
out.sort_unstable();
out.dedup();
Ok(out)
}
fn cmd_list(
storage: WikiStorage<'_>,
tag_names: Vec<String>,
with_backlink_tag: Vec<String>,
without_backlink_tag: Vec<String>,
with_backlink_type: Vec<String>,
without_backlink_type: Vec<String>,
all: bool,
) -> Result<String> {
let report = storage.view(|view| {
let wanted: Vec<BTreeSet<Id>> = tag_names
.iter()
.map(|name| tag_ids_named(&view.facts, &view.reader, name))
.collect::<Result<_>>()?;
let with_backlink_tags: Vec<BTreeSet<Id>> = with_backlink_tag
.iter()
.map(|name| tag_ids_named(&view.facts, &view.reader, name))
.collect::<Result<_>>()?;
let without_backlink_tags: Vec<BTreeSet<Id>> = without_backlink_tag
.iter()
.map(|name| tag_ids_named(&view.facts, &view.reader, name))
.collect::<Result<_>>()?;
for (name, ids) in tag_names
.iter()
.zip(&wanted)
.chain(with_backlink_tag.iter().zip(&with_backlink_tags))
.chain(without_backlink_tag.iter().zip(&without_backlink_tags))
{
if ids.is_empty() {
bail!("unknown tag '{}'", name.trim());
}
}
let with_backlink_types: Vec<String> = with_backlink_type
.iter()
.map(|kind| kind.to_ascii_lowercase())
.collect();
let without_backlink_types: Vec<String> = without_backlink_type
.iter()
.map(|kind| kind.to_ascii_lowercase())
.collect();
let has_backlink_filter = !with_backlink_tags.is_empty()
|| !without_backlink_tags.is_empty()
|| !with_backlink_types.is_empty()
|| !without_backlink_types.is_empty();
let backlink_summaries = if has_backlink_filter {
Some(backlink_summaries(&view.reader, &view.facts)?)
} else {
None
};
let mut entries = wiki_model::entries(&view.facts, &view.latest);
if !all {
entries.retain(|entry| {
!entry
.frontier
.iter()
.all(|revision| revision.tags.contains(&schema::TAG_ARCHIVED_ID))
});
}
let mut report = String::new();
for entry in entries {
if !wanted.is_empty()
&& !entry
.frontier
.iter()
.any(|head| wanted.iter().all(|ids| !head.tags.is_disjoint(ids)))
{
continue;
}
if let Some(backlink_summaries) = &backlink_summaries {
let mut incoming_tags = BTreeSet::new();
let mut incoming_types = BTreeSet::new();
for target in entry.members.iter() {
if let Some(summary) = backlink_summaries.get(target) {
incoming_tags.extend(summary.tags.iter().copied());
incoming_types.extend(summary.types.iter().cloned());
}
}
if !with_backlink_tags
.iter()
.all(|ids| !incoming_tags.is_disjoint(ids))
|| without_backlink_tags
.iter()
.any(|ids| !incoming_tags.is_disjoint(ids))
|| !with_backlink_types
.iter()
.all(|kind| incoming_types.contains(kind))
|| without_backlink_types
.iter()
.any(|kind| incoming_types.contains(kind))
{
continue;
}
}
writeln!(
report,
"{}{}",
entry_label(&entry),
if entry.frontier.len() > 1 {
" [fork]"
} else {
""
}
)
.unwrap();
for head in &entry.frontier {
writeln!(
report,
" {:x} {}{}",
head.id,
revision_title(&view.reader, head)?,
format_tags(&view.facts, &view.reader, &head.tags)?
)
.unwrap();
}
}
Ok(report)
})?;
Ok(report)
}
fn cmd_history(storage: WikiStorage<'_>, id: String) -> Result<String> {
let report = storage.view(|view| {
let entry = mutation_entry(view, &id)?;
let mut report = String::new();
writeln!(report, "# History: {}", entry_label(&entry)).unwrap();
for (index, revision) in wiki_model::entry_history(&view.facts, &entry)
.iter()
.enumerate()
{
writeln!(
report,
"v{} {:x} {} parents=[{}]{}",
index + 1,
revision.id,
revision_title(&view.reader, revision)?,
revision
.supersedes
.iter()
.map(|id| format!("{id:x}"))
.collect::<Vec<_>>()
.join(","),
if entry.frontier.iter().any(|head| head.id == revision.id) {
" [head]"
} else {
""
}
)
.unwrap();
}
Ok(report)
})?;
Ok(report)
}
fn cmd_tag_list(storage: WikiStorage<'_>, out: &mut Out<'_>) -> Result<()> {
let rows = storage.view(|view| {
let mut counts = HashMap::new();
for id in wiki_model::revision_ids(&view.facts) {
for revision in wiki_model::revision_records(&view.facts, id) {
for tag in &revision.tags {
*counts.entry(*tag).or_insert(0usize) += 1;
}
}
}
let mut rows = Vec::new();
for (id, handle) in find!(
(id: Id, handle: schema::TextHandle),
pattern!(&view.facts, [{ ?id @ metadata::name: ?handle }])
) {
rows.push((
read_string(&view.reader, handle)?,
id,
counts.get(&id).copied().unwrap_or(0),
));
}
rows.sort_by(|a, b| b.2.cmp(&a.2).then_with(|| a.0.cmp(&b.0)));
Ok(rows)
})?;
for (name, id, count) in rows {
out.line(format!("{id:x} {name} ({count})"))?;
}
Ok(())
}
fn cmd_tag_mint(storage: WikiStorage<'_>, name: String) -> Result<Id> {
let ids = storage.views(&[], |view, _| {
tag_ids_named(&view.facts, &view.reader, &name)
})?;
if let Some(id) = ids.first() {
return Ok(*id);
}
let (fragment, id, _) = wiki_model::tag_record(&name)?;
storage.publish(fragment)?;
Ok(id)
}
fn cmd_import(
storage: WikiStorage<'_>,
documents: Vec<ImportDocument>,
tags: Vec<String>,
) -> Result<Vec<Id>> {
let (view, files_catalog, tags, mut fragment) =
storage.view_with_scope(FILES_SCOPE_ID, "Files", |view, files| {
let mut fragment = Fragment::empty();
let tags = resolve_tags(&view.facts, &view.reader, &tags, &mut fragment)?;
Ok((view.clone(), files.clone(), tags, fragment))
})?;
let mut ids = Vec::new();
for document in documents {
let content = document.content;
let content = prepare_content(&content, &view.facts, Some(&files_catalog), true)?;
let title = content
.lines()
.find_map(|line| line.strip_prefix("= "))
.map(str::to_owned)
.unwrap_or(document.title);
let revision = stage_revision(storage, &mut fragment, None, title, content, tags.clone())?;
ids.push(revision);
}
if !fragment.facts().is_empty() {
storage.publish(fragment)?;
}
Ok(ids)
}
fn cmd_search(storage: WikiStorage<'_>, query: String, context: bool, all: bool) -> Result<String> {
let needle = query.to_ascii_lowercase();
let report = storage.view(|view| {
let mut entries = wiki_model::entries(&view.facts, &view.latest);
if !all {
entries.retain(|entry| {
!entry
.frontier
.iter()
.all(|revision| revision.tags.contains(&schema::TAG_ARCHIVED_ID))
});
}
let mut report = String::new();
for entry in entries {
for head in &entry.frontier {
let title = revision_title(&view.reader, head)?;
let content_text = revision_content(&view.reader, head)?;
if title.to_ascii_lowercase().contains(&needle)
|| content_text.to_ascii_lowercase().contains(&needle)
{
writeln!(
report,
"{:x} {title}{}",
head.id,
if entry.frontier.len() > 1 {
" [fork]"
} else {
""
}
)
.unwrap();
if context {
for line in content_text
.lines()
.filter(|line| line.to_ascii_lowercase().contains(&needle))
{
writeln!(report, " {}", line.trim()).unwrap();
}
}
}
}
}
Ok(report)
})?;
Ok(report)
}
fn cmd_check(storage: WikiStorage<'_>, compile: bool, out: &mut Out<'_>) -> Result<()> {
let (summary, diagnostics, issues) = storage.view(|view| {
let model = FrontierModel::load(&view.reader, &view.facts, &view.latest)?;
let mut diagnostics = String::new();
let mut issues = 0usize;
let mut legacy = 0usize;
let mut unwritten = 0usize;
let mut archived = 0usize;
let entries = wiki_model::entries(&view.facts, &view.latest);
for entry in &entries {
let live = entry
.frontier
.iter()
.any(|head| !head.tags.contains(&schema::TAG_ARCHIVED_ID));
if !live {
archived += 1;
}
for head in &entry.frontier {
let content = revision_content(&view.reader, head)?;
for raw in extract_link_targets(&content).into_iter().filter(|_| live) {
let id = Id::from_hex(&raw).expect("extractor returns full ids");
match model.classify(id) {
LinkClass::Live(_) | LinkClass::Ambiguous(_) => {}
LinkClass::Retired(target) => {
writeln!(
diagnostics,
"BROKEN_LINK {:x} wiki:{raw} -> archived entry wiki:{:x}",
head.id, model.entries[target].label
)
.unwrap();
issues += 1;
}
LinkClass::Legacy { .. } => {
writeln!(diagnostics, "LEGACY_LINK {:x} wiki:{raw}", head.id)
.unwrap();
legacy += 1;
}
LinkClass::Unwritten(_) => unwritten += 1,
}
}
if compile {
if let Err(error) = validate_typst(&content) {
writeln!(diagnostics, "TYPST_ERROR {:x} {error}", head.id).unwrap();
issues += 1;
}
}
}
}
let entries = entries.len();
let summary = format!(
"Checked {} live entries ({archived} archived, links not scanned), \
{issues} issues ({legacy} legacy anchor, {unwritten} unwritten target)",
entries - archived
);
Ok((summary, diagnostics, issues))
})?;
out.text(format!("{diagnostics}"))?;
out.line(format!("{summary}"))?;
if issues == 0 {
out.line(format!("All clear!"))?;
}
Ok(())
}
enum ReferenceLineResolution {
AlreadyFull,
Expanded(String),
}
fn resolve_reference_line(
line: &str,
resolver: ReferenceResolver<'_, FactArchive>,
) -> Result<ReferenceLineResolution> {
let (scheme, rest) = line
.split_once(':')
.ok_or_else(|| anyhow!("no scheme:selector format"))?;
let expanded = resolver.expand(scheme, rest)?;
let canonical = format!("{scheme}:{expanded}");
if canonical == line {
Ok(ReferenceLineResolution::AlreadyFull)
} else {
Ok(ReferenceLineResolution::Expanded(canonical))
}
}
fn cmd_fix_truncated(storage: WikiStorage<'_>, input: String, out: &mut Out<'_>) -> Result<()> {
let (view, files) = storage.view_with_scope(FILES_SCOPE_ID, "Files", |view, files| {
Ok((view.clone(), files.clone()))
})?;
let resolver = ReferenceResolver {
wiki: &view.facts,
files: Some(&files),
};
for line in input.lines().map(str::trim).filter(|line| !line.is_empty()) {
match resolve_reference_line(line, resolver) {
Ok(ReferenceLineResolution::AlreadyFull) => {}
Ok(ReferenceLineResolution::Expanded(value)) => out.line(format!("{line}\t{value}"))?,
Err(error) => out.line(format!("FAILED: {line} — {error}"))?,
}
}
Ok(())
}
fn cmd_lint(storage: WikiStorage<'_>, fix: bool, check: bool, out: &mut Out<'_>) -> Result<()> {
let (report, changed, revisions) =
storage.view_with_scope(FILES_SCOPE_ID, "Files", |view, files| {
let resolver = ReferenceResolver {
wiki: &view.facts,
files: Some(files),
};
let mut report = String::new();
let mut revisions = Vec::new();
let mut changed = 0usize;
for entry in wiki_model::entries(&view.facts, &view.latest) {
for head in &entry.frontier {
let content = revision_content(&view.reader, head)?;
let revised = lint_fix(&content, resolver);
if revised == content {
continue;
}
changed += 1;
if !check {
writeln!(
report,
"would fix {:x} ({})",
head.id,
revision_title(&view.reader, head)?
)
.unwrap();
}
if fix {
if entry.frontier.len() != 1 {
bail!(
"cannot lint-fix forked entry {} without an explicit content resolution",
entry_label(&entry)
);
}
let title = revision_title(&view.reader, head)?;
let tags = head.tags.iter().copied().collect();
revisions.push((entry.clone(), title, revised, tags));
break;
}
}
}
Ok((report, changed, revisions))
})?;
let mut fragment = Fragment::empty();
for (entry, title, content, tags) in revisions {
stage_revision(storage, &mut fragment, Some(&entry), title, content, tags)?;
}
if fix && !fragment.facts().is_empty() {
storage.publish(fragment)?;
}
out.text(format!("{report}"))?;
out.line(format!("{changed} revision(s) need lint fixes"))?;
if check && changed > 0 {
bail!("lint check failed")
}
Ok(())
}
fn cmd_batch_export(storage: WikiStorage<'_>) -> Result<Vec<(Id, String)>> {
let exports = storage.view(|view| {
let mut exports = Vec::new();
for entry in wiki_model::entries(&view.facts, &view.latest) {
for head in &entry.frontier {
exports.push((head.id, revision_content(&view.reader, head)?));
}
}
Ok(exports)
})?;
Ok(exports)
}
fn cmd_batch_import(storage: WikiStorage<'_>, imports: Vec<(Id, String)>) -> Result<()> {
let revisions = storage.views(&[], |view, _| {
let mut revisions = Vec::new();
for (revision_id, content) in &imports {
let revision_id = *revision_id;
let entry = wiki_model::entry(&view.facts, &view.latest, revision_id)
.ok_or_else(|| anyhow!("unknown revision {revision_id:x}"))?;
if entry.frontier.len() != 1 || entry.frontier[0].id != revision_id {
bail!("stale batch file {revision_id:x}: entry frontier changed");
}
let head = &entry.frontier[0];
if content == &revision_content(&view.reader, head)? {
continue;
}
revisions.push((
entry.clone(),
revision_title(&view.reader, head)?,
content.clone(),
head.tags.iter().copied().collect(),
));
}
Ok(revisions)
})?;
let mut fragment = Fragment::empty();
for (entry, title, content, tags) in revisions {
stage_revision(storage, &mut fragment, Some(&entry), title, content, tags)?;
}
if !fragment.facts().is_empty() {
storage.publish(fragment)?;
}
Ok(())
}
#[cfg(feature = "local-embed")]
fn l2_normalize(mut values: Vec<f32>) -> Vec<f32> {
let norm = values.iter().map(|value| value * value).sum::<f32>().sqrt();
if norm > 0.0 {
for value in &mut values {
*value /= norm;
}
}
values
}
#[cfg(feature = "local-embed")]
fn cmd_embed(storage: WikiStorage<'_>, out: &mut Out<'_>) -> Result<()> {
let documents = storage.view_with_scope(
EMBEDDINGS_SCOPE_ID,
"Embeddings",
|view, embedding_facts| {
let existing: BTreeSet<Id> = find!(
revision: Id,
pattern!(embedding_facts, [{ ?revision @ embeddings::attr::embedding: _?handle }])
)
.collect();
let mut documents = Vec::new();
for entry in wiki_model::entries(&view.facts, &view.latest)
.into_iter()
.filter(|entry| {
!entry
.frontier
.iter()
.all(|revision| revision.tags.contains(&schema::TAG_ARCHIVED_ID))
})
{
for head in &entry.frontier {
if existing.contains(&head.id) {
continue;
}
documents.push((head.id, revision_content(&view.reader, head)?));
}
}
Ok(documents)
},
)?;
let embedder = crate::nomic::load_text_embedder()?;
let mut fragment = Fragment::empty();
for (revision, content) in documents {
let vector = l2_normalize(embedder.embed_document(&content)?);
let handle = fragment.put::<Embedding768, _>(vector);
fragment +=
entity! { ExclusiveId::force_ref(&revision) @ embeddings::attr::embedding: handle };
}
if fragment.facts().is_empty() {
out.line(format!("all current revisions already embedded"))?;
} else {
storage.publish_scope(EMBEDDINGS_SCOPE_ID, fragment)?;
}
Ok(())
}
#[cfg(not(feature = "local-embed"))]
fn cmd_embed(_storage: WikiStorage<'_>, _out: &mut Out<'_>) -> Result<()> {
bail!("`wiki embed` needs --features local-embed")
}
#[cfg(feature = "local-embed")]
fn cmd_similar(storage: WikiStorage<'_>, query: String) -> Result<String> {
let embedder = crate::nomic::load_text_embedder()?;
let query = l2_normalize(embedder.embed_query(&query)?);
let report = storage.view_with_scope(
EMBEDDINGS_SCOPE_ID,
"Embeddings",
|view, embedding_facts| {
let current: BTreeSet<Id> = wiki_model::entries(&view.facts, &view.latest)
.into_iter()
.filter(|entry| {
!entry
.frontier
.iter()
.all(|revision| revision.tags.contains(&schema::TAG_ARCHIVED_ID))
})
.flat_map(|entry| entry.frontier.into_iter().map(|head| head.id))
.collect();
let mut pairs = Vec::new();
for (revision, handle) in find!(
(revision: Id, handle: Inline<inlineencodings::Handle<Embedding768>>),
pattern!(embedding_facts, [{ ?revision @ embeddings::attr::embedding: ?handle }])
) {
if !current.contains(&revision) {
continue;
}
let vector: anybytes::View<[f32]> = BlobStoreGet::get(&view.reader, handle)?;
pairs.push((revision, vector.as_ref().to_vec()));
}
let mut report = String::new();
for (score, revision) in embeddings::nearest(&pairs, &query, 0.0)?
.into_iter()
.take(10)
{
let title = wiki_model::revision_records(&view.facts, revision)
.first()
.map(|row| revision_title(&view.reader, row))
.transpose()?
.unwrap_or_default();
writeln!(report, "{score:6.3} {revision:x} {title}").unwrap();
}
Ok(report)
},
)?;
Ok(report)
}
#[cfg(not(feature = "local-embed"))]
fn cmd_similar(_storage: WikiStorage<'_>, _query: String) -> Result<String> {
bail!("`wiki similar` needs --features local-embed")
}
mod typst_validate {
use typst::diag::FileResult;
use typst::foundations::{Bytes, Datetime};
use typst::layout::PagedDocument;
use typst::syntax::{FileId, Source, VirtualPath};
use typst::text::{Font, FontBook};
use typst::utils::LazyHash;
use typst::{Library, LibraryExt, World};
pub struct ValidateWorld {
library: LazyHash<Library>,
book: LazyHash<FontBook>,
main_id: FileId,
source: Source,
}
impl ValidateWorld {
pub fn new(content: &str) -> Self {
let main_id = FileId::new(None, VirtualPath::new("main.typ"));
Self {
library: LazyHash::new(Library::default()),
book: LazyHash::new(FontBook::new()),
main_id,
source: Source::new(main_id, content.to_owned()),
}
}
pub fn validate(&self) -> Result<(), Vec<String>> {
match typst::compile::<PagedDocument>(self).output {
Ok(_) => Ok(()),
Err(errors) => {
let errors: Vec<String> = errors
.iter()
.filter(|error| !error.message.contains("no font"))
.map(|error| error.message.to_string())
.collect();
if errors.is_empty() {
Ok(())
} else {
Err(errors)
}
}
}
}
}
impl World for ValidateWorld {
fn library(&self) -> &LazyHash<Library> {
&self.library
}
fn book(&self) -> &LazyHash<FontBook> {
&self.book
}
fn main(&self) -> FileId {
self.main_id
}
fn source(&self, id: FileId) -> FileResult<Source> {
if id == self.main_id {
Ok(self.source.clone())
} else {
Err(typst::diag::FileError::NotFound(
id.vpath().as_rootless_path().into(),
))
}
}
fn file(&self, id: FileId) -> FileResult<Bytes> {
Err(typst::diag::FileError::NotFound(
id.vpath().as_rootless_path().into(),
))
}
fn font(&self, _index: usize) -> Option<Font> {
None
}
fn today(&self, _offset: Option<i64>) -> Option<Datetime> {
None
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::storage::load_signer;
use anybytes::Bytes;
use std::fs::File;
use triblespace::core::blob::MemoryBlobStoreSnapshot;
use triblespace::core::repo::{BlobStoreList, MissingBlob, WantRead};
#[test]
fn typst_validation_world_refuses_external_sources_and_files() {
use typst::syntax::{FileId, VirtualPath};
use typst::World;
let world = typst_validate::ValidateWorld::new("resident text");
let external = FileId::new(None, VirtualPath::new("external.typ"));
assert!(world.source(external).is_err());
assert!(world.file(external).is_err());
assert!(world.source(world.main()).is_ok());
assert!(validate_typst("#include \"external.typ\"").is_err());
assert!(validate_typst("#read(\"external.typ\")").is_err());
}
struct Fixture {
_directory: tempfile::TempDir,
pile: PathBuf,
key: PathBuf,
storage: Storage,
}
impl Fixture {
fn new() -> Self {
let directory = tempfile::tempdir().unwrap();
let pile = directory.path().join("wiki.pile");
let key = directory.path().join("wiki.key");
File::create(&pile).unwrap();
crate::storage::initialize_signer(&pile, Some(&key)).unwrap();
Self {
_directory: directory,
storage: Storage::new(pile.clone(), Some(key.clone())),
pile,
key,
}
}
fn storage(&self) -> WikiStorage<'_> {
WikiStorage {
storage: &self.storage,
}
}
fn carry(&self, auxiliaries: &[Collection<blobencodings::SimpleArchive>]) {
self.storage
.with_pile(|pile, signer| {
crate::wiki::carry_for_tests(pile, signer);
for &source in auxiliaries {
crate::storage::carry_facts(pile, source, signer);
}
Ok(())
})
.unwrap();
}
}
#[test]
fn successive_publications_advance_the_views_a_reader_prepares() {
let fixture = Fixture::new();
let storage = fixture.storage();
let (succinct, rank9, latest) = fixture
.storage
.with_pile(|pile, signer| {
let source =
open_configured(pile, schema::DEFAULT_SCOPE_ID, signer.verifying_key())?;
let succinct = pile.attach::<SuccinctArchiveBlob>(source, ())?;
let rank9 = pile.attach::<Rank9AcceleratedSuccinctArchiveBlob>(source, succinct)?;
Ok((
succinct,
rank9,
wiki_model::latest_for_source(pile, source)?,
))
})
.unwrap();
let observe = || {
fixture
.storage
.with_pile(|pile, signer| {
let snapshot = pollster::block_on(async {
drop(pile.maintain_attached(succinct, signer).await?);
drop(pile.maintain_attached(rank9, signer).await?);
pile.maintain_attached(latest, signer).await
})?;
Ok((
snapshot.read_facts(rank9)?,
snapshot
.attached(latest)?
.read::<LatestIndex>()?
.into_value(),
))
})
.unwrap()
};
let mut first = Fragment::empty();
let root = stage_revision(
storage,
&mut first,
None,
"first".into(),
"body".into(),
BTreeSet::new(),
)
.unwrap();
storage.publish(first).unwrap();
let (first_facts, first_latest) = observe();
let entry = wiki_model::entry(&first_facts, &first_latest, root).unwrap();
let mut successor = Fragment::empty();
let next = stage_revision(
storage,
&mut successor,
Some(&entry),
"next".into(),
"new body".into(),
BTreeSet::new(),
)
.unwrap();
storage.publish(successor).unwrap();
let (facts, latest) = observe();
let current = wiki_model::entry(&facts, &latest, root).unwrap();
assert_eq!(
current
.frontier
.iter()
.map(|head| head.id)
.collect::<Vec<_>>(),
vec![next]
);
assert_eq!(
entry
.frontier
.iter()
.map(|head| head.id)
.collect::<Vec<_>>(),
vec![root]
);
assert!(first_latest.contains(root));
assert!(!latest.contains(root));
assert!(latest.contains(next));
}
#[test]
fn reads_answer_for_every_revision_the_facts_hold_before_the_worker_carries() {
let fixture = Fixture::new();
let storage = fixture.storage();
let source = storage
.with_pile(|pile, signer, _| {
Ok(crate::collection_names::open(
pile,
schema::DEFAULT_SCOPE_ID,
signer.verifying_key(),
)?)
})
.unwrap();
let mut genesis = Fragment::empty();
let root = stage_revision(
storage,
&mut genesis,
None,
"original".to_owned(),
"old body".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage
.with_pile(|pile, signer, _| {
pile.commit(source, signer, genesis)?;
Ok(())
})
.unwrap();
fixture.carry(&[]);
let owner_read = || {
storage
.with_pile(|pile, signer, runtime| {
views_in(pile, source, signer, &[], runtime, |view, _| {
Ok(view.clone())
})
})
.unwrap()
};
let current = owner_read();
let entry = wiki_model::entry(¤t.facts, ¤t.latest, root).unwrap();
let mut successor = Fragment::empty();
let next = stage_revision(
storage,
&mut successor,
Some(&entry),
"current".to_owned(),
"new body".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage
.with_pile(|pile, signer, _| {
pile.commit(source, signer, successor)?;
Ok(())
})
.unwrap();
let before = storage
.with_pile(|pile, _, _| {
Ok(pile.snapshot()?.records()?.collect::<Result<Vec<_>, _>>()?)
})
.unwrap();
let resident = owner_read();
let entry = wiki_model::entry(&resident.facts, &resident.latest, root).unwrap();
assert_eq!(
entry
.frontier
.iter()
.map(|head| head.id)
.collect::<Vec<_>>(),
[next]
);
assert_eq!(
revision_content(&resident.reader, &entry.frontier[0]).unwrap(),
"new body"
);
assert!(wiki_model::revision_ids(&resident.facts).contains(&next));
assert_eq!(
storage
.with_pile(|pile, _, _| Ok(pile
.snapshot()?
.records()?
.collect::<Result<Vec<_>, _>>()?))
.unwrap(),
before,
"a read publishes nothing",
);
fixture.carry(&[]);
let current = owner_read();
let entry = wiki_model::entry(¤t.facts, ¤t.latest, root).unwrap();
assert_eq!(
entry
.frontier
.iter()
.map(|head| head.id)
.collect::<Vec<_>>(),
[next]
);
assert_eq!(
revision_content(¤t.reader, &entry.frontier[0]).unwrap(),
"new body"
);
}
#[test]
fn owner_reads_report_cold_selected_source_or_auxiliary_members() {
for cold_auxiliary in [false, true] {
let fixture = Fixture::new();
let storage = fixture.storage();
let mut genesis = Fragment::empty();
let root = stage_revision(
storage,
&mut genesis,
None,
"resident revision".to_owned(),
"resident body".to_owned(),
BTreeSet::new(),
)
.unwrap();
let auxiliary_marker = ufoid();
let (source, auxiliary) = storage
.with_pile(|pile, signer, _| {
let source = crate::collection_names::open(
pile,
schema::DEFAULT_SCOPE_ID,
signer.verifying_key(),
)?;
let auxiliary = crate::collection_names::open(
pile,
FILES_SCOPE_ID,
signer.verifying_key(),
)?;
pile.commit(source, signer, genesis)?;
pile.commit(
auxiliary,
signer,
entity! { &auxiliary_marker @ metadata::tag: &auxiliary_marker },
)?;
Ok((source, auxiliary))
})
.unwrap();
fixture.carry(&[auxiliary]);
let warm = storage
.with_pile(|pile, signer, runtime| {
views_in(
pile,
source,
signer,
&[(auxiliary, "Files")],
runtime,
|view, auxiliaries| {
let entry = wiki_model::entry(&view.facts, &view.latest, root).unwrap();
assert_eq!(
entry.frontier.iter().map(|head| head.id).collect::<Vec<_>>(),
[root],
);
assert_eq!(
revision_content(&view.reader, &entry.frontier[0])?,
"resident body",
);
assert_eq!(
find!(id: Id, pattern!(&auxiliaries[0], [{ ?id @ metadata::tag: &auxiliary_marker }]))
.collect::<Vec<_>>(),
[*auxiliary_marker],
);
Ok(view.clone())
},
)
})
.unwrap();
let entry = wiki_model::entry(&warm.facts, &warm.latest, root).unwrap();
let later = if cold_auxiliary {
let marker = ufoid();
entity! { &marker @ metadata::tag: &marker }
} else {
let mut fragment = Fragment::empty();
stage_revision(
storage,
&mut fragment,
Some(&entry),
"cold revision".to_owned(),
"cold body".to_owned(),
BTreeSet::new(),
)
.unwrap();
fragment
};
storage
.with_pile(|pile, signer, runtime| {
let mut remote = MemoryRepo::default();
let arriving = remote.commit(
if cold_auxiliary { auxiliary } else { source },
signer,
later,
)?;
pile.insert(triblespace::core::collection::CollectionRecord::Commit(arriving))?;
let cold = inlineencodings::Handle::<blobencodings::SimpleArchive>::from_hash(
arriving.data(),
);
let before = pile.snapshot()?;
let selected_source = if cold_auxiliary { auxiliary } else { source };
assert!(selected_source.admitted(&before)?.contains(cold));
assert!(!before.contains_blob(cold)?);
let records = before.records()?.collect::<Result<Vec<_>, _>>()?;
assert!(pile.health().started_at.is_none());
assert!(!pile.health().store.serving_snapshot);
let error = views_in(
pile,
source,
signer,
&[(auxiliary, "Files")],
runtime,
|_, _| -> Result<()> {
panic!("an incomplete observation must not reach the reader")
},
)
.unwrap_err();
let incomplete = error
.downcast_ref::<crate::storage::IncompleteAttachedRead>()
.unwrap();
assert_eq!(incomplete.unread.collection(), selected_source);
assert_eq!(incomplete.unread.members().collect::<Vec<_>>(), vec![cold]);
let after = pile.snapshot()?;
assert!(!after.contains_blob(cold)?);
assert_eq!(after.records()?.collect::<Result<Vec<_>, _>>()?, records);
assert_eq!(after.wants()?.count(), 0);
let health = pile.health();
assert!(!health.store.serving_snapshot);
assert!(health.store.last_snapshot_published_at.is_none());
Ok(())
})
.unwrap();
}
}
fn supply_selected_blob(
fixture: &Fixture,
remote: &MemoryBlobStoreSnapshot,
error: &anyhow::Error,
) -> Inline<inlineencodings::Handle<blobencodings::UnknownBlob>> {
let missing = error
.chain()
.find_map(|error| error.downcast_ref::<MissingBlob>())
.expect("the selected payload must identify its exact missing handle");
let bytes: Bytes = remote.get(missing.handle).unwrap();
let mut arrival = crate::storage::open_pile_strict(&fixture.pile).unwrap();
assert_eq!(
arrival.put::<blobencodings::UnknownBlob, _>(bytes).unwrap(),
missing.handle
);
arrival.close().unwrap();
missing.handle
}
#[test]
fn sparse_payload_retry_keeps_selected_revision_and_publishes_only_after_preparation() {
let fixture = Fixture::new();
let storage = fixture.storage();
let mut genesis = Fragment::empty();
let (tag_fragment, tag, _) = wiki_model::tag_record("selected-tag").unwrap();
genesis += tag_fragment;
let root = stage_revision(
storage,
&mut genesis,
None,
"selected title".to_owned(),
"selected original body".to_owned(),
BTreeSet::from([tag]),
)
.unwrap();
let unrelated = stage_revision(
storage,
&mut genesis,
None,
"unrelated title".to_owned(),
"unrelated body stays cold".to_owned(),
BTreeSet::new(),
)
.unwrap();
let selected = wiki_model::revision_records(genesis.facts(), root).remove(0);
let cold = wiki_model::revision_records(genesis.facts(), unrelated).remove(0);
let mut remote = genesis.blobs().clone();
let remote = remote.snapshot().unwrap();
genesis.blobs_mut().keep([]);
storage.publish(genesis).unwrap();
let (_, author) = storage.author_fragment().unwrap();
let (arrival, later) = wiki_model::revision_record(RevisionDraft {
title: "later title".to_owned(),
content: "later body".to_owned(),
tags: BTreeSet::from([tag]),
predecessors: BTreeSet::from([root]),
author,
authored_at: point(2.0),
})
.unwrap();
let mut arrival = Some(arrival);
let mut requested = Vec::new();
let mut original = None::<FacultySnapshot>;
let (report, entry, title, content) = storage
.view(|view| {
original.get_or_insert_with(|| view.reader.clone());
let entry = mutation_entry(view, &format!("{root:x}"))?;
assert_eq!(
entry
.frontier
.iter()
.map(|head| head.id)
.collect::<Vec<_>>(),
[root]
);
let head = &entry.frontier[0];
let prepared = (|| {
Ok((
render_revision(&view.facts, &view.reader, head)?,
entry.clone(),
revision_title(&view.reader, head)?,
revision_content(&view.reader, head)?,
))
})();
if let Err(error) = &prepared {
requested.push(supply_selected_blob(&fixture, &remote, error));
if let Some(arrival) = arrival.take() {
let signer = load_signer(&fixture.pile, Some(&fixture.key))?;
let mut writer = crate::storage::open_pile_strict(&fixture.pile)?;
let source = open_configured(
&mut writer,
schema::DEFAULT_SCOPE_ID,
signer.verifying_key(),
)?;
writer.commit(source, &signer, arrival)?;
writer.close()?;
}
}
prepared
})
.unwrap();
assert!(report.contains("selected original body"));
assert!(!report.contains("later body"));
assert_eq!(
requested.len(),
3,
"only the selected title, tag name, and body are read"
);
let original = original.unwrap();
for handle in &requested {
assert!(!original.contains_blob(*handle).unwrap());
}
assert!(requested.contains(&selected.title.transmute()));
assert!(requested.contains(&selected.content.transmute()));
assert!(!requested.contains(&cold.title.transmute()));
assert!(!requested.contains(&cold.content.transmute()));
let mut edit = Fragment::empty();
let written = stage_revision(
storage,
&mut edit,
Some(&entry),
title,
format!("{content}\nprepared once"),
BTreeSet::from([tag]),
)
.unwrap();
storage.publish(edit).unwrap();
storage
.view(|view| {
let entry = mutation_entry(view, &format!("{root:x}"))?;
assert_eq!(
entry
.frontier
.iter()
.map(|head| head.id)
.collect::<BTreeSet<_>>(),
BTreeSet::from([later, written])
);
let written = wiki_model::revision_records(&view.facts, written).remove(0);
assert_eq!(
written.authorships.len(),
1,
"retrying text must not repeat authored observations"
);
assert!(!view.reader.contains_blob(cold.title).unwrap());
assert!(!view.reader.contains_blob(cold.content).unwrap());
assert_eq!(view.reader.wants().unwrap().count(), 0);
Ok(())
})
.unwrap();
}
#[test]
fn a_cold_tag_name_is_not_absence_and_mint_does_not_republish_it() {
let fixture = Fixture::new();
let storage = fixture.storage();
let (mut tag, expected, _) = wiki_model::tag_record("known-cold-tag").unwrap();
let mut remote = tag.blobs().clone();
let remote = remote.snapshot().unwrap();
tag.blobs_mut().keep([]);
storage.publish(tag).unwrap();
let mut requested = Vec::new();
let ids = storage
.view(|view| {
let result = tag_ids_named(&view.facts, &view.reader, "known-cold-tag");
if let Err(error) = &result {
requested.push(supply_selected_blob(&fixture, &remote, error));
}
result
})
.unwrap();
assert_eq!(ids, BTreeSet::from([expected]));
assert_eq!(requested.len(), 1);
let before = fs::read(&fixture.pile).unwrap();
cmd_tag_mint(storage, "known-cold-tag".to_owned()).unwrap();
assert_eq!(
fs::read(&fixture.pile).unwrap(),
before,
"an existing cold tag must not be republished"
);
}
#[test]
fn edit_joins_the_complete_current_frontier() {
let fixture = Fixture::new();
let storage = fixture.storage();
let mut genesis = Fragment::empty();
let root = stage_revision(
storage,
&mut genesis,
None,
"root".to_owned(),
"body".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage.publish(genesis).unwrap();
let current = storage.view(|view| Ok(view.clone())).unwrap();
let entry = wiki_model::entry(¤t.facts, ¤t.latest, root).unwrap();
let mut forks = Fragment::empty();
let left = stage_revision(
storage,
&mut forks,
Some(&entry),
"left".to_owned(),
"left".to_owned(),
BTreeSet::new(),
)
.unwrap();
let right = stage_revision(
storage,
&mut forks,
Some(&entry),
"right".to_owned(),
"right".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage.publish(forks).unwrap();
cmd_edit(
storage,
format!("{left:x}"),
Some("joined".to_owned()),
Some("joined".to_owned()),
Vec::new(),
true,
)
.unwrap();
let after = storage.view(|view| Ok(view.clone())).unwrap();
let entry = wiki_model::entry(&after.facts, &after.latest, left).unwrap();
assert_eq!(entry.frontier.len(), 1);
assert_eq!(entry.frontier[0].supersedes, BTreeSet::from([left, right]));
}
fn superseded_pair(storage: WikiStorage<'_>) -> (Id, Id) {
let mut genesis = Fragment::empty();
let root = stage_revision(
storage,
&mut genesis,
None,
"page".to_owned(),
"first draft".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage.publish(genesis).unwrap();
cmd_edit(
storage,
format!("{root:x}"),
Some("second draft".to_owned()),
None,
Vec::new(),
true,
)
.unwrap();
let after = storage.view(|view| Ok(view.clone())).unwrap();
let entry = wiki_model::entry(&after.facts, &after.latest, root).unwrap();
assert_eq!(entry.frontier.len(), 1);
(root, entry.frontier[0].id)
}
#[test]
fn show_follows_a_superseded_id_to_the_frontier_by_default() {
let fixture = Fixture::new();
let storage = fixture.storage();
let (root, head) = superseded_pair(storage);
assert_ne!(root, head, "the fixture must actually supersede something");
let view = storage.view(|view| Ok(view.clone())).unwrap();
let shown = selector_revisions(&view, root, true).unwrap();
assert_eq!(
shown.iter().map(|r| r.id).collect::<Vec<_>>(),
vec![head],
"a superseded selector must resolve forward to the head"
);
assert_eq!(
revision_content(&view.reader, &shown[0]).unwrap(),
"second draft"
);
cmd_show(storage, format!("{root:x}"), false).unwrap();
}
#[test]
fn exact_pins_the_named_revision() {
let fixture = Fixture::new();
let storage = fixture.storage();
let (root, head) = superseded_pair(storage);
let view = storage.view(|view| Ok(view.clone())).unwrap();
let pinned = selector_revisions(&view, root, false).unwrap();
assert_eq!(pinned.iter().map(|r| r.id).collect::<Vec<_>>(), vec![root]);
assert_eq!(
revision_content(&view.reader, &pinned[0]).unwrap(),
"first draft"
);
cmd_show(storage, format!("{root:x}"), true).unwrap();
assert_eq!(
selector_revisions(&view, head, true)
.unwrap()
.iter()
.map(|r| r.id)
.collect::<Vec<_>>(),
vec![head]
);
}
#[test]
fn an_id_that_names_nothing_still_fails_loudly() {
let fixture = Fixture::new();
let storage = fixture.storage();
let (_root, _head) = superseded_pair(storage);
let view = storage.view(|view| Ok(view.clone())).unwrap();
let absent = "f40312df406d1bf1bb5c94ec954e490b";
let error = resolve_prefix(&view.facts, absent).unwrap_err().to_string();
assert!(error.contains("no Wiki id matches"), "got: {error}");
assert!(cmd_show(storage, absent.to_owned(), false).is_err());
assert!(cmd_show(storage, absent.to_owned(), true).is_err());
}
#[test]
fn a_forked_frontier_shows_every_head_and_export_refuses() {
let fixture = Fixture::new();
let storage = fixture.storage();
let mut genesis = Fragment::empty();
let root = stage_revision(
storage,
&mut genesis,
None,
"root".to_owned(),
"body".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage.publish(genesis).unwrap();
let current = storage.view(|view| Ok(view.clone())).unwrap();
let entry = wiki_model::entry(¤t.facts, ¤t.latest, root).unwrap();
let mut forks = Fragment::empty();
let left = stage_revision(
storage,
&mut forks,
Some(&entry),
"left".to_owned(),
"left".to_owned(),
BTreeSet::new(),
)
.unwrap();
let right = stage_revision(
storage,
&mut forks,
Some(&entry),
"right".to_owned(),
"right".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage.publish(forks).unwrap();
let view = storage.view(|view| Ok(view.clone())).unwrap();
let heads: BTreeSet<Id> = selector_revisions(&view, root, true)
.unwrap()
.iter()
.map(|r| r.id)
.collect();
assert_eq!(heads, BTreeSet::from([left, right]));
cmd_show(storage, format!("{root:x}"), false).unwrap();
let error = cmd_export(storage, format!("{root:x}"), false)
.unwrap_err()
.to_string();
assert!(error.contains("fork"), "got: {error}");
assert!(error.contains(&format!("{left:x}")), "got: {error}");
cmd_export(storage, format!("{left:x}"), true).unwrap();
}
#[test]
fn unanchored_native_revision_is_a_cli_selector() {
let fixture = Fixture::new();
let storage = fixture.storage();
let files = storage
.view_with_scope(FILES_SCOPE_ID, "Files", |_, files| Ok(files.clone()))
.unwrap();
assert!(find!(
id: Id,
pattern!(&files, [{ ?id @ metadata::tag: _?kind }])
)
.next()
.is_none());
let mut fragment = Fragment::empty();
let revision = stage_revision(
storage,
&mut fragment,
None,
"native".to_owned(),
"body".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage.publish(fragment).unwrap();
let after = storage.view(|view| Ok(view.clone())).unwrap();
assert_eq!(
resolve_prefix(&after.facts, &format!("{revision:x}")).unwrap(),
revision
);
let entry = wiki_model::entry(&after.facts, &after.latest, revision).unwrap();
assert_eq!(entry.roots, vec![revision]);
}
#[test]
fn lint_preserves_typed_link_while_expanding_the_selector() {
let fixture = Fixture::new();
let storage = fixture.storage();
let mut fragment = Fragment::empty();
let revision = stage_revision(
storage,
&mut fragment,
None,
"target".to_owned(),
"body".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage.publish(fragment).unwrap();
let after = storage.view(|view| Ok(view.clone())).unwrap();
let short = &format!("{revision:x}")[..8];
let fixed = lint_fix(
&format!("[review](wiki:reviews:{short})"),
ReferenceResolver {
wiki: &after.facts,
files: None,
},
);
assert_eq!(
fixed,
format!("#link(\"wiki:reviews:{revision:x}\")[review]")
);
}
#[test]
fn backlinks_name_the_citing_revision_not_its_successor() {
let fixture = Fixture::new();
let storage = fixture.storage();
let mut genesis = Fragment::empty();
let target = stage_revision(
storage,
&mut genesis,
None,
"target".to_owned(),
"body".to_owned(),
BTreeSet::new(),
)
.unwrap();
let citing = stage_revision(
storage,
&mut genesis,
None,
"source".to_owned(),
format!("cites #link(\"wiki:{target:x}\")[target]"),
BTreeSet::new(),
)
.unwrap();
storage.publish(genesis).unwrap();
let current = storage.view(|view| Ok(view.clone())).unwrap();
let source_entry = wiki_model::entry(¤t.facts, ¤t.latest, citing).unwrap();
let mut edit = Fragment::empty();
let dropped = stage_revision(
storage,
&mut edit,
Some(&source_entry),
"source".to_owned(),
"no citation any more".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage.publish(edit).unwrap();
let after = storage.view(|view| Ok(view.clone())).unwrap();
let source_entry = wiki_model::entry(&after.facts, &after.latest, citing).unwrap();
assert_eq!(
source_entry
.frontier
.iter()
.map(|head| head.id)
.collect::<Vec<_>>(),
vec![dropped]
);
let target_entry = wiki_model::entry(&after.facts, &after.latest, target).unwrap();
let incoming = incoming_revisions(&after, &target_entry).unwrap();
assert!(
incoming.contains(&citing),
"the revision that wrote the citation must be listed"
);
assert!(
!incoming.contains(&dropped),
"a revision whose text does not cite the target must not be listed"
);
}
#[test]
fn backlink_summaries_carry_the_citing_revision_tags_only() {
let fixture = Fixture::new();
let storage = fixture.storage();
let mut genesis = Fragment::empty();
let target = stage_revision(
storage,
&mut genesis,
None,
"target".to_owned(),
"body".to_owned(),
BTreeSet::new(),
)
.unwrap();
let (citing_tag_fragment, citing_tag, _) = wiki_model::tag_record("citing").unwrap();
let (later_tag_fragment, later_tag, _) = wiki_model::tag_record("later").unwrap();
genesis += citing_tag_fragment;
genesis += later_tag_fragment;
let citing = stage_revision(
storage,
&mut genesis,
None,
"source".to_owned(),
format!("cites #link(\"wiki:{target:x}\")[target]"),
BTreeSet::from([citing_tag]),
)
.unwrap();
storage.publish(genesis).unwrap();
let current = storage.view(|view| Ok(view.clone())).unwrap();
let source_entry = wiki_model::entry(¤t.facts, ¤t.latest, citing).unwrap();
let mut edit = Fragment::empty();
stage_revision(
storage,
&mut edit,
Some(&source_entry),
"source".to_owned(),
"no citation any more".to_owned(),
BTreeSet::from([later_tag]),
)
.unwrap();
storage.publish(edit).unwrap();
let after = storage.view(|view| Ok(view.clone())).unwrap();
let summaries = backlink_summaries(&after.reader, &after.facts).unwrap();
assert_eq!(
summaries.get(&target).unwrap().tags,
BTreeSet::from([citing_tag])
);
}
#[test]
fn backlink_summaries_index_typed_links_and_source_tags() {
let fixture = Fixture::new();
let storage = fixture.storage();
let mut fragment = Fragment::empty();
let target = stage_revision(
storage,
&mut fragment,
None,
"target".to_owned(),
"body".to_owned(),
BTreeSet::new(),
)
.unwrap();
let (tag_fragment, source_tag, _) = wiki_model::tag_record("source").unwrap();
fragment += tag_fragment;
stage_revision(
storage,
&mut fragment,
None,
"source".to_owned(),
format!("#link(\"wiki:Reviews:{target:x}\")[review]"),
BTreeSet::from([source_tag]),
)
.unwrap();
storage.publish(fragment).unwrap();
let after = storage.view(|view| Ok(view.clone())).unwrap();
let summaries = backlink_summaries(&after.reader, &after.facts).unwrap();
let incoming = summaries.get(&target).unwrap();
assert_eq!(incoming.tags, BTreeSet::from([source_tag]));
assert_eq!(incoming.types, BTreeSet::from(["reviews".to_owned()]));
}
use triblespace::core::metadata;
fn point(seconds: f64) -> Inline<inlineencodings::NsTAIInterval> {
let epoch = Epoch::from_tai_seconds(seconds);
(epoch, epoch).try_to_inline().unwrap()
}
fn legacy_anchor_pair() -> (Fragment, Id, Id, Id) {
let anchor = genid().id;
let mut fragment = Fragment::empty();
let title: schema::TextHandle = fragment.put("T".to_owned());
let older: schema::TextHandle = fragment.put("first text".to_owned());
let newer: schema::TextHandle = fragment.put("second text".to_owned());
let v1 = genid().id;
let v2 = genid().id;
fragment += entity! { ExclusiveId::force_ref(&v1) @
metadata::tag: &schema::KIND_VERSION_ID,
schema::attrs::fragment: anchor,
schema::attrs::title: title,
schema::attrs::content: older,
metadata::created_at: point(1.0),
};
fragment += entity! { ExclusiveId::force_ref(&v2) @
metadata::tag: &schema::KIND_VERSION_ID,
schema::attrs::fragment: anchor,
schema::attrs::title: title,
schema::attrs::content: newer,
metadata::created_at: point(2.0),
metadata::supersedes: v1,
};
(fragment, anchor, v1, v2)
}
#[test]
fn a_legacy_anchor_is_not_a_selector_and_is_left_untouched() {
let (fragment, anchor, v1, _v2) = legacy_anchor_pair();
let resolver = ReferenceResolver {
wiki: fragment.facts(),
files: None,
};
assert!(
!wiki_model::revision_records(fragment.facts(), v1).is_empty(),
"the fixture's legacy versions must load, or this proves nothing"
);
let error = resolve_prefix(fragment.facts(), &format!("{anchor:x}"))
.unwrap_err()
.to_string();
assert!(error.contains("no Wiki id matches"), "got: {error}");
for content in [
format!("see #link(\"wiki:{anchor:x}\")[the page]\n"),
format!("#link(\"wiki:{anchor:x}\")[wiki:{anchor:x}]\n"),
format!("context: wiki:{anchor:x} says so\n"),
] {
assert_eq!(lint_fix(&content, resolver), content);
}
}
#[test]
fn lint_leaves_revision_citations_byte_unchanged() {
let (fragment, _anchor, v1, v2) = legacy_anchor_pair();
let resolver = ReferenceResolver {
wiki: fragment.facts(),
files: None,
};
for target in [v1, v2] {
let content = format!("cites #link(\"wiki:{target:x}\")[pinned]\n");
assert_eq!(lint_fix(&content, resolver), content);
}
let once = lint_fix(&format!("wiki:{}", &format!("{v2:x}")[..12]), resolver);
assert_eq!(once, format!("wiki:{v2:x}"));
assert_eq!(lint_fix(&once, resolver), once);
}
#[test]
fn lint_fix_mints_a_successor_and_never_edits_the_original() {
let fixture = Fixture::new();
let storage = fixture.storage();
let mut genesis = Fragment::empty();
let target = stage_revision(
storage,
&mut genesis,
None,
"target".to_owned(),
"body".to_owned(),
BTreeSet::new(),
)
.unwrap();
storage.publish(genesis).unwrap();
let truncated = format!("{target:x}")[..12].to_owned();
let mut fragment = Fragment::empty();
let citing = stage_revision(
storage,
&mut fragment,
None,
"source".to_owned(),
format!("see #link(\"wiki:{truncated}\")[the page]"),
BTreeSet::new(),
)
.unwrap();
storage.publish(fragment).unwrap();
cmd_lint(storage, true, false, &mut Out::new(&mut |_| Ok(()))).unwrap();
let after = storage.view(|view| Ok(view.clone())).unwrap();
let original = wiki_model::revision_records(&after.facts, citing)
.into_iter()
.next()
.unwrap();
assert_eq!(
read_string(&after.reader, original.content).unwrap(),
format!("see #link(\"wiki:{truncated}\")[the page]"),
"the original revision is content-addressed and must be untouched"
);
let entry = wiki_model::entry(&after.facts, &after.latest, citing).unwrap();
assert_eq!(entry.frontier.len(), 1);
let head = &entry.frontier[0];
assert_ne!(head.id, citing, "the fix is a successor, not a mutation");
assert_eq!(head.supersedes, BTreeSet::from([citing]));
assert_eq!(
read_string(&after.reader, head.content).unwrap(),
format!("see #link(\"wiki:{target:x}\")[the page]")
);
}
#[test]
fn a_failed_selector_says_whether_it_is_an_anchor_or_nothing() {
let fixture = Fixture::new();
let signer = load_signer(&fixture.pile, Some(&fixture.key)).unwrap();
let (author_fragment, _) = wiki_model::author_record(&signer.verifying_key());
let (legacy, anchor, _v1, _v2) = legacy_anchor_pair();
let mut pile =
crate::storage::open_pile_strict_as(&fixture.pile, signer.verifying_key()).unwrap();
let collection = crate::collection_names::open(
&mut pile,
schema::DEFAULT_SCOPE_ID,
signer.verifying_key(),
)
.unwrap();
pile.commit(collection, &signer, author_fragment + legacy)
.unwrap();
crate::wiki::carry_for_tests(&mut pile, &signer);
pile.close().unwrap();
let storage = fixture.storage();
let view = storage.view(|view| Ok(view.clone())).unwrap();
let anchor_hex = format!("{anchor:x}");
let reported = explain_selector(
&view,
&anchor_hex,
anyhow!("no Wiki id matches '{anchor_hex}'"),
)
.unwrap()
.to_string();
assert!(
reported.contains("LEGACY FRAGMENT ANCHOR"),
"an anchor must be named as one; got: {reported}"
);
let never = "ffffffffffffffffffffffffffffffff";
let reported = explain_selector(&view, never, anyhow!("no Wiki id matches '{never}'"))
.unwrap()
.to_string();
assert!(
reported.contains("no fragment has ever had it") && !reported.contains("ANCHOR"),
"an id no fragment ever had must not be called an anchor; got: {reported}"
);
}
}