use crate::fed;
mod shell;
pub use crate::split::{
Evidence, Proposal, Proposed, Ranked, propose, rank, row_weight, structure,
uniform_evidence,
};
use std::path::{Path, PathBuf};
pub const HORIZON: usize = 3;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Task {
pub node: PathBuf,
pub id: String,
pub status: char,
pub text: String,
pub cites: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kind {
NodeFn,
Cli,
MultiFile,
NotCode,
Replaces,
NotAFunction,
NoModule,
Frozen,
}
impl Kind {
#[must_use]
pub fn actionable(self) -> bool {
self == Kind::NodeFn
}
#[must_use]
pub fn why(self) -> &'static str {
match self {
Kind::NodeFn => "single-node function",
Kind::Cli => "needs main.rs -- tdd edits one node's mod.rs only",
Kind::MultiFile => "writes files beyond its own module",
Kind::NotCode => "research or reporting, not code",
Kind::Replaces => "replaces existing code -- the loop only appends",
Kind::NotAFunction => {
"no new function to add -- edits specs or wiring"
}
Kind::NoModule => "root row -- no mod.rs to add to",
Kind::Frozen => "node FROZEN by the root spec until its rung lands",
}
}
}
#[must_use]
pub fn frozen_nodes(root: &Path) -> Vec<PathBuf> {
let Ok(text) = std::fs::read_to_string(root.join("SPEC.md")) else {
return Vec::new();
};
text.lines()
.filter(|l| l.contains("FROZEN"))
.flat_map(|l| l.split('`').skip(1).step_by(2))
.map(|p| root.join(p))
.filter(|p| fed::is_node(p))
.collect()
}
#[must_use]
pub fn classify(node: &Path, text: &str) -> Kind {
if !node.join("mod.rs").is_file() {
return Kind::NoModule;
}
let t = text.to_lowercase();
let has = |ks: &[&str]| ks.iter().any(|k| t.contains(k));
let words: Vec<&str> = t
.split(|c: char| !c.is_ascii_alphanumeric())
.filter(|w| !w.is_empty())
.collect();
let word = |ks: &[&str]| {
ks.iter().any(|k| {
let stem = k.trim_end_matches('e');
words.iter().any(|w| w.starts_with(stem))
})
};
if word(&[
"replace",
"remove",
"port",
"migrate",
"rewrite",
"delete",
"supersede",
"around",
"wrap",
"inside",
]) {
Kind::Replaces
} else if word(&[
"blocked", "needs", "promote", "wire", "move", "record", "flip",
"plant",
]) {
Kind::NotAFunction
} else if has(&["cmd", "cli", "verb", "`sherd ", "flag", "--"]) {
Kind::Cli
} else if word(&["upstream", "audit", "corpus", "fixture"])
|| has(&["ci ", "gate:"])
{
Kind::NotCode
} else if has(&["\u{a7}n", "sync", "baseline", ".md`", "write own"]) {
Kind::MultiFile
} else {
Kind::NodeFn
}
}
#[must_use]
pub fn unreadable(root: &Path) -> Vec<(PathBuf, String)> {
fed::discover(root)
.into_iter()
.filter_map(|n| {
let p = n.join("SPEC.md");
std::fs::read_to_string(&p)
.err()
.map(|e| (p, e.to_string()))
})
.collect()
}
#[must_use]
pub fn open_tasks(root: &Path) -> Vec<Task> {
let mut out = Vec::new();
for node in fed::discover(root) {
let Ok(text) = std::fs::read_to_string(node.join("SPEC.md")) else {
continue;
};
let mut in_t = false;
for line in text.lines() {
if line.starts_with("## \u{a7}") {
in_t = line.starts_with("## \u{a7}T");
continue;
}
if !in_t {
continue;
}
let cells: Vec<&str> = line.split('|').collect();
let [id, state, text, ..] = cells.as_slice() else {
continue;
};
let status = state.chars().next().unwrap_or(' ');
if !id.starts_with('T') || (status != '.' && status != '~') {
continue;
}
out.push(Task {
node: node.strip_prefix(root).unwrap_or(&node).to_path_buf(),
id: (*id).to_string(),
status,
text: (*text).to_string(),
cites: cells.get(3).unwrap_or(&"-").to_string(),
});
}
}
out.sort_by_key(|t| {
(t.node.components().count(), t.node.clone(), t.id.clone())
});
out
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Confidence {
Next,
Likely,
Tentative,
}
impl Confidence {
#[must_use]
pub fn of(step: usize) -> Self {
match step {
0 => Confidence::Next,
1 => Confidence::Likely,
_ => Confidence::Tentative,
}
}
#[must_use]
pub fn label(self) -> &'static str {
match self {
Confidence::Next => "NEXT ",
Confidence::Likely => "LIKELY ",
Confidence::Tentative => "TENTATIVE ",
}
}
#[must_use]
pub fn invalidated_by(self) -> &'static str {
match self {
Confidence::Next => {
"judge rejects the test 3x · gate still red after 3 repairs"
}
Confidence::Likely => {
"step 1 adds a §B row to this node, changing its authoring prompt (tdd B9)"
}
Confidence::Tentative => {
"any §T row added by steps above it; ordering has no declared `needs`"
}
}
}
#[must_use]
pub fn invalidators(self) -> &'static [&'static str] {
match self {
Confidence::Next => &[
"judge rejects the test 3x",
"gate still red after 3 repairs",
],
Confidence::Likely => &[
"step 1 adds a §B row to this node, changing its authoring prompt (tdd B9)",
],
Confidence::Tentative => &[
"any §T row added by steps above it; ordering has no declared `needs`",
],
}
}
}
#[must_use]
pub fn row_key(t: &Task) -> String {
format!("{}:{}", t.node.display(), t.id)
}
#[must_use]
pub fn row_hash(t: &Task) -> String {
crate::state::content_hash(t.text.as_bytes())
}
#[must_use]
pub fn already_applied(st: &crate::state::State, t: &Task) -> bool {
st.get("applied", &row_key(t))
.is_some_and(|v| v.starts_with(&row_hash(t)))
}
#[derive(Debug)]
pub struct Plan {
pub steps: Vec<Task>,
pub unmanaged: Vec<(Task, Kind)>,
pub total_open: usize,
}
#[must_use]
pub fn in_milestone(
root: &Path,
rows: Vec<Task>,
milestone: &str,
) -> (Vec<Task>, usize) {
let mut parts: std::collections::BTreeMap<
PathBuf,
Vec<(String, Vec<u32>)>,
> = std::collections::BTreeMap::new();
let mut kept = Vec::new();
let mut undeclared = 0usize;
for t in rows {
let node = parts.entry(t.node.clone()).or_insert_with(|| {
std::fs::read_to_string(root.join(&t.node).join("SPEC.md"))
.map(|s| crate::spec::milestones(&s))
.unwrap_or_default()
});
if node.is_empty() {
undeclared = undeclared.saturating_add(1);
continue;
}
let claimed = task_number(&t.id).is_some_and(|n| {
node.iter()
.any(|(m, tasks)| m == milestone && tasks.contains(&n))
});
if claimed {
kept.push(t);
}
}
(kept, undeclared)
}
fn task_number(id: &str) -> Option<u32> {
let digits: String = id
.strip_prefix('T')?
.chars()
.take_while(char::is_ascii_digit)
.collect();
digits.parse().ok()
}
#[must_use]
pub fn milestone_declared(root: &Path, milestone: &str) -> bool {
fed::discover(root).iter().any(|n| {
std::fs::read_to_string(n.join("SPEC.md")).is_ok_and(|s| {
crate::spec::milestones(&s)
.iter()
.any(|(m, _)| m == milestone)
})
})
}
#[must_use]
pub fn plan(root: &Path) -> Plan {
plan_in(root, None).0
}
#[must_use]
pub fn plan_in(root: &Path, milestone: Option<&str>) -> (Plan, usize) {
let (all, outside) = match milestone {
Some(m) => in_milestone(root, open_tasks(root), m),
None => (open_tasks(root), 0),
};
let total_open = all.len();
let mut steps = Vec::new();
let mut unmanaged = Vec::new();
let st = crate::state::State::load();
let frozen = frozen_nodes(root);
let scripts = shell::Scripts::read(root);
let mut candidates = Vec::new();
for mut t in all {
let shell = shell::home(root, &t, &scripts);
if let shell::Home::At(node) = &shell {
t.node.clone_from(node);
}
let dir = root.join(&t.node);
let k = match shell {
_ if frozen.contains(&dir) => Kind::Frozen,
shell::Home::At(_) => Kind::NodeFn,
shell::Home::Span => Kind::MultiFile,
shell::Home::NotShell => classify(&dir, &t.text),
};
if k.actionable() && already_applied(&st, &t) {
continue; }
if k.actionable() {
candidates.push(t);
} else {
unmanaged.push((t, k));
}
}
candidates.sort_by(|a, b| {
believability(&b.node)
.total_cmp(&believability(&a.node))
.then_with(|| {
a.node
.components()
.count()
.cmp(&b.node.components().count())
})
.then_with(|| a.id.cmp(&b.id))
});
steps.extend(candidates.into_iter().take(HORIZON));
(
Plan {
steps,
unmanaged,
total_open,
},
outside,
)
}
#[must_use]
pub fn to_json(
st: &crate::state::State,
p: &Plan,
(milestone, outside): (Option<&str>, usize),
context_tokens: &[u64],
) -> String {
to_json_with(st, p, (milestone, outside), context_tokens, &[])
}
#[must_use]
pub fn to_json_with(
st: &crate::state::State,
p: &Plan,
(milestone, outside): (Option<&str>, usize),
context_tokens: &[u64],
touches: &[Option<Vec<String>>],
) -> String {
let steps: Vec<String> = (1usize..)
.zip(&p.steps)
.map(|(rank, t)| {
let c = Confidence::of(rank.saturating_sub(1));
let (tried, kept) = record_in(st, &t.node);
let foot = touches.get(rank.saturating_sub(1)).and_then(Option::as_deref);
let why: Vec<String> = invalidators_of(c, t, foot)
.iter()
.map(|w| json_str(w))
.collect();
format!(
"{{\"rank\":{rank},\"kind\":{},\"node\":{},\"id\":{},\"text\":{},\
\"believability\":{},\"tried\":{tried},\"kept\":{kept},\
\"context_tokens\":{},\"invalidated_by\":[{}]{}}}",
json_str(c.label().trim()),
json_node(&t.node),
json_str(&t.id),
json_str(&t.text),
believability_in(st, &t.node),
context_tokens
.get(rank.saturating_sub(1))
.copied()
.unwrap_or(0),
why.join(","),
foot.map_or_else(String::new, touches_key)
)
})
.collect();
let unmanaged: Vec<String> = p
.unmanaged
.iter()
.map(|(t, k)| {
format!(
"{{\"node\":{},\"id\":{},\"text\":{},\"reason\":{}}}",
json_node(&t.node),
json_str(&t.id),
json_str(&t.text),
json_str(k.why())
)
})
.collect();
format!(
"{{\"horizon\":{},\"open\":{},\"unmanaged\":{},\"milestone\":{},\
\"outside_milestones\":{outside},\"steps\":[{}],\"unmanaged_rows\":[{}]}}",
p.steps.len(),
p.total_open,
p.unmanaged.len(),
milestone.map_or_else(|| "null".to_string(), json_str),
steps.join(","),
unmanaged.join(",")
)
}
fn touches_key(scripts: &[String]) -> String {
if scripts.is_empty() {
return ",\"touches\":null".into();
}
let list: Vec<String> = scripts.iter().map(|s| json_str(s)).collect();
format!(",\"touches\":[{}]", list.join(","))
}
#[must_use]
pub fn footprint(root: &Path, t: &Task) -> Option<Vec<String>> {
if root.join(&t.node).join("mod.rs").is_file() {
return None;
}
Some(shell::touches(root, t, &shell::Scripts::read(root)))
}
#[must_use]
pub fn invalidators_of(
c: Confidence,
t: &Task,
touches: Option<&[String]>,
) -> Vec<String> {
let Some(touches) = touches else {
return c.invalidators().iter().map(|w| (*w).to_string()).collect();
};
let mut out: Vec<String> =
touches.iter().map(|s| format!("a change to {s}")).collect();
if touches.is_empty() {
out.push(format!("a change to any script under {}", t.node.display()));
}
out.extend(
shell::cited_rows(t)
.iter()
.map(|r| format!("a change to {r}")),
);
if c == Confidence::Tentative {
out.extend(c.invalidators().iter().map(|w| (*w).to_string()));
}
out
}
pub(crate) fn json_node(node: &Path) -> String {
if node.as_os_str().is_empty() {
json_str(".")
} else {
json_str(&node.to_string_lossy())
}
}
pub(crate) fn json_str(s: &str) -> String {
let mut out = String::with_capacity(s.len().saturating_add(2));
out.push('"');
for c in s.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
c if c.is_control() && u32::from(c) < 0x20 => {
out.push_str(&format!("\\u{:04x}", u32::from(c)));
}
c => out.push(c),
}
}
out.push('"');
out
}
#[must_use]
pub fn cited_invariant(t: &Task) -> Option<(std::path::PathBuf, String)> {
for raw in t.cites.split(',') {
let c = raw.trim().trim_matches('`');
let (owner, id) = match c.rsplit_once(':') {
Some((path, id)) => (path, id),
None => ("", c),
};
if id.starts_with('V')
&& id.len() > 1
&& id[1..].chars().all(|d| d.is_ascii_digit())
{
let node = match owner {
"" => t.node.clone(), "." => std::path::PathBuf::new(), p => std::path::PathBuf::from(p),
};
return Some((node, id.to_string()));
}
}
None
}
#[cfg(test)]
#[path = "tests/plan.rs"]
mod tests;
pub fn record_outcome_in(
st: &mut crate::state::State,
node: &Path,
kept: bool,
) {
let key = node.to_string_lossy().to_string();
let mut bump = |k: &str| {
let n = st.get_u64("score", &format!("{key}.{k}")).unwrap_or(0) + 1;
st.set("score", &format!("{key}.{k}"), n.to_string());
};
bump("tried");
if kept {
bump("kept");
}
}
pub fn record_outcome(node: &Path, kept: bool) {
let mut st = crate::state::State::load();
record_outcome_in(&mut st, node, kept);
st.save();
}
#[must_use]
pub fn believability_in(st: &crate::state::State, node: &Path) -> f64 {
let (tried, kept) = record_in(st, node);
#[allow(clippy::cast_precision_loss)]
{
(kept as f64 + 1.0) / (tried as f64 + 2.0)
}
}
#[must_use]
pub fn believability(node: &Path) -> f64 {
believability_in(&crate::state::State::load(), node)
}
#[must_use]
pub fn record_in(st: &crate::state::State, node: &Path) -> (u64, u64) {
let key = node.to_string_lossy().to_string();
(
st.get_u64("score", &format!("{key}.tried")).unwrap_or(0),
st.get_u64("score", &format!("{key}.kept")).unwrap_or(0),
)
}
#[must_use]
pub fn record(node: &Path) -> (u64, u64) {
record_in(&crate::state::State::load(), node)
}
#[must_use]
pub fn triage(root: &Path) -> Vec<(Task, Kind, Proposal)> {
open_tasks(root)
.into_iter()
.map(|t| {
let k = classify(&root.join(&t.node), &t.text);
let mut p = propose(&t.text);
if let Proposal::Move(n) = p
&& t.node.file_name().is_some_and(|f| f == n)
{
p = Proposal::Keep;
}
(t, k, p)
})
.filter(|(_, k, _)| !k.actionable())
.collect()
}
#[cfg(feature = "ollama")]
fn preflight(root: &Path) -> Result<String, String> {
let git = |args: &[&str]| {
crate::git::at(root, args)
.output()
.ok()
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
};
let branch =
git(&["rev-parse", "--abbrev-ref", "HEAD"]).unwrap_or_default();
if branch.is_empty() {
return Err("not a git repo -- apply commits, so it needs one".into());
}
if !git(&["status", "--porcelain"])
.unwrap_or_default()
.is_empty()
{
return Err("working tree dirty -- commit or stash first, so the \
generated diff is the only thing in the commit"
.into());
}
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_err(|e| e.to_string())?
.as_secs();
let want = crate::land::run_branch(&branch, secs);
if want != branch {
crate::git::at(root, &["checkout", "-q", "-b", &want])
.status()
.map_err(|e| e.to_string())?;
eprintln!("apply: on {branch} -- generated code goes to {want}");
return Ok(want);
}
Ok(branch)
}
#[cfg(feature = "ollama")]
pub fn apply(root: &Path, max_repair: usize) -> Result<String, String> {
let branch = preflight(root)?;
if let Some((p, e)) = unreadable(root).first() {
return Err(format!(
"{}: cannot read -- {e}. apply refuses to plan around a node it could not read (V27)",
p.display()
));
}
let p = plan(root);
let step = p
.steps
.iter()
.find(|t| footprint(root, t).is_none())
.ok_or("nothing actionable to apply -- shell steps are planned, not applied")?;
let (owner, inv) = cited_invariant(step).ok_or_else(|| {
format!(
"{} {} cites no §V id ({}) -- apply drives an INVARIANT, not prose",
step.node.display(),
step.id,
step.cites
)
})?;
eprintln!(
"apply: {} {} on {branch}\n invariant {inv} (declared in {})\n task {}",
step.node.display(),
step.id,
if owner.as_os_str().is_empty() {
".".into()
} else {
owner.display().to_string()
},
step.text
);
let node = root.join(&step.node);
let log = match crate::tdd::drive_from(
root,
&node,
&root.join(&owner),
&inv,
&step.text,
max_repair,
) {
Ok(l) => l,
Err(e) => {
record_outcome(&step.node, false);
return Err(e);
}
};
let sent: u64 = log.iter().map(|s| s.prompt_tokens).sum();
let max = log.iter().map(|s| s.prompt_tokens).max().unwrap_or(0);
let body = format!(
"feat({}): {} via sherd apply\n\n\
{inv}: driven from {} {}.\n\n\
Written by a local model through the red -> judge -> green -> gate\n\
loop: {} round-trips, {sent} tokens sent, max single call {max}.\n\
The gate ({}) passed before this commit existed.\n\n\
Generated code. Read the diff -- a green gate is not correctness\n\
(.:tdd V11), and this loop has twice produced code that passed both\n\
gates while testing the wrong thing.\n",
step.node
.file_name()
.and_then(|s| s.to_str())
.unwrap_or("sherd"),
step.text,
step.node.display(),
step.id,
log.len(),
"cargo test + sherd check"
);
let git = |args: &[&str]| {
crate::git::at(root, args)
.status()
.map_err(|e| e.to_string())
};
git(&["add", "-A"])?;
let st = crate::git::at(root, &["commit", "-q", "-m", &body])
.status()
.map_err(|e| e.to_string())?;
if !st.success() {
return Err(
"commit refused by the gate -- generated code is still in \
the tree, uncommitted"
.into(),
);
}
let sha = crate::git::at(root, &["rev-parse", "--short", "HEAD"])
.output()
.map_err(|e| e.to_string())?;
let sha = String::from_utf8_lossy(&sha.stdout).trim().to_string();
match crate::review::commit(root, &sha) {
Ok(f) if f.is_empty() => eprintln!(" review: no findings"),
Ok(f) => {
eprintln!(
"\n REVIEW DISAGREES WITH THE GATE -- {} finding(s):",
f.len()
);
for (file, x) in &f {
eprintln!(" {}: {}: {}", file.display(), x.rule, x.detail);
}
eprintln!(
" advisory (review V3). Read the diff before `sherd outcome ... kept`."
);
}
Err(e) => eprintln!(" review: could not run -- {e}"),
}
crate::land::push_branch(root, &branch);
record_outcome(&step.node, false);
let mut state = crate::state::State::load();
state.set(
"applied",
&row_key(step),
format!("{} {sha}", row_hash(step)),
);
state.clear_kind("plan"); state.save();
Ok(sha)
}
#[cfg(test)]
#[path = "tests/git.rs"]
mod git_tests;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Route {
Hit(std::path::PathBuf, Vec<String>),
Ambiguous(Vec<std::path::PathBuf>),
Miss,
}
fn node_words(node: &Path) -> Vec<String> {
let mut words = Vec::new();
if let Some(name) = node.file_name().and_then(|n| n.to_str()) {
words.push(name.to_lowercase());
}
if let Ok(text) = std::fs::read_to_string(node.join("SPEC.md")) {
words.extend(goal_words(&text));
}
words.sort();
words.dedup();
words
}
#[must_use]
pub fn vocabulary(root: &Path) -> Vec<(std::path::PathBuf, Vec<String>)> {
fed::discover(root)
.into_iter()
.map(|node| {
let words = node_words(&node);
(node, words)
})
.collect()
}
fn goal_words(spec: &str) -> Vec<String> {
crate::spec::sections(spec)
.into_iter()
.find(|(name, _)| {
name.trim_start_matches('#')
.trim_start()
.starts_with("\u{a7}G")
})
.map(|(_, body)| {
body.split(|c: char| !c.is_alphanumeric())
.filter(|w| w.chars().count() >= 4)
.map(str::to_lowercase)
.collect()
})
.unwrap_or_default()
}
#[must_use]
fn route_hits(
root: &Path,
terms: &[&str],
) -> Vec<(std::path::PathBuf, Vec<String>)> {
let mut hits: Vec<(std::path::PathBuf, Vec<String>)> = Vec::new();
for (node, words) in vocabulary(root) {
if node == root {
continue;
}
let matched: Vec<String> = words
.into_iter()
.filter(|w| terms.iter().any(|t| t == w))
.collect();
if !matched.is_empty() {
hits.push((node, matched));
}
}
hits
}
#[must_use]
pub fn route(root: &Path, query: &str) -> Route {
let q = query.to_lowercase();
let terms: Vec<&str> = q
.split(|c: char| !c.is_alphanumeric())
.filter(|w| w.chars().count() >= 4)
.collect();
let mut hits = route_hits(root, &terms);
let best = hits.iter().map(|(_, w)| w.len()).max().unwrap_or(0);
hits.retain(|(_, w)| w.len() == best);
match hits.len() {
0 => Route::Miss,
1 => hits
.pop()
.map_or(Route::Miss, |(node, why)| Route::Hit(node, why)),
_ => Route::Ambiguous(hits.into_iter().map(|(n, _)| n).collect()),
}
}
#[cfg(test)]
#[path = "tests/route.rs"]
mod route_tests;