1use crate::candidate::{ChangeClass, Judgement, Metric};
36use crate::config::Config;
37use crate::message::Effort;
38use anyhow::{Context, Result};
39use serde::{Deserialize, Serialize};
40use std::os::fd::AsRawFd;
41use std::path::{Path, PathBuf};
42
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
48pub enum OverrideKey {
49 CompactAtTokens,
50 MaxTurns,
51 MaxOutputTokens,
52 Effort,
53}
54
55impl OverrideKey {
56 pub const ALL: [OverrideKey; 4] = [
57 OverrideKey::CompactAtTokens,
58 OverrideKey::MaxTurns,
59 OverrideKey::MaxOutputTokens,
60 OverrideKey::Effort,
61 ];
62
63 pub fn parse(key: &str) -> Option<OverrideKey> {
64 match key {
65 "compact_at_tokens" => Some(OverrideKey::CompactAtTokens),
66 "max_turns" => Some(OverrideKey::MaxTurns),
67 "max_output_tokens" => Some(OverrideKey::MaxOutputTokens),
68 "effort" => Some(OverrideKey::Effort),
69 _ => None,
70 }
71 }
72
73 pub fn as_str(self) -> &'static str {
74 match self {
75 OverrideKey::CompactAtTokens => "compact_at_tokens",
76 OverrideKey::MaxTurns => "max_turns",
77 OverrideKey::MaxOutputTokens => "max_output_tokens",
78 OverrideKey::Effort => "effort",
79 }
80 }
81
82 pub fn names() -> String {
84 Self::ALL
85 .iter()
86 .map(|k| k.as_str())
87 .collect::<Vec<_>>()
88 .join(", ")
89 }
90}
91
92#[derive(Debug, Clone, PartialEq)]
96pub struct ConfigChange {
97 pub key: OverrideKey,
98 pub value: String,
99}
100
101pub fn parse_change(spec: &str) -> Result<ConfigChange> {
107 let (key, value) = spec
108 .split_once('=')
109 .with_context(|| format!("expected KEY=VALUE, got `{spec}`"))?;
110 let key = key.trim();
111 let value = value.trim();
112 let key = OverrideKey::parse(key).with_context(|| {
113 format!(
114 "`{key}` is not in the closed override set ({})",
115 OverrideKey::names()
116 )
117 })?;
118 let canonical = match key {
119 OverrideKey::CompactAtTokens => {
120 let n: u64 = value
121 .parse()
122 .with_context(|| format!("compact_at_tokens takes a number, got `{value}`"))?;
123 anyhow::ensure!(
124 n >= 1000,
125 "compact_at_tokens below 1000 would compact on nearly every turn"
126 );
127 n.to_string()
128 }
129 OverrideKey::MaxTurns => {
130 let n: u32 = value
131 .parse()
132 .with_context(|| format!("max_turns takes a number, got `{value}`"))?;
133 anyhow::ensure!(n >= 1, "max_turns must be at least 1");
134 n.to_string()
135 }
136 OverrideKey::MaxOutputTokens => {
137 let n: u64 = value
138 .parse()
139 .with_context(|| format!("max_output_tokens takes a number, got `{value}`"))?;
140 anyhow::ensure!(n >= 1, "max_output_tokens must be at least 1");
141 n.to_string()
142 }
143 OverrideKey::Effort => value
144 .parse::<Effort>()
145 .map_err(|e| anyhow::anyhow!("{e}"))?
146 .as_str()
147 .to_string(),
148 };
149 Ok(ConfigChange {
150 key,
151 value: canonical,
152 })
153}
154
155impl ConfigChange {
156 pub fn apply_to_agent(&self, agent: &mut crate::config::AgentConfig) -> Result<()> {
160 match self.key {
161 OverrideKey::CompactAtTokens => agent.compact_at_tokens = Some(self.value.parse()?),
162 OverrideKey::MaxTurns => agent.max_turns = self.value.parse()?,
163 OverrideKey::MaxOutputTokens => agent.max_output_tokens = Some(self.value.parse()?),
164 OverrideKey::Effort => {
165 agent.effort = Some(self.value.parse().map_err(|e| anyhow::anyhow!("{e}"))?)
166 }
167 }
168 Ok(())
169 }
170
171 pub fn spec(&self) -> String {
172 format!("{}={}", self.key.as_str(), self.value)
173 }
174}
175
176pub const STATUS_STAGED: &str = "staged";
181pub const STATUS_ACCEPTED: &str = "accepted";
183pub const STATUS_REJECTED: &str = "rejected";
185pub const STATUS_REVERTED: &str = "reverted";
187
188#[derive(Debug, Clone, Serialize, Deserialize)]
196pub struct HarnessCandidate {
197 pub id: String,
198 pub created_at: String,
199 pub class: ChangeClass,
200 pub change: String,
202 pub metric: Metric,
204 pub rationale: String,
205 pub evidence: String,
207 #[serde(default)]
209 pub model: Option<String>,
210 pub status: String,
212 #[serde(default)]
213 pub measurement: Option<Measurement>,
214 #[serde(default)]
215 pub resolved_at: Option<String>,
216 #[serde(default)]
218 pub reason: Option<String>,
219}
220
221impl HarnessCandidate {
222 pub fn pending(&self) -> bool {
224 self.status == STATUS_STAGED
225 }
226}
227
228#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
229pub struct TallyRecord {
230 pub wins: usize,
231 pub losses: usize,
232 pub ties: usize,
233}
234
235impl std::fmt::Display for TallyRecord {
236 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
237 write!(f, "{}+ {}- {}=", self.wins, self.losses, self.ties)
238 }
239}
240
241#[derive(Debug, Clone, Serialize, Deserialize)]
243pub struct Measurement {
244 pub measured_at: String,
245 pub model: String,
246 pub disposition: String,
249 pub reason: String,
251 pub selection: TallyRecord,
252 pub holdout: TallyRecord,
253 pub work_baseline: u64,
254 pub work_candidate: u64,
255 pub episodes: Vec<String>,
257 #[serde(default)]
269 pub holdout_episodes: Vec<String>,
270 #[serde(default)]
271 pub seed: u64,
272 pub diverged: Vec<String>,
280 #[serde(default, skip_serializing_if = "Vec::is_empty")]
291 pub replay_caveats: Vec<String>,
292 pub skipped: usize,
295}
296
297pub struct Drawn {
302 pub episodes: Vec<String>,
304 pub holdout_episodes: Vec<String>,
305 pub seed: u64,
306 pub diverged: Vec<String>,
307 pub replay_caveats: Vec<String>,
309 pub skipped: usize,
310}
311
312impl Measurement {
313 pub fn record(
314 judgement: &Judgement,
315 model: &str,
316 measured_at: String,
317 drawn: Drawn,
318 ) -> Measurement {
319 let Drawn {
320 episodes,
321 holdout_episodes,
322 seed,
323 diverged,
324 replay_caveats,
325 skipped,
326 } = drawn;
327 use crate::candidate::Disposition;
328 let (disposition, reason) = match &judgement.disposition {
329 Disposition::Accept => ("accept", String::new()),
330 Disposition::Propose(r) => ("propose", r.clone()),
331 Disposition::Reject(r) => ("reject", r.clone()),
332 };
333 let tally = |t: &crate::candidate::Tally| TallyRecord {
334 wins: t.wins,
335 losses: t.losses,
336 ties: t.ties,
337 };
338 Measurement {
339 measured_at,
340 model: model.to_string(),
341 disposition: disposition.to_string(),
342 reason,
343 selection: tally(&judgement.selection),
344 holdout: tally(&judgement.holdout),
345 work_baseline: judgement.work_baseline,
346 work_candidate: judgement.work_candidate,
347 episodes,
348 holdout_episodes,
349 seed,
350 diverged,
351 replay_caveats,
352 skipped,
353 }
354 }
355}
356
357#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
361pub struct AcceptedOverride {
362 pub key: String,
365 pub value: String,
366 pub candidate: String,
368 pub accepted_at: String,
369}
370
371#[derive(Debug, Default, Serialize, Deserialize)]
372struct OverridesFile {
373 #[serde(default, rename = "override")]
374 overrides: Vec<AcceptedOverride>,
375}
376
377pub struct HarnessStore {
379 root: PathBuf,
380}
381
382impl HarnessStore {
383 pub fn default_root() -> Result<PathBuf> {
386 Ok(crate::learning::LearningStore::default_root()?.join("harness"))
387 }
388
389 pub fn open(root: impl Into<PathBuf>) -> Result<HarnessStore> {
390 let root = root.into();
391 crate::create_private_dir(&root.join("candidates"))
392 .with_context(|| format!("creating {}", root.display()))?;
393 Ok(HarnessStore { root })
394 }
395
396 pub fn open_default() -> Result<HarnessStore> {
397 Self::open(Self::default_root()?)
398 }
399
400 pub fn root(&self) -> &Path {
401 &self.root
402 }
403
404 pub fn write(&self, c: &HarnessCandidate) -> Result<()> {
407 let path = self.root.join("candidates").join(format!("{}.json", c.id));
408 let tmp = path.with_extension("json.tmp");
409 std::fs::write(&tmp, serde_json::to_string_pretty(c)?)?;
410 std::fs::rename(&tmp, &path)?;
411 Ok(())
412 }
413
414 pub fn all(&self) -> Result<Vec<HarnessCandidate>> {
417 let dir = self.root.join("candidates");
418 if !dir.is_dir() {
419 return Ok(Vec::new());
420 }
421 let mut out = Vec::new();
422 for entry in std::fs::read_dir(&dir)? {
423 let path = entry?.path();
424 if path.extension().and_then(|e| e.to_str()) != Some("json") {
425 continue;
426 }
427 match serde_json::from_str(&std::fs::read_to_string(&path)?) {
428 Ok(c) => out.push(c),
429 Err(e) => tracing::warn!("skipping unreadable candidate {}: {e}", path.display()),
430 }
431 }
432 out.sort_by(|a: &HarnessCandidate, b: &HarnessCandidate| a.id.cmp(&b.id));
433 Ok(out)
434 }
435
436 pub fn find(&self, id: &str) -> Result<HarnessCandidate> {
439 let all = self.all()?;
440 let matches: Vec<&HarnessCandidate> = all.iter().filter(|c| c.id.starts_with(id)).collect();
441 match matches.len() {
442 0 => anyhow::bail!("no candidate matching `{id}`"),
443 1 => Ok(matches[0].clone()),
444 n => anyhow::bail!(
445 "`{id}` matches {n} candidates: {}",
446 matches
447 .iter()
448 .map(|c| c.id.as_str())
449 .collect::<Vec<_>>()
450 .join(", ")
451 ),
452 }
453 }
454
455 pub fn overrides_path(&self) -> PathBuf {
456 self.root.join("overrides.toml")
457 }
458
459 pub fn overrides(&self) -> Result<Vec<AcceptedOverride>> {
461 let path = self.overrides_path();
462 if !path.exists() {
463 return Ok(Vec::new());
464 }
465 let text = std::fs::read_to_string(&path)
466 .with_context(|| format!("reading {}", path.display()))?;
467 let file: OverridesFile =
468 toml::from_str(&text).with_context(|| format!("parsing {}", path.display()))?;
469 Ok(file.overrides)
470 }
471
472 pub fn set_override(&self, ov: AcceptedOverride) -> Result<Option<AcceptedOverride>> {
475 let _lock = self.lock()?;
476 let mut all = self.overrides()?;
477 let replaced = all
478 .iter()
479 .position(|o| o.key == ov.key)
480 .map(|i| all.remove(i));
481 all.push(ov);
482 self.write_overrides(&all)?;
483 Ok(replaced)
484 }
485
486 pub fn remove_override(&self, key: &str) -> Result<Option<AcceptedOverride>> {
490 let _lock = self.lock()?;
491 let mut all = self.overrides()?;
492 let removed = all.iter().position(|o| o.key == key).map(|i| all.remove(i));
493 if removed.is_some() {
494 self.write_overrides(&all)?;
495 }
496 Ok(removed)
497 }
498
499 fn write_overrides(&self, all: &[AcceptedOverride]) -> Result<()> {
500 let path = self.overrides_path();
501 let file = OverridesFile {
502 overrides: all.to_vec(),
503 };
504 let tmp = path.with_extension("toml.tmp");
505 std::fs::write(&tmp, toml::to_string_pretty(&file)?)?;
506 std::fs::rename(&tmp, &path)?;
507 Ok(())
508 }
509
510 fn lock(&self) -> Result<std::fs::File> {
513 let file = std::fs::OpenOptions::new()
514 .create(true)
515 .truncate(false)
516 .write(true)
517 .open(self.root.join(".lock"))?;
518 if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) } != 0 {
520 return Err(std::io::Error::last_os_error()).context("locking the harness store");
521 }
522 Ok(file)
523 }
524
525 pub fn mint_id() -> String {
528 let now = chrono::Utc::now();
529 format!(
530 "hc-{}-{:04x}",
531 now.format("%Y%m%dT%H%M%S"),
532 (now.timestamp_subsec_nanos() ^ std::process::id()) & 0xffff
533 )
534 }
535}
536
537pub fn apply_accepted_overrides(cfg: &mut Config) {
549 let Ok(root) = HarnessStore::default_root() else {
550 return;
551 };
552 apply_overrides_file(cfg, &root.join("overrides.toml"));
553}
554
555pub fn apply_overrides_file(cfg: &mut Config, path: &Path) {
557 if !path.exists() {
558 return;
559 }
560 let text = match std::fs::read_to_string(path) {
561 Ok(t) => t,
562 Err(e) => {
563 tracing::warn!("harness overrides unreadable ({}): {e}", path.display());
564 return;
565 }
566 };
567 let file: OverridesFile = match toml::from_str(&text) {
568 Ok(f) => f,
569 Err(e) => {
570 tracing::warn!(
571 "harness overrides malformed ({}): {e} — applying none",
572 path.display()
573 );
574 return;
575 }
576 };
577 for ov in file.overrides {
578 let change = match parse_change(&format!("{}={}", ov.key, ov.value)) {
581 Ok(c) => c,
582 Err(e) => {
583 tracing::warn!(
584 "harness override `{}={}` skipped: {e:#} (from candidate {})",
585 ov.key,
586 ov.value,
587 ov.candidate
588 );
589 continue;
590 }
591 };
592 if let Err(e) = change.apply_to_agent(&mut cfg.agent) {
593 tracing::warn!(
594 "harness override `{}` failed to apply: {e:#}",
595 change.spec()
596 );
597 }
598 }
599}
600
601#[cfg(test)]
602mod tests {
603 use super::*;
604
605 fn temp_dir() -> std::path::PathBuf {
606 let dir = std::env::temp_dir()
607 .join("mecha-harness-test")
608 .join(uuid::Uuid::new_v4().to_string());
609 std::fs::create_dir_all(&dir).unwrap();
610 dir
611 }
612
613 #[test]
614 fn the_closed_set_refuses_everything_outside_it() {
615 assert!(parse_change("compact_at_tokens=24000").is_ok());
617 assert!(parse_change("max_turns=30").is_ok());
618 assert!(parse_change("max_output_tokens=8000").is_ok());
619 assert!(parse_change("effort=low").is_ok());
620
621 for hostile in [
623 "sandbox=none",
624 "trifecta=allow",
625 "outbox.tools=",
626 "context_window=999999",
627 "temperature=2.0",
628 ] {
629 assert!(parse_change(hostile).is_err(), "{hostile} must be refused");
630 }
631 assert!(parse_change("just prose").is_err());
633 }
634
635 #[test]
636 fn values_are_validated_not_just_typed() {
637 assert!(parse_change("compact_at_tokens=1").is_err());
638 assert!(parse_change("max_turns=0").is_err());
639 assert!(parse_change("max_turns=notanumber").is_err());
640 assert!(parse_change("effort=extreme").is_err());
641 let c = parse_change(" effort = LOW ").unwrap();
643 assert_eq!(c.value, "low");
644 }
645
646 #[test]
647 fn overrides_apply_beneath_the_user_and_unknown_keys_are_skipped() {
648 let dir = temp_dir();
649 let path = dir.join("overrides.toml");
650 std::fs::write(
651 &path,
652 r#"
653[[override]]
654key = "compact_at_tokens"
655value = "24000"
656candidate = "hc-test"
657accepted_at = "2026-08-22T00:00:00Z"
658
659[[override]]
660key = "sandbox"
661value = "none"
662candidate = "hc-evil"
663accepted_at = "2026-08-22T00:00:00Z"
664
665[[override]]
666key = "max_turns"
667value = "notanumber"
668candidate = "hc-corrupt"
669accepted_at = "2026-08-22T00:00:00Z"
670"#,
671 )
672 .unwrap();
673
674 let mut cfg = Config::default();
675 apply_overrides_file(&mut cfg, &path);
676 assert_eq!(cfg.agent.compact_at_tokens, Some(24000));
678 assert_eq!(cfg.agent.max_turns, 40);
681 }
682
683 #[test]
684 fn a_malformed_overrides_file_applies_nothing() {
685 let dir = temp_dir();
686 let path = dir.join("overrides.toml");
687 std::fs::write(&path, "this is not toml [[[").unwrap();
688 let mut cfg = Config::default();
689 let before = cfg.agent.max_turns;
690 apply_overrides_file(&mut cfg, &path);
691 assert_eq!(cfg.agent.max_turns, before);
692 }
693
694 #[test]
695 fn set_and_remove_override_round_trip_and_replacement_is_returned() {
696 let dir = temp_dir();
697 let store = HarnessStore::open(&dir).unwrap();
698
699 let first = AcceptedOverride {
700 key: "compact_at_tokens".into(),
701 value: "24000".into(),
702 candidate: "hc-a".into(),
703 accepted_at: "2026-08-22T00:00:00Z".into(),
704 };
705 assert!(store.set_override(first.clone()).unwrap().is_none());
706
707 let second = AcceptedOverride {
710 key: "compact_at_tokens".into(),
711 value: "20000".into(),
712 candidate: "hc-b".into(),
713 ..first.clone()
714 };
715 let replaced = store.set_override(second).unwrap();
716 assert_eq!(replaced.unwrap().candidate, "hc-a");
717
718 let all = store.overrides().unwrap();
719 assert_eq!(all.len(), 1);
720 assert_eq!(all[0].value, "20000");
721
722 let mut cfg = Config::default();
724 apply_overrides_file(&mut cfg, &store.overrides_path());
725 assert_eq!(cfg.agent.compact_at_tokens, Some(20000));
726
727 let removed = store.remove_override("compact_at_tokens").unwrap();
729 assert_eq!(removed.unwrap().value, "20000");
730 let mut cfg = Config::default();
731 apply_overrides_file(&mut cfg, &store.overrides_path());
732 assert_eq!(cfg.agent.compact_at_tokens, None);
733 assert!(store
735 .remove_override("compact_at_tokens")
736 .unwrap()
737 .is_none());
738 }
739
740 #[test]
741 fn candidates_round_trip_and_an_unknown_status_still_loads() {
742 let dir = temp_dir();
743 let store = HarnessStore::open(&dir).unwrap();
744 let c = HarnessCandidate {
745 id: "hc-20260822T000000-0001".into(),
746 created_at: "2026-08-22T00:00:00Z".into(),
747 class: ChangeClass::Config,
748 change: "compact_at_tokens=24000".into(),
749 metric: Metric::CutShort,
750 rationale: "runs are dying at the ceiling".into(),
751 evidence: "runs: 64".into(),
752 model: Some("qwen3.6-35b-a3b".into()),
753 status: STATUS_STAGED.into(),
754 measurement: None,
755 resolved_at: None,
756 reason: None,
757 };
758 store.write(&c).unwrap();
759 let read = store.find("hc-2026").unwrap();
760 assert_eq!(read.change, c.change);
761 assert!(read.pending());
762
763 let mut future = c.clone();
765 future.id = "hc-20260823T000000-0002".into();
766 future.status = "escalated".into();
767 store.write(&future).unwrap();
768 assert_eq!(store.all().unwrap().len(), 2);
769 }
770}