codewhale_workflow/
experimental_search.rs

1//! Provider-neutral experimental search authoring for Workflow.
2//!
3//! This module is an authoring and freeze boundary, not a new runtime or
4//! scheduler. A validated search still has to be lowered by the Workflow host
5//! into Fleet workers plus a runtime-owned evaluator. In particular, worker
6//! self-reports are never promoted to hard-gate evidence here.
7
8use serde::{Deserialize, Serialize};
9use sha2::{Digest, Sha256};
10use std::path::{Component, Path};
11use thiserror::Error;
12
13use crate::{DEFAULT_FLEET_WORKFLOW_MAX_AGENTS, experimental_search::SearchSpecError::*};
14
15pub const WORKFLOW_SEARCH_SCHEMA_VERSION: u32 = 1;
16/// Live-worker ceiling for one search admission batch.
17///
18/// 16 matches the Workflow host's live-child ceiling today
19/// (`codewhale_workflow_js::WORKFLOW_MAX_CONCURRENT`, from which the tui
20/// driver sizes its per-run admission semaphore). This crate cannot import
21/// that constant directly because `codewhale-workflow-js` depends on
22/// `codewhale-workflow`, so 16 is documented here as today's default — not a
23/// new configuration knob. Keep it in sync with the host constant.
24pub const WORKFLOW_SEARCH_MAX_CONCURRENT: u16 = 16;
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub struct WorkflowSearchSpec {
28    #[serde(default = "default_schema_version")]
29    pub schema_version: u32,
30    pub name: String,
31    pub objective: String,
32    pub population: u16,
33    pub rounds: Vec<u16>,
34    pub concurrency: u16,
35    pub worker: SearchWorkerSpec,
36    #[serde(default)]
37    pub budget: SearchBudgetSpec,
38    pub hard_gates: SearchHardGateSpec,
39    pub score: SearchScoreSpec,
40    #[serde(default)]
41    pub selection: SearchSelectionSpec,
42    #[serde(default)]
43    pub integration_policy: SearchIntegrationPolicy,
44}
45
46impl WorkflowSearchSpec {
47    pub fn from_toml(source: &str) -> Result<Self, SearchSpecError> {
48        let spec: Self = toml::from_str(source).map_err(|error| Parse(error.to_string()))?;
49        spec.validate()?;
50        Ok(spec)
51    }
52
53    pub fn validate(&self) -> Result<(), SearchSpecError> {
54        if self.schema_version != WORKFLOW_SEARCH_SCHEMA_VERSION {
55            return Err(UnsupportedSchemaVersion(self.schema_version));
56        }
57        validate_name(&self.name)?;
58        validate_text("objective", &self.objective, 32_768)?;
59        if !(2..=DEFAULT_FLEET_WORKFLOW_MAX_AGENTS as u16).contains(&self.population) {
60            return Err(InvalidPopulation(self.population));
61        }
62        if self.concurrency == 0
63            || self.concurrency > WORKFLOW_SEARCH_MAX_CONCURRENT
64            || self.concurrency > self.population
65        {
66            return Err(InvalidConcurrency {
67                concurrency: self.concurrency,
68                population: self.population,
69            });
70        }
71        validate_rounds(self.population, &self.rounds)?;
72        validate_text("worker.model", &self.worker.model, 256)?;
73        if self.worker.write_roots.is_empty() && self.worker.exact_files.is_empty() {
74            return Err(UnboundedWriteScope);
75        }
76        validate_repo_relative_paths("worker.write_roots", &self.worker.write_roots, 128)?;
77        validate_repo_relative_paths("worker.exact_files", &self.worker.exact_files, 256)?;
78        if self.budget.max_cost_microusd == Some(0) {
79            return Err(ZeroBudget("max_cost_microusd"));
80        }
81        if self.budget.max_tokens == Some(0) {
82            return Err(ZeroBudget("max_tokens"));
83        }
84        if !self.hard_gates.forbid_test_changes {
85            return Err(TestWeakeningAllowed);
86        }
87        if self.hard_gates.commands.is_empty() {
88            return Err(MissingHardGates);
89        }
90        validate_string_list("hard_gates.commands", &self.hard_gates.commands, 32, 4_096)?;
91        validate_string_list(
92            "hard_gates.protected_paths",
93            &self.hard_gates.protected_paths,
94            256,
95            1_024,
96        )?;
97        validate_text("score.command", &self.score.command, 4_096)?;
98        validate_text("score.metric", &self.score.metric, 256)?;
99        if !(1..=25).contains(&self.score.trials) {
100            return Err(InvalidTrials(self.score.trials));
101        }
102        if self.score.tie_breakers.is_empty() {
103            return Err(MissingTieBreakers);
104        }
105        Ok(())
106    }
107
108    /// Freeze the exact public inputs and evaluator identity before admission.
109    /// The evaluator bytes are hashed, not exposed to generation workers.
110    pub fn freeze(
111        &self,
112        baseline_commit: &str,
113        resolved_model: &str,
114        public_evidence: &[u8],
115        evaluator: &[u8],
116    ) -> Result<FrozenWorkflowSearch, SearchSpecError> {
117        self.validate()?;
118        validate_commit(baseline_commit)?;
119        validate_text("resolved_model", resolved_model, 256)?;
120        if evaluator.is_empty() {
121            return Err(EmptyEvaluator);
122        }
123
124        let public_evidence_hash = sha256_label(public_evidence);
125        let evaluator_hash = sha256_label(evaluator);
126        let freeze_input = FreezeInput {
127            spec: self,
128            baseline_commit,
129            requested_model: &self.worker.model,
130            resolved_model,
131            public_evidence_hash: &public_evidence_hash,
132            evaluator_hash: &evaluator_hash,
133        };
134        let encoded =
135            serde_json::to_vec(&freeze_input).map_err(|error| FreezeEncoding(error.to_string()))?;
136        let preregistration_hash = sha256_label(&encoded);
137        let search_id = format!("search-{}", &preregistration_hash[7..23]);
138
139        Ok(FrozenWorkflowSearch {
140            schema_version: self.schema_version,
141            search_id,
142            baseline_commit: baseline_commit.to_string(),
143            preregistration_hash,
144            public_evidence_hash,
145            evaluator_hash,
146            requested_model: self.worker.model.clone(),
147            resolved_model: resolved_model.to_string(),
148            candidate_ids: self.candidate_ids(),
149        })
150    }
151
152    #[must_use]
153    pub fn candidate_ids(&self) -> Vec<String> {
154        let width = self.population.to_string().len().max(3);
155        (1..=self.population)
156            .map(|index| format!("cand_{index:0width$}"))
157            .collect()
158    }
159
160    /// Deterministic admission batches. Fleet owns actual scheduling and may
161    /// run fewer workers when its configured pool or provider quota is lower.
162    pub fn admission_batches(&self) -> Result<Vec<Vec<String>>, SearchSpecError> {
163        self.validate()?;
164        Ok(self
165            .candidate_ids()
166            .chunks(usize::from(self.concurrency))
167            .map(<[String]>::to_vec)
168            .collect())
169    }
170}
171
172#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
173pub struct SearchWorkerSpec {
174    #[serde(default)]
175    pub provider: Option<String>,
176    pub model: String,
177    #[serde(default)]
178    pub reasoning_effort: SearchReasoningEffort,
179    #[serde(default)]
180    pub write_authority: SearchWriteAuthority,
181    #[serde(default)]
182    pub write_roots: Vec<String>,
183    #[serde(default)]
184    pub exact_files: Vec<String>,
185}
186
187#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
188#[serde(rename_all = "snake_case")]
189pub enum SearchReasoningEffort {
190    Off,
191    Low,
192    Medium,
193    #[default]
194    High,
195    Max,
196}
197
198#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
199#[serde(rename_all = "snake_case")]
200pub enum SearchWriteAuthority {
201    #[default]
202    WorktreeWrite,
203}
204
205#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
206pub struct SearchBudgetSpec {
207    #[serde(default)]
208    pub max_cost_microusd: Option<u64>,
209    #[serde(default)]
210    pub max_tokens: Option<u64>,
211}
212
213#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
214pub struct SearchHardGateSpec {
215    pub commands: Vec<String>,
216    #[serde(default = "default_true")]
217    pub forbid_test_changes: bool,
218    #[serde(default)]
219    pub protected_paths: Vec<String>,
220}
221
222#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
223pub struct SearchScoreSpec {
224    pub command: String,
225    pub metric: String,
226    #[serde(default)]
227    pub direction: SearchDirection,
228    #[serde(default = "default_trials")]
229    pub trials: u16,
230    #[serde(default)]
231    pub tie_breakers: Vec<SearchTieBreaker>,
232}
233
234#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
235#[serde(rename_all = "snake_case")]
236pub enum SearchDirection {
237    #[default]
238    Minimize,
239    Maximize,
240}
241
242#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
243#[serde(rename_all = "snake_case")]
244pub enum SearchTieBreaker {
245    DiffLines,
246    CostMicrousd,
247    Score,
248}
249
250#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
251pub struct SearchSelectionSpec {
252    #[serde(default)]
253    pub policy: SearchSelectionPolicy,
254    #[serde(default = "default_true")]
255    pub retain_diversity: bool,
256    #[serde(default)]
257    pub ordering: Vec<SearchSelectionMetric>,
258}
259
260impl Default for SearchSelectionSpec {
261    fn default() -> Self {
262        Self {
263            policy: SearchSelectionPolicy::Pareto,
264            retain_diversity: true,
265            ordering: Vec::new(),
266        }
267    }
268}
269
270#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
271#[serde(rename_all = "snake_case")]
272pub enum SearchSelectionPolicy {
273    #[default]
274    Pareto,
275    Ordered,
276}
277
278#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
279#[serde(rename_all = "snake_case")]
280pub enum SearchSelectionMetric {
281    Score,
282    Runtime,
283    DiffLines,
284    CostMicrousd,
285}
286
287#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
288#[serde(rename_all = "snake_case")]
289pub enum SearchIntegrationPolicy {
290    /// Produce a verified, reviewable winner or NONE. Never apply or merge it.
291    #[default]
292    ReviewOnly,
293}
294
295#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
296pub struct FrozenWorkflowSearch {
297    pub schema_version: u32,
298    pub search_id: String,
299    pub baseline_commit: String,
300    pub preregistration_hash: String,
301    pub public_evidence_hash: String,
302    pub evaluator_hash: String,
303    pub requested_model: String,
304    pub resolved_model: String,
305    pub candidate_ids: Vec<String>,
306}
307
308#[derive(Serialize)]
309struct FreezeInput<'a> {
310    spec: &'a WorkflowSearchSpec,
311    baseline_commit: &'a str,
312    requested_model: &'a str,
313    resolved_model: &'a str,
314    public_evidence_hash: &'a str,
315    evaluator_hash: &'a str,
316}
317
318#[derive(Debug, Clone, PartialEq, Eq, Error)]
319pub enum SearchSpecError {
320    #[error("failed to parse Workflow search TOML: {0}")]
321    Parse(String),
322    #[error("unsupported Workflow search schema version {0}")]
323    UnsupportedSchemaVersion(u32),
324    #[error("search name must be a 1-96 character lowercase token")]
325    InvalidName,
326    #[error("{field} must be non-empty and no longer than {max} characters")]
327    InvalidText { field: &'static str, max: usize },
328    #[error("population {0} must be between 2 and 1000")]
329    InvalidPopulation(u16),
330    #[error(
331        "concurrency {concurrency} must be between 1 and 16 and no greater than population {population}"
332    )]
333    InvalidConcurrency { concurrency: u16, population: u16 },
334    #[error("rounds must start at population, decrease strictly, and end at 1")]
335    InvalidRounds,
336    #[error("write-capable search workers require write_roots or exact_files")]
337    UnboundedWriteScope,
338    #[error("{field} contains an empty, oversized, or duplicate entry")]
339    InvalidStringList { field: &'static str },
340    #[error("{field} entries must be bounded repo-relative paths without parent traversal")]
341    InvalidWriteScope { field: &'static str },
342    #[error("{0} must be greater than zero when set")]
343    ZeroBudget(&'static str),
344    #[error("experimental search must forbid test changes")]
345    TestWeakeningAllowed,
346    #[error("experimental search requires at least one runtime-owned hard-gate command")]
347    MissingHardGates,
348    #[error("score.trials must be between 1 and 25, got {0}")]
349    InvalidTrials(u16),
350    #[error("experimental search requires at least one deterministic tie-breaker")]
351    MissingTieBreakers,
352    #[error("baseline_commit must be a 7-64 character hexadecimal commit id")]
353    InvalidBaselineCommit,
354    #[error("evaluator bytes must be non-empty")]
355    EmptyEvaluator,
356    #[error("failed to encode frozen Workflow search: {0}")]
357    FreezeEncoding(String),
358}
359
360fn default_schema_version() -> u32 {
361    WORKFLOW_SEARCH_SCHEMA_VERSION
362}
363
364fn default_trials() -> u16 {
365    5
366}
367
368fn default_true() -> bool {
369    true
370}
371
372fn validate_name(value: &str) -> Result<(), SearchSpecError> {
373    if value.is_empty()
374        || value.len() > 96
375        || !value
376            .bytes()
377            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"-_".contains(&byte))
378    {
379        return Err(InvalidName);
380    }
381    Ok(())
382}
383
384fn validate_text(field: &'static str, value: &str, max: usize) -> Result<(), SearchSpecError> {
385    if value.trim().is_empty() || value.chars().count() > max {
386        return Err(InvalidText { field, max });
387    }
388    Ok(())
389}
390
391fn validate_rounds(population: u16, rounds: &[u16]) -> Result<(), SearchSpecError> {
392    if rounds.len() < 2
393        || rounds.first() != Some(&population)
394        || rounds.last() != Some(&1)
395        || rounds.windows(2).any(|pair| pair[0] <= pair[1])
396    {
397        return Err(InvalidRounds);
398    }
399    Ok(())
400}
401
402fn validate_string_list(
403    field: &'static str,
404    values: &[String],
405    max_items: usize,
406    max_chars: usize,
407) -> Result<(), SearchSpecError> {
408    if values.len() > max_items
409        || values
410            .iter()
411            .any(|value| value.trim().is_empty() || value.chars().count() > max_chars)
412        || values
413            .iter()
414            .enumerate()
415            .any(|(index, value)| values[..index].contains(value))
416    {
417        return Err(InvalidStringList { field });
418    }
419    Ok(())
420}
421
422fn validate_repo_relative_paths(
423    field: &'static str,
424    values: &[String],
425    max_items: usize,
426) -> Result<(), SearchSpecError> {
427    validate_string_list(field, values, max_items, 1_024)?;
428    if values.iter().any(|value| {
429        let trimmed = value.trim();
430        let normalized = trimmed.replace('\\', "/");
431        let windows_drive = normalized.as_bytes().get(1) == Some(&b':')
432            && normalized
433                .as_bytes()
434                .first()
435                .is_some_and(u8::is_ascii_alphabetic);
436        trimmed != value
437            || trimmed.chars().any(|ch| matches!(ch, '\0' | '\r' | '\n'))
438            || windows_drive
439            || Path::new(&normalized).is_absolute()
440            || Path::new(&normalized).components().any(|component| {
441                matches!(
442                    component,
443                    Component::ParentDir | Component::RootDir | Component::Prefix(_)
444                )
445            })
446    }) {
447        return Err(InvalidWriteScope { field });
448    }
449    Ok(())
450}
451
452fn validate_commit(value: &str) -> Result<(), SearchSpecError> {
453    if !(7..=64).contains(&value.len()) || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
454        return Err(InvalidBaselineCommit);
455    }
456    Ok(())
457}
458
459fn sha256_label(bytes: &[u8]) -> String {
460    let digest = Sha256::digest(bytes);
461    let mut output = String::with_capacity(71);
462    output.push_str("sha256:");
463    for byte in digest {
464        use std::fmt::Write as _;
465        let _ = write!(output, "{byte:02x}");
466    }
467    output
468}
469
470#[cfg(test)]
471mod tests {
472    use super::*;
473
474    const SPEC: &str = r#"
475name = "speed-up-certificate"
476objective = "Reduce runtime without changing exact results"
477population = 32
478rounds = [32, 8, 3, 1]
479concurrency = 16
480integration_policy = "review_only"
481
482[worker]
483provider = "deepseek"
484model = "deepseek-v4-flash"
485reasoning_effort = "max"
486write_authority = "worktree_write"
487write_roots = ["code"]
488
489[budget]
490max_cost_microusd = 5000000
491max_tokens = 10000000
492
493[hard_gates]
494commands = ["cargo test --locked", "git diff --exit-code -- expected.json"]
495forbid_test_changes = true
496protected_paths = ["tests", "expected.json"]
497
498[score]
499command = "./scripts/benchmark_candidate.sh"
500direction = "minimize"
501metric = "median_runtime_ms"
502trials = 5
503tie_breakers = ["diff_lines", "cost_microusd"]
504
505[selection]
506policy = "pareto"
507retain_diversity = true
508"#;
509
510    #[test]
511    fn parses_valid_search_and_queues_through_live_cap() {
512        let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
513
514        assert_eq!(spec.population, 32);
515        let batches = spec.admission_batches().expect("validated admission");
516        assert_eq!(batches.len(), 2);
517        assert_eq!(batches[0].len(), 16);
518        assert_eq!(spec.candidate_ids()[0], "cand_001");
519        assert_eq!(spec.candidate_ids()[31], "cand_032");
520    }
521
522    #[test]
523    fn freeze_is_deterministic_and_model_version_sensitive() {
524        let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
525        let first = spec
526            .freeze(
527                "33bc6a98",
528                "DeepSeek-V4-Flash-0731",
529                b"public evidence",
530                b"private evaluator",
531            )
532            .expect("freeze succeeds");
533        let replay = spec
534            .freeze(
535                "33bc6a98",
536                "DeepSeek-V4-Flash-0731",
537                b"public evidence",
538                b"private evaluator",
539            )
540            .expect("freeze succeeds");
541        let drifted = spec
542            .freeze(
543                "33bc6a98",
544                "DeepSeek-V4-Flash-next",
545                b"public evidence",
546                b"private evaluator",
547            )
548            .expect("freeze succeeds");
549
550        assert_eq!(first, replay);
551        assert_ne!(first.search_id, drifted.search_id);
552        assert_ne!(first.preregistration_hash, drifted.preregistration_hash);
553        assert_eq!(first.requested_model, "deepseek-v4-flash");
554        assert_eq!(first.resolved_model, "DeepSeek-V4-Flash-0731");
555    }
556
557    #[test]
558    fn rejects_unsafe_or_unbounded_searches() {
559        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
560        spec.hard_gates.forbid_test_changes = false;
561        assert_eq!(spec.validate(), Err(SearchSpecError::TestWeakeningAllowed));
562
563        spec.hard_gates.forbid_test_changes = true;
564        spec.worker.write_roots.clear();
565        assert_eq!(spec.validate(), Err(SearchSpecError::UnboundedWriteScope));
566    }
567
568    #[test]
569    fn rejects_invalid_rounds_and_excess_live_concurrency() {
570        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
571        spec.rounds = vec![32, 8, 8, 1];
572        assert_eq!(spec.validate(), Err(SearchSpecError::InvalidRounds));
573
574        spec.rounds = vec![32, 1];
575        spec.concurrency = 17;
576        assert_eq!(
577            spec.validate(),
578            Err(SearchSpecError::InvalidConcurrency {
579                concurrency: 17,
580                population: 32,
581            })
582        );
583    }
584
585    #[test]
586    fn admission_refuses_unvalidated_zero_concurrency_without_panicking() {
587        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
588        spec.concurrency = 0;
589
590        assert_eq!(
591            spec.admission_batches(),
592            Err(SearchSpecError::InvalidConcurrency {
593                concurrency: 0,
594                population: 32,
595            })
596        );
597    }
598
599    #[test]
600    fn rejects_write_scopes_that_escape_or_obscure_the_repo_boundary() {
601        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
602        for unsafe_path in ["../outside", "/tmp/outside", r"C:\outside"] {
603            spec.worker.write_roots = vec![unsafe_path.to_string()];
604            assert_eq!(
605                spec.validate(),
606                Err(SearchSpecError::InvalidWriteScope {
607                    field: "worker.write_roots",
608                }),
609                "path should be rejected: {unsafe_path}"
610            );
611        }
612    }
613
614    #[test]
615    fn deserialization_refuses_auto_merge_policy() {
616        let source = SPEC.replace("review_only", "auto_merge");
617        let error = WorkflowSearchSpec::from_toml(&source).expect_err("must reject auto merge");
618
619        assert!(matches!(error, SearchSpecError::Parse(_)));
620    }
621}
codewhale_workflow/experimental_search.rs

codewhale_workflow/
experimental_search.rs
