codewhale_workflow/
experimental_search.rs

1//! Provider-neutral experimental search authoring for Workflow.
2//!
3//! This module is an authoring and freeze boundary, not a new runtime or
4//! scheduler. A validated search still has to be lowered by the Workflow host
5//! into Fleet workers plus a runtime-owned evaluator. In particular, worker
6//! self-reports are never promoted to hard-gate evidence here.
7
8use serde::{Deserialize, Serialize};
9use sha2::{Digest, Sha256};
10use std::path::{Component, Path};
11use thiserror::Error;
12
13use crate::{DEFAULT_FLEET_WORKFLOW_MAX_AGENTS, experimental_search::SearchSpecError::*};
14
15pub const WORKFLOW_SEARCH_SCHEMA_VERSION: u32 = 1;
16/// Fallback live-worker ceiling for one search admission batch.
17///
18/// This is the answer when the host resolves no Fleet concurrency limit at
19/// all — not a second knob. 16 matches the Workflow host's live-child ceiling
20/// today (`codewhale_workflow_js::WORKFLOW_MAX_CONCURRENT`, from which the tui
21/// driver sizes its per-run admission semaphore). This crate cannot import
22/// that constant directly because `codewhale-workflow-js` depends on
23/// `codewhale-workflow`.
24///
25/// Prefer passing the live limit: every validation entry point has a
26/// `*_with_limit` twin that takes the resolved Fleet ceiling, and the frozen
27/// receipt records which of the two actually bounded the search.
28pub const WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT: u16 = 16;
29
30/// Where a search's live-worker ceiling came from.
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
32#[serde(rename_all = "snake_case")]
33pub enum SearchConcurrencySource {
34    /// Resolved from the host's Fleet concurrency configuration.
35    FleetLimit,
36    /// No Fleet limit resolved, so [`WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT`]
37    /// applied.
38    #[default]
39    Default,
40}
41
42/// The live-worker ceiling that bounded a search, plus where it came from.
43///
44/// Carried on the frozen receipt so an operator reading a run can tell a
45/// deliberately small Fleet pool from this crate's fallback.
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
47pub struct ResolvedSearchConcurrency {
48    pub limit: u16,
49    pub source: SearchConcurrencySource,
50}
51
52impl Default for ResolvedSearchConcurrency {
53    fn default() -> Self {
54        Self::resolve(None)
55    }
56}
57
58impl ResolvedSearchConcurrency {
59    /// Resolve the ceiling from an already-resolved Fleet limit.
60    ///
61    /// `None` — and a nonsensical `Some(0)`, which would admit nothing — fall
62    /// back to [`WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT`].
63    #[must_use]
64    pub fn resolve(fleet_limit: Option<u16>) -> Self {
65        match fleet_limit.filter(|limit| *limit > 0) {
66            Some(limit) => Self {
67                limit,
68                source: SearchConcurrencySource::FleetLimit,
69            },
70            None => Self {
71                limit: WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT,
72                source: SearchConcurrencySource::Default,
73            },
74        }
75    }
76
77    /// Resolve straight from the two config seams the host already owns:
78    /// `[workflow] max_concurrent` (the per-run live-agent ceiling, see
79    /// `codewhale_config::WorkflowConfigToml::max_concurrent`) and a Fleet
80    /// profile's `delegation.max_concurrency` hint
81    /// (`codewhale_config::FleetDelegationHints::max_concurrency`).
82    ///
83    /// The lower of the present values wins: a profile that asks for fewer
84    /// workers than the run allows is a real bound, and a run ceiling below a
85    /// profile hint is the admission the host will actually grant.
86    #[must_use]
87    pub fn from_fleet_config(
88        workflow_max_concurrent: Option<u32>,
89        profile_max_concurrency: Option<usize>,
90    ) -> Self {
91        let workflow =
92            workflow_max_concurrent.map(|value| u16::try_from(value).unwrap_or(u16::MAX));
93        let profile = profile_max_concurrency.map(|value| u16::try_from(value).unwrap_or(u16::MAX));
94        let resolved = match (workflow.filter(|v| *v > 0), profile.filter(|v| *v > 0)) {
95            (Some(left), Some(right)) => Some(left.min(right)),
96            (Some(value), None) | (None, Some(value)) => Some(value),
97            (None, None) => None,
98        };
99        Self::resolve(resolved)
100    }
101
102    /// One-line receipt copy naming the ceiling and its origin.
103    #[must_use]
104    pub fn receipt_line(&self) -> String {
105        let origin = match self.source {
106            SearchConcurrencySource::FleetLimit => "resolved Fleet limit",
107            SearchConcurrencySource::Default => "default, no Fleet limit resolved",
108        };
109        format!("live-worker ceiling {limit} ({origin})", limit = self.limit)
110    }
111}
112
113#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
114pub struct WorkflowSearchSpec {
115    #[serde(default = "default_schema_version")]
116    pub schema_version: u32,
117    pub name: String,
118    pub objective: String,
119    pub population: u16,
120    pub rounds: Vec<u16>,
121    pub concurrency: u16,
122    pub worker: SearchWorkerSpec,
123    #[serde(default)]
124    pub budget: SearchBudgetSpec,
125    pub hard_gates: SearchHardGateSpec,
126    pub score: SearchScoreSpec,
127    #[serde(default)]
128    pub selection: SearchSelectionSpec,
129    #[serde(default)]
130    pub integration_policy: SearchIntegrationPolicy,
131}
132
133impl WorkflowSearchSpec {
134    /// Parse and validate against the fallback ceiling. Hosts that know their
135    /// Fleet limit should call [`Self::from_toml_with_limit`].
136    pub fn from_toml(source: &str) -> Result<Self, SearchSpecError> {
137        Self::from_toml_with_limit(source, None)
138    }
139
140    /// Parse and validate against the resolved Fleet concurrency limit.
141    pub fn from_toml_with_limit(
142        source: &str,
143        fleet_limit: Option<u16>,
144    ) -> Result<Self, SearchSpecError> {
145        let spec: Self = toml::from_str(source).map_err(|error| Parse(error.to_string()))?;
146        spec.validate_with_limit(fleet_limit)?;
147        Ok(spec)
148    }
149
150    pub fn validate(&self) -> Result<(), SearchSpecError> {
151        self.validate_with_limit(None).map(|_| ())
152    }
153
154    /// Validate against the resolved Fleet concurrency limit, returning the
155    /// ceiling that applied so the caller can echo it in a receipt.
156    pub fn validate_with_limit(
157        &self,
158        fleet_limit: Option<u16>,
159    ) -> Result<ResolvedSearchConcurrency, SearchSpecError> {
160        let concurrency_ceiling = ResolvedSearchConcurrency::resolve(fleet_limit);
161        if self.schema_version != WORKFLOW_SEARCH_SCHEMA_VERSION {
162            return Err(UnsupportedSchemaVersion(self.schema_version));
163        }
164        validate_name(&self.name)?;
165        validate_text("objective", &self.objective, 32_768)?;
166        if !(2..=DEFAULT_FLEET_WORKFLOW_MAX_AGENTS as u16).contains(&self.population) {
167            return Err(InvalidPopulation(self.population));
168        }
169        if self.concurrency == 0
170            || self.concurrency > concurrency_ceiling.limit
171            || self.concurrency > self.population
172        {
173            return Err(InvalidConcurrency {
174                concurrency: self.concurrency,
175                population: self.population,
176                limit: concurrency_ceiling.limit,
177            });
178        }
179        validate_rounds(self.population, &self.rounds)?;
180        validate_text("worker.model", &self.worker.model, 256)?;
181        if self.worker.write_roots.is_empty() && self.worker.exact_files.is_empty() {
182            return Err(UnboundedWriteScope);
183        }
184        validate_repo_relative_paths("worker.write_roots", &self.worker.write_roots, 128)?;
185        validate_repo_relative_paths("worker.exact_files", &self.worker.exact_files, 256)?;
186        if self.budget.max_cost_microusd == Some(0) {
187            return Err(ZeroBudget("max_cost_microusd"));
188        }
189        if self.budget.max_tokens == Some(0) {
190            return Err(ZeroBudget("max_tokens"));
191        }
192        if !self.hard_gates.forbid_test_changes {
193            return Err(TestWeakeningAllowed);
194        }
195        if self.hard_gates.commands.is_empty() {
196            return Err(MissingHardGates);
197        }
198        validate_string_list("hard_gates.commands", &self.hard_gates.commands, 32, 4_096)?;
199        validate_string_list(
200            "hard_gates.protected_paths",
201            &self.hard_gates.protected_paths,
202            256,
203            1_024,
204        )?;
205        validate_text("score.command", &self.score.command, 4_096)?;
206        validate_text("score.metric", &self.score.metric, 256)?;
207        if !(1..=25).contains(&self.score.trials) {
208            return Err(InvalidTrials(self.score.trials));
209        }
210        if self.score.tie_breakers.is_empty() {
211            return Err(MissingTieBreakers);
212        }
213        Ok(concurrency_ceiling)
214    }
215
216    /// Freeze the exact public inputs and evaluator identity before admission.
217    /// The evaluator bytes are hashed, not exposed to generation workers.
218    pub fn freeze(
219        &self,
220        baseline_commit: &str,
221        resolved_model: &str,
222        public_evidence: &[u8],
223        evaluator: &[u8],
224    ) -> Result<FrozenWorkflowSearch, SearchSpecError> {
225        self.freeze_with_limit(
226            baseline_commit,
227            resolved_model,
228            public_evidence,
229            evaluator,
230            None,
231        )
232    }
233
234    /// Freeze against the resolved Fleet concurrency limit.
235    ///
236    /// The resolved ceiling is recorded on the receipt but deliberately kept
237    /// out of the preregistration hash: the hash freezes the scientific inputs
238    /// (spec, model, public evidence, evaluator identity), and how many
239    /// workers the operator's pool happened to allow is an operational fact
240    /// about the run, not part of what was preregistered.
241    pub fn freeze_with_limit(
242        &self,
243        baseline_commit: &str,
244        resolved_model: &str,
245        public_evidence: &[u8],
246        evaluator: &[u8],
247        fleet_limit: Option<u16>,
248    ) -> Result<FrozenWorkflowSearch, SearchSpecError> {
249        let resolved_concurrency = self.validate_with_limit(fleet_limit)?;
250        validate_commit(baseline_commit)?;
251        validate_text("resolved_model", resolved_model, 256)?;
252        if evaluator.is_empty() {
253            return Err(EmptyEvaluator);
254        }
255
256        let public_evidence_hash = sha256_label(public_evidence);
257        let evaluator_hash = sha256_label(evaluator);
258        let freeze_input = FreezeInput {
259            spec: self,
260            baseline_commit,
261            requested_model: &self.worker.model,
262            resolved_model,
263            public_evidence_hash: &public_evidence_hash,
264            evaluator_hash: &evaluator_hash,
265        };
266        let encoded =
267            serde_json::to_vec(&freeze_input).map_err(|error| FreezeEncoding(error.to_string()))?;
268        let preregistration_hash = sha256_label(&encoded);
269        let search_id = format!("search-{}", &preregistration_hash[7..23]);
270
271        Ok(FrozenWorkflowSearch {
272            schema_version: self.schema_version,
273            search_id,
274            baseline_commit: baseline_commit.to_string(),
275            preregistration_hash,
276            public_evidence_hash,
277            evaluator_hash,
278            requested_model: self.worker.model.clone(),
279            resolved_model: resolved_model.to_string(),
280            candidate_ids: self.candidate_ids(),
281            resolved_concurrency,
282        })
283    }
284
285    #[must_use]
286    pub fn candidate_ids(&self) -> Vec<String> {
287        let width = self.population.to_string().len().max(3);
288        (1..=self.population)
289            .map(|index| format!("cand_{index:0width$}"))
290            .collect()
291    }
292
293    /// Deterministic admission batches. Fleet owns actual scheduling and may
294    /// run fewer workers when its configured pool or provider quota is lower.
295    pub fn admission_batches(&self) -> Result<Vec<Vec<String>>, SearchSpecError> {
296        self.admission_batches_with_limit(None)
297    }
298
299    /// Deterministic admission batches, validated against the resolved Fleet
300    /// concurrency limit.
301    pub fn admission_batches_with_limit(
302        &self,
303        fleet_limit: Option<u16>,
304    ) -> Result<Vec<Vec<String>>, SearchSpecError> {
305        self.validate_with_limit(fleet_limit)?;
306        Ok(self
307            .candidate_ids()
308            .chunks(usize::from(self.concurrency))
309            .map(<[String]>::to_vec)
310            .collect())
311    }
312}
313
314#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
315pub struct SearchWorkerSpec {
316    #[serde(default)]
317    pub provider: Option<String>,
318    pub model: String,
319    #[serde(default)]
320    pub reasoning_effort: SearchReasoningEffort,
321    #[serde(default)]
322    pub write_authority: SearchWriteAuthority,
323    #[serde(default)]
324    pub write_roots: Vec<String>,
325    #[serde(default)]
326    pub exact_files: Vec<String>,
327}
328
329#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
330#[serde(rename_all = "snake_case")]
331pub enum SearchReasoningEffort {
332    Off,
333    Low,
334    Medium,
335    #[default]
336    High,
337    Max,
338}
339
340#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
341#[serde(rename_all = "snake_case")]
342pub enum SearchWriteAuthority {
343    #[default]
344    WorktreeWrite,
345}
346
347#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
348pub struct SearchBudgetSpec {
349    #[serde(default)]
350    pub max_cost_microusd: Option<u64>,
351    #[serde(default)]
352    pub max_tokens: Option<u64>,
353}
354
355#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
356pub struct SearchHardGateSpec {
357    pub commands: Vec<String>,
358    #[serde(default = "default_true")]
359    pub forbid_test_changes: bool,
360    #[serde(default)]
361    pub protected_paths: Vec<String>,
362}
363
364#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
365pub struct SearchScoreSpec {
366    pub command: String,
367    pub metric: String,
368    #[serde(default)]
369    pub direction: SearchDirection,
370    #[serde(default = "default_trials")]
371    pub trials: u16,
372    #[serde(default)]
373    pub tie_breakers: Vec<SearchTieBreaker>,
374}
375
376#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
377#[serde(rename_all = "snake_case")]
378pub enum SearchDirection {
379    #[default]
380    Minimize,
381    Maximize,
382}
383
384#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
385#[serde(rename_all = "snake_case")]
386pub enum SearchTieBreaker {
387    DiffLines,
388    CostMicrousd,
389    Score,
390}
391
392#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
393pub struct SearchSelectionSpec {
394    #[serde(default)]
395    pub policy: SearchSelectionPolicy,
396    #[serde(default = "default_true")]
397    pub retain_diversity: bool,
398    #[serde(default)]
399    pub ordering: Vec<SearchSelectionMetric>,
400}
401
402impl Default for SearchSelectionSpec {
403    fn default() -> Self {
404        Self {
405            policy: SearchSelectionPolicy::Pareto,
406            retain_diversity: true,
407            ordering: Vec::new(),
408        }
409    }
410}
411
412#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
413#[serde(rename_all = "snake_case")]
414pub enum SearchSelectionPolicy {
415    #[default]
416    Pareto,
417    Ordered,
418}
419
420#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
421#[serde(rename_all = "snake_case")]
422pub enum SearchSelectionMetric {
423    Score,
424    Runtime,
425    DiffLines,
426    CostMicrousd,
427}
428
429#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
430#[serde(rename_all = "snake_case")]
431pub enum SearchIntegrationPolicy {
432    /// Produce a verified, reviewable winner or NONE. Never apply or merge it.
433    #[default]
434    ReviewOnly,
435}
436
437#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
438pub struct FrozenWorkflowSearch {
439    pub schema_version: u32,
440    pub search_id: String,
441    pub baseline_commit: String,
442    pub preregistration_hash: String,
443    pub public_evidence_hash: String,
444    pub evaluator_hash: String,
445    pub requested_model: String,
446    pub resolved_model: String,
447    pub candidate_ids: Vec<String>,
448    /// The live-worker ceiling that bounded this search, and whether it came
449    /// from Fleet config or the crate fallback. `#[serde(default)]` so
450    /// receipts written before the bound was recorded still load.
451    #[serde(default)]
452    pub resolved_concurrency: ResolvedSearchConcurrency,
453}
454
455impl FrozenWorkflowSearch {
456    /// Receipt line naming the ceiling that actually bounded this run.
457    #[must_use]
458    pub fn concurrency_receipt_line(&self) -> String {
459        self.resolved_concurrency.receipt_line()
460    }
461}
462
463#[derive(Serialize)]
464struct FreezeInput<'a> {
465    spec: &'a WorkflowSearchSpec,
466    baseline_commit: &'a str,
467    requested_model: &'a str,
468    resolved_model: &'a str,
469    public_evidence_hash: &'a str,
470    evaluator_hash: &'a str,
471}
472
473#[derive(Debug, Clone, PartialEq, Eq, Error)]
474pub enum SearchSpecError {
475    #[error("failed to parse Workflow search TOML: {0}")]
476    Parse(String),
477    #[error("unsupported Workflow search schema version {0}")]
478    UnsupportedSchemaVersion(u32),
479    #[error("search name must be a 1-96 character lowercase token")]
480    InvalidName,
481    #[error("{field} must be non-empty and no longer than {max} characters")]
482    InvalidText { field: &'static str, max: usize },
483    #[error("population {0} must be between 2 and 1000")]
484    InvalidPopulation(u16),
485    #[error(
486        "concurrency {concurrency} must be between 1 and {limit} (resolved live-worker ceiling) and no greater than population {population}"
487    )]
488    InvalidConcurrency {
489        concurrency: u16,
490        population: u16,
491        limit: u16,
492    },
493    #[error("rounds must start at population, decrease strictly, and end at 1")]
494    InvalidRounds,
495    #[error("write-capable search workers require write_roots or exact_files")]
496    UnboundedWriteScope,
497    #[error("{field} contains an empty, oversized, or duplicate entry")]
498    InvalidStringList { field: &'static str },
499    #[error("{field} entries must be bounded repo-relative paths without parent traversal")]
500    InvalidWriteScope { field: &'static str },
501    #[error("{0} must be greater than zero when set")]
502    ZeroBudget(&'static str),
503    #[error("experimental search must forbid test changes")]
504    TestWeakeningAllowed,
505    #[error("experimental search requires at least one runtime-owned hard-gate command")]
506    MissingHardGates,
507    #[error("score.trials must be between 1 and 25, got {0}")]
508    InvalidTrials(u16),
509    #[error("experimental search requires at least one deterministic tie-breaker")]
510    MissingTieBreakers,
511    #[error("baseline_commit must be a 7-64 character hexadecimal commit id")]
512    InvalidBaselineCommit,
513    #[error("evaluator bytes must be non-empty")]
514    EmptyEvaluator,
515    #[error("failed to encode frozen Workflow search: {0}")]
516    FreezeEncoding(String),
517}
518
519fn default_schema_version() -> u32 {
520    WORKFLOW_SEARCH_SCHEMA_VERSION
521}
522
523fn default_trials() -> u16 {
524    5
525}
526
527fn default_true() -> bool {
528    true
529}
530
531fn validate_name(value: &str) -> Result<(), SearchSpecError> {
532    if value.is_empty()
533        || value.len() > 96
534        || !value
535            .bytes()
536            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"-_".contains(&byte))
537    {
538        return Err(InvalidName);
539    }
540    Ok(())
541}
542
543fn validate_text(field: &'static str, value: &str, max: usize) -> Result<(), SearchSpecError> {
544    if value.trim().is_empty() || value.chars().count() > max {
545        return Err(InvalidText { field, max });
546    }
547    Ok(())
548}
549
550fn validate_rounds(population: u16, rounds: &[u16]) -> Result<(), SearchSpecError> {
551    if rounds.len() < 2
552        || rounds.first() != Some(&population)
553        || rounds.last() != Some(&1)
554        || rounds.windows(2).any(|pair| pair[0] <= pair[1])
555    {
556        return Err(InvalidRounds);
557    }
558    Ok(())
559}
560
561fn validate_string_list(
562    field: &'static str,
563    values: &[String],
564    max_items: usize,
565    max_chars: usize,
566) -> Result<(), SearchSpecError> {
567    if values.len() > max_items
568        || values
569            .iter()
570            .any(|value| value.trim().is_empty() || value.chars().count() > max_chars)
571        || values
572            .iter()
573            .enumerate()
574            .any(|(index, value)| values[..index].contains(value))
575    {
576        return Err(InvalidStringList { field });
577    }
578    Ok(())
579}
580
581fn validate_repo_relative_paths(
582    field: &'static str,
583    values: &[String],
584    max_items: usize,
585) -> Result<(), SearchSpecError> {
586    validate_string_list(field, values, max_items, 1_024)?;
587    if values.iter().any(|value| {
588        let trimmed = value.trim();
589        let normalized = trimmed.replace('\\', "/");
590        let windows_drive = normalized.as_bytes().get(1) == Some(&b':')
591            && normalized
592                .as_bytes()
593                .first()
594                .is_some_and(u8::is_ascii_alphabetic);
595        trimmed != value
596            || trimmed.chars().any(|ch| matches!(ch, '\0' | '\r' | '\n'))
597            || windows_drive
598            || Path::new(&normalized).is_absolute()
599            || Path::new(&normalized).components().any(|component| {
600                matches!(
601                    component,
602                    Component::ParentDir | Component::RootDir | Component::Prefix(_)
603                )
604            })
605    }) {
606        return Err(InvalidWriteScope { field });
607    }
608    Ok(())
609}
610
611fn validate_commit(value: &str) -> Result<(), SearchSpecError> {
612    if !(7..=64).contains(&value.len()) || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
613        return Err(InvalidBaselineCommit);
614    }
615    Ok(())
616}
617
618fn sha256_label(bytes: &[u8]) -> String {
619    let digest = Sha256::digest(bytes);
620    let mut output = String::with_capacity(71);
621    output.push_str("sha256:");
622    for byte in digest {
623        use std::fmt::Write as _;
624        let _ = write!(output, "{byte:02x}");
625    }
626    output
627}
628
629#[cfg(test)]
630mod tests {
631    use super::*;
632
633    const SPEC: &str = r#"
634name = "speed-up-certificate"
635objective = "Reduce runtime without changing exact results"
636population = 32
637rounds = [32, 8, 3, 1]
638concurrency = 16
639integration_policy = "review_only"
640
641[worker]
642provider = "deepseek"
643model = "deepseek-v4-flash"
644reasoning_effort = "max"
645write_authority = "worktree_write"
646write_roots = ["code"]
647
648[budget]
649max_cost_microusd = 5000000
650max_tokens = 10000000
651
652[hard_gates]
653commands = ["cargo test --locked", "git diff --exit-code -- expected.json"]
654forbid_test_changes = true
655protected_paths = ["tests", "expected.json"]
656
657[score]
658command = "./scripts/benchmark_candidate.sh"
659direction = "minimize"
660metric = "median_runtime_ms"
661trials = 5
662tie_breakers = ["diff_lines", "cost_microusd"]
663
664[selection]
665policy = "pareto"
666retain_diversity = true
667"#;
668
669    #[test]
670    fn parses_valid_search_and_queues_through_live_cap() {
671        let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
672
673        assert_eq!(spec.population, 32);
674        let batches = spec.admission_batches().expect("validated admission");
675        assert_eq!(batches.len(), 2);
676        assert_eq!(batches[0].len(), 16);
677        assert_eq!(spec.candidate_ids()[0], "cand_001");
678        assert_eq!(spec.candidate_ids()[31], "cand_032");
679    }
680
681    #[test]
682    fn freeze_is_deterministic_and_model_version_sensitive() {
683        let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
684        let first = spec
685            .freeze(
686                "33bc6a98",
687                "DeepSeek-V4-Flash-0731",
688                b"public evidence",
689                b"private evaluator",
690            )
691            .expect("freeze succeeds");
692        let replay = spec
693            .freeze(
694                "33bc6a98",
695                "DeepSeek-V4-Flash-0731",
696                b"public evidence",
697                b"private evaluator",
698            )
699            .expect("freeze succeeds");
700        let drifted = spec
701            .freeze(
702                "33bc6a98",
703                "DeepSeek-V4-Flash-next",
704                b"public evidence",
705                b"private evaluator",
706            )
707            .expect("freeze succeeds");
708
709        assert_eq!(first, replay);
710        assert_ne!(first.search_id, drifted.search_id);
711        assert_ne!(first.preregistration_hash, drifted.preregistration_hash);
712        assert_eq!(first.requested_model, "deepseek-v4-flash");
713        assert_eq!(first.resolved_model, "DeepSeek-V4-Flash-0731");
714    }
715
716    #[test]
717    fn rejects_unsafe_or_unbounded_searches() {
718        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
719        spec.hard_gates.forbid_test_changes = false;
720        assert_eq!(spec.validate(), Err(SearchSpecError::TestWeakeningAllowed));
721
722        spec.hard_gates.forbid_test_changes = true;
723        spec.worker.write_roots.clear();
724        assert_eq!(spec.validate(), Err(SearchSpecError::UnboundedWriteScope));
725    }
726
727    #[test]
728    fn rejects_invalid_rounds_and_excess_live_concurrency() {
729        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
730        spec.rounds = vec![32, 8, 8, 1];
731        assert_eq!(spec.validate(), Err(SearchSpecError::InvalidRounds));
732
733        spec.rounds = vec![32, 1];
734        spec.concurrency = 17;
735        assert_eq!(
736            spec.validate(),
737            Err(SearchSpecError::InvalidConcurrency {
738                concurrency: 17,
739                population: 32,
740                limit: 16,
741            })
742        );
743    }
744
745    #[test]
746    fn ceiling_follows_the_resolved_fleet_limit() {
747        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
748
749        // A smaller Fleet pool rejects the spec's 16-wide batches...
750        assert_eq!(
751            spec.validate_with_limit(Some(8)),
752            Err(SearchSpecError::InvalidConcurrency {
753                concurrency: 16,
754                population: 32,
755                limit: 8,
756            })
757        );
758
759        // ...and a larger one admits concurrency the fallback would refuse.
760        spec.concurrency = 24;
761        let resolved = spec
762            .validate_with_limit(Some(32))
763            .expect("24 workers fit a 32-wide Fleet limit");
764        assert_eq!(
765            resolved,
766            ResolvedSearchConcurrency {
767                limit: 32,
768                source: SearchConcurrencySource::FleetLimit,
769            }
770        );
771        let batches = spec
772            .admission_batches_with_limit(Some(32))
773            .expect("validated admission");
774        assert_eq!(batches.len(), 2);
775        assert_eq!(batches[0].len(), 24);
776    }
777
778    #[test]
779    fn fallback_ceiling_stays_sixteen_when_no_limit_resolves() {
780        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
781        assert_eq!(
782            spec.validate_with_limit(None),
783            Ok(ResolvedSearchConcurrency {
784                limit: WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT,
785                source: SearchConcurrencySource::Default,
786            })
787        );
788        assert_eq!(WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT, 16);
789
790        // A zero limit is not a ceiling of zero — it is an unresolved one.
791        assert_eq!(
792            spec.validate_with_limit(Some(0)),
793            spec.validate_with_limit(None)
794        );
795
796        spec.concurrency = 17;
797        assert_eq!(
798            spec.validate_with_limit(None),
799            Err(SearchSpecError::InvalidConcurrency {
800                concurrency: 17,
801                population: 32,
802                limit: 16,
803            })
804        );
805    }
806
807    #[test]
808    fn fleet_config_seam_takes_the_lower_present_bound() {
809        assert_eq!(
810            ResolvedSearchConcurrency::from_fleet_config(Some(16), Some(4)),
811            ResolvedSearchConcurrency {
812                limit: 4,
813                source: SearchConcurrencySource::FleetLimit,
814            }
815        );
816        assert_eq!(
817            ResolvedSearchConcurrency::from_fleet_config(Some(6), None),
818            ResolvedSearchConcurrency {
819                limit: 6,
820                source: SearchConcurrencySource::FleetLimit,
821            }
822        );
823        assert_eq!(
824            ResolvedSearchConcurrency::from_fleet_config(None, None),
825            ResolvedSearchConcurrency::default()
826        );
827        assert_eq!(
828            ResolvedSearchConcurrency::default().source,
829            SearchConcurrencySource::Default
830        );
831    }
832
833    #[test]
834    fn freeze_receipt_echoes_the_bound_that_applied() {
835        let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
836
837        let fallback = spec
838            .freeze("33bc6a98", "DeepSeek-V4-Flash-0731", b"evidence", b"eval")
839            .expect("freeze succeeds");
840        assert_eq!(
841            fallback.resolved_concurrency,
842            ResolvedSearchConcurrency {
843                limit: 16,
844                source: SearchConcurrencySource::Default,
845            }
846        );
847        assert_eq!(
848            fallback.concurrency_receipt_line(),
849            "live-worker ceiling 16 (default, no Fleet limit resolved)"
850        );
851
852        let bounded = spec
853            .freeze_with_limit(
854                "33bc6a98",
855                "DeepSeek-V4-Flash-0731",
856                b"evidence",
857                b"eval",
858                Some(16),
859            )
860            .expect("freeze succeeds");
861        assert_eq!(
862            bounded.resolved_concurrency,
863            ResolvedSearchConcurrency {
864                limit: 16,
865                source: SearchConcurrencySource::FleetLimit,
866            }
867        );
868        assert_eq!(
869            bounded.concurrency_receipt_line(),
870            "live-worker ceiling 16 (resolved Fleet limit)"
871        );
872
873        // The preregistration identity is unchanged by the operational bound.
874        assert_eq!(
875            fallback.preregistration_hash, bounded.preregistration_hash,
876            "the resolved ceiling must not perturb the preregistration hash"
877        );
878
879        let receipt = serde_json::to_value(&bounded).expect("receipt serializes");
880        assert_eq!(receipt["resolved_concurrency"]["limit"], 16);
881        assert_eq!(receipt["resolved_concurrency"]["source"], "fleet_limit");
882    }
883
884    #[test]
885    fn admission_refuses_unvalidated_zero_concurrency_without_panicking() {
886        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
887        spec.concurrency = 0;
888
889        assert_eq!(
890            spec.admission_batches(),
891            Err(SearchSpecError::InvalidConcurrency {
892                concurrency: 0,
893                population: 32,
894                limit: 16,
895            })
896        );
897    }
898
899    #[test]
900    fn rejects_write_scopes_that_escape_or_obscure_the_repo_boundary() {
901        let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec");
902        for unsafe_path in ["../outside", "/tmp/outside", r"C:\outside"] {
903            spec.worker.write_roots = vec![unsafe_path.to_string()];
904            assert_eq!(
905                spec.validate(),
906                Err(SearchSpecError::InvalidWriteScope {
907                    field: "worker.write_roots",
908                }),
909                "path should be rejected: {unsafe_path}"
910            );
911        }
912    }
913
914    #[test]
915    fn deserialization_refuses_auto_merge_policy() {
916        let source = SPEC.replace("review_only", "auto_merge");
917        let error = WorkflowSearchSpec::from_toml(&source).expect_err("must reject auto merge");
918
919        assert!(matches!(error, SearchSpecError::Parse(_)));
920    }
921}
codewhale_workflow/experimental_search.rs

codewhale_workflow/
experimental_search.rs
