1#[cfg(test)]
16pub use molo_agent::ReActAgent;
17pub use molo_agent::{agent, tool};
18#[cfg(test)]
19pub use molo_core::ToolCall;
20pub use molo_core::{effect, message, observability, provider, run};
21
22use crate::agent::{
23 AgentAction, AgentError, AgentKernel, ModelObservation, ModelRequest, Observation,
24};
25use crate::effect::{
26 DisplayOutput, EffectKind, EffectObservation, EffectOutput, EffectRequest, EffectStatus,
27 RiskLevel,
28};
29use crate::provider::{Provider, ProviderError, ProviderRequestContext};
30use crate::run::{Artifact, RunContext, RunMetadata, RunOutput, RunRequest};
31use async_trait::async_trait;
32use futures::stream::{FuturesUnordered, StreamExt};
33use serde::{Deserialize, Serialize};
34use std::collections::{BTreeMap, HashMap};
35use std::fmt;
36use std::sync::{Arc, Mutex};
37use std::time::Duration;
38
39pub use crate::observability::RedactionRecord;
40
41#[derive(Debug)]
47pub struct HarnessRuntime<P, H> {
48 provider: P,
49 harness: H,
50 config: HarnessRuntimeConfig,
51}
52
53#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
55#[serde(default)]
56#[non_exhaustive]
57pub struct HarnessRuntimeConfig {
58 pub(crate) max_agent_steps: usize,
60 pub(crate) max_effect_batch_concurrency: usize,
62 pub(crate) fail_fast_effect_batches: bool,
68}
69
70impl Default for HarnessRuntimeConfig {
71 fn default() -> Self {
72 Self {
73 max_agent_steps: 256,
74 max_effect_batch_concurrency: 1,
75 fail_fast_effect_batches: false,
76 }
77 }
78}
79
80impl HarnessRuntimeConfig {
81 pub fn new() -> Self {
83 Self::default()
84 }
85
86 pub fn max_agent_steps(&self) -> usize {
88 self.max_agent_steps
89 }
90
91 pub fn with_max_agent_steps(mut self, max_agent_steps: usize) -> Self {
93 self.max_agent_steps = max_agent_steps;
94 self
95 }
96
97 pub fn max_effect_batch_concurrency(&self) -> usize {
99 self.max_effect_batch_concurrency
100 }
101
102 pub fn with_max_effect_batch_concurrency(
104 mut self,
105 max_effect_batch_concurrency: usize,
106 ) -> Self {
107 self.max_effect_batch_concurrency = max_effect_batch_concurrency;
108 self
109 }
110
111 pub fn fail_fast_effect_batches(&self) -> bool {
113 self.fail_fast_effect_batches
114 }
115
116 pub fn with_fail_fast_effect_batches(mut self, fail_fast_effect_batches: bool) -> Self {
118 self.fail_fast_effect_batches = fail_fast_effect_batches;
119 self
120 }
121}
122
123impl<P, H> HarnessRuntime<P, H>
124where
125 P: Provider,
126 H: Harness,
127{
128 pub fn new(provider: P, harness: H) -> Self {
130 Self {
131 provider,
132 harness,
133 config: HarnessRuntimeConfig::default(),
134 }
135 }
136
137 pub fn with_config(mut self, config: HarnessRuntimeConfig) -> Self {
139 self.config = config;
140 self
141 }
142
143 pub async fn run<K>(
153 &self,
154 kernel: &mut K,
155 request: RunRequest,
156 context: RunContext,
157 ) -> Result<RunOutput, HarnessRuntimeError>
158 where
159 K: AgentKernel,
160 {
161 check_run_context(&context)?;
162 let mut action = kernel.start(request, &context).await?;
163 for _ in 0..self.config.max_agent_steps {
164 check_run_context(&context)?;
165 let observation = match action {
166 AgentAction::Respond { output } => return Ok(output),
167 AgentAction::RequestModel { request } => {
168 Observation::Model(self.execute_model_request(request, &context).await?)
169 }
170 AgentAction::RequestEffect { request } => {
171 let observation = self.harness.execute(request, &context).await?;
172 Observation::Effect(observation)
173 }
174 AgentAction::RequestEffects { requests } => {
175 let observations = self.execute_effect_batch(requests, &context).await?;
176 Observation::Effects(observations)
177 }
178 _ => {
179 return Err(
180 AgentError::InvalidStep("unsupported agent action".to_string()).into(),
181 );
182 }
183 };
184 action = kernel.observe(observation, &context).await?;
185 }
186 Err(HarnessRuntimeError::TooManyAgentSteps(
187 self.config.max_agent_steps,
188 ))
189 }
190
191 async fn execute_effect_batch(
192 &self,
193 requests: Vec<EffectRequest>,
194 context: &RunContext,
195 ) -> Result<Vec<EffectObservation>, HarnessError> {
196 check_run_context(context)?;
197 if requests.is_empty() {
198 return Ok(Vec::new());
199 }
200
201 let concurrency = self.config.max_effect_batch_concurrency.max(1);
202 if concurrency == 1 {
203 let mut observations = Vec::with_capacity(requests.len());
204 for request in requests {
205 let effect_id = request.id.clone();
206 match self.harness.execute(request, context).await {
207 Ok(observation) => observations.push(observation),
208 Err(error) if self.config.fail_fast_effect_batches => return Err(error),
209 Err(error) if is_terminal_context_error(&error) => return Err(error),
210 Err(error) => {
211 observations.push(observation_from_harness_error(effect_id, error))
212 }
213 }
214 }
215 return Ok(observations);
216 }
217
218 let request_count = requests.len();
219 let mut pending = requests.into_iter().enumerate();
220 let mut in_flight = FuturesUnordered::new();
221 let mut observations: Vec<Option<EffectObservation>> = std::iter::repeat_with(|| None)
222 .take(request_count)
223 .collect();
224
225 for _ in 0..concurrency {
226 let Some((index, request)) = pending.next() else {
227 break;
228 };
229 in_flight.push(execute_indexed_effect(
230 &self.harness,
231 index,
232 request,
233 context,
234 ));
235 }
236
237 while let Some((index, effect_id, result)) = in_flight.next().await {
238 match result {
239 Ok(observation) => observations[index] = Some(observation),
240 Err(error) if self.config.fail_fast_effect_batches => return Err(error),
241 Err(error) if is_terminal_context_error(&error) => return Err(error),
242 Err(error) => {
243 observations[index] = Some(observation_from_harness_error(effect_id, error))
244 }
245 }
246
247 if let Some((next_index, request)) = pending.next() {
248 in_flight.push(execute_indexed_effect(
249 &self.harness,
250 next_index,
251 request,
252 context,
253 ));
254 }
255 }
256
257 Ok(observations
258 .into_iter()
259 .map(|observation| {
260 observation.expect("batch scheduler must fill every requested observation")
261 })
262 .collect())
263 }
264
265 async fn execute_model_request(
266 &self,
267 request: ModelRequest,
268 context: &RunContext,
269 ) -> Result<ModelObservation, ProviderError> {
270 let request_id = request.id;
271 let provider_context = ProviderRequestContext::from_run_context(&request_id, context);
272 let response = self
273 .provider
274 .chat_with_context(request.chat, &provider_context)
275 .await?;
276 Ok(ModelObservation::new(request_id, response))
277 }
278}
279
280async fn execute_indexed_effect<'a, H>(
281 harness: &'a H,
282 index: usize,
283 request: EffectRequest,
284 context: &'a RunContext,
285) -> (usize, String, Result<EffectObservation, HarnessError>)
286where
287 H: Harness,
288{
289 let effect_id = request.id.clone();
290 let result = harness.execute(request, context).await;
291 (index, effect_id, result)
292}
293
294#[async_trait]
296pub trait Harness: Send + Sync {
297 async fn execute(
299 &self,
300 request: EffectRequest,
301 context: &RunContext,
302 ) -> Result<EffectObservation, HarnessError>;
303
304 async fn execute_batch(
309 &self,
310 requests: Vec<EffectRequest>,
311 context: &RunContext,
312 ) -> Result<Vec<EffectObservation>, HarnessError> {
313 let mut observations = Vec::with_capacity(requests.len());
314 for request in requests {
315 observations.push(self.execute(request, context).await?);
316 }
317 Ok(observations)
318 }
319}
320
321#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
323pub struct ClassifiedEffect {
324 pub request: EffectRequest,
326 pub requested_risk: RiskLevel,
328 pub effective_risk: RiskLevel,
330 pub reasons: Vec<String>,
332 pub metadata: RunMetadata,
334}
335
336#[async_trait]
338pub trait RiskClassifier: Send + Sync {
339 async fn classify(
341 &self,
342 request: EffectRequest,
343 context: &RunContext,
344 ) -> Result<ClassifiedEffect, HarnessError>;
345}
346
347#[derive(Debug, Default, Clone, Copy)]
349pub struct DefaultRiskClassifier;
350
351#[async_trait]
352impl RiskClassifier for DefaultRiskClassifier {
353 async fn classify(
354 &self,
355 request: EffectRequest,
356 _context: &RunContext,
357 ) -> Result<ClassifiedEffect, HarnessError> {
358 validate_effect_request(&request)?;
359 let requested_risk = request.risk;
360 let kind_floor = risk_floor_for_kind(&request.kind);
361 let mut effective_risk = max_risk(requested_risk, kind_floor);
362 let mut reasons = vec![format!("kind floor: {:?}", kind_floor)];
363
364 let payload_text = request.payload.to_string().to_ascii_lowercase();
365 let description = request.description.to_ascii_lowercase();
366 let combined = format!("{description} {payload_text}");
367 if contains_critical_pattern(&combined) {
368 effective_risk = max_risk(effective_risk, RiskLevel::Critical);
369 reasons.push("critical payload pattern".to_string());
370 } else if contains_high_pattern(&combined) {
371 effective_risk = max_risk(effective_risk, RiskLevel::High);
372 reasons.push("high-risk payload pattern".to_string());
373 }
374
375 Ok(ClassifiedEffect {
376 request,
377 requested_risk,
378 effective_risk,
379 reasons,
380 metadata: RunMetadata::new(),
381 })
382 }
383}
384
385#[derive(Debug, Clone, PartialEq, Eq)]
387#[non_exhaustive]
388pub enum PolicyDecision {
389 Allow,
391 Deny {
393 reason: String,
395 },
396 RequireApproval {
398 reason: String,
400 },
401}
402
403#[async_trait]
405pub trait PolicyEngine: Send + Sync {
406 async fn evaluate(
408 &self,
409 effect: &ClassifiedEffect,
410 context: &RunContext,
411 ) -> Result<PolicyDecision, HarnessError>;
412}
413
414#[derive(Debug, Default, Clone, Copy)]
419pub struct DefaultPolicyEngine;
420
421#[async_trait]
422impl PolicyEngine for DefaultPolicyEngine {
423 async fn evaluate(
424 &self,
425 effect: &ClassifiedEffect,
426 _context: &RunContext,
427 ) -> Result<PolicyDecision, HarnessError> {
428 Ok(match effect.effective_risk {
429 RiskLevel::Low | RiskLevel::Medium => PolicyDecision::Allow,
430 RiskLevel::High => PolicyDecision::RequireApproval {
431 reason: "high-risk effect requires approval".to_string(),
432 },
433 RiskLevel::Critical => PolicyDecision::Deny {
434 reason: "critical-risk effect denied by default policy".to_string(),
435 },
436 _ => PolicyDecision::RequireApproval {
437 reason: "unknown-risk effect requires approval".to_string(),
438 },
439 })
440 }
441}
442
443#[derive(Debug, Clone, PartialEq, Eq)]
445pub struct ApprovalRequest {
446 pub run_id: String,
448 pub effect_id: String,
450 pub kind: EffectKind,
452 pub description: String,
454 pub risk: RiskLevel,
456 pub reason: String,
458 pub payload_summary: String,
460 pub sandbox: SandboxPolicy,
462 pub network: NetworkPolicy,
464 pub metadata: RunMetadata,
466}
467
468#[derive(Debug, Clone, PartialEq, Eq)]
470#[non_exhaustive]
471pub enum ApprovalDecision {
472 AllowOnce,
474 AllowForSession,
476 Deny {
478 reason: String,
480 },
481}
482
483#[async_trait]
485pub trait ApprovalBroker: Send + Sync {
486 async fn approve(
488 &self,
489 request: ApprovalRequest,
490 context: &RunContext,
491 ) -> Result<ApprovalDecision, ApprovalError>;
492}
493
494#[derive(Debug, Default, Clone, Copy)]
496pub struct AlwaysAllowApprovalBroker;
497
498#[async_trait]
499impl ApprovalBroker for AlwaysAllowApprovalBroker {
500 async fn approve(
501 &self,
502 _request: ApprovalRequest,
503 _context: &RunContext,
504 ) -> Result<ApprovalDecision, ApprovalError> {
505 Ok(ApprovalDecision::AllowOnce)
506 }
507}
508
509#[derive(Debug, Default, Clone, Copy)]
511pub struct AlwaysDenyApprovalBroker;
512
513#[async_trait]
514impl ApprovalBroker for AlwaysDenyApprovalBroker {
515 async fn approve(
516 &self,
517 _request: ApprovalRequest,
518 _context: &RunContext,
519 ) -> Result<ApprovalDecision, ApprovalError> {
520 Ok(ApprovalDecision::Deny {
521 reason: "denied by approval broker".to_string(),
522 })
523 }
524}
525
526#[derive(Debug, Clone)]
528pub struct StaticApprovalBroker {
529 decision: ApprovalDecision,
530}
531
532impl StaticApprovalBroker {
533 pub fn new(decision: ApprovalDecision) -> Self {
535 Self { decision }
536 }
537}
538
539#[async_trait]
540impl ApprovalBroker for StaticApprovalBroker {
541 async fn approve(
542 &self,
543 _request: ApprovalRequest,
544 _context: &RunContext,
545 ) -> Result<ApprovalDecision, ApprovalError> {
546 Ok(self.decision.clone())
547 }
548}
549
550#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
552#[non_exhaustive]
553pub enum SandboxPolicy {
554 ReadOnly,
556 WorkspaceWrite,
558 FullAccess,
560 Custom(String),
562}
563
564#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
566#[non_exhaustive]
567pub enum NetworkPolicy {
568 Deny,
570 AllowListed(Vec<String>),
572 AllowAll,
574 Custom(String),
576}
577
578#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
580#[serde(default)]
581#[non_exhaustive]
582pub struct OutputLimit {
583 pub(crate) model_bytes: usize,
585 pub(crate) display_bytes: usize,
587 pub(crate) debug_bytes: usize,
589}
590
591impl Default for OutputLimit {
592 fn default() -> Self {
593 Self {
594 model_bytes: 64 * 1024,
595 display_bytes: 256 * 1024,
596 debug_bytes: 16 * 1024,
597 }
598 }
599}
600
601impl OutputLimit {
602 pub fn new(model_bytes: usize, display_bytes: usize, debug_bytes: usize) -> Self {
604 Self {
605 model_bytes,
606 display_bytes,
607 debug_bytes,
608 }
609 }
610
611 pub fn model_bytes(&self) -> usize {
613 self.model_bytes
614 }
615
616 pub fn with_model_bytes(mut self, model_bytes: usize) -> Self {
618 self.model_bytes = model_bytes;
619 self
620 }
621
622 pub fn display_bytes(&self) -> usize {
624 self.display_bytes
625 }
626
627 pub fn with_display_bytes(mut self, display_bytes: usize) -> Self {
629 self.display_bytes = display_bytes;
630 self
631 }
632
633 pub fn debug_bytes(&self) -> usize {
635 self.debug_bytes
636 }
637
638 pub fn with_debug_bytes(mut self, debug_bytes: usize) -> Self {
640 self.debug_bytes = debug_bytes;
641 self
642 }
643}
644
645#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
647#[serde(default)]
648#[non_exhaustive]
649pub struct ExecutionPolicy {
650 pub(crate) sandbox: SandboxPolicy,
652 pub(crate) network: NetworkPolicy,
654 pub(crate) timeout: Option<Duration>,
656 pub(crate) output_limit: OutputLimit,
658}
659
660impl Default for ExecutionPolicy {
661 fn default() -> Self {
662 Self {
663 sandbox: SandboxPolicy::ReadOnly,
664 network: NetworkPolicy::Deny,
665 timeout: Some(Duration::from_secs(30)),
666 output_limit: OutputLimit::default(),
667 }
668 }
669}
670
671impl ExecutionPolicy {
672 pub fn new(sandbox: SandboxPolicy, network: NetworkPolicy) -> Self {
674 Self {
675 sandbox,
676 network,
677 ..Self::default()
678 }
679 }
680
681 pub fn sandbox(&self) -> &SandboxPolicy {
683 &self.sandbox
684 }
685
686 pub fn with_sandbox(mut self, sandbox: SandboxPolicy) -> Self {
688 self.sandbox = sandbox;
689 self
690 }
691
692 pub fn network(&self) -> &NetworkPolicy {
694 &self.network
695 }
696
697 pub fn with_network(mut self, network: NetworkPolicy) -> Self {
699 self.network = network;
700 self
701 }
702
703 pub fn timeout(&self) -> Option<Duration> {
705 self.timeout
706 }
707
708 pub fn with_timeout(mut self, timeout: Option<Duration>) -> Self {
710 self.timeout = timeout;
711 self
712 }
713
714 pub fn output_limit(&self) -> &OutputLimit {
716 &self.output_limit
717 }
718
719 pub fn with_output_limit(mut self, output_limit: OutputLimit) -> Self {
721 self.output_limit = output_limit;
722 self
723 }
724}
725
726#[async_trait]
728pub trait EffectExecutor: Send + Sync {
729 async fn execute(
731 &self,
732 request: &EffectRequest,
733 policy: &ExecutionPolicy,
734 context: &RunContext,
735 ) -> Result<RawEffectOutput, ExecutionError>;
736}
737
738#[derive(Debug, Clone, PartialEq)]
740pub struct RawEffectOutput {
741 pub observation_for_model: String,
743 pub display: Option<DisplayOutput>,
745 pub artifacts: Vec<Artifact>,
747 pub metadata: RunMetadata,
749 pub debug: Option<String>,
751}
752
753impl RawEffectOutput {
754 pub fn text(observation_for_model: impl Into<String>) -> Self {
756 Self {
757 observation_for_model: observation_for_model.into(),
758 display: None,
759 artifacts: Vec::new(),
760 metadata: RunMetadata::new(),
761 debug: None,
762 }
763 }
764
765 pub fn with_display(mut self, display: DisplayOutput) -> Self {
767 self.display = Some(display);
768 self
769 }
770
771 pub fn with_artifacts(mut self, artifacts: Vec<Artifact>) -> Self {
773 self.artifacts = artifacts;
774 self
775 }
776
777 pub fn with_metadata(mut self, metadata: RunMetadata) -> Self {
779 self.metadata = metadata;
780 self
781 }
782
783 pub fn with_debug(mut self, debug: impl Into<String>) -> Self {
785 self.debug = Some(debug.into());
786 self
787 }
788}
789
790#[derive(Debug, Default, Clone, Copy)]
792pub struct NoopEffectExecutor;
793
794#[async_trait]
795impl EffectExecutor for NoopEffectExecutor {
796 async fn execute(
797 &self,
798 request: &EffectRequest,
799 _policy: &ExecutionPolicy,
800 _context: &RunContext,
801 ) -> Result<RawEffectOutput, ExecutionError> {
802 Err(ExecutionError::Unsupported(format!(
803 "effect kind {:?} is not supported by NoopEffectExecutor",
804 request.kind
805 )))
806 }
807}
808
809#[derive(Debug, Clone, Default)]
811pub struct StaticEffectExecutor {
812 outputs: BTreeMap<String, Result<RawEffectOutput, ExecutionError>>,
813}
814
815impl StaticEffectExecutor {
816 pub fn new() -> Self {
818 Self::default()
819 }
820
821 pub fn with_output(mut self, effect_id: impl Into<String>, output: RawEffectOutput) -> Self {
823 self.outputs.insert(effect_id.into(), Ok(output));
824 self
825 }
826
827 pub fn with_error(mut self, effect_id: impl Into<String>, error: ExecutionError) -> Self {
829 self.outputs.insert(effect_id.into(), Err(error));
830 self
831 }
832}
833
834#[async_trait]
835impl EffectExecutor for StaticEffectExecutor {
836 async fn execute(
837 &self,
838 request: &EffectRequest,
839 _policy: &ExecutionPolicy,
840 _context: &RunContext,
841 ) -> Result<RawEffectOutput, ExecutionError> {
842 match self.outputs.get(&request.id) {
843 Some(Ok(output)) => Ok(output.clone()),
844 Some(Err(error)) => Err(error.clone()),
845 None => Err(ExecutionError::Unsupported(format!(
846 "no static output for effect {}",
847 request.id
848 ))),
849 }
850 }
851}
852
853#[derive(Default, Clone)]
855pub struct RouterEffectExecutor {
856 routes: HashMap<EffectKindKey, Arc<dyn EffectExecutor>>,
857}
858
859impl fmt::Debug for RouterEffectExecutor {
860 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
861 f.debug_struct("RouterEffectExecutor")
862 .field("routes", &self.routes.keys().collect::<Vec<_>>())
863 .finish()
864 }
865}
866
867impl RouterEffectExecutor {
868 pub fn new() -> Self {
870 Self::default()
871 }
872
873 pub fn route(mut self, kind: EffectKind, executor: impl EffectExecutor + 'static) -> Self {
875 self.routes
876 .insert(EffectKindKey::from(kind), Arc::new(executor));
877 self
878 }
879}
880
881#[async_trait]
882impl EffectExecutor for RouterEffectExecutor {
883 async fn execute(
884 &self,
885 request: &EffectRequest,
886 policy: &ExecutionPolicy,
887 context: &RunContext,
888 ) -> Result<RawEffectOutput, ExecutionError> {
889 let key = EffectKindKey::from(request.kind.clone());
890 let Some(executor) = self.routes.get(&key) else {
891 return Err(ExecutionError::Unsupported(format!(
892 "no executor registered for effect kind {:?}",
893 request.kind
894 )));
895 };
896 executor.execute(request, policy, context).await
897 }
898}
899
900#[derive(Debug, Clone, PartialEq)]
902pub struct LimitedOutput {
903 pub output: EffectOutput,
905 pub truncated: bool,
907 pub redactions: Vec<RedactionRecord>,
909 pub debug: Option<String>,
911}
912
913#[derive(Debug, Clone, PartialEq, Eq)]
915pub struct RedactedText {
916 pub text: String,
918 pub redactions: Vec<RedactionRecord>,
920}
921
922pub trait Redactor: Send + Sync {
924 fn redact_model_text(&self, text: &str) -> RedactedText;
926
927 fn redact_display_text(&self, text: &str) -> RedactedText;
929
930 fn redact_debug_text(&self, text: &str) -> RedactedText;
932
933 fn redact_metadata(&self, metadata: RunMetadata) -> RunMetadata;
935}
936
937#[derive(Debug, Default, Clone, Copy)]
939pub struct NoopRedactor;
940
941impl Redactor for NoopRedactor {
942 fn redact_model_text(&self, text: &str) -> RedactedText {
943 RedactedText {
944 text: text.to_string(),
945 redactions: Vec::new(),
946 }
947 }
948
949 fn redact_display_text(&self, text: &str) -> RedactedText {
950 RedactedText {
951 text: text.to_string(),
952 redactions: Vec::new(),
953 }
954 }
955
956 fn redact_debug_text(&self, text: &str) -> RedactedText {
957 RedactedText {
958 text: text.to_string(),
959 redactions: Vec::new(),
960 }
961 }
962
963 fn redact_metadata(&self, metadata: RunMetadata) -> RunMetadata {
964 metadata
965 }
966}
967
968#[derive(Debug, Clone)]
973pub struct PatternRedactor {
974 patterns: Vec<String>,
975 replacement: String,
976}
977
978impl PatternRedactor {
979 pub fn new(patterns: impl IntoIterator<Item = impl Into<String>>) -> Self {
981 Self {
982 patterns: patterns.into_iter().map(Into::into).collect(),
983 replacement: "[REDACTED]".to_string(),
984 }
985 }
986
987 pub fn with_replacement(mut self, replacement: impl Into<String>) -> Self {
989 self.replacement = replacement.into();
990 self
991 }
992
993 fn redact_field(&self, field: &str, text: &str) -> RedactedText {
994 let mut redacted = text.to_string();
995 let mut records = Vec::new();
996 for pattern in &self.patterns {
997 if pattern.is_empty() || !redacted.contains(pattern) {
998 continue;
999 }
1000 redacted = redacted.replace(pattern, &self.replacement);
1001 records.push(RedactionRecord {
1002 field: field.to_string(),
1003 reason: "pattern match".to_string(),
1004 });
1005 }
1006 RedactedText {
1007 text: redacted,
1008 redactions: records,
1009 }
1010 }
1011}
1012
1013impl Redactor for PatternRedactor {
1014 fn redact_model_text(&self, text: &str) -> RedactedText {
1015 self.redact_field("model", text)
1016 }
1017
1018 fn redact_display_text(&self, text: &str) -> RedactedText {
1019 self.redact_field("display", text)
1020 }
1021
1022 fn redact_debug_text(&self, text: &str) -> RedactedText {
1023 self.redact_field("debug", text)
1024 }
1025
1026 fn redact_metadata(&self, mut metadata: RunMetadata) -> RunMetadata {
1027 for value in metadata.values_mut() {
1028 redact_json_value(value, &self.patterns, &self.replacement);
1029 }
1030 metadata
1031 }
1032}
1033
1034#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1036#[non_exhaustive]
1037pub enum AuditEvent {
1038 EffectRequested {
1040 effect_id: String,
1042 kind: EffectKind,
1044 description: String,
1046 risk: RiskLevel,
1048 },
1049 EffectClassified {
1051 effect_id: String,
1053 requested_risk: RiskLevel,
1055 effective_risk: RiskLevel,
1057 reasons: Vec<String>,
1059 },
1060 PolicyDecided {
1062 effect_id: String,
1064 decision: String,
1066 },
1067 ApprovalRequested {
1069 effect_id: String,
1071 reason: String,
1073 },
1074 ApprovalDecided {
1076 effect_id: String,
1078 decision: String,
1080 },
1081 EffectStarted {
1083 effect_id: String,
1085 policy: ExecutionPolicySummary,
1087 },
1088 EffectCompleted {
1090 effect_id: String,
1092 truncated: bool,
1094 },
1095 EffectDenied {
1097 effect_id: String,
1099 reason: String,
1101 },
1102 EffectFailed {
1104 effect_id: String,
1106 reason: String,
1108 },
1109 EffectTimedOut {
1111 effect_id: String,
1113 reason: String,
1115 },
1116 EffectCancelled {
1118 effect_id: String,
1120 reason: String,
1122 },
1123}
1124
1125#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1127pub struct ExecutionPolicySummary {
1128 pub sandbox: String,
1130 pub network: String,
1132 pub timeout_ms: Option<u64>,
1134}
1135
1136#[async_trait]
1138pub trait AuditSink: Send + Sync {
1139 async fn record(&self, event: AuditEvent, context: &RunContext) -> Result<(), AuditError>;
1141}
1142
1143#[derive(Debug, Default, Clone, Copy)]
1148pub struct NoopAuditSink;
1149
1150#[async_trait]
1151impl AuditSink for NoopAuditSink {
1152 async fn record(&self, _event: AuditEvent, _context: &RunContext) -> Result<(), AuditError> {
1153 Ok(())
1154 }
1155}
1156
1157#[derive(Debug, Default, Clone)]
1159pub struct VecAuditSink {
1160 events: Arc<Mutex<Vec<AuditEvent>>>,
1161}
1162
1163impl VecAuditSink {
1164 pub fn new() -> Self {
1166 Self::default()
1167 }
1168
1169 pub fn events(&self) -> Vec<AuditEvent> {
1171 self.events
1172 .lock()
1173 .expect("VecAuditSink lock poisoned")
1174 .clone()
1175 }
1176}
1177
1178#[async_trait]
1179impl AuditSink for VecAuditSink {
1180 async fn record(&self, event: AuditEvent, _context: &RunContext) -> Result<(), AuditError> {
1181 self.events
1182 .lock()
1183 .expect("VecAuditSink lock poisoned")
1184 .push(event);
1185 Ok(())
1186 }
1187}
1188
1189#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1191#[non_exhaustive]
1192pub enum TranscriptRecord {
1193 RunStarted {
1195 run_id: String,
1197 request: RunRequest,
1199 },
1200 AgentAction {
1202 run_id: String,
1204 action: AgentActionSummary,
1206 },
1207 EffectRequested {
1209 run_id: String,
1211 effect_id: String,
1213 kind: EffectKind,
1215 description: String,
1217 risk: RiskLevel,
1219 },
1220 EffectClassified {
1222 run_id: String,
1224 effect_id: String,
1226 requested_risk: RiskLevel,
1228 effective_risk: RiskLevel,
1230 reasons: Vec<String>,
1232 },
1233 PolicyDecided {
1235 run_id: String,
1237 effect_id: String,
1239 decision: String,
1241 },
1242 ApprovalRequested {
1244 run_id: String,
1246 effect_id: String,
1248 reason: String,
1250 },
1251 ApprovalDecided {
1253 run_id: String,
1255 effect_id: String,
1257 decision: String,
1259 },
1260 ExecutorStarted {
1262 run_id: String,
1264 effect_id: String,
1266 policy: ExecutionPolicySummary,
1268 },
1269 ExecutorCompleted {
1271 run_id: String,
1273 effect_id: String,
1275 status: EffectStatus,
1277 },
1278 ModelObservation {
1280 run_id: String,
1282 request_id: String,
1284 summary: ModelSummary,
1286 },
1287 EffectObservation {
1289 run_id: String,
1291 effect_id: String,
1293 status: EffectStatus,
1295 },
1296 RunCompleted {
1298 run_id: String,
1300 output: RunOutput,
1302 },
1303 RunFailed {
1305 run_id: String,
1307 error: String,
1309 },
1310}
1311
1312#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1314#[non_exhaustive]
1315pub enum AgentActionSummary {
1316 Respond,
1318 RequestModel {
1320 request_id: String,
1322 },
1323 RequestEffect {
1325 effect_id: String,
1327 },
1328 RequestEffects {
1330 effect_ids: Vec<String>,
1332 },
1333}
1334
1335#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1337pub struct ModelSummary {
1338 pub content_bytes: usize,
1340 pub tool_calls: usize,
1342}
1343
1344#[async_trait]
1346pub trait TranscriptStore: Send + Sync {
1347 async fn append(
1349 &self,
1350 record: TranscriptRecord,
1351 context: &RunContext,
1352 ) -> Result<(), TranscriptError>;
1353}
1354
1355#[derive(Debug, Default, Clone, Copy)]
1357pub struct NoopTranscriptStore;
1358
1359#[async_trait]
1360impl TranscriptStore for NoopTranscriptStore {
1361 async fn append(
1362 &self,
1363 _record: TranscriptRecord,
1364 _context: &RunContext,
1365 ) -> Result<(), TranscriptError> {
1366 Ok(())
1367 }
1368}
1369
1370#[derive(Debug, Default, Clone)]
1372pub struct VecTranscriptStore {
1373 records: Arc<Mutex<Vec<TranscriptRecord>>>,
1374}
1375
1376impl VecTranscriptStore {
1377 pub fn new() -> Self {
1379 Self::default()
1380 }
1381
1382 pub fn records(&self) -> Vec<TranscriptRecord> {
1384 self.records
1385 .lock()
1386 .expect("VecTranscriptStore lock poisoned")
1387 .clone()
1388 }
1389}
1390
1391#[async_trait]
1392impl TranscriptStore for VecTranscriptStore {
1393 async fn append(
1394 &self,
1395 record: TranscriptRecord,
1396 _context: &RunContext,
1397 ) -> Result<(), TranscriptError> {
1398 self.records
1399 .lock()
1400 .expect("VecTranscriptStore lock poisoned")
1401 .push(record);
1402 Ok(())
1403 }
1404}
1405
1406pub struct BasicHarness<E, P, A, S, T> {
1408 executor: E,
1409 policy: P,
1410 approval: A,
1411 audit: S,
1412 transcript: T,
1413 classifier: DefaultRiskClassifier,
1414 redactor: Arc<dyn Redactor>,
1415 session_approvals: Mutex<Vec<SessionApproval>>,
1416 config: HarnessConfig,
1417}
1418
1419impl<E, P, A, S, T> fmt::Debug for BasicHarness<E, P, A, S, T>
1420where
1421 E: fmt::Debug,
1422 P: fmt::Debug,
1423 A: fmt::Debug,
1424 S: fmt::Debug,
1425 T: fmt::Debug,
1426{
1427 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1428 f.debug_struct("BasicHarness")
1429 .field("executor", &self.executor)
1430 .field("policy", &self.policy)
1431 .field("approval", &self.approval)
1432 .field("audit", &self.audit)
1433 .field("transcript", &self.transcript)
1434 .field("classifier", &self.classifier)
1435 .field("redactor", &"dyn Redactor")
1436 .field("session_approvals", &self.session_approvals)
1437 .field("config", &self.config)
1438 .finish()
1439 }
1440}
1441
1442#[derive(Debug, Clone, PartialEq, Eq)]
1443struct SessionApproval {
1444 run_id: String,
1445 kind: EffectKindKey,
1446 risk_ceiling: RiskLevel,
1447}
1448
1449#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1451#[serde(default)]
1452#[non_exhaustive]
1453pub struct HarnessConfig {
1454 pub(crate) default_sandbox: SandboxPolicy,
1456 pub(crate) default_network: NetworkPolicy,
1458 pub(crate) default_timeout: Duration,
1460 pub(crate) output_limit: OutputLimit,
1462 pub(crate) fail_closed_on_audit_error: bool,
1464 pub(crate) fail_closed_on_transcript_error: bool,
1466}
1467
1468impl Default for HarnessConfig {
1469 fn default() -> Self {
1470 Self {
1471 default_sandbox: SandboxPolicy::ReadOnly,
1472 default_network: NetworkPolicy::Deny,
1473 default_timeout: Duration::from_secs(30),
1474 output_limit: OutputLimit::default(),
1475 fail_closed_on_audit_error: true,
1476 fail_closed_on_transcript_error: false,
1477 }
1478 }
1479}
1480
1481impl HarnessConfig {
1482 pub fn new() -> Self {
1484 Self::default()
1485 }
1486
1487 pub fn default_sandbox(&self) -> &SandboxPolicy {
1489 &self.default_sandbox
1490 }
1491
1492 pub fn with_default_sandbox(mut self, default_sandbox: SandboxPolicy) -> Self {
1494 self.default_sandbox = default_sandbox;
1495 self
1496 }
1497
1498 pub fn default_network(&self) -> &NetworkPolicy {
1500 &self.default_network
1501 }
1502
1503 pub fn with_default_network(mut self, default_network: NetworkPolicy) -> Self {
1505 self.default_network = default_network;
1506 self
1507 }
1508
1509 pub fn default_timeout(&self) -> Duration {
1511 self.default_timeout
1512 }
1513
1514 pub fn with_default_timeout(mut self, default_timeout: Duration) -> Self {
1516 self.default_timeout = default_timeout;
1517 self
1518 }
1519
1520 pub fn output_limit(&self) -> &OutputLimit {
1522 &self.output_limit
1523 }
1524
1525 pub fn with_output_limit(mut self, output_limit: OutputLimit) -> Self {
1527 self.output_limit = output_limit;
1528 self
1529 }
1530
1531 pub fn fail_closed_on_audit_error(&self) -> bool {
1533 self.fail_closed_on_audit_error
1534 }
1535
1536 pub fn with_fail_closed_on_audit_error(mut self, fail_closed_on_audit_error: bool) -> Self {
1538 self.fail_closed_on_audit_error = fail_closed_on_audit_error;
1539 self
1540 }
1541
1542 pub fn fail_closed_on_transcript_error(&self) -> bool {
1544 self.fail_closed_on_transcript_error
1545 }
1546
1547 pub fn with_fail_closed_on_transcript_error(
1549 mut self,
1550 fail_closed_on_transcript_error: bool,
1551 ) -> Self {
1552 self.fail_closed_on_transcript_error = fail_closed_on_transcript_error;
1553 self
1554 }
1555}
1556
1557impl
1558 BasicHarness<
1559 NoopEffectExecutor,
1560 DefaultPolicyEngine,
1561 AlwaysDenyApprovalBroker,
1562 NoopAuditSink,
1563 NoopTranscriptStore,
1564 >
1565{
1566 pub fn noop() -> Self {
1568 Self::new(
1569 NoopEffectExecutor,
1570 DefaultPolicyEngine,
1571 AlwaysDenyApprovalBroker,
1572 NoopAuditSink,
1573 NoopTranscriptStore,
1574 )
1575 }
1576}
1577
1578impl<E, P, A, S, T> BasicHarness<E, P, A, S, T>
1579where
1580 E: EffectExecutor,
1581 P: PolicyEngine,
1582 A: ApprovalBroker,
1583 S: AuditSink,
1584 T: TranscriptStore,
1585{
1586 pub fn new(executor: E, policy: P, approval: A, audit: S, transcript: T) -> Self {
1588 Self {
1589 executor,
1590 policy,
1591 approval,
1592 audit,
1593 transcript,
1594 classifier: DefaultRiskClassifier,
1595 redactor: Arc::new(NoopRedactor),
1596 session_approvals: Mutex::new(Vec::new()),
1597 config: HarnessConfig::default(),
1598 }
1599 }
1600
1601 pub fn with_config(mut self, config: HarnessConfig) -> Self {
1603 self.config = config;
1604 self
1605 }
1606
1607 pub fn with_noop_redactor(mut self) -> Self {
1609 self.redactor = Arc::new(NoopRedactor);
1610 self
1611 }
1612
1613 pub fn with_redactor(mut self, redactor: impl Redactor + 'static) -> Self {
1616 self.redactor = Arc::new(redactor);
1617 self
1618 }
1619
1620 async fn audit(&self, event: AuditEvent, context: &RunContext) -> Result<(), HarnessError> {
1621 match self.audit.record(event, context).await {
1622 Ok(()) => Ok(()),
1623 Err(error) if self.config.fail_closed_on_audit_error => Err(error.into()),
1624 Err(_) => Ok(()),
1625 }
1626 }
1627
1628 async fn transcript(
1629 &self,
1630 record: TranscriptRecord,
1631 context: &RunContext,
1632 ) -> Result<(), HarnessError> {
1633 match self.transcript.append(record, context).await {
1634 Ok(()) => Ok(()),
1635 Err(error) if self.config.fail_closed_on_transcript_error => Err(error.into()),
1636 Err(_) => Ok(()),
1637 }
1638 }
1639}
1640
1641#[async_trait]
1642impl<E, P, A, S, T> Harness for BasicHarness<E, P, A, S, T>
1643where
1644 E: EffectExecutor,
1645 P: PolicyEngine,
1646 A: ApprovalBroker,
1647 S: AuditSink,
1648 T: TranscriptStore,
1649{
1650 async fn execute(
1651 &self,
1652 request: EffectRequest,
1653 context: &RunContext,
1654 ) -> Result<EffectObservation, HarnessError> {
1655 check_run_context(context)?;
1656 validate_effect_request(&request)?;
1657 self.audit(
1658 AuditEvent::EffectRequested {
1659 effect_id: request.id.clone(),
1660 kind: request.kind.clone(),
1661 description: request.description.clone(),
1662 risk: request.risk,
1663 },
1664 context,
1665 )
1666 .await?;
1667 self.transcript(
1668 TranscriptRecord::EffectRequested {
1669 run_id: context.run_id.clone(),
1670 effect_id: request.id.clone(),
1671 kind: request.kind.clone(),
1672 description: request.description.clone(),
1673 risk: request.risk,
1674 },
1675 context,
1676 )
1677 .await?;
1678
1679 let classified = self.classifier.classify(request, context).await?;
1680 self.audit(
1681 AuditEvent::EffectClassified {
1682 effect_id: classified.request.id.clone(),
1683 requested_risk: classified.requested_risk,
1684 effective_risk: classified.effective_risk,
1685 reasons: classified.reasons.clone(),
1686 },
1687 context,
1688 )
1689 .await?;
1690 self.transcript(
1691 TranscriptRecord::EffectClassified {
1692 run_id: context.run_id.clone(),
1693 effect_id: classified.request.id.clone(),
1694 requested_risk: classified.requested_risk,
1695 effective_risk: classified.effective_risk,
1696 reasons: classified.reasons.clone(),
1697 },
1698 context,
1699 )
1700 .await?;
1701
1702 let execution_policy = self.execution_policy(&classified, context);
1703 let decision = self.policy.evaluate(&classified, context).await?;
1704 let decision_summary = policy_decision_summary(&decision);
1705 self.audit(
1706 AuditEvent::PolicyDecided {
1707 effect_id: classified.request.id.clone(),
1708 decision: decision_summary.clone(),
1709 },
1710 context,
1711 )
1712 .await?;
1713 self.transcript(
1714 TranscriptRecord::PolicyDecided {
1715 run_id: context.run_id.clone(),
1716 effect_id: classified.request.id.clone(),
1717 decision: decision_summary,
1718 },
1719 context,
1720 )
1721 .await?;
1722
1723 match decision {
1724 PolicyDecision::Allow => {}
1725 PolicyDecision::Deny { reason } => {
1726 return self
1727 .denied_observation(classified.request, reason, context)
1728 .await;
1729 }
1730 PolicyDecision::RequireApproval { reason } => {
1731 if self.is_session_approved(&classified, context) {
1732 self.audit(
1733 AuditEvent::ApprovalDecided {
1734 effect_id: classified.request.id.clone(),
1735 decision: "allow for session".to_string(),
1736 },
1737 context,
1738 )
1739 .await?;
1740 self.transcript(
1741 TranscriptRecord::ApprovalDecided {
1742 run_id: context.run_id.clone(),
1743 effect_id: classified.request.id.clone(),
1744 decision: "allow for session".to_string(),
1745 },
1746 context,
1747 )
1748 .await?;
1749 } else {
1750 let approval_request = ApprovalRequest {
1751 run_id: context.run_id.clone(),
1752 effect_id: classified.request.id.clone(),
1753 kind: classified.request.kind.clone(),
1754 description: classified.request.description.clone(),
1755 risk: classified.effective_risk,
1756 reason: reason.clone(),
1757 payload_summary: payload_summary(&classified.request.payload),
1758 sandbox: execution_policy.sandbox.clone(),
1759 network: execution_policy.network.clone(),
1760 metadata: classified.request.metadata.clone(),
1761 };
1762 self.audit(
1763 AuditEvent::ApprovalRequested {
1764 effect_id: classified.request.id.clone(),
1765 reason: reason.clone(),
1766 },
1767 context,
1768 )
1769 .await?;
1770 self.transcript(
1771 TranscriptRecord::ApprovalRequested {
1772 run_id: context.run_id.clone(),
1773 effect_id: classified.request.id.clone(),
1774 reason,
1775 },
1776 context,
1777 )
1778 .await?;
1779 let approval = self.approval.approve(approval_request, context).await?;
1780 let approval_summary = approval_decision_summary(&approval);
1781 self.audit(
1782 AuditEvent::ApprovalDecided {
1783 effect_id: classified.request.id.clone(),
1784 decision: approval_summary.clone(),
1785 },
1786 context,
1787 )
1788 .await?;
1789 self.transcript(
1790 TranscriptRecord::ApprovalDecided {
1791 run_id: context.run_id.clone(),
1792 effect_id: classified.request.id.clone(),
1793 decision: approval_summary,
1794 },
1795 context,
1796 )
1797 .await?;
1798 if let ApprovalDecision::Deny { reason } = approval {
1799 return self
1800 .denied_observation(classified.request, reason, context)
1801 .await;
1802 }
1803 if matches!(approval, ApprovalDecision::AllowForSession) {
1804 self.remember_session_approval(&classified, context);
1805 }
1806 }
1807 }
1808 }
1809
1810 self.audit(
1811 AuditEvent::EffectStarted {
1812 effect_id: classified.request.id.clone(),
1813 policy: ExecutionPolicySummary::from_policy(&execution_policy),
1814 },
1815 context,
1816 )
1817 .await?;
1818 self.transcript(
1819 TranscriptRecord::ExecutorStarted {
1820 run_id: context.run_id.clone(),
1821 effect_id: classified.request.id.clone(),
1822 policy: ExecutionPolicySummary::from_policy(&execution_policy),
1823 },
1824 context,
1825 )
1826 .await?;
1827 let effect_id = classified.request.id.clone();
1828 let execution = run_executor_with_context(
1829 &self.executor,
1830 &classified.request,
1831 &execution_policy,
1832 context,
1833 )
1834 .await;
1835 match execution {
1836 Ok(raw) => {
1837 let limited =
1838 limit_and_redact(raw, &self.config.output_limit, self.redactor.as_ref());
1839 self.audit(
1840 AuditEvent::EffectCompleted {
1841 effect_id: effect_id.clone(),
1842 truncated: limited.truncated,
1843 },
1844 context,
1845 )
1846 .await?;
1847 self.transcript(
1848 TranscriptRecord::ExecutorCompleted {
1849 run_id: context.run_id.clone(),
1850 effect_id: effect_id.clone(),
1851 status: EffectStatus::Succeeded,
1852 },
1853 context,
1854 )
1855 .await?;
1856 let mut observation_metadata = RunMetadata::new();
1857 observation_metadata.insert(
1858 "truncated".to_string(),
1859 serde_json::json!(limited.truncated),
1860 );
1861 observation_metadata.insert(
1862 "redactions_applied".to_string(),
1863 serde_json::json!(limited.redactions.len()),
1864 );
1865 let observation = EffectObservation {
1866 effect_id: effect_id.clone(),
1867 status: EffectStatus::Succeeded,
1868 output: limited.output,
1869 metadata: observation_metadata,
1870 };
1871 self.transcript(
1872 TranscriptRecord::EffectObservation {
1873 run_id: context.run_id.clone(),
1874 effect_id,
1875 status: observation.status.clone(),
1876 },
1877 context,
1878 )
1879 .await?;
1880 Ok(observation)
1881 }
1882 Err(ExecutionError::TimedOut(reason)) => {
1883 self.audit(
1884 AuditEvent::EffectTimedOut {
1885 effect_id: effect_id.clone(),
1886 reason: reason.clone(),
1887 },
1888 context,
1889 )
1890 .await?;
1891 self.transcript(
1892 TranscriptRecord::ExecutorCompleted {
1893 run_id: context.run_id.clone(),
1894 effect_id: effect_id.clone(),
1895 status: EffectStatus::TimedOut,
1896 },
1897 context,
1898 )
1899 .await?;
1900 self.terminal_observation(
1901 effect_id,
1902 EffectStatus::TimedOut,
1903 format!("effect timed out: {reason}"),
1904 context,
1905 )
1906 .await
1907 }
1908 Err(ExecutionError::Denied(reason)) => {
1909 self.audit(
1910 AuditEvent::EffectDenied {
1911 effect_id: effect_id.clone(),
1912 reason: reason.clone(),
1913 },
1914 context,
1915 )
1916 .await?;
1917 self.transcript(
1918 TranscriptRecord::ExecutorCompleted {
1919 run_id: context.run_id.clone(),
1920 effect_id: effect_id.clone(),
1921 status: EffectStatus::Denied,
1922 },
1923 context,
1924 )
1925 .await?;
1926 self.terminal_observation(
1927 effect_id,
1928 EffectStatus::Denied,
1929 format!("effect denied: {reason}"),
1930 context,
1931 )
1932 .await
1933 }
1934 Err(ExecutionError::Cancelled(reason)) => {
1935 self.audit(
1936 AuditEvent::EffectCancelled {
1937 effect_id: effect_id.clone(),
1938 reason: reason.clone(),
1939 },
1940 context,
1941 )
1942 .await?;
1943 self.transcript(
1944 TranscriptRecord::ExecutorCompleted {
1945 run_id: context.run_id.clone(),
1946 effect_id: effect_id.clone(),
1947 status: EffectStatus::Cancelled,
1948 },
1949 context,
1950 )
1951 .await?;
1952 self.terminal_observation(
1953 effect_id,
1954 EffectStatus::Cancelled,
1955 format!("effect cancelled: {reason}"),
1956 context,
1957 )
1958 .await
1959 }
1960 Err(error) => {
1961 let reason = error.to_string();
1962 self.audit(
1963 AuditEvent::EffectFailed {
1964 effect_id: effect_id.clone(),
1965 reason: reason.clone(),
1966 },
1967 context,
1968 )
1969 .await?;
1970 self.transcript(
1971 TranscriptRecord::ExecutorCompleted {
1972 run_id: context.run_id.clone(),
1973 effect_id: effect_id.clone(),
1974 status: EffectStatus::Failed,
1975 },
1976 context,
1977 )
1978 .await?;
1979 self.terminal_observation(
1980 effect_id,
1981 EffectStatus::Failed,
1982 format!("effect failed: {reason}"),
1983 context,
1984 )
1985 .await
1986 }
1987 }
1988 }
1989}
1990
1991impl<E, P, A, S, T> BasicHarness<E, P, A, S, T>
1992where
1993 E: EffectExecutor,
1994 P: PolicyEngine,
1995 A: ApprovalBroker,
1996 S: AuditSink,
1997 T: TranscriptStore,
1998{
1999 fn execution_policy(
2000 &self,
2001 classified: &ClassifiedEffect,
2002 context: &RunContext,
2003 ) -> ExecutionPolicy {
2004 let mut timeout = Some(self.config.default_timeout);
2005 if let Some(request_timeout) = classified.request.timeout {
2006 timeout = Some(timeout.map_or(request_timeout, |default| default.min(request_timeout)));
2007 }
2008 if let Some(remaining) = context.remaining() {
2009 timeout = Some(timeout.map_or(remaining, |current| current.min(remaining)));
2010 }
2011 ExecutionPolicy {
2012 sandbox: self.config.default_sandbox.clone(),
2013 network: self.config.default_network.clone(),
2014 timeout,
2015 output_limit: self.config.output_limit.clone(),
2016 }
2017 }
2018
2019 fn is_session_approved(&self, classified: &ClassifiedEffect, context: &RunContext) -> bool {
2020 let kind = EffectKindKey::from(classified.request.kind.clone());
2021 let approvals = self
2022 .session_approvals
2023 .lock()
2024 .expect("BasicHarness session approval lock poisoned");
2025 approvals.iter().any(|approval| {
2026 approval.run_id == context.run_id
2027 && approval.kind == kind
2028 && risk_rank(classified.effective_risk) <= risk_rank(approval.risk_ceiling)
2029 })
2030 }
2031
2032 fn remember_session_approval(&self, classified: &ClassifiedEffect, context: &RunContext) {
2033 let approval = SessionApproval {
2034 run_id: context.run_id.clone(),
2035 kind: EffectKindKey::from(classified.request.kind.clone()),
2036 risk_ceiling: classified.effective_risk,
2037 };
2038 let mut approvals = self
2039 .session_approvals
2040 .lock()
2041 .expect("BasicHarness session approval lock poisoned");
2042 if !approvals.contains(&approval) {
2043 approvals.push(approval);
2044 }
2045 }
2046
2047 async fn denied_observation(
2048 &self,
2049 request: EffectRequest,
2050 reason: String,
2051 context: &RunContext,
2052 ) -> Result<EffectObservation, HarnessError>
2053 where
2054 S: AuditSink,
2055 T: TranscriptStore,
2056 {
2057 self.audit(
2058 AuditEvent::EffectDenied {
2059 effect_id: request.id.clone(),
2060 reason: reason.clone(),
2061 },
2062 context,
2063 )
2064 .await?;
2065 self.terminal_observation(
2066 request.id,
2067 EffectStatus::Denied,
2068 format!("effect denied: {reason}"),
2069 context,
2070 )
2071 .await
2072 }
2073
2074 async fn terminal_observation(
2075 &self,
2076 effect_id: String,
2077 status: EffectStatus,
2078 observation_for_model: String,
2079 context: &RunContext,
2080 ) -> Result<EffectObservation, HarnessError>
2081 where
2082 T: TranscriptStore,
2083 {
2084 let observation = EffectObservation {
2085 effect_id: effect_id.clone(),
2086 status,
2087 output: EffectOutput::text(observation_for_model),
2088 metadata: RunMetadata::new(),
2089 };
2090 self.transcript(
2091 TranscriptRecord::EffectObservation {
2092 run_id: context.run_id.clone(),
2093 effect_id,
2094 status: observation.status.clone(),
2095 },
2096 context,
2097 )
2098 .await?;
2099 Ok(observation)
2100 }
2101}
2102
2103#[derive(Debug, thiserror::Error)]
2105#[non_exhaustive]
2106pub enum HarnessError {
2107 #[error("invalid effect request: {0}")]
2109 InvalidRequest(String),
2110 #[error("policy error: {0}")]
2112 Policy(String),
2113 #[error("approval error: {0}")]
2115 Approval(#[from] ApprovalError),
2116 #[error("execution error: {0}")]
2118 Execution(#[from] ExecutionError),
2119 #[error("audit error: {0}")]
2121 Audit(#[from] AuditError),
2122 #[error("transcript error: {0}")]
2124 Transcript(#[from] TranscriptError),
2125 #[error("run cancelled")]
2127 Cancelled,
2128 #[error("run deadline exceeded")]
2130 DeadlineExceeded,
2131}
2132
2133#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
2135#[non_exhaustive]
2136pub enum ApprovalError {
2137 #[error("{0}")]
2139 Broker(String),
2140}
2141
2142#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
2144#[non_exhaustive]
2145pub enum ExecutionError {
2146 #[error("unsupported effect: {0}")]
2148 Unsupported(String),
2149 #[error("failed: {0}")]
2151 Failed(String),
2152 #[error("denied: {0}")]
2154 Denied(String),
2155 #[error("timed out: {0}")]
2157 TimedOut(String),
2158 #[error("cancelled: {0}")]
2160 Cancelled(String),
2161}
2162
2163#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
2165#[non_exhaustive]
2166pub enum AuditError {
2167 #[error("{0}")]
2169 Sink(String),
2170}
2171
2172#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
2174#[non_exhaustive]
2175pub enum TranscriptError {
2176 #[error("{0}")]
2178 Store(String),
2179}
2180
2181#[derive(Debug, thiserror::Error)]
2183#[non_exhaustive]
2184pub enum HarnessRuntimeError {
2185 #[error("agent error: {0}")]
2187 Agent(#[from] AgentError),
2188 #[error("provider error: {0}")]
2190 Provider(#[from] ProviderError),
2191 #[error("harness error: {0}")]
2193 Harness(#[from] HarnessError),
2194 #[error("too many agent steps: {0}")]
2196 TooManyAgentSteps(usize),
2197}
2198
2199impl ExecutionPolicySummary {
2200 fn from_policy(policy: &ExecutionPolicy) -> Self {
2201 Self {
2202 sandbox: format!("{:?}", policy.sandbox),
2203 network: format!("{:?}", policy.network),
2204 timeout_ms: policy
2205 .timeout
2206 .map(|timeout| timeout.as_millis().min(u128::from(u64::MAX)) as u64),
2207 }
2208 }
2209}
2210
2211#[derive(Debug, Clone, PartialEq, Eq, Hash)]
2212enum EffectKindKey {
2213 ReadFile,
2214 WriteFile,
2215 ApplyPatch,
2216 Search,
2217 ExecuteCommand,
2218 Git,
2219 Network,
2220 Browser,
2221 Mcp,
2222 Custom(String),
2223}
2224
2225impl From<EffectKind> for EffectKindKey {
2226 fn from(kind: EffectKind) -> Self {
2227 match kind {
2228 EffectKind::ReadFile => Self::ReadFile,
2229 EffectKind::WriteFile => Self::WriteFile,
2230 EffectKind::ApplyPatch => Self::ApplyPatch,
2231 EffectKind::Search => Self::Search,
2232 EffectKind::ExecuteCommand => Self::ExecuteCommand,
2233 EffectKind::Git => Self::Git,
2234 EffectKind::Network => Self::Network,
2235 EffectKind::Browser => Self::Browser,
2236 EffectKind::Mcp => Self::Mcp,
2237 EffectKind::Custom(kind) => Self::Custom(kind),
2238 _ => Self::Custom("unknown".to_string()),
2239 }
2240 }
2241}
2242
2243fn validate_effect_request(request: &EffectRequest) -> Result<(), HarnessError> {
2244 if request.id.trim().is_empty() {
2245 return Err(HarnessError::InvalidRequest(
2246 "effect id must not be empty".to_string(),
2247 ));
2248 }
2249 if request.description.trim().is_empty() {
2250 return Err(HarnessError::InvalidRequest(
2251 "effect description must not be empty".to_string(),
2252 ));
2253 }
2254 Ok(())
2255}
2256
2257fn check_run_context(context: &RunContext) -> Result<(), HarnessError> {
2258 if context.is_cancelled() {
2259 Err(HarnessError::Cancelled)
2260 } else if context.is_expired() {
2261 Err(HarnessError::DeadlineExceeded)
2262 } else {
2263 Ok(())
2264 }
2265}
2266
2267fn is_terminal_context_error(error: &HarnessError) -> bool {
2268 matches!(
2269 error,
2270 HarnessError::Cancelled | HarnessError::DeadlineExceeded
2271 )
2272}
2273
2274fn observation_from_harness_error(effect_id: String, error: HarnessError) -> EffectObservation {
2275 let mut metadata = RunMetadata::new();
2276 metadata.insert("harness_error".to_string(), serde_json::json!(true));
2277 EffectObservation {
2278 effect_id,
2279 status: EffectStatus::Failed,
2280 output: EffectOutput::text(format!("effect failed: {error}")),
2281 metadata,
2282 }
2283}
2284
2285async fn run_executor_with_context<E>(
2286 executor: &E,
2287 request: &EffectRequest,
2288 policy: &ExecutionPolicy,
2289 context: &RunContext,
2290) -> Result<RawEffectOutput, ExecutionError>
2291where
2292 E: EffectExecutor,
2293{
2294 if context.is_cancelled() {
2295 return Err(ExecutionError::Cancelled("run cancelled".to_string()));
2296 }
2297 if context.is_expired() {
2298 return Err(ExecutionError::TimedOut(
2299 "run deadline exceeded".to_string(),
2300 ));
2301 }
2302 match policy.timeout {
2303 Some(timeout) if timeout.is_zero() => {
2304 Err(ExecutionError::TimedOut("timeout elapsed".to_string()))
2305 }
2306 Some(timeout) => {
2307 tokio::select! {
2308 _ = context.cancellation.cancelled() => {
2309 Err(ExecutionError::Cancelled("run cancelled".to_string()))
2310 }
2311 _ = tokio::time::sleep(timeout) => {
2312 Err(ExecutionError::TimedOut(format!("exceeded {timeout:?}")))
2313 }
2314 output = executor.execute(request, policy, context) => output,
2315 }
2316 }
2317 None => {
2318 tokio::select! {
2319 _ = context.cancellation.cancelled() => {
2320 Err(ExecutionError::Cancelled("run cancelled".to_string()))
2321 }
2322 output = executor.execute(request, policy, context) => output,
2323 }
2324 }
2325 }
2326}
2327
2328fn limit_and_redact(
2329 raw: RawEffectOutput,
2330 limit: &OutputLimit,
2331 redactor: &(impl Redactor + ?Sized),
2332) -> LimitedOutput {
2333 let model_redacted = redactor.redact_model_text(&raw.observation_for_model);
2334 let (mut model_text, model_truncated) = truncate_with_marker(
2335 model_redacted.text,
2336 limit.model_bytes,
2337 "\n[output truncated]",
2338 );
2339
2340 let mut redactions = model_redacted.redactions;
2341 let mut truncated = model_truncated;
2342 if model_truncated && !model_text.contains("[output truncated]") {
2343 model_text.push_str("\n[output truncated]");
2344 }
2345
2346 let display = raw.display.map(|display| {
2347 let display_redacted = redactor.redact_display_text(&display.content);
2348 redactions.extend(display_redacted.redactions);
2349 let (content, was_truncated) =
2350 truncate_with_marker(display_redacted.text, limit.display_bytes, "\n[truncated]");
2351 truncated |= was_truncated;
2352 DisplayOutput {
2353 content,
2354 metadata: redactor.redact_metadata(display.metadata),
2355 ..display
2356 }
2357 });
2358
2359 let debug = raw.debug.map(|debug| {
2360 let debug_redacted = redactor.redact_debug_text(&debug);
2361 redactions.extend(debug_redacted.redactions);
2362 let (debug, was_truncated) =
2363 truncate_with_marker(debug_redacted.text, limit.debug_bytes, "\n[truncated]");
2364 truncated |= was_truncated;
2365 debug
2366 });
2367
2368 LimitedOutput {
2369 output: EffectOutput::text(model_text)
2370 .with_artifacts(raw.artifacts)
2371 .with_metadata(redactor.redact_metadata(raw.metadata)),
2372 truncated,
2373 redactions,
2374 debug,
2375 }
2376 .with_display(display)
2377}
2378
2379impl LimitedOutput {
2380 fn with_display(mut self, display: Option<DisplayOutput>) -> Self {
2381 self.output.display = display;
2382 self
2383 }
2384}
2385
2386fn truncate_with_marker(mut text: String, limit: usize, marker: &str) -> (String, bool) {
2387 if text.len() <= limit {
2388 return (text, false);
2389 }
2390 if limit == 0 {
2391 return (String::new(), true);
2392 }
2393
2394 let marker_len = marker.len().min(limit);
2395 let keep = limit.saturating_sub(marker_len);
2396 let mut end = keep.min(text.len());
2397 while !text.is_char_boundary(end) {
2398 end -= 1;
2399 }
2400 text.truncate(end);
2401 if marker_len == marker.len() {
2402 text.push_str(marker);
2403 }
2404 (text, true)
2405}
2406
2407fn risk_floor_for_kind(kind: &EffectKind) -> RiskLevel {
2408 match kind {
2409 EffectKind::ReadFile | EffectKind::Search => RiskLevel::Low,
2410 EffectKind::WriteFile
2411 | EffectKind::ApplyPatch
2412 | EffectKind::ExecuteCommand
2413 | EffectKind::Git
2414 | EffectKind::Network
2415 | EffectKind::Browser
2416 | EffectKind::Mcp
2417 | EffectKind::Custom(_) => RiskLevel::Medium,
2418 _ => RiskLevel::Medium,
2419 }
2420}
2421
2422fn max_risk(left: RiskLevel, right: RiskLevel) -> RiskLevel {
2423 if risk_rank(left) >= risk_rank(right) {
2424 left
2425 } else {
2426 right
2427 }
2428}
2429
2430fn risk_rank(risk: RiskLevel) -> u8 {
2431 match risk {
2432 RiskLevel::Low => 0,
2433 RiskLevel::Medium => 1,
2434 RiskLevel::High => 2,
2435 RiskLevel::Critical => 3,
2436 _ => 3,
2437 }
2438}
2439
2440fn contains_high_pattern(text: &str) -> bool {
2441 ["sudo", "force push", "--force", " outside workspace"]
2442 .iter()
2443 .any(|pattern| text.contains(pattern))
2444}
2445
2446fn contains_critical_pattern(text: &str) -> bool {
2447 [
2448 "rm -rf /",
2449 "mkfs",
2450 "dd if=",
2451 "shutdown",
2452 "reboot",
2453 "chmod -r 777 /",
2454 ]
2455 .iter()
2456 .any(|pattern| text.contains(pattern))
2457}
2458
2459fn payload_summary(payload: &serde_json::Value) -> String {
2460 payload.to_string().chars().take(512).collect()
2461}
2462
2463fn policy_decision_summary(decision: &PolicyDecision) -> String {
2464 match decision {
2465 PolicyDecision::Allow => "allow".to_string(),
2466 PolicyDecision::Deny { reason } => format!("deny: {reason}"),
2467 PolicyDecision::RequireApproval { reason } => format!("require approval: {reason}"),
2468 }
2469}
2470
2471fn approval_decision_summary(decision: &ApprovalDecision) -> String {
2472 match decision {
2473 ApprovalDecision::AllowOnce => "allow once".to_string(),
2474 ApprovalDecision::AllowForSession => "allow for session".to_string(),
2475 ApprovalDecision::Deny { reason } => format!("deny: {reason}"),
2476 }
2477}
2478
2479fn redact_json_value(value: &mut serde_json::Value, patterns: &[String], replacement: &str) {
2480 match value {
2481 serde_json::Value::String(text) => {
2482 for pattern in patterns {
2483 if !pattern.is_empty() && text.contains(pattern) {
2484 *text = text.replace(pattern, replacement);
2485 }
2486 }
2487 }
2488 serde_json::Value::Array(values) => {
2489 for value in values {
2490 redact_json_value(value, patterns, replacement);
2491 }
2492 }
2493 serde_json::Value::Object(map) => {
2494 for value in map.values_mut() {
2495 redact_json_value(value, patterns, replacement);
2496 }
2497 }
2498 serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => {}
2499 }
2500}
2501
2502#[cfg(test)]
2503mod tests {
2504 use super::*;
2505 use crate::agent::{AgentKernel, ModelRequest};
2506 use crate::message::Message;
2507 use crate::provider::{FakeProvider, FakeReply, ProviderError};
2508 use crate::tool::{Tool, ToolContext, ToolError, ToolRegistry, ToolResult, ToolSchema};
2509 use serde_json::json;
2510
2511 struct SingleModelKernel;
2512
2513 #[async_trait]
2514 impl AgentKernel for SingleModelKernel {
2515 async fn start(
2516 &mut self,
2517 _request: RunRequest,
2518 _context: &RunContext,
2519 ) -> Result<AgentAction, AgentError> {
2520 Ok(AgentAction::RequestModel {
2521 request: ModelRequest::new("model-1", Default::default()),
2522 })
2523 }
2524
2525 async fn observe(
2526 &mut self,
2527 observation: Observation,
2528 context: &RunContext,
2529 ) -> Result<AgentAction, AgentError> {
2530 let Observation::Model(observation) = observation else {
2531 return Err(AgentError::InvalidStep(
2532 "expected model observation".to_string(),
2533 ));
2534 };
2535 let answer = match &observation.response.message {
2536 Message::Assistant { content, .. } => content.clone(),
2537 _ => String::new(),
2538 };
2539 Ok(AgentAction::Respond {
2540 output: RunOutput {
2541 run_id: context.run_id.clone(),
2542 answer,
2543 summary: Default::default(),
2544 final_message: observation.response.message,
2545 artifacts: Vec::new(),
2546 metadata: RunMetadata::new(),
2547 },
2548 })
2549 }
2550 }
2551
2552 struct EffectKernel {
2553 observed: bool,
2554 }
2555
2556 #[async_trait]
2557 impl AgentKernel for EffectKernel {
2558 async fn start(
2559 &mut self,
2560 _request: RunRequest,
2561 _context: &RunContext,
2562 ) -> Result<AgentAction, AgentError> {
2563 Ok(AgentAction::RequestEffect {
2564 request: EffectRequest::new(EffectKind::ReadFile, "read config", json!({}))
2565 .with_id("effect-1"),
2566 })
2567 }
2568
2569 async fn observe(
2570 &mut self,
2571 observation: Observation,
2572 context: &RunContext,
2573 ) -> Result<AgentAction, AgentError> {
2574 let Observation::Effect(observation) = observation else {
2575 return Err(AgentError::InvalidStep(
2576 "expected effect observation".to_string(),
2577 ));
2578 };
2579 self.observed = true;
2580 Ok(AgentAction::Respond {
2581 output: RunOutput {
2582 run_id: context.run_id.clone(),
2583 answer: observation.output.observation_for_model,
2584 summary: Default::default(),
2585 final_message: Message::assistant("done"),
2586 artifacts: Vec::new(),
2587 metadata: RunMetadata::new(),
2588 },
2589 })
2590 }
2591 }
2592
2593 #[tokio::test]
2594 async fn runtime_drives_model_request() {
2595 let provider = FakeProvider::new([FakeReply::Text("hello".to_string())]);
2596 let harness = BasicHarness::noop();
2597 let runtime = HarnessRuntime::new(provider, harness);
2598 let output = runtime
2599 .run(
2600 &mut SingleModelKernel,
2601 RunRequest::text("hi"),
2602 RunContext::new("run-model"),
2603 )
2604 .await
2605 .unwrap();
2606 assert_eq!(output.answer, "hello");
2607 }
2608
2609 #[tokio::test]
2610 async fn runtime_drives_effect_request() {
2611 let provider = FakeProvider::new([]);
2612 let executor = StaticEffectExecutor::new()
2613 .with_output("effect-1", RawEffectOutput::text("file contents"));
2614 let audit = VecAuditSink::new();
2615 let harness = BasicHarness::new(
2616 executor,
2617 DefaultPolicyEngine,
2618 AlwaysAllowApprovalBroker,
2619 audit.clone(),
2620 NoopTranscriptStore,
2621 );
2622 let runtime = HarnessRuntime::new(provider, harness);
2623 let output = runtime
2624 .run(
2625 &mut EffectKernel { observed: false },
2626 RunRequest::text("read"),
2627 RunContext::new("run-effect"),
2628 )
2629 .await
2630 .unwrap();
2631 assert_eq!(output.answer, "file contents");
2632 assert!(audit
2633 .events()
2634 .iter()
2635 .any(|event| matches!(event, AuditEvent::EffectCompleted { effect_id, .. } if effect_id == "effect-1")));
2636 }
2637
2638 #[tokio::test]
2639 async fn runtime_returns_provider_error() {
2640 let provider = FakeProvider::new([FakeReply::Error(ProviderError::Api {
2641 status: 500,
2642 code: None,
2643 message: "provider down".to_string(),
2644 })]);
2645 let harness = BasicHarness::noop();
2646 let runtime = HarnessRuntime::new(provider, harness);
2647 let err = runtime
2648 .run(
2649 &mut SingleModelKernel,
2650 RunRequest::text("hi"),
2651 RunContext::new("run-provider-error"),
2652 )
2653 .await
2654 .unwrap_err();
2655
2656 assert!(matches!(err, HarnessRuntimeError::Provider(_)));
2657 }
2658
2659 struct LoopKernel;
2660
2661 #[async_trait]
2662 impl AgentKernel for LoopKernel {
2663 async fn start(
2664 &mut self,
2665 _request: RunRequest,
2666 _context: &RunContext,
2667 ) -> Result<AgentAction, AgentError> {
2668 Ok(AgentAction::RequestModel {
2669 request: ModelRequest::new("model-1", Default::default()),
2670 })
2671 }
2672
2673 async fn observe(
2674 &mut self,
2675 _observation: Observation,
2676 _context: &RunContext,
2677 ) -> Result<AgentAction, AgentError> {
2678 Ok(AgentAction::RequestModel {
2679 request: ModelRequest::new("model-loop", Default::default()),
2680 })
2681 }
2682 }
2683
2684 #[tokio::test]
2685 async fn runtime_limits_agent_steps() {
2686 let provider = FakeProvider::new([
2687 FakeReply::Text("one".to_string()),
2688 FakeReply::Text("two".to_string()),
2689 FakeReply::Text("three".to_string()),
2690 ]);
2691 let harness = BasicHarness::noop();
2692 let runtime = HarnessRuntime::new(provider, harness).with_config(HarnessRuntimeConfig {
2693 max_agent_steps: 1,
2694 ..Default::default()
2695 });
2696 let err = runtime
2697 .run(
2698 &mut LoopKernel,
2699 RunRequest::text("hi"),
2700 RunContext::new("run-step-limit"),
2701 )
2702 .await
2703 .unwrap_err();
2704
2705 assert!(matches!(err, HarnessRuntimeError::TooManyAgentSteps(1)));
2706 }
2707
2708 #[tokio::test]
2709 async fn react_kernel_runs_effect_through_runtime() {
2710 let provider = FakeProvider::new([
2711 FakeReply::ToolCalls {
2712 content: String::new(),
2713 calls: vec![crate::ToolCall {
2714 id: "call-1".to_string(),
2715 name: "read_file".to_string(),
2716 arguments: "{}".to_string(),
2717 }],
2718 },
2719 FakeReply::Text("done after observation".to_string()),
2720 ]);
2721 let executor =
2722 StaticEffectExecutor::new().with_output("effect-1", RawEffectOutput::text("observed"));
2723 let harness = BasicHarness::new(
2724 executor,
2725 DefaultPolicyEngine,
2726 AlwaysAllowApprovalBroker,
2727 NoopAuditSink,
2728 NoopTranscriptStore,
2729 );
2730 let runtime = HarnessRuntime::new(provider, harness);
2731 let mut registry = ToolRegistry::new();
2732 registry.register(EffectTool);
2733 let mut kernel = crate::ReActAgent::kernel(registry, "");
2734
2735 let output = runtime
2736 .run(
2737 &mut kernel,
2738 RunRequest::text("read"),
2739 RunContext::new("run-react-runtime"),
2740 )
2741 .await
2742 .unwrap();
2743
2744 assert_eq!(output.answer, "done after observation");
2745 }
2746
2747 #[tokio::test]
2748 async fn basic_harness_denies_critical_effect() {
2749 let harness = BasicHarness::new(
2750 StaticEffectExecutor::new(),
2751 DefaultPolicyEngine,
2752 AlwaysAllowApprovalBroker,
2753 NoopAuditSink,
2754 NoopTranscriptStore,
2755 );
2756 let observation = harness
2757 .execute(
2758 EffectRequest::new(
2759 EffectKind::ExecuteCommand,
2760 "run rm -rf /",
2761 json!({"cmd": "rm -rf /"}),
2762 )
2763 .with_id("critical"),
2764 &RunContext::new("run-deny"),
2765 )
2766 .await
2767 .unwrap();
2768 assert_eq!(observation.status, EffectStatus::Denied);
2769 assert!(
2770 observation
2771 .output
2772 .observation_for_model
2773 .contains("effect denied")
2774 );
2775 }
2776
2777 #[tokio::test]
2778 async fn basic_harness_batch_can_mix_denied_and_succeeded() {
2779 let harness = BasicHarness::new(
2780 StaticEffectExecutor::new()
2781 .with_output("read-ok", RawEffectOutput::text("read succeeded")),
2782 DefaultPolicyEngine,
2783 AlwaysAllowApprovalBroker,
2784 NoopAuditSink,
2785 NoopTranscriptStore,
2786 );
2787 let observations = harness
2788 .execute_batch(
2789 vec![
2790 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id("read-ok"),
2791 EffectRequest::new(
2792 EffectKind::ExecuteCommand,
2793 "run rm -rf /",
2794 json!({"cmd": "rm -rf /"}),
2795 )
2796 .with_id("deny-critical"),
2797 ],
2798 &RunContext::new("run-batch"),
2799 )
2800 .await
2801 .unwrap();
2802
2803 assert_eq!(observations[0].status, EffectStatus::Succeeded);
2804 assert_eq!(observations[1].status, EffectStatus::Denied);
2805 }
2806
2807 #[derive(Debug, Clone)]
2808 struct ConcurrencyHarness {
2809 in_flight: Arc<std::sync::atomic::AtomicUsize>,
2810 max_seen: Arc<std::sync::atomic::AtomicUsize>,
2811 }
2812
2813 #[async_trait]
2814 impl Harness for ConcurrencyHarness {
2815 async fn execute(
2816 &self,
2817 request: EffectRequest,
2818 _context: &RunContext,
2819 ) -> Result<EffectObservation, HarnessError> {
2820 let current = self
2821 .in_flight
2822 .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
2823 + 1;
2824 self.max_seen
2825 .fetch_max(current, std::sync::atomic::Ordering::SeqCst);
2826 let delay = if request.id == "effect-1" { 30 } else { 5 };
2827 tokio::time::sleep(Duration::from_millis(delay)).await;
2828 self.in_flight
2829 .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
2830 Ok(EffectObservation::succeeded(
2831 request.id.clone(),
2832 format!("done {}", request.id),
2833 ))
2834 }
2835 }
2836
2837 #[tokio::test]
2838 async fn runtime_batch_honors_concurrency_limit_and_request_order() {
2839 let in_flight = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2840 let max_seen = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2841 let harness = ConcurrencyHarness {
2842 in_flight,
2843 max_seen: max_seen.clone(),
2844 };
2845 let runtime = HarnessRuntime::new(FakeProvider::new([]), harness)
2846 .with_config(HarnessRuntimeConfig::default().with_max_effect_batch_concurrency(2));
2847 let observations = runtime
2848 .execute_effect_batch(
2849 vec![
2850 EffectRequest::new(EffectKind::ReadFile, "one", json!({})).with_id("effect-1"),
2851 EffectRequest::new(EffectKind::ReadFile, "two", json!({})).with_id("effect-2"),
2852 EffectRequest::new(EffectKind::ReadFile, "three", json!({}))
2853 .with_id("effect-3"),
2854 ],
2855 &RunContext::new("run-batch-runtime"),
2856 )
2857 .await
2858 .unwrap();
2859
2860 assert_eq!(
2861 observations
2862 .iter()
2863 .map(|observation| observation.effect_id.as_str())
2864 .collect::<Vec<_>>(),
2865 ["effect-1", "effect-2", "effect-3"]
2866 );
2867 assert_eq!(max_seen.load(std::sync::atomic::Ordering::SeqCst), 2);
2868 }
2869
2870 #[derive(Debug, Clone)]
2871 struct ErroringHarness {
2872 seen: Arc<Mutex<Vec<String>>>,
2873 }
2874
2875 #[async_trait]
2876 impl Harness for ErroringHarness {
2877 async fn execute(
2878 &self,
2879 request: EffectRequest,
2880 _context: &RunContext,
2881 ) -> Result<EffectObservation, HarnessError> {
2882 self.seen
2883 .lock()
2884 .expect("seen lock poisoned")
2885 .push(request.id.clone());
2886 if request.id == "bad" {
2887 return Err(HarnessError::Execution(ExecutionError::Failed(
2888 "boom".to_string(),
2889 )));
2890 }
2891 Ok(EffectObservation::succeeded(request.id, "ok"))
2892 }
2893 }
2894
2895 #[tokio::test]
2896 async fn runtime_batch_fail_fast_stops_after_first_harness_error() {
2897 let seen = Arc::new(Mutex::new(Vec::new()));
2898 let runtime = HarnessRuntime::new(
2899 FakeProvider::new([]),
2900 ErroringHarness { seen: seen.clone() },
2901 )
2902 .with_config(
2903 HarnessRuntimeConfig::default()
2904 .with_max_effect_batch_concurrency(1)
2905 .with_fail_fast_effect_batches(true),
2906 );
2907 let error = runtime
2908 .execute_effect_batch(
2909 vec![
2910 EffectRequest::new(EffectKind::ReadFile, "bad", json!({})).with_id("bad"),
2911 EffectRequest::new(EffectKind::ReadFile, "later", json!({})).with_id("later"),
2912 ],
2913 &RunContext::new("run-batch-fail-fast"),
2914 )
2915 .await
2916 .unwrap_err();
2917
2918 assert!(matches!(error, HarnessError::Execution(_)));
2919 assert_eq!(seen.lock().expect("seen lock poisoned").as_slice(), ["bad"]);
2920 }
2921
2922 #[tokio::test]
2923 async fn runtime_batch_fail_open_collects_error_observation_and_continues() {
2924 let runtime = HarnessRuntime::new(
2925 FakeProvider::new([]),
2926 ErroringHarness {
2927 seen: Arc::new(Mutex::new(Vec::new())),
2928 },
2929 )
2930 .with_config(
2931 HarnessRuntimeConfig::default()
2932 .with_max_effect_batch_concurrency(1)
2933 .with_fail_fast_effect_batches(false),
2934 );
2935 let observations = runtime
2936 .execute_effect_batch(
2937 vec![
2938 EffectRequest::new(EffectKind::ReadFile, "bad", json!({})).with_id("bad"),
2939 EffectRequest::new(EffectKind::ReadFile, "later", json!({})).with_id("later"),
2940 ],
2941 &RunContext::new("run-batch-fail-open"),
2942 )
2943 .await
2944 .unwrap();
2945
2946 assert_eq!(observations[0].effect_id, "bad");
2947 assert_eq!(observations[0].status, EffectStatus::Failed);
2948 assert_eq!(observations[1].status, EffectStatus::Succeeded);
2949 }
2950
2951 #[tokio::test]
2952 async fn approval_deny_becomes_denied_observation() {
2953 let harness = BasicHarness::new(
2954 StaticEffectExecutor::new(),
2955 DefaultPolicyEngine,
2956 AlwaysDenyApprovalBroker,
2957 NoopAuditSink,
2958 NoopTranscriptStore,
2959 );
2960 let observation = harness
2961 .execute(
2962 EffectRequest::new(
2963 EffectKind::ExecuteCommand,
2964 "run high-risk command",
2965 json!({}),
2966 )
2967 .with_id("approval-deny")
2968 .with_risk(RiskLevel::High),
2969 &RunContext::new("run-approval-deny"),
2970 )
2971 .await
2972 .unwrap();
2973
2974 assert_eq!(observation.status, EffectStatus::Denied);
2975 assert!(
2976 observation
2977 .output
2978 .observation_for_model
2979 .contains("approval broker")
2980 );
2981 }
2982
2983 #[derive(Debug, Clone)]
2984 struct CountingApprovalBroker {
2985 calls: Arc<Mutex<usize>>,
2986 decision: ApprovalDecision,
2987 }
2988
2989 #[async_trait]
2990 impl ApprovalBroker for CountingApprovalBroker {
2991 async fn approve(
2992 &self,
2993 _request: ApprovalRequest,
2994 _context: &RunContext,
2995 ) -> Result<ApprovalDecision, ApprovalError> {
2996 *self.calls.lock().expect("approval counter lock poisoned") += 1;
2997 Ok(self.decision.clone())
2998 }
2999 }
3000
3001 #[tokio::test]
3002 async fn allow_for_session_skips_repeated_approval_for_same_run_and_kind() {
3003 let calls = Arc::new(Mutex::new(0));
3004 let broker = CountingApprovalBroker {
3005 calls: calls.clone(),
3006 decision: ApprovalDecision::AllowForSession,
3007 };
3008 let harness = BasicHarness::new(
3009 StaticEffectExecutor::new()
3010 .with_output("effect-1", RawEffectOutput::text("first"))
3011 .with_output("effect-2", RawEffectOutput::text("second")),
3012 DefaultPolicyEngine,
3013 broker,
3014 NoopAuditSink,
3015 NoopTranscriptStore,
3016 );
3017 let context = RunContext::new("run-session-approval");
3018
3019 let first = harness
3020 .execute(
3021 EffectRequest::new(EffectKind::ExecuteCommand, "run command", json!({}))
3022 .with_id("effect-1")
3023 .with_risk(RiskLevel::High),
3024 &context,
3025 )
3026 .await
3027 .unwrap();
3028 let second = harness
3029 .execute(
3030 EffectRequest::new(EffectKind::ExecuteCommand, "run another command", json!({}))
3031 .with_id("effect-2")
3032 .with_risk(RiskLevel::High),
3033 &context,
3034 )
3035 .await
3036 .unwrap();
3037
3038 assert_eq!(first.status, EffectStatus::Succeeded);
3039 assert_eq!(second.status, EffectStatus::Succeeded);
3040 assert_eq!(*calls.lock().expect("approval counter lock poisoned"), 1);
3041 }
3042
3043 #[tokio::test]
3044 async fn executor_failure_is_effect_observation() {
3045 let harness = BasicHarness::new(
3046 StaticEffectExecutor::new()
3047 .with_error("effect-1", ExecutionError::Failed("boom".to_string())),
3048 DefaultPolicyEngine,
3049 AlwaysAllowApprovalBroker,
3050 NoopAuditSink,
3051 NoopTranscriptStore,
3052 );
3053 let observation = harness
3054 .execute(
3055 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id("effect-1"),
3056 &RunContext::new("run-fail"),
3057 )
3058 .await
3059 .unwrap();
3060 assert_eq!(observation.status, EffectStatus::Failed);
3061 assert!(observation.output.observation_for_model.contains("boom"));
3062 }
3063
3064 #[tokio::test]
3065 async fn executor_denial_is_denied_observation() {
3066 let harness = BasicHarness::new(
3067 StaticEffectExecutor::new().with_error(
3068 "effect-1",
3069 ExecutionError::Denied("capability mismatch".to_string()),
3070 ),
3071 DefaultPolicyEngine,
3072 AlwaysAllowApprovalBroker,
3073 NoopAuditSink,
3074 NoopTranscriptStore,
3075 );
3076 let observation = harness
3077 .execute(
3078 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id("effect-1"),
3079 &RunContext::new("run-denied-by-executor"),
3080 )
3081 .await
3082 .unwrap();
3083
3084 assert_eq!(observation.status, EffectStatus::Denied);
3085 assert!(
3086 observation
3087 .output
3088 .observation_for_model
3089 .contains("capability mismatch")
3090 );
3091 }
3092
3093 #[derive(Debug, Default, Clone, Copy)]
3094 struct SlowExecutor;
3095
3096 #[async_trait]
3097 impl EffectExecutor for SlowExecutor {
3098 async fn execute(
3099 &self,
3100 _request: &EffectRequest,
3101 _policy: &ExecutionPolicy,
3102 _context: &RunContext,
3103 ) -> Result<RawEffectOutput, ExecutionError> {
3104 tokio::time::sleep(Duration::from_millis(50)).await;
3105 Ok(RawEffectOutput::text("late"))
3106 }
3107 }
3108
3109 #[tokio::test]
3110 async fn executor_timeout_becomes_timed_out_observation() {
3111 let harness = BasicHarness::new(
3112 SlowExecutor,
3113 DefaultPolicyEngine,
3114 AlwaysAllowApprovalBroker,
3115 NoopAuditSink,
3116 NoopTranscriptStore,
3117 )
3118 .with_config(HarnessConfig {
3119 default_timeout: Duration::from_millis(1),
3120 ..Default::default()
3121 });
3122 let observation = harness
3123 .execute(
3124 EffectRequest::new(EffectKind::ReadFile, "read slowly", json!({}))
3125 .with_id("slow-effect"),
3126 &RunContext::new("run-timeout"),
3127 )
3128 .await
3129 .unwrap();
3130
3131 assert_eq!(observation.status, EffectStatus::TimedOut);
3132 }
3133
3134 #[tokio::test]
3135 async fn output_is_limited_and_redacted() {
3136 let raw =
3137 RawEffectOutput::text("secret-token-1234567890 plus a long tail that must be cut")
3138 .with_debug("debug secret-token");
3139 let limit = OutputLimit {
3140 model_bytes: 32,
3141 display_bytes: 20,
3142 debug_bytes: 20,
3143 };
3144 let redactor = PatternRedactor::new(["secret-token"]);
3145 let output = limit_and_redact(raw, &limit, &redactor);
3146 assert!(output.truncated);
3147 assert!(!output.output.observation_for_model.contains("secret-token"));
3148 assert!(output.output.observation_for_model.contains("[REDACTED]"));
3149 }
3150
3151 #[tokio::test]
3152 async fn basic_harness_uses_configured_redactor() {
3153 let harness = BasicHarness::new(
3154 StaticEffectExecutor::new()
3155 .with_output("effect-1", RawEffectOutput::text("secret-token in output")),
3156 DefaultPolicyEngine,
3157 AlwaysAllowApprovalBroker,
3158 NoopAuditSink,
3159 NoopTranscriptStore,
3160 )
3161 .with_redactor(PatternRedactor::new(["secret-token"]));
3162
3163 let observation = harness
3164 .execute(
3165 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id("effect-1"),
3166 &RunContext::new("run-redactor"),
3167 )
3168 .await
3169 .unwrap();
3170
3171 assert_eq!(observation.status, EffectStatus::Succeeded);
3172 assert!(
3173 !observation
3174 .output
3175 .observation_for_model
3176 .contains("secret-token")
3177 );
3178 assert!(
3179 observation
3180 .output
3181 .observation_for_model
3182 .contains("[REDACTED]")
3183 );
3184 assert_eq!(
3185 observation
3186 .metadata
3187 .get("redactions_applied")
3188 .and_then(serde_json::Value::as_u64),
3189 Some(1)
3190 );
3191 }
3192
3193 #[tokio::test]
3194 async fn redacted_effect_output_is_used_for_observation_audit_and_transcript() {
3195 let audit = VecAuditSink::new();
3196 let transcript = VecTranscriptStore::new();
3197 let mut metadata = RunMetadata::new();
3198 metadata.insert(
3199 "token".to_string(),
3200 serde_json::Value::String("secret-token".to_string()),
3201 );
3202 let harness = BasicHarness::new(
3203 StaticEffectExecutor::new().with_output(
3204 "effect-1",
3205 RawEffectOutput::text("model sees secret-token")
3206 .with_display(DisplayOutput::new(
3207 crate::effect::DisplayFormat::PlainText,
3208 "display sees secret-token",
3209 ))
3210 .with_metadata(metadata)
3211 .with_debug("debug sees secret-token"),
3212 ),
3213 DefaultPolicyEngine,
3214 AlwaysAllowApprovalBroker,
3215 audit.clone(),
3216 transcript.clone(),
3217 )
3218 .with_redactor(PatternRedactor::new(["secret-token"]));
3219
3220 let observation = harness
3221 .execute(
3222 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id("effect-1"),
3223 &RunContext::new("run-redaction-records"),
3224 )
3225 .await
3226 .unwrap();
3227
3228 assert_eq!(observation.status, EffectStatus::Succeeded);
3229 let observation_json = serde_json::to_string(&observation).unwrap();
3230 assert!(!observation_json.contains("secret-token"));
3231 assert!(observation_json.contains("[REDACTED]"));
3232 assert!(
3233 observation
3234 .metadata
3235 .get("redactions_applied")
3236 .and_then(serde_json::Value::as_u64)
3237 .unwrap_or_default()
3238 > 0
3239 );
3240
3241 let audit_json = serde_json::to_string(&audit.events()).unwrap();
3242 assert!(!audit_json.contains("secret-token"));
3243 let transcript_json = serde_json::to_string(&transcript.records()).unwrap();
3244 assert!(!transcript_json.contains("secret-token"));
3245 }
3246
3247 #[tokio::test]
3248 async fn transcript_records_effect_lifecycle() {
3249 let transcript = VecTranscriptStore::new();
3250 let harness = BasicHarness::new(
3251 StaticEffectExecutor::new().with_output("effect-1", RawEffectOutput::text("ok")),
3252 DefaultPolicyEngine,
3253 AlwaysAllowApprovalBroker,
3254 NoopAuditSink,
3255 transcript.clone(),
3256 );
3257 let observation = harness
3258 .execute(
3259 EffectRequest::new(EffectKind::ReadFile, "read config", json!({}))
3260 .with_id("effect-1"),
3261 &RunContext::new("run-transcript-lifecycle"),
3262 )
3263 .await
3264 .unwrap();
3265 assert_eq!(observation.status, EffectStatus::Succeeded);
3266
3267 let records = transcript.records();
3268 assert!(records.iter().any(
3269 |record| matches!(record, TranscriptRecord::EffectRequested { effect_id, .. } if effect_id == "effect-1")
3270 ));
3271 assert!(records.iter().any(
3272 |record| matches!(record, TranscriptRecord::EffectClassified { effect_id, .. } if effect_id == "effect-1")
3273 ));
3274 assert!(records.iter().any(
3275 |record| matches!(record, TranscriptRecord::PolicyDecided { effect_id, .. } if effect_id == "effect-1")
3276 ));
3277 assert!(records.iter().any(
3278 |record| matches!(record, TranscriptRecord::ExecutorStarted { effect_id, .. } if effect_id == "effect-1")
3279 ));
3280 assert!(records.iter().any(
3281 |record| matches!(record, TranscriptRecord::ExecutorCompleted { effect_id, status, .. } if effect_id == "effect-1" && *status == EffectStatus::Succeeded)
3282 ));
3283 assert!(records.iter().any(
3284 |record| matches!(record, TranscriptRecord::EffectObservation { effect_id, status, .. } if effect_id == "effect-1" && *status == EffectStatus::Succeeded)
3285 ));
3286 }
3287
3288 #[derive(Debug, Default, Clone, Copy)]
3289 struct FailingAuditSink;
3290
3291 #[async_trait]
3292 impl AuditSink for FailingAuditSink {
3293 async fn record(
3294 &self,
3295 _event: AuditEvent,
3296 _context: &RunContext,
3297 ) -> Result<(), AuditError> {
3298 Err(AuditError::Sink("audit offline".to_string()))
3299 }
3300 }
3301
3302 #[tokio::test]
3303 async fn audit_failure_fails_closed_before_execution() {
3304 let harness = BasicHarness::new(
3305 StaticEffectExecutor::new()
3306 .with_output("effect-1", RawEffectOutput::text("should not execute")),
3307 DefaultPolicyEngine,
3308 AlwaysAllowApprovalBroker,
3309 FailingAuditSink,
3310 NoopTranscriptStore,
3311 );
3312 let err = harness
3313 .execute(
3314 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id("effect-1"),
3315 &RunContext::new("run-audit-fail"),
3316 )
3317 .await
3318 .unwrap_err();
3319
3320 assert!(matches!(err, HarnessError::Audit(_)));
3321 }
3322
3323 #[derive(Debug, Default, Clone, Copy)]
3324 struct FailingTranscriptStore;
3325
3326 #[async_trait]
3327 impl TranscriptStore for FailingTranscriptStore {
3328 async fn append(
3329 &self,
3330 _record: TranscriptRecord,
3331 _context: &RunContext,
3332 ) -> Result<(), TranscriptError> {
3333 Err(TranscriptError::Store("transcript offline".to_string()))
3334 }
3335 }
3336
3337 #[tokio::test]
3338 async fn transcript_failure_defaults_to_fail_open() {
3339 let harness = BasicHarness::new(
3340 StaticEffectExecutor::new().with_output("effect-1", RawEffectOutput::text("ok")),
3341 DefaultPolicyEngine,
3342 AlwaysAllowApprovalBroker,
3343 NoopAuditSink,
3344 FailingTranscriptStore,
3345 );
3346 let observation = harness
3347 .execute(
3348 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id("effect-1"),
3349 &RunContext::new("run-transcript-open"),
3350 )
3351 .await
3352 .unwrap();
3353
3354 assert_eq!(observation.status, EffectStatus::Succeeded);
3355 }
3356
3357 #[tokio::test]
3358 async fn transcript_failure_can_fail_closed() {
3359 let harness = BasicHarness::new(
3360 StaticEffectExecutor::new().with_output("effect-1", RawEffectOutput::text("ok")),
3361 DefaultPolicyEngine,
3362 AlwaysAllowApprovalBroker,
3363 NoopAuditSink,
3364 FailingTranscriptStore,
3365 )
3366 .with_config(HarnessConfig {
3367 fail_closed_on_transcript_error: true,
3368 ..Default::default()
3369 });
3370 let err = harness
3371 .execute(
3372 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id("effect-1"),
3373 &RunContext::new("run-transcript-closed"),
3374 )
3375 .await
3376 .unwrap_err();
3377
3378 assert!(matches!(err, HarnessError::Transcript(_)));
3379 }
3380
3381 #[tokio::test]
3382 async fn invalid_effect_request_returns_harness_error() {
3383 let harness = BasicHarness::noop();
3384 let err = harness
3385 .execute(
3386 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id(""),
3387 &RunContext::new("run-invalid"),
3388 )
3389 .await
3390 .unwrap_err();
3391 assert!(matches!(err, HarnessError::InvalidRequest(_)));
3392 }
3393
3394 struct EffectTool;
3395
3396 #[async_trait]
3397 impl Tool for EffectTool {
3398 fn schema(&self) -> ToolSchema {
3399 ToolSchema::new("read_file", "Read a file", json!({}))
3400 }
3401
3402 async fn call(
3403 &self,
3404 _arguments: serde_json::Value,
3405 _context: ToolContext<'_>,
3406 ) -> Result<ToolResult, ToolError> {
3407 Ok(ToolResult::Effect(
3408 EffectRequest::new(EffectKind::ReadFile, "read", json!({})).with_id("effect-1"),
3409 ))
3410 }
3411 }
3412
3413 #[tokio::test]
3414 async fn registry_effect_tool_stays_external_to_harness() {
3415 let mut registry = ToolRegistry::new();
3416 registry.register(EffectTool);
3417 assert_eq!(registry.names(), ["read_file"]);
3418 }
3419}