use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use base64::Engine as _;
use incurs_codemode::{ArtifactStore as _, McpClient};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};
use crate::code::StdioMcpClient;
use crate::execution::{
BUILTIN_MACOS_PROVIDER_GENERATION, BUILTIN_MACOS_PROVIDER_NAME, ProviderExecutionControl,
ProviderExecutionEvent, ProviderExecutionGateway, ProviderExecutionRequest,
};
use crate::image::{image_store, import_verified_bundle};
use crate::model::{
CausalEvent, ExecutionArtifactRef, ExecutionExportSpec, ExecutionExportWhen,
ImageImportResponse, IsolationMutationResponse, IsolationStatus, OperationReceipt,
OperationTarget, ProcessController, ProviderConfig, ProviderKind, ReceiptState, now_ms,
};
use crate::store::Store;
pub struct RegisteredProviderExecutionGateway {
store: Arc<Store>,
builtin_config: Option<ProviderConfig>,
builtin_client: tokio::sync::Mutex<Option<Arc<StdioMcpClient>>>,
isolation_mutation: tokio::sync::Mutex<()>,
}
impl RegisteredProviderExecutionGateway {
pub fn new(store: Arc<Store>) -> Self {
Self {
store,
builtin_config: None,
builtin_client: tokio::sync::Mutex::new(None),
isolation_mutation: tokio::sync::Mutex::new(()),
}
}
#[cfg(test)]
fn with_builtin_config_for_test(store: Arc<Store>, config: ProviderConfig) -> Self {
Self {
store,
builtin_config: Some(config),
builtin_client: tokio::sync::Mutex::new(None),
isolation_mutation: tokio::sync::Mutex::new(()),
}
}
fn config(&self, name: &str, generation: &str) -> Result<ProviderConfig, String> {
if name == BUILTIN_MACOS_PROVIDER_NAME {
if generation != BUILTIN_MACOS_PROVIDER_GENERATION {
return Err(format!(
"Built-in macOS execution generation mismatch: requested {generation}, expected {BUILTIN_MACOS_PROVIDER_GENERATION}"
));
}
return self
.builtin_config
.clone()
.map(Ok)
.unwrap_or_else(builtin_macos_provider_config);
}
let value = self
.store
.get_provider(name)?
.ok_or_else(|| format!("Execution provider \"{name}\" is not registered"))?;
let config = serde_json::from_value::<ProviderConfig>(value)
.map_err(|error| format!("Invalid provider configuration for {name}: {error}"))?;
if !config.enabled {
return Err(format!("Execution provider \"{name}\" is disabled"));
}
if config.generation != generation {
return Err(format!(
"Execution provider generation mismatch for {name}: requested {generation}, registered {}",
config.generation
));
}
if config.kind != ProviderKind::McpStdio {
return Err(format!(
"Execution provider \"{name}\" must use the mcp_stdio transport"
));
}
Ok(config)
}
async fn client(
&self,
provider: &str,
generation: &str,
) -> Result<Arc<StdioMcpClient>, String> {
let config = self.config(provider, generation)?;
if provider != BUILTIN_MACOS_PROVIDER_NAME {
return StdioMcpClient::new(&config).map(Arc::new);
}
let mut retained = self.builtin_client.lock().await;
if let Some(client) = retained.as_ref() {
return Ok(Arc::clone(client));
}
let client = Arc::new(StdioMcpClient::new(&config)?);
*retained = Some(Arc::clone(&client));
Ok(client)
}
async fn discard_builtin_client(&self, failed: &Arc<StdioMcpClient>) {
let mut retained = self.builtin_client.lock().await;
if retained
.as_ref()
.is_some_and(|current| Arc::ptr_eq(current, failed))
{
*retained = None;
}
}
pub async fn isolation_status(&self) -> Result<IsolationStatus, String> {
let client = self
.client(
BUILTIN_MACOS_PROVIDER_NAME,
BUILTIN_MACOS_PROVIDER_GENERATION,
)
.await?;
let result = client
.call_tool("isolation_status", serde_json::json!({}))
.await;
match result {
Ok(value) => serde_json::from_value(value)
.map_err(|error| format!("Invalid isolation_status result: {error}")),
Err(error) => {
self.discard_builtin_client(&client).await;
Err(error)
}
}
}
pub async fn image_import(
&self,
source: PathBuf,
idempotency_key: &str,
controller: &ProcessController,
) -> Result<ImageImportResponse, String> {
if idempotency_key.is_empty() {
return Err("execution.image.import requires idempotency_key".to_string());
}
let _guard = self.isolation_mutation.lock().await;
let operation = "execution.image.import";
let source_bundle = source.to_string_lossy().into_owned();
let fingerprint = isolation_fingerprint(
operation,
&serde_json::json!({"source_bundle": source_bundle}),
)?;
let storage_key = format!("image-import:{idempotency_key}");
let started_at = now_ms();
let initial = ImageImportReplay {
operation: operation.to_string(),
fingerprint: fingerprint.clone(),
response: ImageImportResponse {
receipt: OperationReceipt {
id: uuid::Uuid::now_v7().to_string(),
idempotency_key: idempotency_key.to_string(),
operation: operation.to_string(),
actor: controller.actor.clone(),
execution_id: controller.execution_id.clone(),
state: ReceiptState::Unknown,
targets: vec![OperationTarget {
key: "image-store".to_string(),
applied: false,
error_code: None,
error: None,
}],
started_at,
completed_at: None,
},
source_bundle: source_bundle.clone(),
target_bundle: None,
image_digest: None,
files: 0,
bytes: 0,
already_present: false,
replayed: false,
},
};
if let Some(mut replay) =
self.store
.claim_receipt(&storage_key, "image_import", &initial)?
{
if replay.operation != operation || replay.fingerprint != fingerprint {
return Err(format!(
"IMAGE_IMPORT_IDEMPOTENCY_CONFLICT: key \"{idempotency_key}\" was already used for a different request"
));
}
replay.response.replayed = true;
return match replay.response.receipt.state {
ReceiptState::Applied => Ok(replay.response),
ReceiptState::Partial | ReceiptState::Failed | ReceiptState::Unknown => {
Err(image_import_error(&replay.response))
}
};
}
let intent = self.store.append_causal_event(CausalEvent {
sequence: 0,
id: uuid::Uuid::now_v7().to_string(),
at: started_at,
kind: format!("{operation}.intent"),
subject: source_bundle.clone(),
actor: controller.actor.clone(),
correlation_id: initial.response.receipt.id.clone(),
causation_id: None,
execution_id: controller.execution_id.clone(),
session_id: None,
payload_id: None,
details: serde_json::json!({
"idempotency_key": idempotency_key,
"fingerprint": fingerprint
}),
})?;
let mut response = initial.response.clone();
let imported = tokio::task::spawn_blocking(move || {
image_store().and_then(|store| import_verified_bundle(&source, &store))
})
.await
.map_err(|error| format!("Image import worker failed: {error}"))?;
match imported {
Ok(imported) => {
response.receipt.state = ReceiptState::Applied;
response.receipt.completed_at = Some(now_ms());
response.receipt.targets[0].key = format!("image:{}", imported.image_digest);
response.receipt.targets[0].applied = true;
response.source_bundle = imported.source_bundle;
response.target_bundle = Some(imported.target_bundle);
response.image_digest = Some(imported.image_digest);
response.files = imported.files;
response.bytes = imported.bytes;
response.already_present = imported.already_present;
}
Err(error) => finish_image_import_receipt(&mut response, error),
}
self.store.put_receipt(
&storage_key,
"image_import",
&ImageImportReplay {
operation: operation.to_string(),
fingerprint,
response: response.clone(),
},
)?;
self.store.append_causal_event(CausalEvent {
sequence: 0,
id: uuid::Uuid::now_v7().to_string(),
at: response.receipt.completed_at.unwrap_or_else(now_ms),
kind: format!("{operation}.result"),
subject: response
.image_digest
.as_ref()
.map(|digest| format!("image:{digest}"))
.unwrap_or(source_bundle),
actor: controller.actor.clone(),
correlation_id: response.receipt.id.clone(),
causation_id: Some(intent.id),
execution_id: controller.execution_id.clone(),
session_id: None,
payload_id: None,
details: serde_json::to_value(&response).map_err(|error| error.to_string())?,
})?;
match response.receipt.state {
ReceiptState::Applied => Ok(response),
ReceiptState::Partial | ReceiptState::Failed | ReceiptState::Unknown => {
Err(image_import_error(&response))
}
}
}
pub async fn isolation_mutation(
&self,
action: &str,
arguments: Value,
idempotency_key: &str,
controller: &ProcessController,
) -> Result<IsolationMutationResponse, String> {
if !matches!(action, "warmup" | "drain") {
return Err(format!("Unsupported isolation mutation \"{action}\""));
}
let _guard = self.isolation_mutation.lock().await;
let operation = format!("execution.isolation.{action}");
let fingerprint = isolation_fingerprint(&operation, &arguments)?;
let storage_key = format!("isolation:{idempotency_key}");
let started_at = now_ms();
let initial = IsolationMutationReplay {
operation: operation.clone(),
fingerprint: fingerprint.clone(),
response: IsolationMutationResponse {
receipt: OperationReceipt {
id: uuid::Uuid::now_v7().to_string(),
idempotency_key: idempotency_key.to_string(),
operation: operation.clone(),
actor: controller.actor.clone(),
execution_id: controller.execution_id.clone(),
state: ReceiptState::Unknown,
targets: vec![OperationTarget {
key: "isolation/macos-pool".to_string(),
applied: false,
error_code: None,
error: None,
}],
started_at,
completed_at: None,
},
status: None,
replayed: false,
},
};
if let Some(mut replay) =
self.store
.claim_receipt(&storage_key, "isolation_mutation", &initial)?
{
if replay.operation != operation || replay.fingerprint != fingerprint {
return Err(format!(
"ISOLATION_IDEMPOTENCY_CONFLICT: key \"{idempotency_key}\" was already used for a different request"
));
}
replay.response.replayed = true;
return Ok(replay.response);
}
let intent = self.store.append_causal_event(CausalEvent {
sequence: 0,
id: uuid::Uuid::now_v7().to_string(),
at: started_at,
kind: format!("{operation}.intent"),
subject: "isolation/macos-pool".to_string(),
actor: controller.actor.clone(),
correlation_id: initial.response.receipt.id.clone(),
causation_id: None,
execution_id: controller.execution_id.clone(),
session_id: None,
payload_id: None,
details: serde_json::json!({
"idempotency_key": idempotency_key,
"fingerprint": fingerprint
}),
})?;
let mut response = initial.response.clone();
match self
.client(
BUILTIN_MACOS_PROVIDER_NAME,
BUILTIN_MACOS_PROVIDER_GENERATION,
)
.await
{
Err(error) => {
finish_isolation_receipt(
&mut response,
ReceiptState::Failed,
"ISOLATION_PROVIDER_UNAVAILABLE",
error,
);
}
Ok(client) => {
let tool = format!("isolation_{action}");
match client.call_tool(&tool, arguments).await {
Ok(value) => match serde_json::from_value::<IsolationStatus>(value) {
Ok(status) => {
response.receipt.state = ReceiptState::Applied;
response.receipt.completed_at = Some(now_ms());
response.receipt.targets[0].applied = true;
response.status = Some(status);
}
Err(error) => finish_isolation_receipt(
&mut response,
ReceiptState::Unknown,
"ISOLATION_PROVIDER_RESULT_INVALID",
error.to_string(),
),
},
Err(error) => {
self.discard_builtin_client(&client).await;
finish_isolation_receipt(
&mut response,
ReceiptState::Unknown,
"ISOLATION_PROVIDER_RESULT_UNKNOWN",
error,
);
}
}
}
}
self.store.put_receipt(
&storage_key,
"isolation_mutation",
&IsolationMutationReplay {
operation: operation.clone(),
fingerprint,
response: response.clone(),
},
)?;
self.store.append_causal_event(CausalEvent {
sequence: 0,
id: uuid::Uuid::now_v7().to_string(),
at: response.receipt.completed_at.unwrap_or_else(now_ms),
kind: format!("{operation}.result"),
subject: "isolation/macos-pool".to_string(),
actor: controller.actor.clone(),
correlation_id: response.receipt.id.clone(),
causation_id: Some(intent.id),
execution_id: controller.execution_id.clone(),
session_id: None,
payload_id: None,
details: serde_json::to_value(&response).map_err(|error| error.to_string())?,
})?;
Ok(response)
}
}
#[derive(Clone, Serialize, Deserialize)]
struct IsolationMutationReplay {
operation: String,
fingerprint: String,
response: IsolationMutationResponse,
}
#[derive(Clone, Serialize, Deserialize)]
struct ImageImportReplay {
operation: String,
fingerprint: String,
response: ImageImportResponse,
}
fn isolation_fingerprint(operation: &str, arguments: &Value) -> Result<String, String> {
let encoded = serde_json::to_vec(&(operation, arguments)).map_err(|error| error.to_string())?;
Ok(format!("sha256:{:x}", Sha256::digest(encoded)))
}
fn finish_isolation_receipt(
response: &mut IsolationMutationResponse,
state: ReceiptState,
error_code: &str,
error: String,
) {
response.receipt.state = state;
response.receipt.completed_at = Some(now_ms());
response.receipt.targets[0].error_code = Some(error_code.to_string());
response.receipt.targets[0].error = Some(error);
}
fn finish_image_import_receipt(response: &mut ImageImportResponse, error: String) {
response.receipt.state = ReceiptState::Failed;
response.receipt.completed_at = Some(now_ms());
response.receipt.targets[0].error_code = Some("IMAGE_IMPORT_FAILED".to_string());
response.receipt.targets[0].error = Some(error);
}
fn image_import_error(response: &ImageImportResponse) -> String {
response.receipt.targets[0]
.error
.clone()
.unwrap_or_else(|| {
format!(
"Image import receipt \"{}\" has no applied result",
response.receipt.id
)
})
}
fn builtin_macos_provider_config() -> Result<ProviderConfig, String> {
let executable = resolve_builtin_macos_provider()?;
let now = now_ms();
let mut env = std::collections::BTreeMap::new();
if let Some(bundle) = resolve_builtin_vm_bundle(&executable) {
env.insert(
"APOC_EXECUTOR_VM_BUNDLE".to_string(),
bundle.to_string_lossy().into_owned(),
);
}
Ok(ProviderConfig {
name: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
kind: ProviderKind::McpStdio,
url: None,
base_url: None,
command: Some(executable.to_string_lossy().into_owned()),
args: vec!["--mcp".to_string()],
cwd: executable
.parent()
.map(|parent| parent.to_string_lossy().into_owned()),
env,
secret_env: std::collections::BTreeMap::new(),
headers: std::collections::BTreeMap::new(),
secret_headers: std::collections::BTreeMap::new(),
profile_mode: crate::model::ProfileMode::None,
profile_env: BTreeMap::new(),
profile_headers: BTreeMap::new(),
instructions: None,
enabled: true,
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
created_at: now,
updated_at: now,
})
}
fn resolve_builtin_vm_bundle(provider: &std::path::Path) -> Option<std::path::PathBuf> {
let configured = std::env::var_os("APOC_EXECUTOR_VM_BUNDLE").map(std::path::PathBuf::from);
let sibling = provider.parent().map(|parent| parent.join("vm-bundle"));
let repository = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
configured
.into_iter()
.chain(sibling)
.chain([
repository.join("extensions/macos-native/.build/vm-bundle"),
repository.join("extensions/macos-native/vm-bundle"),
])
.find(|path| path.is_dir())
}
fn resolve_builtin_macos_provider() -> Result<std::path::PathBuf, String> {
let configured = std::env::var_os("APOC_MACOS_PROVIDER").map(std::path::PathBuf::from);
let sibling = std::env::current_exe()
.ok()
.and_then(|path| path.parent().map(|parent| parent.join("apoc-macos-vz")));
let repository = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let candidates = configured.into_iter().chain(sibling).chain([
repository.join("target/release/apoc-macos-vz"),
repository.join("target/debug/apoc-macos-vz"),
]);
candidates
.into_iter()
.find(|path| path.is_file())
.ok_or_else(|| {
"BUILTIN_MACOS_PROVIDER_UNAVAILABLE: install apoc-macos-vz beside apoc or set the explicit APOC_MACOS_PROVIDER test override"
.to_string()
})
}
#[async_trait::async_trait]
impl ProviderExecutionGateway for RegisteredProviderExecutionGateway {
fn validate(&self, provider: &str, generation: &str) -> Result<(), String> {
self.config(provider, generation).map(|_| ())
}
async fn execute(
&self,
request: ProviderExecutionRequest,
events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
cancellation: tokio_util::sync::CancellationToken,
) -> Result<(), String> {
self.execute_with_retries(request, events, cancellation)
.await
}
async fn input(&self, control: ProviderExecutionControl, data: &[u8]) -> Result<(), String> {
let client = self.control_client(&control).await?;
client
.call_tool(
"execution_input",
serde_json::json!({
"execution_id": control.provider_execution_id,
"data_base64": base64::engine::general_purpose::STANDARD.encode(data)
}),
)
.await?;
Ok(())
}
async fn signal(&self, control: ProviderExecutionControl, signal: &str) -> Result<(), String> {
let client = self.control_client(&control).await?;
client
.call_tool(
"execution_signal",
serde_json::json!({
"execution_id": control.provider_execution_id,
"signal": signal
}),
)
.await?;
Ok(())
}
}
impl RegisteredProviderExecutionGateway {
async fn execute_with_retries(
&self,
request: ProviderExecutionRequest,
events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
cancellation: tokio_util::sync::CancellationToken,
) -> Result<(), String> {
let max_attempts = request.retry.max_attempts.max(1);
for attempt in 1..=max_attempts {
let client = self.client(&request.provider, &request.generation).await?;
let request_id = if max_attempts == 1 {
request.execution_id.clone()
} else {
format!("{}:attempt:{attempt}", request.execution_id)
};
let (attempt_events, incoming) = tokio::sync::mpsc::unbounded_channel();
let (call_result, terminal_result) = tokio::join!(
execute_with_client(
&self.store,
client.as_ref(),
&request,
&request_id,
attempt_events,
cancellation.clone(),
),
collect_attempt_events(incoming, &events)
);
if request.provider == BUILTIN_MACOS_PROVIDER_NAME && call_result.is_err() {
self.discard_builtin_client(&client).await;
}
let terminal = terminal_result?;
let failure = provider_attempt_failure(&call_result, terminal.as_ref());
if let Some((exit_code, error_code, error)) = failure.as_ref()
&& attempt < max_attempts
&& !cancellation.is_cancelled()
{
events
.send(ProviderExecutionEvent::Retrying {
exit_code: *exit_code,
error_code: error_code.clone(),
error: error.clone(),
will_retry: true,
})
.map_err(|_| "apoc stopped receiving provider events".to_string())?;
let exponent = attempt.saturating_sub(1).min(31);
let delay = request
.retry
.backoff_ms
.saturating_mul(1_u64 << exponent)
.min(request.retry.max_backoff_ms);
if delay > 0 {
tokio::select! {
_ = tokio::time::sleep(Duration::from_millis(delay)) => {}
_ = cancellation.cancelled() => {
return Err("Provider execution was canceled during retry backoff".to_string());
}
}
}
continue;
}
if let Err(error) = call_result {
if let Some((exit_code, error_code, message)) = failure {
events
.send(ProviderExecutionEvent::Retrying {
exit_code,
error_code,
error: message,
will_retry: false,
})
.map_err(|_| "apoc stopped receiving provider events".to_string())?;
}
return Err(error);
}
let Some(terminal) = terminal else {
if let Some((exit_code, error_code, error)) = failure {
events
.send(ProviderExecutionEvent::Retrying {
exit_code,
error_code,
error,
will_retry: false,
})
.map_err(|_| "apoc stopped receiving provider events".to_string())?;
}
return Err("Provider returned without a terminal event".to_string());
};
events
.send(terminal)
.map_err(|_| "apoc stopped receiving provider events".to_string())?;
return Ok(());
}
unreachable!("max_attempts is at least one")
}
async fn control_client(
&self,
control: &ProviderExecutionControl,
) -> Result<Arc<StdioMcpClient>, String> {
if control.provider != BUILTIN_MACOS_PROVIDER_NAME
|| control.generation != BUILTIN_MACOS_PROVIDER_GENERATION
{
return Err(format!(
"EXECUTION_CONTROL_UNSUPPORTED: provider {}@{} does not retain a control transport",
control.provider, control.generation
));
}
self.builtin_client.lock().await.clone().ok_or_else(|| {
format!(
"Execution \"{}\" has no active built-in provider transport",
control.execution_id
)
})
}
}
async fn collect_attempt_events(
mut incoming: tokio::sync::mpsc::UnboundedReceiver<ProviderExecutionEvent>,
events: &tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
) -> Result<Option<ProviderExecutionEvent>, String> {
let mut terminal = None;
while let Some(event) = incoming.recv().await {
if matches!(
event,
ProviderExecutionEvent::Exited { .. }
| ProviderExecutionEvent::Canceled
| ProviderExecutionEvent::Error { .. }
) {
if terminal.replace(event).is_some() {
return Err("Provider emitted more than one terminal event".to_string());
}
continue;
}
events
.send(event)
.map_err(|_| "apoc stopped receiving provider events".to_string())?;
}
Ok(terminal)
}
fn provider_attempt_failure(
call_result: &Result<(), String>,
terminal: Option<&ProviderExecutionEvent>,
) -> Option<(Option<i32>, String, String)> {
if let Err(error) = call_result {
return Some((None, "PROVIDER_EXECUTION_FAILED".to_string(), error.clone()));
}
match terminal {
Some(ProviderExecutionEvent::Exited { exit_code }) if *exit_code != 0 => Some((
Some(*exit_code),
"PROVIDER_EXIT_NONZERO".to_string(),
format!("Provider execution exited with code {exit_code}"),
)),
Some(ProviderExecutionEvent::Error { code, message }) => {
Some((None, code.clone(), message.clone()))
}
None => Some((
None,
"PROVIDER_TERMINAL_EVENT_MISSING".to_string(),
"Provider returned without a terminal event".to_string(),
)),
_ => None,
}
}
async fn execute_with_client(
store: &Store,
client: &StdioMcpClient,
request: &ProviderExecutionRequest,
request_id: &str,
events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
cancellation: tokio_util::sync::CancellationToken,
) -> Result<(), String> {
let exports = request
.options
.get("exports")
.cloned()
.map(serde_json::from_value::<Vec<ExecutionExportSpec>>)
.transpose()
.map_err(|error| format!("Invalid isolated export declarations: {error}"))?
.unwrap_or_default();
let artifact_limit = request
.artifact_bytes
.unwrap_or(MAX_PROVIDER_ARTIFACT_BYTES)
.min(MAX_PROVIDER_ARTIFACT_BYTES);
let mut export_bytes = 0_u64;
let mut active_exports = BTreeMap::<usize, Vec<u8>>::new();
let mut finished_exports = BTreeMap::<usize, PendingExport>::new();
let transport_env = transport_env(request)?;
let start_arguments = start_arguments(request, request_id, &transport_env);
let started = tokio::select! {
result = client.call_tool("execution_start", start_arguments) => result?,
_ = cancellation.cancelled() => {
return Err("Provider execution was canceled before acceptance".to_string());
}
};
let provider_execution_id = execution_id(&started).unwrap_or_else(|| request_id.to_string());
let mut cursor = None::<String>;
loop {
let mut arguments = serde_json::json!({
"execution_id": provider_execution_id,
"wait_ms": 250
});
if let Some(cursor) = &cursor {
arguments
.as_object_mut()
.expect("event arguments are an object")
.insert("cursor".to_string(), cursor.clone().into());
}
let response = tokio::select! {
result = client.call_tool("execution_events", arguments) => result?,
_ = cancellation.cancelled() => {
client.call_tool(
"execution_cancel",
serde_json::json!({
"execution_id": provider_execution_id,
"request_id": request_id
}),
).await?;
events.send(ProviderExecutionEvent::Canceled)
.map_err(|_| "apoc stopped receiving provider events".to_string())?;
return Ok(());
}
};
let batch = response
.get("events")
.and_then(Value::as_array)
.ok_or_else(|| "execution_events returned no events array".to_string())?;
for value in batch {
let kind = value
.get("type")
.and_then(Value::as_str)
.unwrap_or_default();
if kind.starts_with("export_") {
let finished = handle_export_event(
&exports,
artifact_limit,
&mut export_bytes,
&mut active_exports,
value,
)
.await?;
if let Some(export) = finished
&& finished_exports.insert(export.index, export).is_some()
{
return Err("Provider finished one export more than once".to_string());
}
continue;
}
if matches!(kind, "exit" | "error" | "canceled") && !active_exports.is_empty() {
return Err("Provider terminated with an incomplete export stream".to_string());
}
if kind == "exit" {
let exit_code = value
.get("exit_code")
.and_then(Value::as_i64)
.ok_or_else(|| "Provider exit event has no exit_code".to_string())?;
validate_finished_exports(&exports, &finished_exports, exit_code)?;
for (index, declaration) in exports.iter().enumerate() {
if !export_matches_exit(declaration.when, exit_code) {
continue;
}
let export = finished_exports
.remove(&index)
.expect("validated matching export is present");
let artifact =
store_export_artifact(store, request, declaration, export).await?;
events
.send(ProviderExecutionEvent::Artifact(artifact))
.map_err(|_| "apoc stopped receiving provider events".to_string())?;
}
}
let Some(event) = provider_event(value)? else {
continue;
};
let terminal = matches!(
event,
ProviderExecutionEvent::Exited { .. }
| ProviderExecutionEvent::Canceled
| ProviderExecutionEvent::Error { .. }
);
events
.send(event)
.map_err(|_| "apoc stopped receiving provider events".to_string())?;
if terminal {
return Ok(());
}
}
let next = response
.get("cursor")
.and_then(Value::as_str)
.map(ToString::to_string);
if next == cursor && batch.is_empty() {
tokio::time::sleep(Duration::from_millis(25)).await;
}
cursor = next;
}
}
const MAX_PROVIDER_ARTIFACT_BYTES: u64 = 64 * 1024 * 1024;
#[derive(Debug)]
struct PendingExport {
index: usize,
bytes: Vec<u8>,
guest_sha256: String,
}
fn export_matches_exit(when: ExecutionExportWhen, exit_code: i64) -> bool {
match when {
ExecutionExportWhen::Success => exit_code == 0,
ExecutionExportWhen::Failure => exit_code != 0,
ExecutionExportWhen::Always => true,
}
}
fn validate_finished_exports(
exports: &[ExecutionExportSpec],
finished: &BTreeMap<usize, PendingExport>,
exit_code: i64,
) -> Result<(), String> {
for (index, declaration) in exports.iter().enumerate() {
match (
export_matches_exit(declaration.when, exit_code),
finished.contains_key(&index),
) {
(true, false) => {
return Err(format!(
"Provider omitted declared export index {index} for exit code {exit_code}"
));
}
(false, true) => {
return Err(format!(
"Provider emitted export index {index} outside its declared timing for exit code {exit_code}"
));
}
_ => {}
}
}
if let Some(index) = finished.keys().find(|index| **index >= exports.len()) {
return Err(format!(
"Provider completed undeclared export index {index}"
));
}
Ok(())
}
async fn handle_export_event(
exports: &[ExecutionExportSpec],
artifact_limit: u64,
total_bytes: &mut u64,
active: &mut BTreeMap<usize, Vec<u8>>,
value: &Value,
) -> Result<Option<PendingExport>, String> {
let kind = value
.get("type")
.and_then(Value::as_str)
.ok_or_else(|| "Provider export event has no type".to_string())?;
let index = value
.get("export_index")
.and_then(Value::as_u64)
.and_then(|index| usize::try_from(index).ok())
.ok_or_else(|| format!("Provider {kind} event has no valid export_index"))?;
exports.get(index).ok_or_else(|| {
format!("Provider {kind} event references undeclared export index {index}")
})?;
match kind {
"export_started" => {
if active.insert(index, Vec::new()).is_some() {
return Err(format!(
"Provider started export index {index} more than once"
));
}
Ok(None)
}
"export_chunk" => {
let encoded = value
.get("data")
.and_then(Value::as_str)
.ok_or_else(|| format!("Provider export chunk {index} has no data"))?;
let chunk = base64::engine::general_purpose::STANDARD
.decode(encoded)
.map_err(|error| format!("Provider export chunk {index} is not base64: {error}"))?;
let chunk_bytes = u64::try_from(chunk.len())
.map_err(|_| "Provider export chunk length is outside u64".to_string())?;
let next_total = total_bytes
.checked_add(chunk_bytes)
.ok_or_else(|| "Provider export byte count overflowed".to_string())?;
if next_total > artifact_limit {
return Err(format!(
"EXECUTION_ARTIFACT_BUDGET_EXCEEDED: export bytes {next_total} exceed limit {artifact_limit}"
));
}
active
.get_mut(&index)
.ok_or_else(|| format!("Provider sent export chunk {index} before export_started"))?
.extend_from_slice(&chunk);
*total_bytes = next_total;
Ok(None)
}
"export_finished" => {
let declared_digest = value
.get("data")
.and_then(Value::as_str)
.filter(|digest| {
digest.len() == 64
&& digest
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
})
.ok_or_else(|| format!("Provider export {index} has no canonical SHA-256"))?;
let bytes = active
.remove(&index)
.ok_or_else(|| format!("Provider finished export {index} before export_started"))?;
let actual_digest = format!("{:x}", Sha256::digest(&bytes));
if actual_digest != declared_digest {
return Err(format!(
"Provider export {index} digest mismatch: declared {declared_digest}, computed {actual_digest}"
));
}
Ok(Some(PendingExport {
index,
bytes,
guest_sha256: format!("sha256:{actual_digest}"),
}))
}
other => Err(format!("Unknown provider export event type \"{other}\"")),
}
}
async fn store_export_artifact(
store: &Store,
request: &ProviderExecutionRequest,
declaration: &ExecutionExportSpec,
export: PendingExport,
) -> Result<ExecutionArtifactRef, String> {
let envelope = serde_json::json!({
"kind": "isolated_export",
"name": declaration.destination,
"source": declaration.source,
"media_type": "application/x-tar",
"guest_sha256": export.guest_sha256,
"data_base64": base64::engine::general_purpose::STANDARD.encode(&export.bytes)
});
let stored = store.put(&request.execution_id, &envelope).await?;
Ok(ExecutionArtifactRef {
name: declaration.destination.clone(),
source: declaration.source.clone(),
id: stored.id,
bytes: u64::try_from(export.bytes.len())
.map_err(|_| "Provider export length is outside u64".to_string())?,
guest_sha256: export.guest_sha256,
media_type: "application/x-tar".to_string(),
})
}
fn transport_env(
request: &ProviderExecutionRequest,
) -> Result<std::collections::BTreeMap<String, String>, String> {
let mut env = request.env.clone();
for (name, secret) in &request.secrets {
if secret.provider != "env" {
return Err(format!("SECRET_PROVIDER_UNAVAILABLE: {name}"));
}
let value =
std::env::var(&secret.reference).map_err(|_| format!("SECRET_NOT_FOUND: {name}"))?;
env.insert(name.clone(), value);
}
Ok(env)
}
fn start_arguments(
request: &ProviderExecutionRequest,
request_id: &str,
env: &std::collections::BTreeMap<String, String>,
) -> Value {
let mut arguments = serde_json::json!({
"protocol_version": 1,
"request_id": request_id,
"argv": request.argv,
"shell": request.shell,
"env": env.iter().map(|(name, value)| format!("{name}={value}")).collect::<Vec<_>>(),
"tty": matches!(request.io, crate::model::IoMode::Pty),
});
let object = arguments
.as_object_mut()
.expect("launch arguments are an object");
if let Some(cwd) = &request.cwd {
object.insert("cwd".to_string(), cwd.clone().into());
}
if let Some(timeout_ms) = request.timeout_ms {
object.insert("timeout_ms".to_string(), timeout_ms.into());
}
if request.provider == BUILTIN_MACOS_PROVIDER_NAME {
if let Some(options) = request.options.as_object() {
object.extend(options.clone());
}
} else {
object.insert("options".to_string(), request.options.clone());
}
arguments
}
fn execution_id(value: &Value) -> Option<String> {
value
.get("provider_execution_id")
.or_else(|| value.get("execution_id"))
.or_else(|| value.get("request_id"))
.and_then(Value::as_str)
.map(ToString::to_string)
}
fn provider_event(value: &Value) -> Result<Option<ProviderExecutionEvent>, String> {
let kind = value
.get("type")
.and_then(Value::as_str)
.ok_or_else(|| "Provider event has no type".to_string())?;
match kind {
"accepted" => Ok(None),
"started" => Ok(Some(ProviderExecutionEvent::Started {
provider_execution_id: execution_id(value)
.ok_or_else(|| "Provider started event has no execution identity".to_string())?,
})),
"stdout" | "stderr" => {
let data = if let Some(encoded) = value.get("data_base64").and_then(Value::as_str) {
base64::engine::general_purpose::STANDARD
.decode(encoded)
.map_err(|error| {
format!("Provider {kind} event has invalid data_base64: {error}")
})?
} else {
value
.get("data")
.and_then(Value::as_str)
.ok_or_else(|| format!("Provider {kind} event has no data or data_base64"))?
.as_bytes()
.to_vec()
};
Ok(Some(if kind == "stdout" {
ProviderExecutionEvent::Stdout(data)
} else {
ProviderExecutionEvent::Stderr(data)
}))
}
"exit" => {
let exit_code = value
.get("exit_code")
.and_then(Value::as_i64)
.ok_or_else(|| "Provider exit event has no exit_code".to_string())?;
let exit_code = i32::try_from(exit_code)
.map_err(|_| "Provider exit_code is outside the i32 range".to_string())?;
Ok(Some(ProviderExecutionEvent::Exited { exit_code }))
}
"canceled" => Ok(Some(ProviderExecutionEvent::Canceled)),
"error" => Ok(Some(ProviderExecutionEvent::Error {
code: value
.get("error_code")
.and_then(Value::as_str)
.unwrap_or("PROVIDER_ERROR")
.to_string(),
message: value
.get("data")
.or_else(|| value.get("message"))
.and_then(Value::as_str)
.unwrap_or("Provider execution failed")
.to_string(),
})),
value if value.starts_with("export_") => Err(format!(
"Provider export event {value} is unsupported until apoc owns artifact transfer"
)),
other => Err(format!("Unknown provider execution event type \"{other}\"")),
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use super::*;
use crate::execution::ExecutionManager;
use crate::model::{
ExecutionBackendSpec, ExecutionBudget, ExecutionProgram, ExecutionSpec, ExecutionStatus,
IoMode, LeaseOwnerKind, ProcessController, RetryPolicy, SCHEMA_VERSION, WorkspaceContext,
now_ms,
};
use crate::run::RunManager;
#[test]
fn decodes_executor_events_without_losing_bytes_or_terminal_state() {
assert_eq!(
provider_event(&serde_json::json!({
"type": "started",
"request_id": "sandbox-1"
}))
.unwrap(),
Some(crate::execution::ProviderExecutionEvent::Started {
provider_execution_id: "sandbox-1".to_string()
})
);
assert_eq!(
provider_event(&serde_json::json!({
"type": "stdout",
"data": "hello\n"
}))
.unwrap(),
Some(crate::execution::ProviderExecutionEvent::Stdout(
b"hello\n".to_vec()
))
);
assert_eq!(
provider_event(&serde_json::json!({
"type": "stdout",
"data_base64": "aGVsbG8K"
}))
.unwrap(),
Some(crate::execution::ProviderExecutionEvent::Stdout(
b"hello\n".to_vec()
))
);
assert_eq!(
provider_event(&serde_json::json!({
"type": "exit",
"exit_code": 7
}))
.unwrap(),
Some(crate::execution::ProviderExecutionEvent::Exited { exit_code: 7 })
);
}
#[tokio::test]
async fn registered_mcp_provider_runs_one_executor_event_stream() {
let directory = tempfile::tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let fixture = r#"
import json, sys
for line in sys.stdin:
request = json.loads(line)
method = request.get('method')
if 'id' not in request:
continue
if method == 'initialize':
result = {'protocolVersion':'2025-06-18','capabilities':{'tools':{}},'serverInfo':{'name':'fixture','version':'1'}}
elif method == 'tools/call':
name = request['params']['name']
arguments = request['params']['arguments']
if name == 'execution_start':
result = {'structuredContent':{'provider_execution_id':arguments['request_id']}}
elif name == 'execution_events':
result = {'structuredContent':{'cursor':'4','events':[
{'type':'started','request_id':arguments['execution_id']},
{'type':'stdout','data':'guest output\n'},
{'type':'stderr','data':'guest warning\n'},
{'type':'exit','exit_code':0}
]}}
else:
result = {'isError':True,'content':[{'type':'text','text':'unexpected tool'}]}
else:
result = {}
print(json.dumps({'jsonrpc':'2.0','id':request['id'],'result':result}), flush=True)
"#;
let generation = "sha256:fixture-v1";
let config = ProviderConfig {
name: "executor-fixture".to_string(),
kind: ProviderKind::McpStdio,
url: None,
base_url: None,
command: Some("/usr/bin/python3".to_string()),
args: vec!["-u".to_string(), "-c".to_string(), fixture.to_string()],
cwd: None,
env: BTreeMap::new(),
secret_env: BTreeMap::new(),
headers: BTreeMap::new(),
secret_headers: BTreeMap::new(),
profile_mode: crate::model::ProfileMode::None,
profile_env: BTreeMap::new(),
profile_headers: BTreeMap::new(),
instructions: None,
enabled: true,
generation: generation.to_string(),
created_at: now_ms(),
updated_at: now_ms(),
};
store
.put_provider("executor-fixture", &serde_json::to_value(config).unwrap())
.unwrap();
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store.clone()).unwrap(),
Arc::new(RegisteredProviderExecutionGateway::new(store)),
);
let execution = manager
.start(
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: Some("provider-fixture".to_string()),
name: "provider-fixture".to_string(),
purpose: Some("prove registered provider execution".to_string()),
labels: BTreeMap::new(),
program: ExecutionProgram::Command {
argv: vec!["/bin/echo".to_string(), "guest".to_string()],
shell: false,
},
backend: ExecutionBackendSpec::Provider {
name: "executor-fixture".to_string(),
generation: generation.to_string(),
options: serde_json::json!({ "image_ref": "local" }),
},
workspace: WorkspaceContext {
cwd: Some("/workspace".to_string()),
..WorkspaceContext::default()
},
env: BTreeMap::new(),
secrets: BTreeMap::new(),
profile_selection: None,
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: RetryPolicy::default(),
budget: ExecutionBudget::default(),
telemetry: crate::model::TelemetryPolicy::default(),
expect: None,
},
ProcessController::capture(false, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
let logs = manager.logs(&execution.id, 1024).unwrap();
assert_eq!(finished.status, ExecutionStatus::Completed);
assert_eq!(logs.stdout, "guest output\n");
assert_eq!(logs.stderr, "guest warning\n");
}
#[tokio::test]
async fn builtin_isolation_reuses_one_stdio_process_across_executions() {
let directory = tempfile::tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let fixture = r#"
import json, sys
sequence = 0
executions = {}
for line in sys.stdin:
request = json.loads(line)
if 'id' not in request:
continue
method = request.get('method')
if method == 'initialize':
result = {'protocolVersion':'2025-06-18','capabilities':{'tools':{}},'serverInfo':{'name':'builtin-fixture','version':'1'}}
elif method == 'tools/call':
name = request['params']['name']
arguments = request['params']['arguments']
if name == 'execution_start':
sequence += 1
execution_id = arguments['request_id']
executions[execution_id] = sequence
result = {'structuredContent':{'provider_execution_id':execution_id}}
elif name == 'execution_events':
execution_id = arguments['execution_id']
current = executions[execution_id]
result = {'structuredContent':{'cursor':'3','events':[
{'type':'started','request_id':execution_id},
{'type':'stdout','data':f'provider-process-sequence={current}\n'},
{'type':'exit','exit_code':0}
]}}
else:
result = {'isError':True,'content':[{'type':'text','text':'unexpected tool'}]}
else:
result = {}
print(json.dumps({'jsonrpc':'2.0','id':request['id'],'result':result}), flush=True)
"#;
let config = ProviderConfig {
name: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
kind: ProviderKind::McpStdio,
url: None,
base_url: None,
command: Some("/usr/bin/python3".to_string()),
args: vec!["-u".to_string(), "-c".to_string(), fixture.to_string()],
cwd: None,
env: BTreeMap::new(),
secret_env: BTreeMap::new(),
headers: BTreeMap::new(),
secret_headers: BTreeMap::new(),
profile_mode: crate::model::ProfileMode::None,
profile_env: BTreeMap::new(),
profile_headers: BTreeMap::new(),
instructions: None,
enabled: true,
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
created_at: now_ms(),
updated_at: now_ms(),
};
let gateway = Arc::new(
RegisteredProviderExecutionGateway::with_builtin_config_for_test(store.clone(), config),
);
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store).unwrap(),
gateway,
);
for (index, expected) in [
"provider-process-sequence=1\n",
"provider-process-sequence=2\n",
]
.into_iter()
.enumerate()
{
let execution = manager
.start(
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: Some(format!("builtin-persistent-{index}")),
name: format!("builtin-persistent-{index}"),
purpose: Some("prove built-in provider process ownership".to_string()),
labels: BTreeMap::new(),
program: ExecutionProgram::Command {
argv: vec!["/bin/true".to_string()],
shell: false,
},
backend: ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::None,
guest_cwd: None,
memory_mb: None,
cpu_count: None,
pids_max: None,
network: crate::model::IsolatedNetworkPolicy::None,
exports: Vec::new(),
},
},
workspace: WorkspaceContext::default(),
env: BTreeMap::new(),
secrets: BTreeMap::new(),
profile_selection: None,
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: RetryPolicy::default(),
budget: ExecutionBudget::default(),
telemetry: crate::model::TelemetryPolicy::default(),
expect: None,
},
ProcessController::capture(false, None),
None,
None,
)
.await
.unwrap();
manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
assert_eq!(manager.logs(&execution.id, 1024).unwrap().stdout, expected);
}
}
#[tokio::test]
async fn builtin_isolation_imports_a_verified_export_as_an_execution_artifact() {
use incurs_codemode::ArtifactStore as _;
let directory = tempfile::tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let archive = b"fixture-tar-archive";
let encoded = base64::engine::general_purpose::STANDARD.encode(archive);
let digest = format!("{:x}", Sha256::digest(archive));
let fixture = format!(
r#"
import json, sys
for line in sys.stdin:
request = json.loads(line)
if 'id' not in request:
continue
method = request.get('method')
if method == 'initialize':
result = {{'protocolVersion':'2025-06-18','capabilities':{{'tools':{{}}}},'serverInfo':{{'name':'export-fixture','version':'1'}}}}
elif method == 'tools/call':
name = request['params']['name']
arguments = request['params']['arguments']
if name == 'execution_start':
result = {{'structuredContent':{{'provider_execution_id':arguments['request_id']}}}}
elif name == 'execution_events':
result = {{'structuredContent':{{'cursor':'5','events':[
{{'type':'started','request_id':arguments['execution_id']}},
{{'type':'export_started','export_index':0}},
{{'type':'export_chunk','export_index':0,'data':'{encoded}'}},
{{'type':'export_finished','export_index':0,'data':'{digest}'}},
{{'type':'exit','exit_code':0}}
]}}}}
else:
result = {{'isError':True,'content':[{{'type':'text','text':'unexpected tool'}}]}}
else:
result = {{}}
print(json.dumps({{'jsonrpc':'2.0','id':request['id'],'result':result}}), flush=True)
"#
);
let config = ProviderConfig {
name: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
kind: ProviderKind::McpStdio,
url: None,
base_url: None,
command: Some("/usr/bin/python3".to_string()),
args: vec!["-u".to_string(), "-c".to_string(), fixture],
cwd: None,
env: BTreeMap::new(),
secret_env: BTreeMap::new(),
headers: BTreeMap::new(),
secret_headers: BTreeMap::new(),
profile_mode: crate::model::ProfileMode::None,
profile_env: BTreeMap::new(),
profile_headers: BTreeMap::new(),
instructions: None,
enabled: true,
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
created_at: now_ms(),
updated_at: now_ms(),
};
let gateway = Arc::new(
RegisteredProviderExecutionGateway::with_builtin_config_for_test(store.clone(), config),
);
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store.clone()).unwrap(),
gateway,
);
let execution = manager
.start(
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: Some("builtin-export-1".to_string()),
name: "builtin-export".to_string(),
purpose: Some("prove isolated export ownership".to_string()),
labels: BTreeMap::new(),
program: ExecutionProgram::Command {
argv: vec!["/bin/true".to_string()],
shell: false,
},
backend: ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::None,
guest_cwd: None,
memory_mb: None,
cpu_count: None,
pids_max: None,
network: crate::model::IsolatedNetworkPolicy::None,
exports: vec![crate::model::ExecutionExportSpec {
source: "/tmp/result".to_string(),
destination: "result".to_string(),
when: crate::model::ExecutionExportWhen::Always,
}],
},
},
workspace: WorkspaceContext::default(),
env: BTreeMap::new(),
secrets: BTreeMap::new(),
profile_selection: None,
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: RetryPolicy::default(),
budget: ExecutionBudget {
artifact_bytes: Some(1024),
..ExecutionBudget::default()
},
telemetry: crate::model::TelemetryPolicy::default(),
expect: None,
},
ProcessController::capture(false, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Completed, "{finished:#?}");
let artifact = &finished.result.as_ref().unwrap()["artifacts"][0];
assert_eq!(artifact["name"], "result");
assert_eq!(artifact["source"], "/tmp/result");
assert_eq!(artifact["guest_sha256"], format!("sha256:{digest}"));
assert_eq!(artifact["bytes"], archive.len());
let artifact_id = artifact["id"].as_str().unwrap();
let stored = store
.get(&execution.id, artifact_id)
.await
.unwrap()
.unwrap();
assert_eq!(stored["data_base64"], encoded);
assert_eq!(stored["guest_sha256"], format!("sha256:{digest}"));
}
fn export_test_declarations() -> Vec<ExecutionExportSpec> {
vec![ExecutionExportSpec {
source: "/tmp/result".to_string(),
destination: "result".to_string(),
when: crate::model::ExecutionExportWhen::Always,
}]
}
#[tokio::test]
async fn isolated_export_rejects_a_digest_mismatch_before_artifact_storage() {
let exports = export_test_declarations();
let mut total_bytes = 0;
let mut active = BTreeMap::new();
handle_export_event(
&exports,
1024,
&mut total_bytes,
&mut active,
&serde_json::json!({"type":"export_started","export_index":0}),
)
.await
.unwrap();
handle_export_event(
&exports,
1024,
&mut total_bytes,
&mut active,
&serde_json::json!({
"type":"export_chunk",
"export_index":0,
"data":base64::engine::general_purpose::STANDARD.encode(b"archive")
}),
)
.await
.unwrap();
let error = handle_export_event(
&exports,
1024,
&mut total_bytes,
&mut active,
&serde_json::json!({"type":"export_finished","export_index":0,"data":"0".repeat(64)}),
)
.await
.unwrap_err();
assert!(error.contains("digest mismatch"), "{error}");
}
#[tokio::test]
async fn isolated_export_rejects_chunks_beyond_the_execution_artifact_budget() {
let exports = export_test_declarations();
let mut total_bytes = 0;
let mut active = BTreeMap::new();
handle_export_event(
&exports,
3,
&mut total_bytes,
&mut active,
&serde_json::json!({"type":"export_started","export_index":0}),
)
.await
.unwrap();
let error = handle_export_event(
&exports,
3,
&mut total_bytes,
&mut active,
&serde_json::json!({
"type":"export_chunk",
"export_index":0,
"data":base64::engine::general_purpose::STANDARD.encode(b"archive")
}),
)
.await
.unwrap_err();
assert!(
error.contains("EXECUTION_ARTIFACT_BUDGET_EXCEEDED"),
"{error}"
);
assert_eq!(total_bytes, 0);
assert!(active[&0].is_empty());
}
#[test]
fn isolated_export_terminalization_enforces_every_declared_timing() {
let declarations = vec![
ExecutionExportSpec {
source: "/tmp/success".to_string(),
destination: "success".to_string(),
when: ExecutionExportWhen::Success,
},
ExecutionExportSpec {
source: "/tmp/failure".to_string(),
destination: "failure".to_string(),
when: ExecutionExportWhen::Failure,
},
ExecutionExportSpec {
source: "/tmp/always".to_string(),
destination: "always".to_string(),
when: ExecutionExportWhen::Always,
},
];
let pending = |index| PendingExport {
index,
bytes: Vec::new(),
guest_sha256: format!("sha256:{}", "0".repeat(64)),
};
let success = BTreeMap::from([(0, pending(0)), (2, pending(2))]);
validate_finished_exports(&declarations, &success, 0).unwrap();
let failure = BTreeMap::from([(1, pending(1)), (2, pending(2))]);
validate_finished_exports(&declarations, &failure, 1).unwrap();
let missing = BTreeMap::from([(0, pending(0))]);
assert!(
validate_finished_exports(&declarations, &missing, 0)
.unwrap_err()
.contains("omitted declared export index 2")
);
let wrong_timing = BTreeMap::from([(0, pending(0)), (1, pending(1)), (2, pending(2))]);
assert!(
validate_finished_exports(&declarations, &wrong_timing, 0)
.unwrap_err()
.contains("outside its declared timing")
);
}
#[tokio::test]
async fn builtin_isolation_discards_a_failed_stdio_process_without_replay() {
let directory = tempfile::tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let process_count = directory.path().join("provider-process-count");
let fixture = r#"
import json, os, pathlib, sys
counter_path = pathlib.Path(os.environ['PROVIDER_PROCESS_COUNT'])
count = int(counter_path.read_text()) + 1 if counter_path.exists() else 1
counter_path.write_text(str(count))
for line in sys.stdin:
request = json.loads(line)
if 'id' not in request:
continue
method = request.get('method')
if method == 'initialize':
result = {'protocolVersion':'2025-06-18','capabilities':{'tools':{}},'serverInfo':{'name':'failing-builtin-fixture','version':'1'}}
elif method == 'tools/call':
name = request['params']['name']
arguments = request['params']['arguments']
if name == 'execution_start':
result = {'structuredContent':{'provider_execution_id':arguments['request_id']}}
elif name == 'execution_events' and count == 1:
sys.exit(41)
elif name == 'execution_events':
result = {'structuredContent':{'cursor':'2','events':[
{'type':'started','request_id':arguments['execution_id']},
{'type':'exit','exit_code':0}
]}}
else:
result = {'isError':True,'content':[{'type':'text','text':'unexpected tool'}]}
else:
result = {}
print(json.dumps({'jsonrpc':'2.0','id':request['id'],'result':result}), flush=True)
"#;
let config = ProviderConfig {
name: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
kind: ProviderKind::McpStdio,
url: None,
base_url: None,
command: Some("/usr/bin/python3".to_string()),
args: vec!["-u".to_string(), "-c".to_string(), fixture.to_string()],
cwd: None,
env: BTreeMap::from([(
"PROVIDER_PROCESS_COUNT".to_string(),
process_count.to_string_lossy().into_owned(),
)]),
secret_env: BTreeMap::new(),
headers: BTreeMap::new(),
secret_headers: BTreeMap::new(),
profile_mode: crate::model::ProfileMode::None,
profile_env: BTreeMap::new(),
profile_headers: BTreeMap::new(),
instructions: None,
enabled: true,
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
created_at: now_ms(),
updated_at: now_ms(),
};
let gateway =
RegisteredProviderExecutionGateway::with_builtin_config_for_test(store.clone(), config);
let request = |execution_id: &str| ProviderExecutionRequest {
execution_id: execution_id.to_string(),
provider: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
argv: vec!["/bin/true".to_string()],
shell: false,
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
io: IoMode::Pipes,
timeout_ms: None,
artifact_bytes: None,
retry: crate::model::RetryPolicy::default(),
options: serde_json::json!({}),
};
let (first_events, _first_receiver) = tokio::sync::mpsc::unbounded_channel();
let first = gateway
.execute(
request("first-ambiguous-request"),
first_events,
tokio_util::sync::CancellationToken::new(),
)
.await;
assert!(first.is_err(), "the first provider process must fail");
assert_eq!(std::fs::read_to_string(&process_count).unwrap(), "1");
let (second_events, mut second_receiver) = tokio::sync::mpsc::unbounded_channel();
gateway
.execute(
request("second-distinct-request"),
second_events,
tokio_util::sync::CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(std::fs::read_to_string(&process_count).unwrap(), "2");
assert_eq!(
second_receiver.recv().await,
Some(ProviderExecutionEvent::Started {
provider_execution_id: "second-distinct-request".to_string(),
})
);
assert_eq!(
second_receiver.recv().await,
Some(ProviderExecutionEvent::Exited { exit_code: 0 })
);
}
#[tokio::test]
async fn provider_retry_uses_distinct_attempt_ids_and_one_event_stream() {
let directory = tempfile::tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let request_ids = directory.path().join("provider-request-ids");
let fixture = r#"
import json, os, pathlib, sys
request_ids = pathlib.Path(os.environ['PROVIDER_REQUEST_IDS'])
for line in sys.stdin:
request = json.loads(line)
if 'id' not in request:
continue
method = request.get('method')
if method == 'initialize':
result = {'protocolVersion':'2025-06-18','capabilities':{'tools':{}},'serverInfo':{'name':'retry-builtin-fixture','version':'1'}}
elif method == 'tools/call':
name = request['params']['name']
arguments = request['params']['arguments']
if name == 'execution_start':
execution_id = arguments['request_id']
with request_ids.open('a') as stream:
stream.write(execution_id + '\n')
result = {'structuredContent':{'provider_execution_id':execution_id}}
elif name == 'execution_events':
execution_id = arguments['execution_id']
exit_code = 7 if execution_id.endswith(':attempt:1') else 0
result = {'structuredContent':{'cursor':'2','events':[
{'type':'started','request_id':execution_id},
{'type':'exit','exit_code':exit_code}
]}}
else:
result = {'isError':True,'content':[{'type':'text','text':'unexpected tool'}]}
else:
result = {}
print(json.dumps({'jsonrpc':'2.0','id':request['id'],'result':result}), flush=True)
"#;
let config = ProviderConfig {
name: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
kind: ProviderKind::McpStdio,
url: None,
base_url: None,
command: Some("/usr/bin/python3".to_string()),
args: vec!["-u".to_string(), "-c".to_string(), fixture.to_string()],
cwd: None,
env: BTreeMap::from([(
"PROVIDER_REQUEST_IDS".to_string(),
request_ids.to_string_lossy().into_owned(),
)]),
secret_env: BTreeMap::new(),
headers: BTreeMap::new(),
secret_headers: BTreeMap::new(),
profile_mode: crate::model::ProfileMode::None,
profile_env: BTreeMap::new(),
profile_headers: BTreeMap::new(),
instructions: None,
enabled: true,
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
created_at: now_ms(),
updated_at: now_ms(),
};
let gateway = Arc::new(
RegisteredProviderExecutionGateway::with_builtin_config_for_test(store, config),
);
let request = ProviderExecutionRequest {
execution_id: "retry-parent".to_string(),
provider: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
argv: vec!["/bin/true".to_string()],
shell: false,
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
io: IoMode::Pipes,
timeout_ms: None,
artifact_bytes: None,
retry: crate::model::RetryPolicy {
max_attempts: 2,
backoff_ms: 0,
max_backoff_ms: 0,
},
options: serde_json::json!({}),
};
let (events, mut incoming) = tokio::sync::mpsc::unbounded_channel();
gateway
.execute(
request.clone(),
events,
tokio_util::sync::CancellationToken::new(),
)
.await
.unwrap();
let mut observed = Vec::new();
while let Ok(event) = incoming.try_recv() {
observed.push(event);
}
assert_eq!(
std::fs::read_to_string(request_ids).unwrap(),
"retry-parent:attempt:1\nretry-parent:attempt:2\n"
);
assert_eq!(observed.len(), 4, "{observed:#?}");
assert!(matches!(
observed[0],
ProviderExecutionEvent::Started { .. }
));
assert_eq!(
observed[1],
ProviderExecutionEvent::Retrying {
exit_code: Some(7),
error_code: "PROVIDER_EXIT_NONZERO".to_string(),
error: "Provider execution exited with code 7".to_string(),
will_retry: true,
}
);
assert!(matches!(
observed[2],
ProviderExecutionEvent::Started { .. }
));
assert_eq!(observed[3], ProviderExecutionEvent::Exited { exit_code: 0 });
let mut canceled_request = request;
canceled_request.execution_id = "retry-canceled".to_string();
canceled_request.retry.backoff_ms = 5_000;
canceled_request.retry.max_backoff_ms = 5_000;
let cancellation = tokio_util::sync::CancellationToken::new();
let task_gateway = gateway.clone();
let task_cancellation = cancellation.clone();
let (canceled_events, mut canceled_incoming) = tokio::sync::mpsc::unbounded_channel();
let task = tokio::spawn(async move {
task_gateway
.execute(canceled_request, canceled_events, task_cancellation)
.await
});
assert!(matches!(
canceled_incoming.recv().await,
Some(ProviderExecutionEvent::Started { .. })
));
assert!(matches!(
canceled_incoming.recv().await,
Some(ProviderExecutionEvent::Retrying { .. })
));
cancellation.cancel();
let error = tokio::time::timeout(Duration::from_secs(1), task)
.await
.unwrap()
.unwrap()
.unwrap_err();
assert!(error.contains("canceled during retry backoff"), "{error}");
assert_eq!(
std::fs::read_to_string(directory.path().join("provider-request-ids")).unwrap(),
"retry-parent:attempt:1\nretry-parent:attempt:2\nretry-canceled:attempt:1\n"
);
}
#[tokio::test]
async fn builtin_provider_death_restarts_transport_for_the_next_attempt() {
let directory = tempfile::tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let process_count = directory.path().join("provider-process-count");
let fixture = r#"
import json, os, pathlib, sys
counter = pathlib.Path(os.environ['PROVIDER_PROCESS_COUNT'])
count = int(counter.read_text()) + 1 if counter.exists() else 1
counter.write_text(str(count))
for line in sys.stdin:
request = json.loads(line)
if 'id' not in request:
continue
method = request.get('method')
if method == 'initialize':
result = {'protocolVersion':'2025-06-18','capabilities':{'tools':{}},'serverInfo':{'name':'dying-retry-fixture','version':'1'}}
elif method == 'tools/call':
name = request['params']['name']
arguments = request['params']['arguments']
if name == 'execution_start':
result = {'structuredContent':{'provider_execution_id':arguments['request_id']}}
elif name == 'execution_events' and count == 1:
sys.exit(41)
elif name == 'execution_events':
result = {'structuredContent':{'cursor':'2','events':[
{'type':'started','request_id':arguments['execution_id']},
{'type':'exit','exit_code':0}
]}}
else:
result = {'isError':True,'content':[{'type':'text','text':'unexpected tool'}]}
else:
result = {}
print(json.dumps({'jsonrpc':'2.0','id':request['id'],'result':result}), flush=True)
"#;
let config = ProviderConfig {
name: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
kind: ProviderKind::McpStdio,
url: None,
base_url: None,
command: Some("/usr/bin/python3".to_string()),
args: vec!["-u".to_string(), "-c".to_string(), fixture.to_string()],
cwd: None,
env: BTreeMap::from([(
"PROVIDER_PROCESS_COUNT".to_string(),
process_count.to_string_lossy().into_owned(),
)]),
secret_env: BTreeMap::new(),
headers: BTreeMap::new(),
secret_headers: BTreeMap::new(),
profile_mode: crate::model::ProfileMode::None,
profile_env: BTreeMap::new(),
profile_headers: BTreeMap::new(),
instructions: None,
enabled: true,
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
created_at: now_ms(),
updated_at: now_ms(),
};
let gateway =
RegisteredProviderExecutionGateway::with_builtin_config_for_test(store, config);
let request = ProviderExecutionRequest {
execution_id: "provider-death-retry".to_string(),
provider: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
argv: vec!["/bin/true".to_string()],
shell: false,
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
io: IoMode::Pipes,
timeout_ms: None,
artifact_bytes: None,
retry: crate::model::RetryPolicy {
max_attempts: 2,
backoff_ms: 0,
max_backoff_ms: 0,
},
options: serde_json::json!({}),
};
let (events, mut incoming) = tokio::sync::mpsc::unbounded_channel();
gateway
.execute(request, events, tokio_util::sync::CancellationToken::new())
.await
.unwrap();
let mut observed = Vec::new();
while let Ok(event) = incoming.try_recv() {
observed.push(event);
}
assert_eq!(std::fs::read_to_string(process_count).unwrap(), "2");
assert!(matches!(
observed[0],
ProviderExecutionEvent::Retrying { .. }
));
assert!(matches!(
observed[1],
ProviderExecutionEvent::Started { .. }
));
assert_eq!(observed[2], ProviderExecutionEvent::Exited { exit_code: 0 });
}
#[tokio::test]
#[ignore = "requires a signed apoc-macos-vz executable and VM bundle"]
async fn registered_macos_provider_runs_one_live_vz_execution() {
let repository = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let provider = repository.join("target/debug/apoc-macos-vz");
let bundle = repository.join("extensions/macos-native/.build/vm-bundle");
assert!(
provider.is_file(),
"build and sign apoc-macos-vz before this test"
);
assert!(
bundle.is_dir(),
"build the executor VM bundle before this test"
);
let directory = tempfile::tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let config = ProviderConfig {
name: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
kind: ProviderKind::McpStdio,
url: None,
base_url: None,
command: Some(provider.to_string_lossy().into_owned()),
args: vec!["--mcp".to_string()],
cwd: Some(repository.to_string_lossy().into_owned()),
env: BTreeMap::new(),
secret_env: BTreeMap::new(),
headers: BTreeMap::new(),
secret_headers: BTreeMap::new(),
profile_mode: crate::model::ProfileMode::None,
profile_env: BTreeMap::new(),
profile_headers: BTreeMap::new(),
instructions: None,
enabled: true,
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
created_at: now_ms(),
updated_at: now_ms(),
};
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store.clone()).unwrap(),
Arc::new(
RegisteredProviderExecutionGateway::with_builtin_config_for_test(store, config),
),
);
let execution = manager
.start(
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: Some("live-vz-execution-v1".to_string()),
name: "live-vz-execution".to_string(),
purpose: Some("prove a live VZ-backed apoc execution".to_string()),
labels: BTreeMap::new(),
program: ExecutionProgram::Command {
argv: vec!["/bin/echo".to_string(), "apoc-vz-live".to_string()],
shell: false,
},
backend: ExecutionBackendSpec::Provider {
name: BUILTIN_MACOS_PROVIDER_NAME.to_string(),
generation: BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
options: serde_json::json!({
"image_ref": "local",
"vm_bundle": bundle.to_string_lossy()
}),
},
workspace: WorkspaceContext {
cwd: Some("/".to_string()),
..WorkspaceContext::default()
},
env: BTreeMap::new(),
secrets: BTreeMap::new(),
profile_selection: None,
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: RetryPolicy::default(),
budget: ExecutionBudget {
wall_time_ms: Some(60_000),
..ExecutionBudget::default()
},
telemetry: crate::model::TelemetryPolicy::default(),
expect: None,
},
ProcessController::capture(false, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(60)))
.await
.unwrap();
let logs = manager.logs(&execution.id, 1024).unwrap();
assert_eq!(
finished.status,
ExecutionStatus::Completed,
"live execution: {finished:#?}; logs: {logs:#?}"
);
assert_eq!(logs.stdout, "apoc-vz-live\n");
assert_eq!(logs.stderr, "");
}
}