use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{AttemptId, CommandId, PageId};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ControlActionEvidence {
pub operation: crate::FormControlOperation,
pub target: crate::FormControlTarget,
pub state: crate::FormControlState,
pub validity: crate::FormControlValidity,
pub node_replaced: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub revealed_controls: Vec<crate::forms::RevealedControl>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct AccessibilityTarget {
pub role: String,
pub accessible_name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ordinal: Option<usize>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub frame_path: Vec<crate::forms::SemanticTargetSegment>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct ContextAnswer {
pub target: AccessibilityTarget,
pub confidence: f32,
#[serde(default)]
pub observed_at: ContextObservedAt,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source: Option<ContextAnswerSource>,
}
impl ContextAnswer {
pub fn page_derived(&self) -> bool {
true
}
}
impl Default for ContextObservedAt {
fn default() -> Self {
Self::Generation { generation: 0 }
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(tag = "kind", rename_all = "camelCase")]
pub enum ContextObservedAt {
Generation { generation: u64 },
Persisted,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "kebab-case")]
pub enum ContextAnswerSource {
Observed,
VisionPromoted,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct ContextNeighbors {
pub answer: ContextAnswer,
pub form: String,
pub page_pattern: String,
pub controls: Vec<ContextNeighborControl>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct ContextNeighborControl {
pub role: String,
pub accessible_name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ordinal: Option<usize>,
pub intents: std::collections::BTreeMap<String, ContextNeighborStats>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct ContextNeighborStats {
pub success_count: u64,
pub failure_count: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub last_verified_day: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source: Option<ContextAnswerSource>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct ContextSiteView {
pub site_key: String,
pub pages: std::collections::BTreeMap<
String,
std::collections::BTreeMap<String, Vec<ContextNeighborControl>>,
>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
pub enum ContextMissReason {
#[serde(rename = "notRemembered")]
NotRemembered,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
pub enum ContextNextStep {
#[serde(rename = "a11y_snapshot")]
A11ySnapshot,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ContextAskResponse {
pub answer: Option<ContextAnswer>,
pub hit: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reason: Option<ContextMissReason>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub next_step: Option<ContextNextStep>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct ContextAskResponseWire {
answer: Option<ContextAnswer>,
hit: bool,
#[serde(default)]
reason: Option<ContextMissReason>,
#[serde(default)]
next_step: Option<ContextNextStep>,
#[serde(default)]
page_derived: Option<bool>,
}
impl<'de> Deserialize<'de> for ContextAskResponse {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let wire = ContextAskResponseWire::deserialize(deserializer)?;
if wire.page_derived == Some(false) {
return Err(serde::de::Error::custom("pageDerived must be true"));
}
Ok(Self {
answer: wire.answer,
hit: wire.hit,
reason: wire.reason,
next_step: wire.next_step,
})
}
}
impl ContextAskResponse {
pub fn page_derived(&self) -> bool {
true
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ContextNeighborsResponse {
pub neighbors: Option<ContextNeighbors>,
pub hit: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reason: Option<ContextMissReason>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub next_step: Option<ContextNextStep>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct ContextNeighborsResponseWire {
neighbors: Option<ContextNeighbors>,
hit: bool,
#[serde(default)]
reason: Option<ContextMissReason>,
#[serde(default)]
next_step: Option<ContextNextStep>,
#[serde(default)]
page_derived: Option<bool>,
}
impl<'de> Deserialize<'de> for ContextNeighborsResponse {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let wire = ContextNeighborsResponseWire::deserialize(deserializer)?;
if wire.page_derived == Some(false) {
return Err(serde::de::Error::custom("pageDerived must be true"));
}
Ok(Self {
neighbors: wire.neighbors,
hit: wire.hit,
reason: wire.reason,
next_step: wire.next_step,
})
}
}
impl ContextNeighborsResponse {
pub fn page_derived(&self) -> bool {
true
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ContextSiteResponse {
pub site: Option<ContextSiteView>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct ContextSiteResponseWire {
site: Option<ContextSiteView>,
#[serde(default)]
page_derived: Option<bool>,
}
impl<'de> Deserialize<'de> for ContextSiteResponse {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let wire = ContextSiteResponseWire::deserialize(deserializer)?;
if wire.page_derived == Some(false) {
return Err(serde::de::Error::custom("pageDerived must be true"));
}
Ok(Self { site: wire.site })
}
}
impl ContextSiteResponse {
pub fn page_derived(&self) -> bool {
true
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct AccessibilityNode {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target: Option<AccessibilityTarget>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub value: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub required: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub disabled: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub read_only: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub invalid: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub checked: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub autocomplete: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub value_min: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub value_max: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub children: Vec<AccessibilityNode>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(
tag = "status",
rename_all = "camelCase",
rename_all_fields = "camelCase"
)]
pub enum CommandOutcome {
Completed {
command_id: CommandId,
evidence: Vec<Evidence>,
},
RetryableFailure {
command_id: CommandId,
error: CommandError,
},
NeedsReconciliation {
command_id: CommandId,
error: CommandError,
evidence: Vec<Evidence>,
},
PolicyDenied {
command_id: CommandId,
error: CommandError,
},
ResourceExhausted {
command_id: CommandId,
error: CommandError,
retry_after_ms: u64,
},
Restarted {
command_id: CommandId,
prior_attempt_id: AttemptId,
attempt_id: AttemptId,
reason: String,
#[serde(default)]
evidence: Vec<Evidence>,
},
Failed {
command_id: CommandId,
error: CommandError,
#[serde(default)]
evidence: Vec<Evidence>,
},
}
pub const MAX_WAIT_OBSERVED_CHARS: usize = 512;
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub enum SubmitSettlementOutcome {
Settled,
ValidationRejected,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(
tag = "kind",
rename_all = "camelCase",
rename_all_fields = "camelCase"
)]
pub enum Evidence {
ExecutionPath {
path: ExecutionPath,
reason: ExecutionReason,
state_version: u64,
elapsed_ms: u64,
bytes: Option<u64>,
sha256: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
final_url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
content_type: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
status: Option<u16>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
redirect_chain: Vec<String>,
},
Navigation {
url: String,
title: String,
},
Inspection {
selector: Option<String>,
url: String,
title: String,
text: String,
html: Option<String>,
},
SubmitSettlement {
outcome: SubmitSettlementOutcome,
},
Element {
selector: String,
text: Option<String>,
},
Upload {
selector: String,
paths: Vec<String>,
},
Page {
page_id: PageId,
url: String,
title: String,
},
PageGeneration {
page_id: PageId,
generation: u64,
},
Pages {
pages: Vec<PageEvidence>,
},
Popup {
opener_page_id: PageId,
page_id: PageId,
url: String,
title: String,
},
PopupClosed {
popup_page_id: PageId,
opener_page_id: PageId,
},
Download {
filename: String,
path: String,
bytes: u64,
sha256: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
saved_to: Option<String>,
},
Configuration {
name: String,
value: String,
},
Resolution {
target: Box<crate::TargetSpec>,
fingerprint: Box<TargetFingerprint>,
candidates: Vec<CandidateEvidence>,
best_match_authorized: bool,
},
Wait {
condition: crate::WaitCondition,
elapsed_ms: u64,
observations: u64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
excluded_classes: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
observed: Option<String>,
},
Screenshot {
artifact_id: String,
media_type: String,
width: u32,
height: u32,
bytes: u64,
sha256: String,
},
BrowserExecution {
engine: String,
browser_version: String,
profile_id: String,
interaction_path: String,
},
JavaScriptResult {
value: serde_json::Value,
truncated: bool,
},
AccessibilitySnapshot {
page_id: PageId,
nodes: Vec<AccessibilityNode>,
truncated: bool,
},
FormSnapshot {
snapshot: crate::FormSnapshot,
},
FormValidation {
issues: Vec<crate::FormValidationIssue>,
},
ControlAction {
action: ControlActionEvidence,
},
StructuredExtraction {
page_id: PageId,
value: serde_json::Value,
truncated: bool,
},
ChallengeDetection {
confidence: f32,
detection: Option<crate::challenges::ChallengeDetection>,
#[serde(default, skip_serializing_if = "Option::is_none")]
prior_kind: Option<String>,
},
CookieState {
page_id: Option<PageId>,
cookies: Vec<crate::CookieRecord>,
},
PdfArtifact {
artifact_id: String,
media_type: String,
bytes: u64,
sha256: String,
},
Dialog {
dialog_type: String,
message: String,
action: String,
},
Emulation {
viewport: Option<crate::ViewportSize>,
geolocation: Option<crate::GeolocationCoordinates>,
},
HarArtifact {
artifact_id: String,
media_type: String,
bytes: u64,
sha256: String,
entries: u32,
},
IntentExecution {
record: ExecutionRecord,
},
Humanization {
engine: String,
actions: u32,
synthesized_ms: u64,
},
Extraction {
field: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
value: Option<String>,
resolution_path: IntentResolutionPath,
#[serde(default, skip_serializing_if = "Option::is_none")]
error_code: Option<ErrorCode>,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub enum IntentResolutionPath {
Deterministic,
VisionFallback,
VisionPrefill,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct ExecutionRecord {
pub intent_kind: String,
pub purpose: Option<String>,
pub resolution_path: IntentResolutionPath,
pub plan_summary: String,
pub candidates: Vec<CandidateEvidence>,
pub wait_elapsed_ms: Option<u64>,
pub verification: String,
pub artifact_ids: Vec<String>,
pub vision_proposal_sha256: Option<String>,
}
impl Evidence {
pub fn page_derived(&self) -> bool {
matches!(
self,
Self::Inspection { .. }
| Self::AccessibilitySnapshot { .. }
| Self::FormSnapshot { .. }
| Self::StructuredExtraction { .. }
)
}
pub fn journal_safe(&self) -> Self {
fn safe_url(value: &str) -> String {
let Ok(mut url) = url::Url::parse(value) else {
return "[redacted-invalid-url]".into();
};
let _ = url.set_username("");
let _ = url.set_password(None);
url.set_query(None);
url.set_fragment(None);
url.to_string()
}
let mut safe = self.clone();
match &mut safe {
Self::ExecutionPath {
final_url,
redirect_chain,
..
} => {
if let Some(url) = final_url {
*url = safe_url(url);
}
for url in redirect_chain {
*url = safe_url(url);
}
}
Self::Navigation { url, .. }
| Self::Inspection { url, .. }
| Self::Page { url, .. }
| Self::Popup { url, .. } => *url = safe_url(url),
Self::Pages { pages } => {
for page in pages {
page.url = safe_url(&page.url);
}
}
Self::Upload { paths, .. } => {
for (index, path) in paths.iter_mut().enumerate() {
*path = format!("upload://evidence/{index}");
}
}
Self::Download {
path,
sha256,
saved_to,
..
} => {
*path = format!("artifact://sha256/{sha256}");
*saved_to = None;
}
Self::PopupClosed { .. } => {}
Self::BrowserExecution { .. } => {}
Self::IntentExecution { .. } => {}
Self::Extraction { .. } => {}
_ => {}
}
safe
}
}
pub fn annotate_page_derived_evidence(outcome: &mut serde_json::Value) {
let Some(evidence) = outcome
.get_mut("evidence")
.and_then(serde_json::Value::as_array_mut)
else {
return;
};
for item in evidence {
if matches!(
item.get("kind").and_then(serde_json::Value::as_str),
Some("inspection" | "accessibilitySnapshot" | "formSnapshot" | "structuredExtraction")
) {
item["pageDerived"] = serde_json::Value::Bool(true);
}
}
}
impl CommandOutcome {
pub fn journal_safe(&self) -> Self {
let mut safe = self.clone();
match &mut safe {
Self::Completed { evidence, .. }
| Self::NeedsReconciliation { evidence, .. }
| Self::Restarted { evidence, .. }
| Self::Failed { evidence, .. } => {
*evidence = evidence.iter().map(Evidence::journal_safe).collect();
}
_ => {}
}
match &mut safe {
Self::RetryableFailure { error, .. }
| Self::NeedsReconciliation { error, .. }
| Self::PolicyDenied { error, .. }
| Self::ResourceExhausted { error, .. }
| Self::Failed { error, .. } => error.message = "redacted durable diagnostic".into(),
_ => {}
}
safe
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub enum ExecutionPath {
DirectHttp,
#[serde(alias = "chromium")]
Browser,
#[serde(alias = "chromiumFallback")]
BrowserFallback,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub enum ExecutionReason {
EligibleStaticDocument,
EligibleExplicitDownload,
IneligibleCommand,
SemanticTargetRequired,
JavascriptRequired,
UnsupportedContentType,
StateConflict,
PolicyRequired,
PageMutated,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct CandidateEvidence {
pub role: Option<String>,
pub name: Option<String>,
pub score: i32,
pub reasons: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct TargetFingerprint {
pub page_id: PageId,
pub frame: Option<String>,
pub role: Option<String>,
pub name: Option<String>,
pub stable_attributes: std::collections::BTreeMap<String, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct PageEvidence {
pub page_id: PageId,
pub url: String,
pub title: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct CommandError {
pub code: ErrorCode,
pub message: String,
pub layer: ErrorLayer,
pub retryable: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub enum ErrorCode {
InvalidRequest,
NotFound,
DeadlineExceeded,
BrowserLaunchFailed,
BrowserCommandFailed,
VerificationFailed,
JournalFailed,
ResourceExhausted,
PolicyDenied,
Internal,
TargetNotFound,
TargetAmbiguous,
FrameNotFound,
ShadowRootUnavailable,
TargetDetached,
TargetObscured,
TargetOutOfBounds,
WaitConditionTimedOut,
ScreenshotCaptureFailed,
NetworkPolicyDenied,
HttpResponseTooLarge,
HttpTransferFailed,
HttpStateConflict,
HttpEquivalenceUnproven,
IntentCompileFailed,
IntentActionMismatch,
ObstructionSuspected,
VisionAssistDenied,
VisionAssistFailed,
ExpectedStatePreSatisfied,
BoundaryAlreadyExecuted,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub enum ErrorLayer {
Interface,
Broker,
Workflow,
Page,
Driver,
Browser,
Network,
Site,
Journal,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub enum CommandPhase {
Accepted,
Prepared,
Executing,
ResultPrepared,
Verifying,
Recovering,
Completed,
Failed,
}
#[derive(Debug, Error, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
pub enum RuntimeError {
#[error("not found: {0}")]
NotFound(String),
#[error("invalid request: {0}")]
InvalidRequest(String),
#[error("{0}")]
EngineUnreachable(String),
#[error("internal error: {0}")]
Internal(String),
}