glass/browser/session/workflow/
recorder.rs

1use super::*;
2/// Semantic target captured by the workflow recorder.
3#[derive(Debug, Clone, Serialize, Deserialize)]
4#[serde(rename_all = "camelCase")]
5pub struct WorkflowRecordedTarget {
6    pub role: String,
7    pub name: String,
8    #[serde(default, skip_serializing_if = "Option::is_none")]
9    pub context: Option<String>,
10    #[serde(default, skip_serializing_if = "Option::is_none")]
11    pub region_kind: Option<super::super::SemanticRegionKind>,
12}
13
14/// Bounded route evidence captured by a semantic recorder.
15///
16/// Browser target and frame handles are hashed because they are useful for
17/// comparing a recording with later evidence but are not valid replay
18/// selectors. Query strings and fragments are removed from the retained URL.
19#[derive(Debug, Clone, Serialize, Deserialize)]
20#[serde(rename_all = "camelCase")]
21pub struct WorkflowRecordedRoute {
22    pub target_digest: String,
23    pub frame_digest: String,
24    pub url: String,
25}
26
27/// Resolution evidence retained with one semantic draft step.
28#[derive(Debug, Clone, Serialize, Deserialize)]
29#[serde(rename_all = "camelCase")]
30pub struct WorkflowRecordedSemantic {
31    pub intent: String,
32    pub normalized_intent: String,
33    pub action: SemanticIntentAction,
34    pub resolution: SemanticResolution,
35    pub policy_decision: IntentPolicyDecision,
36    pub candidate_count: usize,
37    pub excluded_count: usize,
38    pub ambiguous: bool,
39    #[serde(default, skip_serializing_if = "Option::is_none")]
40    pub revision: Option<u64>,
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub route: Option<WorkflowRecordedRoute>,
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub target_fingerprint: Option<String>,
45}
46
47/// Confidence attached to a recorded draft, never to a runtime guarantee.
48#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
49#[serde(rename_all = "snake_case")]
50pub enum WorkflowRecordingConfidence {
51    High,
52    Medium,
53    Low,
54}
55
56/// One reviewable semantic recorder draft step.
57#[derive(Debug, Clone, Serialize, Deserialize)]
58#[serde(rename_all = "camelCase")]
59pub struct WorkflowDraftStep {
60    pub id: String,
61    pub action: BatchStep,
62    #[serde(skip_serializing_if = "Option::is_none")]
63    pub intent: Option<WorkflowIntentStep>,
64    #[serde(skip_serializing_if = "Option::is_none")]
65    pub target: Option<WorkflowRecordedTarget>,
66    #[serde(skip_serializing_if = "Option::is_none")]
67    pub semantic: Option<WorkflowRecordedSemantic>,
68    #[serde(skip_serializing_if = "Option::is_none")]
69    pub expect: Option<VerificationPredicate>,
70    pub transaction: WorkflowTransactionClass,
71    pub confidence: WorkflowRecordingConfidence,
72    pub review_required: bool,
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub input_name: Option<String>,
75    #[serde(default)]
76    pub sensitive_input: bool,
77}
78
79/// A bounded recorder output that remains a draft until explicitly reviewed.
80#[derive(Debug, Clone, Serialize, Deserialize)]
81#[serde(rename_all = "camelCase")]
82pub struct WorkflowDraft {
83    pub schema_version: u32,
84    pub name: String,
85    pub workflow_version: String,
86    pub steps: Vec<WorkflowDraftStep>,
87}
88
89/// In-memory recorder for semantic workflow drafts.
90#[derive(Debug, Clone)]
91pub struct WorkflowRecorder {
92    draft: WorkflowDraft,
93}
94
95impl WorkflowRecorder {
96    /// Start a bounded recorder draft. Recording is local and does not attach
97    /// to Chrome or intercept browser traffic.
98    pub fn new(name: impl Into<String>, workflow_version: impl Into<String>) -> Self {
99        Self {
100            draft: WorkflowDraft {
101                schema_version: WORKFLOW_SCHEMA_VERSION,
102                name: name.into(),
103                workflow_version: workflow_version.into(),
104                steps: Vec::new(),
105            },
106        }
107    }
108
109    /// Record a semantic click draft using an explicit role and accessible name.
110    pub fn record_click(
111        &mut self,
112        id: impl Into<String>,
113        role: impl Into<String>,
114        name: impl Into<String>,
115        expect: Option<VerificationPredicate>,
116    ) -> Result<(), WorkflowValidationError> {
117        let target = recorded_target(role.into(), name.into(), None, None)?;
118        let locator = format!("role={};name={}", target.role, target.name);
119        self.push(WorkflowDraftStep {
120            id: id.into(),
121            action: BatchStep::Click { target: locator },
122            intent: None,
123            target: Some(target),
124            semantic: None,
125            expect,
126            transaction: WorkflowTransactionClass::Unknown,
127            confidence: WorkflowRecordingConfidence::High,
128            review_required: true,
129            input_name: None,
130            sensitive_input: false,
131        })
132    }
133
134    /// Record text as a typed input placeholder, never as a literal value.
135    pub fn record_type_input(
136        &mut self,
137        id: impl Into<String>,
138        role: impl Into<String>,
139        name: impl Into<String>,
140        input_name: impl Into<String>,
141    ) -> Result<(), WorkflowValidationError> {
142        let target = recorded_target(role.into(), name.into(), None, None)?;
143        let input_name = input_name.into();
144        validate_name("inputName", &input_name)?;
145        let sensitive_input = looks_sensitive_input_name(&input_name);
146        let locator = format!("role={};name={}", target.role, target.name);
147        self.push(WorkflowDraftStep {
148            id: id.into(),
149            action: BatchStep::Type {
150                text: format!("${{inputs.{input_name}}}"),
151                target: Some(locator),
152            },
153            intent: None,
154            target: Some(target),
155            semantic: None,
156            expect: None,
157            transaction: WorkflowTransactionClass::Unknown,
158            confidence: WorkflowRecordingConfidence::High,
159            review_required: true,
160            input_name: Some(input_name),
161            sensitive_input,
162        })
163    }
164
165    /// Record a read-only observation draft.
166    pub fn record_observe(&mut self, id: impl Into<String>) -> Result<(), WorkflowValidationError> {
167        self.push(WorkflowDraftStep {
168            id: id.into(),
169            action: BatchStep::Observe {
170                include_dom: false,
171                include_screenshot: false,
172                include_form_values: false,
173            },
174            intent: None,
175            target: None,
176            semantic: None,
177            expect: None,
178            transaction: WorkflowTransactionClass::ReadOnly,
179            confidence: WorkflowRecordingConfidence::High,
180            review_required: true,
181            input_name: None,
182            sensitive_input: false,
183        })
184    }
185
186    /// Record a semantic resolution as a reviewable workflow intent step.
187    ///
188    /// The result may be ambiguous, rejected, or lack a selected candidate;
189    /// those states are retained as evidence and never turned into a replay
190    /// target. Value-bearing actions receive an input placeholder only.
191    pub fn record_semantic_intent(
192        &mut self,
193        id: impl Into<String>,
194        request: &SemanticIntentRequest,
195        result: &SemanticIntentResult,
196        input_name: Option<impl Into<String>>,
197        transaction: WorkflowTransactionClass,
198        expect: Option<VerificationPredicate>,
199    ) -> Result<(), WorkflowValidationError> {
200        request
201            .validate()
202            .map_err(|error| WorkflowValidationError::new("semantic.request", error.to_string()))?;
203        result
204            .validate()
205            .map_err(|error| WorkflowValidationError::new("semantic.result", error.to_string()))?;
206        if request.action != result.action || request.intent != result.intent {
207            return Err(WorkflowValidationError::new(
208                "semantic",
209                "request and result action/intent do not match",
210            ));
211        }
212
213        let input_name = input_name.map(Into::into);
214        let value = match request.action {
215            SemanticIntentAction::Type | SemanticIntentAction::Select => {
216                let input_name = input_name.as_deref().ok_or_else(|| {
217                    WorkflowValidationError::new(
218                        "inputName",
219                        "type and select recordings require an input name",
220                    )
221                })?;
222                validate_name("inputName", input_name)?;
223                Some(format!("${{inputs.{input_name}}}"))
224            }
225            _ if input_name.is_some() => {
226                return Err(WorkflowValidationError::new(
227                    "inputName",
228                    "only type and select recordings accept an input name",
229                ));
230            }
231            _ => None,
232        };
233
234        let selected = result.selected_candidate.as_deref().and_then(|id| {
235            result
236                .candidates
237                .iter()
238                .find(|candidate| candidate.id == id)
239        });
240        let target = selected
241            .map(|candidate| {
242                recorded_target(
243                    candidate.role.clone(),
244                    candidate.name.clone(),
245                    candidate.region_kind.map(|kind| format!("{kind:?}")),
246                    candidate.region_kind,
247                )
248            })
249            .transpose()?;
250        let target_fingerprint = selected.and_then(|candidate| {
251            candidate.fingerprint.as_ref().map(|fingerprint| {
252                target_fingerprint_digest(
253                    &candidate.role,
254                    &candidate.name,
255                    candidate.input_type.as_deref(),
256                    candidate.region_kind,
257                    fingerprint.purpose,
258                )
259            })
260        });
261        let confidence = selected
262            .map(|candidate| recording_confidence(candidate.confidence))
263            .unwrap_or(WorkflowRecordingConfidence::Low);
264        let semantic = WorkflowRecordedSemantic {
265            intent: result.intent.clone(),
266            normalized_intent: result.normalized_intent.clone(),
267            action: result.action,
268            resolution: result.resolution,
269            policy_decision: result.policy_decision,
270            candidate_count: result.candidates.len(),
271            excluded_count: result.excluded_count,
272            ambiguous: matches!(result.resolution, SemanticResolution::Ambiguous),
273            revision: result.revision,
274            route: result.route.as_ref().map(recorded_route),
275            target_fingerprint,
276        };
277        let intent = WorkflowIntentStep {
278            action: request.action,
279            purpose: None,
280            intent: Some(request.intent.clone()),
281            scope: request.scope.clone(),
282            constraints: request.constraints.clone(),
283            resolution_policy: request.resolution_policy,
284            value,
285        };
286        self.push(WorkflowDraftStep {
287            id: id.into(),
288            action: BatchStep::Observe {
289                include_dom: false,
290                include_screenshot: false,
291                include_form_values: false,
292            },
293            intent: Some(intent),
294            target,
295            semantic: Some(semantic),
296            expect,
297            transaction,
298            confidence,
299            review_required: true,
300            sensitive_input: input_name
301                .as_deref()
302                .is_some_and(looks_sensitive_input_name),
303            input_name,
304        })
305    }
306
307    pub fn draft(&self) -> &WorkflowDraft {
308        &self.draft
309    }
310
311    /// Infer declarations for the placeholders observed in the draft.
312    ///
313    /// No value is retained. Names that look sensitive are marked sensitive;
314    /// callers still decide whether the resulting declaration is appropriate
315    /// before compiling the final workflow.
316    pub fn inferred_inputs(&self) -> BTreeMap<String, WorkflowInput> {
317        let mut inputs = BTreeMap::new();
318        for step in &self.draft.steps {
319            let Some(name) = &step.input_name else {
320                continue;
321            };
322            let sensitive = step.sensitive_input;
323            inputs
324                .entry(name.clone())
325                .and_modify(|input: &mut WorkflowInput| {
326                    if sensitive {
327                        input.sensitive = Some(true);
328                    }
329                })
330                .or_insert_with(|| WorkflowInput {
331                    value_type: WorkflowValueType::String,
332                    required: true,
333                    max_length: None,
334                    sensitive: sensitive.then_some(true),
335                });
336        }
337        inputs
338    }
339
340    /// Convert a reviewed draft into the normal validated workflow contract.
341    pub fn into_definition(
342        self,
343        inputs: BTreeMap<String, WorkflowInput>,
344        budgets: WorkflowBudgets,
345        terminal_condition: VerificationPredicate,
346        outputs: BTreeMap<String, WorkflowOutputDeclaration>,
347    ) -> Result<WorkflowDefinition, WorkflowValidationError> {
348        let definition = WorkflowDefinition {
349            schema_version: self.draft.schema_version,
350            name: self.draft.name,
351            workflow_version: self.draft.workflow_version,
352            description: Some("Recorded draft; review before execution.".into()),
353            inputs,
354            budgets,
355            preconditions: Vec::new(),
356            steps: self
357                .draft
358                .steps
359                .into_iter()
360                .map(|step| WorkflowStep {
361                    id: step.id,
362                    action: step.action,
363                    intent: step.intent,
364                    when: None,
365                    expect: step.expect,
366                    before_retry: None,
367                    transaction: step.transaction,
368                    idempotency_key: None,
369                    max_retries: 0,
370                    repeat: 1,
371                })
372                .collect(),
373            terminal_condition,
374            outputs,
375        };
376        definition.validate()?;
377        Ok(definition)
378    }
379
380    fn push(&mut self, step: WorkflowDraftStep) -> Result<(), WorkflowValidationError> {
381        if self.draft.steps.len() >= MAX_STEPS {
382            return Err(WorkflowValidationError::new(
383                "steps",
384                format!("must contain at most {MAX_STEPS} entries"),
385            ));
386        }
387        validate_name("steps.id", &step.id)?;
388        if self.draft.steps.iter().any(|item| item.id == step.id) {
389            return Err(WorkflowValidationError::new(
390                "steps.id",
391                format!("duplicate step ID {:?}", step.id),
392            ));
393        }
394        self.draft.steps.push(step);
395        Ok(())
396    }
397}
398
399fn recorded_target(
400    role: String,
401    name: String,
402    context: Option<String>,
403    region_kind: Option<super::super::SemanticRegionKind>,
404) -> Result<WorkflowRecordedTarget, WorkflowValidationError> {
405    validate_bytes("target.role", &role, 1, 128)?;
406    validate_bytes("target.name", &name, 1, MAX_TARGET_BYTES)?;
407    if role.contains([';', '\n', '\r']) || name.contains([';', '\n', '\r']) {
408        return Err(WorkflowValidationError::new(
409            "target",
410            "semantic target fields cannot contain locator separators or newlines",
411        ));
412    }
413    if let Some(context) = &context {
414        validate_bytes("target.context", context, 1, 256)?;
415    }
416    Ok(WorkflowRecordedTarget {
417        role,
418        name,
419        context,
420        region_kind,
421    })
422}
423
424fn recording_confidence(confidence: IntentConfidence) -> WorkflowRecordingConfidence {
425    match confidence {
426        IntentConfidence::Exact | IntentConfidence::High => WorkflowRecordingConfidence::High,
427        IntentConfidence::Medium => WorkflowRecordingConfidence::Medium,
428        IntentConfidence::Low | IntentConfidence::Insufficient => WorkflowRecordingConfidence::Low,
429    }
430}
431
432fn looks_sensitive_input_name(name: &str) -> bool {
433    let normalized = name.to_ascii_lowercase();
434    [
435        "password", "passwd", "secret", "token", "api_key", "apikey", "cookie",
436    ]
437    .iter()
438    .any(|term| normalized.contains(term))
439}
440
441fn recorded_route(route: &SemanticRouteIdentity) -> WorkflowRecordedRoute {
442    let url = Url::parse(&route.url)
443        .map(|mut parsed| {
444            let _ = parsed.set_username("");
445            let _ = parsed.set_password(None);
446            parsed.set_query(None);
447            parsed.set_fragment(None);
448            parsed.to_string()
449        })
450        .unwrap_or_else(|_| bound_workflow_text(&route.url, 2_048));
451    WorkflowRecordedRoute {
452        target_digest: hash_recorded_identifier(&route.target_id),
453        frame_digest: hash_recorded_identifier(&route.frame_id),
454        url,
455    }
456}
457
458fn hash_recorded_identifier(value: &str) -> String {
459    let digest = Sha256::digest(value.as_bytes());
460    format!("sha256:{digest:x}")
461}
glass/browser/session/workflow/recorder.rs

glass/browser/session/workflow/
recorder.rs
