agentic_vision/
types.rs

1//! Core data types for visual observations and memory.
2
3use serde::{Deserialize, Serialize};
4
5/// A captured visual observation stored in visual memory.
6#[derive(Debug, Clone, Serialize, Deserialize)]
7pub struct VisualObservation {
8    pub id: u64,
9    pub timestamp: u64,
10    pub session_id: u32,
11    pub source: CaptureSource,
12    pub embedding: Vec<f32>,
13    pub thumbnail: Vec<u8>,
14    pub metadata: ObservationMeta,
15    pub memory_link: Option<u64>,
16}
17
18/// How the image was captured.
19#[derive(Debug, Clone, Serialize, Deserialize)]
20#[serde(tag = "type", rename_all = "snake_case")]
21pub enum CaptureSource {
22    File { path: String },
23    Base64 { mime: String },
24    Screenshot { region: Option<Rect> },
25    Clipboard,
26}
27
28/// Metadata about a visual observation.
29#[derive(Debug, Clone, Serialize, Deserialize)]
30pub struct ObservationMeta {
31    pub width: u32,
32    pub height: u32,
33    pub original_width: u32,
34    pub original_height: u32,
35    pub labels: Vec<String>,
36    pub description: Option<String>,
37}
38
39/// Pixel-level diff between two captures.
40#[derive(Debug, Clone, Serialize, Deserialize)]
41pub struct VisualDiff {
42    pub before_id: u64,
43    pub after_id: u64,
44    pub similarity: f32,
45    pub changed_regions: Vec<Rect>,
46    pub pixel_diff_ratio: f32,
47}
48
49/// A rectangle region.
50#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
51pub struct Rect {
52    pub x: u32,
53    pub y: u32,
54    pub w: u32,
55    pub h: u32,
56}
57
58/// A similarity match result.
59#[derive(Debug, Clone, Serialize, Deserialize)]
60pub struct SimilarityMatch {
61    pub id: u64,
62    pub similarity: f32,
63}
64
65/// In-memory container for all visual observations.
66#[derive(Debug, Clone)]
67pub struct VisualMemoryStore {
68    pub observations: Vec<VisualObservation>,
69    pub embedding_dim: u32,
70    pub next_id: u64,
71    pub session_count: u32,
72    pub created_at: u64,
73    pub updated_at: u64,
74}
75
76impl VisualMemoryStore {
77    /// Create a new empty store.
78    pub fn new(embedding_dim: u32) -> Self {
79        let now = std::time::SystemTime::now()
80            .duration_since(std::time::UNIX_EPOCH)
81            .unwrap_or_default()
82            .as_secs();
83
84        Self {
85            observations: Vec::new(),
86            embedding_dim,
87            next_id: 1,
88            session_count: 0,
89            created_at: now,
90            updated_at: now,
91        }
92    }
93
94    /// Get an observation by ID.
95    pub fn get(&self, id: u64) -> Option<&VisualObservation> {
96        self.observations.iter().find(|o| o.id == id)
97    }
98
99    /// Get a mutable observation by ID.
100    pub fn get_mut(&mut self, id: u64) -> Option<&mut VisualObservation> {
101        self.observations.iter_mut().find(|o| o.id == id)
102    }
103
104    /// Add an observation and return its assigned ID.
105    pub fn add(&mut self, mut obs: VisualObservation) -> u64 {
106        let id = self.next_id;
107        obs.id = id;
108        self.next_id += 1;
109        self.updated_at = std::time::SystemTime::now()
110            .duration_since(std::time::UNIX_EPOCH)
111            .unwrap_or_default()
112            .as_secs();
113        self.observations.push(obs);
114        id
115    }
116
117    /// Return the number of observations.
118    pub fn count(&self) -> usize {
119        self.observations.len()
120    }
121
122    /// Get observations filtered by session ID.
123    pub fn by_session(&self, session_id: u32) -> Vec<&VisualObservation> {
124        self.observations
125            .iter()
126            .filter(|o| o.session_id == session_id)
127            .collect()
128    }
129
130    /// Get observations in a timestamp range.
131    pub fn in_time_range(&self, start: u64, end: u64) -> Vec<&VisualObservation> {
132        self.observations
133            .iter()
134            .filter(|o| o.timestamp >= start && o.timestamp <= end)
135            .collect()
136    }
137
138    /// Get the most recent observations.
139    pub fn recent(&self, limit: usize) -> Vec<&VisualObservation> {
140        let mut sorted: Vec<_> = self.observations.iter().collect();
141        sorted.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
142        sorted.truncate(limit);
143        sorted
144    }
145}
146
147/// Errors that can occur in the vision library.
148#[derive(thiserror::Error, Debug)]
149pub enum VisionError {
150    #[error("Image error: {0}")]
151    Image(#[from] image::ImageError),
152
153    #[error("IO error: {0}")]
154    Io(#[from] std::io::Error),
155
156    #[error("Embedding error: {0}")]
157    Embedding(String),
158
159    #[error("Storage error: {0}")]
160    Storage(String),
161
162    #[error("Capture not found: {0}")]
163    CaptureNotFound(u64),
164
165    #[error("Invalid input: {0}")]
166    InvalidInput(String),
167
168    #[error("Capture error: {0}")]
169    Capture(String),
170
171    #[error("Model not available: {0}")]
172    ModelNotAvailable(String),
173}
174
175/// Convenience result type.
176pub type VisionResult<T> = Result<T, VisionError>;
agentic_vision/types.rs

agentic_vision/
types.rs
