feagi_agent/sdk/motor/perception/
decoder.rs

1//! Perception decoder implementation.
2
3use std::path::PathBuf;
4use std::time::Instant;
5
6use serde::Serialize;
7
8use crate::core::SdkError;
9use crate::sdk::base::{CorticalTopology, TopologyCache};
10use crate::sdk::motor::perception::config::PerceptionDecoderConfig;
11use crate::sdk::motor::traits::MotorDecoder;
12use crate::sdk::types::{
13    ColorChannelLayout, ColorSpace, CorticalChannelCount, CorticalChannelIndex, CorticalUnitIndex,
14    FrameChangeHandling, ImageFrame, ImageFrameProperties, ImageXYResolution, MiscData,
15    MiscDataDimensions, MotorCorticalUnit,
16};
17use feagi_sensorimotor::caching::MotorDeviceCache;
18use feagi_sensorimotor::data_types::text_token::decode_token_id_from_misc_data_with_depth;
19use feagi_structures::neuron_voxels::xyzp::CorticalMappedXYZPNeuronVoxels;
20use std::sync::Mutex;
21
22/// Semantic segmentation frame decoded from OSEG motor output.
23#[derive(Debug, Clone, Serialize)]
24pub struct OsegFrame {
25    pub width: u32,
26    pub height: u32,
27    pub depth: u32,
28    pub channels: u32,
29    pub x: Vec<u32>,
30    pub y: Vec<u32>,
31    pub z: Vec<u32>,
32    pub p: Vec<f32>,
33}
34
35/// RGB image frame decoded from OIMG motor output.
36#[derive(Debug, Clone, Serialize)]
37pub struct OimgFrame {
38    pub width: u32,
39    pub height: u32,
40    pub depth: u32,
41    pub channels: u32,
42    pub x: Vec<u32>,
43    pub y: Vec<u32>,
44    pub z: Vec<u32>,
45    pub p: Vec<u32>,
46}
47
48/// High-level perception frame returned to controllers/UI.
49#[derive(Debug, Clone, Serialize)]
50pub struct PerceptionFrame {
51    pub timestamp_ms: u64,
52    pub source_width: Option<u32>,
53    pub source_height: Option<u32>,
54    pub oseg: Option<OsegFrame>,
55    pub oimg: Option<OimgFrame>,
56    pub oten_text: Option<String>,
57    pub oten_token_id: Option<u32>,
58}
59
60/// Decoder for FEAGI motor outputs into higher-level perception frames.
61pub struct PerceptionDecoder {
62    config: PerceptionDecoderConfig,
63    motor_cache: Mutex<MotorDeviceCache>,
64    oseg_topology: CorticalTopology,
65    oimg_topology: CorticalTopology,
66    oten_topology: CorticalTopology,
67    #[cfg(feature = "sdk-text")]
68    tokenizer: Option<tokenizers::Tokenizer>,
69}
70
71impl PerceptionDecoder {
72    /// Create a new perception decoder (fetches required topologies).
73    pub async fn new(
74        config: PerceptionDecoderConfig,
75        topology_cache: &TopologyCache,
76        tokenizer_path: Option<PathBuf>,
77    ) -> Result<Self, SdkError> {
78        let unit = CorticalUnitIndex::from(config.cortical_unit_id);
79        let frame = FrameChangeHandling::Absolute;
80
81        let oseg_id =
82            MotorCorticalUnit::get_cortical_ids_array_for_object_segmentation_with_parameters(
83                frame, unit,
84            )[0];
85        let oimg_id =
86            MotorCorticalUnit::get_cortical_ids_array_for_simple_vision_output_with_parameters(
87                frame, unit,
88            )[0];
89        let oten_id =
90            MotorCorticalUnit::get_cortical_ids_array_for_text_english_output_with_parameters(
91                frame, unit,
92            )[0];
93
94        let oseg_topology = topology_cache.get_topology(&oseg_id).await?;
95        let oimg_topology = topology_cache.get_topology(&oimg_id).await?;
96        let oten_topology = topology_cache.get_topology(&oten_id).await?;
97
98        let mut motor_cache = MotorDeviceCache::new();
99        register_oseg(
100            &mut motor_cache,
101            unit,
102            oseg_topology,
103            FrameChangeHandling::Absolute,
104        )?;
105        register_oimg(
106            &mut motor_cache,
107            unit,
108            oimg_topology,
109            FrameChangeHandling::Absolute,
110        )?;
111        register_oten(
112            &mut motor_cache,
113            unit,
114            oten_topology,
115            FrameChangeHandling::Absolute,
116        )?;
117
118        #[cfg(feature = "sdk-text")]
119        let tokenizer = match tokenizer_path {
120            Some(path) => Some(
121                tokenizers::Tokenizer::from_file(path)
122                    .map_err(|e| SdkError::Other(format!("Tokenizer load failed: {e}")))?,
123            ),
124            None => None,
125        };
126
127        Ok(Self {
128            config,
129            motor_cache: Mutex::new(motor_cache),
130            oseg_topology,
131            oimg_topology,
132            oten_topology,
133            #[cfg(feature = "sdk-text")]
134            tokenizer,
135        })
136    }
137
138    /// Returns availability flags for (oseg, oimg, oten).
139    pub fn available_areas(&self) -> (bool, bool, bool) {
140        (
141            self.oseg_topology.width > 0,
142            self.oimg_topology.width > 0,
143            self.oten_topology.width > 0,
144        )
145    }
146
147    fn decode_oseg(&self, unit: CorticalUnitIndex) -> Result<Option<OsegFrame>, SdkError> {
148        // TODO: Support multiple channels and channel-wise aggregation.
149        let channel = CorticalChannelIndex::from(0u32);
150        let wrapped = self
151            .motor_cache
152            .lock()
153            .map_err(|_| SdkError::Other("OSEG cache lock poisoned".to_string()))?
154            .object_segmentation_read_postprocessed_cache_value(unit, channel)
155            .map_err(|e| SdkError::Other(format!("OSEG read failed: {e}")))?;
156        let misc: MiscData = wrapped;
157        let dims = misc.get_dimensions();
158
159        let mut x = Vec::new();
160        let mut y = Vec::new();
161        let mut z = Vec::new();
162        let mut p = Vec::new();
163        for ((xi, yi, zi), val) in misc.get_internal_data().indexed_iter() {
164            if val.abs() <= f32::EPSILON {
165                continue;
166            }
167            x.push(xi as u32);
168            y.push(yi as u32);
169            z.push(zi as u32);
170            p.push(*val);
171        }
172
173        Ok(Some(OsegFrame {
174            width: dims.width,
175            height: dims.height,
176            depth: dims.depth,
177            channels: self.oseg_topology.channels.max(1),
178            x,
179            y,
180            z,
181            p,
182        }))
183    }
184
185    fn decode_oimg(&self, unit: CorticalUnitIndex) -> Result<Option<OimgFrame>, SdkError> {
186        // TODO: Support multiple channels and channel-wise aggregation.
187        let channel = CorticalChannelIndex::from(0u32);
188        let wrapped = self
189            .motor_cache
190            .lock()
191            .map_err(|_| SdkError::Other("OIMG cache lock poisoned".to_string()))?
192            .simple_vision_output_read_postprocessed_cache_value(unit, channel)
193            .map_err(|e| SdkError::Other(format!("OIMG read failed: {e}")))?;
194        let image: ImageFrame = wrapped;
195
196        let resolution = image.get_image_frame_properties().get_image_resolution();
197        let mut x = Vec::new();
198        let mut y = Vec::new();
199        let mut z = Vec::new();
200        let mut p = Vec::new();
201
202        let height = resolution.height;
203        for ((row, col, channel), val) in image.get_internal_data().indexed_iter() {
204            if *val == 0 {
205                continue;
206            }
207            x.push(col as u32);
208            y.push(height - 1 - (row as u32));
209            z.push(channel as u32);
210            p.push(*val as u32);
211        }
212
213        Ok(Some(OimgFrame {
214            width: resolution.width,
215            height: resolution.height,
216            depth: image
217                .get_image_frame_properties()
218                .get_color_channel_layout() as u32,
219            channels: self.oimg_topology.channels.max(1),
220            x,
221            y,
222            z,
223            p,
224        }))
225    }
226
227    fn decode_oten(
228        &self,
229        unit: CorticalUnitIndex,
230    ) -> Result<(Option<u32>, Option<String>), SdkError> {
231        let channel = CorticalChannelIndex::from(0u32);
232        let wrapped = self
233            .motor_cache
234            .lock()
235            .map_err(|_| SdkError::Other("OTEN cache lock poisoned".to_string()))?
236            .text_english_output_read_postprocessed_cache_value(unit, channel)
237            .map_err(|e| SdkError::Other(format!("OTEN read failed: {e}")))?;
238        let misc: MiscData = wrapped;
239        let token_id = decode_token_id_from_misc_data_with_depth(&misc, self.oten_topology.depth)
240            .map_err(|e| SdkError::Other(format!("OTEN token decode failed: {e}")))?;
241
242        #[cfg(feature = "sdk-text")]
243        let text = if let (Some(token_id), Some(tokenizer)) = (token_id, &self.tokenizer) {
244            tokenizer.decode(&[token_id], true).ok()
245        } else {
246            None
247        };
248
249        #[cfg(not(feature = "sdk-text"))]
250        let text: Option<String> = None;
251
252        Ok((token_id, text))
253    }
254}
255
256impl MotorDecoder for PerceptionDecoder {
257    type Input = CorticalMappedXYZPNeuronVoxels;
258    type Output = PerceptionFrame;
259
260    fn decode(&self, input: &Self::Input) -> Result<Self::Output, SdkError> {
261        {
262            let mut cache = self
263                .motor_cache
264                .lock()
265                .map_err(|_| SdkError::Other("Motor cache lock poisoned".to_string()))?;
266            cache
267                .ingest_neuron_data_and_run_callbacks(input.clone(), Instant::now())
268                .map_err(|e| SdkError::Other(format!("Motor cache ingest failed: {e}")))?;
269        }
270
271        let unit = CorticalUnitIndex::from(self.config.cortical_unit_id);
272        let oseg = self.decode_oseg(unit)?;
273        let oimg = self.decode_oimg(unit)?;
274        let (oten_token_id, oten_text) = self.decode_oten(unit)?;
275
276        let timestamp_ms = std::time::SystemTime::now()
277            .duration_since(std::time::UNIX_EPOCH)
278            .map_err(|e| SdkError::Other(format!("Timestamp error: {e}")))?
279            .as_millis() as u64;
280
281        Ok(PerceptionFrame {
282            timestamp_ms,
283            source_width: oimg.as_ref().map(|f| f.width),
284            source_height: oimg.as_ref().map(|f| f.height),
285            oseg,
286            oimg,
287            oten_text,
288            oten_token_id,
289        })
290    }
291}
292
293fn register_oseg(
294    motor_cache: &mut MotorDeviceCache,
295    unit: CorticalUnitIndex,
296    topology: CorticalTopology,
297    frame: FrameChangeHandling,
298) -> Result<(), SdkError> {
299    let channels = CorticalChannelCount::new(topology.channels)
300        .map_err(|e| SdkError::Other(format!("Invalid OSEG channel count: {e}")))?;
301    let dims = MiscDataDimensions::new(topology.width, topology.height, topology.depth)
302        .map_err(|e| SdkError::Other(format!("Invalid OSEG dimensions: {e}")))?;
303    motor_cache
304        .object_segmentation_register(unit, channels, frame, dims)
305        .map_err(|e| SdkError::Other(format!("OSEG register failed: {e}")))
306}
307
308fn register_oimg(
309    motor_cache: &mut MotorDeviceCache,
310    unit: CorticalUnitIndex,
311    topology: CorticalTopology,
312    frame: FrameChangeHandling,
313) -> Result<(), SdkError> {
314    let channels = CorticalChannelCount::new(topology.channels)
315        .map_err(|e| SdkError::Other(format!("Invalid OIMG channel count: {e}")))?;
316    let resolution = ImageXYResolution::new(topology.width, topology.height)
317        .map_err(|e| SdkError::Other(format!("Invalid OIMG resolution: {e}")))?;
318    let layout = match topology.depth {
319        1 => ColorChannelLayout::GrayScale,
320        3 => ColorChannelLayout::RGB,
321        _ => {
322            return Err(SdkError::Other(format!(
323                "Unsupported OIMG depth: {}",
324                topology.depth
325            )))
326        }
327    };
328    // TODO: allow caller-configurable ColorSpace.
329    let props = ImageFrameProperties::new(resolution, ColorSpace::Gamma, layout)
330        .map_err(|e| SdkError::Other(format!("OIMG properties error: {e}")))?;
331    motor_cache
332        .simple_vision_output_register(unit, channels, frame, props)
333        .map_err(|e| SdkError::Other(format!("OIMG register failed: {e}")))
334}
335
336fn register_oten(
337    motor_cache: &mut MotorDeviceCache,
338    unit: CorticalUnitIndex,
339    topology: CorticalTopology,
340    frame: FrameChangeHandling,
341) -> Result<(), SdkError> {
342    let channels = CorticalChannelCount::new(topology.channels)
343        .map_err(|e| SdkError::Other(format!("Invalid OTEN channel count: {e}")))?;
344    let dims = MiscDataDimensions::new(topology.width, topology.height, topology.depth)
345        .map_err(|e| SdkError::Other(format!("Invalid OTEN dimensions: {e}")))?;
346    motor_cache
347        .text_english_output_register(unit, channels, frame, dims)
348        .map_err(|e| SdkError::Other(format!("OTEN register failed: {e}")))
349}
feagi_agent/sdk/motor/perception/decoder.rs

feagi_agent/sdk/motor/perception/
decoder.rs
