feagi_sensorimotor/feedbacks/
feedback_registration.rs1use crate::caching::{MotorDeviceCache, SensorDeviceCache};
2use crate::data_pipeline::{PipelineStageProperties, PipelineStagePropertyIndex};
3use crate::data_types::descriptors::{
4 ColorChannelLayout, ColorSpace, ImageFrameProperties, SegmentedImageFrameProperties,
5 SegmentedXYImageResolutions,
6};
7use crate::data_types::{GazeProperties, ImageFilteringSettings};
8use crate::feedbacks::feedback_registrar::FeedbackRegistrar;
9use crate::feedbacks::feedback_registration_targets::FeedbackRegistrationTargets;
10use crate::wrapped_io_data::WrappedIOData;
11use feagi_structures::genomic::{MotorCorticalUnit, SensoryCorticalUnit};
12use feagi_structures::{FeagiDataError, FeagiSignalIndex};
13use serde::{Deserialize, Serialize};
14use std::fmt::{Display, Formatter};
15use std::sync::{Arc, Mutex};
16
17#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
18pub enum FeedBackRegistration {
19 SegmentedVisionWithGaze {},
20 SegmentedVisionWithImageFiltering {},
21 VisionWithImageFiltering {},
22}
23
24impl Display for FeedBackRegistration {
25 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
26 match self {
27 FeedBackRegistration::SegmentedVisionWithGaze {} => {
28 write!(f, "SegmentedVisionWithGaze")
29 }
30 FeedBackRegistration::SegmentedVisionWithImageFiltering {} => {
31 write!(f, "SegmentedVisionWithImageFiltering")
32 }
33 FeedBackRegistration::VisionWithImageFiltering {} => {
34 write!(f, "VisionWithImageFiltering")
35 }
36 }
37 }
38}
39
40impl FeedBackRegistration {
41 pub(crate) fn try_registering_feedback_and_save(
42 &self,
43 feedback_registrar: &mut FeedbackRegistrar,
44 sensor_cache: Arc<Mutex<SensorDeviceCache>>,
45 motor_cache: Arc<Mutex<MotorDeviceCache>>,
46 target: FeedbackRegistrationTargets,
47 ) -> Result<(), FeagiDataError> {
48 self.try_registering_feedbacks(sensor_cache, motor_cache, target.clone())?;
49 let _ = feedback_registrar.push_verified_feedback(target, self.clone());
50 Ok(())
51 }
52
53 pub(crate) fn try_registering_feedbacks(
54 &self,
55 sensor_cache: Arc<Mutex<SensorDeviceCache>>,
56 motor_cache: Arc<Mutex<MotorDeviceCache>>,
57 target: FeedbackRegistrationTargets,
58 ) -> Result<(), FeagiDataError> {
59 let (sensory_cortical_unit, motor_cortical_unit) = self.get_sensor_motor_cortical_units();
60
61 let target_stage_index: PipelineStagePropertyIndex;
62
63 {
65 let sensors = sensor_cache.lock().unwrap();
66 sensors.verify_existence(
67 sensory_cortical_unit,
68 target.get_sensor_unit_index(),
69 target.get_sensor_channel_index(),
70 )?;
71 target_stage_index = sensors.try_get_index_of_first_stage_property_type_of(
72 sensory_cortical_unit,
73 target.get_sensor_unit_index(),
74 target.get_sensor_channel_index(),
75 &self.create_example_property(),
76 )?
77 }
78 {
79 let motors = motor_cache.lock().unwrap();
80 motors.verify_existence(
81 motor_cortical_unit,
82 target.get_motor_unit_index(),
83 target.get_motor_channel_index(),
84 )?;
85 }
86
87 match self {
88 FeedBackRegistration::SegmentedVisionWithGaze {} => {
89 feedback_segmented_vision_with_gaze(
90 &target,
91 sensor_cache.clone(),
92 motor_cache.clone(),
93 target_stage_index,
94 )?;
95 }
96 FeedBackRegistration::SegmentedVisionWithImageFiltering {} => {
97 feedback_segmented_vision_with_image_filtering(
98 &target,
99 sensor_cache.clone(),
100 motor_cache.clone(),
101 target_stage_index,
102 )?;
103 }
104 FeedBackRegistration::VisionWithImageFiltering {} => {
105 feedback_simple_vision_with_image_filtering(
106 &target,
107 sensor_cache.clone(),
108 motor_cache.clone(),
109 target_stage_index,
110 )?;
111 }
112 }
113 Ok(())
114 }
115
116 fn get_sensor_motor_cortical_units(&self) -> (SensoryCorticalUnit, MotorCorticalUnit) {
117 match &self {
118 FeedBackRegistration::SegmentedVisionWithGaze {} => (
119 SensoryCorticalUnit::SegmentedVision,
120 MotorCorticalUnit::Gaze,
121 ),
122 FeedBackRegistration::SegmentedVisionWithImageFiltering {} => (
123 SensoryCorticalUnit::SegmentedVision,
124 MotorCorticalUnit::DynamicImageProcessing,
125 ),
126 FeedBackRegistration::VisionWithImageFiltering {} => (
127 SensoryCorticalUnit::Vision,
128 MotorCorticalUnit::DynamicImageProcessing,
129 ),
130 }
131 }
132
133 fn create_example_property(&self) -> PipelineStageProperties {
134 match &self {
135 FeedBackRegistration::SegmentedVisionWithGaze {} => {
136 PipelineStageProperties::ImageFrameSegmentator {
137 input_image_properties: ImageFrameProperties::new(
138 (1, 1).try_into().unwrap(),
139 ColorSpace::Linear,
140 ColorChannelLayout::GrayScale,
141 )
142 .unwrap(),
143 output_image_properties: SegmentedImageFrameProperties::new(
144 SegmentedXYImageResolutions::create_with_same_sized_peripheral(
145 (1, 1).try_into().unwrap(),
146 (1, 1).try_into().unwrap(),
147 ),
148 ColorChannelLayout::GrayScale,
149 ColorChannelLayout::GrayScale,
150 ColorSpace::Linear,
151 ),
152 segmentation_gaze: GazeProperties::create_default_centered(),
153 }
154 }
155 FeedBackRegistration::SegmentedVisionWithImageFiltering {} => {
156 todo!()
157 }
158 FeedBackRegistration::VisionWithImageFiltering {} => {
159 PipelineStageProperties::ImageQuickDiff {
160 per_pixel_allowed_range: 0..=1,
161 acceptable_amount_of_activity_in_image: 0.0.try_into().unwrap()
162 ..=0.1.try_into().unwrap(),
163 image_properties: ImageFrameProperties::new(
164 (1, 1).try_into().unwrap(),
165 ColorSpace::Linear,
166 ColorChannelLayout::GrayScale,
167 )
168 .unwrap(),
169 }
170 }
171 }
172 }
173}
174
175fn feedback_segmented_vision_with_gaze(
176 target: &FeedbackRegistrationTargets,
177 sensors: Arc<Mutex<SensorDeviceCache>>,
178 motors: Arc<Mutex<MotorDeviceCache>>,
179 stage_index: PipelineStagePropertyIndex,
180) -> Result<FeagiSignalIndex, FeagiDataError> {
181 let sensor_unit = target.get_sensor_unit_index();
182 let sensor_channel = target.get_sensor_channel_index();
183
184 let sensor_ref = sensors.clone();
185
186 let closure = move |wrapped_data: &WrappedIOData| {
187 let gaze_properties: GazeProperties = wrapped_data.try_into().unwrap();
188
189 let mut sensors = sensor_ref.lock().unwrap();
190 let stage_properties = sensors
191 .segmented_vision_get_single_stage_properties(sensor_unit, sensor_channel, stage_index)
192 .unwrap();
193 let new_properties: PipelineStageProperties = match stage_properties {
194 PipelineStageProperties::ImageFrameSegmentator {
195 input_image_properties,
196 output_image_properties,
197 segmentation_gaze: _,
198 } => PipelineStageProperties::ImageFrameSegmentator {
199 input_image_properties,
200 output_image_properties,
201 segmentation_gaze: gaze_properties,
202 },
203 _ => {
204 panic!("Invalid pipeline stage properties for segmented gaze vision feedback!")
205 }
206 };
207
208 _ = sensors.segmented_vision_replace_single_stage(
209 sensor_unit,
210 sensor_channel,
211 0.into(),
212 new_properties,
213 );
214 };
215
216 let motor_ref = motors.clone();
217 let mut motors = motor_ref.lock().unwrap();
218
219 let index = motors.gaze_try_register_motor_callback(
220 target.get_motor_unit_index(),
221 target.get_motor_channel_index(),
222 closure,
223 )?;
224 Ok(index)
225}
226
227fn feedback_segmented_vision_with_image_filtering(
228 target: &FeedbackRegistrationTargets,
229 sensors: Arc<Mutex<SensorDeviceCache>>,
230 motors: Arc<Mutex<MotorDeviceCache>>,
231 stage_index: PipelineStagePropertyIndex,
232) -> Result<FeagiSignalIndex, FeagiDataError> {
233 let sensor_unit = target.get_sensor_unit_index();
234 let sensor_channel = target.get_sensor_channel_index();
235
236 let sensor_ref = sensors.clone();
237
238 let closure = move |wrapped_data: &WrappedIOData| {
239 let image_filtering_settings: ImageFilteringSettings = wrapped_data.try_into().unwrap();
240
241 let mut sensors = sensor_ref.lock().unwrap();
242 let stage_properties = sensors
243 .segmented_vision_get_single_stage_properties(sensor_unit, sensor_channel, stage_index)
244 .unwrap();
245 let new_properties: PipelineStageProperties = match stage_properties {
246 PipelineStageProperties::ImageQuickDiff {
247 image_properties, ..
248 } => {
249 let pixel_range = image_filtering_settings
250 .per_pixel_diff_threshold
251 .a
252 .get_as_u8()
253 ..=image_filtering_settings
254 .per_pixel_diff_threshold
255 .b
256 .get_as_u8();
257 let image_range = image_filtering_settings.image_diff_threshold.a
258 ..=image_filtering_settings.image_diff_threshold.b;
259
260 PipelineStageProperties::ImageQuickDiff {
261 per_pixel_allowed_range: pixel_range,
262 acceptable_amount_of_activity_in_image: image_range,
263 image_properties,
264 }
265 }
266 _ => {
267 panic!("Invalid pipeline stage properties for image transform!")
268 }
269 };
270
271 _ = sensors.segmented_vision_replace_single_stage(
272 sensor_unit,
273 sensor_channel,
274 0.into(),
275 new_properties,
276 );
277 };
278
279 let motor_ref = motors.clone();
280 let mut motors = motor_ref.lock().unwrap();
281
282 let index = motors.dynamic_image_processing_try_register_motor_callback(
283 target.get_motor_unit_index(),
284 target.get_motor_channel_index(),
285 closure,
286 )?;
287 Ok(index)
288}
289
290fn feedback_simple_vision_with_image_filtering(
291 target: &FeedbackRegistrationTargets,
292 sensors: Arc<Mutex<SensorDeviceCache>>,
293 motors: Arc<Mutex<MotorDeviceCache>>,
294 stage_index: PipelineStagePropertyIndex,
295) -> Result<FeagiSignalIndex, FeagiDataError> {
296 let sensor_unit = target.get_sensor_unit_index();
297 let sensor_channel = target.get_sensor_channel_index();
298
299 let sensor_ref = sensors.clone();
300
301 let closure = move |wrapped_data: &WrappedIOData| {
302 let image_filtering_settings: ImageFilteringSettings = wrapped_data.try_into().unwrap();
303
304 let mut sensors = sensor_ref.lock().unwrap();
305 let stage_properties = sensors
306 .vision_get_single_stage_properties(sensor_unit, sensor_channel, stage_index)
307 .unwrap();
308 let new_properties: PipelineStageProperties = match stage_properties {
309 PipelineStageProperties::ImageQuickDiff {
310 image_properties, ..
311 } => {
312 let pixel_range = image_filtering_settings
313 .per_pixel_diff_threshold
314 .a
315 .get_as_u8()
316 ..=image_filtering_settings
317 .per_pixel_diff_threshold
318 .b
319 .get_as_u8();
320 let image_range = image_filtering_settings.image_diff_threshold.a
321 ..=image_filtering_settings.image_diff_threshold.b;
322
323 PipelineStageProperties::ImageQuickDiff {
324 per_pixel_allowed_range: pixel_range,
325 acceptable_amount_of_activity_in_image: image_range,
326 image_properties,
327 }
328 }
329 _ => {
330 panic!("Invalid pipeline stage properties for image transform!")
331 }
332 };
333
334 _ = sensors.vision_replace_single_stage(
335 sensor_unit,
336 sensor_channel,
337 0.into(),
338 new_properties,
339 );
340 };
341
342 let motor_ref = motors.clone();
343 let mut motors = motor_ref.lock().unwrap();
344
345 let index = motors.dynamic_image_processing_try_register_motor_callback(
346 target.get_motor_unit_index(),
347 target.get_motor_channel_index(),
348 closure,
349 )?;
350 Ok(index)
351}