ados_protocol/
vision_rpc.rs

1//! The `args` of the plugin-facing vision requests the engine serves on
2//! `vision.sock`, and the reply `vision.infer` returns.
3//!
4//! A plugin's vision request crosses three parties: the SDK builds it, the
5//! plugin host forwards it to the engine, and the engine decodes it. Each shape
6//! lives here once, so the Rust SDK and the engine build and read the same
7//! bytes; the Python SDK mirrors them field for field.
8//!
9//! The host stamps the calling plugin's id into `vision.register_model` and
10//! `vision.infer` as a string [`OWNER_FIELD`] before forwarding, so the engine
11//! keeps each plugin's models apart. A plugin never sets it: the host replaces
12//! whatever the request carried. When the host stops serving a plugin it sends
13//! [`UNREGISTER_OWNER`] `{owner}` so the engine drops that plugin's models.
14//!
15//! A contract payload rides as a msgpack binary field holding its
16//! [`crate::framebus`] encoding — the form the engine's own pushes already use
17//! (`vision.deliver {descriptor}`, `vision.deliver_detection {batch}`) — so it
18//! crosses byte for byte and its version is checked where it is decoded:
19//!
20//! | method | args |
21//! |---|---|
22//! | `vision.publish_detection` | `{batch: bin(DetectionBatch)}` |
23//! | `vision.register_model` | `{model: bin(ModelMetadata)}` |
24//! | `vision.infer` | `{model_id: str, descriptor: bin(FrameDescriptor)}`; reply `{batch: bin(DetectionBatch)}` |
25//! | `vision.designate_track` | `{camera_id: str, bbox: {x, y, width, height}, class_label?: str, confidence?: num}` |
26//!
27//! `vision.designate_track` is a flat map instead, because the control plane's
28//! click-to-follow route sends the same request built from the operator's JSON
29//! pick.
30
31use rmpv::Value;
32use thiserror::Error;
33
34use crate::framebus::{BoundingBox, Detection, DetectionBatch, FrameDescriptor, ModelMetadata};
35
36/// The group that may read the camera frame rings in `/dev/shm`. The engine
37/// creates each ring `0640` in this group; a plugin unit joins it only while
38/// its `vision.frame.read` grant is held.
39pub const VISION_READERS_GROUP: &str = "ados-vision-readers";
40
41/// The args field carrying the plugin id the host stamped on a request.
42pub const OWNER_FIELD: &str = "owner";
43
44/// Host-to-engine request dropping every model one plugin registered. Not a
45/// plugin-facing method: no capability maps to it and the host never routes a
46/// plugin's request to it.
47pub const UNREGISTER_OWNER: &str = "vision.unregister_owner";
48
49/// `args` with [`OWNER_FIELD`] set to `owner`, replacing any value the caller
50/// put there. Non-map args come back as `{owner}` alone, which the engine then
51/// refuses for the missing payload.
52pub fn with_owner(args: &Value, owner: &str) -> Value {
53    let mut entries: Vec<(Value, Value)> = match args {
54        Value::Map(entries) => entries
55            .iter()
56            .filter(|(k, _)| k.as_str() != Some(OWNER_FIELD))
57            .cloned()
58            .collect(),
59        _ => Vec::new(),
60    };
61    entries.push((Value::from(OWNER_FIELD), Value::from(owner)));
62    Value::Map(entries)
63}
64
65/// The non-empty [`OWNER_FIELD`] a request carries.
66pub fn owner_of(args: &Value) -> Result<&str, VisionArgsError> {
67    field(args, OWNER_FIELD)?
68        .as_str()
69        .filter(|s| !s.is_empty())
70        .ok_or_else(|| decode_err(OWNER_FIELD, "not a non-empty string"))
71}
72
73/// [`UNREGISTER_OWNER`] args: `{owner}`.
74pub fn unregister_owner_args(owner: &str) -> Value {
75    map(vec![(OWNER_FIELD, Value::from(owner))])
76}
77
78/// The id a plugin's model or published batch carries on the engine: the
79/// plugin's own id under its namespace (`<plugin_id>/<id>`), so a plugin can
80/// neither replace nor pose as the engine's configured models or another
81/// plugin's. An id the plugin already namespaced is kept as is.
82pub fn owned_model_id(owner: &str, id: &str) -> String {
83    match id
84        .strip_prefix(owner)
85        .and_then(|rest| rest.strip_prefix('/'))
86    {
87        Some(_) => id.to_string(),
88        None => format!("{owner}/{id}"),
89    }
90}
91
92/// A vision request or reply that does not have its method's shape.
93#[derive(Debug, Error)]
94pub enum VisionArgsError {
95    #[error("vision args are not a map")]
96    NotAMap,
97    #[error("missing `{0}`")]
98    Missing(&'static str),
99    #[error("`{field}` does not decode: {reason}")]
100    Decode { field: &'static str, reason: String },
101    #[error("`{field}` does not encode: {reason}")]
102    Encode { field: &'static str, reason: String },
103}
104
105/// `vision.publish_detection` args: `{batch}`.
106pub fn publish_detection_args(batch: &DetectionBatch) -> Result<Value, VisionArgsError> {
107    Ok(map(vec![("batch", blob("batch", batch.to_msgpack())?)]))
108}
109
110/// Decode `vision.publish_detection` args, rejecting a batch whose version
111/// this build does not speak.
112pub fn decode_publish_detection(args: &Value) -> Result<DetectionBatch, VisionArgsError> {
113    DetectionBatch::from_msgpack(binary(args, "batch")?).map_err(|e| decode_err("batch", e))
114}
115
116/// `vision.register_model` args: `{model}`.
117pub fn register_model_args(model: &ModelMetadata) -> Result<Value, VisionArgsError> {
118    Ok(map(vec![("model", blob("model", model.to_msgpack())?)]))
119}
120
121/// Decode `vision.register_model` args.
122pub fn decode_register_model(args: &Value) -> Result<ModelMetadata, VisionArgsError> {
123    ModelMetadata::from_msgpack(binary(args, "model")?).map_err(|e| decode_err("model", e))
124}
125
126/// A decoded `vision.infer` request: the model to run and the frame to run it
127/// on, named by the descriptor the engine published for it.
128#[derive(Debug, Clone, PartialEq)]
129pub struct InferRequest {
130    pub model_id: String,
131    pub frame: FrameDescriptor,
132}
133
134/// `vision.infer` args: `{model_id, descriptor}`. The pixels stay in the ring
135/// the descriptor names.
136pub fn infer_args(model_id: &str, frame: &FrameDescriptor) -> Result<Value, VisionArgsError> {
137    Ok(map(vec![
138        ("model_id", Value::from(model_id)),
139        ("descriptor", blob("descriptor", frame.to_msgpack())?),
140    ]))
141}
142
143/// Decode `vision.infer` args, rejecting a descriptor whose version this build
144/// does not speak.
145pub fn decode_infer(args: &Value) -> Result<InferRequest, VisionArgsError> {
146    let model_id = field(args, "model_id")?
147        .as_str()
148        .ok_or_else(|| decode_err("model_id", "not a string"))?
149        .to_string();
150    let frame = FrameDescriptor::from_msgpack(binary(args, "descriptor")?)
151        .map_err(|e| decode_err("descriptor", e))?;
152    Ok(InferRequest { model_id, frame })
153}
154
155/// The `vision.infer` reply: `{batch}`, the model's detections on that frame.
156pub fn infer_reply(batch: &DetectionBatch) -> Result<Value, VisionArgsError> {
157    publish_detection_args(batch)
158}
159
160/// Decode the `vision.infer` reply.
161pub fn decode_infer_reply(args: &Value) -> Result<DetectionBatch, VisionArgsError> {
162    decode_publish_detection(args)
163}
164
165/// A decoded `vision.designate_track` request: the camera whose tracker locks,
166/// and the box it locks onto.
167#[derive(Debug, Clone, PartialEq)]
168pub struct DesignateTrack {
169    pub camera_id: String,
170    pub target: Detection,
171}
172
173/// `vision.designate_track` args from the detection to lock onto. Only its box,
174/// label and confidence cross; the box is required.
175pub fn designate_track_args(camera_id: &str, target: &Detection) -> Result<Value, VisionArgsError> {
176    let bbox = target.bbox.ok_or(VisionArgsError::Missing("bbox"))?;
177    Ok(map(vec![
178        ("camera_id", Value::from(camera_id)),
179        (
180            "bbox",
181            map(vec![
182                ("x", Value::from(f64::from(bbox.x))),
183                ("y", Value::from(f64::from(bbox.y))),
184                ("width", Value::from(f64::from(bbox.width))),
185                ("height", Value::from(f64::from(bbox.height))),
186            ]),
187        ),
188        ("class_label", Value::from(target.class_label.as_str())),
189        ("confidence", Value::from(f64::from(target.confidence))),
190    ]))
191}
192
193/// Decode `vision.designate_track` args. Box fields read with numeric coercion,
194/// so an int- or float-encoded value decodes the same. Every box field is
195/// required and must be finite, `x`/`y` non-negative and `width`/`height`
196/// positive: a NaN or zero-size box would seed a tracker that never associates
197/// or predicts NaN. `class_label` defaults to empty and `confidence` to full,
198/// clamped to `0..=1`: the operator's pick overrides the auto-lock regardless.
199pub fn decode_designate_track(args: &Value) -> Result<DesignateTrack, VisionArgsError> {
200    let camera_id = field(args, "camera_id")?
201        .as_str()
202        .ok_or_else(|| decode_err("camera_id", "not a string"))?
203        .to_string();
204    let bbox = field(args, "bbox")?;
205    if !bbox.is_map() {
206        return Err(decode_err("bbox", "not a map"));
207    }
208    let side = |k: &'static str| -> Result<f32, VisionArgsError> {
209        let v = field(bbox, k)
210            .ok()
211            .and_then(number)
212            .ok_or_else(|| decode_err("bbox", format!("`{k}` is missing or not a number")))?;
213        if v.is_finite() {
214            Ok(v)
215        } else {
216            Err(decode_err("bbox", format!("`{k}` is not finite")))
217        }
218    };
219    let (x, y, width, height) = (side("x")?, side("y")?, side("width")?, side("height")?);
220    if x < 0.0 || y < 0.0 {
221        return Err(decode_err("bbox", "x and y must be non-negative"));
222    }
223    if width <= 0.0 || height <= 0.0 {
224        return Err(decode_err("bbox", "width and height must be positive"));
225    }
226    let confidence = field(args, "confidence")
227        .ok()
228        .and_then(number)
229        .filter(|c| c.is_finite())
230        .unwrap_or(1.0)
231        .clamp(0.0, 1.0);
232    let target = Detection {
233        bbox: Some(BoundingBox {
234            x,
235            y,
236            width,
237            height,
238        }),
239        class_label: field(args, "class_label")
240            .ok()
241            .and_then(Value::as_str)
242            .unwrap_or_default()
243            .to_string(),
244        confidence,
245        track_id: None,
246        assoc_confidence: None,
247        lock_state: None,
248        attributes: None,
249        mask: None,
250        keypoints: None,
251        depth: None,
252        world_pos: None,
253    };
254    Ok(DesignateTrack { camera_id, target })
255}
256
257fn map(pairs: Vec<(&str, Value)>) -> Value {
258    Value::Map(
259        pairs
260            .into_iter()
261            .map(|(k, v)| (Value::from(k), v))
262            .collect(),
263    )
264}
265
266fn blob(
267    field: &'static str,
268    encoded: Result<Vec<u8>, rmp_serde::encode::Error>,
269) -> Result<Value, VisionArgsError> {
270    encoded
271        .map(Value::Binary)
272        .map_err(|e| VisionArgsError::Encode {
273            field,
274            reason: e.to_string(),
275        })
276}
277
278fn field<'a>(args: &'a Value, key: &'static str) -> Result<&'a Value, VisionArgsError> {
279    let Value::Map(entries) = args else {
280        return Err(VisionArgsError::NotAMap);
281    };
282    entries
283        .iter()
284        .find(|(k, _)| k.as_str() == Some(key))
285        .map(|(_, v)| v)
286        .ok_or(VisionArgsError::Missing(key))
287}
288
289fn binary<'a>(args: &'a Value, key: &'static str) -> Result<&'a [u8], VisionArgsError> {
290    match field(args, key)? {
291        Value::Binary(bytes) => Ok(bytes),
292        _ => Err(decode_err(key, "not binary")),
293    }
294}
295
296fn decode_err(field: &'static str, reason: impl ToString) -> VisionArgsError {
297    VisionArgsError::Decode {
298        field,
299        reason: reason.to_string(),
300    }
301}
302
303/// Any msgpack number as f32.
304fn number(v: &Value) -> Option<f32> {
305    v.as_f64()
306        .or_else(|| v.as_i64().map(|i| i as f64))
307        .or_else(|| v.as_u64().map(|u| u as f64))
308        .map(|f| f as f32)
309}
310
311#[cfg(test)]
312mod tests {
313    use super::*;
314    use crate::framebus::VISION_DETECTION_VERSION;
315
316    #[test]
317    fn a_batch_of_an_unknown_version_is_refused_at_decode() {
318        let batch = DetectionBatch {
319            v: VISION_DETECTION_VERSION + 1,
320            model_id: "m".into(),
321            camera_id: "c".into(),
322            frame_id: 1,
323            ts_ms: 0,
324            frame_width: 0,
325            frame_height: 0,
326            detections: vec![],
327        };
328        let args = publish_detection_args(&batch).unwrap();
329        assert!(matches!(
330            decode_publish_detection(&args),
331            Err(VisionArgsError::Decode { field: "batch", .. })
332        ));
333    }
334
335    #[test]
336    fn designate_reads_integer_boxes_and_defaults_label_and_confidence() {
337        // The control route builds this map from the operator's JSON pick,
338        // where a whole-pixel box arrives as integers.
339        let args = map(vec![
340            ("camera_id", Value::from("cam-0")),
341            (
342                "bbox",
343                map(vec![
344                    ("x", Value::from(10)),
345                    ("y", Value::from(20u64)),
346                    ("width", Value::from(30.5)),
347                    ("height", Value::from(40)),
348                ]),
349            ),
350        ]);
351        let d = decode_designate_track(&args).unwrap();
352        assert_eq!(d.camera_id, "cam-0");
353        assert_eq!(
354            d.target.bbox,
355            Some(BoundingBox {
356                x: 10.0,
357                y: 20.0,
358                width: 30.5,
359                height: 40.0
360            })
361        );
362        assert_eq!(d.target.class_label, "");
363        assert_eq!(d.target.confidence, 1.0);
364    }
365
366    #[test]
367    fn designate_without_a_box_is_refused_on_both_ends() {
368        let args = map(vec![("camera_id", Value::from("cam-0"))]);
369        assert!(matches!(
370            decode_designate_track(&args),
371            Err(VisionArgsError::Missing("bbox"))
372        ));
373        let boxless = Detection {
374            bbox: None,
375            class_label: "person".into(),
376            confidence: 0.9,
377            track_id: None,
378            assoc_confidence: None,
379            lock_state: None,
380            attributes: None,
381            mask: None,
382            keypoints: None,
383            depth: None,
384            world_pos: None,
385        };
386        assert!(matches!(
387            designate_track_args("cam-0", &boxless),
388            Err(VisionArgsError::Missing("bbox"))
389        ));
390    }
391
392    #[test]
393    fn designate_refuses_non_finite_and_empty_boxes_and_clamps_confidence() {
394        let with = |x: f64, w: f64, conf: f64| {
395            map(vec![
396                ("camera_id", Value::from("cam-0")),
397                (
398                    "bbox",
399                    map(vec![
400                        ("x", Value::from(x)),
401                        ("y", Value::from(1.0)),
402                        ("width", Value::from(w)),
403                        ("height", Value::from(5.0)),
404                    ]),
405                ),
406                ("confidence", Value::from(conf)),
407            ])
408        };
409        for bad in [
410            with(f64::NAN, 5.0, 0.5),
411            with(f64::INFINITY, 5.0, 0.5),
412            with(-1.0, 5.0, 0.5),
413            with(1.0, 0.0, 0.5),
414            with(1.0, -3.0, 0.5),
415        ] {
416            assert!(matches!(
417                decode_designate_track(&bad),
418                Err(VisionArgsError::Decode { field: "bbox", .. })
419            ));
420        }
421        let d = decode_designate_track(&with(1.0, 5.0, 7.0)).unwrap();
422        assert_eq!(d.target.confidence, 1.0);
423        let d = decode_designate_track(&with(1.0, 5.0, -2.0)).unwrap();
424        assert_eq!(d.target.confidence, 0.0);
425    }
426
427    #[test]
428    fn the_stamped_owner_replaces_whatever_the_plugin_sent() {
429        let spoofed = map(vec![
430            ("model_id", Value::from("det")),
431            (OWNER_FIELD, Value::from("com.example.victim")),
432        ]);
433        let stamped = with_owner(&spoofed, "com.example.caller");
434        assert_eq!(owner_of(&stamped).unwrap(), "com.example.caller");
435        let Value::Map(entries) = &stamped else {
436            panic!("not a map");
437        };
438        assert_eq!(
439            entries
440                .iter()
441                .filter(|(k, _)| k.as_str() == Some(OWNER_FIELD))
442                .count(),
443            1
444        );
445        assert_eq!(field(&stamped, "model_id").unwrap().as_str(), Some("det"));
446        assert!(owner_of(&map(vec![(OWNER_FIELD, Value::from(""))])).is_err());
447    }
448}
ados_protocol/vision_rpc.rs

ados_protocol/
vision_rpc.rs
