1#![allow(unused)]
16
17#[cfg(feature = "data_bincode")]
18extern crate bincode;
19
20#[cfg(feature = "data_cbor")]
21extern crate serde_cbor;
22
23#[cfg(feature = "data_json")]
24extern crate serde_json;
25
26use crate::model::record::DataFlowRecord;
27use crate::model::registry::RegistryNode;
28use crate::runtime::{RuntimeConfig, RuntimeInfo, RuntimeStatus};
29use crate::zfresult::ErrorKind;
30use crate::Result;
31use crate::{bail, zferror};
32use async_std::pin::Pin;
33use async_std::stream::Stream;
34use async_std::task::{Context, Poll};
35use futures::StreamExt;
36use futures_lite::FutureExt;
37use pin_project_lite::pin_project;
38use serde::{de::DeserializeOwned, Serialize};
39use std::convert::TryFrom;
40use std::sync::Arc;
41use uhlc::HLC;
42use uuid::Uuid;
43use zenoh::prelude::r#async::*;
44use zenoh::query::Reply;
45
46use super::Job;
47
48pub static ROOT_PLUGIN_RUNTIME_PREFIX: &str = "@/router/";
52pub static ROOT_PLUGIN_RUNTIME_SUFFIX: &str = "plugin/zenoh-flow";
54pub static ROOT_STANDALONE: &str = "zenoh-flow";
56
57pub static KEY_RUNTIMES: &str = "runtimes";
59pub static KEY_REGISTRY: &str = "registry";
61
62pub static KEY_FLOWS: &str = "flows";
64pub static KEY_GRAPHS: &str = "graphs";
66
67pub static KEY_INFO: &str = "info";
69pub static KEY_STATUS: &str = "status";
71pub static KEY_CONFIGURATION: &str = "configuration";
73
74pub static KEY_JOB_QUEUE: &str = "job-queue";
76
77pub static KEY_JOB_SUBMITTED: &str = "sumbitted";
79
80pub static KEY_JOB_STARTED: &str = "started";
82
83pub static KEY_JOB_DONE: &str = "done";
85
86pub static KEY_JOB_FAILED: &str = "failed";
88
89#[macro_export]
91macro_rules! RT_INFO_PATH {
92 ($prefix:expr, $rtid:expr) => {
93 format!(
94 "{}/{}/{}/{}",
95 $prefix,
96 $crate::runtime::resources::KEY_RUNTIMES,
97 $rtid,
98 $crate::runtime::resources::KEY_INFO
99 )
100 };
101}
102
103#[macro_export]
105macro_rules! RT_STATUS_PATH {
106 ($prefix:expr, $rtid:expr) => {
107 format!(
108 "{}/{}/{}/{}",
109 $prefix,
110 $crate::runtime::resources::KEY_RUNTIMES,
111 $rtid,
112 $crate::runtime::resources::KEY_STATUS
113 )
114 };
115}
116#[macro_export]
118macro_rules! RT_CONFIGURATION_PATH {
119 ($prefix:expr, $rtid:expr) => {
120 format!(
121 "{}/{}/{}/{}",
122 $prefix,
123 $crate::runtime::resources::KEY_RUNTIMES,
124 $rtid,
125 $crate::runtime::resources::KEY_CONFIGURATION
126 )
127 };
128}
129
130#[macro_export]
132macro_rules! RT_FLOW_PATH {
133 ($prefix:expr, $rtid:expr, $fid:expr, $iid:expr) => {
134 format!(
135 "{}/{}/{}/{}/{}/{}",
136 $prefix,
137 $crate::runtime::resources::KEY_RUNTIMES,
138 $rtid,
139 $crate::runtime::resources::KEY_FLOWS,
140 $fid,
141 $iid
142 )
143 };
144}
145
146#[macro_export]
148macro_rules! RT_FLOW_SELECTOR_BY_INSTANCE {
149 ($prefix:expr, $rtid:expr, $iid:expr) => {
150 format!(
151 "{}/{}/{}/{}/*/{}",
152 $prefix,
153 $crate::runtime::resources::KEY_RUNTIMES,
154 $rtid,
155 $crate::runtime::resources::KEY_FLOWS,
156 $iid
157 )
158 };
159}
160
161#[macro_export]
163macro_rules! RT_FLOW_SELECTOR_BY_FLOW {
164 ($prefix:expr, $rtid:expr, $fid:expr) => {
165 format!(
166 "{}/{}/{}/{}/{}/*",
167 $prefix,
168 $crate::runtime::resources::KEY_RUNTIMES,
169 $rtid,
170 $crate::runtime::resources::KEY_FLOWS,
171 $fid
172 )
173 };
174}
175
176#[macro_export]
178macro_rules! RT_FLOW_SELECTOR_ALL {
179 ($prefix:expr, $rtid:expr) => {
180 format!(
181 "{}/{}/{}/{}/*/*",
182 $prefix,
183 $crate::runtime::resources::KEY_RUNTIMES,
184 $rtid,
185 $crate::runtime::resources::KEY_FLOWS
186 )
187 };
188}
189
190#[macro_export]
192macro_rules! FLOW_SELECTOR_BY_INSTANCE {
193 ($prefix:expr, $iid:expr) => {
194 format!(
195 "{}/{}/*/{}/*/{}",
196 $prefix,
197 $crate::runtime::resources::KEY_RUNTIMES,
198 $crate::runtime::resources::KEY_FLOWS,
199 $iid
200 )
201 };
202}
203#[macro_export]
205macro_rules! FLOW_SELECTOR_BY_FLOW {
206 ($prefix:expr, $fid:expr) => {
207 format!(
208 "{}/{}/*/{}/{}/*",
209 $prefix,
210 $crate::runtime::resources::KEY_RUNTIMES,
211 $crate::runtime::resources::KEY_FLOWS,
212 $fid
213 )
214 };
215}
216
217#[macro_export]
219macro_rules! REG_GRAPH_SELECTOR {
220 ($prefix:expr, $fid:expr) => {
221 format!(
222 "{}/{}/{}/{}",
223 $prefix,
224 $crate::runtime::resources::KEY_REGISTRY,
225 $crate::runtime::resources::KEY_GRAPHS,
226 $fid
227 )
228 };
229}
230
231#[macro_export]
233macro_rules! JQ_SUMBITTED_SEL {
234 ($prefix:expr, $rid:expr) => {
235 format!(
236 "{}/{}/{}/{}/{}/*",
237 $prefix,
238 $crate::runtime::resources::KEY_RUNTIMES,
239 $rid,
240 $crate::runtime::resources::KEY_JOB_QUEUE,
241 $crate::runtime::resources::KEY_JOB_SUBMITTED
242 )
243 };
244}
245
246#[macro_export]
248macro_rules! JQ_SUMBITTED_JOB {
249 ($prefix:expr, $rid:expr, $jid: expr) => {
250 format!(
251 "{}/{}/{}/{}/{}/{}",
252 $prefix,
253 $crate::runtime::resources::KEY_RUNTIMES,
254 $rid,
255 $crate::runtime::resources::KEY_JOB_QUEUE,
256 $crate::runtime::resources::KEY_JOB_SUBMITTED,
257 $jid
258 )
259 };
260}
261
262#[macro_export]
264macro_rules! JQ_STARTED_JOB {
265 ($prefix:expr, $rid:expr, $jid: expr) => {
266 format!(
267 "{}/{}/{}/{}/{}/{}",
268 $prefix,
269 $crate::runtime::resources::KEY_RUNTIMES,
270 $rid,
271 $crate::runtime::resources::KEY_JOB_QUEUE,
272 $crate::runtime::resources::KEY_JOB_STARTED,
273 $jid
274 )
275 };
276}
277
278#[macro_export]
280macro_rules! JQ_DONE_JOB {
281 ($prefix:expr, $rid:expr, $jid: expr) => {
282 format!(
283 "{}/{}/{}/{}/{}/{}",
284 $prefix,
285 $crate::runtime::resources::KEY_RUNTIMES,
286 $rid,
287 $crate::runtime::resources::KEY_JOB_QUEUE,
288 $crate::runtime::resources::KEY_JOB_DONE,
289 $jid
290 )
291 };
292}
293
294#[macro_export]
296macro_rules! JQ_FAILED_JOB {
297 ($prefix:expr, $rid:expr, $jid: expr) => {
298 format!(
299 "{}/{}/{}/{}/{}/{}",
300 $prefix,
301 $crate::runtime::resources::KEY_RUNTIMES,
302 $rid,
303 $crate::runtime::resources::KEY_JOB_QUEUE,
304 $crate::runtime::resources::KEY_JOB_FAILED,
305 $jid
306 )
307 };
308}
309
310pub fn deserialize_data<T>(raw_data: &[u8]) -> Result<T>
318where
319 T: DeserializeOwned,
320{
321 #[cfg(feature = "data_bincode")]
322 return Ok(bincode::deserialize::<T>(&raw_data)?);
323
324 #[cfg(feature = "data_cbor")]
325 return Ok(serde_cbor::from_slice::<T>(&raw_data)?);
326
327 #[cfg(feature = "data_json")]
328 return Ok(serde_json::from_str::<T>(std::str::from_utf8(raw_data)?)?);
329}
330
331#[cfg(feature = "data_bincode")]
337
338pub fn serialize_data<T: ?Sized>(data: &T) -> FResult<Vec<u8>>
339where
340 T: Serialize,
341{
342 Ok(bincode::serialize(data)?)
343}
344
345#[cfg(feature = "data_json")]
351pub fn serialize_data<T: ?Sized>(data: &T) -> Result<Vec<u8>>
352where
353 T: Serialize,
354{
355 Ok(serde_json::to_string(data)?.into_bytes())
356}
357
358#[cfg(feature = "data_cbor")]
364pub fn serialize_data<T>(data: &T) -> FResult<Vec<u8>>
365where
366 T: Serialize,
367{
368 Ok(serde_cbor::to_vec(data)?)
369}
370pub fn convert<T>(sample: Sample) -> Result<T>
381where
382 T: DeserializeOwned,
383{
384 match sample.kind {
385 SampleKind::Put => match sample.value.encoding {
386 Encoding::APP_OCTET_STREAM => {
387 match deserialize_data::<T>(&sample.value.payload.contiguous()) {
388 Ok(data) => Ok(data),
389 Err(e) => Err(e),
390 }
391 }
392 _ => {
393 log::warn!(
394 "Received sample with wrong encoding {:?}, dropping",
395 sample.value.encoding
396 );
397 Err(zferror!(
398 ErrorKind::DeserializationError,
399 "Received sample with wrong encoding {:?}, dropping",
400 sample.value.encoding
401 )
402 .into())
403 }
404 },
405 SampleKind::Delete => {
406 log::warn!("Received delete sample drop it");
407 Err(zferror!(
408 ErrorKind::DeserializationError,
409 "Received delete sample dropping it"
410 )
411 .into())
412 }
413 }
414}
415
416#[derive(Clone)]
419pub struct DataStore {
420 z: Arc<zenoh::Session>,
422}
423
424impl DataStore {
425 pub fn new(z: Arc<zenoh::Session>) -> Self {
427 Self { z }
428 }
429
430 pub async fn get_runtime_info(&self, rtid: &ZenohId) -> Result<RuntimeInfo> {
437 let selector = RT_INFO_PATH!(ROOT_STANDALONE, rtid);
438
439 self.get_from_zenoh::<RuntimeInfo>(&selector).await
440 }
441
442 pub async fn get_all_runtime_info(&self) -> Result<Vec<RuntimeInfo>> {
450 let selector = RT_INFO_PATH!(ROOT_STANDALONE, "*");
451
452 self.get_vec_from_zenoh::<RuntimeInfo>(&selector).await
453 }
454
455 pub async fn get_runtime_info_by_name(&self, rtid: &str) -> Result<RuntimeInfo> {
463 let selector = RT_INFO_PATH!(ROOT_STANDALONE, "*");
464 let rts = self.get_vec_from_zenoh::<RuntimeInfo>(&selector).await?;
465 for rt in &rts {
466 if *rt.name == *rtid {
467 return Ok(rt.clone());
468 }
469 }
470 bail!(ErrorKind::NotFound)
471 }
472
473 pub async fn remove_runtime_info(&self, rtid: &ZenohId) -> Result<()> {
478 let path = RT_INFO_PATH!(ROOT_STANDALONE, rtid);
479
480 self.z.delete(&path).res().await
481 }
482
483 pub async fn add_runtime_info(&self, rtid: &ZenohId, rt_info: &RuntimeInfo) -> Result<()> {
491 let path = RT_INFO_PATH!(ROOT_STANDALONE, rtid);
492
493 let encoded_info = serialize_data(rt_info)?;
494 self.z.put(&path, encoded_info).res().await
495 }
496
497 pub async fn get_runtime_config(&self, rtid: &ZenohId) -> Result<RuntimeConfig> {
504 let selector = RT_CONFIGURATION_PATH!(ROOT_STANDALONE, rtid);
505 self.get_from_zenoh::<RuntimeConfig>(&selector).await
506 }
507
508 pub async fn subscribe_runtime_config(
516 &self,
517 rtid: &ZenohId,
518 ) -> Result<zenoh::subscriber::Subscriber<'static, flume::Receiver<Sample>>> {
519 bail!(ErrorKind::Unimplemented)
526 }
527
528 pub async fn remove_runtime_config(&self, rtid: &ZenohId) -> Result<()> {
533 let path = RT_CONFIGURATION_PATH!(ROOT_STANDALONE, rtid);
534
535 self.z.delete(&path).res().await
536 }
537
538 pub async fn add_runtime_config(&self, rtid: &ZenohId, rt_info: &RuntimeConfig) -> Result<()> {
546 let path = RT_CONFIGURATION_PATH!(ROOT_STANDALONE, rtid);
547
548 let encoded_info = serialize_data(rt_info)?;
549 self.z.put(&path, encoded_info).res().await
550 }
551
552 pub async fn get_runtime_status(&self, rtid: &ZenohId) -> Result<RuntimeStatus> {
559 let selector = RT_STATUS_PATH!(ROOT_STANDALONE, rtid);
560 self.get_from_zenoh::<RuntimeStatus>(&selector).await
561 }
562
563 pub async fn remove_runtime_status(&self, rtid: &ZenohId) -> Result<()> {
568 let path = RT_STATUS_PATH!(ROOT_STANDALONE, rtid);
569
570 self.z.delete(&path).res().await
571 }
572
573 pub async fn add_runtime_status(&self, rtid: &ZenohId, rt_info: &RuntimeStatus) -> Result<()> {
582 let path = RT_STATUS_PATH!(ROOT_STANDALONE, rtid);
583
584 let encoded_info = serialize_data(rt_info)?;
585 self.z.put(&path, encoded_info).res().await
586 }
587
588 pub async fn get_runtime_flow_by_instance(
596 &self,
597 rtid: &ZenohId,
598 iid: &Uuid,
599 ) -> Result<DataFlowRecord> {
600 let selector = RT_FLOW_SELECTOR_BY_INSTANCE!(ROOT_STANDALONE, rtid, iid);
601
602 self.get_from_zenoh::<DataFlowRecord>(&selector).await
603 }
604
605 pub async fn get_flow_by_instance(&self, iid: &Uuid) -> Result<DataFlowRecord> {
613 let selector = RT_FLOW_SELECTOR_BY_INSTANCE!(ROOT_STANDALONE, "*", iid);
614 self.get_from_zenoh::<DataFlowRecord>(&selector).await
615 }
616
617 pub async fn get_runtime_flow_instances(
626 &self,
627 rtid: &ZenohId,
628 fid: &str,
629 ) -> Result<Vec<DataFlowRecord>> {
630 let selector = RT_FLOW_SELECTOR_BY_FLOW!(ROOT_STANDALONE, rtid, fid);
631
632 self.get_vec_from_zenoh::<DataFlowRecord>(&selector).await
633 }
634
635 pub async fn get_flow_instances(&self, fid: &str) -> Result<Vec<DataFlowRecord>> {
644 let selector = FLOW_SELECTOR_BY_FLOW!(ROOT_STANDALONE, fid);
645 self.get_vec_from_zenoh::<DataFlowRecord>(&selector).await
646 }
647
648 pub async fn get_all_instances(&self) -> Result<Vec<DataFlowRecord>> {
651 let selector = FLOW_SELECTOR_BY_FLOW!(ROOT_STANDALONE, "*");
652 self.get_vec_from_zenoh::<DataFlowRecord>(&selector).await
653 }
654
655 pub async fn get_flow_instance_runtimes(&self, iid: &Uuid) -> Result<Vec<ZenohId>> {
657 let selector = RT_FLOW_SELECTOR_BY_INSTANCE!(ROOT_STANDALONE, "*", iid);
658
659 let mut ds = self.z.get(&selector).res().await?;
660
661 let mut runtimes = Vec::new();
662
663 for kv in ds.into_iter() {
664 if let Ok(sample) = &kv.sample {
665 let id = sample
666 .key_expr
667 .as_str()
668 .split('/')
669 .nth(2) .ok_or_else(|| {
671 log::error!(
672 "Could not extract the instance id from key expression: {}",
673 sample.key_expr.as_str()
674 );
675 zferror!(ErrorKind::DeserializationError)
676 })?;
677 runtimes.push(id.parse::<ZenohId>()?);
678 }
679 }
680
681 Ok(runtimes)
682 }
683
684 pub async fn remove_runtime_flow_instance(
690 &self,
691 rtid: &ZenohId,
692 fid: &str,
693 iid: &Uuid,
694 ) -> Result<()> {
695 let path = RT_FLOW_PATH!(ROOT_STANDALONE, rtid, fid, iid);
696
697 self.z.delete(&path).res().await
698 }
699
700 pub async fn add_runtime_flow(
708 &self,
709 rtid: &ZenohId,
710 flow_instance: &DataFlowRecord,
711 ) -> Result<()> {
712 let path = RT_FLOW_PATH!(
713 ROOT_STANDALONE,
714 rtid,
715 flow_instance.flow,
716 flow_instance.uuid
717 );
718
719 let encoded_info = serialize_data(flow_instance)?;
720 self.z.put(&path, encoded_info).res().await
721 }
722
723 pub async fn add_graph(&self, graph: &RegistryNode) -> Result<()> {
733 let path = REG_GRAPH_SELECTOR!(ROOT_STANDALONE, &graph.id);
734
735 let encoded_info = serialize_data(graph)?;
736 self.z.put(&path, encoded_info).res().await
737 }
738
739 pub async fn get_graph(&self, graph_id: &str) -> Result<RegistryNode> {
747 let selector = REG_GRAPH_SELECTOR!(ROOT_STANDALONE, graph_id);
748 self.get_from_zenoh::<RegistryNode>(&selector).await
749 }
750
751 pub async fn get_all_graphs(&self) -> Result<Vec<RegistryNode>> {
760 let selector = REG_GRAPH_SELECTOR!(ROOT_STANDALONE, "*");
761 self.get_vec_from_zenoh::<RegistryNode>(&selector).await
762 }
763
764 pub async fn delete_graph(&self, graph_id: &str) -> Result<()> {
766 let path = REG_GRAPH_SELECTOR!(ROOT_STANDALONE, &graph_id);
767
768 self.z.delete(&path).res().await
769 }
770
771 pub async fn subscribe_sumbitted_jobs(
780 &self,
781 rtid: &ZenohId,
782 ) -> Result<zenoh::subscriber::Subscriber<'static, flume::Receiver<Sample>>> {
783 let selector = JQ_SUMBITTED_SEL!(ROOT_STANDALONE, rtid);
784 self.z.declare_subscriber(&selector).res().await
785 }
786
787 pub async fn add_submitted_job(&self, rtid: &ZenohId, job: &Job) -> Result<()> {
794 let path = JQ_SUMBITTED_JOB!(ROOT_STANDALONE, rtid, &job.id);
795 let encoded_info = serialize_data(job)?;
796 self.z.put(&path, encoded_info).res().await
797 }
798
799 pub async fn del_submitted_job(&self, rtid: &ZenohId, id: &Uuid) -> Result<()> {
800 let path = JQ_SUMBITTED_JOB!(ROOT_STANDALONE, rtid, id);
801 self.z.delete(&path).res().await
802 }
803
804 pub async fn add_started_job(&self, rtid: &ZenohId, job: &Job) -> Result<()> {
811 let path = JQ_STARTED_JOB!(ROOT_STANDALONE, rtid, &job.id);
812 let encoded_info = serialize_data(job)?;
813 self.z.put(&path, encoded_info).res().await
814 }
815
816 pub async fn add_done_job(&self, rtid: &ZenohId, job: &Job) -> Result<()> {
823 let path = JQ_DONE_JOB!(ROOT_STANDALONE, rtid, &job.id);
824 let encoded_info = serialize_data(job)?;
825 self.z.put(&path, encoded_info).res().await
826 }
827
828 pub async fn add_failed_job(&self, rtid: &ZenohId, job: &Job) -> Result<()> {
835 let path = JQ_FAILED_JOB!(ROOT_STANDALONE, rtid, &job.id);
836 let encoded_info = serialize_data(job)?;
837 self.z.put(&path, encoded_info).res().await
838 }
839
840 async fn get_from_zenoh<T>(&self, path: &str) -> Result<T>
851 where
852 T: DeserializeOwned,
853 {
854 let mut ds = self.z.get(path).res().await?;
855 let data = ds.into_iter().collect::<Vec<Reply>>();
856 match data.len() {
857 0 => Err(zferror!(ErrorKind::Empty).into()),
858 _ => {
859 let kv = &data[0];
860 match &kv.sample {
861 Ok(sample) => match &sample.value.encoding {
862 &Encoding::APP_OCTET_STREAM => {
863 let ni = deserialize_data::<T>(&sample.value.payload.contiguous())?;
864 Ok(ni)
865 }
866 _ => Err(zferror!(ErrorKind::DeserializationError).into()),
867 },
868 _ => Err(zferror!(ErrorKind::DeserializationError).into()),
869 }
870 }
871 }
872 }
873
874 async fn get_vec_from_zenoh<T>(&self, selector: &str) -> Result<Vec<T>>
882 where
883 T: DeserializeOwned,
884 {
885 let mut ds = self.z.get(selector).res().await?;
886
887 let mut zf_data: Vec<T> = Vec::new();
888
889 for kv in ds.into_iter() {
890 match &kv.sample {
891 Ok(sample) => match &sample.value.encoding {
892 &Encoding::APP_OCTET_STREAM => {
893 let ni = deserialize_data::<T>(&sample.value.payload.contiguous())?;
894 zf_data.push(ni);
895 }
896 _ => return Err(zferror!(ErrorKind::DeserializationError).into()),
897 },
898 _ => return Err(zferror!(ErrorKind::DeserializationError).into()),
899 }
900 }
901 Ok(zf_data)
902 }
903}