Skip to main content

weirdnet_plugin/
connector.rs

1//! The ingest-connector host methods (a code space of kind `ingest-connector`;
2//! see `docs/connectors.md`). A connector gathers data, stores it raw
3//! ([`Host::store_raw`]), pipes it into its model ([`Host::pipe`]) and runs its rule set
4//! ([`Host::run_ruleset`]). Only main calls these, and only in a connector run: the
5//! host's scheduler or a dry run, which records the calls instead of storing, piping or
6//! running rules.
7use crate::{Error, Host, Json, Result};
8
9/// What this run is for (`host.context()`).
10#[derive(Debug, Clone, Default, PartialEq)]
11pub struct Context {
12    pub schedule_id: Option<String>,
13    pub run_id: Option<String>,
14    pub ruleset_id: Option<String>,
15    /// The source raw items go into (its name).
16    pub source: Option<String>,
17    /// The pipeline [`Host::pipe`] runs (its name).
18    pub pipeline: Option<String>,
19    /// Epoch ms (UTC) of the schedule's last successful run.
20    pub last_success_at: Option<i64>,
21    /// The schedule's options, as text.
22    pub options: Vec<(String, String)>,
23    /// True in `--connector-dry-run`: nothing is stored, piped or run.
24    pub dry_run: bool,
25}
26
27/// What [`Host::pipe`] did.
28#[derive(Debug, Clone, Default, PartialEq)]
29pub struct Piped {
30    pub inserted: usize,
31    /// Each model sink: `{"model", "inserted"}`.
32    pub sinks: Vec<Json>,
33}
34
35/// What [`Host::run_ruleset`] did.
36#[derive(Debug, Clone, Default, PartialEq)]
37pub struct RulesetRun {
38    pub rules_run: usize,
39    pub errors: Vec<String>,
40}
41
42/// An HTTP request for [`Host::fetch`].
43#[derive(Debug, Clone, PartialEq)]
44pub struct Request {
45    pub method: String,
46    pub url: String,
47    pub headers: Vec<(String, String)>,
48    pub body: Option<Vec<u8>>,
49}
50
51impl Request {
52    pub fn get(url: &str) -> Request {
53        Request { method: "GET".into(), url: url.into(), headers: vec![], body: None }
54    }
55    pub fn post(url: &str, body: impl Into<Vec<u8>>) -> Request {
56        Request { method: "POST".into(), url: url.into(), headers: vec![], body: Some(body.into()) }
57    }
58    pub fn header(mut self, k: &str, v: &str) -> Request {
59        self.headers.push((k.into(), v.into()));
60        self
61    }
62    fn to_json(&self) -> Json {
63        let h = Json::Obj(self.headers.iter().map(|(k, v)| (k.clone(), Json::from(v.as_str()))).collect());
64        let mut j = Json::obj().with("method", self.method.as_str()).with("url", self.url.as_str()).with("headers", h);
65        if let Some(b) = &self.body {
66            j = j.with("body_base64", b64_encode(b));
67        }
68        j
69    }
70}
71
72/// A [`Host::fetch`] reply.
73#[derive(Debug, Clone, PartialEq)]
74pub struct Response {
75    pub status: u16,
76    pub headers: Vec<(String, String)>,
77    pub body: Vec<u8>,
78}
79
80impl Response {
81    pub fn text(&self) -> String {
82        String::from_utf8_lossy(&self.body).into_owned()
83    }
84    pub fn json(&self) -> Result<Json> {
85        Json::parse(&self.text()).map_err(|e| Error(format!("response is not JSON: {e}")))
86    }
87    pub fn header(&self, name: &str) -> Option<&str> {
88        self.headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str())
89    }
90}
91
92fn s(j: &Json, k: &str) -> Option<String> {
93    j.get(k).and_then(|v| v.as_str().map(str::to_string))
94}
95
96fn n(j: &Json, k: &str) -> usize {
97    j.get(k).and_then(Json::as_f64).unwrap_or(0.0) as usize
98}
99
100impl Host {
101    /// The run's context: schedule, run, rule set, source, pipeline, last success, options.
102    pub fn context(&self) -> Result<Context> {
103        let j = self.call("context", &Json::Null)?;
104        Ok(Context {
105            schedule_id: s(&j, "schedule_id"),
106            run_id: s(&j, "run_id"),
107            ruleset_id: s(&j, "ruleset_id"),
108            source: s(&j, "source"),
109            pipeline: s(&j, "pipeline"),
110            last_success_at: j.get("last_success_at").and_then(Json::as_i64),
111            options: j
112                .get("options")
113                .and_then(Json::as_object)
114                .unwrap_or(&[])
115                .iter()
116                .map(|(k, v)| (k.clone(), v.as_str().map(str::to_string).unwrap_or_else(|| v.to_string())))
117                .collect(),
118            dry_run: j.get("dry_run").and_then(Json::as_bool).unwrap_or(false),
119        })
120    }
121
122    /// The context as the host sent it.
123    pub fn context_json(&self) -> Result<Json> {
124        self.call("context", &Json::Null)
125    }
126
127    /// Store one raw item (kept byte for byte, never parsed) in the run's source; returns
128    /// its id. `format` is a hint for the pipeline: `csv`, `json`, `ndjson`, `text`, `bytes`.
129    pub fn store_raw(&self, name: &str, format: &str, bytes: &[u8]) -> Result<String> {
130        let r = self.call("store_raw", &Json::obj().with("name", name).with("format", format).with("bytes_base64", b64_encode(bytes)))?;
131        Ok(s(&r, "id").unwrap_or_default())
132    }
133
134    /// Run the schedule's pipeline over this run's raw items.
135    pub fn pipe(&self) -> Result<Piped> {
136        let r = self.call("pipe", &Json::Null)?;
137        Ok(Piped { inserted: n(&r, "inserted"), sinks: r.get("sinks").and_then(Json::as_array).unwrap_or(&[]).to_vec() })
138    }
139
140    /// Record `connector.run` and run a rule set (`None`: the schedule's).
141    pub fn run_ruleset(&self, ruleset_id: Option<&str>) -> Result<RulesetRun> {
142        let req = match ruleset_id {
143            Some(id) => Json::obj().with("ruleset_id", id),
144            None => Json::obj(),
145        };
146        let r = self.call("run_ruleset", &req)?;
147        Ok(RulesetRun {
148            rules_run: n(&r, "rules_run"),
149            errors: r
150                .get("errors")
151                .and_then(Json::as_array)
152                .unwrap_or(&[])
153                .iter()
154                .map(|e| e.as_str().map(str::to_string).unwrap_or_else(|| e.to_string()))
155                .collect(),
156        })
157    }
158
159    /// HTTP through the host's client (its user agent, timeouts and retries).
160    pub fn fetch(&self, req: &Request) -> Result<Response> {
161        let r = self.call("fetch", &req.to_json())?;
162        let body = match r.get("body_base64").and_then(Json::as_str) {
163            Some(b) => b64_decode(b)?,
164            None => Vec::new(),
165        };
166        Ok(Response {
167            status: r.get("status").and_then(Json::as_f64).unwrap_or(0.0) as u16,
168            headers: r
169                .get("headers")
170                .and_then(Json::as_object)
171                .unwrap_or(&[])
172                .iter()
173                .map(|(k, v)| (k.clone(), v.as_str().map(str::to_string).unwrap_or_default()))
174                .collect(),
175            body,
176        })
177    }
178
179    /// A value of the schedule's small state (a cursor such as the last timestamp).
180    pub fn state_get(&self, key: &str) -> Result<Option<Json>> {
181        let r = self.call("state_get", &Json::obj().with("key", key))?;
182        Ok(r.get("value").filter(|v| !v.is_null()).cloned())
183    }
184
185    /// Set (or with `Json::Null`, remove) a state value; kept only when the run succeeds.
186    pub fn state_set(&self, key: &str, value: impl Into<Json>) -> Result<()> {
187        self.call("state_set", &Json::obj().with("key", key).with("value", value.into())).map(|_| ())
188    }
189}
190
191const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
192
193/// Standard base64 with padding (how bytes cross the JSON ABI).
194pub fn b64_encode(b: &[u8]) -> String {
195    let mut out = String::with_capacity(b.len().div_ceil(3) * 4);
196    for c in b.chunks(3) {
197        let v = (c[0] as u32) << 16 | (*c.get(1).unwrap_or(&0) as u32) << 8 | *c.get(2).unwrap_or(&0) as u32;
198        for i in 0..4 {
199            if i <= c.len() {
200                out.push(B64[(v >> (18 - 6 * i) & 63) as usize] as char);
201            } else {
202                out.push('=');
203            }
204        }
205    }
206    out
207}
208
209/// Decodes [`b64_encode`]'s output (whitespace ignored).
210pub fn b64_decode(s: &str) -> Result<Vec<u8>> {
211    let mut out = Vec::with_capacity(s.len() / 4 * 3);
212    let (mut acc, mut bits) = (0u32, 0);
213    for c in s.bytes().filter(|c| !c.is_ascii_whitespace() && *c != b'=') {
214        let v = B64.iter().position(|&x| x == c).ok_or_else(|| Error(format!("bad base64 byte `{}`", c as char)))? as u32;
215        acc = acc << 6 | v;
216        bits += 6;
217        if bits >= 8 {
218            bits -= 8;
219            out.push((acc >> bits & 0xff) as u8);
220        }
221    }
222    Ok(out)
223}
224
225#[cfg(test)]
226mod tests {
227    use super::*;
228    #[test]
229    fn base64_round_trips() {
230        for s in ["", "f", "fo", "foo", "foob", "fooba", "foobar"] {
231            assert_eq!(b64_decode(&b64_encode(s.as_bytes())).unwrap(), s.as_bytes());
232        }
233        assert_eq!(b64_encode(b"foobar"), "Zm9vYmFy");
234        assert_eq!(b64_encode(b"fo"), "Zm8=");
235        let all: Vec<u8> = (0..=255).collect();
236        assert_eq!(b64_decode(&b64_encode(&all)).unwrap(), all);
237    }
238}
239
240#[cfg(test)]
241mod macro_tests {
242    use crate::prelude::*;
243    fn run(_: &Host) -> Result<View> {
244        Ok(View::text("t", "x"))
245    }
246    crate::plugin!(run, kind = "ingest-connector");
247
248    #[test]
249    fn info_names_the_kind_and_sdk() {
250        let i = weirdnet_plugin_info();
251        // SAFETY: the info string is static UTF-8
252        let s = unsafe { std::str::from_utf8(std::slice::from_raw_parts(i.ptr, i.len)).unwrap() };
253        assert_eq!(s, r#"{"sdk":"0.1.0","lang":"rust","kind":"ingest-connector"}"#);
254        assert_eq!(crate::SDK_VERSION, "0.1.0");
255    }
256}