dataflow_rs/function/
templated.rs

1//! `Templated`: the config of a plugin function whose fields are declared at
2//! runtime.
3
4use super::{__private, Ctx, FunctionConfig};
5use crate::base::AuthoredPath;
6use crate::base::{IssueCode, Result, Severity};
7use crate::expr::{
8    CompileCx, ConfigField, Expr, ExprEngine, ExprVisitor, FieldKind, FieldSpec, Interpolated,
9    Output, Path, Resolver, ValueKind, Visit,
10};
11use datavalue::OwnedDataValue;
12use serde_json::Value;
13use std::borrow::Cow;
14use std::str::FromStr;
15
16/// One compiled runtime field.
17#[derive(Debug)]
18enum TemplatedField {
19    Expr(Expr<OwnedDataValue>),
20    /// An `Expr(ValueKind::Path)` field, compiled as the derive's `Expr<Path>`
21    /// is, so it is refused the same way.
22    Path(Expr<Path>),
23    Interpolated(Interpolated),
24    Literal(OwnedDataValue),
25    Output(Output),
26}
27
28/// A compiled runtime field with its name, and whether the engine records
29/// its result ([`FieldSpec::records`]).
30#[derive(Debug)]
31struct Field {
32    name: Box<str>,
33    records: bool,
34    value: TemplatedField,
35}
36
37/// A plugin function's config: the fields its manifest declared, compiled by
38/// their kinds. The fields come from the [`CompileCx`]'s runtime fields.
39///
40/// A call reads them by name with [`Templated::get`], or all at once with
41/// [`Templated::resolved`]. Its primary output is the only `Output` field the
42/// manifest declares, or, of several, the one marked
43/// [`primary`](FieldSpec::primary).
44#[derive(Debug)]
45pub struct Templated {
46    /// The fields with a value, set by the input or by a default, in
47    /// declaration order.
48    fields: Vec<Field>,
49    /// The primary output's position in `fields`, when it has a value.
50    primary: Option<usize>,
51}
52
53impl TemplatedField {
54    /// The field's value through `r`, as the derive's field of the same kind
55    /// resolves: through that kind's [`ConfigField::resolve`]. `None` when
56    /// the resolver leaves the field out, as a supplied primary output may.
57    fn resolve(&self, r: &mut dyn Resolver) -> Result<Option<OwnedDataValue>> {
58        match self {
59            Self::Expr(expr) => expr.resolve(r),
60            Self::Path(expr) => expr.resolve(r),
61            Self::Interpolated(doc) => doc.resolve(r),
62            Self::Literal(value) => value.resolve(r),
63            Self::Output(out) => out.resolve(r),
64        }
65    }
66
67    /// The field's value for one call: a literal and a folded constant lent
68    /// from the config, anything else resolved, as [`Self::resolve`] gives
69    /// it, owned. A field the resolver leaves out reads `null`, as
70    /// [`Templated::get`] has no absence to give.
71    fn lend<'s>(&'s self, ctx: &Ctx<'_>) -> Result<Cow<'s, OwnedDataValue>> {
72        match self {
73            Self::Literal(value) => return Ok(Cow::Borrowed(value)),
74            Self::Expr(expr) => {
75                if let Some(constant) = expr.constant() {
76                    return Ok(Cow::Borrowed(constant));
77                }
78            }
79            _ => {}
80        }
81        self.resolve(&mut &*ctx)
82            .map(|value| Cow::Owned(value.unwrap_or(OwnedDataValue::Null)))
83    }
84}
85
86/// What makes `specs` break the derive's rule for outputs, if anything:
87/// of several `Output` fields, exactly one is marked
88/// [`primary`](FieldSpec::primary). [`Registry::register`](super::Registry::register)
89/// refuses a `Templated` function whose runtime fields break it, as the
90/// derive refuses such a struct at expansion.
91pub(super) fn primary_problem(specs: &[FieldSpec]) -> Option<String> {
92    let (outputs, primaries) = specs
93        .iter()
94        .filter(|spec| matches!(spec.kind, FieldKind::Output))
95        .fold((0, 0), |(n, p), spec| {
96            (n + 1, p + usize::from(spec.primary))
97        });
98    (outputs > 1 && primaries != 1).then(|| {
99        format!(
100            "its runtime fields hold {outputs} outputs, {primaries} of them primary: mark \
101             exactly one of them `primary`"
102        )
103    })
104}
105
106/// The name of the primary output among `specs`: the only `Output` field,
107/// or, of several, the one marked primary, which `register` makes sure of.
108pub(super) fn primary_name(specs: &[FieldSpec]) -> Option<&str> {
109    let is_output = |spec: &&FieldSpec| matches!(spec.kind, FieldKind::Output);
110    let mut outputs = specs.iter().filter(is_output);
111    let first = outputs.next()?;
112    if outputs.next().is_none() {
113        return Some(&first.name);
114    }
115    specs
116        .iter()
117        .filter(is_output)
118        .find(|spec| spec.primary)
119        .map(|spec| &*spec.name)
120}
121
122fn raw_fits(kind: ValueKind, raw: &Value) -> bool {
123    match kind {
124        ValueKind::Any => true,
125        ValueKind::String => raw.is_string(),
126        ValueKind::Number => raw.is_number(),
127        ValueKind::Bool => raw.is_boolean(),
128        ValueKind::Object => raw.is_object(),
129        ValueKind::Array => raw.is_array(),
130        ValueKind::Path => raw.is_string(),
131    }
132}
133
134fn constant_fits(kind: ValueKind, value: &OwnedDataValue) -> bool {
135    match kind {
136        ValueKind::Any => true,
137        ValueKind::String => matches!(value, OwnedDataValue::String(_)),
138        ValueKind::Number => matches!(value, OwnedDataValue::Number(_)),
139        ValueKind::Bool => matches!(value, OwnedDataValue::Bool(_)),
140        ValueKind::Object => matches!(value, OwnedDataValue::Object(_)),
141        ValueKind::Array => matches!(value, OwnedDataValue::Array(_)),
142        ValueKind::Path => matches!(value, OwnedDataValue::String(s) if Path::from_str(s).is_ok()),
143    }
144}
145
146fn expected(kind: ValueKind) -> &'static str {
147    match kind {
148        ValueKind::String => "a string",
149        ValueKind::Number => "a number",
150        ValueKind::Bool => "a boolean",
151        ValueKind::Object => "an object",
152        ValueKind::Array => "an array",
153        ValueKind::Path => "a path",
154        ValueKind::Any => "a value",
155    }
156}
157
158/// Compiles `raw` as the spec's kind, at the current path.
159fn field(kind: FieldKind, raw: &Value, cx: &mut CompileCx<'_>) -> Option<TemplatedField> {
160    match kind {
161        FieldKind::Expr(ValueKind::Path) => {
162            <Expr<Path>>::compile(raw, cx).map(TemplatedField::Path)
163        }
164        FieldKind::Expr(vk) => {
165            let expr = <Expr<OwnedDataValue>>::compile(raw, cx)?;
166            if expr.constant().is_some_and(|c| !constant_fits(vk, c)) {
167                cx.issue(IssueCode::InputParse, format!("expected {}", expected(vk)));
168                return None;
169            }
170            Some(TemplatedField::Expr(expr))
171        }
172        FieldKind::Interpolated => Interpolated::compile(raw, cx).map(TemplatedField::Interpolated),
173        FieldKind::Literal(vk) => {
174            if !raw_fits(vk, raw) {
175                cx.issue(IssueCode::InputParse, format!("expected {}", expected(vk)));
176                return None;
177            }
178            let value = OwnedDataValue::from(raw);
179            if vk == ValueKind::Path && !constant_fits(vk, &value) {
180                cx.issue(IssueCode::InvalidPath, "not a valid context path");
181                return None;
182            }
183            Some(TemplatedField::Literal(value))
184        }
185        FieldKind::Output => Output::compile(raw, cx).map(TemplatedField::Output),
186        _ => {
187            cx.issue(IssueCode::InputParse, "unsupported runtime field kind");
188            None
189        }
190    }
191}
192
193/// Whether `spec`'s declared default compiles through its kind. A host bug,
194/// not the author's: `Some(why)` names it, and
195/// [`Registry::register`](super::Registry::register) fails once instead of
196/// every compile that leaves the key out.
197///
198/// Kinds a `FieldSpec` alone does not describe (a nested config, a list, a
199/// map, a runtime set) are left to the compile that owns them.
200pub(super) fn default_problem(exprs: &ExprEngine, spec: &FieldSpec) -> Option<String> {
201    let text = spec.default.as_deref()?;
202    if matches!(
203        spec.kind,
204        FieldKind::Templated | FieldKind::List(_) | FieldKind::Map(_) | FieldKind::Nested(_)
205    ) {
206        return None;
207    }
208    let raw: Value = match serde_json::from_str(text) {
209        Ok(raw) => raw,
210        Err(e) => {
211            return Some(format!(
212                "its field `{}`'s default is not JSON ({e})",
213                spec.name
214            ));
215        }
216    };
217    let mut issues = Vec::new();
218    let mut cx = CompileCx::new(exprs, &mut issues);
219    let compiled = field(spec.kind, &raw, &mut cx);
220    let rejected = issues
221        .iter()
222        .find(|issue| issue.severity == Severity::Rejected);
223    if compiled.is_some() && rejected.is_none() {
224        return None;
225    }
226    let why = rejected
227        .map(|issue| issue.message.clone())
228        .unwrap_or_else(|| "it does not compile".to_owned());
229    Some(format!(
230        "its field `{}`'s default is invalid: {why}",
231        spec.name
232    ))
233}
234
235impl Templated {
236    fn compile_fields(input: &Value, cx: &mut CompileCx<'_>) -> Option<Self> {
237        let Some(specs) = cx.runtime_fields() else {
238            cx.issue(
239                IssueCode::InputParse,
240                "no runtime fields are declared for this function",
241            );
242            return None;
243        };
244        let object = __private::object(input, cx)?;
245        let before = cx.rejected();
246        let names: Vec<&str> = specs.iter().map(|s| &*s.name).collect();
247        __private::unknown_keys(object, &names, cx);
248        let mut fields = Vec::new();
249        for spec in specs {
250            cx.at(&*spec.name, |cx| {
251                if let Some(value) = Self::one(spec, object.get(&*spec.name), cx) {
252                    fields.push(Field {
253                        name: Box::from(&*spec.name),
254                        records: spec.records,
255                        value,
256                    });
257                }
258            });
259        }
260        let primary = primary_name(specs)
261            .and_then(|name| fields.iter().position(|field| &*field.name == name));
262        (cx.rejected() == before).then_some(Self { fields, primary })
263    }
264
265    /// The field `name`'s value for this call: a literal, and an expression
266    /// that folded to a constant, lent from the config; an expression
267    /// evaluated, a document rendered, and an output's or a path's
268    /// destination as a string, owned. `None` for a field with no value (an
269    /// optional one the input left unset, with no default), and for a name
270    /// no field has.
271    ///
272    /// # Errors
273    ///
274    /// [`Ctx::eval`]'s, for an expression, a path or an output.
275    pub fn get<'s>(&'s self, ctx: &Ctx<'_>, name: &str) -> Result<Option<Cow<'s, OwnedDataValue>>> {
276        self.fields
277            .iter()
278            .find(|field| &*field.name == name)
279            .map(|field| field.value.lend(ctx))
280            .transpose()
281    }
282
283    /// Every field with a value, set by the input or by its default, each as
284    /// [`Self::get`] gives it, as an object in declaration order (an
285    /// optional field left unset is absent): what a derived config with the
286    /// same fields resolves to ([`FunctionConfig::resolve`], on the same
287    /// per-field code). Secrets are read, as the call reads them.
288    ///
289    /// # Errors
290    ///
291    /// The first field's that fails, as [`Self::get`]'s.
292    pub fn resolved(&self, ctx: &Ctx<'_>) -> Result<OwnedDataValue> {
293        self.resolve_fields(&mut &*ctx)
294    }
295
296    /// Every field with a value, through `r`, as an object in declaration
297    /// order: [`FunctionConfig::resolve`], which a derived config with the
298    /// same fields gives alike. A field the resolver leaves out is absent,
299    /// as an optional field left unset is.
300    fn resolve_fields(&self, r: &mut dyn Resolver) -> Result<OwnedDataValue> {
301        let mut entries = Vec::with_capacity(self.fields.len());
302        for field in &self.fields {
303            if let Some(value) = field.value.resolve(r)? {
304                entries.push((field.name.to_string(), value));
305            }
306        }
307        Ok(OwnedDataValue::Object(entries))
308    }
309
310    /// One runtime field, or `None` when it is absent or rejected.
311    ///
312    /// An absent key takes the spec's default, and a required field with none
313    /// is missing. An explicit `null` is compiled when the field's kind takes
314    /// `null` as a value: an expression, a document, or a literal of any
315    /// JSON. On any other kind it reads as absent without the default, so a
316    /// required field is missing even when it declares one; an output alone
317    /// keeps its default, as a derived config's does.
318    fn one(
319        spec: &FieldSpec,
320        raw: Option<&Value>,
321        cx: &mut CompileCx<'_>,
322    ) -> Option<TemplatedField> {
323        let takes_null = matches!(
324            spec.kind,
325            FieldKind::Expr(_) | FieldKind::Interpolated | FieldKind::Literal(ValueKind::Any)
326        );
327        let default = match raw {
328            Some(raw) if takes_null || !raw.is_null() => return field(spec.kind, raw, cx),
329            // A `null` the kind has no value for: absent, with no default.
330            Some(_) if !matches!(spec.kind, FieldKind::Output) => None,
331            _ => spec.default.as_deref(),
332        };
333        if let Some(text) = default {
334            return match serde_json::from_str::<Value>(text) {
335                Ok(default) => field(spec.kind, &default, cx),
336                Err(e) => {
337                    cx.issue(
338                        IssueCode::InputParse,
339                        format!("the default of `{}` is not JSON ({e})", spec.name),
340                    );
341                    None
342                }
343            };
344        }
345        if spec.required {
346            __private::missing(&spec.name, cx);
347        }
348        None
349    }
350
351    fn walk(&self, v: &mut Visit<'_>) {
352        for field in &self.fields {
353            v.at(&*field.name, |v| {
354                v.recording(field.records, |v| match &field.value {
355                    TemplatedField::Expr(expr) => v.expr(expr.raw()),
356                    TemplatedField::Path(expr) => v.expr(expr.raw()),
357                    // As recorded, as an `Output` always is.
358                    TemplatedField::Output(output) => output.visit(v),
359                    TemplatedField::Interpolated(doc) => v.interpolated(doc),
360                    TemplatedField::Literal(_) => {}
361                });
362            });
363        }
364    }
365}
366
367impl ConfigField for Templated {
368    fn compile(raw: &Value, cx: &mut CompileCx<'_>) -> Option<Self> {
369        Self::compile_fields(raw, cx)
370    }
371    fn visit(&self, v: &mut Visit<'_>) {
372        self.walk(v);
373    }
374    fn resolve(&self, r: &mut dyn Resolver) -> Result<Option<OwnedDataValue>> {
375        self.resolve_fields(r).map(Some)
376    }
377    fn kind() -> FieldKind {
378        FieldKind::Templated
379    }
380}
381
382impl FunctionConfig for Templated {
383    const TEMPLATED: bool = true;
384    fn compile(input: &Value, cx: &mut CompileCx<'_>) -> Option<Self> {
385        Self::compile_fields(input, cx)
386    }
387    fn visit(&self, v: &mut dyn ExprVisitor) {
388        self.walk(&mut Visit::new(v, AuthoredPath::root()));
389    }
390    fn fields() -> &'static [FieldSpec] {
391        &[]
392    }
393
394    fn primary_output(&self) -> Option<&Output> {
395        match &self.fields[self.primary?].value {
396            TemplatedField::Output(out) => Some(out),
397            _ => None,
398        }
399    }
400
401    /// [`Templated::resolved`], through `r`.
402    fn resolve(&self, r: &mut dyn Resolver) -> Result<OwnedDataValue> {
403        self.resolve_fields(r)
404    }
405}
406
407#[cfg(test)]
408mod tests {
409    use super::*;
410    use crate::base::{Issue, Severity};
411    use crate::expr::ExprEngine;
412    use serde_json::json;
413
414    fn specs() -> Vec<FieldSpec> {
415        vec![
416            FieldSpec::runtime("label", FieldKind::Literal(ValueKind::String), false),
417            FieldSpec::runtime("mode", FieldKind::Literal(ValueKind::String), false)
418                .with_default("\"fast\""),
419            FieldSpec::runtime("out", FieldKind::Output, false),
420            FieldSpec::runtime("name", FieldKind::Literal(ValueKind::String), true),
421            FieldSpec::runtime("value", FieldKind::Expr(ValueKind::Any), false),
422            FieldSpec::runtime("doc", FieldKind::Interpolated, false),
423            FieldSpec::runtime("any", FieldKind::Literal(ValueKind::Any), false),
424            FieldSpec::runtime("dest", FieldKind::Output, false).with_default("\"data.dest\""),
425            // Required and defaulted: a host can build this through the
426            // public fields.
427            FieldSpec::runtime("both", FieldKind::Literal(ValueKind::String), true)
428                .with_default("\"fallback\""),
429        ]
430    }
431
432    fn compile(input: &Value) -> (Option<Templated>, Vec<Issue>) {
433        let engine = ExprEngine::default();
434        let specs = specs();
435        let mut issues = Vec::new();
436        let out = {
437            let mut cx = CompileCx::new(&engine, &mut issues)
438                .at_path("tasks[0].function.input".parse().unwrap())
439                .with_runtime_fields(&specs);
440            <Templated as FunctionConfig>::compile(input, &mut cx)
441        };
442        (out, issues)
443    }
444
445    fn get<'t>(templated: &'t Templated, name: &str) -> Option<&'t TemplatedField> {
446        templated
447            .fields
448            .iter()
449            .find(|f| &*f.name == name)
450            .map(|f| &f.value)
451    }
452
453    #[test]
454    fn a_null_reads_as_absent_unless_the_kind_takes_null() {
455        let (out, issues) = compile(&json!({
456            "label": null, "mode": null, "out": null, "name": "n",
457            "value": null, "doc": null, "any": null, "dest": null,
458        }));
459        assert!(issues.is_empty(), "{issues:?}");
460        let out = out.unwrap();
461        // A literal string has no null: absent, and its declared default is
462        // not applied.
463        assert!(get(&out, "label").is_none());
464        assert!(get(&out, "mode").is_none());
465        // An output has no null either, but keeps its default.
466        assert!(get(&out, "out").is_none());
467        assert!(matches!(get(&out, "dest"), Some(TemplatedField::Output(_))));
468        // An expression, a document and any JSON take `null` as a value.
469        assert!(matches!(
470            get(&out, "value"),
471            Some(TemplatedField::Expr(e)) if e.constant() == Some(&OwnedDataValue::Null)
472        ));
473        assert!(matches!(
474            get(&out, "doc"),
475            Some(TemplatedField::Interpolated(_))
476        ));
477        assert!(matches!(
478            get(&out, "any"),
479            Some(TemplatedField::Literal(OwnedDataValue::Null))
480        ));
481    }
482
483    #[test]
484    fn the_primary_output_is_the_only_output_or_the_marked_one() {
485        let primary = |specs: &[FieldSpec], input: &Value| {
486            let engine = ExprEngine::default();
487            let mut issues = Vec::new();
488            let mut cx = CompileCx::new(&engine, &mut issues).with_runtime_fields(specs);
489            let templated = <Templated as FunctionConfig>::compile(input, &mut cx).unwrap();
490            templated
491                .primary_output()
492                .map(|out| out.constant().unwrap().to_string())
493        };
494        let input = json!({"name": "n", "out": "data.out"});
495        // Two outputs, `out` and `dest`, neither marked: none.
496        assert_eq!(primary(&specs(), &input), None);
497        // One of them marked: it, whether the input set it or its default did.
498        let mut marked = specs();
499        marked
500            .iter_mut()
501            .find(|s| s.name == "dest")
502            .unwrap()
503            .primary = true;
504        assert_eq!(primary(&marked, &input).as_deref(), Some("data.dest"));
505        // The only output, unmarked; none when the input leaves it unset.
506        let one: Vec<FieldSpec> = specs().into_iter().filter(|s| s.name != "dest").collect();
507        assert_eq!(primary(&one, &input).as_deref(), Some("data.out"));
508        assert_eq!(primary(&one, &json!({"name": "n"})), None);
509    }
510
511    #[test]
512    fn a_null_on_a_required_literal_is_a_missing_field() {
513        let (out, issues) = compile(&json!({"name": null}));
514        assert!(out.is_none());
515        assert_eq!(issues.len(), 1, "{issues:?}");
516        assert_eq!(issues[0].code, IssueCode::InputParse);
517        assert_eq!(issues[0].severity, Severity::Rejected);
518        assert_eq!(issues[0].path.to_string(), "tasks[0].function.input.name");
519    }
520
521    #[test]
522    fn a_null_on_a_required_field_with_a_default_is_missing_and_absence_is_not() {
523        // An absent key takes the default.
524        let (out, issues) = compile(&json!({"name": "n"}));
525        assert!(issues.is_empty(), "{issues:?}");
526        assert!(matches!(
527            get(&out.unwrap(), "both"),
528            Some(TemplatedField::Literal(OwnedDataValue::String(s))) if s == "fallback"
529        ));
530        // An explicit `null` reads as absent with no default: missing.
531        let (out, issues) = compile(&json!({"name": "n", "both": null}));
532        assert!(out.is_none());
533        assert_eq!(issues.len(), 1, "{issues:?}");
534        assert_eq!(issues[0].code, IssueCode::InputParse);
535        assert_eq!(issues[0].severity, Severity::Rejected);
536        assert_eq!(issues[0].path.to_string(), "tasks[0].function.input.both");
537    }
538}
dataflow_rs/function/templated.rs

dataflow_rs/function/
templated.rs
