dataflow_rs/function/
templated.rs

1//! `Templated`: the config of a plugin function whose fields are declared at
2//! runtime.
3
4use super::{__private, Ctx, FunctionConfig};
5use crate::base::AuthoredPath;
6use crate::base::{IssueCode, Result, Severity};
7use crate::expr::{
8    CompileCx, ConfigField, Expr, ExprEngine, ExprVisitor, FieldKind, FieldSpec, Interpolated,
9    Output, Path, Resolver, ValueKind, Visit,
10};
11use datavalue::OwnedDataValue;
12use serde_json::Value;
13use std::borrow::Cow;
14use std::str::FromStr;
15
16/// One compiled runtime field.
17#[derive(Debug)]
18enum TemplatedField {
19    Expr(Expr<OwnedDataValue>),
20    /// An `Expr(ValueKind::Path)` field, compiled as the derive's `Expr<Path>`
21    /// is, so it is refused the same way.
22    Path(Expr<Path>),
23    Interpolated(Interpolated),
24    Literal(OwnedDataValue),
25    Output(Output),
26}
27
28/// A compiled runtime field with its name, and whether the engine records
29/// its result ([`FieldSpec::records`]).
30#[derive(Debug)]
31struct Field {
32    name: Box<str>,
33    records: bool,
34    value: TemplatedField,
35}
36
37/// A plugin function's config: the fields its manifest declared, compiled by
38/// their kinds. The fields come from the [`CompileCx`]'s runtime fields.
39///
40/// A call reads them by name with [`Templated::get`], or all at once with
41/// [`Templated::resolved`]. Its primary output is the only `Output` field the
42/// manifest declares, or, of several, the one marked
43/// [`primary`](FieldSpec::primary).
44#[derive(Debug)]
45pub struct Templated {
46    /// The fields with a value, set by the input or by a default, in
47    /// declaration order.
48    fields: Vec<Field>,
49    /// The primary output's position in `fields`, when it has a value.
50    primary: Option<usize>,
51}
52
53impl TemplatedField {
54    /// The field's value through `r`, as the derive's field of the same kind
55    /// resolves: through that kind's [`ConfigField::resolve`].
56    fn resolve(&self, r: &mut dyn Resolver) -> Result<OwnedDataValue> {
57        let value = match self {
58            Self::Expr(expr) => expr.resolve(r)?,
59            Self::Path(expr) => expr.resolve(r)?,
60            Self::Interpolated(doc) => doc.resolve(r)?,
61            Self::Literal(value) => value.resolve(r)?,
62            Self::Output(out) => out.resolve(r)?,
63        };
64        // Every kind a runtime field compiles to is set once compiled.
65        Ok(value.unwrap_or(OwnedDataValue::Null))
66    }
67
68    /// The field's value for one call: a literal and a folded constant lent
69    /// from the config, anything else resolved, as [`Self::resolve`] gives
70    /// it, owned.
71    fn lend<'s>(&'s self, ctx: &Ctx<'_>) -> Result<Cow<'s, OwnedDataValue>> {
72        match self {
73            Self::Literal(value) => return Ok(Cow::Borrowed(value)),
74            Self::Expr(expr) => {
75                if let Some(constant) = expr.constant() {
76                    return Ok(Cow::Borrowed(constant));
77                }
78            }
79            _ => {}
80        }
81        self.resolve(&mut &*ctx).map(Cow::Owned)
82    }
83}
84
85/// What makes `specs` break the derive's rule for outputs, if anything:
86/// of several `Output` fields, exactly one is marked
87/// [`primary`](FieldSpec::primary). [`Registry::register`](super::Registry::register)
88/// refuses a `Templated` function whose runtime fields break it, as the
89/// derive refuses such a struct at expansion.
90pub(super) fn primary_problem(specs: &[FieldSpec]) -> Option<String> {
91    let (outputs, primaries) = specs
92        .iter()
93        .filter(|spec| matches!(spec.kind, FieldKind::Output))
94        .fold((0, 0), |(n, p), spec| {
95            (n + 1, p + usize::from(spec.primary))
96        });
97    (outputs > 1 && primaries != 1).then(|| {
98        format!(
99            "its runtime fields hold {outputs} outputs, {primaries} of them primary: mark \
100             exactly one of them `primary`"
101        )
102    })
103}
104
105/// The name of the primary output among `specs`: the only `Output` field,
106/// or, of several, the one marked primary, which `register` makes sure of.
107pub(super) fn primary_name(specs: &[FieldSpec]) -> Option<&str> {
108    let is_output = |spec: &&FieldSpec| matches!(spec.kind, FieldKind::Output);
109    let mut outputs = specs.iter().filter(is_output);
110    let first = outputs.next()?;
111    if outputs.next().is_none() {
112        return Some(&first.name);
113    }
114    specs
115        .iter()
116        .filter(is_output)
117        .find(|spec| spec.primary)
118        .map(|spec| &*spec.name)
119}
120
121fn raw_fits(kind: ValueKind, raw: &Value) -> bool {
122    match kind {
123        ValueKind::Any => true,
124        ValueKind::String => raw.is_string(),
125        ValueKind::Number => raw.is_number(),
126        ValueKind::Bool => raw.is_boolean(),
127        ValueKind::Object => raw.is_object(),
128        ValueKind::Array => raw.is_array(),
129        ValueKind::Path => raw.is_string(),
130    }
131}
132
133fn constant_fits(kind: ValueKind, value: &OwnedDataValue) -> bool {
134    match kind {
135        ValueKind::Any => true,
136        ValueKind::String => matches!(value, OwnedDataValue::String(_)),
137        ValueKind::Number => matches!(value, OwnedDataValue::Number(_)),
138        ValueKind::Bool => matches!(value, OwnedDataValue::Bool(_)),
139        ValueKind::Object => matches!(value, OwnedDataValue::Object(_)),
140        ValueKind::Array => matches!(value, OwnedDataValue::Array(_)),
141        ValueKind::Path => matches!(value, OwnedDataValue::String(s) if Path::from_str(s).is_ok()),
142    }
143}
144
145fn expected(kind: ValueKind) -> &'static str {
146    match kind {
147        ValueKind::String => "a string",
148        ValueKind::Number => "a number",
149        ValueKind::Bool => "a boolean",
150        ValueKind::Object => "an object",
151        ValueKind::Array => "an array",
152        ValueKind::Path => "a path",
153        ValueKind::Any => "a value",
154    }
155}
156
157/// Compiles `raw` as the spec's kind, at the current path.
158fn field(kind: FieldKind, raw: &Value, cx: &mut CompileCx<'_>) -> Option<TemplatedField> {
159    match kind {
160        FieldKind::Expr(ValueKind::Path) => {
161            <Expr<Path>>::compile(raw, cx).map(TemplatedField::Path)
162        }
163        FieldKind::Expr(vk) => {
164            let expr = <Expr<OwnedDataValue>>::compile(raw, cx)?;
165            if expr.constant().is_some_and(|c| !constant_fits(vk, c)) {
166                cx.issue(IssueCode::InputParse, format!("expected {}", expected(vk)));
167                return None;
168            }
169            Some(TemplatedField::Expr(expr))
170        }
171        FieldKind::Interpolated => Interpolated::compile(raw, cx).map(TemplatedField::Interpolated),
172        FieldKind::Literal(vk) => {
173            if !raw_fits(vk, raw) {
174                cx.issue(IssueCode::InputParse, format!("expected {}", expected(vk)));
175                return None;
176            }
177            let value = OwnedDataValue::from(raw);
178            if vk == ValueKind::Path && !constant_fits(vk, &value) {
179                cx.issue(IssueCode::InvalidPath, "not a valid context path");
180                return None;
181            }
182            Some(TemplatedField::Literal(value))
183        }
184        FieldKind::Output => Output::compile(raw, cx).map(TemplatedField::Output),
185        _ => {
186            cx.issue(IssueCode::InputParse, "unsupported runtime field kind");
187            None
188        }
189    }
190}
191
192/// Whether `spec`'s declared default compiles through its kind. A host bug,
193/// not the author's: `Some(why)` names it, and
194/// [`Registry::register`](super::Registry::register) fails once instead of
195/// every compile that leaves the key out.
196///
197/// Kinds a `FieldSpec` alone does not describe (a nested config, a list, a
198/// map, a runtime set) are left to the compile that owns them.
199pub(super) fn default_problem(exprs: &ExprEngine, spec: &FieldSpec) -> Option<String> {
200    let text = spec.default.as_deref()?;
201    if matches!(
202        spec.kind,
203        FieldKind::Templated | FieldKind::List(_) | FieldKind::Map(_) | FieldKind::Nested(_)
204    ) {
205        return None;
206    }
207    let raw: Value = match serde_json::from_str(text) {
208        Ok(raw) => raw,
209        Err(e) => {
210            return Some(format!(
211                "its field `{}`'s default is not JSON ({e})",
212                spec.name
213            ));
214        }
215    };
216    let mut issues = Vec::new();
217    let mut cx = CompileCx::new(exprs, &mut issues);
218    let compiled = field(spec.kind, &raw, &mut cx);
219    let rejected = issues
220        .iter()
221        .find(|issue| issue.severity == Severity::Rejected);
222    if compiled.is_some() && rejected.is_none() {
223        return None;
224    }
225    let why = rejected
226        .map(|issue| issue.message.clone())
227        .unwrap_or_else(|| "it does not compile".to_owned());
228    Some(format!(
229        "its field `{}`'s default is invalid: {why}",
230        spec.name
231    ))
232}
233
234impl Templated {
235    fn compile_fields(input: &Value, cx: &mut CompileCx<'_>) -> Option<Self> {
236        let Some(specs) = cx.runtime_fields() else {
237            cx.issue(
238                IssueCode::InputParse,
239                "no runtime fields are declared for this function",
240            );
241            return None;
242        };
243        let object = __private::object(input, cx)?;
244        let before = cx.rejected();
245        let names: Vec<&str> = specs.iter().map(|s| &*s.name).collect();
246        __private::unknown_keys(object, &names, cx);
247        let mut fields = Vec::new();
248        for spec in specs {
249            cx.at(&*spec.name, |cx| {
250                if let Some(value) = Self::one(spec, object.get(&*spec.name), cx) {
251                    fields.push(Field {
252                        name: Box::from(&*spec.name),
253                        records: spec.records,
254                        value,
255                    });
256                }
257            });
258        }
259        let primary = primary_name(specs)
260            .and_then(|name| fields.iter().position(|field| &*field.name == name));
261        (cx.rejected() == before).then_some(Self { fields, primary })
262    }
263
264    /// The field `name`'s value for this call: a literal, and an expression
265    /// that folded to a constant, lent from the config; an expression
266    /// evaluated, a document rendered, and an output's or a path's
267    /// destination as a string, owned. `None` for a field with no value (an
268    /// optional one the input left unset, with no default), and for a name
269    /// no field has.
270    ///
271    /// # Errors
272    ///
273    /// [`Ctx::eval`]'s, for an expression, a path or an output.
274    pub fn get<'s>(&'s self, ctx: &Ctx<'_>, name: &str) -> Result<Option<Cow<'s, OwnedDataValue>>> {
275        self.fields
276            .iter()
277            .find(|field| &*field.name == name)
278            .map(|field| field.value.lend(ctx))
279            .transpose()
280    }
281
282    /// Every field with a value, set by the input or by its default, each as
283    /// [`Self::get`] gives it, as an object in declaration order (an
284    /// optional field left unset is absent): what a derived config with the
285    /// same fields resolves to ([`FunctionConfig::resolve`], on the same
286    /// per-field code). Secrets are read, as the call reads them.
287    ///
288    /// # Errors
289    ///
290    /// The first field's that fails, as [`Self::get`]'s.
291    pub fn resolved(&self, ctx: &Ctx<'_>) -> Result<OwnedDataValue> {
292        self.resolve_fields(&mut &*ctx)
293    }
294
295    /// Every field with a value, through `r`, as an object in declaration
296    /// order: [`FunctionConfig::resolve`], which a derived config with the
297    /// same fields gives alike.
298    fn resolve_fields(&self, r: &mut dyn Resolver) -> Result<OwnedDataValue> {
299        let entries = self
300            .fields
301            .iter()
302            .map(|field| Ok((field.name.to_string(), field.value.resolve(r)?)))
303            .collect::<Result<Vec<_>>>()?;
304        Ok(OwnedDataValue::Object(entries))
305    }
306
307    /// One runtime field, or `None` when it is absent or rejected.
308    ///
309    /// An absent key takes the spec's default, and a required field with none
310    /// is missing. An explicit `null` is compiled when the field's kind takes
311    /// `null` as a value: an expression, a document, or a literal of any
312    /// JSON. On any other kind it reads as absent without the default, so a
313    /// required field is missing even when it declares one; an output alone
314    /// keeps its default, as a derived config's does.
315    fn one(
316        spec: &FieldSpec,
317        raw: Option<&Value>,
318        cx: &mut CompileCx<'_>,
319    ) -> Option<TemplatedField> {
320        let takes_null = matches!(
321            spec.kind,
322            FieldKind::Expr(_) | FieldKind::Interpolated | FieldKind::Literal(ValueKind::Any)
323        );
324        let default = match raw {
325            Some(raw) if takes_null || !raw.is_null() => return field(spec.kind, raw, cx),
326            // A `null` the kind has no value for: absent, with no default.
327            Some(_) if !matches!(spec.kind, FieldKind::Output) => None,
328            _ => spec.default.as_deref(),
329        };
330        if let Some(text) = default {
331            return match serde_json::from_str::<Value>(text) {
332                Ok(default) => field(spec.kind, &default, cx),
333                Err(e) => {
334                    cx.issue(
335                        IssueCode::InputParse,
336                        format!("the default of `{}` is not JSON ({e})", spec.name),
337                    );
338                    None
339                }
340            };
341        }
342        if spec.required {
343            __private::missing(&spec.name, cx);
344        }
345        None
346    }
347
348    fn walk(&self, v: &mut Visit<'_>) {
349        for field in &self.fields {
350            v.at(&*field.name, |v| {
351                v.recording(field.records, |v| match &field.value {
352                    TemplatedField::Expr(expr) => v.expr(expr.raw()),
353                    TemplatedField::Path(expr) => v.expr(expr.raw()),
354                    // As recorded, as an `Output` always is.
355                    TemplatedField::Output(output) => output.visit(v),
356                    TemplatedField::Interpolated(doc) => v.interpolated(doc),
357                    TemplatedField::Literal(_) => {}
358                });
359            });
360        }
361    }
362}
363
364impl ConfigField for Templated {
365    fn compile(raw: &Value, cx: &mut CompileCx<'_>) -> Option<Self> {
366        Self::compile_fields(raw, cx)
367    }
368    fn visit(&self, v: &mut Visit<'_>) {
369        self.walk(v);
370    }
371    fn resolve(&self, r: &mut dyn Resolver) -> Result<Option<OwnedDataValue>> {
372        self.resolve_fields(r).map(Some)
373    }
374    fn kind() -> FieldKind {
375        FieldKind::Templated
376    }
377}
378
379impl FunctionConfig for Templated {
380    const TEMPLATED: bool = true;
381    fn compile(input: &Value, cx: &mut CompileCx<'_>) -> Option<Self> {
382        Self::compile_fields(input, cx)
383    }
384    fn visit(&self, v: &mut dyn ExprVisitor) {
385        self.walk(&mut Visit::new(v, AuthoredPath::root()));
386    }
387    fn fields() -> &'static [FieldSpec] {
388        &[]
389    }
390
391    fn primary_output(&self) -> Option<&Output> {
392        match &self.fields[self.primary?].value {
393            TemplatedField::Output(out) => Some(out),
394            _ => None,
395        }
396    }
397
398    /// [`Templated::resolved`], through `r`.
399    fn resolve(&self, r: &mut dyn Resolver) -> Result<OwnedDataValue> {
400        self.resolve_fields(r)
401    }
402}
403
404#[cfg(test)]
405mod tests {
406    use super::*;
407    use crate::base::{Issue, Severity};
408    use crate::expr::ExprEngine;
409    use serde_json::json;
410
411    fn specs() -> Vec<FieldSpec> {
412        vec![
413            FieldSpec::runtime("label", FieldKind::Literal(ValueKind::String), false),
414            FieldSpec::runtime("mode", FieldKind::Literal(ValueKind::String), false)
415                .with_default("\"fast\""),
416            FieldSpec::runtime("out", FieldKind::Output, false),
417            FieldSpec::runtime("name", FieldKind::Literal(ValueKind::String), true),
418            FieldSpec::runtime("value", FieldKind::Expr(ValueKind::Any), false),
419            FieldSpec::runtime("doc", FieldKind::Interpolated, false),
420            FieldSpec::runtime("any", FieldKind::Literal(ValueKind::Any), false),
421            FieldSpec::runtime("dest", FieldKind::Output, false).with_default("\"data.dest\""),
422            // Required and defaulted: a host can build this through the
423            // public fields.
424            FieldSpec::runtime("both", FieldKind::Literal(ValueKind::String), true)
425                .with_default("\"fallback\""),
426        ]
427    }
428
429    fn compile(input: &Value) -> (Option<Templated>, Vec<Issue>) {
430        let engine = ExprEngine::default();
431        let specs = specs();
432        let mut issues = Vec::new();
433        let out = {
434            let mut cx = CompileCx::new(&engine, &mut issues)
435                .at_path("tasks[0].function.input".parse().unwrap())
436                .with_runtime_fields(&specs);
437            <Templated as FunctionConfig>::compile(input, &mut cx)
438        };
439        (out, issues)
440    }
441
442    fn get<'t>(templated: &'t Templated, name: &str) -> Option<&'t TemplatedField> {
443        templated
444            .fields
445            .iter()
446            .find(|f| &*f.name == name)
447            .map(|f| &f.value)
448    }
449
450    #[test]
451    fn a_null_reads_as_absent_unless_the_kind_takes_null() {
452        let (out, issues) = compile(&json!({
453            "label": null, "mode": null, "out": null, "name": "n",
454            "value": null, "doc": null, "any": null, "dest": null,
455        }));
456        assert!(issues.is_empty(), "{issues:?}");
457        let out = out.unwrap();
458        // A literal string has no null: absent, and its declared default is
459        // not applied.
460        assert!(get(&out, "label").is_none());
461        assert!(get(&out, "mode").is_none());
462        // An output has no null either, but keeps its default.
463        assert!(get(&out, "out").is_none());
464        assert!(matches!(get(&out, "dest"), Some(TemplatedField::Output(_))));
465        // An expression, a document and any JSON take `null` as a value.
466        assert!(matches!(
467            get(&out, "value"),
468            Some(TemplatedField::Expr(e)) if e.constant() == Some(&OwnedDataValue::Null)
469        ));
470        assert!(matches!(
471            get(&out, "doc"),
472            Some(TemplatedField::Interpolated(_))
473        ));
474        assert!(matches!(
475            get(&out, "any"),
476            Some(TemplatedField::Literal(OwnedDataValue::Null))
477        ));
478    }
479
480    #[test]
481    fn the_primary_output_is_the_only_output_or_the_marked_one() {
482        let primary = |specs: &[FieldSpec], input: &Value| {
483            let engine = ExprEngine::default();
484            let mut issues = Vec::new();
485            let mut cx = CompileCx::new(&engine, &mut issues).with_runtime_fields(specs);
486            let templated = <Templated as FunctionConfig>::compile(input, &mut cx).unwrap();
487            templated
488                .primary_output()
489                .map(|out| out.constant().unwrap().to_string())
490        };
491        let input = json!({"name": "n", "out": "data.out"});
492        // Two outputs, `out` and `dest`, neither marked: none.
493        assert_eq!(primary(&specs(), &input), None);
494        // One of them marked: it, whether the input set it or its default did.
495        let mut marked = specs();
496        marked
497            .iter_mut()
498            .find(|s| s.name == "dest")
499            .unwrap()
500            .primary = true;
501        assert_eq!(primary(&marked, &input).as_deref(), Some("data.dest"));
502        // The only output, unmarked; none when the input leaves it unset.
503        let one: Vec<FieldSpec> = specs().into_iter().filter(|s| s.name != "dest").collect();
504        assert_eq!(primary(&one, &input).as_deref(), Some("data.out"));
505        assert_eq!(primary(&one, &json!({"name": "n"})), None);
506    }
507
508    #[test]
509    fn a_null_on_a_required_literal_is_a_missing_field() {
510        let (out, issues) = compile(&json!({"name": null}));
511        assert!(out.is_none());
512        assert_eq!(issues.len(), 1, "{issues:?}");
513        assert_eq!(issues[0].code, IssueCode::InputParse);
514        assert_eq!(issues[0].severity, Severity::Rejected);
515        assert_eq!(issues[0].path.to_string(), "tasks[0].function.input.name");
516    }
517
518    #[test]
519    fn a_null_on_a_required_field_with_a_default_is_missing_and_absence_is_not() {
520        // An absent key takes the default.
521        let (out, issues) = compile(&json!({"name": "n"}));
522        assert!(issues.is_empty(), "{issues:?}");
523        assert!(matches!(
524            get(&out.unwrap(), "both"),
525            Some(TemplatedField::Literal(OwnedDataValue::String(s))) if s == "fallback"
526        ));
527        // An explicit `null` reads as absent with no default: missing.
528        let (out, issues) = compile(&json!({"name": "n", "both": null}));
529        assert!(out.is_none());
530        assert_eq!(issues.len(), 1, "{issues:?}");
531        assert_eq!(issues[0].code, IssueCode::InputParse);
532        assert_eq!(issues[0].severity, Severity::Rejected);
533        assert_eq!(issues[0].path.to_string(), "tasks[0].function.input.both");
534    }
535}
dataflow_rs/function/templated.rs

dataflow_rs/function/
templated.rs
