use crate::engine::error::{DataflowError, Result};
use crate::engine::functions::FunctionConfig;
use crate::engine::task::Task;
use chrono::{DateTime, Utc};
use datalogic_rs::Logic;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::fs;
use std::path::Path;
use std::sync::Arc;
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct Rollout {
pub bucket_start: u8,
pub bucket_end: u8,
}
impl Rollout {
#[inline]
pub fn accepts(&self, bucket: u8) -> bool {
bucket >= self.bucket_start && bucket < self.bucket_end
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
pub struct LoopConfig {
#[serde(default)]
pub counter: Option<String>,
#[serde(default)]
pub init: i64,
#[serde(default = "default_increment")]
pub increment: i64,
pub max: i64,
#[doc(hidden)]
#[serde(skip)]
pub counter_parts: Arc<[Arc<str>]>,
}
fn default_increment() -> i64 {
1
}
impl LoopConfig {
fn validate(&self, workflow_id: &str) -> Result<()> {
if self.increment < 1 {
return Err(DataflowError::Workflow(format!(
"Workflow {workflow_id}: loop increment must be >= 1, got {} \
(a non-advancing counter would never reach max)",
self.increment
)));
}
if self.max <= self.init {
return Err(DataflowError::Workflow(format!(
"Workflow {workflow_id}: loop max ({}) must be greater than init ({}) — \
the bound is half-open, so this could never run a sweep",
self.max, self.init
)));
}
if let Some(counter) = &self.counter {
if counter.is_empty() || counter.split('.').any(str::is_empty) {
return Err(DataflowError::Workflow(format!(
"Workflow {workflow_id}: loop counter must be a non-empty \
temp_data field path, got {counter:?}"
)));
}
}
Ok(())
}
#[doc(hidden)]
pub fn precompute_counter_path(&mut self) {
self.counter_parts = match &self.counter {
Some(counter) => crate::engine::utils::compute_path_parts("temp_data", counter),
None => Arc::from([] as [Arc<str>; 0]),
};
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum WorkflowStatus {
#[default]
Active,
Paused,
Archived,
}
#[derive(Clone, Debug, Deserialize)]
pub struct Workflow {
pub id: String,
#[doc(hidden)]
#[serde(skip)]
pub id_arc: Arc<str>,
pub name: String,
#[serde(default)]
pub priority: u32,
pub description: Option<String>,
#[serde(default = "crate::engine::utils::default_condition")]
pub condition: Value,
#[doc(hidden)]
#[serde(skip)]
pub compiled_condition: Option<Arc<Logic>>,
#[doc(hidden)]
#[serde(skip, default)]
pub fully_sync: bool,
pub tasks: Vec<Task>,
#[serde(default)]
pub continue_on_error: bool,
#[serde(default = "default_channel")]
pub channel: String,
#[serde(default = "default_version")]
pub version: u32,
#[serde(default)]
pub status: WorkflowStatus,
#[serde(default)]
pub rollout: Option<Rollout>,
#[serde(default, rename = "loop")]
pub loop_config: Option<LoopConfig>,
#[serde(default)]
pub tags: Vec<String>,
#[serde(default)]
pub created_at: Option<DateTime<Utc>>,
#[serde(default)]
pub updated_at: Option<DateTime<Utc>>,
}
fn default_channel() -> String {
"default".to_string()
}
fn default_version() -> u32 {
1
}
impl Default for Workflow {
fn default() -> Self {
Self::new()
}
}
impl Workflow {
pub fn new() -> Self {
Workflow {
id: String::new(),
id_arc: Arc::from(""),
name: String::new(),
priority: 0,
description: None,
condition: Value::Bool(true),
compiled_condition: None,
fully_sync: false,
tasks: Vec::new(),
continue_on_error: false,
channel: default_channel(),
version: 1,
status: WorkflowStatus::Active,
rollout: None,
loop_config: None,
tags: Vec::new(),
created_at: None,
updated_at: None,
}
}
pub fn rule(id: &str, name: &str, condition: Value, tasks: Vec<Task>) -> Self {
Workflow {
id: id.to_string(),
id_arc: Arc::from(id),
name: name.to_string(),
priority: 0,
description: None,
condition,
compiled_condition: None,
fully_sync: false,
tasks,
continue_on_error: false,
channel: default_channel(),
version: 1,
status: WorkflowStatus::Active,
rollout: None,
loop_config: None,
tags: Vec::new(),
created_at: None,
updated_at: None,
}
}
pub fn from_json(json_str: &str) -> Result<Self> {
serde_json::from_str(json_str).map_err(DataflowError::from_serde)
}
pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
let json_str = fs::read_to_string(path).map_err(DataflowError::from_io)?;
Self::from_json(&json_str)
}
pub fn validate(&self) -> Result<()> {
if self.id.is_empty() {
return Err(DataflowError::Workflow(
"Workflow id cannot be empty".to_string(),
));
}
if self.name.is_empty() {
return Err(DataflowError::Workflow(
"Workflow name cannot be empty".to_string(),
));
}
if self.tasks.is_empty() {
return Err(DataflowError::Workflow(
"Workflow must have at least one task".to_string(),
));
}
let mut task_ids = std::collections::HashSet::new();
for task in &self.tasks {
if !task_ids.insert(&task.id) {
return Err(DataflowError::Workflow(format!(
"Duplicate task ID '{}' in workflow",
task.id
)));
}
}
if let Some(loop_config) = &self.loop_config {
loop_config.validate(&self.id)?;
}
Ok(())
}
}
#[derive(Debug, Clone, Copy)]
pub struct ConnectorRef<'a> {
pub workflow_id: &'a str,
pub task_id: &'a str,
pub function: &'a str,
pub connector: &'a str,
pub config: &'a FunctionConfig,
}
impl Workflow {
pub fn connector_refs(&self) -> impl Iterator<Item = ConnectorRef<'_>> {
self.tasks.iter().filter_map(move |task| {
task.function.connector().map(|connector| ConnectorRef {
workflow_id: &self.id,
task_id: &task.id,
function: task.function.function_name(),
connector,
config: &task.function,
})
})
}
}
#[cfg(test)]
mod tests {
use super::*;
fn wf(tasks_json: &str) -> Workflow {
Workflow::from_json(&format!(
r#"{{ "id": "w", "name": "w", "priority": 0, "condition": true,
"tasks": [{tasks_json}] }}"#
))
.expect("workflow should parse")
}
const HTTP: &str = r#"{ "id": "call", "name": "call", "function": {
"name": "http_call", "input": { "connector": "user_service" } } }"#;
const KAFKA: &str = r#"{ "id": "pub", "name": "pub", "function": {
"name": "publish_kafka",
"input": { "connector": "events", "topic": "t" } } }"#;
const MAP: &str = r#"{ "id": "m", "name": "m", "function": {
"name": "map", "input": { "mappings": [] } } }"#;
const LOG: &str = r#"{ "id": "l", "name": "l", "function": {
"name": "log", "input": { "message": "hi" } } }"#;
#[test]
fn connector_refs_yields_only_connector_tasks_in_task_order() {
let workflow = wf(&format!("{MAP},{HTTP},{LOG},{KAFKA}"));
let refs: Vec<_> = workflow.connector_refs().collect();
assert_eq!(refs.len(), 2);
assert_eq!(refs[0].task_id, "call");
assert_eq!(refs[0].function, "http_call");
assert_eq!(refs[0].connector, "user_service");
assert_eq!(refs[1].task_id, "pub");
assert_eq!(refs[1].function, "publish_kafka");
assert_eq!(refs[1].connector, "events");
}
#[test]
fn connector_refs_carries_the_owning_workflow_id() {
let workflow = wf(HTTP);
assert!(workflow.connector_refs().all(|r| r.workflow_id == "w"));
let empty = Workflow::new();
assert_eq!(empty.id, "");
assert_eq!(empty.connector_refs().count(), 0);
}
#[test]
fn connector_refs_is_empty_for_no_tasks() {
assert_eq!(Workflow::new().connector_refs().count(), 0);
}
#[test]
fn connector_refs_does_not_deduplicate() {
let a = r#"{ "id": "a", "name": "a", "function": {
"name": "http_call", "input": { "connector": "same" } } }"#;
let b = r#"{ "id": "b", "name": "b", "function": {
"name": "enrich",
"input": { "connector": "same", "merge_path": "data.out" } } }"#;
let workflow = wf(&format!("{a},{b}"));
let refs: Vec<_> = workflow.connector_refs().collect();
assert_eq!(refs.len(), 2, "one item per task, not a distinct set");
assert!(refs.iter().all(|r| r.connector == "same"));
}
#[test]
fn connector_refs_works_on_an_uncompiled_workflow() {
let workflow = wf(HTTP);
assert!(workflow.compiled_condition.is_none());
assert_eq!(workflow.connector_refs().count(), 1);
}
#[test]
fn connector_ref_is_copy() {
let workflow = wf(HTTP);
let r = workflow.connector_refs().next().unwrap();
let copied = r;
assert_eq!(r.connector, copied.connector);
assert_eq!(r.task_id, copied.task_id);
}
#[test]
fn connector_ref_config_supports_a_cross_field_rule() {
let custom = r#"{ "id": "db", "name": "db", "function": {
"name": "pg_query",
"input": { "connector": "pg_main", "database": "orders" } } }"#;
let workflow = wf(custom);
let r = workflow.connector_refs().next().expect("custom connector");
assert_eq!(r.connector, "pg_main");
match r.config {
FunctionConfig::Custom { input, .. } => {
assert_eq!(
input.get("database").and_then(|v| v.as_str()),
Some("orders")
);
}
other => panic!("expected Custom, got {other:?}"),
}
}
#[test]
fn rollout_accepts_is_a_half_open_range() {
let all = Rollout {
bucket_start: 0,
bucket_end: 100,
};
assert!(all.accepts(0));
assert!(all.accepts(99));
let lower = Rollout {
bucket_start: 0,
bucket_end: 50,
};
assert!(lower.accepts(0));
assert!(lower.accepts(49));
assert!(!lower.accepts(50), "bucket_end is exclusive");
assert!(!lower.accepts(99));
let upper = Rollout {
bucket_start: 50,
bucket_end: 100,
};
assert!(upper.accepts(50), "bucket_start is inclusive");
assert!(upper.accepts(99));
assert!(!upper.accepts(49));
for b in 0u8..=99 {
assert_ne!(
lower.accepts(b),
upper.accepts(b),
"bucket {b} must be served by exactly one half"
);
}
}
#[test]
fn rollout_empty_and_inverted_ranges_accept_nothing() {
let empty = Rollout {
bucket_start: 50,
bucket_end: 50,
};
let inverted = Rollout {
bucket_start: 60,
bucket_end: 20,
};
for b in 0u8..=99 {
assert!(!empty.accepts(b), "empty range accepted {b}");
assert!(!inverted.accepts(b), "inverted range accepted {b}");
}
}
#[test]
fn rollout_end_of_100_is_representable_without_overflow() {
let r = Rollout {
bucket_start: 99,
bucket_end: 100,
};
assert!(r.accepts(99));
assert!(!r.accepts(98));
}
#[test]
fn rollout_defaults_to_none_on_every_construction_path() {
assert_eq!(Workflow::new().rollout, None);
assert_eq!(Workflow::default().rollout, None);
assert_eq!(
Workflow::rule("r", "r", Value::Bool(true), Vec::new()).rollout,
None
);
assert_eq!(wf(MAP).rollout, None, "absent JSON key gives None");
}
fn loop_wf(loop_json: &str) -> Result<Workflow> {
let workflow = Workflow::from_json(&format!(
r#"{{ "id": "w", "name": "w", "loop": {loop_json}, "tasks": [{MAP}] }}"#
))?;
workflow.validate()?;
Ok(workflow)
}
#[test]
fn loop_config_defaults_init_zero_increment_one() {
let cfg = loop_wf(r#"{"max": 5}"#)
.expect("valid loop")
.loop_config
.expect("loop config present");
assert_eq!(cfg.init, 0);
assert_eq!(cfg.increment, 1);
assert_eq!(cfg.max, 5);
assert_eq!(cfg.counter, None);
}
#[test]
fn loop_config_is_absent_on_every_construction_path() {
assert!(wf(MAP).loop_config.is_none(), "absent JSON key gives None");
assert!(Workflow::new().loop_config.is_none());
assert!(Workflow::default().loop_config.is_none());
assert!(
Workflow::rule("r", "r", Value::Bool(true), Vec::new())
.loop_config
.is_none()
);
}
#[test]
fn loop_config_rejects_a_bound_that_could_never_run_a_sweep() {
assert!(loop_wf(r#"{"max": 0}"#).is_err());
assert!(loop_wf(r#"{"init": 5, "max": 5}"#).is_err());
assert!(loop_wf(r#"{"init": 5, "max": 2}"#).is_err());
}
#[test]
fn loop_config_rejects_a_non_advancing_increment() {
assert!(loop_wf(r#"{"max": 5, "increment": 0}"#).is_err());
assert!(loop_wf(r#"{"max": 5, "increment": -1}"#).is_err());
}
#[test]
fn loop_config_rejects_an_empty_counter_path() {
assert!(loop_wf(r#"{"max": 5, "counter": ""}"#).is_err());
assert!(loop_wf(r#"{"max": 5, "counter": "a..b"}"#).is_err());
assert!(loop_wf(r#"{"max": 5, "counter": "a."}"#).is_err());
}
#[test]
fn loop_config_requires_max() {
assert!(
Workflow::from_json(r#"{ "id": "w", "name": "w", "loop": {}, "tasks": [] }"#).is_err()
);
}
#[test]
fn loop_config_deserializes_every_combination_of_optional_fields() {
for (json, counter, init, increment) in [
(r#"{"max": 9}"#, None, 0, 1),
(r#"{"max": 9, "counter": "i"}"#, Some("i"), 0, 1),
(r#"{"max": 9, "init": 4}"#, None, 4, 1),
(r#"{"max": 9, "increment": 3}"#, None, 0, 3),
(r#"{"max": 9, "counter": "i", "init": 4}"#, Some("i"), 4, 1),
(
r#"{"max": 9, "counter": "i", "increment": 3}"#,
Some("i"),
0,
3,
),
(r#"{"max": 9, "init": 4, "increment": 3}"#, None, 4, 3),
(
r#"{"max": 9, "counter": "i", "init": 4, "increment": 3}"#,
Some("i"),
4,
3,
),
] {
let cfg = loop_wf(json)
.unwrap_or_else(|e| panic!("{json} should be valid: {e}"))
.loop_config
.expect("loop config present");
assert_eq!(cfg.counter.as_deref(), counter, "counter for {json}");
assert_eq!(cfg.init, init, "init for {json}");
assert_eq!(cfg.increment, increment, "increment for {json}");
assert_eq!(cfg.max, 9, "max for {json}");
}
}
#[test]
fn loop_config_validation_matrix_over_init_increment_and_max() {
for (init, increment, max, valid) in [
(0_i64, 1_i64, 1_i64, true),
(0, 1, 100, true),
(0, 7, 3, true), (5, 1, 6, true),
(-5, 1, 0, true),
(-5, 2, -4, true),
(-1, 1, 1, true),
(0, 1, 0, false),
(5, 1, 5, false),
(5, 1, 4, false),
(0, 1, -1, false),
(-5, 1, -5, false),
(0, 0, 10, false),
(0, -1, 10, false),
(0, -100, 10, false),
] {
let json = format!(r#"{{"init": {init}, "increment": {increment}, "max": {max}}}"#);
assert_eq!(
loop_wf(&json).is_ok(),
valid,
"init={init} increment={increment} max={max} should be {}",
if valid { "accepted" } else { "rejected" }
);
}
}
#[test]
fn loop_config_counter_path_matrix() {
for (counter, valid) in [
("i", true),
("index", true),
("cursor.index", true),
("a.b.c.d", true),
("#7", true), ("", false),
(".", false),
("a.", false),
(".a", false),
("a..b", false),
] {
let json = format!(r#"{{"max": 5, "counter": "{counter}"}}"#);
assert_eq!(
loop_wf(&json).is_ok(),
valid,
"counter {counter:?} should be {}",
if valid { "accepted" } else { "rejected" }
);
}
}
#[test]
fn precompute_counter_path_matrix() {
for (counter, expected) in [
("i", vec!["temp_data", "i"]),
("cursor.index", vec!["temp_data", "cursor", "index"]),
("a.b.c", vec!["temp_data", "a", "b", "c"]),
("#7", vec!["temp_data", "#7"]),
] {
let mut cfg = loop_wf(&format!(r#"{{"max": 5, "counter": "{counter}"}}"#))
.expect("valid loop")
.loop_config
.expect("loop config present");
cfg.precompute_counter_path();
let parts: Vec<&str> = cfg.counter_parts.iter().map(Arc::as_ref).collect();
assert_eq!(parts, expected, "for counter {counter:?}");
}
}
#[test]
fn precompute_counter_path_is_idempotent() {
let mut cfg = loop_wf(r#"{"max": 5, "counter": "cursor.index"}"#)
.expect("valid loop")
.loop_config
.expect("loop config present");
cfg.precompute_counter_path();
let first: Vec<Arc<str>> = cfg.counter_parts.to_vec();
cfg.precompute_counter_path();
assert_eq!(cfg.counter_parts.to_vec(), first);
}
#[test]
fn loop_config_rejects_a_non_object_and_a_non_numeric_max() {
for json in [r#""five""#, "5", "[]", r#"{"max": "5"}"#, "true"] {
assert!(loop_wf(json).is_err(), "{json} is not a valid loop config");
}
}
#[test]
fn an_explicit_null_loop_means_no_loop() {
let workflow = loop_wf("null").expect("explicit null should be accepted");
assert!(workflow.loop_config.is_none());
}
#[test]
fn a_workflow_with_a_loop_still_validates_its_other_rules() {
let duplicate_tasks = Workflow::from_json(
r#"{ "id": "w", "name": "w", "loop": {"max": 5}, "tasks": [
{"id": "t", "name": "t", "function": {"name": "map", "input": {"mappings": []}}},
{"id": "t", "name": "t", "function": {"name": "map", "input": {"mappings": []}}}] }"#,
)
.expect("should parse");
assert!(duplicate_tasks.validate().is_err(), "duplicate task ids");
let no_tasks =
Workflow::from_json(r#"{ "id": "w", "name": "w", "loop": {"max": 5}, "tasks": [] }"#)
.expect("should parse");
assert!(no_tasks.validate().is_err(), "empty task list");
}
#[test]
fn loop_config_coexists_with_every_other_workflow_field() {
let workflow = Workflow::from_json(&format!(
r#"{{ "id": "w", "name": "w", "priority": 7, "description": "d",
"condition": {{"==": [1, 1]}},
"loop": {{"counter": "i", "max": 5}},
"continue_on_error": true, "channel": "c", "version": 3,
"status": "paused",
"rollout": {{"bucket_start": 0, "bucket_end": 50}},
"tags": ["x"], "tasks": [{MAP}] }}"#
))
.expect("should parse");
workflow.validate().expect("should validate");
assert_eq!(workflow.priority, 7);
assert_eq!(workflow.channel, "c");
assert_eq!(workflow.version, 3);
assert_eq!(workflow.status, WorkflowStatus::Paused);
assert!(workflow.continue_on_error);
assert_eq!(
workflow.rollout,
Some(Rollout {
bucket_start: 0,
bucket_end: 50
})
);
assert_eq!(workflow.tags, ["x"]);
assert_eq!(
workflow
.loop_config
.expect("loop present")
.counter
.as_deref(),
Some("i")
);
}
#[test]
fn loop_config_accepts_a_valid_counter() {
let cfg = loop_wf(r#"{"max": 5, "counter": "cursor.index"}"#)
.expect("valid loop")
.loop_config
.expect("loop config present");
assert_eq!(cfg.counter.as_deref(), Some("cursor.index"));
}
#[test]
fn precompute_counter_path_prefixes_temp_data() {
let mut cfg = loop_wf(r#"{"max": 5, "counter": "cursor.index"}"#)
.expect("valid loop")
.loop_config
.expect("loop config present");
assert!(
cfg.counter_parts.is_empty(),
"uncompiled workflows start with no pre-split path"
);
cfg.precompute_counter_path();
let parts: Vec<&str> = cfg.counter_parts.iter().map(Arc::as_ref).collect();
assert_eq!(parts, ["temp_data", "cursor", "index"]);
}
#[test]
fn precompute_counter_path_is_empty_without_a_counter_name() {
let mut cfg = loop_wf(r#"{"max": 5}"#)
.expect("valid loop")
.loop_config
.expect("loop config present");
cfg.precompute_counter_path();
assert!(cfg.counter_parts.is_empty());
}
#[test]
fn rollout_deserializes_from_json() {
let workflow = Workflow::from_json(
r#"{ "id": "w", "name": "w", "condition": true,
"rollout": { "bucket_start": 0, "bucket_end": 50 },
"tasks": [] }"#,
)
.unwrap();
assert_eq!(
workflow.rollout,
Some(Rollout {
bucket_start: 0,
bucket_end: 50
})
);
}
}