use std::collections::HashMap;
use crate::storage::models::{Channel, Workflow};
use crate::storage::repositories::workflows::{
workflow_to_dataflow, workflow_to_dataflow_with_rollout,
};
pub fn filter_channels(
channels: Vec<Channel>,
config: &crate::config::ChannelFilterConfig,
) -> Vec<Channel> {
if config.include.is_empty() && config.exclude.is_empty() {
return channels;
}
channels
.into_iter()
.filter(|ch| {
if !config.include.is_empty() && !config.include.iter().any(|p| glob_match(p, &ch.name))
{
return false;
}
!config.exclude.iter().any(|p| glob_match(p, &ch.name))
})
.collect()
}
fn glob_match(pattern: &str, name: &str) -> bool {
let p: Vec<char> = pattern.chars().collect();
let n: Vec<char> = name.chars().collect();
let (mut pi, mut ni) = (0usize, 0usize);
let mut star: Option<usize> = None;
let mut mark = 0usize;
while ni < n.len() {
if pi < p.len() && (p[pi] == n[ni]) {
pi += 1;
ni += 1;
} else if pi < p.len() && p[pi] == '*' {
star = Some(pi);
mark = ni;
pi += 1;
} else if let Some(s) = star {
pi = s + 1;
mark += 1;
ni = mark;
} else {
return false;
}
}
while pi < p.len() && p[pi] == '*' {
pi += 1;
}
pi == p.len()
}
pub const CUSTOM_HANDLER_FUNCTIONS: &[&str] = &[
"cache_read",
"cache_write",
"channel_call",
"crypto",
"data_query",
"data_write",
"db_read",
"db_write",
"http_call",
"jwt_sign",
"jwt_verify",
"mongo_aggregate",
"mongo_read",
"mongo_write",
"publish_kafka",
"send_email",
"storage_head",
"storage_presign",
];
pub trait HandlerScreen {
fn check_workflow(&self, workflow: &dataflow_rs::Workflow) -> Vec<dataflow_rs::WorkflowIssue>;
}
impl HandlerScreen for dataflow_rs::Engine {
fn check_workflow(&self, workflow: &dataflow_rs::Workflow) -> Vec<dataflow_rs::WorkflowIssue> {
dataflow_rs::Engine::check_workflow(self, workflow)
}
}
impl HandlerScreen for dataflow_rs::engine::EngineBuilder {
fn check_workflow(&self, workflow: &dataflow_rs::Workflow) -> Vec<dataflow_rs::WorkflowIssue> {
dataflow_rs::engine::EngineBuilder::check_workflow(self, workflow)
}
}
fn screen_workflow(
workflow: &dataflow_rs::Workflow,
screen: &dyn HandlerScreen,
) -> Result<(), String> {
let issues = screen.check_workflow(workflow);
if issues.is_empty() {
return Ok(());
}
Err(issues
.iter()
.map(|issue| match &issue.task_id {
Some(task_id) => format!("task '{task_id}': {}", issue.message),
None => issue.message.clone(),
})
.collect::<Vec<_>>()
.join("; "))
}
pub fn build_single(
workflow: dataflow_rs::Workflow,
handlers: HashMap<String, dataflow_rs::BoxedFunctionHandler>,
secrets: &crate::engine::ResolvedSecrets,
) -> Result<dataflow_rs::Engine, crate::errors::OrionError> {
let builder = crate::engine::operators::with_orion_engine_defaults(
dataflow_rs::Engine::builder(),
secrets,
)
.with_handlers(handlers);
screen_workflow(&workflow, &builder).map_err(|e| {
crate::errors::OrionError::validation(format!(
"workflow '{}' has an unusable task: {e}",
workflow.id
))
})?;
builder
.with_workflow(workflow)
.build()
.map_err(crate::errors::OrionError::Engine)
}
pub fn build_engine_workflows(
channels: &[Channel],
workflows: &[Workflow],
screen: &dyn HandlerScreen,
) -> (
Vec<dataflow_rs::Workflow>,
Vec<crate::channel::ChannelLoadIssue>,
) {
let mut workflow_map: HashMap<String, Vec<&Workflow>> = HashMap::new();
for workflow in workflows {
workflow_map
.entry(workflow.workflow_id.clone())
.or_default()
.push(workflow);
}
let mut result = Vec::new();
let mut issues: Vec<crate::channel::ChannelLoadIssue> = Vec::new();
for channel in channels {
let Some(ref wf_id) = channel.workflow_id else {
issues.push(crate::channel::ChannelLoadIssue {
channel: channel.name.clone(),
reason: "channel has no workflow_id".to_string(),
});
continue;
};
let Some(wf_versions) = workflow_map.get(wf_id) else {
issues.push(crate::channel::ChannelLoadIssue {
channel: channel.name.clone(),
reason: format!("workflow '{wf_id}' not found among active workflows"),
});
continue;
};
if wf_versions.len() == 1 && wf_versions[0].rollout_percentage == 100 {
match workflow_to_dataflow(wf_versions[0], &channel.name) {
Ok(w) => match screen_workflow(&w, screen) {
Ok(()) => result.push(w),
Err(e) => {
issues.push(crate::channel::ChannelLoadIssue {
channel: channel.name.clone(),
reason: format!("workflow '{wf_id}' has an unusable task: {e}"),
});
}
},
Err(e) => {
issues.push(crate::channel::ChannelLoadIssue {
channel: channel.name.clone(),
reason: format!("workflow '{wf_id}' failed to convert: {e}"),
});
}
}
} else {
let mut sorted: Vec<&&Workflow> = wf_versions.iter().collect();
sorted.sort_by_key(|b| std::cmp::Reverse(b.version));
let percentages: Vec<u8> = sorted
.iter()
.map(|wf| u8::try_from(wf.rollout_percentage).unwrap_or(u8::MAX))
.collect();
match dataflow_rs::Rollout::partition(&percentages) {
Ok(ranges) => {
let mut converted = Vec::new();
let mut failed = false;
for (wf, rollout) in sorted.iter().zip(ranges) {
match workflow_to_dataflow_with_rollout(wf, &channel.name, rollout)
.map_err(|e| e.to_string())
.and_then(|w| screen_workflow(&w, screen).map(|()| w))
{
Ok(w) => converted.push(w),
Err(e) => {
issues.push(crate::channel::ChannelLoadIssue {
channel: channel.name.clone(),
reason: format!(
"workflow '{}' v{} failed to convert: {e}",
wf.workflow_id, wf.version
),
});
failed = true;
break;
}
}
}
if !failed {
result.append(&mut converted);
}
}
Err(e) => issues.push(crate::channel::ChannelLoadIssue {
channel: channel.name.clone(),
reason: format!(
"rollout percentages for workflow '{wf_id}' do not partition the \
bucket space: {e}"
),
}),
}
}
}
(result, issues)
}
#[cfg(test)]
mod tests {
use super::*;
fn screen() -> dataflow_rs::engine::EngineBuilder {
dataflow_rs::Engine::builder()
}
#[test]
fn test_glob_match_exact() {
assert!(glob_match("orders", "orders"));
assert!(!glob_match("orders", "events"));
}
#[test]
fn test_glob_match_prefix_wildcard() {
assert!(glob_match("internal-*", "internal-debug"));
assert!(glob_match("internal-*", "internal-"));
assert!(!glob_match("internal-*", "external-debug"));
}
#[test]
fn test_glob_match_suffix_wildcard() {
assert!(glob_match("*-debug", "internal-debug"));
assert!(!glob_match("*-debug", "internal-prod"));
}
#[test]
fn test_glob_match_star_only() {
assert!(glob_match("*", "anything"));
assert!(glob_match("*", ""));
}
#[test]
fn test_glob_match_middle_wildcard() {
assert!(glob_match("pre*suf", "presuf"));
assert!(glob_match("pre*suf", "pre-middle-suf"));
assert!(!glob_match("pre*suf", "pre-middle"));
}
#[test]
fn test_glob_match_backtracking() {
assert!(glob_match("a*bc", "abxbc"));
assert!(glob_match("a*bc", "abcbc"));
assert!(!glob_match("a*bc", "abxbd"));
}
#[test]
fn test_glob_match_multi_star() {
assert!(glob_match("a*b*c", "a-x-b-y-c"));
assert!(glob_match("*orders*", "internal-orders-debug"));
assert!(!glob_match("a*b*c", "a-x-c-y-b"));
assert!(glob_match("**", "anything"));
}
fn make_channel(name: &str) -> Channel {
Channel {
tags_json: "[]".to_string(),
channel_id: name.to_string(),
name: name.to_string(),
version: 1,
status: crate::storage::models::EntityStatus::Active
.as_str()
.to_string(),
channel_type: "sync".to_string(),
protocol: crate::storage::models::ChannelProtocol::Http
.as_str()
.to_string(),
methods_json: Some("POST".to_string()),
workflow_id: None,
topic: None,
consumer_group: None,
route_pattern: None,
description: None,
transport_config_json: "{}".to_string(),
config_json: "{}".to_string(),
priority: 0,
created_at: chrono::NaiveDateTime::default(),
updated_at: chrono::NaiveDateTime::default(),
}
}
fn make_workflow(wf_id: &str, version: i64, rollout: i64) -> Workflow {
Workflow {
workflow_id: wf_id.to_string(),
version,
name: format!("{wf_id}-v{version}"),
description: None,
priority: 0,
status: "active".to_string(),
rollout_percentage: rollout,
condition_json: "true".to_string(),
tasks_json:
r#"[{"id":"t1","name":"log","function":{"name":"log","input":{"message":"x"}}}]"#
.to_string(),
tags_json: "[]".to_string(),
loop_json: None,
continue_on_error: false,
created_at: chrono::NaiveDateTime::default(),
updated_at: chrono::NaiveDateTime::default(),
}
}
#[test]
fn test_rollout_sum_must_be_100() {
let mut channel = make_channel("rollout-ch");
channel.workflow_id = Some("wf".to_string());
let wfs = vec![make_workflow("wf", 1, 30), make_workflow("wf", 2, 50)];
let (converted, issues) = build_engine_workflows(&[channel.clone()], &wfs, &screen());
assert!(converted.is_empty(), "under-100 rollout must not half-load");
assert_eq!(issues.len(), 1);
assert!(
issues[0].reason.contains("sum to 80"),
"{}",
issues[0].reason
);
let wfs = vec![make_workflow("wf", 1, 60), make_workflow("wf", 2, 60)];
let (converted, issues) = build_engine_workflows(&[channel.clone()], &wfs, &screen());
assert!(converted.is_empty());
assert_eq!(issues.len(), 1);
assert!(
issues[0].reason.contains("sum to 120"),
"{}",
issues[0].reason
);
let wfs = vec![make_workflow("wf", 1, 50), make_workflow("wf", 2, 50)];
let (converted, issues) = build_engine_workflows(&[channel], &wfs, &screen());
assert_eq!(converted.len(), 2);
assert!(issues.is_empty(), "{issues:?}");
}
fn bucket_range(wf: &dataflow_rs::Workflow) -> (u8, u8) {
let rollout = wf
.rollout
.expect("converted rollout workflow carries a range");
(rollout.bucket_start, rollout.bucket_end)
}
#[test]
fn test_rollout_bucket_offsets_partition_newest_first() {
let mut channel = make_channel("rollout-ch");
channel.workflow_id = Some("wf".to_string());
let wfs = vec![
make_workflow("wf", 1, 30),
make_workflow("wf", 2, 50),
make_workflow("wf", 3, 20),
];
let (converted, issues) = build_engine_workflows(&[channel], &wfs, &screen());
assert!(issues.is_empty(), "{issues:?}");
assert_eq!(converted.len(), 3);
let by_id: std::collections::HashMap<String, (u8, u8)> = converted
.iter()
.map(|w| (w.id.clone(), bucket_range(w)))
.collect();
assert_eq!(
by_id["wf:v3"],
(0, 20),
"newest version gets the first bucket"
);
assert_eq!(by_id["wf:v2"], (20, 70));
assert_eq!(by_id["wf:v1"], (70, 100));
for bucket in 0u8..100 {
let serving: Vec<&str> = converted
.iter()
.filter(|w| w.rollout.is_some_and(|r| r.accepts(bucket)))
.map(|w| w.id.as_str())
.collect();
assert_eq!(
serving.len(),
1,
"bucket {bucket} is served by {serving:?}, not exactly one version"
);
}
}
#[test]
fn test_missing_and_unknown_workflow_are_reported_as_issues() {
let no_wf = make_channel("no-wf");
let mut unknown = make_channel("unknown-wf");
unknown.workflow_id = Some("ghost".to_string());
let (converted, issues) = build_engine_workflows(&[no_wf, unknown], &[], &screen());
assert!(converted.is_empty());
assert_eq!(issues.len(), 2);
assert!(
issues[0].reason.contains("no workflow_id"),
"{}",
issues[0].reason
);
assert!(
issues[1].reason.contains("'ghost' not found"),
"{}",
issues[1].reason
);
}
#[test]
fn test_partial_rollout_conversion_failure_loads_nothing() {
let mut channel = make_channel("rollout-ch");
channel.workflow_id = Some("wf".to_string());
let mut bad_v1 = make_workflow("wf", 1, 50);
bad_v1.tasks_json = "not json".to_string();
let wfs = vec![bad_v1, make_workflow("wf", 2, 50)];
let (converted, issues) = build_engine_workflows(&[channel], &wfs, &screen());
assert!(
converted.is_empty(),
"the successfully-converted v2 must be discarded with v1"
);
assert_eq!(issues.len(), 1);
assert!(
issues[0].reason.contains("v1 failed to convert"),
"{}",
issues[0].reason
);
}
#[test]
fn test_filter_channels_no_config() {
let channels = vec![make_channel("orders"), make_channel("events")];
let config = crate::config::ChannelFilterConfig::default();
let filtered = filter_channels(channels, &config);
assert_eq!(filtered.len(), 2);
}
#[test]
fn test_filter_channels_include_only() {
let channels = vec![
make_channel("orders"),
make_channel("events"),
make_channel("internal-debug"),
];
let config = crate::config::ChannelFilterConfig {
include: vec!["orders".to_string(), "events".to_string()],
exclude: vec![],
};
let filtered = filter_channels(channels, &config);
assert_eq!(filtered.len(), 2);
assert!(filtered.iter().all(|c| c.name != "internal-debug"));
}
#[test]
fn test_filter_channels_exclude_only() {
let channels = vec![
make_channel("orders"),
make_channel("events"),
make_channel("internal-debug"),
];
let config = crate::config::ChannelFilterConfig {
include: vec![],
exclude: vec!["internal-*".to_string()],
};
let filtered = filter_channels(channels, &config);
assert_eq!(filtered.len(), 2);
assert!(filtered.iter().all(|c| c.name != "internal-debug"));
}
#[test]
fn test_filter_channels_include_and_exclude() {
let channels = vec![
make_channel("orders"),
make_channel("orders-debug"),
make_channel("events"),
];
let config = crate::config::ChannelFilterConfig {
include: vec!["orders*".to_string()],
exclude: vec!["*-debug".to_string()],
};
let filtered = filter_channels(channels, &config);
assert_eq!(filtered.len(), 1);
assert_eq!(filtered[0].name, "orders");
}
fn workflow_with_raw_tasks(wf_id: &str, tasks_json: &str) -> Workflow {
let mut wf = make_workflow(wf_id, 1, 100);
wf.tasks_json = tasks_json.to_string();
wf
}
#[test]
fn unregistered_custom_function_quarantines_only_its_channel() {
let mut good = make_channel("good");
good.workflow_id = Some("wf-good".to_string());
let mut bad = make_channel("bad");
bad.workflow_id = Some("wf-bad".to_string());
let wfs = vec![
make_workflow("wf-good", 1, 100),
workflow_with_raw_tasks(
"wf-bad",
r#"[{"id":"t1","name":"oops","function":{"name":"totally_not_a_function",
"input":{}}}]"#,
),
];
let (converted, issues) = build_engine_workflows(&[good, bad], &wfs, &screen());
assert_eq!(
converted.len(),
1,
"the healthy channel must still be built"
);
assert_eq!(converted[0].channel, "good");
assert_eq!(issues.len(), 1, "issues = {issues:?}");
assert_eq!(issues[0].channel, "bad");
assert!(
issues[0].reason.contains("'totally_not_a_function'")
&& issues[0].reason.contains("task 't1'"),
"reason should name the offending task and function: {}",
issues[0].reason
);
}
fn one_task_workflow(function: &str, input: serde_json::Value) -> dataflow_rs::Workflow {
serde_json::from_value(serde_json::json!({
"id": "wf-1",
"name": "wf-1",
"condition": true,
"tasks": [{
"id": "t1",
"name": "t1",
"function": { "name": function, "input": input }
}],
}))
.expect("a well-formed workflow")
}
#[test]
fn build_single_refuses_a_built_in_with_no_handler_behind_it() {
let Err(err) = build_single(
one_task_workflow(
"enrich",
serde_json::json!({ "connector": "c", "merge_path": "data" }),
),
std::collections::HashMap::new(),
&crate::engine::ResolvedSecrets::empty(),
) else {
unreachable!(
"`enrich` has no handler in any Orion engine — building it is \
the bug this screen exists to catch"
);
};
let msg = err.to_string();
assert!(
msg.contains("wf-1") && msg.contains("t1"),
"the refusal must name the workflow and the task: {msg}"
);
assert!(
matches!(err, crate::errors::OrionError::Validation { .. }),
"an unusable task is the author's document to fix, not a server \
fault: {err:?}"
);
}
#[test]
fn build_single_builds_a_workflow_every_engine_can_run() {
assert!(
build_single(
one_task_workflow("log", serde_json::json!({ "message": "x" })),
std::collections::HashMap::new(),
&crate::engine::ResolvedSecrets::empty(),
)
.is_ok(),
"`log` is self-contained — every engine dispatches it"
);
}
}