use crate::errors::BntoError;
use crate::metadata::InputCardinality;
use crate::pipeline::{PipelineFile, PipelineNode, is_container_node};
use crate::registry::NodeRegistry;
use super::{ChainPosition, NodeExecutionResult, PipelineContext, run_node_chain};
pub(super) fn run_auto_iteration<F: Fn() -> u64 + Copy>(
ctx: &PipelineContext<F>,
nodes: &[&PipelineNode],
files: Vec<PipelineFile>,
) -> Result<NodeExecutionResult, BntoError> {
let runs = partition_into_runs(nodes, ctx.registry);
let mut current_files = files;
let mut total_processed: usize = 0;
let mut all_warnings: Vec<String> = Vec::new();
for run in &runs {
let result = run_one(ctx, nodes, run, current_files)?;
total_processed += result.files_processed;
all_warnings.extend(result.warnings);
current_files = result.output_files;
}
Ok(NodeExecutionResult {
output_files: current_files,
files_processed: total_processed,
warnings: all_warnings,
})
}
fn run_one<F: Fn() -> u64 + Copy>(
ctx: &PipelineContext<F>,
nodes: &[&PipelineNode],
run: &Run,
files: Vec<PipelineFile>,
) -> Result<NodeExecutionResult, BntoError> {
match run {
Run::PerFileSequence { start, len } => {
let seq = &nodes[*start..*start + *len];
let position = ChainPosition::starting_at(*start, nodes.len());
run_per_file_sequence(ctx, seq, files, position)
}
Run::Container { index } | Run::Batch { index } => {
run_single_node(ctx, nodes, *index, files)
}
Run::Source { index } => run_single_node(ctx, nodes, *index, vec![]),
}
}
fn run_single_node<F: Fn() -> u64 + Copy>(
ctx: &PipelineContext<F>,
nodes: &[&PipelineNode],
index: usize,
files: Vec<PipelineFile>,
) -> Result<NodeExecutionResult, BntoError> {
let node = &nodes[index..index + 1];
let position = ChainPosition::starting_at(index, nodes.len());
run_node_chain(ctx, node, files, 0, position)
}
enum Run {
PerFileSequence { start: usize, len: usize },
Container { index: usize },
Batch { index: usize },
Source { index: usize },
}
fn is_batch_node(node: &PipelineNode, registry: &NodeRegistry) -> bool {
let params = &node.params;
registry
.resolve(&node.node_type, params)
.map(|p| p.metadata().input_cardinality == InputCardinality::Batch)
.unwrap_or(false)
}
fn is_source_node(node: &PipelineNode, registry: &NodeRegistry) -> bool {
let params = &node.params;
registry
.resolve(&node.node_type, params)
.map(|p| p.metadata().input_cardinality == InputCardinality::Source)
.unwrap_or(false)
}
fn partition_into_runs(nodes: &[&PipelineNode], registry: &NodeRegistry) -> Vec<Run> {
let mut runs = Vec::new();
let mut seq_start: Option<usize> = None;
for (i, node) in nodes.iter().enumerate() {
if is_container_node(&node.node_type) {
if let Some(start) = seq_start.take() {
runs.push(Run::PerFileSequence {
start,
len: i - start,
});
}
runs.push(Run::Container { index: i });
} else if is_batch_node(node, registry) {
if let Some(start) = seq_start.take() {
runs.push(Run::PerFileSequence {
start,
len: i - start,
});
}
runs.push(Run::Batch { index: i });
} else if is_source_node(node, registry) {
if let Some(start) = seq_start.take() {
runs.push(Run::PerFileSequence {
start,
len: i - start,
});
}
runs.push(Run::Source { index: i });
} else if seq_start.is_none() {
seq_start = Some(i);
}
}
if let Some(start) = seq_start {
runs.push(Run::PerFileSequence {
start,
len: nodes.len() - start,
});
}
runs
}
fn run_per_file_sequence<F: Fn() -> u64 + Copy>(
ctx: &PipelineContext<F>,
nodes: &[&PipelineNode],
files: Vec<PipelineFile>,
position: ChainPosition,
) -> Result<NodeExecutionResult, BntoError> {
let mut all_output_files: Vec<PipelineFile> = Vec::new();
let mut total_processed: usize = 0;
let mut all_warnings: Vec<String> = Vec::new();
for (i, file) in files.into_iter().enumerate() {
let result = run_node_chain(ctx, nodes, vec![file], i, position)?;
total_processed += result.files_processed;
all_warnings.extend(result.warnings);
all_output_files.extend(result.output_files);
}
Ok(NodeExecutionResult {
files_processed: total_processed,
output_files: all_output_files,
warnings: all_warnings,
})
}