use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use tokio::sync::RwLock;
use crate::event::StreamEvent;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VisualDesignerConfig {
pub enable_auto_layout: bool,
pub enable_validation: bool,
pub enable_optimization: bool,
pub max_nodes: usize,
pub max_edges: usize,
pub enable_real_time_debug: bool,
pub debug_buffer_size: usize,
pub enable_profiling: bool,
pub export_formats: Vec<ExportFormat>,
}
impl Default for VisualDesignerConfig {
fn default() -> Self {
Self {
enable_auto_layout: true,
enable_validation: true,
enable_optimization: true,
max_nodes: 1000,
max_edges: 5000,
enable_real_time_debug: true,
debug_buffer_size: 10000,
enable_profiling: true,
export_formats: vec![
ExportFormat::Json,
ExportFormat::Yaml,
ExportFormat::Dot,
ExportFormat::Mermaid,
],
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum ExportFormat {
Json,
Yaml,
Dot, Mermaid, Svg, Png, }
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineNode {
pub id: String,
pub name: String,
pub node_type: NodeType,
pub position: Position,
pub config: NodeConfig,
pub metadata: NodeMetadata,
pub status: NodeStatus,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum NodeType {
Source(SourceType),
Map,
Filter,
FlatMap,
Reduce,
Aggregate,
Join,
Window,
Transform(TransformType),
MLModel(MLModelType),
Sink(SinkType),
Router,
Splitter,
Merger,
Breakpoint,
Logger,
Profiler,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum SourceType {
Kafka,
Nats,
Redis,
Memory,
File,
WebSocket,
Http,
Custom(String),
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum TransformType {
RdfTransform,
SparqlQuery,
GraphPattern,
Custom(String),
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum MLModelType {
OnlineLearning,
AnomalyDetection,
Prediction,
Classification,
Clustering,
Custom(String),
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum SinkType {
Kafka,
Nats,
Redis,
Memory,
Database,
File,
WebSocket,
Http,
Custom(String),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Position {
pub x: f64,
pub y: f64,
pub z: Option<f64>, }
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NodeConfig {
pub parameters: HashMap<String, ConfigValue>,
pub input_ports: Vec<Port>,
pub output_ports: Vec<Port>,
pub resource_limits: ResourceLimits,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum ConfigValue {
String(String),
Number(f64),
Boolean(bool),
Array(Vec<ConfigValue>),
Object(HashMap<String, ConfigValue>),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Port {
pub id: String,
pub name: String,
pub port_type: PortType,
pub data_type: DataType,
pub required: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum PortType {
Input,
Output,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum DataType {
StreamEvent,
RdfTriple,
SparqlResult,
Json,
Binary,
Custom(String),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResourceLimits {
pub max_memory_mb: Option<u64>,
pub max_cpu_percent: Option<f64>,
pub max_execution_time_ms: Option<u64>,
pub max_events_per_second: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NodeMetadata {
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub version: String,
pub author: Option<String>,
pub description: Option<String>,
pub tags: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum NodeStatus {
Idle,
Running,
Paused,
Error(String),
Completed,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineEdge {
pub id: String,
pub source_node_id: String,
pub source_port_id: String,
pub target_node_id: String,
pub target_port_id: String,
pub edge_type: EdgeType,
pub config: EdgeConfig,
pub metadata: EdgeMetadata,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum EdgeType {
DataFlow,
ControlFlow,
Conditional(Condition),
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Condition {
pub expression: String,
pub predicate_type: PredicateType,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum PredicateType {
Equals,
NotEquals,
GreaterThan,
LessThan,
Contains,
Matches,
Custom(String),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EdgeConfig {
pub buffer_size: usize,
pub backpressure_strategy: BackpressureStrategy,
pub error_handling: ErrorHandling,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum BackpressureStrategy {
Drop,
Buffer,
Block,
Exponential,
Adaptive,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ErrorHandling {
Propagate,
Ignore,
Retry { max_attempts: u32 },
DeadLetter,
Custom(String),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EdgeMetadata {
pub created_at: DateTime<Utc>,
pub label: Option<String>,
pub style: EdgeStyle,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EdgeStyle {
pub color: String,
pub thickness: f64,
pub line_type: LineType,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum LineType {
Solid,
Dashed,
Dotted,
Curved,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VisualPipeline {
pub id: String,
pub name: String,
pub description: Option<String>,
pub version: String,
pub nodes: HashMap<String, PipelineNode>,
pub edges: HashMap<String, PipelineEdge>,
pub metadata: PipelineMetadata,
pub validation_result: Option<ValidationResult>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineMetadata {
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub author: Option<String>,
pub tags: Vec<String>,
pub properties: HashMap<String, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationResult {
pub is_valid: bool,
pub errors: Vec<ValidationError>,
pub warnings: Vec<ValidationWarning>,
pub validated_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationError {
pub error_type: ValidationErrorType,
pub message: String,
pub node_id: Option<String>,
pub edge_id: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum ValidationErrorType {
MissingRequiredPort,
IncompatibleDataTypes,
CyclicDependency,
DisconnectedNode,
InvalidConfiguration,
ResourceLimitExceeded,
DuplicateNodeId,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ValidationWarning {
pub warning_type: ValidationWarningType,
pub message: String,
pub node_id: Option<String>,
pub suggestion: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum ValidationWarningType {
UnusedPort,
SuboptimalConfiguration,
PerformanceBottleneck,
MemoryPressure,
DeprecatedNode,
}
#[derive(Debug)]
pub struct PipelineDebugger {
pub pipeline: VisualPipeline,
pub config: DebuggerConfig,
pub state: Arc<RwLock<DebuggerState>>,
pub breakpoints: Arc<RwLock<HashMap<String, Breakpoint>>>,
pub event_history: Arc<RwLock<VecDeque<DebugEvent>>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DebuggerConfig {
pub enable_breakpoints: bool,
pub enable_event_capture: bool,
pub max_event_history: usize,
pub enable_time_travel: bool,
pub enable_profiling: bool,
pub capture_intermediate_results: bool,
}
impl Default for DebuggerConfig {
fn default() -> Self {
Self {
enable_breakpoints: true,
enable_event_capture: true,
max_event_history: 10000,
enable_time_travel: true,
enable_profiling: true,
capture_intermediate_results: true,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DebuggerState {
pub is_running: bool,
pub is_paused: bool,
pub current_node_id: Option<String>,
pub execution_stack: Vec<String>,
pub variables: HashMap<String, DebugVariable>,
pub metrics: DebugMetrics,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DebugVariable {
pub name: String,
pub value: String,
pub var_type: String,
pub scope: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct DebugMetrics {
pub events_processed: u64,
pub events_dropped: u64,
pub average_latency_ms: f64,
pub throughput_per_second: f64,
pub memory_usage_mb: f64,
pub cpu_usage_percent: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Breakpoint {
pub id: String,
pub node_id: String,
pub condition: Option<String>,
pub enabled: bool,
pub hit_count: u64,
pub max_hits: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DebugEvent {
pub timestamp: DateTime<Utc>,
pub node_id: String,
pub event_type: DebugEventType,
pub data: StreamEvent,
pub metadata: HashMap<String, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum DebugEventType {
NodeEnter,
NodeExit,
BreakpointHit,
Error,
Warning,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineInfo {
pub id: String,
pub name: String,
pub version: String,
pub node_count: usize,
pub edge_count: usize,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OptimizationResult {
pub original_metrics: PipelineMetrics,
pub suggestions: Vec<OptimizationSuggestion>,
pub optimized_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineMetrics {
pub node_count: usize,
pub edge_count: usize,
pub avg_chain_length: f64,
pub max_chain_length: usize,
pub parallel_opportunities: usize,
pub bottleneck_nodes: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OptimizationSuggestion {
pub suggestion_type: OptimizationType,
pub impact: ImpactLevel,
pub description: String,
pub estimated_improvement: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum OptimizationType {
ReduceChainLength,
IncreaseParallelism,
OptimizeBufferSize,
ReduceMemoryUsage,
ImproveLocality,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum ImpactLevel {
Low,
Medium,
High,
Critical,
}
pub struct PipelineValidator {
pub(crate) config: VisualDesignerConfig,
}
pub struct PipelineOptimizer {
pub(crate) config: VisualDesignerConfig,
}