rs3gw 0.2.1

High-Performance AI/HPC Object Storage Gateway powered by scirs2-io
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

//! Preprocessing pipeline definition and builder.

use super::error::{PreprocessingError, PreprocessingResult};
use super::image::PreprocessingStepType;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;

/// A single preprocessing step in a pipeline
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PreprocessingStep {
    /// Unique identifier for this step
    pub id: String,
    /// Step type
    pub step_type: PreprocessingStepType,
    /// Configuration (JSON-serialized)
    pub config: serde_json::Value,
    /// Whether to cache results of this step
    pub cache_results: bool,
    /// Step description
    pub description: Option<String>,
}

impl PreprocessingStep {
    pub fn new(id: String, step_type: PreprocessingStepType) -> Self {
        Self {
            id,
            step_type,
            config: serde_json::Value::Null,
            cache_results: false,
            description: None,
        }
    }

    pub fn with_config<T: Serialize>(mut self, config: T) -> PreprocessingResult<Self> {
        self.config = serde_json::to_value(config)
            .map_err(|e| PreprocessingError::SerializationError(e.to_string()))?;
        Ok(self)
    }

    pub fn with_cache(mut self, cache: bool) -> Self {
        self.cache_results = cache;
        self
    }

    pub fn with_description(mut self, desc: String) -> Self {
        self.description = Some(desc);
        self
    }
}

/// Preprocessing pipeline definition
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineDefinition {
    /// Unique pipeline identifier
    pub id: String,
    /// Pipeline name
    pub name: String,
    /// Pipeline version
    pub version: String,
    /// Pipeline description
    pub description: Option<String>,
    /// List of preprocessing steps (executed in order)
    pub steps: Vec<PreprocessingStep>,
    /// Pipeline metadata
    pub metadata: HashMap<String, String>,
    /// Creation timestamp
    pub created_at: DateTime<Utc>,
    /// Last modified timestamp
    pub modified_at: DateTime<Utc>,
}

impl PipelineDefinition {
    pub fn new(id: String, name: String, version: String) -> Self {
        let now = Utc::now();
        Self {
            id,
            name,
            version,
            description: None,
            steps: Vec::new(),
            metadata: HashMap::new(),
            created_at: now,
            modified_at: now,
        }
    }

    pub fn add_step(&mut self, step: PreprocessingStep) {
        self.steps.push(step);
        self.modified_at = Utc::now();
    }

    pub fn with_description(mut self, desc: String) -> Self {
        self.description = Some(desc);
        self
    }

    pub fn add_metadata(&mut self, key: String, value: String) {
        self.metadata.insert(key, value);
        self.modified_at = Utc::now();
    }
}

/// Builder for preprocessing pipelines
pub struct PipelineBuilder {
    definition: PipelineDefinition,
}

impl PipelineBuilder {
    pub fn new(id: String, name: String) -> Self {
        Self {
            definition: PipelineDefinition::new(id, name, "1.0.0".to_string()),
        }
    }

    pub fn version(mut self, version: String) -> Self {
        self.definition.version = version;
        self
    }

    pub fn description(mut self, desc: String) -> Self {
        self.definition.description = Some(desc);
        self
    }

    pub fn add_step(mut self, step: PreprocessingStep) -> Self {
        self.definition.add_step(step);
        self
    }

    pub fn metadata(mut self, key: String, value: String) -> Self {
        self.definition.add_metadata(key, value);
        self
    }

    pub fn build(self) -> PipelineDefinition {
        self.definition
    }
}