helix_core/compiler/
serializer.rs

1use crate::codegen::HelixIR;
2use crate::compiler::binary::{
3    HelixBinary, BinaryFlags, BinaryMetadata, DataSection, SectionType, SymbolTable,
4    Instruction, Value, CompressionMethod,
5};
6use std::path::Path;
7use std::fs::File;
8use std::io::{Write, Read};
9use bincode;
10pub struct BinarySerializer {
11    enable_compression: bool,
12    compression_method: CompressionMethod,
13}
14impl BinarySerializer {
15    pub fn new(enable_compression: bool) -> Self {
16        Self {
17            enable_compression,
18            compression_method: CompressionMethod::Lz4,
19        }
20    }
21    pub fn with_compression_method(mut self, method: CompressionMethod) -> Self {
22        self.compression_method = method;
23        self
24    }
25    pub fn serialize(
26        &self,
27        ir: HelixIR,
28        source_path: Option<&Path>,
29    ) -> Result<HelixBinary, SerializationError> {
30        let mut binary = HelixBinary::new();
31        binary.metadata = BinaryMetadata {
32            created_at: std::time::SystemTime::now()
33                .duration_since(std::time::UNIX_EPOCH)
34                .unwrap()
35                .as_secs(),
36            compiler_version: env!("CARGO_PKG_VERSION").to_string(),
37            source_hash: self.calculate_source_hash(&ir),
38            optimization_level: 2,
39            platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
40            source_path: source_path.map(|p| p.display().to_string()),
41            extra: Default::default(),
42        };
43        binary.flags = BinaryFlags {
44            compressed: self.enable_compression,
45            optimized: true,
46            encrypted: false,
47            signed: false,
48            custom: 0,
49        };
50        binary.symbol_table = self.convert_symbol_table(&ir);
51        binary.data_sections = self.create_data_sections(&ir)?;
52        if self.enable_compression {
53            for section in &mut binary.data_sections {
54                section.compress(self.compression_method.clone())?;
55            }
56        }
57        binary.checksum = binary.calculate_checksum();
58        Ok(binary)
59    }
60    pub fn write_to_file(
61        &self,
62        binary: &HelixBinary,
63        path: &Path,
64    ) -> Result<(), SerializationError> {
65        let data = bincode::serialize(binary)
66            .map_err(|e| SerializationError::BincodeError(e.to_string()))?;
67        let mut file = File::create(path)
68            .map_err(|e| SerializationError::IoError(e.to_string()))?;
69        file.write_all(&data).map_err(|e| SerializationError::IoError(e.to_string()))?;
70        Ok(())
71    }
72    pub fn read_from_file(
73        &self,
74        path: &Path,
75    ) -> Result<HelixBinary, SerializationError> {
76        let mut file = File::open(path)
77            .map_err(|e| SerializationError::IoError(e.to_string()))?;
78        let mut data = Vec::new();
79        file.read_to_end(&mut data)
80            .map_err(|e| SerializationError::IoError(e.to_string()))?;
81        let binary: HelixBinary = bincode::deserialize(&data)
82            .map_err(|e| SerializationError::BincodeError(e.to_string()))?;
83        binary.validate().map_err(|e| SerializationError::ValidationError(e))?;
84        Ok(binary)
85    }
86    pub fn deserialize_to_ir(
87        &self,
88        binary: &HelixBinary,
89    ) -> Result<HelixIR, SerializationError> {
90        let mut ir = HelixIR {
91            version: binary.version,
92            metadata: self.convert_metadata(&binary.metadata),
93            symbol_table: self.convert_symbol_table_to_ir(&binary.symbol_table),
94            instructions: Vec::new(),
95            string_pool: crate::codegen::StringPool {
96                strings: binary.symbol_table.strings.clone(),
97                index: binary.symbol_table.string_map.clone(),
98            },
99            constants: crate::codegen::ConstantPool::new(),
100        };
101        for section in &binary.data_sections {
102            let mut section_clone = section.clone();
103            if section.compression.is_some() {
104                section_clone
105                    .decompress()
106                    .map_err(|e| SerializationError::DecompressionError(e))?;
107            }
108            match section.section_type {
109                SectionType::Instructions => {
110                    ir.instructions = self
111                        .deserialize_instructions(&section_clone.data)?;
112                }
113                _ => {}
114            }
115        }
116        Ok(ir)
117    }
118    fn convert_symbol_table(&self, ir: &HelixIR) -> SymbolTable {
119        let mut table = SymbolTable::default();
120        table.strings = ir.string_pool.strings.clone();
121        for (i, s) in table.strings.iter().enumerate() {
122            table.string_map.insert(s.clone(), i as u32);
123        }
124        for (id, agent) in &ir.symbol_table.agents {
125            if let Some(name) = ir.string_pool.get(agent.name_idx) {
126                table.agents.insert(name.clone(), *id);
127            }
128        }
129        for (id, workflow) in &ir.symbol_table.workflows {
130            if let Some(name) = ir.string_pool.get(workflow.name_idx) {
131                table.workflows.insert(name.clone(), *id);
132            }
133        }
134        for (id, context) in &ir.symbol_table.contexts {
135            if let Some(name) = ir.string_pool.get(context.name_idx) {
136                table.contexts.insert(name.clone(), *id);
137            }
138        }
139        for (id, crew) in &ir.symbol_table.crews {
140            if let Some(name) = ir.string_pool.get(crew.name_idx) {
141                table.crews.insert(name.clone(), *id);
142            }
143        }
144        table
145    }
146    fn convert_symbol_table_to_ir(
147        &self,
148        table: &SymbolTable,
149    ) -> crate::codegen::SymbolTable {
150        use crate::codegen::{AgentSymbol, WorkflowSymbol, ContextSymbol, CrewSymbol};
151        use std::collections::HashMap;
152        let mut symbol_table = crate::codegen::SymbolTable::default();
153        for (name, id) in &table.agents {
154            let name_idx = table.string_map.get(name).copied().unwrap_or(0);
155            symbol_table
156                .agents
157                .insert(
158                    *id,
159                    AgentSymbol {
160                        id: *id,
161                        name_idx,
162                        model_idx: 0,
163                        role_idx: 0,
164                        temperature: None,
165                        max_tokens: None,
166                        capabilities: Vec::new(),
167                        backstory_idx: None,
168                    },
169                );
170        }
171        for (name, id) in &table.workflows {
172            let name_idx = table.string_map.get(name).copied().unwrap_or(0);
173            symbol_table
174                .workflows
175                .insert(
176                    *id,
177                    WorkflowSymbol {
178                        id: *id,
179                        name_idx,
180                        trigger_type: crate::codegen::TriggerType::Manual,
181                        steps: Vec::new(),
182                        pipeline: None,
183                    },
184                );
185        }
186        for (name, id) in &table.contexts {
187            let name_idx = table.string_map.get(name).copied().unwrap_or(0);
188            symbol_table
189                .contexts
190                .insert(
191                    *id,
192                    ContextSymbol {
193                        id: *id,
194                        name_idx,
195                        environment_idx: 0,
196                        debug: false,
197                        max_tokens: None,
198                        secrets: HashMap::new(),
199                    },
200                );
201        }
202        for (name, id) in &table.crews {
203            let name_idx = table.string_map.get(name).copied().unwrap_or(0);
204            symbol_table
205                .crews
206                .insert(
207                    *id,
208                    CrewSymbol {
209                        id: *id,
210                        name_idx,
211                        agent_ids: Vec::new(),
212                        process_type: crate::codegen::ProcessTypeIR::Sequential,
213                        manager_id: None,
214                    },
215                );
216        }
217        symbol_table
218    }
219    fn convert_metadata(&self, metadata: &BinaryMetadata) -> crate::codegen::Metadata {
220        crate::codegen::Metadata {
221            source_file: metadata.source_path.clone(),
222            compile_time: metadata.created_at,
223            compiler_version: metadata.compiler_version.clone(),
224            checksum: None,
225        }
226    }
227    fn create_data_sections(
228        &self,
229        ir: &HelixIR,
230    ) -> Result<Vec<DataSection>, SerializationError> {
231        let mut sections = Vec::new();
232        if !ir.instructions.is_empty() {
233            let instruction_data = self.serialize_instructions(&ir.instructions)?;
234            sections.push(DataSection::new(SectionType::Instructions, instruction_data));
235        }
236        if !ir.symbol_table.agents.is_empty() {
237            let agent_data = bincode::serialize(&ir.symbol_table.agents)
238                .map_err(|e| SerializationError::BincodeError(e.to_string()))?;
239            sections.push(DataSection::new(SectionType::Agents, agent_data));
240        }
241        if !ir.symbol_table.workflows.is_empty() {
242            let workflow_data = bincode::serialize(&ir.symbol_table.workflows)
243                .map_err(|e| SerializationError::BincodeError(e.to_string()))?;
244            sections.push(DataSection::new(SectionType::Workflows, workflow_data));
245        }
246        Ok(sections)
247    }
248    fn serialize_instructions(
249        &self,
250        instructions: &[crate::codegen::Instruction],
251    ) -> Result<Vec<u8>, SerializationError> {
252        let binary_instructions: Vec<Instruction> = instructions
253            .iter()
254            .map(|inst| self.convert_instruction(inst))
255            .collect();
256        bincode::serialize(&binary_instructions)
257            .map_err(|e| SerializationError::BincodeError(e.to_string()))
258    }
259    fn deserialize_instructions(
260        &self,
261        data: &[u8],
262    ) -> Result<Vec<crate::codegen::Instruction>, SerializationError> {
263        let binary_instructions: Vec<Instruction> = bincode::deserialize(data)
264            .map_err(|e| SerializationError::BincodeError(e.to_string()))?;
265        Ok(
266            binary_instructions
267                .iter()
268                .map(|inst| self.convert_instruction_to_ir(inst))
269                .collect(),
270        )
271    }
272    fn convert_instruction(&self, inst: &crate::codegen::Instruction) -> Instruction {
273        match inst {
274            crate::codegen::Instruction::DeclareAgent(id) => {
275                Instruction::InvokeAgent(*id)
276            }
277            crate::codegen::Instruction::DeclareWorkflow(_id) => Instruction::Nop,
278            crate::codegen::Instruction::DeclareContext(_id) => Instruction::Nop,
279            crate::codegen::Instruction::DeclareCrew(id) => Instruction::InvokeCrew(*id),
280            crate::codegen::Instruction::SetProperty { .. } => Instruction::Nop,
281            crate::codegen::Instruction::SetCapability { .. } => Instruction::Nop,
282            crate::codegen::Instruction::SetSecret { .. } => Instruction::Nop,
283            crate::codegen::Instruction::DefineStep { .. } => Instruction::Nop,
284            crate::codegen::Instruction::DefinePipeline { workflow, .. } => {
285                Instruction::Pipeline(*workflow)
286            }
287            crate::codegen::Instruction::ResolveReference { .. } => Instruction::Nop,
288            crate::codegen::Instruction::SetMetadata { .. } => Instruction::Nop,
289        }
290    }
291    fn convert_instruction_to_ir(
292        &self,
293        inst: &Instruction,
294    ) -> crate::codegen::Instruction {
295        match inst {
296            Instruction::InvokeAgent(id) => {
297                crate::codegen::Instruction::DeclareAgent(*id)
298            }
299            Instruction::InvokeCrew(id) => crate::codegen::Instruction::DeclareCrew(*id),
300            Instruction::Pipeline(id) => {
301                crate::codegen::Instruction::DeclareWorkflow(*id)
302            }
303            _ => crate::codegen::Instruction::DeclareAgent(0),
304        }
305    }
306    #[allow(dead_code)]
307    fn convert_value(&self, val: &crate::types::Value) -> Value {
308        match val {
309            crate::types::Value::Bool(b) => Value::Bool(*b),
310            crate::types::Value::Number(n) => Value::Float(*n),
311            crate::types::Value::String(_s) => {
312                let id = 0;
313                Value::String(id)
314            }
315            crate::types::Value::Duration(d) => {
316                let secs = match &d.unit {
317                    crate::types::TimeUnit::Seconds => d.value,
318                    crate::types::TimeUnit::Minutes => d.value * 60,
319                    crate::types::TimeUnit::Hours => d.value * 3600,
320                    crate::types::TimeUnit::Days => d.value * 86400,
321                } as u64;
322                Value::Duration(secs)
323            }
324            crate::types::Value::Array(_) => Value::Null,
325            crate::types::Value::Object(_) => Value::Null,
326            crate::types::Value::Reference(_) => Value::Null,
327        }
328    }
329    #[allow(dead_code)]
330    fn convert_value_to_ir(&self, val: &Value) -> crate::types::Value {
331        match val {
332            Value::Null => crate::types::Value::String(String::new()),
333            Value::Bool(b) => crate::types::Value::Bool(*b),
334            Value::Int(i) => crate::types::Value::Number(*i as f64),
335            Value::Float(f) => crate::types::Value::Number(*f),
336            Value::String(_id) => crate::types::Value::String(String::new()),
337            Value::Duration(secs) => {
338                crate::types::Value::Duration(crate::types::Duration {
339                    value: (*secs / 60) as u64,
340                    unit: crate::types::TimeUnit::Minutes,
341                })
342            }
343            Value::Reference(_id) => crate::types::Value::Reference(String::new()),
344            Value::Array(arr) => {
345                crate::types::Value::Array(
346                    arr.iter().map(|v| self.convert_value_to_ir(v)).collect(),
347                )
348            }
349            Value::Object(obj) => {
350                let mut map = std::collections::HashMap::new();
351                for (key_idx, value) in obj {
352                    let key = format!("key_{}", key_idx);
353                    map.insert(key, self.convert_value_to_ir(value));
354                }
355                crate::types::Value::Object(map)
356            }
357        }
358    }
359    fn calculate_source_hash(&self, ir: &HelixIR) -> String {
360        use std::collections::hash_map::DefaultHasher;
361        use std::hash::{Hash, Hasher};
362        let mut hasher = DefaultHasher::new();
363        ir.version.hash(&mut hasher);
364        ir.string_pool.strings.len().hash(&mut hasher);
365        ir.instructions.len().hash(&mut hasher);
366        format!("{:x}", hasher.finish())
367    }
368}
369#[derive(Debug)]
370pub enum SerializationError {
371    IoError(String),
372    BincodeError(String),
373    CompressionError(String),
374    DecompressionError(String),
375    ValidationError(String),
376}
377impl std::fmt::Display for SerializationError {
378    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
379        match self {
380            Self::IoError(e) => write!(f, "I/O error: {}", e),
381            Self::BincodeError(e) => write!(f, "Bincode error: {}", e),
382            Self::CompressionError(e) => write!(f, "Compression error: {}", e),
383            Self::DecompressionError(e) => write!(f, "Decompression error: {}", e),
384            Self::ValidationError(e) => write!(f, "Validation error: {}", e),
385        }
386    }
387}
388impl std::error::Error for SerializationError {}
389impl From<String> for SerializationError {
390    fn from(s: String) -> Self {
391        Self::CompressionError(s)
392    }
393}
394#[cfg(test)]
395mod tests {
396    use super::*;
397    use crate::codegen::{StringPool, Metadata, ConstantPool};
398    #[test]
399    fn test_serialization_roundtrip() {
400        let mut string_pool = StringPool::new();
401        string_pool.intern("test");
402        let ir = HelixIR {
403            version: 1,
404            metadata: Metadata::default(),
405            symbol_table: crate::codegen::SymbolTable::default(),
406            instructions: vec![
407                crate ::codegen::Instruction::DeclareAgent(1), crate
408                ::codegen::Instruction::DeclareWorkflow(2),
409            ],
410            string_pool,
411            constants: ConstantPool::default(),
412        };
413        let serializer = BinarySerializer::new(false);
414        let binary = serializer.serialize(ir.clone(), None).unwrap();
415        let deserialized = serializer.deserialize_to_ir(&binary).unwrap();
416        assert_eq!(ir.version, deserialized.version);
417        assert_eq!(ir.instructions.len(), deserialized.instructions.len());
418    }
419}
helix_core/compiler/serializer.rs

helix_core/compiler/
serializer.rs
