helix_core/compiler/
loader.rs

1use crate::compiler::binary::{HelixBinary, DataSection, SectionType};
2use crate::types::HelixConfig;
3use std::path::Path;
4use std::fs::File;
5use std::io::Read;
6use std::collections::HashMap;
7use memmap2::MmapOptions;
8use bincode;
9pub struct BinaryLoader {
10    enable_mmap: bool,
11    enable_lazy: bool,
12    _cache_enabled: bool,
13}
14impl BinaryLoader {
15    pub fn new() -> Self {
16        Self {
17            enable_mmap: true,
18            enable_lazy: false,
19            _cache_enabled: true,
20        }
21    }
22    pub fn with_mmap(mut self, enable: bool) -> Self {
23        self.enable_mmap = enable;
24        self
25    }
26    pub fn with_lazy(mut self, enable: bool) -> Self {
27        self.enable_lazy = enable;
28        self
29    }
30    pub fn load_file<P: AsRef<Path>>(&self, path: P) -> Result<HelixBinary, LoadError> {
31        let path = path.as_ref();
32        if self.enable_mmap {
33            self.load_with_mmap(path)
34        } else {
35            self.load_standard(path)
36        }
37    }
38    fn load_with_mmap(&self, path: &Path) -> Result<HelixBinary, LoadError> {
39        let file = File::open(path)
40            .map_err(|e| LoadError::IoError(format!("Failed to open file: {}", e)))?;
41        let mmap = unsafe {
42            MmapOptions::new()
43                .map(&file)
44                .map_err(|e| LoadError::MmapError(format!("Failed to map file: {}", e)))?
45        };
46        let binary: HelixBinary = bincode::deserialize(&mmap)
47            .map_err(|e| LoadError::DeserializationError(
48                format!("Failed to deserialize: {}", e),
49            ))?;
50        binary.validate().map_err(|e| LoadError::ValidationError(e))?;
51        Ok(binary)
52    }
53    fn load_standard(&self, path: &Path) -> Result<HelixBinary, LoadError> {
54        let mut file = File::open(path)
55            .map_err(|e| LoadError::IoError(format!("Failed to open file: {}", e)))?;
56        let mut buffer = Vec::new();
57        file.read_to_end(&mut buffer)
58            .map_err(|e| LoadError::IoError(format!("Failed to read file: {}", e)))?;
59        let binary: HelixBinary = bincode::deserialize(&buffer)
60            .map_err(|e| LoadError::DeserializationError(
61                format!("Failed to deserialize: {}", e),
62            ))?;
63        binary.validate().map_err(|e| LoadError::ValidationError(e))?;
64        Ok(binary)
65    }
66    pub fn load_to_config<P: AsRef<Path>>(
67        &self,
68        path: P,
69    ) -> Result<HelixConfig, LoadError> {
70        let binary = self.load_file(path)?;
71        self.binary_to_config(binary)
72    }
73    pub fn binary_to_config(
74        &self,
75        binary: HelixBinary,
76    ) -> Result<HelixConfig, LoadError> {
77        let mut config = HelixConfig::default();
78        for section in binary.data_sections {
79            let mut section = section;
80            if section.compression.is_some() {
81                section.decompress().map_err(|e| LoadError::DecompressionError(e))?;
82            }
83            match section.section_type {
84                SectionType::Agents => {
85                    self.load_agents_section(&section, &mut config)?;
86                }
87                SectionType::Workflows => {
88                    self.load_workflows_section(&section, &mut config)?;
89                }
90                SectionType::Contexts => {
91                    self.load_contexts_section(&section, &mut config)?;
92                }
93                SectionType::Memory => {
94                    self.load_memory_section(&section, &mut config)?;
95                }
96                SectionType::Crews => {
97                    self.load_crews_section(&section, &mut config)?;
98                }
99                SectionType::Pipelines => {
100                    self.load_pipelines_section(&section, &mut config)?;
101                }
102                _ => {}
103            }
104        }
105        Ok(config)
106    }
107    fn load_agents_section(
108        &self,
109        section: &DataSection,
110        config: &mut HelixConfig,
111    ) -> Result<(), LoadError> {
112        use crate::types::AgentConfig;
113        let agents: HashMap<String, AgentConfig> = bincode::deserialize(&section.data)
114            .map_err(|e| LoadError::DeserializationError(
115                format!("Failed to deserialize agents: {}", e),
116            ))?;
117        for (name, agent) in agents {
118            config.agents.insert(name, agent);
119        }
120        Ok(())
121    }
122    fn load_workflows_section(
123        &self,
124        section: &DataSection,
125        config: &mut HelixConfig,
126    ) -> Result<(), LoadError> {
127        use crate::types::WorkflowConfig;
128        let workflows: HashMap<String, WorkflowConfig> = bincode::deserialize(
129                &section.data,
130            )
131            .map_err(|e| LoadError::DeserializationError(
132                format!("Failed to deserialize workflows: {}", e),
133            ))?;
134        for (name, workflow) in workflows {
135            config.workflows.insert(name, workflow);
136        }
137        Ok(())
138    }
139    fn load_contexts_section(
140        &self,
141        section: &DataSection,
142        config: &mut HelixConfig,
143    ) -> Result<(), LoadError> {
144        use crate::types::ContextConfig;
145        let contexts: HashMap<String, ContextConfig> = bincode::deserialize(
146                &section.data,
147            )
148            .map_err(|e| LoadError::DeserializationError(
149                format!("Failed to deserialize contexts: {}", e),
150            ))?;
151        for (name, context) in contexts {
152            config.contexts.insert(name, context);
153        }
154        Ok(())
155    }
156    fn load_memory_section(
157        &self,
158        section: &DataSection,
159        config: &mut HelixConfig,
160    ) -> Result<(), LoadError> {
161        use crate::types::MemoryConfig;
162        let memory: MemoryConfig = bincode::deserialize(&section.data)
163            .map_err(|e| LoadError::DeserializationError(
164                format!("Failed to deserialize memory config: {}", e),
165            ))?;
166        config.memory = Some(memory);
167        Ok(())
168    }
169    fn load_crews_section(
170        &self,
171        section: &DataSection,
172        config: &mut HelixConfig,
173    ) -> Result<(), LoadError> {
174        use crate::types::CrewConfig;
175        let crews: HashMap<String, CrewConfig> = bincode::deserialize(&section.data)
176            .map_err(|e| LoadError::DeserializationError(
177                format!("Failed to deserialize crews: {}", e),
178            ))?;
179        for (name, crew) in crews {
180            config.crews.insert(name, crew);
181        }
182        Ok(())
183    }
184    fn load_pipelines_section(
185        &self,
186        section: &DataSection,
187        _config: &mut HelixConfig,
188    ) -> Result<(), LoadError> {
189        use crate::types::PipelineConfig;
190        let pipelines: Vec<PipelineConfig> = bincode::deserialize(&section.data)
191            .map_err(|e| LoadError::DeserializationError(
192                format!("Failed to deserialize pipelines: {}", e),
193            ))?;
194        if !pipelines.is_empty() {
195            eprintln!(
196                "Warning: Loaded {} pipelines but HelixConfig has no dedicated pipeline storage",
197                pipelines.len()
198            );
199        }
200        Ok(())
201    }
202}
203impl Default for BinaryLoader {
204    fn default() -> Self {
205        Self::new()
206    }
207}
208pub struct LazyBinaryLoader {
209    binary: HelixBinary,
210    loaded_sections: std::collections::HashSet<usize>,
211}
212impl LazyBinaryLoader {
213    pub fn new(binary: HelixBinary) -> Self {
214        Self {
215            binary,
216            loaded_sections: std::collections::HashSet::new(),
217        }
218    }
219    pub fn load_section(
220        &mut self,
221        section_type: SectionType,
222    ) -> Result<&DataSection, LoadError> {
223        for (idx, section) in self.binary.data_sections.iter_mut().enumerate() {
224            if std::mem::discriminant(&section.section_type)
225                == std::mem::discriminant(&section_type)
226            {
227                if !self.loaded_sections.contains(&idx) {
228                    if section.compression.is_some() {
229                        section
230                            .decompress()
231                            .map_err(|e| LoadError::DecompressionError(e))?;
232                    }
233                    self.loaded_sections.insert(idx);
234                }
235                return Ok(section);
236            }
237        }
238        Err(LoadError::SectionNotFound(format!("{:?}", section_type)))
239    }
240    pub fn is_loaded(&self, section_type: &SectionType) -> bool {
241        for (idx, section) in self.binary.data_sections.iter().enumerate() {
242            if std::mem::discriminant(&section.section_type)
243                == std::mem::discriminant(section_type)
244            {
245                return self.loaded_sections.contains(&idx);
246            }
247        }
248        false
249    }
250    pub fn stats(&self) -> LoaderStats {
251        LoaderStats {
252            total_sections: self.binary.data_sections.len(),
253            loaded_sections: self.loaded_sections.len(),
254            total_size: self.binary.size(),
255            loaded_size: self.calculate_loaded_size(),
256        }
257    }
258    fn calculate_loaded_size(&self) -> usize {
259        self.binary
260            .data_sections
261            .iter()
262            .enumerate()
263            .filter(|(idx, _)| self.loaded_sections.contains(idx))
264            .map(|(_, section)| section.size as usize)
265            .sum()
266    }
267}
268#[derive(Debug)]
269pub struct LoaderStats {
270    pub total_sections: usize,
271    pub loaded_sections: usize,
272    pub total_size: usize,
273    pub loaded_size: usize,
274}
275#[derive(Debug)]
276pub enum LoadError {
277    IoError(String),
278    MmapError(String),
279    DeserializationError(String),
280    DecompressionError(String),
281    ValidationError(String),
282    SectionNotFound(String),
283}
284impl std::fmt::Display for LoadError {
285    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
286        match self {
287            Self::IoError(e) => write!(f, "I/O error: {}", e),
288            Self::MmapError(e) => write!(f, "Memory mapping error: {}", e),
289            Self::DeserializationError(e) => write!(f, "Deserialization error: {}", e),
290            Self::DecompressionError(e) => write!(f, "Decompression error: {}", e),
291            Self::ValidationError(e) => write!(f, "Validation error: {}", e),
292            Self::SectionNotFound(e) => write!(f, "Section not found: {}", e),
293        }
294    }
295}
296impl std::error::Error for LoadError {}
297#[cfg(test)]
298mod tests {
299    use super::*;
300    #[test]
301    fn test_loader_creation() {
302        let loader = BinaryLoader::new();
303        assert!(loader.enable_mmap);
304        assert!(! loader.enable_lazy);
305        assert!(loader.cache_enabled);
306    }
307    #[test]
308    fn test_loader_builder() {
309        let loader = BinaryLoader::new().with_mmap(false).with_lazy(true);
310        assert!(! loader.enable_mmap);
311        assert!(loader.enable_lazy);
312    }
313}
helix_core/compiler/loader.rs

helix_core/compiler/
loader.rs
