ctxpack 0.1.0

Configuration-driven source packing for LLM context, with a versioning protocol for incremental patch/replace
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

use super::traits::TransformStage;

pub struct Pipeline {
    stages: Vec<Box<dyn TransformStage>>,
}

impl Pipeline {
    pub fn new() -> Self {
        Pipeline { stages: Vec::new() }
    }

    pub fn add_stage(&mut self, stage: impl TransformStage + 'static) {
        self.stages.push(Box::new(stage));
    }

    pub fn encode_all(&self, input: &str) -> String {
        let mut current = input.to_string();
        for stage in &self.stages {
            current = stage.encode(&current);
        }
        current
    }

    pub fn decode_all(&self, input: &str) -> String {
        let mut current = input.to_string();
        for stage in self.stages.iter().rev() {
            current = stage.decode(&current);
        }
        current
    }
}

impl Default for Pipeline {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::encoding_layer::{AnchorEncoder, IndentEncoder};

    #[test]
    fn test_pipeline_round_trip() {
        let mut pipeline = Pipeline::new();
        pipeline.add_stage(IndentEncoder::new(4));
        pipeline.add_stage(AnchorEncoder::new(10));

        let original = "fn main() {\n    let x = 1;\n}\n";
        let encoded = pipeline.encode_all(original);
        let decoded = pipeline.decode_all(&encoded);
        assert_eq!(decoded, original);
    }

    #[test]
    fn test_pipeline_large_file_round_trip() {
        let mut pipeline = Pipeline::new();
        pipeline.add_stage(IndentEncoder::new(4));
        pipeline.add_stage(AnchorEncoder::new(10));

        let lines: Vec<String> =
            (1..=120).map(|i| format!("{}line {}", "    ".repeat(i % 5), i)).collect();
        let original = lines.join("\n") + "\n";
        let encoded = pipeline.encode_all(&original);
        let decoded = pipeline.decode_all(&encoded);
        assert_eq!(decoded, original);
    }

    #[test]
    fn test_empty_pipeline() {
        let pipeline = Pipeline::new();
        let text = "hello world\n";
        assert_eq!(pipeline.encode_all(text), text);
        assert_eq!(pipeline.decode_all(text), text);
    }
}