codetether-rlm 0.1.0

Recursive Language Model processing for CodeTether
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13

//! Top-level compress function combining chunk + select + reassemble.

use super::chunk::chunk;
use super::reassemble::reassemble;
use super::select::select_chunks;
use super::types::ChunkOptions;

/// Intelligently compress content to fit within token budget.
pub fn compress(content: &str, max_tokens: usize, options: Option<ChunkOptions>) -> String {
    let chunks = chunk(content, options);
    let selected = select_chunks(&chunks, max_tokens);
    reassemble(&selected)
}