sqz_engine/
lib.rs

1//! # sqz_engine
2//!
3//! The core compression engine behind sqz. Takes text — JSON, CLI output, code,
4//! logs, prose — and squeezes it down to use fewer LLM tokens while keeping the
5//! important stuff intact.
6//!
7//! ## Quick start
8//!
9//! ```rust
10//! use sqz_engine::SqzEngine;
11//!
12//! let engine = SqzEngine::new().expect("failed to init engine");
13//!
14//! // Compress some text
15//! let result = engine.compress("hello world").unwrap();
16//! println!("compressed: {}", result.data);
17//! println!("tokens: {} → {}", result.tokens_original, result.tokens_compressed);
18//!
19//! // JSON gets TOON-encoded automatically
20//! let json_result = engine.compress(r#"{"name": "Alice", "age": 30}"#).unwrap();
21//! assert!(json_result.data.starts_with("TOON:"));
22//! ```
23//!
24//! ## How it works
25//!
26//! Content flows through a multi-stage pipeline:
27//!
28//! 1. **Content routing** — the confidence router classifies input (JSON, code,
29//!    logs, prose) and picks a compression mode (safe, default, aggressive).
30//! 2. **Stage pipeline** — configurable stages run in priority order: ANSI
31//!    stripping, null removal, repeated-line condensing, git diff folding,
32//!    string truncation, array collapsing, and custom transforms.
33//! 3. **Post-processing** — RLE compression, sliding-window dedup, entropy
34//!    truncation, and token pruning for prose.
35//! 4. **TOON encoding** — JSON gets encoded into Token-Optimized Object
36//!    Notation, which drops unnecessary quotes and whitespace for 30-60%
37//!    fewer tokens.
38//! 5. **Verification** — a two-pass verifier checks that error lines, JSON
39//!    keys, and diff hunks survived compression. If confidence is low, the
40//!    engine falls back to safe mode.
41//!
42//! ## Key types
43//!
44//! - [`SqzEngine`] — top-level facade, wires everything together
45//! - [`CompressionPipeline`] — the stage-based compression orchestrator
46//! - [`CacheManager`] — SHA-256 content-hash dedup cache
47//! - [`SessionStore`] — SQLite-backed session and cache persistence
48//! - [`Preset`] — TOML-configurable compression settings
49//! - [`ToonEncoder`] — JSON → TOON lossless encoding
50//! - [`CompressedContent`] — compression result with token counts and metadata
51
52pub mod adaptive_tree;
53pub mod advanced_search;
54pub mod ansi_strip;
55pub mod api_proxy;
56pub mod ast_delta;
57pub mod ast_parser;
58pub mod benchmarks;
59#[cfg(test)]
60mod cmd_formatter_bench;
61pub mod bpe_compressor;
62pub mod cascade_compressor;
63pub mod claude_md_integration;
64pub mod cmd_formatters;
65pub mod codex_integration;
66pub mod compression_quality;
67pub mod confidence_router;
68pub mod context_evictor;
69pub mod crp_engine;
70pub mod dashboard;
71pub mod delta_encoder;
72pub mod dependency_mapper;
73pub mod dict_compressor;
74pub mod entropy_analyzer;
75pub mod entropy_truncator;
76pub mod exit_marker;
77pub mod file_reader;
78pub mod image_compressor;
79pub mod json_projection;
80pub mod kv_cache_optimizer;
81pub mod litm_positioner;
82pub mod mdl_selector;
83pub mod minhash_lsh;
84pub mod ngram_abbreviator;
85pub mod opencode_plugin;
86pub mod rle_compressor;
87pub mod simhash;
88pub mod table_compactor;
89pub mod textrank;
90pub mod token_pruner;
91pub mod tool_hooks;
92pub mod tool_selector;
93pub mod engine;
94pub mod hook_manager;
95pub mod budget_tracker;
96pub mod cache_manager;
97pub mod correction_log;
98pub mod cost_calculator;
99pub mod ctx_format;
100pub mod error;
101pub mod model_router;
102pub mod parse_tree_compressor;
103pub mod pipeline;
104pub mod pin_manager;
105pub mod plugin_api;
106pub mod preset;
107pub mod progressive_throttle;
108pub mod prompt_cache;
109pub mod regret_tracker;
110pub mod sandbox_executor;
111pub mod session_continuity;
112pub mod session_store;
113pub mod kiro_integration;
114pub mod stages;
115pub mod structural_summary;
116pub mod text_boundary;
117pub mod zed_integration;
118pub mod tee_mode;
119pub mod terse_mode;
120pub mod token_counter;
121pub mod toon;
122pub mod transparency;
123pub mod types;
124pub mod url_indexer;
125pub mod verifier;
126
127pub use advanced_search::{AdvancedSearch, SearchResult};
128pub use ansi_strip::AnsiStripper;
129pub use api_proxy::{compress_request, parse_http_request, build_http_response, ApiFormat, ProxyConfig, ProxyStats};
130pub use ast_parser::{AstParser, ClassDefinition, CodeSummary, FunctionSignature, ImportDecl, TypeDeclaration};
131pub use bpe_compressor::{bpe_compress, BpeConfig, BpeResult};
132pub use compression_quality::{measure_quality, format_quality_report, CompressionQuality, QualityGrade};
133pub use confidence_router::{ConfidenceRouter, CompressionMode};
134pub use context_evictor::{evict, should_evict, ContextItem, EvictionConfig, EvictionResult};
135pub use cmd_formatters::format_command;
136pub use delta_encoder::{DeltaConfig, DeltaEncoder, DeltaResult};
137pub use dependency_mapper::DependencyMapper;
138pub use dict_compressor::{DictCompressor, DictConfig, DictCompressResult};
139pub use entropy_analyzer::{EntropyAnalyzer, InfoLevel, AnalyzedBlock};
140pub use entropy_truncator::{EntropyTruncator, EntropyTruncConfig, EntropyTruncResult, EntropyTruncArrayResult};
141pub use file_reader::{FileReadMode, FileReader, ReadResult, BlockEntropy, compute_entropy, analyze_block_entropies};
142pub use image_compressor::{ImageCompressor, ImageDescription};
143pub use json_projection::{project_json, ProjectionConfig, ProjectionResult};
144pub use litm_positioner::{ContextSection, LitmPositioner, LitmStrategy, SectionType};
145pub use ngram_abbreviator::{NgramAbbreviator, AbbreviatorConfig, AbbreviationResult};
146pub use rle_compressor::{rle_compress, sliding_window_dedup, RleResult, SlidingWindowResult};
147pub use simhash::{simhash, SimHashFingerprint};
148pub use structural_summary::{summarize as structural_summarize, summarize_multi, SummaryConfig, StructuralSummaryResult};
149pub use textrank::{textrank_compress, TextRankConfig, TextRankResult};
150pub use mdl_selector::{select_stages, profile_content, ContentProfile, MdlSelection};
151pub use tool_hooks::{process_hook, process_hook_cursor, process_hook_gemini, process_hook_kiro, process_hook_windsurf, process_hook_copilot, generate_hook_configs, install_tool_hooks, install_tool_hooks_scoped, install_tool_hooks_scoped_filtered, claude_user_settings_path, claude_project_settings_needs_update, remove_claude_global_hook, copilot_hooks_path, install_copilot_hook, remove_copilot_hook, ci_hooks_path, ci_hook_content, install_ci_hook, remove_ci_hook, canonicalize_tool_name, parse_tool_list, InstallScope, ToolFilter, ToolHookConfig, HookScope, HookPlatform, SUPPORTED_TOOL_NAMES};
152pub use opencode_plugin::{
153    generate_opencode_plugin, install_opencode_plugin, update_opencode_config,
154    update_opencode_config_detailed, find_opencode_config,
155    opencode_config_has_comments,
156    plan_opencode_config_change, PlannedOpencodeChange,
157    remove_sqz_from_opencode_config, strip_jsonc_comments,
158    process_opencode_hook, opencode_plugin_path,
159};
160pub use codex_integration::{
161    agents_md_guidance_block, agents_md_path,
162    codex_config_path,
163    install_agents_md_guidance, install_codex_mcp_config,
164    remove_agents_md_guidance, remove_codex_mcp_config,
165};
166pub use claude_md_integration::{
167    claude_md_guidance_block, claude_md_path, claude_user_json_path,
168    install_claude_md_guidance, install_claude_mcp_config,
169    remove_claude_md_guidance, remove_claude_mcp_config,
170};
171pub use zed_integration::{
172    install_zed_guidance, install_zed_mcp_config, remove_zed_guidance,
173    remove_zed_mcp_config, zed_guidance_block, zed_guidance_target,
174    zed_mcp_snippet, zed_settings_path, ZedMcpInstall, ZedMcpRemove,
175};
176pub use kiro_integration::{
177    install_kiro_mcp_config, install_kiro_steering, kiro_legacy_hook_path,
178    kiro_mcp_path, kiro_steering_path, remove_kiro_legacy_hook,
179    remove_kiro_mcp_config, remove_kiro_steering, KiroMcpInstall, KiroMcpRemove,
180};
181pub use token_pruner::{TokenPruner, PrunerConfig, PruneResult};
182pub use tool_selector::{ToolDefinition, ToolSelector};
183pub use budget_tracker::{
184    AgentBudget, BudgetTracker, BudgetWarning, UsagePrediction, UsageReport,
185};
186pub use cache_manager::{CacheManager, CacheResult, DedupHit, ExpandResult};
187pub use correction_log::ContextWindow;
188pub use crp_engine::{CrpEngine, CrpLevel};
189pub use cost_calculator::{
190    CostBreakdown, CostCalculator, ModelPricing, PricingConfig, SavedCostEstimate,
191    SavingsAssumptions, SessionCostSummary, TokenUsage, ToolCost, estimate_saved_cost,
192};
193pub use ctx_format::{CtxEnvelope, CtxFormat, CtxMetadata};
194pub use error::{Result, SqzError, SourceLocation};
195pub use exit_marker::{posix_rewrite, powershell_rewrite, strip_exit_marker};
196pub use model_router::{ModelRouter, RoutingDecision, TaskContext};
197pub use pipeline::{CompressionPipeline, SessionContext};
198pub use pin_manager::PinManager;
199pub use plugin_api::{PluginLoader, PluginManifest, PluginSource, SqzPlugin};
200pub use prompt_cache::{CacheBoundary, Message, PromptCacheDetector};
201pub use session_store::{CommandStats, CompressionStats, DailyGain, RecallHit, RegretStats, SessionStore, SessionSummary};
202pub use session_continuity::{
203    SessionContinuityManager, SessionGuide, Snapshot, SnapshotEvent, SnapshotEventType,
204};
205pub use toon::ToonEncoder;
206pub use tee_mode::{TeeMode, TeeManager, TeeEntry};
207pub use terse_mode::TerseMode;
208pub use token_counter::TokenCounter;
209pub use types::*;
210pub use preset::{
211    BudgetConfig, CollapseArraysConfig, CompressionConfig, CondenseConfig, CustomTransformsConfig,
212    FlattenConfig, GitDiffFoldConfig, KeepFieldsConfig, ModelConfig, ModelPricingConfig, Preset,
213    PresetHeader, PresetMeta, PresetParser, StripFieldsConfig, StripNullsConfig, TerseLevel,
214    TerseModeConfig, ToolSelectionConfig, TruncateStringsConfig,
215};
216pub use progressive_throttle::{ProgressiveThrottler, ThrottleConfig, ThrottleLevel};
217pub use dashboard::{
218    CommandBreakdown, DashboardConfig, DashboardHtml, DashboardMetrics, DashboardServer,
219    SessionHistoryEntry, ToolBreakdown,
220};
221pub use engine::SqzEngine;
222pub use hook_manager::{
223    generate_platform_config, known_platforms, Hook, HookAction, HookContext, HookManager, HookType,
224};
225pub use sandbox_executor::{SandboxExecutor, SandboxResult, RuntimeInfo, FilteredOutput};
226pub use url_indexer::{ContentFetcher, IndexedChunk, IndexResult, UrlIndexer};
227pub use verifier::Verifier;
228
229pub use adaptive_tree::{compress_to_budget, build_tree, SemanticNode};
230pub use ast_delta::{ast_diff, encode_delta, AstDelta, AstChange, ChangeKind};
231pub use kv_cache_optimizer::{compress_with_sinks, compress_with_custom_sinks};
232pub use minhash_lsh::{MinHashLsh, MinHashSignature};
233pub use parse_tree_compressor::{compress_code, char_entropy};
234pub use cascade_compressor::{cascade_compress, CascadeLevel, CascadeThresholds, CascadeResult};
235pub use regret_tracker::{RegretTracker, RegretEvent, RegretKind, FileProfile};
236pub use transparency::{CompressionAnnotation};
sqz_engine/lib.rs

sqz_engine/
lib.rs
