llm_utl/
lib.rs

1//! # llm-utl
2//!
3//! A high-performance library for converting code repositories into LLM prompts.
4//!
5//! ## Features
6//!
7//! - Parallel file scanning with `.gitignore` support
8//! - Smart chunking with configurable token limits
9//! - Multiple output formats (Markdown, XML, JSON)
10//! - Atomic file operations with automatic backups
11//! - Advanced tokenization strategies
12//!
13//! ## Quick Start
14//!
15//! ```no_run
16//! use llm_utl::{Config, Pipeline, OutputFormat};
17//!
18//! # fn main() -> anyhow::Result<()> {
19//! let config = Config::builder()
20//!     .root_dir("./src")
21//!     .output_dir("./prompts")
22//!     .format(OutputFormat::Markdown)
23//!     .max_tokens(100_000)
24//!     .build()?;
25//!
26//! Pipeline::new(config)?.run()?;
27//! # Ok(())
28//! # }
29//! ```
30//!
31//! ## Architecture
32//!
33//! The library follows a pipeline architecture:
34//! 1. **Scanner**: Discovers files respecting `.gitignore`
35//! 2. **Tokenizer**: Estimates token counts
36//! 3. **Splitter**: Divides content into optimal chunks
37//! 4. **Writer**: Renders and persists output files
38
39#![warn(
40    missing_docs,
41    rust_2018_idioms,
42    unreachable_pub,
43    clippy::all,
44    clippy::pedantic,
45    clippy::nursery
46)]
47#![allow(clippy::module_name_repetitions)]
48
49mod config;
50mod error;
51mod file;
52mod filter;
53mod pipeline;
54mod scanner;
55mod splitter;
56mod template;
57mod template_validator;
58mod token;
59mod writer;
60
61pub mod preset;
62pub mod api;
63
64pub use config::{Config, ConfigBuilder, OutputFormat};
65pub use error::{Error, Result};
66pub use file::FileData;
67pub use filter::{CodeFilter, FileFilterConfig, FilterConfig};
68pub use pipeline::{Pipeline, PipelineStats};
69pub use preset::{LLMPreset, PresetKind};
70pub use splitter::Chunk;
71pub use token::{TokenEstimator, TokenizerKind};
72
73/// Runs the complete conversion pipeline with the given configuration.
74///
75/// This is the main entry point for the library.
76///
77/// # Errors
78///
79/// Returns an error if:
80/// - Configuration is invalid
81/// - Root directory doesn't exist or is inaccessible
82/// - No processable files are found
83/// - Output directory cannot be created
84/// - File operations fail
85///
86/// # Examples
87///
88/// ```no_run
89/// use llm_utl::{Config, run};
90///
91/// # fn main() -> anyhow::Result<()> {
92/// let config = Config::builder()
93///     .root_dir(".")
94///     .build()?;
95///
96/// run(config)?;
97/// # Ok(())
98/// # }
99/// ```
100pub fn run(config: Config) -> Result<PipelineStats> {
101    Pipeline::new(config)?.run()
102}
llm_utl/lib.rs

llm_utl/
lib.rs
