semtree_parse/
lib.rs

1//! **Tree-sitter parsing and chunk extraction for semtree.**
2//!
3//! Turns source files into structured [`Chunk`]s -
4//! functions, methods, structs, classes - aligned to real syntax boundaries
5//! instead of arbitrary line windows. Supports Rust, Python, JavaScript,
6//! TypeScript and Go; non-code text falls back to fixed-size windows.
7//!
8//! ```no_run
9//! use semtree_parse::extract_file;
10//!
11//! let chunks = extract_file(std::path::Path::new("src/lib.rs"))?;
12//! for c in &chunks {
13//!     println!("{:?} {:?}", c.kind, c.name);
14//! }
15//! # Ok::<(), semtree_parse::ParseError>(())
16//! ```
17
18mod error;
19mod extractor;
20mod lang;
21mod parser;
22mod text;
23
24pub use error::ParseError;
25pub use extractor::Extractor;
26pub use parser::{ParsedTree, SemtreeParser};
27pub use text::{chunk_text, is_text_file};
28
29use semtree_core::{Chunk, Language};
30
31pub fn parse_and_extract(source: &str, language: Language) -> Result<Vec<Chunk>, ParseError> {
32    let tree = SemtreeParser::parse(source, language)?;
33    lang::extract(&tree).map_err(|e| ParseError::Extract(e.to_string()))
34}
35
36pub fn parse_and_extract_file(path: &std::path::Path) -> Result<Vec<Chunk>, ParseError> {
37    let tree = SemtreeParser::parse_file(path)?;
38    lang::extract(&tree).map_err(|e| ParseError::Extract(e.to_string()))
39}
40
41/// Extract chunks from any supported file - code or plain text.
42pub fn extract_file(path: &std::path::Path) -> Result<Vec<Chunk>, ParseError> {
43    if is_text_file(path) {
44        let source = std::fs::read_to_string(path)?;
45        return Ok(chunk_text(path, &source, 40, 5));
46    }
47    parse_and_extract_file(path)
48}
semtree_parse/lib.rs

semtree_parse/
lib.rs
