okc 0.1.0

A local-first tool for AI agents to browse, parse, search, and reason over Open Knowledge Format (OKF) repositories
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

//! Parallel filesystem walker for markdown file discovery.
//!
//! [`Scanner`] uses the `ignore` crate for efficient, parallel directory traversal
//! with support for:
//! - `.gitignore` and standard ignore patterns
//! - Configurable exclude patterns
//! - Symlink following (optional)
//! - File size limits
//! - Extension filtering (`.md` files only)

use std::path::{Path, PathBuf};
use std::sync::mpsc;
use std::thread;
use std::time::SystemTime;

use ignore::{WalkBuilder, WalkState};
use tracing::info;

use crate::config::OkcConfig;
use crate::model::document::{FileRecord, LimitError};

/// Discovers markdown files in configured repository roots.
///
/// Runs a parallel walk for each root directory, sending discovered
/// `.md` files through a channel for collection and sorting.
pub struct Scanner;

impl Scanner {
    /// Discover all markdown files under configured roots.
    ///
    /// Returns a sorted vector of [`FileRecord`] with path, size, and mtime.
    /// Respects exclude patterns, size limits, and symlink settings from config.
    pub fn discover(config: &OkcConfig) -> Result<Vec<FileRecord>, LimitError> {
        let (tx, rx) = mpsc::channel();
        let max_size = config.max_file_size;
        let follow_symlinks = config.follow_symlinks;
        let exclude_patterns = config.exclude_patterns.clone();
        let roots = config.roots.clone();

        thread::spawn(move || {
            for root in roots {
                info!("Scanning root: {:?}", root);
                let mut builder = WalkBuilder::new(&root);
                builder
                    .standard_filters(true)
                    .hidden(false)
                    .follow_links(follow_symlinks)
                    .max_depth(None);

                for pattern in &exclude_patterns {
                    builder.add_ignore(pattern);
                }

                let root_clone = root.clone();
                builder.build_parallel().run(|| {
                    let tx = tx.clone();
                    let root_clone = root_clone.clone();
                    Box::new(move |entry| {
                        match entry {
                            Ok(entry) => {
                                let path = entry.path();
                                if path.extension().is_some_and(|e| e == "md") {
                                    if let Ok(metadata) = entry.metadata() {
                                        if metadata.is_file() {
                                            let size = metadata.len();
                                            if size > max_size {
                                                let _ = tx.send(Err(LimitError::new(
                                                    "max_file_size",
                                                    &max_size.to_string(),
                                                    &format!(
                                                        "File exceeds maximum size of {} bytes",
                                                        max_size
                                                    ),
                                                )
                                                .with_actual(&size.to_string())));
                                                return WalkState::Continue;
                                            }

                                            let modified_at = metadata
                                                .modified()
                                                .ok()
                                                .and_then(|t| {
                                                    t.duration_since(SystemTime::UNIX_EPOCH).ok()
                                                })
                                                .map_or(0, |d| d.as_secs() as i64);

                                            let rel_path = pathdiff(path, &root_clone)
                                                .unwrap_or_else(|| path.to_path_buf());

                                            let _ = tx.send(Ok(FileRecord {
                                                path: rel_path.to_string_lossy().to_string(),
                                                absolute_path: path.to_string_lossy().to_string(),
                                                size,
                                                modified_at,
                                            }));
                                        }
                                    }
                                }
                            }
                            Err(err) => {
                                info!("Walk error: {}", err);
                            }
                        }
                        WalkState::Continue
                    })
                });
            }
        });

        let mut files: Vec<FileRecord> = Vec::new();
        for result in rx.iter() {
            let file = result?;
            files.push(file);
        }
        files.sort_by(|a, b| a.path.cmp(&b.path));
        Ok(files)
    }
}

/// Compute relative path from root to path.
fn pathdiff(path: &Path, root: &Path) -> Option<PathBuf> {
    path.strip_prefix(root).ok().map(|p| p.to_path_buf())
}