weavatrix_clone/
repository.rs

1use crate::block::{BlockDetection, BlockSource, detect_blocks};
2use crate::canonical::suppress_contained;
3use crate::cluster::families_for_pairs;
4use crate::fragment::fragment_file;
5use crate::{
6    CloneDetector, CloneError, CloneReport, DetectionMode, Language, Result, SourceFragment,
7};
8use std::sync::{Arc, Mutex};
9use weavatrix_scan::{
10    ContentDiscoveryMode, ContentFileStatus, ContentValidationPolicy, ContentVisitControl,
11    ContentVisitEvent, ScanOptions, Scanner,
12};
13
14const DEFAULT_EXTENSIONS: &[&str] = &[
15    "rs", "go", "c", "h", "cc", "cpp", "cxx", "hh", "hpp", "hxx", "sh", "bash", "zsh", "sql",
16    "psql", "js", "jsx", "mjs", "cjs", "ts", "tsx", "mts", "cts", "py", "pyi", "java", "cs",
17    "html", "htm", "xml", "vue", "svelte", "md", "mdx",
18];
19
20#[derive(Debug, Clone, PartialEq, Eq)]
21pub struct RepositoryOptions {
22    pub max_file_bytes: u64,
23    pub min_fragment_lines: usize,
24    pub max_fragment_lines: usize,
25    pub parallelism: usize,
26    pub cross_extensions: bool,
27    pub extensions: Vec<String>,
28}
29
30impl Default for RepositoryOptions {
31    fn default() -> Self {
32        Self {
33            max_file_bytes: 1_500_000,
34            min_fragment_lines: 3,
35            max_fragment_lines: 400,
36            parallelism: 0,
37            cross_extensions: false,
38            extensions: DEFAULT_EXTENSIONS
39                .iter()
40                .map(|value| (*value).to_owned())
41                .collect(),
42        }
43    }
44}
45
46impl RepositoryOptions {
47    #[must_use]
48    pub fn with_extensions<I, S>(mut self, extensions: I) -> Self
49    where
50        I: IntoIterator<Item = S>,
51        S: AsRef<str>,
52    {
53        self.extensions = extensions
54            .into_iter()
55            .map(|value| value.as_ref().trim_start_matches('.').to_ascii_lowercase())
56            .filter(|value| !value.is_empty())
57            .collect();
58        self
59    }
60}
61
62#[derive(Debug, Clone)]
63pub struct RepositoryCloneDetector {
64    detector: CloneDetector,
65    options: RepositoryOptions,
66}
67
68impl RepositoryCloneDetector {
69    #[must_use]
70    pub fn new(detector: CloneDetector) -> Self {
71        Self {
72            detector,
73            options: RepositoryOptions::default(),
74        }
75    }
76
77    #[must_use]
78    pub fn options(mut self, options: RepositoryOptions) -> Self {
79        self.options = options;
80        self
81    }
82
83    /// Scans a repository once, fragments selected source files, and detects
84    /// deterministic clone families.
85    ///
86    /// # Errors
87    ///
88    /// Rejects incomplete scans, invalid UTF-8 source, fragment limits, and
89    /// clone-core validation failures without returning partial output.
90    pub fn detect(&self, root: impl Into<std::path::PathBuf>) -> Result<CloneReport> {
91        self.validate_options()?;
92        let files = Arc::new(Mutex::new(Vec::<CollectedFile>::new()));
93        let sink = Arc::clone(&files);
94        let mut scan_options = ScanOptions::default()
95            .with_extensions(&self.options.extensions)
96            .with_content_discovery(ContentDiscoveryMode::BufferedParallel)
97            .with_content_validation(ContentValidationPolicy::Strict)
98            .with_parallelism(self.options.parallelism)
99            .selected_files_only();
100        scan_options.max_file_bytes = self.options.max_file_bytes;
101        let summary = Scanner::new(root)
102            .options(scan_options)
103            .visit_content(move |_worker| {
104                let sink = Arc::clone(&sink);
105                let mut current = None::<CollectedFile>;
106                move |event| {
107                    match event {
108                        ContentVisitEvent::FileStart { file, .. } => {
109                            current = Some(CollectedFile {
110                                path: file.relative.to_owned(),
111                                bytes: Vec::with_capacity(usize::try_from(file.bytes).unwrap_or(0)),
112                            });
113                        }
114                        ContentVisitEvent::Chunk { bytes, .. } => {
115                            if let Some(file) = &mut current {
116                                file.bytes.extend_from_slice(bytes);
117                            }
118                        }
119                        ContentVisitEvent::FileEnd {
120                            status: ContentFileStatus::Selected,
121                            ..
122                        } => {
123                            if let Some(file) = current.take() {
124                                sink.lock()
125                                    .unwrap_or_else(std::sync::PoisonError::into_inner)
126                                    .push(file);
127                            }
128                        }
129                        ContentVisitEvent::FileEnd { .. } => current = None,
130                    }
131                    ContentVisitControl::Continue
132                }
133            })
134            .map_err(|error| CloneError::Repository(error.to_string()))?;
135        if !summary.complete || summary.stopped {
136            return Err(CloneError::Repository(
137                "scan did not produce a complete repository view".to_owned(),
138            ));
139        }
140        self.build_report(&files)
141    }
142
143    fn validate_options(&self) -> Result<()> {
144        if self.options.max_file_bytes == 0
145            || self.options.min_fragment_lines == 0
146            || self.options.max_fragment_lines < self.options.min_fragment_lines
147            || self.options.extensions.is_empty()
148        {
149            return Err(CloneError::Repository(
150                "invalid repository fragment or byte limits".to_owned(),
151            ));
152        }
153        Ok(())
154    }
155
156    fn build_report(&self, files: &Mutex<Vec<CollectedFile>>) -> Result<CloneReport> {
157        let mut files = {
158            let mut collected = files
159                .lock()
160                .unwrap_or_else(std::sync::PoisonError::into_inner);
161            std::mem::take(&mut *collected)
162        };
163        files.sort_unstable_by(|left, right| left.path.cmp(&right.path));
164        let source_files = files.len();
165        let mut fragments = Vec::<SourceFragment>::new();
166        let mut block_sources = Vec::<BlockSource>::with_capacity(files.len());
167        for file in files {
168            let language = Language::from_path(&file.path);
169            let text = String::from_utf8(file.bytes).map_err(|_| {
170                CloneError::Repository(format!("selected source is not UTF-8: {}", file.path))
171            })?;
172            if self.detector.config().mode != DetectionMode::Exact {
173                fragments.extend(fragment_file(
174                    &file.path,
175                    language,
176                    &text,
177                    self.options.min_fragment_lines,
178                    self.options.max_fragment_lines,
179                )?);
180            }
181            block_sources.push(BlockSource {
182                path: file.path,
183                language,
184                text,
185            });
186        }
187        let mut report = self.detector.detect(&fragments)?;
188        let blocks = self.detect_blocks(block_sources)?;
189        report.pairs.extend(blocks.pairs);
190        report.pairs = suppress_contained(report.pairs);
191        report.statistics.source_files = source_files;
192        report.statistics.source_tokens = blocks.tokens;
193        report.statistics.exact_block_candidates = blocks.candidates;
194        report.statistics.candidate_pairs = report
195            .statistics
196            .candidate_pairs
197            .saturating_add(blocks.candidates);
198        report.statistics.suppressed_exact_buckets = blocks.suppressed_buckets;
199        report.statistics.verified_pairs = report.pairs.len();
200        report.families = families_for_pairs(&report.pairs);
201        Ok(report)
202    }
203
204    fn detect_blocks(&self, sources: Vec<BlockSource>) -> Result<BlockDetection> {
205        if self.options.cross_extensions || sources.len() < 2 {
206            return detect_blocks(
207                &sources,
208                self.detector.config(),
209                self.options.min_fragment_lines,
210                self.options.parallelism,
211            );
212        }
213        let mut by_extension = std::collections::BTreeMap::<String, Vec<BlockSource>>::new();
214        for source in sources {
215            let extension = std::path::Path::new(&source.path)
216                .extension()
217                .and_then(|value| value.to_str())
218                .unwrap_or_default()
219                .to_ascii_lowercase();
220            by_extension.entry(extension).or_default().push(source);
221        }
222        let groups = by_extension.into_values().collect::<Vec<_>>();
223        let available = std::thread::available_parallelism().map_or(1, usize::from);
224        let workers = if self.options.parallelism == 0 {
225            available
226        } else {
227            self.options.parallelism
228        }
229        .clamp(1, groups.len());
230        let lexer_workers = if self.options.parallelism == 0 {
231            0
232        } else {
233            self.options.parallelism.div_ceil(workers)
234        };
235        let mut assignments = (0..workers)
236            .map(|_| Vec::<Vec<BlockSource>>::new())
237            .collect::<Vec<_>>();
238        for (index, group) in groups.into_iter().enumerate() {
239            assignments[index % workers].push(group);
240        }
241        std::thread::scope(|scope| {
242            let handles = assignments
243                .into_iter()
244                .map(|assignment| {
245                    scope.spawn(move || {
246                        let mut combined = BlockDetection::default();
247                        for group in assignment {
248                            combined.merge(detect_blocks(
249                                &group,
250                                self.detector.config(),
251                                self.options.min_fragment_lines,
252                                lexer_workers,
253                            )?);
254                        }
255                        Ok::<_, CloneError>(combined)
256                    })
257                })
258                .collect::<Vec<_>>();
259            let mut combined = BlockDetection::default();
260            for handle in handles {
261                combined.merge(
262                    handle.join().map_err(|_| {
263                        CloneError::Repository("block worker panicked".to_owned())
264                    })??,
265                );
266            }
267            Ok(combined)
268        })
269    }
270}
271
272struct CollectedFile {
273    path: String,
274    bytes: Vec<u8>,
275}
weavatrix_clone/repository.rs

weavatrix_clone/
repository.rs
