1use std::path::PathBuf;
18
19use serde::Serialize;
20
21pub const SNAPSHOT_SCHEMA_VERSION: u32 = 1;
23
24pub const SNAPSHOTS_DIR: &str = ".ctx/snapshots";
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
30pub enum CaptureMode {
31 Live,
34 Backfill,
38}
39
40impl CaptureMode {
41 pub fn as_str(self) -> &'static str {
43 match self {
44 CaptureMode::Live => "live",
45 CaptureMode::Backfill => "backfill",
46 }
47 }
48}
49
50#[derive(Debug, Clone)]
52pub struct CaptureOptions {
53 pub out_dir: PathBuf,
55 pub churn_window: String,
60 pub capture_mode: CaptureMode,
62 pub force: bool,
64}
65
66#[derive(Debug, Clone, Serialize)]
72pub struct SnapshotReport {
73 pub commit_sha: String,
75 pub committed_at: String,
77 pub partition_dir: String,
79 pub files: usize,
81 pub symbols: usize,
83 pub dup_pairs: usize,
85 pub violations: i64,
87 pub skipped_existing: bool,
89}
90
91#[derive(Debug, Clone)]
93pub struct BackfillOptions {
94 pub since: String,
98 pub every: usize,
101 pub churn_window: String,
103 pub out_dir: PathBuf,
106}
107
108pub fn partition_dir_name(sha: &str) -> String {
110 format!("sha={}", sha)
111}
112
113#[cfg(feature = "duckdb")]
114pub use engine::{backfill, capture};
115
116#[cfg(feature = "duckdb")]
117mod engine {
118 use std::collections::HashMap;
119 use std::fs;
120 use std::path::{Path, PathBuf};
121 use std::process::Command;
122
123 use time::format_description::well_known::Rfc3339;
124 use time::OffsetDateTime;
125
126 use super::*;
127 use crate::analytics::Analytics;
128 use crate::check;
129 use crate::error::{CtxError, Result};
130 use crate::fingerprint::{self, DuplicatePair};
131 use crate::gitutil;
132 use crate::index::Indexer;
133 use crate::rules;
134 use crate::score::{DUP_MIN_TOKENS, DUP_THRESHOLD};
135 use crate::walker::WalkerConfig;
136
137 pub fn capture(root: &Path, opts: &CaptureOptions) -> Result<SnapshotReport> {
145 if !gitutil::is_git_repo_in(root) {
146 return Err(CtxError::NotGitRepo);
147 }
148 let (sha, committed_at) = gitutil::head_commit_in(root)?;
149
150 let partition = opts.out_dir.join(partition_dir_name(&sha));
151 if partition.exists() && !opts.force {
152 return Ok(skipped_report(&sha, &committed_at, &partition));
153 }
154
155 let mut indexer = Indexer::with_config(root, false, WalkerConfig::default())?;
158 indexer.index()?;
159 let db = indexer.db;
160
161 let pairs = fingerprint::find_near_duplicates(&db, DUP_THRESHOLD, DUP_MIN_TOKENS, None)?;
163 let (violations_total, violations_by_file) = violation_counts(root)?;
164 let until = match opts.capture_mode {
165 CaptureMode::Backfill => Some(committed_at.as_str()),
166 CaptureMode::Live => None,
167 };
168 let churn = gitutil::churn_between_in(root, &opts.churn_window, until)?;
169
170 let analytics = Analytics::open_export(root)?;
173 let conn = analytics.connection();
174 load_side_tables(conn, &churn, &violations_by_file, &pairs)?;
175
176 let staging = opts
178 .out_dir
179 .join(format!("{}.tmp", partition_dir_name(&sha)));
180 let guard = StagingGuard::new(&staging)?;
181 write_parquet_files(conn, &staging, &sha, &committed_at, opts.capture_mode)?;
182
183 if partition.exists() {
184 fs::remove_dir_all(&partition)?;
186 }
187 fs::rename(&staging, &partition)?;
188 guard.defuse();
189
190 let count = |sql: &str| -> Result<usize> {
191 let n: i64 = conn.query_row(sql, [], |row| row.get(0))?;
192 Ok(n.max(0) as usize)
193 };
194 Ok(SnapshotReport {
195 commit_sha: sha,
196 committed_at,
197 partition_dir: partition.display().to_string(),
198 files: count("SELECT COUNT(*) FROM v1.files")?,
199 symbols: count("SELECT COUNT(*) FROM v1.symbols")?,
200 dup_pairs: pairs.len(),
201 violations: violations_total,
202 skipped_existing: false,
203 })
204 }
205
206 pub fn backfill(root: &Path, opts: &BackfillOptions) -> Result<Vec<SnapshotReport>> {
219 if !gitutil::is_git_repo_in(root) {
220 return Err(CtxError::NotGitRepo);
221 }
222
223 let mut shas = gitutil::rev_list_first_parent_in(root, &format!("{}..HEAD", opts.since))?;
224 if let Some(since_sha) = resolve_commit(root, &opts.since) {
225 if !shas.contains(&since_sha) {
226 shas.insert(0, since_sha);
227 }
228 }
229 let sampled = sample_every(shas, opts.every.max(1));
230
231 let worktrees_root =
233 std::env::temp_dir().join(format!("ctx-backfill-{}", std::process::id()));
234 fs::create_dir_all(&worktrees_root)?;
235
236 let total = sampled.len();
237 let mut reports = Vec::new();
238 let mut failed = 0usize;
239 for (i, sha) in sampled.iter().enumerate() {
240 let short = &sha[..12.min(sha.len())];
241 let partition = opts.out_dir.join(partition_dir_name(sha));
242 if partition.exists() {
243 eprintln!(
244 "[{}/{}] {}: partition exists, skipping",
245 i + 1,
246 total,
247 short
248 );
249 let committed_at = commit_date(root, sha).unwrap_or_default();
250 reports.push(skipped_report(sha, &committed_at, &partition));
251 continue;
252 }
253
254 eprintln!("[{}/{}] {}: capturing…", i + 1, total, short);
255 match capture_commit(root, sha, &worktrees_root, opts) {
256 Ok(report) => reports.push(report),
257 Err(e) => {
258 failed += 1;
259 eprintln!("[{}/{}] {}: failed: {}", i + 1, total, short, e);
260 }
261 }
262 }
263 let _ = fs::remove_dir(&worktrees_root);
265
266 if failed > 0 {
267 eprintln!(
268 "backfill: {} succeeded, {} failed of {} commits",
269 reports.len(),
270 failed,
271 total
272 );
273 }
274 Ok(reports)
275 }
276
277 fn capture_commit(
279 root: &Path,
280 sha: &str,
281 worktrees_root: &Path,
282 opts: &BackfillOptions,
283 ) -> Result<SnapshotReport> {
284 let wt_path = worktrees_root.join(sha);
285 let output = Command::new("git")
286 .args(["worktree", "add", "--detach"])
287 .arg(&wt_path)
288 .arg(sha)
289 .current_dir(root)
290 .output()?;
291 if !output.status.success() {
292 return Err(CtxError::git(
293 String::from_utf8_lossy(&output.stderr).trim().to_string(),
294 ));
295 }
296 let _guard = WorktreeGuard {
297 repo: root.to_path_buf(),
298 path: wt_path.clone(),
299 };
300
301 capture(
302 &wt_path,
303 &CaptureOptions {
304 out_dir: opts.out_dir.clone(),
305 churn_window: opts.churn_window.clone(),
306 capture_mode: CaptureMode::Backfill,
307 force: false,
308 },
309 )
310 }
311
312 fn load_side_tables(
319 conn: &duckdb::Connection,
320 churn: &HashMap<String, u32>,
321 violations_by_file: &HashMap<String, i64>,
322 pairs: &[DuplicatePair],
323 ) -> Result<()> {
324 conn.execute_batch(
325 r#"
326 CREATE TABLE churn (path VARCHAR, churn_commits INTEGER);
327 CREATE TABLE violations (path VARCHAR, violation_count INTEGER);
328 CREATE TABLE dup_pairs (
329 file_a VARCHAR,
330 symbol_a VARCHAR,
331 file_b VARCHAR,
332 symbol_b VARCHAR,
333 similarity DOUBLE,
334 token_count_a BIGINT,
335 token_count_b BIGINT
336 );
337 "#,
338 )?;
339
340 let mut app = conn.appender("churn")?;
341 for (path, commits) in churn {
342 app.append_row(duckdb::params![path, *commits])?;
343 }
344 app.flush()?;
345
346 let mut app = conn.appender("violations")?;
347 for (path, count) in violations_by_file {
348 app.append_row(duckdb::params![path, *count])?;
349 }
350 app.flush()?;
351
352 let mut app = conn.appender("dup_pairs")?;
353 for pair in pairs {
354 app.append_row(duckdb::params![
355 pair.a.file_path,
356 pair.a.name,
357 pair.b.file_path,
358 pair.b.name,
359 pair.similarity,
360 pair.token_count_a,
361 pair.token_count_b,
362 ])?;
363 }
364 app.flush()?;
365
366 Ok(())
367 }
368
369 fn write_parquet_files(
372 conn: &duckdb::Connection,
373 staging: &Path,
374 sha: &str,
375 committed_at: &str,
376 mode: CaptureMode,
377 ) -> Result<()> {
378 let committed_utc = to_utc_timestamp_literal(committed_at)?;
381 let captured_at = OffsetDateTime::now_utc()
382 .format(&Rfc3339)
383 .unwrap_or_default();
384
385 let stamp = format!(
389 "'{}' AS commit_sha, TIMESTAMP '{}' AS committed_at",
390 sql_escape(sha),
391 sql_escape(&committed_utc)
392 );
393
394 copy_to_parquet(
395 conn,
396 &format!(
397 "SELECT {stamp},
398 id, name, qualified_name, kind, file,
399 line_start, line_end, is_public,
400 complexity, fan_in, fan_out
401 FROM v1.symbols"
402 ),
403 &staging.join("symbols.parquet"),
404 )?;
405
406 copy_to_parquet(
407 conn,
408 &format!(
409 "SELECT {stamp},
410 f.path, f.language, f.symbol_count, f.total_complexity,
411 COALESCE(mx.max_complexity, 0) AS max_complexity,
412 COALESCE(c.churn_commits, 0) AS churn_commits,
413 COALESCE(v.violation_count, 0) AS violation_count
414 FROM v1.files f
415 LEFT JOIN (
416 SELECT file, MAX(complexity) AS max_complexity
417 FROM v1.symbols GROUP BY file
418 ) mx ON mx.file = f.path
419 LEFT JOIN churn c ON c.path = f.path
420 LEFT JOIN violations v ON v.path = f.path"
421 ),
422 &staging.join("files.parquet"),
423 )?;
424
425 copy_to_parquet(
426 conn,
427 &format!(
428 "SELECT {stamp},
429 file_a, symbol_a, file_b, symbol_b,
430 similarity, token_count_a, token_count_b
431 FROM dup_pairs"
432 ),
433 &staging.join("dup_pairs.parquet"),
434 )?;
435
436 copy_to_parquet(
437 conn,
438 &format!(
439 "SELECT {stamp},
440 '{captured_at}' AS captured_at,
441 '{version}' AS ctx_version,
442 CAST({schema} AS INTEGER) AS snapshot_schema_version,
443 '{mode}' AS capture_mode",
444 captured_at = sql_escape(&captured_at),
445 version = sql_escape(env!("CARGO_PKG_VERSION")),
446 schema = SNAPSHOT_SCHEMA_VERSION,
447 mode = mode.as_str(),
448 ),
449 &staging.join("meta.parquet"),
450 )?;
451
452 Ok(())
453 }
454
455 fn copy_to_parquet(conn: &duckdb::Connection, select: &str, path: &Path) -> Result<()> {
457 let escaped = sql_escape(&path.display().to_string());
458 conn.execute_batch(&format!("COPY ({select}) TO '{escaped}' (FORMAT PARQUET);"))?;
459 Ok(())
460 }
461
462 fn sql_escape(s: &str) -> String {
464 s.replace('\'', "''")
465 }
466
467 fn to_utc_timestamp_literal(iso: &str) -> Result<String> {
470 let parsed = OffsetDateTime::parse(iso, &Rfc3339)
471 .map_err(|e| CtxError::Other(format!("invalid commit date {:?}: {}", iso, e)))?;
472 let utc = parsed.to_offset(time::UtcOffset::UTC);
473 Ok(format!(
474 "{:04}-{:02}-{:02} {:02}:{:02}:{:02}",
475 utc.year(),
476 u8::from(utc.month()),
477 utc.day(),
478 utc.hour(),
479 utc.minute(),
480 utc.second()
481 ))
482 }
483
484 fn violation_counts(root: &Path) -> Result<(i64, HashMap<String, i64>)> {
491 let rules_path = root.join(rules::DEFAULT_RULES_PATH);
492 if !rules_path.exists() {
493 return Ok((0, HashMap::new()));
494 }
495 let context = check::load_context(root, None)?;
496 let violations = check::collect_violations(root, &context, None)?;
497 let mut by_file: HashMap<String, i64> = HashMap::new();
498 for violation in &violations {
499 *by_file.entry(violation.file.clone()).or_insert(0) += 1;
500 }
501 Ok((violations.len() as i64, by_file))
502 }
503
504 fn skipped_report(sha: &str, committed_at: &str, partition: &Path) -> SnapshotReport {
507 SnapshotReport {
508 commit_sha: sha.to_string(),
509 committed_at: committed_at.to_string(),
510 partition_dir: partition.display().to_string(),
511 files: 0,
512 symbols: 0,
513 dup_pairs: 0,
514 violations: 0,
515 skipped_existing: true,
516 }
517 }
518
519 fn git_trimmed(root: &Path, args: &[&str]) -> Option<String> {
526 let output = Command::new("git")
527 .args(args)
528 .current_dir(root)
529 .output()
530 .ok()?;
531 if !output.status.success() {
532 return None;
533 }
534 let out = String::from_utf8_lossy(&output.stdout).trim().to_string();
535 (!out.is_empty()).then_some(out)
536 }
537
538 fn resolve_commit(root: &Path, rev: &str) -> Option<String> {
540 let spec = format!("{}^{{commit}}", rev);
541 git_trimmed(root, &["rev-parse", "--verify", "--quiet", &spec])
542 }
543
544 fn commit_date(root: &Path, sha: &str) -> Option<String> {
546 git_trimmed(root, &["log", "-1", "--format=%cI", sha])
547 }
548
549 fn sample_every(shas: Vec<String>, every: usize) -> Vec<String> {
553 let n = shas.len();
554 shas.into_iter()
555 .enumerate()
556 .filter(|(i, _)| (n - 1 - i).is_multiple_of(every))
557 .map(|(_, sha)| sha)
558 .collect()
559 }
560
561 struct StagingGuard {
568 path: PathBuf,
569 defused: bool,
570 }
571
572 impl StagingGuard {
573 fn new(path: &Path) -> Result<StagingGuard> {
574 if path.exists() {
576 fs::remove_dir_all(path)?;
577 }
578 fs::create_dir_all(path)?;
579 Ok(StagingGuard {
580 path: path.to_path_buf(),
581 defused: false,
582 })
583 }
584
585 fn defuse(mut self) {
586 self.defused = true;
587 }
588 }
589
590 impl Drop for StagingGuard {
591 fn drop(&mut self) {
592 if !self.defused {
593 let _ = fs::remove_dir_all(&self.path);
594 }
595 }
596 }
597
598 struct WorktreeGuard {
601 repo: PathBuf,
602 path: PathBuf,
603 }
604
605 impl Drop for WorktreeGuard {
606 fn drop(&mut self) {
607 let _ = Command::new("git")
608 .args(["worktree", "remove", "--force"])
609 .arg(&self.path)
610 .current_dir(&self.repo)
611 .output();
612 if self.path.exists() {
613 let _ = fs::remove_dir_all(&self.path);
616 let _ = Command::new("git")
617 .args(["worktree", "prune"])
618 .current_dir(&self.repo)
619 .output();
620 }
621 }
622 }
623
624 #[cfg(test)]
625 mod tests {
626 use super::*;
627 use crate::testutil::GitRepo;
628 use tempfile::TempDir;
629
630 #[test]
634 fn parquet_copy_round_trip() {
635 let dir = TempDir::new().unwrap();
636 let path = dir.path().join("t.parquet");
637 let escaped = sql_escape(&path.display().to_string());
638
639 let conn = duckdb::Connection::open_in_memory().unwrap();
640 conn.execute_batch(&format!(
641 "COPY (SELECT 1 AS x) TO '{}' (FORMAT PARQUET);",
642 escaped
643 ))
644 .unwrap();
645 assert!(path.exists(), "COPY must write the parquet file");
646
647 let x: i64 = conn
648 .query_row(
649 &format!("SELECT x FROM read_parquet('{}')", escaped),
650 [],
651 |row| row.get(0),
652 )
653 .unwrap();
654 assert_eq!(x, 1);
655 }
656
657 const MAIN_RS: &str = r#"
658pub fn alpha() -> i64 {
659 beta()
660}
661
662pub fn beta() -> i64 {
663 1
664}
665"#;
666
667 fn snapshot_fixture() -> (TempDir, GitRepo) {
668 let temp = TempDir::new().unwrap();
669 let repo = GitRepo::init(temp.path());
670 repo.write("src/a.rs", MAIN_RS);
671 repo.commit_all_with_date("v1", "2024-03-04T05:06:07 +0200");
672 let mut indexer =
673 Indexer::with_config(&repo.root, false, WalkerConfig::default()).unwrap();
674 indexer.index().unwrap();
675 (temp, repo)
676 }
677
678 #[test]
681 fn capture_writes_readable_partition() {
682 let (_temp, repo) = snapshot_fixture();
683 let out_dir = repo.root.join(SNAPSHOTS_DIR);
684
685 let report = capture(
686 &repo.root,
687 &CaptureOptions {
688 out_dir: out_dir.clone(),
689 churn_window: "90 days ago".to_string(),
690 capture_mode: CaptureMode::Live,
691 force: false,
692 },
693 )
694 .unwrap();
695
696 assert!(!report.skipped_existing);
697 assert!(report.symbols >= 2, "symbols: {}", report.symbols);
698 assert!(report.files >= 1, "files: {}", report.files);
699
700 let partition = out_dir.join(partition_dir_name(&report.commit_sha));
701 assert_eq!(report.partition_dir, partition.display().to_string());
702 let conn = duckdb::Connection::open_in_memory().unwrap();
703 for name in ["symbols", "files", "dup_pairs", "meta"] {
704 let path = partition.join(format!("{}.parquet", name));
705 assert!(path.exists(), "missing {}", path.display());
706 let escaped = sql_escape(&path.display().to_string());
707 let sha: String = conn
708 .query_row(
709 &format!("SELECT commit_sha FROM read_parquet('{}') LIMIT 1", escaped),
710 [],
711 |row| row.get(0),
712 )
713 .unwrap_or_default();
714 if name != "dup_pairs" {
715 assert_eq!(sha, report.commit_sha, "table {}", name);
717 }
718 }
719
720 let meta = partition.join("meta.parquet");
723 let escaped = sql_escape(&meta.display().to_string());
724 let (committed, mode, schema): (String, String, i32) = conn
725 .query_row(
726 &format!(
727 "SELECT CAST(committed_at AS VARCHAR), capture_mode, \
728 snapshot_schema_version FROM read_parquet('{}')",
729 escaped
730 ),
731 [],
732 |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
733 )
734 .unwrap();
735 assert_eq!(committed, "2024-03-04 03:06:07");
736 assert_eq!(mode, "live");
737 assert_eq!(schema as u32, SNAPSHOT_SCHEMA_VERSION);
738
739 assert!(!out_dir
741 .join(format!("{}.tmp", partition_dir_name(&report.commit_sha)))
742 .exists());
743
744 let skipped = capture(
746 &repo.root,
747 &CaptureOptions {
748 out_dir: out_dir.clone(),
749 churn_window: "90 days ago".to_string(),
750 capture_mode: CaptureMode::Live,
751 force: false,
752 },
753 )
754 .unwrap();
755 assert!(skipped.skipped_existing);
756 assert_eq!(skipped.symbols, 0, "skipped reports zero counts");
757
758 let forced = capture(
759 &repo.root,
760 &CaptureOptions {
761 out_dir,
762 churn_window: "90 days ago".to_string(),
763 capture_mode: CaptureMode::Live,
764 force: true,
765 },
766 )
767 .unwrap();
768 assert!(!forced.skipped_existing);
769 assert_eq!(forced.symbols, report.symbols);
770 }
771
772 #[test]
773 fn sample_every_always_keeps_newest() {
774 let shas: Vec<String> = (0..7).map(|i| format!("c{}", i)).collect();
775 let sampled = sample_every(shas.clone(), 3);
776 assert_eq!(sampled, vec!["c0", "c3", "c6"]);
777 let sampled = sample_every(shas.clone(), 2);
778 assert_eq!(sampled, vec!["c0", "c2", "c4", "c6"]);
779 let sampled = sample_every(shas, 1);
780 assert_eq!(sampled.len(), 7);
781 }
782 }
783}