scryer-engine 0.3.0

Tree-sitter AST indexing and reference resolution engine for Scryer code intelligence
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

use std::fs;
use std::sync::Arc;
use std::time::Instant;
use tempfile::tempdir;

use scryer_db::{ProjectRegistry, Scope, ScryerDb, SourceFile, Symbol};
use scryer_engine::EngineService;

#[tokio::test]
async fn test_concurrency_and_high_throughput_5000_files() -> anyhow::Result<()> {
    let temp = tempdir()?;
    let root = temp.path().join("concurrency_repo");
    fs::create_dir_all(&root)?;

    const FILE_COUNT: usize = 5000;
    println!("Generating {FILE_COUNT} synthetic source files...");

    // Generate 5,000 source files across 50 directory buckets
    for i in 0..FILE_COUNT {
        let dir_index = i / 100;
        let dir_path = root.join(format!("pkg_{dir_index:02}"));
        if i % 100 == 0 {
            fs::create_dir_all(&dir_path)?;
        }

        let file_path = dir_path.join(format!("item_{i:04}.rs"));
        let code = format!(
            r#"
pub struct ItemRecord_{i:04} {{
    pub id: u64,
}}

impl ItemRecord_{i:04} {{
    pub fn compute_{i:04}(&self) -> u64 {{
        self.id * 2
    }}
}}
"#
        );
        fs::write(&file_path, code)?;
    }

    let db = ScryerDb::new_in_memory().await?;
    let registry = Arc::new(ProjectRegistry::new(db.inner()).await?);
    let project = registry
        .register_or_update(&root, Some("concurrency-stress"))
        .await?;

    // Configure EngineService with 16 Rayon worker threads
    let engine = EngineService::with_thread_count(db.clone(), 16);

    println!("Starting 16-thread Rayon parallel indexing to Turso...");
    let start_time = Instant::now();
    let report = engine.index_project(project.id, &root).await?;
    let elapsed = start_time.elapsed();

    println!("Indexed 5,000 files in {:.2?}", elapsed);

    // 1. Validate throughput performance (< 20 seconds in release mode, < 75 seconds in unoptimized debug test profile)
    let max_duration_secs = if cfg!(debug_assertions) { 75 } else { 20 };
    assert!(
        elapsed.as_secs() < max_duration_secs,
        "5,000 file indexing must complete in under {max_duration_secs} seconds, took {:.2?}",
        elapsed
    );

    // 2. Validate report metrics
    assert_eq!(report.scanned_files, FILE_COUNT);
    assert_eq!(report.added_files, FILE_COUNT);
    assert_eq!(report.modified_files, 0);
    assert_eq!(report.deleted_files, 0);

    // 3. Validate 100% data integrity in Turso database
    let mut guard = db.lock().await;

    let db_files = SourceFile::filter(SourceFile::fields().project_id().eq(project.id))
        .exec(&mut *guard)
        .await?;
    assert_eq!(
        db_files.len(),
        FILE_COUNT,
        "Database SourceFile count must match exactly 5,000"
    );

    let db_symbols = Symbol::filter(Symbol::fields().project_id().eq(project.id))
        .exec(&mut *guard)
        .await?;
    // Each file contains 1 struct + 1 fn = 2 symbols * 5000 = 10,000 symbols
    assert_eq!(
        db_symbols.len(),
        FILE_COUNT * 2,
        "Database Symbol count must match exactly 10,000"
    );

    let db_scopes = Scope::filter(Scope::fields().project_id().eq(project.id))
        .exec(&mut *guard)
        .await?;
    // Each file contains at least struct scope + impl scope + fn scope
    assert!(
        db_scopes.len() >= FILE_COUNT * 2,
        "Database Scope count ({}) must match expected scope bounds",
        db_scopes.len()
    );

    println!("✅ 100% data integrity confirmed across 5,000 files and 10,000 symbols!");
    Ok(())
}