1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
//! HEAD-time import-edge extraction and resolution. The first pass tree-sitter
//! parses every live-at-HEAD Tier-1 source file for import statements and
//! inserts one unresolved row per edge into the `imports` table; the companion
//! resolver pass maps the raw targets to repo-relative tracked paths where the
//! per-language resolver succeeds.
use super::FactsDb;
use crate::{CodeLoreError, Options, Result};
impl FactsDb {
/// Walk Tier-1 source files at HEAD, tree-sitter-parse each for
/// import statements, and bulk-insert one row per import edge
/// into the `imports` table. Returns the total row count for
/// diagnostics.
///
/// Mirrors `populate_clones_at_head`'s rayon-then-serial-drain
/// shape: parallel blob-read + extraction, then a single Appender
/// drain on the connection-owning thread (`DuckDB` Connection is
/// `!Send + !Sync`). Per-file duplicates (same raw target listed
/// twice in one file) are deduped before drain to honor the
/// `(rev, src_path, target)` PRIMARY KEY.
///
/// Every row lands with `resolved=false` / `target_path=NULL`;
/// the companion `resolve_imports_at_head` UPDATE pass fills in
/// resolvable targets immediately after.
pub(super) fn populate_imports_at_head<R: crate::repo::Repo>(
&self,
repo: &R,
_opts: &Options,
live_paths: &[String],
head_rev: &str,
) -> Result<usize> {
use crate::imports::{ImportLanguage, RawImport, extract_imports};
use rayon::prelude::*;
use std::collections::HashSet;
// Phase 1 (serial): caller-supplied path set, filtered to Tier-1
// extensions. Same source-of-truth pattern as the complexity +
// clones passes (works on bare repos via the gix ODB;
// `PathsFilter` already ran at ingest time).
let candidates: Vec<(String, ImportLanguage)> = live_paths
.iter()
.filter_map(|rel| {
let lang = ImportLanguage::from_path(std::path::Path::new(rel))?;
Some((rel.clone(), lang))
})
.collect();
// Phase 2 (parallel): read blob + extract imports per file.
// Errors are logged at warn / debug and the file skipped —
// a single malformed file shouldn't fail the whole ingest.
let per_file: Vec<(String, Vec<RawImport>)> = candidates
.into_par_iter()
.map_init(
|| repo.blob_reader_at("HEAD"),
|reader, (rel, lang)| {
let code = match reader.read(&rel) {
Ok(Some(code)) => code,
Ok(None) => {
// Path not tracked at HEAD; skip (non-fatal, the
// rest of the scan continues).
tracing::debug!("imports: {rel} not tracked at HEAD; skipping");
return None;
}
Err(e) => {
// Object-database error (corrupted pack, missing
// shallow object). Surface as a warning and skip
// — the rest of the scan can still complete.
tracing::warn!("imports: blob read failed for {rel}: {e}");
return None;
}
};
if code.len() > crate::constants::DEFAULT_MAX_AST_FILE_BYTES {
tracing::debug!(
"imports: skipping {rel} ({size} bytes > {cap}-byte AST cap)",
size = code.len(),
cap = crate::constants::DEFAULT_MAX_AST_FILE_BYTES,
);
return None;
}
let imports = match extract_imports(&code, lang) {
Ok(v) if !v.is_empty() => v,
Ok(_) => return None,
Err(e) => {
tracing::warn!("imports: extract failed for {rel}: {e}");
return None;
}
};
Some((rel, imports))
},
)
.flatten_iter()
.collect();
// Phase 3 (serial drain): bulk-insert via the DuckDB Appender
// on the connection-owning thread. Dedup within each file's
// target set so the (rev, src_path, target) PK isn't violated
// by a file that lists the same raw target twice (rare but
// legal in JS dynamic-import patterns).
let mut app = self
.conn()
.appender("imports")
.map_err(|e| CodeLoreError::Analysis(format!("appender imports: {e}")))?;
let mut rows_inserted = 0usize;
for (path, imports) in per_file {
let mut seen = HashSet::new();
for imp in &imports {
if !seen.insert(imp.target.clone()) {
continue;
}
app.append_row(duckdb::params![
head_rev,
&path,
&imp.target,
false,
Option::<&str>::None,
imp.kind.as_str(),
])
.map_err(|e| CodeLoreError::Analysis(format!("append imports row: {e}")))?;
rows_inserted += 1;
}
}
app.flush()
.map_err(|e| CodeLoreError::Analysis(format!("flush imports appender: {e}")))?;
Ok(rows_inserted)
}
/// Resolver pass: for every import row whose `target_path` is
/// still NULL, attempt a per-language resolution against the
/// live-at-HEAD tracked path set. On a hit, UPDATE the row to
/// set `resolved=TRUE` and the canonical `target_path`. Returns
/// the total number of rows successfully resolved.
///
/// Covers Rust `crate::` / `self::` / `super::` paths, Python
/// relative + absolute imports, JS/TS `./` / `../` paths, and Java
/// FQNs (resolved to `.java` files by package-path suffix match).
pub(super) fn resolve_imports_at_head(
&self,
live_paths: &[String],
head_rev: &str,
) -> Result<usize> {
use crate::imports::resolve_by_extension;
// 1. Pull every unresolved import row scoped to a language the
// multi-language resolver supports. The extension allow-list
// mirrors the full Tier-1 set (JS/TS, Python, Rust, Java) so
// every extracted edge is fed to its per-language resolver.
let mut stmt = self
.conn()
.prepare(
"SELECT src_path, target FROM imports \
WHERE target_path IS NULL \
AND (
src_path LIKE '%.js' OR src_path LIKE '%.jsx' OR
src_path LIKE '%.mjs' OR src_path LIKE '%.cjs' OR
src_path LIKE '%.ts' OR src_path LIKE '%.tsx' OR
src_path LIKE '%.py' OR src_path LIKE '%.pyi' OR
src_path LIKE '%.rs' OR src_path LIKE '%.java'
)",
)
.map_err(|e| CodeLoreError::Analysis(format!("prepare imports scan: {e}")))?;
let candidates: Vec<(String, String)> = stmt
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))
.map_err(|e| CodeLoreError::Analysis(format!("query imports scan: {e}")))?
.collect::<std::result::Result<Vec<_>, _>>()
.map_err(|e| CodeLoreError::Analysis(format!("collect imports scan: {e}")))?;
// 2. Dispatch to the per-language resolver by file extension
// (shared with the historical `architecture-trend` scan via
// `resolve_by_extension`, so language coverage lives in one
// place). Falls through to `None` for external targets.
//
// The resolvers want an owned-string hash set; build it once
// directly from the live-path slice (the caller hoisted
// `query_live_paths` to compute-once across all HEAD-time
// passes) and share the same `&HashSet<String>` across every
// resolver call instead of rebuilding it per row.
let live_paths_owned: std::collections::HashSet<String> =
live_paths.iter().cloned().collect();
let mut hits: Vec<(String, String, String)> = Vec::new();
for (src_path, target) in candidates {
if let Some(resolved_target_path) =
resolve_by_extension(&src_path, &target, &live_paths_owned)
{
hits.push((resolved_target_path, src_path, target));
}
}
if hits.is_empty() {
return Ok(0);
}
// 4. Apply all resolved hits via a single hash-joined UPDATE…FROM
// instead of N independent UPDATEs. Each per-row UPDATE in the
// old shape forced DuckDB to scan `imports` end-to-end (the
// `(rev, src_path, target)` PK is not a clustered index in
// DuckDB), making the pass O(N × |imports|). The temp-table
// join shape is one hash build + one streaming pass.
//
// `CREATE OR REPLACE TEMPORARY TABLE` mirrors the lineage /
// grouping passes elsewhere in this file.
self.conn()
.execute_batch(
"CREATE OR REPLACE TEMPORARY TABLE _resolved_imports (
target_path TEXT,
src_path TEXT,
target TEXT
)",
)
.map_err(|e| CodeLoreError::Analysis(format!("create resolved temp table: {e}")))?;
{
let mut app = self
.conn()
.appender("_resolved_imports")
.map_err(|e| CodeLoreError::Analysis(format!("appender resolved: {e}")))?;
for (target_path, src_path, target) in &hits {
app.append_row(duckdb::params![target_path, src_path, target])
.map_err(|e| CodeLoreError::Analysis(format!("append resolved row: {e}")))?;
}
app.flush()
.map_err(|e| CodeLoreError::Analysis(format!("flush resolved appender: {e}")))?;
}
let updated = self
.conn()
.execute(
"UPDATE imports
SET resolved = TRUE,
target_path = r.target_path
FROM _resolved_imports r
WHERE imports.rev = ?
AND imports.src_path = r.src_path
AND imports.target = r.target",
duckdb::params![head_rev],
)
.map_err(|e| CodeLoreError::Analysis(format!("bulk imports update: {e}")))?;
self.conn()
.execute_batch("DROP TABLE _resolved_imports")
.map_err(|e| CodeLoreError::Analysis(format!("drop resolved temp table: {e}")))?;
Ok(updated)
}
}