/**
* std/code_librarian — typed code-graph + Cypher surface for Harn scripts.
*
* Import: import "std/code_librarian"
*
* Wraps the `code_index.*` builtin family (issue #2434, PR #2441)
* behind ergonomic functions returning typed records. Consumers call
* one library instead of stitching outputs from many primitives. The
* natural-language entry point `code_librarian_query_nl` (issue #2439)
* compiles NL → Cypher via a cheap LLM and falls back to bounded MCTS
* over the graph when the compile path misses.
*
* The library never rebuilds the index; callers are expected to drive
* `code_index.rebuild` (or its lifecycle equivalent) themselves.
*
* Underlying executor docs: docs/src/stdlib/code-librarian.md
* A typed Cypher row — keys are projected column names from `RETURN`.
*/
import { agent_emit_event } from "std/agent/state"
import { with_cache_envelope } from "std/cache"
import { safe_structured_call } from "std/llm/safe"
import { schema_list, schema_object, schema_refine, schema_string } from "std/schema"
import { verification_file_hash_snapshot } from "std/verification"
type LibrarianRecord = dict
/** Result of a Cypher pass-through: rows plus the active overlay name. */
type LibrarianCypherResult = {rows: list<LibrarianRecord>, overlay: string?}
/** One symbol entry from the file outline, as returned by outline_get. */
type LibrarianOutlineSymbol = {
name: string,
kind: string,
access_level: string?,
start_line: int,
end_line: int,
signature: string,
}
/** A file outline plus its declared and reverse import edges. */
type LibrarianOutline = {
path: string,
file_id: int,
known: bool,
symbols: list<LibrarianOutlineSymbol>,
imports: list<dict>,
importers: list<string>,
}
type LibrarianOutlineSymbols = list<LibrarianOutlineSymbol>
/** A single caller site for a target symbol. */
type LibrarianCallSite = {caller: string?, path: string, symbol: string}
/** One declaration that may own a member surface. */
type LibrarianSymbolDefinition = {path: string, line: int, kind: string}
/** One member declared by a type or module container. */
type LibrarianMember = {
name: string,
kind: string,
path: string,
line: int,
signature: string,
access_level: string?,
}
/** A container's declared members plus enough identity to detect ambiguity. */
type LibrarianMemberSurface = {
container: string,
definitions: list<LibrarianSymbolDefinition>,
members: list<LibrarianMember>,
ambiguous: bool,
}
/** One external call site for a symbol. */
type LibrarianConsumerSite = {path: string, line: int}
/** Conservative by-name external-consumer facts for one defining file. */
type LibrarianExternalConsumers = {
symbol: string,
defining_path: string,
definition_found: bool,
sites: list<LibrarianConsumerSite>,
file_count: int,
sole_definer: bool,
}
/** A single import edge into a target path. */
type LibrarianImport = {importer: string, kind: string}
/** One entry from the change log for a recent-changes window. */
type LibrarianFileChange = {
path: string,
seq: int,
agent_id: int,
op: string,
hash: string,
size: int,
timestamp_ms: int,
}
type LibrarianFileChanges = list<LibrarianFileChange>
/** Freshness signal: indexed vs on-disk state for a known or unknown path. */
type LibrarianFreshness = {
path: string,
known: bool,
stale: bool,
indexed_hash: string?,
indexed_mtime_ms: int?,
disk_hash: string?,
disk_mtime_ms: int?,
}
/** Overlay activation result: branch name (or nil when deactivated) + reuse fraction. */
type LibrarianOverlay = {active: string?, reuse_fraction: float}
/** Seq-bound current file-hash snapshot for verification freshness checks. */
type LibrarianFileHashSnapshot = dict
fn __code_librarian_cypher_literal(value: string) -> string {
return replace(replace(value, "\\", "\\\\"), "'", "\\'")
}
fn __code_librarian_int(value: int?) -> int {
return to_int(value ?? 0) ?? 0
}
fn __code_librarian_optional_string(value: any) -> string? {
if value == nil {
return nil
}
const text = to_string(value)
if text == "" {
return nil
}
return text
}
/**
* Pass a Cypher query straight through to the typed symbol graph executor.
* Returns the row list plus the active overlay name (nil when on base).
*
* @effects: [host]
* @errors: [backend]
* @api_stability: experimental
* @example: code_librarian_query("MATCH (f:Function {name: 'main'}) RETURN f.path AS path")
*/
pub fn code_librarian_query(code_index: HarnessCodeIndex, cypher: string) -> LibrarianCypherResult {
const raw = code_index.cypher({query: cypher})
return {rows: raw?.rows ?? [], overlay: raw?.overlay}
}
/**
* Outline a file: symbols plus the import edges in and out of it. `depth`
* is reserved for future graph-aware traversal (currently 1: the file
* itself); values >= 1 still return the immediate outline so the call
* shape is stable for richer expansions later.
*
* @effects: [host]
* @errors: [invalid_argument]
* @api_stability: experimental
* @example: code_librarian_outline("src/util.ts")
*/
pub fn code_librarian_outline(
code_index: HarnessCodeIndex,
path: string,
depth: int = 1,
) -> LibrarianOutline {
if depth < 1 {
throw "code_librarian_outline: depth must be >= 1"
}
const file_id = to_int(code_index.path_to_id({path: path}))
if file_id == nil || file_id == 0 {
return {file_id: 0, importers: [], imports: [], known: false, path: path, symbols: []}
}
const symbols = schema_expect(
code_index.outline_get({file_id: file_id}),
schema_refine(
schema_of(LibrarianOutlineSymbols),
schema_list(schema_of(LibrarianOutlineSymbol)),
),
)
const imps = schema_expect(
code_index.imports_for({path: path}),
schema_object({imports: schema_list()}),
)
const users = schema_expect(
code_index.importers_of({module: path}),
schema_object({importers: schema_list(schema_string())}),
)
return {
file_id: file_id,
importers: users?.importers ?? [],
imports: imps?.imports ?? [],
known: true,
path: path,
symbols: symbols ?? [],
}
}
/**
* Resolve every call site that targets `symbol`, walking up to
* `max_hops` chained CALLS edges (currently 1 hop — the direct callers).
* Higher hop budgets are reserved for the upcoming transitive query
* shape; the return type stays the same.
*
* Canned query: `MATCH (f:Function {name: $name})<-[:CALLS]-(c:CallSite) RETURN c.path AS path, c.caller AS caller, f.name AS symbol`.
*
* @effects: [host]
* @errors: [backend]
* @api_stability: experimental
* @example: code_librarian_who_calls("formatDate")
*/
pub fn code_librarian_who_calls(
code_index: HarnessCodeIndex,
symbol: string,
max_hops: int = 2,
) -> list<LibrarianCallSite> {
if max_hops < 1 {
throw "code_librarian_who_calls: max_hops must be >= 1"
}
const safe_symbol = __code_librarian_cypher_literal(symbol)
const cypher = "MATCH (f:Function {name: '" + safe_symbol
+ "'})<-[:CALLS]-(c:CallSite) RETURN c.path AS path, c.caller AS caller, f.name AS symbol"
const raw = code_index.cypher({query: cypher})
const rows = raw?.rows ?? []
let out: list<LibrarianCallSite> = []
for row in rows {
out = out + [{caller: row?.caller, path: row?.path ?? "", symbol: row?.symbol ?? symbol}]
}
return out
}
/**
* Resolve the members declared by a project type or module. Container names are
* not globally qualified in the symbol graph, so callers may pass
* `defining_path` after resolving a type definition. Without it, definitions in
* multiple files set `ambiguous = true` instead of presenting a merged surface
* as authoritative. Multiple type/implementation nodes in one file contribute
* one non-ambiguous surface.
*
* @effects: [host]
* @errors: [backend]
* @api_stability: experimental
* @example: code_librarian_member_surface("StatusOr", "include/status.hpp")
*/
pub fn code_librarian_member_surface(
code_index: HarnessCodeIndex,
container: string,
defining_path: string? = nil,
) -> LibrarianMemberSurface {
const target = container.trim()
const requested_path = to_string(defining_path ?? "").trim()
if target == "" {
return {container: target, definitions: [], members: [], ambiguous: false}
}
const safe_target = __code_librarian_cypher_literal(target)
const definition_raw = code_index.cypher(
{
query:
"MATCH (d {name: '${safe_target}'}) WHERE d.kind = 'Type' OR d.kind = 'Module' RETURN d.path AS path, d.line AS line, d.kind AS kind",
},
)
let definitions: list<LibrarianSymbolDefinition> = []
let definition_keys: dict = {}
let definition_paths: dict = {}
for row in definition_raw?.rows ?? [] {
const path = to_string(row?.path ?? "")
const line = __code_librarian_int(row?.line)
const kind = to_string(row?.kind ?? "")
if path == "" || (requested_path != "" && path != requested_path) {
continue
}
const key = path + ":" + to_string(line) + ":" + kind
if !(definition_keys[key] ?? false) {
definitions = definitions + [{path: path, line: line, kind: kind}]
definition_keys = definition_keys + {[key]: true}
definition_paths = definition_paths + {[path]: true}
}
}
if len(definitions) == 0 {
return {container: target, definitions: [], members: [], ambiguous: false}
}
const member_raw = code_index.cypher(
{
query:
"MATCH (owner {name: '${safe_target}'})-[:CONTAINS]->(member) RETURN owner.path AS owner_path, member.name AS name, member.kind AS kind, member.path AS path, member.line AS line, member.signature AS signature, member.access_level AS access_level",
},
)
let members: list<LibrarianMember> = []
let member_keys: dict = {}
for row in member_raw?.rows ?? [] {
const owner_path = to_string(row?.owner_path ?? "")
const path = to_string(row?.path ?? "")
const line = __code_librarian_int(row?.line)
const name = to_string(row?.name ?? "")
const kind = to_string(row?.kind ?? "")
const signature = to_string(row?.signature ?? "")
if !(definition_paths[owner_path] ?? false) || path == "" || name == "" {
continue
}
const key = path + ":" + to_string(line) + ":" + kind + ":" + name + ":" + signature
if !(member_keys[key] ?? false) {
members = members
+ [
{
name: name,
kind: kind,
path: path,
line: line,
signature: signature,
access_level: __code_librarian_optional_string(row?.access_level),
},
]
member_keys = member_keys + {[key]: true}
}
}
return {
container: target,
definitions: definitions,
members: members,
ambiguous: requested_path == "" && len(definition_paths) > 1,
}
}
/**
* Return files outside `defining_path` that call `symbol` by name. This reads
* raw `CallSite` nodes rather than only traversing `CALLS` edges: unresolved and
* grammar-limited calls still carry the final qualified-name segment even when
* the graph could not link an edge. `sole_definer` is true only when the symbol
* is present in `defining_path` and absent from every other indexed file.
*
* `file_count` covers the full result while `sites` is bounded by `max_sites`.
*
* @effects: [host]
* @errors: [backend, invalid_argument]
* @api_stability: experimental
* @example: code_librarian_external_consumers("greet", "src/greeter.rs")
*/
pub fn code_librarian_external_consumers(
code_index: HarnessCodeIndex,
symbol: string,
defining_path: string,
max_sites: int = 20,
) -> LibrarianExternalConsumers {
if max_sites < 1 {
throw "code_librarian_external_consumers: max_sites must be >= 1"
}
const target = symbol.trim()
const owner_path = defining_path.trim()
if target == "" {
return {
symbol: target,
defining_path: owner_path,
definition_found: false,
sites: [],
file_count: 0,
sole_definer: false,
}
}
const safe_target = __code_librarian_cypher_literal(target)
const call_raw = code_index.cypher(
{
query:
"MATCH (c:CallSite) WHERE c.name = '${safe_target}' RETURN c.path AS path, c.line AS line",
},
)
let sites: list<LibrarianConsumerSite> = []
let site_keys: dict = {}
let consumer_paths: dict = {}
for row in call_raw?.rows ?? [] {
const path = to_string(row?.path ?? "")
const line = __code_librarian_int(row?.line)
if path == "" || path == owner_path {
continue
}
consumer_paths = consumer_paths + {[path]: true}
const key = path + ":" + to_string(line)
if !(site_keys[key] ?? false) {
if len(sites) < max_sites {
sites = sites + [{path: path, line: line}]
}
site_keys = site_keys + {[key]: true}
}
}
const definition_raw = code_index.cypher(
{
query:
"MATCH (d {name: '${safe_target}'}) WHERE d.kind = 'Function' OR d.kind = 'Type' OR d.kind = 'Module' RETURN d.path AS path",
},
)
let definition_found = false
let foreign_definition_found = false
for row in definition_raw?.rows ?? [] {
const path = to_string(row?.path ?? "")
if path == owner_path {
definition_found = true
} else if path != "" {
foreign_definition_found = true
}
}
return {
symbol: target,
defining_path: owner_path,
definition_found: definition_found,
sites: sites,
file_count: len(consumer_paths),
sole_definer: definition_found && !foreign_definition_found,
}
}
/**
* Resolve every file that imports `path`. Combines the direct importers
* edge (used by Cypher) with the legacy importers_of view so connectors
* that haven't migrated still surface here.
*
* Canned query: `MATCH (m:Module {path: $path})<-[:IMPORTS]-(s:Module) RETURN s.path AS importer, 'import' AS kind`.
*
* @effects: [host]
* @errors: [backend]
* @api_stability: experimental
* @example: code_librarian_what_imports("src/util.ts")
*/
pub fn code_librarian_what_imports(
code_index: HarnessCodeIndex,
path: string,
) -> list<LibrarianImport> {
const users = code_index.importers_of({module: path})
const importers = users?.importers ?? []
let out: list<LibrarianImport> = []
for item in importers {
out = out + [{importer: item, kind: "import"}]
}
return out
}
/**
* Recent changes since `since_seq` in the version log. The issue spec
* names this `recent_changes(window: Duration)`; Harn has no native
* Duration primitive yet, so the library accepts the same monotonic
* sequence number used by `code_index.current_seq` /
* `changes_since`. `since_seq = 0` returns every recorded change.
*
* Pair with `code_index.current_seq({})` to checkpoint and
* resume.
*
* @effects: [host]
* @errors: [invalid_argument]
* @api_stability: experimental
* @example: code_librarian_recent_changes(0)
*/
pub fn code_librarian_recent_changes(
code_index: HarnessCodeIndex,
since_seq: int = 0,
) -> list<LibrarianFileChange> {
if since_seq < 0 {
throw "code_librarian_recent_changes: since_seq must be >= 0"
}
const raw = code_index.changes_since({seq: since_seq})
return schema_expect(
raw ?? [],
schema_refine(schema_of(LibrarianFileChanges), schema_list(schema_of(LibrarianFileChange))),
)
}
/**
* Staleness signal for `path`: did the on-disk content drift away from
* the indexed snapshot? Wraps `code_index.freshness` directly,
* preserving the `{known, stale, indexed_hash, disk_hash, ...}` shape.
*
* @effects: [host, fs]
* @errors: [backend]
* @api_stability: experimental
* @example: code_librarian_freshness("src/util.ts")
*/
pub fn code_librarian_freshness(code_index: HarnessCodeIndex, path: string) -> LibrarianFreshness {
return schema_expect(code_index.freshness({path: path}), schema_of(LibrarianFreshness))
}
/** How `code_librarian_module_graph` groups files into nodes. */
type ModuleGraphGranularity = "dir" | "module"
/** Options for `code_librarian_module_graph`. Every field is optional. */
type ModuleGraphOptions = {
granularity?: ModuleGraphGranularity,
depth?: int,
roots?: list<string>,
include_external?: bool,
min_weight?: int,
}
/** A directory or module and the source files it groups. */
type ModuleGraphNode = {id: string, files: int, languages: list<string>}
/** Import links between two nodes, with up to three `[importer, imported]` file pairs. */
type ModuleGraphEdge = {from: string, to: string, weight: int, sample: list<list<string>>}
/** An import no workspace file satisfies, counted by importing files per node. */
type ModuleGraphExternal = {from: string, spec: string, count: int}
/** The workspace import graph rolled up to directories or modules. */
type ModuleGraph = {
indexed: bool,
nodes: list<ModuleGraphNode>,
edges: list<ModuleGraphEdge>,
external: list<ModuleGraphExternal>,
unresolved_count: int,
index_seq: int,
}
/**
* The workspace import graph rolled up from files to directories
* (`granularity: "dir"`) or modules (`"module"`, the default: a Swift
* target or Go package, else the nearest package manifest directory,
* else the directory). Edges come from explicit import statements only,
* self-loops are dropped, and every list is sorted, so the same index
* gives the same graph. `indexed` is false when no workspace is indexed.
*
* @effects: [host]
* @errors: [backend, invalid_argument]
* @api_stability: experimental
* @example: code_librarian_module_graph(harness.code_index, {granularity: "dir", depth: 2})
*/
pub fn code_librarian_module_graph(
code_index: HarnessCodeIndex,
options: ModuleGraphOptions? = nil,
) -> ModuleGraph {
return schema_expect(code_index.module_graph(options ?? {}), schema_of(ModuleGraph))
}
/** How a target architecture treats a dependency between two of its nodes. */
type ArchitectureEdgeKind = "allowed" | "required" | "forbidden"
/** A target node and the globs naming the actual graph nodes it owns. */
type ArchitectureNode = {id: string, paths: list<string>}
/** A declared dependency between target nodes. `kind` defaults to `"allowed"`. */
type ArchitectureEdge = {from: string, to: string, kind?: ArchitectureEdgeKind}
/** A target architecture. With `strict`, any undeclared dependency is reported. */
type ArchitectureTarget = {
nodes: list<ArchitectureNode>,
edges: list<ArchitectureEdge>,
strict?: bool,
}
/** One actual edge that contributes to a target-level dependency. */
type ArchitectureActualEdge = {from: string, to: string, weight: int}
/** A target-level dependency that breaks the target, with the code behind it. */
type ArchitectureViolation = {
from: string,
to: string,
weight: int,
sample: list<list<string>>,
actual: list<ArchitectureActualEdge>,
}
/** Where the actual graph departs from a target architecture. */
type ArchitectureDiff = {
conforms: bool,
missing_required: list<ArchitectureViolation>,
forbidden_present: list<ArchitectureViolation>,
unexpected: list<ArchitectureViolation>,
unmatched_nodes: list<string>,
unmapped_actual: list<ModuleGraphNode>,
}
const __ARCHITECTURE_SAMPLE_CAP = 5
/**
* Anchored regex for a path glob. A double star crosses directories (and
* a double star followed by a slash may match no directory at all), a
* single star and `?` stay inside one, and a pattern ending in slash and
* double star also matches the directory itself.
*/
fn __architecture_glob_regex(glob: string) -> string {
const chars = glob.chars()
let out = "^"
let i = 0
while i < len(chars) {
const c = chars[i]
if c == "*" && i + 2 < len(chars) && chars[i + 1] == "*" && chars[i + 2] == "/" {
out = out + "(?:.*/)?"
i = i + 3
continue
}
if c == "*" && i + 1 < len(chars) && chars[i + 1] == "*" {
out = out + ".*"
i = i + 2
continue
}
if c == "/" && i + 3 == len(chars) && chars[i + 1] == "*" && chars[i + 2] == "*" {
out = out + "(?:/.*)?"
i = i + 3
continue
}
if c == "*" {
out = out + "[^/]*"
} else if c == "?" {
out = out + "[^/]"
} else if contains(".+^$()[]{}|\\", c) {
out = out + "\\" + c
} else {
out = out + c
}
i = i + 1
}
return out + "$"
}
fn __architecture_owner(id: string, nodes: list<ArchitectureNode>, patterns: dict) -> string? {
for node in nodes {
for glob in node.paths {
if regex_match(patterns[glob], id) != nil {
return node.id
}
}
}
return nil
}
/** Validate a target once and index it: glob regexes by glob, edge kinds by key. */
fn __architecture_index_target(target: ArchitectureTarget) -> dict {
let patterns: dict = {}
let declared_ids: dict = {}
for node in target.nodes {
if declared_ids[node.id] != nil {
throw "architecture_graph_diff: duplicate target node " + node.id
}
declared_ids = declared_ids + {[node.id]: true}
for glob in node.paths {
patterns = patterns + {[glob]: __architecture_glob_regex(glob)}
}
}
let kinds: dict = {}
for edge in target.edges {
if declared_ids[edge.from] == nil || declared_ids[edge.to] == nil {
throw "architecture_graph_diff: edge " + edge.from + " -> " + edge.to
+ " names an undeclared target node"
}
const key = edge.from + "\n" + edge.to
if kinds[key] != nil {
throw "architecture_graph_diff: duplicate target edge " + edge.from + " -> " + edge.to
}
kinds = kinds + {[key]: edge.kind ?? "allowed"}
}
return {patterns: patterns, kinds: kinds}
}
/**
* Lift actual edges onto target nodes. Returns violations-in-waiting keyed
* by target edge, plus the keys in first-seen order so output is stable.
*/
fn __architecture_lift(edges: list<ModuleGraphEdge>, owner: dict) -> dict {
let lifted: dict = {}
let order: list<string> = []
for edge in edges {
const owner_from = owner[edge.from]
const owner_to = owner[edge.to]
if owner_from == nil || owner_to == nil || owner_from == owner_to {
continue
}
const key = owner_from + "\n" + owner_to
if lifted[key] == nil {
order = order + [key]
}
const prior = lifted[key]
?? {from: owner_from, to: owner_to, weight: 0, sample: [], actual: []}
let sample = prior.sample
for pair in edge.sample {
if len(sample) < __ARCHITECTURE_SAMPLE_CAP {
sample = sample + [pair]
}
}
lifted = lifted
+ {
[key]: prior
+ {
weight: prior.weight + edge.weight,
sample: sample,
actual: prior.actual + [pick(edge, ["from", "to", "weight"])],
},
}
}
return {lifted: lifted, order: order}
}
/**
* Compare an actual module graph with a target architecture. Each actual
* node belongs to the first target node (in declaration order) with a
* path glob matching its id; dependencies inside one target node are
* ignored. Reports required dependencies with no code behind them,
* forbidden ones the code has, and, when `strict`, dependencies the
* target never declared. Every reported dependency carries its weight,
* sample file pairs, and the actual edges behind it. `conforms` is true
* when all three lists are empty and, under `strict`, every actual node
* has an owner. A graph that measured nothing (not indexed, or no nodes)
* throws rather than conforming vacuously. Pure: no index or file access.
*
* @effects: []
* @errors: [invalid_argument]
* @api_stability: experimental
* @example: architecture_graph_diff(graph, {nodes: [{id: "ui", paths: ["src/ui"]}], edges: []})
*/
pub fn architecture_graph_diff(
actual: ModuleGraph,
target: ArchitectureTarget,
) -> ArchitectureDiff {
if !actual.indexed || len(actual.nodes) == 0 {
throw "architecture_graph_diff: the actual graph measured nothing (indexed: "
+ to_string(actual.indexed)
+ ", nodes: 0); rebuild the index or check the roots"
}
const index = __architecture_index_target(target)
const kinds = index.kinds
const strict = target.strict ?? false
let owner: dict = {}
let unmapped_actual: list<ModuleGraphNode> = []
for node in actual.nodes {
const found = __architecture_owner(node.id, target.nodes, index.patterns)
if found == nil {
unmapped_actual = unmapped_actual + [node]
} else {
owner = owner + {[node.id]: found}
}
}
const lift = __architecture_lift(actual.edges, owner)
const lifted = lift.lifted
let missing_required: list<ArchitectureViolation> = []
let forbidden_present: list<ArchitectureViolation> = []
for edge in target.edges {
const key = edge.from + "\n" + edge.to
if kinds[key] == "required" && lifted[key] == nil {
missing_required = missing_required
+ [{from: edge.from, to: edge.to, weight: 0, sample: [], actual: []}]
} else if kinds[key] == "forbidden" && lifted[key] != nil {
forbidden_present = forbidden_present + [lifted[key]]
}
}
let unexpected: list<ArchitectureViolation> = []
if strict {
for key in lift.order {
if kinds[key] == nil {
unexpected = unexpected + [lifted[key]]
}
}
}
const owners = owner.values()
let unmatched_nodes: list<string> = []
for node in target.nodes {
if !contains(owners, node.id) {
unmatched_nodes = unmatched_nodes + [node.id]
}
}
return {
conforms: len(missing_required) == 0
&& len(forbidden_present) == 0
&& len(unexpected) == 0
&& !(strict && len(unmapped_actual) > 0),
missing_required: missing_required,
forbidden_present: forbidden_present,
unexpected: unexpected,
unmatched_nodes: unmatched_nodes,
unmapped_actual: unmapped_actual,
}
}
/**
* Batch current file hashes under one code-index sequence binding. The
* returned `snapshot` map is the direct shape expected by
* `verification_diagnostic_classify`, while `files` preserves per-path
* index/readability metadata for HUDs and debuggers.
*
* @effects: [host, fs]
* @errors: [backend, invalid_argument]
* @api_stability: experimental
* @example: code_librarian_file_hash_snapshot(["src/util.ts"])
*/
pub fn code_librarian_file_hash_snapshot(
code_index: HarnessCodeIndex,
paths: list<string>,
) -> LibrarianFileHashSnapshot {
return verification_file_hash_snapshot(code_index, paths)
}
/**
* Activate a per-branch CDC overlay. If the overlay doesn't exist yet
* it's created as an empty pass-through so the base graph serves it
* untouched (100% reuse baseline). Passing `branch = nil` deactivates
* the active overlay and returns to the base graph.
*
* @effects: [host]
* @errors: [backend, invalid_argument]
* @api_stability: experimental
* @example: code_librarian_branch_overlay("topic/cypher")
*/
pub fn code_librarian_branch_overlay(
code_index: HarnessCodeIndex,
branch: string?,
) -> LibrarianOverlay {
if branch == nil {
return schema_expect(
code_index.branch_overlay({action: "deactivate"}),
schema_of(LibrarianOverlay),
)
}
return schema_expect(
code_index.branch_overlay({action: "activate", branch: branch}),
schema_of(LibrarianOverlay),
)
}
// -------------------------------------------------------------------------------------------------
// Natural-language query layer (#2439) — compile + MCTS fallback + cache
// -------------------------------------------------------------------------------------------------
const __NL_CACHE_NAMESPACE = "code_librarian.query_nl"
const __NL_CACHE_TTL_SECONDS = 300
const __MCTS_DEPTH_MAX_DEFAULT = 4
const __MCTS_EXPANSIONS_MAX_DEFAULT = 200
const __DEFAULT_LLM_OPTIONS = {
provider: "openrouter",
model: "mistralai/devstral-small",
temperature: 0.0,
max_tokens: 400,
}
/**
* code_librarian_query_nl translates a natural-language question into Cypher
* via a cheap LLM with schema-aware structured output, executes it, and
* falls back to bounded MCTS over the typed graph when the compile fails to
* parse or returns zero rows.
*
* Returns a dict with:
* - rows : list<LibrarianRecord> (same shape as the rows
* `code_librarian_query` projects)
* - cypher? : the compile path's Cypher (nil for MCTS-only)
* - path : "compile" | "mcts"
* - cache_hit : bool
* - mcts_expansions: int (zero when path = "compile")
* - mcts_depth : int (zero when path = "compile")
* - elapsed_ms : int
*
* Options:
* - llm : dict overriding the default value-tier model
* (Devstral Small via OpenRouter)
* - mcts_depth_max : int (default 4)
* - mcts_expansions_max : int (default 200)
* - session_id : string, when set the fallback path emits a
* `code_librarian_query_nl_fallback` agent
* event so consumers can pin fallback rate.
*
* Implements the RANGER (arxiv 2509.25257) compile + MCTS-fallback shape;
* results in-process cached for 5 minutes on the lowercased NL text.
*
* @effects: [host, llm.call]
* @errors: [backend]
* @api_stability: experimental
* @example: code_librarian_query_nl(harness, "which file defines formatDate?")
*/
pub fn code_librarian_query_nl(harness: Harness, text: string, options: dict? = nil) -> dict {
const opts = options ?? {}
const started_ms = harness.clock.monotonic_ms()
const cache_key = "v1:" + lowercase(trim(text))
const envelope = with_cache_envelope(
harness,
cache_key,
{ -> __nl_resolve(harness.agent, harness.llm, harness.code_index, text, opts) },
{store: {backend: "mem", namespace: __NL_CACHE_NAMESPACE}, ttl_seconds: __NL_CACHE_TTL_SECONDS},
)
const result = envelope.value
return {
cache_hit: envelope.hit,
cypher: result?.cypher,
elapsed_ms: harness.clock.monotonic_ms() - started_ms,
mcts_depth: to_int(result?.mcts_depth ?? 0),
mcts_expansions: to_int(result?.mcts_expansions ?? 0),
path: result.path,
rows: result.rows,
}
}
fn __nl_resolve(
agent: HarnessAgent,
llm: HarnessLlm,
code_index: HarnessCodeIndex,
text: string,
opts: dict,
) -> dict {
const compiled = __nl_compile(llm, text, opts)
if compiled.ok {
const rows = try {
const raw = code_index.cypher({query: compiled.cypher})
raw?.rows ?? []
} catch (_) {
[]
}
if !is_err(rows) && len(rows) > 0 {
return {
cypher: compiled.cypher,
mcts_depth: 0,
mcts_expansions: 0,
path: "compile",
rows: rows,
}
}
}
// Compile-path missed (parse error or zero rows). Fall back to MCTS.
const mcts = __nl_mcts(code_index, text, opts)
__nl_emit_fallback(agent, opts, text, compiled?.cypher, mcts)
return {
cypher: compiled?.cypher,
mcts_depth: mcts.depth,
mcts_expansions: mcts.expansions,
path: "mcts",
rows: mcts.rows,
}
}
fn __nl_emit_fallback(
agent: HarnessAgent,
opts: dict,
text: string,
attempted_cypher: any,
mcts: dict,
) {
const session_id = to_string(opts?.session_id ?? "")
if session_id == "" {
return
}
const _ = try {
agent_emit_event(
agent,
session_id,
"code_librarian_query_nl_fallback",
{
attempted_cypher: attempted_cypher,
mcts_depth: mcts.depth,
mcts_expansions: mcts.expansions,
result_count: len(mcts.rows),
text: text,
},
)
}
}
const __CYPHER_TEMPLATES = [
"MATCH (f:Function {name: '<NAME>'}) RETURN f.path AS path",
"MATCH (t:Type {name: '<NAME>'}) RETURN t.path AS path",
"MATCH (e:EnumCase {name: '<NAME>'}) RETURN e.path AS path, e.container AS enum",
"MATCH (f:Field {name: '<NAME>'}) RETURN f.path AS path, f.container AS container, f.access_level AS access_level",
"MATCH (m:Module) WHERE m.path = '<PATH>' RETURN m.name AS name",
"MATCH (f:Function {name: '<NAME>'})<-[:CALLS]-(c:CallSite) RETURN c.path AS path",
"MATCH (i:Import)<-[:IMPORTS]-(m:Module) WHERE i.name = '<IMPORT>' RETURN m.path AS path",
"MATCH (m:Module)-[:IMPORTS]->(i:Import) WHERE m.path = '<PATH>' RETURN i.name AS import",
"MATCH (parent:Type {name: '<NAME>'})-[:CONTAINS]->(child:Function) RETURN child.name AS name",
"MATCH (f:Function)-[:CALLS*1..3]->(g:Function {name: '<NAME>'}) RETURN f.name AS name, f.path AS path",
]
const __CYPHER_PROMPT_PREAMBLE =
"You translate code-graph questions into a small Cypher dialect over a typed symbol graph.\n\nNode kinds: Function, Type, Field, EnumCase, Module, Import, CallSite, Macro.\nEdge kinds: CALLS, REFS, IMPORTS, CONTAINS, OVERRIDES (each also has an inverse such as CALLED_BY / IMPORTED_BY).\nNode properties: name, path, language, kind, container, access_level, signature, line, file_id, id.\n\nThe dialect supports MATCH … [WHERE …] RETURN. Variable-length traversal up to depth 4 via `*1..N`. WHERE supports =, <>, !=, <, <=, >, >=, AND, OR, NOT. Always alias every projection with AS.\n\nReturn exactly one Cypher query that answers the question. No prose, no markdown, no leading explanation. Aliases on every projected column.\n\nCanonical templates (substitute literal values):\n"
fn __nl_compile(llm: HarnessLlm, text: string, opts: dict) -> dict {
const prompt = __nl_compile_prompt(text)
const schema = schema_object({cypher: schema_string()})
const call_opts = __nl_llm_options(opts)
const envelope = try {
safe_structured_call(llm, prompt, schema, call_opts)
} catch (_) {
return {cypher: nil, ok: false}
}
if is_err(envelope) || !(envelope?.ok ?? false) {
return {cypher: nil, ok: false}
}
const value = envelope?.value ?? envelope?.data ?? {}
const cypher = to_string(value?.cypher ?? "")
const stripped = trim(cypher)
if stripped == "" {
return {cypher: nil, ok: false}
}
return {cypher: stripped, ok: true}
}
fn __nl_compile_prompt(text: string) -> string {
const templates = join(__CYPHER_TEMPLATES, "\n")
return __CYPHER_PROMPT_PREAMBLE + templates + "\n\nQuestion: " + text + "\nCypher:"
}
fn __nl_llm_options(opts: dict) -> dict {
const overrides = if type_of(opts?.llm) == "dict" {
opts.llm
} else {
{}
}
return {
max_tokens: overrides?.max_tokens ?? __DEFAULT_LLM_OPTIONS.max_tokens,
model: overrides?.model ?? __DEFAULT_LLM_OPTIONS.model,
provider: overrides?.provider ?? __DEFAULT_LLM_OPTIONS.provider,
temperature: overrides?.temperature ?? __DEFAULT_LLM_OPTIONS.temperature,
}
}
/**
* Bounded structural enumeration over the typed symbol graph.
*
* Not a stochastic MCTS — embeddings are out of scope (#2434 design
* note) and the graph is small. Performs a bounded BFS-style
* enumeration of name-matched candidate seeds at depth 0, then expands
* `CALLS` / `CALLED_BY` edges up to `mcts_depth_max`, scoring by token
* overlap with the NL question. Every node visited counts toward
* `mcts_expansions_max`. Top-K scored candidates are surfaced as rows
* projected to `{path, name, kind}` so the result shape matches a
* typical canned-query row.
*
* Mirrors the RANGER design: MCTS is the structural safety net, not
* the precision tool. Bounded, deterministic exploration with a token-
* overlap heuristic suffices for the 30-question NL fixture.
*/
fn __nl_mcts(code_index: HarnessCodeIndex, text: any, opts: dict) -> dict {
const depth_cap = to_int(opts?.mcts_depth_max ?? __MCTS_DEPTH_MAX_DEFAULT)
const expansion_cap = to_int(opts?.mcts_expansions_max ?? __MCTS_EXPANSIONS_MAX_DEFAULT)
const terms = __nl_terms(text)
let seen = {}
let ranked = []
let expansions = 0
let depth_reached = 0
for term in terms {
if expansions < expansion_cap {
const probe = __nl_mcts_probe(
code_index,
term,
terms,
depth_cap,
expansion_cap - expansions,
seen,
)
seen = probe.seen
expansions = expansions + probe.expansions
depth_reached = max(depth_reached, probe.depth_reached)
for cand in probe.candidates {
ranked = ranked + [cand]
}
}
}
const sorted = ranked.sorted_by({ cand -> -cand.score })
const unique = __nl_dedupe_candidates(sorted)
const trimmed = if len(unique) > 10 {
unique[:10]
} else {
unique
}
return {
depth: depth_reached,
expansions: expansions,
rows: trimmed.map({ c -> {kind: c.kind, name: c.name, path: c.path} }),
}
}
fn __nl_mcts_probe(
code_index: HarnessCodeIndex,
term: string,
all_terms: list,
depth_cap: int,
budget: int,
seen: dict,
) -> dict {
if budget <= 0 {
return {candidates: [], depth_reached: 0, expansions: 0, seen: seen}
}
const seeds = try {
const raw = code_index.cypher(
{
query: "MATCH (n) WHERE n.name = '" + __nl_escape(term)
+ "' RETURN n.path AS path, n.name AS name, n.kind AS kind",
},
)
raw?.rows ?? []
} catch (_) {
[]
}
let next_seen = seen
let candidates = []
let expansions = 0
let depth_reached = 0
for seed in seeds {
if expansions < budget {
const seed_key = __nl_node_key(seed)
if next_seen[seed_key] == nil {
next_seen = next_seen + {[seed_key]: true}
expansions = expansions + 1
candidates = candidates
+ [
{
kind: to_string(seed?.kind ?? ""),
name: to_string(seed?.name ?? ""),
path: to_string(seed?.path ?? ""),
score: __nl_score(seed, all_terms),
},
]
}
}
}
let depth = 1
while depth <= depth_cap && expansions < budget {
const cypher = "MATCH (s:Function {name: '" + __nl_escape(term) + "'})-[:CALLS*1.."
+ to_string(depth)
+ "]->(n) RETURN n.path AS path, n.name AS name, n.kind AS kind"
const hop = try {
const raw = code_index.cypher({query: cypher})
raw?.rows ?? []
} catch (_) {
[]
}
for n in hop {
if expansions < budget {
const key = __nl_node_key(n)
if next_seen[key] == nil {
next_seen = next_seen + {[key]: true}
expansions = expansions + 1
candidates = candidates
+ [
{
kind: to_string(n?.kind ?? ""),
name: to_string(n?.name ?? ""),
path: to_string(n?.path ?? ""),
score: __nl_score(n, all_terms),
},
]
}
}
}
if len(hop) > 0 {
depth_reached = depth
}
depth = depth + 1
}
const callers_cypher = "MATCH (s:Function {name: '" + __nl_escape(term)
+ "'})<-[:CALLS]-(c:CallSite) RETURN c.path AS path, c.name AS name, c.kind AS kind"
const callers = try {
const raw = code_index.cypher({query: callers_cypher})
raw?.rows ?? []
} catch (_) {
[]
}
for c in callers {
if expansions < budget {
const key = __nl_node_key(c)
if next_seen[key] == nil {
next_seen = next_seen + {[key]: true}
expansions = expansions + 1
candidates = candidates
+ [
{
kind: to_string(c?.kind ?? ""),
name: to_string(c?.name ?? ""),
path: to_string(c?.path ?? ""),
score: __nl_score(c, all_terms),
},
]
}
}
}
return {
candidates: candidates,
depth_reached: depth_reached,
expansions: expansions,
seen: next_seen,
}
}
fn __nl_node_key(row: dict) -> string {
return to_string(row?.kind ?? "") + ":" + to_string(row?.path ?? "") + ":"
+ to_string(row?.name ?? "")
}
fn __nl_dedupe_candidates(cands: list) -> list {
let seen = {}
let out = []
for c in cands {
const key = c.kind + ":" + c.path + ":" + c.name
if seen[key] == nil {
seen = seen + {[key]: true}
out = out + [c]
}
}
return out
}
fn __nl_score(row: dict, terms: list) -> int {
const blob = lowercase(
to_string(row?.name ?? "") + " " + to_string(row?.path ?? "") + " "
+ to_string(row?.kind ?? ""),
)
let score = 0
for t in terms {
if contains(blob, lowercase(t)) {
score = score + 1
}
}
return score
}
/**
* Extract candidate name/path tokens from an NL question, preserving
* original casing so case-sensitive name lookups against the symbol
* graph (e.g. `formatDate` vs `formatdate`) still hit. Splits on
* non-identifier punctuation, drops stop words via a lowercased copy,
* and dedupes.
*/
fn __nl_terms(text: string?) -> list {
const cleaned = regex_replace("[^A-Za-z0-9_/.]+", " ", text)
const raw = split(cleaned, " ")
let seen = {}
let out = []
for word in raw {
const stripped = trim(word)
const lowered = lowercase(stripped)
const usable = stripped != "" && !__nl_is_stop_word(lowered) && seen[stripped] == nil
if usable {
seen = seen + {[stripped]: true}
out = out + [stripped]
}
}
return out
}
const __NL_STOP_WORDS = [
"a",
"an",
"and",
"any",
"are",
"as",
"at",
"be",
"by",
"call",
"calls",
"code",
"contain",
"contains",
"define",
"defined",
"defines",
"do",
"does",
"file",
"files",
"find",
"for",
"from",
"function",
"functions",
"get",
"has",
"have",
"how",
"i",
"if",
"import",
"imports",
"in",
"into",
"is",
"it",
"list",
"many",
"module",
"modules",
"name",
"named",
"names",
"of",
"on",
"or",
"out",
"show",
"struct",
"symbol",
"symbols",
"that",
"the",
"to",
"type",
"types",
"use",
"uses",
"what",
"when",
"where",
"which",
"who",
"why",
"with",
]
fn __nl_is_stop_word(w: string) -> bool {
return __NL_STOP_WORDS.contains(w)
}
/**
* Escape single-quotes for Cypher string literals. The host executor uses
* `'...'` literals and supports `\'` as an escape (see `unescape` in
* `code_index/cypher.rs`).
*/
fn __nl_escape(s: string) -> string {
return __code_librarian_cypher_literal(s)
}