Skip to main content

lean_ctx/core/bm25_index/
source_gate.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Current source rights for index entries at the moment they are served
3//! (G04).
4//!
5//! An index chunk is text copied out of a source at build time. The index
6//! knows which file it came from (`CodeChunk::file_path`) but nothing in the
7//! chunk changes when that file is deleted, restricted, reclassified or
8//! redacted by a rule that did not exist at the build. The digest an index is
9//! bound to (`BM25Index::admission_policy`) covers the gateway policy only,
10//! and a staleness check is a rebuild trigger, not an authorization.
11//!
12//! So an index its store loaded or handed out is asked again for every chunk
13//! it is about to serve: the source is read NOW through the admitted rooted
14//! reader (`core::admitted_sources::read` — path jail, role, project policy,
15//! gateway, decided on the original bytes), and the chunk is served only if
16//! that read is storable and still contains the chunk's text. A source that
17//! is gone, outside the root, withheld, restricted, lossily decoded, changed
18//! or now rewritten at that place does not serve the chunk. This is the same
19//! question `context_admission::stored::serve_stored` asks of a bound copy;
20//! the reader is the one admission, not a second one.
21//!
22//! Where a source authority is required (`source_authority_required`, asked
23//! for the gate's own project), a provider chunk (`scheme://…`) whose origin
24//! the index has on record (`provider_origin`) is asked of its provider the
25//! same way; without protection it follows the originless rule below. The item is
26//! reacquired NOW through `ProviderOrigin::current_chunks` — the check the
27//! knowledge store and the checkpoint import already use (provider tool
28//! allowed for the role and the project policy, same project, providers
29//! enabled, the provider's reuse binding unchanged, the item still returned
30//! and unchanged, content under the project policy) — then passed through the
31//! store admission it was written through, and the chunk is served only if
32//! that text still contains it. One provider request per distinct item,
33//! bounded per gate; over the bound, or without an answer, it is not served.
34//!
35//! The other chunks without a readable text file have an origin of their
36//! own kind on record (`external_origin`), asked the same way:
37//!
38//! - delivered tool output (where a source authority is required): the
39//!   origin is bound to this chunk and the lineage of its delivery holds
40//!   (`archive::delivered::Lineage::still_admitted` — complete, every source
41//!   re-read under the current policy and role and unchanged). Its sources
42//!   are then published to the running call like a file read;
43//! - downstream tool output (where a source authority is required): closed.
44//!   A tool call cannot be replayed, so nothing proves the text current;
45//!   it is not served, not ranked and not counted until downstream tools
46//!   have a re-read contract (`external_origin::DownstreamOrigin`);
47//! - an extracted document (in every mode, a changed file is stale for
48//!   everyone): the rooted reader's path decision admits the file and its
49//!   bytes are the ones the extraction read (`DocumentOrigin::unchanged`).
50//!
51//! In each case the chunk's text must also pass the content admission
52//! unchanged. A served chunk of a provider or a document has no file
53//! lineage: it marks the call's source set unknown.
54//!
55//! A chunk with no origin on record — a chunk written before origins were
56//! kept, a recomputed source that records none — has no source authority
57//! and follows the originless rule of `serve_stored`: withheld while
58//! protection is active in any layer, re-admitted under the content rules in
59//! Community developer mode (an extracted document must also still exist).
60//!
61//! ONE gateway snapshot per gate: the gateway rules are captured when the
62//! gate is built (or handed in by a caller that already judged something
63//! under a snapshot, `SourceGate::with_admission`) and every decision of
64//! the gate uses them — file reads, provider re-admission, the content
65//! admission of chunk text, originless decisions. A configuration that
66//! changes and changes back during a call cannot admit a source the
67//! snapshot refuses. The project pack and the role are those current at
68//! each decision (a caller that needs them pinned holds a source view).
69//!
70//! The reads run inside the call's source capture and Decision Receipt: a
71//! served chunk gives the result an ordinary file lineage, and a refusal is
72//! counted like any other admission of that call. Nothing is cached beyond
73//! one gate; a gate lives for one search.
74
75use std::collections::HashMap;
76use std::path::{Path, PathBuf};
77use std::sync::Arc;
78
79use super::external_origin::ChunkOrigin;
80use super::{BM25Index, ChunkOrigins, CodeChunk, SearchResult};
81use crate::core::context_admission::stores::StoreAdmission;
82use crate::core::providers::provenance::{ProviderOrigin, with_reuse_deadline_until};
83
84/// Tool identity the sources of a search index are re-read for.
85pub(super) const SEARCH_TOOL: &str = "ctx_search";
86
87/// Delivery lineages one gate re-checks (each re-reads every source of one
88/// delivery). Over it an undecided tool-output chunk is refused.
89const MAX_LINEAGE_CHECKS: usize = 16;
90
91/// Distinct sources one gate reads AND remembers — one bound for both. Over
92/// it the gate refuses (fail closed): a source it has not decided yet is not
93/// read, not served and not added to the memo; sources already decided keep
94/// their decision. Fewer results, never an unchecked one, never unbounded
95/// memory.
96pub(crate) const MAX_SOURCES: usize = 128;
97
98/// Provider items one gate reacquires (each is a provider request with the
99/// provider's own deadline). Over it an undecided provider chunk is refused.
100const MAX_PROVIDER_CHECKS: usize = 16;
101
102/// Time one gate — one search — may spend on provider re-acquisition, counted
103/// from its first provider request. A hard bound: no request is started
104/// after it, and a request in flight ends at it
105/// (`providers::provenance::with_reuse_deadline_until` — the earlier of this
106/// deadline and the provider's own). The provider chunks left undecided are
107/// refused (not remembered) and the search returns what was decided. Three
108/// seconds keeps an interactive search answerable while leaving room for a
109/// few items of a healthy provider.
110const PROVIDER_BUDGET: std::time::Duration = std::time::Duration::from_secs(3);
111
112/// The serve-time check of one search over one index.
113pub struct SourceGate {
114    root: PathBuf,
115    tool: &'static str,
116    /// Per source asked so far: the admitted text as the reader returns it
117    /// now, with line endings normalised; `None` when it is not admitted.
118    sources: HashMap<String, Option<String>>,
119    /// Per source whose origin is not compared with a source text (delivered
120    /// or downstream tool output, a document): whether its authority holds.
121    authorities: HashMap<String, bool>,
122    /// The gateway rules every decision of this gate is made under.
123    admission: StoreAdmission,
124    /// Origins the index has on record, by chunk source.
125    origins: Arc<ChunkOrigins>,
126    provider_checks: usize,
127    lineage_checks: usize,
128    /// End of this gate's provider budget, set at its first provider
129    /// request ([`PROVIDER_BUDGET`]).
130    provider_deadline: Option<std::time::Instant>,
131    /// The root as the request scope names it (canonical when resolvable).
132    scope_root: PathBuf,
133    /// Whether a source authority is required for the project of this
134    /// gate's root; asked once per gate.
135    protected: Option<bool>,
136    withheld: usize,
137}
138
139impl SourceGate {
140    /// A gate that captures the gateway rules as they are configured now
141    /// (`carrier::gateway_snapshot`, unobserved) and decides everything
142    /// under them.
143    pub(crate) fn new(root: &Path, tool: &'static str) -> Self {
144        Self::with_admission(
145            root,
146            tool,
147            &crate::core::context_admission::carrier::gateway_snapshot(),
148        )
149    }
150
151    /// A gate that decides under the gateway snapshot its caller already
152    /// holds: what the caller judged under `admission` and the sources this
153    /// gate reads are then judged under the same gateway rules.
154    pub(crate) fn with_admission(
155        root: &Path,
156        tool: &'static str,
157        admission: &StoreAdmission,
158    ) -> Self {
159        Self {
160            root: root.to_path_buf(),
161            tool,
162            sources: HashMap::new(),
163            authorities: HashMap::new(),
164            admission: admission.clone(),
165            origins: Arc::default(),
166            provider_checks: 0,
167            lineage_checks: 0,
168            provider_deadline: None,
169            scope_root: root.canonicalize().unwrap_or_else(|_| root.to_path_buf()),
170            protected: None,
171            withheld: 0,
172        }
173    }
174
175    /// Run `operation` in the project scope of this gate's root: the gate
176    /// decides for the index it belongs to, whatever project the calling
177    /// request or the working directory names. A request already bound to
178    /// this project keeps its scope untouched (and with it an open source
179    /// view). Under a request bound to ANOTHER project with a source view
180    /// open, `policy::runtime::active()` answers deny-all for the rebound
181    /// scope and invalidates that view: the gate then sees protection, and
182    /// the enclosing operation ends with an error — closed, never the other
183    /// project's rules applied to this index.
184    fn in_root_scope<T>(&self, operation: impl FnOnce() -> T) -> T {
185        use crate::core::policy::runtime::REQUEST_PROJECT;
186        let bound = REQUEST_PROJECT
187            .try_with(|slot| slot.borrow().clone())
188            .ok()
189            .flatten();
190        if bound.is_some_and(|bound| {
191            bound == self.scope_root || bound.canonicalize().is_ok_and(|b| b == self.scope_root)
192        }) {
193            return operation();
194        }
195        REQUEST_PROJECT.sync_scope(
196            std::cell::RefCell::new(Some(self.scope_root.clone())),
197            operation,
198        )
199    }
200
201    /// Whether protection is active in some layer for this gate's project
202    /// (`stored::source_authority_required_under`): the gateway side is read
203    /// from THIS gate's snapshot — the mode cannot differ from the rules the
204    /// gate admits under —, the pack side from the project policy in scope
205    /// (pinned for the whole search by [`pinned`]). Asked once per gate.
206    pub(crate) fn protected(&mut self) -> bool {
207        if let Some(protected) = self.protected {
208            return protected;
209        }
210        let protected = self.in_root_scope(|| {
211            crate::core::context_admission::stored::source_authority_required_under(
212                self.admission.admission_policy(),
213            )
214        });
215        self.protected = Some(protected);
216        protected
217    }
218
219    /// This gate for an index that has `origins` on record.
220    #[must_use]
221    pub(crate) fn with_origins(mut self, origins: &Arc<ChunkOrigins>) -> Self {
222        self.origins = Arc::clone(origins);
223        self
224    }
225
226    /// Whether this gate already holds as many decisions as it may.
227    fn over_bound(&self) -> bool {
228        self.sources.len() + self.authorities.len() >= MAX_SOURCES
229    }
230
231    /// The decision for a source whose origin is not compared with a source
232    /// text, asked once per gate in the gate's project scope. Over the bound:
233    /// refused, neither asked nor remembered.
234    fn authority_holds(&mut self, source: &str, check: impl FnOnce(&Self) -> bool) -> bool {
235        if let Some(&holds) = self.authorities.get(source) {
236            return holds;
237        }
238        if self.over_bound() {
239            return false;
240        }
241        let holds = self.in_root_scope(|| check(self));
242        self.authorities.insert(source.to_owned(), holds);
243        holds
244    }
245
246    /// Whether the content admission, under this gate's gateway snapshot and
247    /// the current project policy, leaves `text` exactly as it is and lets
248    /// it be kept: a copy the rules would withhold, restrict or rewrite is
249    /// not the copy the index holds. `label` is the source the text was
250    /// admitted under when it has one (a path, an `mcp://` reference).
251    fn admits_verbatim(&self, text: &str, label: Option<&str>) -> bool {
252        use crate::core::context_admission as admission;
253        self.in_root_scope(|| {
254            let admitted = match label {
255                Some(label) => admission::admit_source_with_snapshot(
256                    text,
257                    label,
258                    false,
259                    admission::OutputCompleteness::complete(),
260                    &self.admission,
261                )
262                .ok(),
263                None => admission::carrier::admit_carrier_text_under(
264                    text,
265                    Some(self.admission.admission_policy()),
266                    &admission::carrier::STORED,
267                )
268                .ok(),
269            };
270            admitted.is_some_and(|admitted| admitted.storable && admitted.text == text)
271        })
272    }
273
274    /// The originless rule (`serve_stored` without a binding) under this
275    /// gate's snapshot: withheld where a source authority is required,
276    /// otherwise the content admission alone.
277    fn originless(&mut self, text: &str) -> bool {
278        !self.protected() && self.admits_verbatim(text, None)
279    }
280
281    /// A chunk was served whose source the call's capture cannot name (a
282    /// provider item, a downstream result, a document): where a source
283    /// authority is required the call's source set is then unknown, never
284    /// silently the files alone. Always `true`.
285    fn served_without_file_lineage(&mut self) -> bool {
286        if self.protected() {
287            crate::core::archive::authority::mark_unbound();
288        }
289        true
290    }
291
292    /// The decision already on record for `text` of `source`, without a read
293    /// and without counting: `None` while the source is undecided.
294    pub(super) fn decided(&self, source: &str, text: &str) -> Option<bool> {
295        let admitted = self.sources.get(source)?;
296        Some(
297            admitted
298                .as_deref()
299                .is_some_and(|current| current.contains(normalised(text).as_str())),
300        )
301    }
302
303    /// The sources this gate decided are not admitted now.
304    pub(crate) fn refused_sources(&self) -> impl Iterator<Item = &str> {
305        self.sources
306            .iter()
307            .filter(|(_, admitted)| admitted.is_none())
308            .map(|(source, _)| source.as_str())
309    }
310
311    /// Whether `text`, derived from `source` (a path relative to the root,
312    /// or an external reference), may be served now.
313    pub(crate) fn serves(&mut self, source: &str, text: &str) -> bool {
314        self.serves_with(source, text, None)
315    }
316
317    /// [`Self::serves`] for a decision that delivers nothing (the ranking
318    /// corpus, the dense backend's source filter): the origin ledger is left
319    /// alone, so only the chunks a result actually carries taint the session.
320    pub(crate) fn admits_unserved(&mut self, source: &str, text: &str) -> bool {
321        let admitted = self.decide(source, text);
322        if !admitted {
323            self.withheld += 1;
324        }
325        admitted
326    }
327
328    /// [`Self::serves`] for a chunk with its recorded origin. A served chunk
329    /// of an external source (`scheme://`) is noted in the call's origin
330    /// ledger with its origin, or as unknown when it has none (an index
331    /// written before origins were kept, pz21 §1.6): the release point then
332    /// fences it.
333    pub(crate) fn serves_with(
334        &mut self,
335        source: &str,
336        text: &str,
337        input_origin: Option<&crate::core::context_admission::origin::InputOrigin>,
338    ) -> bool {
339        let served = self.decide(source, text);
340        if served {
341            Self::note_served(source, input_origin);
342        } else {
343            self.withheld += 1;
344        }
345        served
346    }
347
348    /// Note a served chunk of an external source (`scheme://`, or one with a
349    /// recorded origin) in the call's origin ledger.
350    pub(crate) fn note_served(
351        source: &str,
352        input_origin: Option<&crate::core::context_admission::origin::InputOrigin>,
353    ) {
354        if source.contains("://") || input_origin.is_some() {
355            crate::core::context_admission::capture::note_origin(
356                &crate::core::context_admission::stores::class_of_path(
357                    Path::new(source),
358                    input_origin,
359                ),
360            );
361        }
362    }
363
364    /// Entries this gate refused. A count only: no path, no text.
365    #[cfg(test)]
366    pub(crate) fn withheld(&self) -> usize {
367        self.withheld
368    }
369
370    fn decide(&mut self, source: &str, text: &str) -> bool {
371        let external = source.contains("://");
372        let document = !external && crate::core::binary_detect::has_binary_extension(source);
373        let origins = Arc::clone(&self.origins);
374        let recorded = origins.get(source);
375        if let (true, Some(ChunkOrigin::Document(origin))) = (document, recorded) {
376            // In every mode: a document whose bytes changed is stale.
377            let label = self.root.join(source);
378            return self.authority_holds(source, |gate| {
379                origin.unchanged(&gate.root, source, gate.tool)
380            }) && self.admits_verbatim(text, label.to_str())
381                && self.served_without_file_lineage();
382        }
383        // An external origin is asked only where a source authority is
384        // required. Without protection (Community developer mode) an
385        // external chunk follows the originless rule: content re-admitted,
386        // no provider round trip, search usable offline.
387        let origin = match recorded {
388            Some(recorded) if external && self.protected() => match recorded {
389                ChunkOrigin::Provider(origin) => Some(&**origin),
390                ChunkOrigin::Delivery(origin) => {
391                    if !origin.binds(text) {
392                        return false;
393                    }
394                    if !self.authorities.contains_key(source) {
395                        if self.lineage_checks >= MAX_LINEAGE_CHECKS {
396                            return false;
397                        }
398                        self.lineage_checks += 1;
399                    }
400                    let served = self.authority_holds(source, |_| origin.still_admitted())
401                        && self.admits_verbatim(text, None);
402                    if served {
403                        origin.publish();
404                    }
405                    return served;
406                }
407                // Closed: a downstream result cannot be re-read, so nothing
408                // proves it current. That its server and tool are still
409                // allowed is not such a proof (`DownstreamOrigin`). The
410                // refusal is the originless one, `store.no_source_authority`.
411                ChunkOrigin::Downstream(_) => return false,
412                ChunkOrigin::Document(_) => None,
413            },
414            _ => None,
415        };
416        if origin.is_none() && (external || document) {
417            // No source that is asked: the originless rule, judged for this
418            // gate's project. An extracted document must at least still exist.
419            return (external || self.root.join(source).is_file()) && self.originless(text);
420        }
421        if let Some(served) = self.decided(source, text) {
422            return served && (origin.is_none() || self.served_without_file_lineage());
423        }
424        if self.over_bound() {
425            // Over the bound: refused, neither read nor remembered.
426            return false;
427        }
428        let admitted = match origin {
429            Some(origin) => {
430                let deadline = *self
431                    .provider_deadline
432                    .get_or_insert_with(|| std::time::Instant::now() + PROVIDER_BUDGET);
433                if self.provider_checks >= MAX_PROVIDER_CHECKS
434                    || std::time::Instant::now() >= deadline
435                {
436                    // Over the budget: refused, neither asked nor remembered.
437                    return false;
438                }
439                self.provider_checks += 1;
440                let admitted = self.read_provider(origin, deadline);
441                if admitted.is_none() && std::time::Instant::now() >= deadline {
442                    // Cut off by the budget, not answered: not a decision.
443                    return false;
444                }
445                admitted
446            }
447            None => self.read(source),
448        };
449        self.sources.insert(source.to_owned(), admitted);
450        self.decided(source, text).unwrap_or(false)
451            && (origin.is_none() || self.served_without_file_lineage())
452    }
453
454    /// What the provider returns for `origin` NOW, as the store admission
455    /// would keep it; `None` when the item is not admitted to this reader.
456    fn read_provider(
457        &self,
458        origin: &ProviderOrigin,
459        deadline: std::time::Instant,
460    ) -> Option<String> {
461        let root = self.root.to_str()?;
462        let view = match crate::core::policy::runtime::with_project_source_view(root, || {
463            with_reuse_deadline_until(Some(deadline), || origin.current_chunks(root))
464        }) {
465            Ok(Ok(view)) => view,
466            Ok(Err(refusal)) => {
467                // The kind only, never the item: a search must not tell an
468                // item it withholds from one that does not exist (K02). The
469                // call's receipt records the refused re-check (C04).
470                crate::core::context_admission::record_recheck_refusal(
471                    &self.admission,
472                    refusal.validity,
473                );
474                return None;
475            }
476            Err(_) => return None,
477        };
478        let mut current = String::new();
479        for chunk in &view.chunks {
480            let admitted = self.admission.admit(
481                &chunk.content,
482                Path::new(&chunk.file_path),
483                chunk.input_origin.as_ref(),
484            )?;
485            current.push_str(&normalised(&admitted));
486            current.push('\n');
487        }
488        Some(current)
489    }
490
491    /// The text of `source` as the admitted reader returns it now, when the
492    /// original may be kept in a derived store (complete, not restricted).
493    /// The gateway rules are this gate's snapshot; each read is bounded by
494    /// the read cap (the gate never shared a byte budget across sources).
495    fn read(&self, source: &str) -> Option<String> {
496        let path = self.root.join(source);
497        // NOTE the argument order of this entry: root first.
498        let read = crate::core::admitted_sources::read_with_admission(
499            self.root.to_str()?,
500            path.to_str()?,
501            self.tool,
502            &self.admission,
503        );
504        read.storable().map(normalised)
505    }
506}
507
508/// Line endings are not content: an index keeps `\n`-joined lines of a
509/// source that may use `\r\n`.
510fn normalised(text: &str) -> String {
511    text.lines().collect::<Vec<_>>().join("\n")
512}
513
514/// Whether `chunk` may be served through `gate`. Without a gate — a view
515/// built in the running call from sources the reader returned then — every
516/// chunk is served.
517pub fn chunk_served(gate: &mut Option<SourceGate>, chunk: &CodeChunk) -> bool {
518    gate.as_mut().is_none_or(|gate| {
519        gate.serves_with(
520            &chunk.file_path,
521            &chunk.content,
522            chunk.input_origin.as_ref(),
523        )
524    })
525}
526
527impl BM25Index {
528    /// This index as its store hands it out for `root`: every search over it
529    /// is gated ([`Self::source_gate`]).
530    #[must_use]
531    pub(crate) fn rooted_at(mut self, root: &Path) -> Self {
532        self.source_root = Some(root.to_path_buf());
533        self
534    }
535
536    /// The gate of one search: `None` for an index without a root.
537    pub fn source_gate(&self) -> Option<SourceGate> {
538        self.source_root
539            .as_deref()
540            .map(|root| SourceGate::new(root, SEARCH_TOOL).with_origins(&self.provider_origins))
541    }
542
543    /// The `top_k` best chunks for `query` that may be served now. Pack and
544    /// role are pinned for the whole search (`pinned`); if either changed
545    /// while it ran, nothing is returned.
546    pub fn search(&self, query: &str, top_k: usize) -> Vec<SearchResult> {
547        pinned(self.source_root.as_deref(), || {
548            self.search_gated(query, top_k, &mut self.source_gate())
549        })
550        .unwrap_or_default()
551    }
552}
553
554/// Run one search over an index handed out for `root` with the project pack
555/// and the role pinned from its first decision to its last
556/// (`policy::runtime::with_project_source_view`): every gate decision inside
557/// sees one pack and one role, and a change of either while the search ran
558/// makes the view report it — the result is then `None` and the caller
559/// returns nothing. Nested calls share the outermost view.
560///
561/// A request bound to ANOTHER project keeps its own view; the gate's
562/// rebinding to its root then answers deny-all and invalidates that view
563/// (`SourceGate::in_root_scope`), so such a search also ends in `None`. A
564/// root that cannot be resolved has no view: `None`. Without a root (a view
565/// built in the running call, no gate) the search runs as it is.
566pub fn pinned<T>(root: Option<&Path>, search: impl FnOnce() -> T) -> Option<T> {
567    let Some(root) = root else {
568        return Some(search());
569    };
570    // An index of a root the request's Workspace does not contain (detached,
571    // or never attached) serves nothing (#2120).
572    if !crate::core::workspace_authority::root_authorized(root) {
573        return None;
574    }
575    crate::core::policy::runtime::with_project_source_view(root.to_str()?, search).ok()
576}
577
578/// How many candidates a similarity search fetches to serve `top_k` through
579/// a gate: a refused candidate must not cost a result slot, and the walk
580/// stays bounded. More refusals than the surplus mean fewer results.
581#[cfg(feature = "embeddings")]
582pub(crate) fn overfetch(top_k: usize) -> usize {
583    top_k.saturating_add(top_k.clamp(16, MAX_SOURCES))
584}
585
586#[cfg(test)]
587mod tests {
588    use super::*;
589    use crate::core::policy::runtime::TestPolicyOverride;
590
591    /// A chunk of a persisted index is served while the admitted reader still
592    /// returns its text, and is no longer served once the source is withdrawn:
593    /// deleted, or rewritten so that the stored text is no longer in it. The
594    /// next chunk takes its place; an index without a root is not asked.
595    #[test]
596    fn a_loaded_index_serves_a_chunk_only_while_its_source_admits_it() {
597        let _data = crate::core::data_dir::isolated_data_dir();
598        let _community = TestPolicyOverride::set(None);
599        let project = tempfile::tempdir().expect("project");
600        let root = project.path();
601        let withdrawn = root.join("billing.rs");
602        std::fs::write(
603            &withdrawn,
604            "pub fn settle_canary() -> u32 {\n    let reconciliation = 7319;\n    reconciliation\n}\n",
605        )
606        .expect("source");
607        std::fs::write(
608            root.join("ledger.rs"),
609            "pub fn settle_ledger() -> u32 {\n    let reconciliation = 42;\n    reconciliation\n}\n",
610        )
611        .expect("source");
612
613        let built = BM25Index::build_from_directory(root);
614        built.save(root).expect("persisted");
615        let files = |index: &BM25Index| -> Vec<String> {
616            index
617                .search("reconciliation", 10)
618                .into_iter()
619                .map(|hit| hit.file_path)
620                .collect()
621        };
622
623        let loaded = BM25Index::load(root).expect("index of the current policy");
624        assert!(loaded.source_root.is_some());
625        let served = files(&loaded);
626        assert!(served.iter().any(|file| file == "billing.rs"), "{served:?}");
627        assert!(served.iter().any(|file| file == "ledger.rs"), "{served:?}");
628        assert!(!loaded.search("settle_canary", 5).is_empty());
629        // The same chunks without a root (taken before the source is
630        // withdrawn: a search that finds it deleted evicts it from the store).
631        let unrooted = BM25Index::load_persisted(root).expect("persisted file");
632
633        // Rewritten: the stored text is no longer what the reader returns.
634        std::fs::write(&withdrawn, "pub fn unrelated() {}\n").expect("rewrite");
635        let mut gate = loaded.source_gate();
636        let hits = loaded.search_gated("reconciliation", 10, &mut gate);
637        assert!(hits.iter().all(|hit| hit.file_path == "ledger.rs"));
638        assert!(!hits.is_empty());
639        assert!(gate.as_ref().is_some_and(|gate| gate.withheld() >= 1));
640
641        // Deleted: nothing of it is served, name and path included, and the
642        // chunk of the other source takes the slot.
643        std::fs::remove_file(&withdrawn).expect("delete");
644        let served = files(&loaded);
645        assert!(!served.is_empty());
646        assert!(served.iter().all(|file| file == "ledger.rs"), "{served:?}");
647        assert!(loaded.search("settle_canary", 5).is_empty());
648
649        // Over the bound: an undecided source is refused and not remembered.
650        let mut full = SourceGate::new(root, SEARCH_TOOL);
651        for n in 0..MAX_SOURCES {
652            full.sources.insert(format!("known-{n}.rs"), None);
653        }
654        assert!(!full.serves("ledger.rs", "reconciliation"));
655        assert_eq!(full.sources.len(), MAX_SOURCES);
656
657        // Without a root: a fresh view, not asked again.
658        assert!(unrooted.source_root.is_none());
659        assert!(files(&unrooted).iter().any(|file| file == "billing.rs"));
660    }
661}