1use std::collections::{HashMap, HashSet};
2use std::path::Path;
3use std::sync::{Arc, Mutex};
4use std::time::Instant;
5
6use crate::config;
7use crate::config::Config;
8use crate::error::CoreError;
9use crate::ingest::canonical::build_canonical_document;
10use crate::ingest::chunk::{
11 chunk_canonical_document, chunk_canonical_document_with_counter, resolve_policy,
12};
13use crate::ingest::extract::default_registry;
14use crate::lock::{LockMode, OperationLock};
15use crate::models;
16use crate::retrieval_context::{EmbeddingDocumentInput, DENSE_DOCUMENT_RENDER_IDENTITY};
17use crate::storage::Storage;
18use crate::storage::{
19 ChunkInsert, ChunkRow, CollectionRow, DocumentGenerationReplace, DocumentRow, DocumentTextRow,
20 SpaceResolution, TantivyEntry,
21};
22use crate::Result;
23use kbolt_types::{
24 ActiveSpace, ActiveSpaceSource, AddCollectionRequest, AddCollectionResult, CollectionInfo,
25 CollectionStatus, DocumentResponse, FileEntry, GetChunkRequest, GetRequest,
26 InitialIndexingBlock, InitialIndexingOutcome, KboltError, Locator, ModelStatus, MultiGetItem,
27 MultiGetItemKind, MultiGetRequest, MultiGetResponse, OmitReason, OmittedItem, ReadLocator,
28 SearchEmptyReason, SearchMode, SearchPipeline, SearchPipelineNotice, SearchPipelineStep,
29 SearchPipelineUnavailableReason, SearchRequest, SearchResponse, SearchResult,
30 SearchResultChunk, SearchSignals, SpaceInfo, SpaceStatus, StatusResponse, UpdateOptions,
31 UpdateReport,
32};
33mod eval_ops;
34mod file_utils;
35pub(crate) mod ignore_helpers;
36mod ignore_ops;
37mod path_utils;
38mod schedule_ops;
39mod schedule_run_ops;
40mod schedule_status_ops;
41mod scoring;
42mod search_ops;
43mod text_helpers;
44mod update_ops;
45use file_utils::{file_error, modified_token, sha256_hex};
46use ignore_helpers::{
47 collection_ignore_file_path, count_ignore_patterns, is_hard_ignored_file,
48 load_collection_ignore_matcher, validate_ignore_pattern,
49};
50use path_utils::{
51 collection_relative_path, extension_allowed, normalize_docid, normalize_list_prefix,
52 normalized_extension_filter, path_matches_prefix, short_docid, split_collection_path,
53};
54use scoring::{dense_distance_to_score, max_option};
55#[cfg(test)]
56pub(crate) use text_helpers::retrieval_text_with_prefix;
57use text_helpers::search_text_with_loaded_canonical_neighbors;
58
59pub struct Engine {
60 storage: Storage,
61 config: Config,
62 embedder: Option<Arc<dyn models::Embedder>>,
63 embedding_document_sizer: Option<Arc<dyn models::EmbeddingDocumentSizer>>,
64 reranker: Option<Arc<dyn models::Reranker>>,
65 expander: Option<Arc<dyn models::Expander>>,
66}
67
68#[derive(Debug, Clone, PartialEq, Eq)]
69pub struct IgnoreListEntry {
70 pub space: String,
71 pub collection: String,
72 pub pattern_count: usize,
73}
74
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct UpdateTarget {
77 pub space: String,
78 pub collection: CollectionRow,
79}
80
81#[derive(Debug, Clone, Copy)]
82struct TargetScope<'a> {
83 space: Option<&'a str>,
84 collections: &'a [String],
85}
86
87#[derive(Debug, Clone)]
88struct SearchCollectionMeta {
89 space: String,
90 collection: String,
91}
92
93#[derive(Debug)]
94struct SearchTargetScope {
95 space: String,
96 collection_ids: Vec<i64>,
97 filtered: bool,
98 document_ids: Vec<i64>,
99 chunk_count: usize,
100 chunk_key_filter: Mutex<Option<Arc<HashSet<u64>>>>,
101 bm25_reloaded: Mutex<bool>,
102}
103
104#[derive(Debug, Clone)]
105struct SearchHitCandidate {
106 chunk_id: i64,
107 bm25_score: f32,
108}
109
110#[derive(Debug, Clone)]
111struct RankedChunk {
112 chunk_id: i64,
113 score: f32,
114 fusion: f32,
115 reranker: Option<f32>,
116 bm25: Option<f32>,
117 dense: Option<f32>,
118 original_rank: Option<usize>,
119}
120
121impl Engine {
122 pub fn new(config_path: Option<&Path>) -> Result<Self> {
123 Self::new_with_recovery_notice(config_path, None)
124 }
125
126 pub fn new_with_recovery_notice(
127 config_path: Option<&Path>,
128 recovery_notice: Option<crate::RecoveryNoticeSink>,
129 ) -> Result<Self> {
130 let config = config::load(config_path)?;
131 let storage = Storage::new(&config.cache_dir)?;
132 let built_models =
133 models::build_inference_clients_with_recovery_notice(&config, recovery_notice)?;
134 Ok(Self {
135 storage,
136 config,
137 embedder: built_models.embedder,
138 embedding_document_sizer: built_models.embedding_document_sizer,
139 reranker: built_models.reranker,
140 expander: built_models.expander,
141 })
142 }
143
144 #[cfg(test)]
145 pub(crate) fn from_parts(storage: Storage, config: Config) -> Self {
146 Self::from_parts_with_models(storage, config, None, None, None)
147 }
148
149 #[cfg(test)]
150 pub(crate) fn from_parts_with_embedder(
151 storage: Storage,
152 config: Config,
153 embedder: Option<Arc<dyn models::Embedder>>,
154 ) -> Self {
155 Self::from_parts_with_inference(storage, config, embedder, None, None, None)
156 }
157
158 #[cfg(test)]
159 pub(crate) fn from_parts_with_embedding_runtime(
160 storage: Storage,
161 config: Config,
162 embedder: Option<Arc<dyn models::Embedder>>,
163 embedding_document_sizer: Option<Arc<dyn models::EmbeddingDocumentSizer>>,
164 ) -> Self {
165 Self::from_parts_with_inference(
166 storage,
167 config,
168 embedder,
169 embedding_document_sizer,
170 None,
171 None,
172 )
173 }
174
175 #[cfg(test)]
176 pub(crate) fn from_parts_with_models(
177 storage: Storage,
178 config: Config,
179 embedder: Option<Arc<dyn models::Embedder>>,
180 reranker: Option<Arc<dyn models::Reranker>>,
181 expander: Option<Arc<dyn models::Expander>>,
182 ) -> Self {
183 Self::from_parts_with_inference(storage, config, embedder, None, reranker, expander)
184 }
185
186 #[cfg(test)]
187 fn from_parts_with_inference(
188 storage: Storage,
189 config: Config,
190 embedder: Option<Arc<dyn models::Embedder>>,
191 embedding_document_sizer: Option<Arc<dyn models::EmbeddingDocumentSizer>>,
192 reranker: Option<Arc<dyn models::Reranker>>,
193 expander: Option<Arc<dyn models::Expander>>,
194 ) -> Self {
195 let built_models =
196 models::build_inference_clients(&config).expect("build inference models");
197 Self {
198 storage,
199 config,
200 embedder: embedder.or(built_models.embedder),
201 embedding_document_sizer: embedding_document_sizer
202 .or(built_models.embedding_document_sizer),
203 reranker: reranker.or(built_models.reranker),
204 expander: expander.or(built_models.expander),
205 }
206 }
207
208 pub fn add_space(&self, name: &str, description: Option<&str>) -> Result<SpaceInfo> {
209 let _lock = self.acquire_operation_lock(LockMode::Exclusive)?;
210 self.storage.create_space(name, description)?;
211 let space = self.storage.get_space(name)?;
212 self.build_space_info(&space)
213 }
214
215 pub fn remove_space(&mut self, name: &str) -> Result<()> {
216 let _lock = self.acquire_operation_lock(LockMode::Exclusive)?;
217 self.storage.delete_space(name)?;
218 if self.config.default_space.as_deref() == Some(name) {
219 self.persist_default_space(None)?;
220 }
221 Ok(())
222 }
223
224 pub fn rename_space(&mut self, old: &str, new: &str) -> Result<()> {
225 let _lock = self.acquire_operation_lock(LockMode::Exclusive)?;
226 self.storage.rename_space(old, new)?;
227 if self.config.default_space.as_deref() == Some(old) {
228 self.persist_default_space(Some(new.to_string()))?;
229 }
230 Ok(())
231 }
232
233 pub fn describe_space(&self, name: &str, description: &str) -> Result<()> {
234 let _lock = self.acquire_operation_lock(LockMode::Exclusive)?;
235 self.storage.update_space_description(name, description)
236 }
237
238 pub fn list_spaces(&self) -> Result<Vec<SpaceInfo>> {
239 let spaces = self.storage.list_spaces()?;
240 let mut infos = Vec::with_capacity(spaces.len());
241 for space in spaces {
242 infos.push(self.build_space_info(&space)?);
243 }
244 Ok(infos)
245 }
246
247 pub fn space_info(&self, name: &str) -> Result<SpaceInfo> {
248 let space = self.storage.get_space(name)?;
249 self.build_space_info(&space)
250 }
251
252 pub fn set_default_space(&mut self, name: Option<&str>) -> Result<Option<String>> {
253 if let Some(space_name) = name {
254 self.storage.get_space(space_name)?;
255 }
256
257 self.persist_default_space(name.map(ToString::to_string))?;
258 Ok(self.config.default_space.clone())
259 }
260
261 fn persist_default_space(&mut self, default_space: Option<String>) -> Result<()> {
262 let previous = self.config.default_space.clone();
263 self.config.default_space = default_space;
264 if let Err(err) = config::save(&self.config) {
265 self.config.default_space = previous;
266 return Err(err);
267 }
268 Ok(())
269 }
270
271 pub fn add_collection(&self, req: AddCollectionRequest) -> Result<AddCollectionResult> {
272 let _lock = self.acquire_operation_lock(LockMode::Exclusive)?;
273 let AddCollectionRequest {
274 path,
275 space: requested_space,
276 name: requested_name,
277 description,
278 extensions,
279 no_index,
280 } = req;
281
282 let space = match requested_space.as_deref() {
283 Some(space_name) => match self.storage.get_space(space_name) {
284 Ok(space) => space,
285 Err(CoreError::Domain(KboltError::SpaceNotFound { .. })) => {
286 self.storage.create_space(space_name, None)?;
287 self.storage.get_space(space_name)?
288 }
289 Err(err) => return Err(err),
290 },
291 None => self.resolve_space_row(None, None)?,
292 };
293 if !path.is_absolute() || !path.is_dir() {
294 return Err(KboltError::InvalidPath(path).into());
295 }
296
297 let name = match requested_name {
298 Some(name) => name,
299 None => path
300 .file_name()
301 .and_then(|name| name.to_str())
302 .map(ToString::to_string)
303 .ok_or_else(|| KboltError::InvalidPath(path.clone()))?,
304 };
305
306 self.storage.create_collection(
307 space.id,
308 &name,
309 &path,
310 description.as_deref(),
311 extensions.as_deref(),
312 )?;
313
314 let initial_indexing = if no_index {
315 InitialIndexingOutcome::Skipped
316 } else {
317 match self.update_unlocked(UpdateOptions {
318 space: Some(space.name.clone()),
319 collections: vec![name.clone()],
320 no_embed: false,
321 dry_run: false,
322 verbose: false,
323 }) {
324 Ok(report) => InitialIndexingOutcome::Indexed(report),
325 Err(CoreError::Domain(KboltError::SpaceDenseRepairRequired { space, reason })) => {
326 InitialIndexingOutcome::Blocked(
327 InitialIndexingBlock::SpaceDenseRepairRequired { space, reason },
328 )
329 }
330 Err(CoreError::Domain(KboltError::ModelNotAvailable { name })) => {
331 InitialIndexingOutcome::Blocked(InitialIndexingBlock::ModelNotAvailable {
332 name,
333 })
334 }
335 Err(err) => return Err(err),
336 }
337 };
338
339 let collection = self.storage.get_collection(space.id, &name)?;
340 Ok(AddCollectionResult {
341 collection: self.build_collection_info(&space.name, &collection)?,
342 initial_indexing,
343 })
344 }
345
346 pub fn remove_collection(&self, space: Option<&str>, name: &str) -> Result<()> {
347 let _lock = self.acquire_operation_lock(LockMode::Exclusive)?;
348 let resolved = self.resolve_space_row(space, Some(name))?;
349 let collection = self.storage.get_collection(resolved.id, name)?;
350 let documents = self.storage.list_documents(collection.id, false)?;
351 let doc_ids = documents.into_iter().map(|doc| doc.id).collect::<Vec<_>>();
352 let chunk_ids = self.collect_document_chunk_ids(&doc_ids)?;
353 self.purge_space_chunks(&resolved.name, &chunk_ids)?;
354 self.storage.delete_collection(resolved.id, name)?;
355
356 let ignore_path =
357 collection_ignore_file_path(&self.config.config_dir, &resolved.name, name);
358 if ignore_path.is_file() {
359 std::fs::remove_file(ignore_path)?;
360 }
361
362 Ok(())
363 }
364
365 pub fn rename_collection(&self, space: Option<&str>, old: &str, new: &str) -> Result<()> {
366 let _lock = self.acquire_operation_lock(LockMode::Exclusive)?;
367 let resolved = self.resolve_space_row(space, Some(old))?;
368 let old_ignore_path =
369 collection_ignore_file_path(&self.config.config_dir, &resolved.name, old);
370 let new_ignore_path =
371 collection_ignore_file_path(&self.config.config_dir, &resolved.name, new);
372 if old_ignore_path.is_file() && new_ignore_path.exists() {
373 return Err(KboltError::Internal(format!(
374 "cannot rename ignore file: destination already exists: {}",
375 new_ignore_path.display()
376 ))
377 .into());
378 }
379
380 self.storage.rename_collection(resolved.id, old, new)?;
381
382 if old_ignore_path.is_file() {
383 if let Some(parent) = new_ignore_path.parent() {
384 std::fs::create_dir_all(parent)?;
385 }
386 if let Err(rename_err) = std::fs::rename(&old_ignore_path, &new_ignore_path) {
387 match self.storage.rename_collection(resolved.id, new, old) {
388 Ok(()) => {
389 return Err(KboltError::Internal(format!(
390 "renamed collection was rolled back after ignore rename failure: {rename_err}"
391 ))
392 .into())
393 }
394 Err(rollback_err) => {
395 return Err(KboltError::Internal(format!(
396 "ignore rename failed: {rename_err}; rollback failed: {rollback_err}"
397 ))
398 .into())
399 }
400 }
401 }
402 }
403
404 Ok(())
405 }
406
407 pub fn describe_collection(&self, space: Option<&str>, name: &str, desc: &str) -> Result<()> {
408 let _lock = self.acquire_operation_lock(LockMode::Exclusive)?;
409 let resolved = self.resolve_space_row(space, Some(name))?;
410 self.storage
411 .update_collection_description(resolved.id, name, desc)
412 }
413
414 pub fn list_collections(&self, space: Option<&str>) -> Result<Vec<CollectionInfo>> {
415 let (space_id_filter, spaces_by_id) = if let Some(space_name) = space {
416 let resolved = self.resolve_space_row(Some(space_name), None)?;
417 let mut map = std::collections::HashMap::new();
418 map.insert(resolved.id, resolved.name.clone());
419 (Some(resolved.id), map)
420 } else {
421 let spaces = self.storage.list_spaces()?;
422 let map = spaces
423 .into_iter()
424 .map(|space| (space.id, space.name))
425 .collect::<std::collections::HashMap<_, _>>();
426 (None, map)
427 };
428
429 let collections = self.storage.list_collections(space_id_filter)?;
430 let mut infos = Vec::with_capacity(collections.len());
431 for collection in collections {
432 let space_name = spaces_by_id
433 .get(&collection.space_id)
434 .ok_or_else(|| {
435 KboltError::Internal(format!(
436 "missing space mapping for collection '{}'",
437 collection.name
438 ))
439 })?
440 .clone();
441 infos.push(self.build_collection_info(&space_name, &collection)?);
442 }
443 Ok(infos)
444 }
445
446 pub fn collection_info(&self, space: Option<&str>, name: &str) -> Result<CollectionInfo> {
447 let resolved = self.resolve_space_row(space, Some(name))?;
448 let collection = self.storage.get_collection(resolved.id, name)?;
449 self.build_collection_info(&resolved.name, &collection)
450 }
451
452 pub fn list_files(
453 &self,
454 space: Option<&str>,
455 collection: &str,
456 prefix: Option<&str>,
457 ) -> Result<Vec<FileEntry>> {
458 let resolved_space = self.resolve_space_row(space, Some(collection))?;
459 let collection_row = self.storage.get_collection(resolved_space.id, collection)?;
460 let normalized_prefix = normalize_list_prefix(prefix)?;
461 let file_rows = self
462 .storage
463 .list_collection_file_rows(collection_row.id, false)?;
464
465 let mut files = Vec::with_capacity(file_rows.len());
466 for file_row in file_rows {
467 if let Some(prefix) = normalized_prefix.as_deref() {
468 if !path_matches_prefix(&file_row.path, prefix) {
469 continue;
470 }
471 }
472
473 let title =
474 crate::engine::file_utils::display_title(file_row.title.as_deref(), &file_row.path);
475 files.push(FileEntry {
476 path: file_row.path,
477 title,
478 docid: short_docid(&file_row.hash),
479 active: file_row.active,
480 chunk_count: file_row.chunk_count,
481 embedded: file_row.chunk_count > 0
482 && file_row.embedded_chunk_count >= file_row.chunk_count,
483 });
484 }
485
486 Ok(files)
487 }
488
489 pub fn get_document(&self, req: GetRequest) -> Result<DocumentResponse> {
490 self.get_document_unlocked(req)
491 }
492
493 fn get_document_unlocked(&self, req: GetRequest) -> Result<DocumentResponse> {
494 let GetRequest {
495 locator,
496 space,
497 offset,
498 limit,
499 } = req;
500
501 let (document, collection_row, space_name) = match locator {
502 Locator::Path(locator_path) => {
503 let (collection_name, relative_path) = split_collection_path(&locator_path)?;
504 let resolved_space =
505 self.resolve_space_row(space.as_deref(), Some(&collection_name))?;
506 let collection = self
507 .storage
508 .get_collection(resolved_space.id, &collection_name)?;
509 let document = self
510 .storage
511 .get_document_by_path(collection.id, &relative_path)?
512 .ok_or_else(|| KboltError::DocumentNotFound {
513 path: locator_path.clone(),
514 })?;
515 (document, collection, resolved_space.name)
516 }
517 Locator::DocId(docid) => self.resolve_document_by_docid(&docid, space.as_deref())?,
518 };
519
520 let document_text = self.storage.get_document_text(document.id)?;
521 let full_path = collection_row.path.join(&document.path);
522 let stale = source_is_stale(&full_path, document.hash.as_str())?;
523
524 let (content, total_lines, returned_lines) =
525 slice_lines(&document_text.text, offset, limit);
526
527 Ok(DocumentResponse {
528 docid: short_docid(&document.hash),
529 path: format!("{}/{}", collection_row.name, document.path),
530 title: crate::engine::file_utils::display_title(
531 document.title.as_deref(),
532 &document.path,
533 ),
534 space: space_name,
535 collection: collection_row.name,
536 content,
537 stale,
538 total_lines,
539 returned_lines,
540 })
541 }
542
543 pub fn get_chunk(&self, req: GetChunkRequest) -> Result<kbolt_types::ChunkResponse> {
544 let GetChunkRequest {
545 locator,
546 space,
547 offset,
548 limit,
549 } = req;
550 let (document, collection_row, space_name) =
551 self.resolve_document_by_docid(&locator.docid, space.as_deref())?;
552
553 let seq = locator
554 .chunk_ordinal
555 .checked_sub(1)
556 .and_then(|value| i32::try_from(value).ok())
557 .ok_or_else(|| {
558 KboltError::InvalidInput("chunk locator ordinal is too large".to_string())
559 })?;
560 let mut chunks_by_doc =
561 self.storage
562 .get_chunks_for_document_seq_ranges(&[(document.id, seq, seq)])?;
563 let chunk = chunks_by_doc
564 .remove(&document.id)
565 .and_then(|mut chunks| chunks.pop())
566 .ok_or_else(|| KboltError::ChunkNotFound {
567 locator: format!("{}@{}", short_docid(&document.hash), locator.chunk_ordinal),
568 })?;
569 let mut texts = self.storage.get_canonical_chunk_texts(&[chunk.id])?;
570 let chunk_text = texts.remove(&chunk.id).ok_or_else(|| {
571 CoreError::Internal(format!("canonical text missing for chunk {}", chunk.id))
572 })?;
573 let full_path = collection_row.path.join(&document.path);
574 let stale = source_is_stale(&full_path, document.hash.as_str())?;
575 let (content, total_lines, returned_lines) =
576 slice_lines_exact_when_unsliced(&chunk_text, offset, limit);
577 let docid = short_docid(&document.hash);
578
579 Ok(kbolt_types::ChunkResponse {
580 locator: format!("#{}@{}", document.hash, locator.chunk_ordinal),
581 docid,
582 chunk_ordinal: locator.chunk_ordinal,
583 path: format!("{}/{}", collection_row.name, document.path),
584 title: crate::engine::file_utils::display_title(
585 document.title.as_deref(),
586 &document.path,
587 ),
588 space: space_name,
589 collection: collection_row.name,
590 heading: chunk.heading,
591 content,
592 stale,
593 total_lines,
594 returned_lines,
595 })
596 }
597
598 fn resolve_document_by_docid(
599 &self,
600 docid: &str,
601 space: Option<&str>,
602 ) -> Result<(DocumentRow, CollectionRow, String)> {
603 let prefix = normalize_docid(docid)?;
604 let mut candidates = self
605 .storage
606 .get_document_by_hash_prefix(&prefix)?
607 .into_iter()
608 .map(|document| {
609 let collection = self.storage.get_collection_by_id(document.collection_id)?;
610 Ok((document, collection))
611 })
612 .collect::<Result<Vec<_>>>()?;
613
614 if let Some(space_name) = space {
615 let resolved_space = self.resolve_space_row(Some(space_name), None)?;
616 candidates.retain(|(_, collection)| collection.space_id == resolved_space.id);
617 }
618
619 if candidates.is_empty() {
620 return Err(KboltError::DocumentNotFound {
621 path: format!("#{prefix}"),
622 }
623 .into());
624 }
625
626 if candidates.len() > 1 {
627 return Err(KboltError::InvalidInput(
628 "docid is ambiguous; provide more characters".to_string(),
629 )
630 .into());
631 }
632
633 let (document, collection) = candidates.pop().expect("candidate exists");
634 let space = self.storage.get_space_by_id(collection.space_id)?;
635 Ok((document, collection, space.name))
636 }
637
638 pub fn multi_get(&self, req: MultiGetRequest) -> Result<MultiGetResponse> {
639 if req.max_files == 0 {
640 return Err(
641 KboltError::InvalidInput("max_files must be greater than 0".to_string()).into(),
642 );
643 }
644 if req.max_bytes == 0 {
645 return Err(
646 KboltError::InvalidInput("max_bytes must be greater than 0".to_string()).into(),
647 );
648 }
649
650 let mut items = Vec::new();
651 let mut omitted = Vec::new();
652 let mut resolved_count = 0usize;
653 let mut warnings = Vec::new();
654 let mut consumed_bytes = 0usize;
655
656 for locator in req.locators {
657 let item = match self.resolve_multi_get_item(locator, req.space.clone()) {
658 Ok(item) => item,
659 Err(err) => match KboltError::from(err) {
660 KboltError::DocumentNotFound { path } => {
661 warnings.push(format!("item not found: {path}"));
662 continue;
663 }
664 KboltError::ChunkNotFound { locator } => {
665 warnings.push(format!("item not found: {locator}"));
666 continue;
667 }
668 KboltError::InvalidInput(message) => {
669 warnings.push(format!("invalid locator: {message}"));
670 continue;
671 }
672 KboltError::InvalidPath(path) => {
673 warnings.push(format!("invalid locator path: {}", path.display()));
674 continue;
675 }
676 KboltError::AmbiguousSpace { collection, spaces } => {
677 warnings.push(format!(
678 "ambiguous locator space for collection '{collection}': {:?}. use --space to disambiguate.",
679 spaces
680 ));
681 continue;
682 }
683 KboltError::SpaceNotFound { name } => {
684 warnings.push(format!("space not found for locator: {name}"));
685 continue;
686 }
687 KboltError::CollectionNotFound { name } => {
688 warnings.push(format!("collection not found for locator: {name}"));
689 continue;
690 }
691 other => return Err(other.into()),
692 },
693 };
694 resolved_count = resolved_count.saturating_add(1);
695
696 let size_bytes = multi_get_item_content_len(&item);
697 if items.len() >= req.max_files {
698 omitted.push(OmittedItem {
699 kind: multi_get_item_kind(&item),
700 locator: multi_get_item_locator(&item).to_string(),
701 path: multi_get_item_path(&item).to_string(),
702 docid: multi_get_item_docid(&item).to_string(),
703 space: multi_get_item_space(&item).to_string(),
704 size_bytes,
705 reason: OmitReason::MaxFiles,
706 });
707 continue;
708 }
709
710 if consumed_bytes.saturating_add(size_bytes) > req.max_bytes {
711 omitted.push(OmittedItem {
712 kind: multi_get_item_kind(&item),
713 locator: multi_get_item_locator(&item).to_string(),
714 path: multi_get_item_path(&item).to_string(),
715 docid: multi_get_item_docid(&item).to_string(),
716 space: multi_get_item_space(&item).to_string(),
717 size_bytes,
718 reason: OmitReason::MaxBytes,
719 });
720 continue;
721 }
722
723 consumed_bytes = consumed_bytes.saturating_add(size_bytes);
724 items.push(item);
725 }
726
727 Ok(MultiGetResponse {
728 items,
729 omitted,
730 resolved_count,
731 warnings,
732 })
733 }
734
735 fn resolve_multi_get_item(
736 &self,
737 locator: ReadLocator,
738 space: Option<String>,
739 ) -> Result<MultiGetItem> {
740 match locator {
741 ReadLocator::Document(locator) => {
742 let document = self.get_document_unlocked(GetRequest {
743 locator,
744 space,
745 offset: None,
746 limit: None,
747 })?;
748 Ok(MultiGetItem::Document(document))
749 }
750 ReadLocator::Chunk(locator) => {
751 let chunk = self.get_chunk(GetChunkRequest {
752 locator,
753 space,
754 offset: None,
755 limit: None,
756 })?;
757 Ok(MultiGetItem::Chunk(chunk))
758 }
759 }
760 }
761
762 pub fn search(&self, req: SearchRequest) -> Result<SearchResponse> {
763 let _profile = crate::profile::SearchProfileGuard::start();
764 let _lock = crate::profile::timed_search_stage("operation_lock", || {
765 self.acquire_operation_lock(LockMode::Shared)
766 })?;
767 let started = Instant::now();
768 let query = req.query.trim();
769 if query.is_empty() {
770 return Err(KboltError::InvalidInput("query cannot be empty".to_string()).into());
771 }
772
773 let requested_mode = req.mode.clone();
774 let rerank_enabled =
775 matches!(requested_mode, SearchMode::Auto | SearchMode::Deep) && !req.no_rerank;
776 let mut pipeline = self.initial_search_pipeline(&requested_mode);
777
778 let targets = crate::profile::timed_search_stage("resolve_targets", || {
779 self.resolve_targets(TargetScope {
780 space: req.space.as_deref(),
781 collections: &req.collections,
782 })
783 })?;
784 crate::profile::increment_search_count("target_collections", targets.len() as u64);
785
786 let staleness_hint = targets
787 .iter()
788 .map(|target| target.collection.updated.clone())
789 .max()
790 .map(|updated| format!("Index last updated: {updated}"));
791
792 if req.limit == 0 || targets.is_empty() {
793 return Ok(SearchResponse {
794 results: Vec::new(),
795 query: req.query,
796 requested_mode: requested_mode.clone(),
797 effective_mode: self.effective_search_mode(&requested_mode, &pipeline),
798 pipeline,
799 empty_reason: None,
800 staleness_hint,
801 elapsed_ms: started.elapsed().as_millis() as u64,
802 });
803 }
804
805 self.validate_search_mode_runtime(&requested_mode)?;
806
807 let mut collections_by_id: HashMap<i64, SearchCollectionMeta> = HashMap::new();
808 for target in &targets {
809 collections_by_id.insert(
810 target.collection.id,
811 SearchCollectionMeta {
812 space: target.space.clone(),
813 collection: target.collection.name.clone(),
814 },
815 );
816 }
817
818 let target_scopes = crate::profile::timed_search_stage("search_target_scopes", || {
819 self.search_target_scopes(&targets, !req.collections.is_empty())
820 })?;
821 crate::profile::increment_search_count("target_scopes", target_scopes.len() as u64);
822 for scope in &target_scopes {
823 crate::profile::increment_search_count("scope_chunks", scope.chunk_count as u64);
824 if scope.filtered {
825 crate::profile::increment_search_count("filtered_scopes", 1);
826 crate::profile::increment_search_count(
827 "scope_document_filters",
828 scope.document_ids.len() as u64,
829 );
830 }
831 }
832 let max_candidates = crate::profile::timed_search_stage("max_search_candidates", || {
833 self.max_search_candidates(&target_scopes)
834 });
835 if max_candidates == 0 {
836 return Ok(SearchResponse {
837 results: Vec::new(),
838 query: req.query,
839 requested_mode: requested_mode.clone(),
840 effective_mode: self.effective_search_mode(&requested_mode, &pipeline),
841 pipeline,
842 empty_reason: Some(SearchEmptyReason::UnindexedScope),
843 staleness_hint,
844 elapsed_ms: started.elapsed().as_millis() as u64,
845 });
846 }
847
848 let mut retrieval_limit =
849 self.initial_search_candidate_limit(&requested_mode, req.limit, rerank_enabled);
850 let final_candidates = loop {
851 crate::profile::increment_search_count("retrieval_iterations", 1);
852 let ranked_chunks = crate::profile::timed_search_stage("rank_chunks", || {
853 self.rank_chunks_for_mode(
854 &requested_mode,
855 &target_scopes,
856 query,
857 retrieval_limit,
858 req.min_score,
859 &mut pipeline,
860 )
861 })?;
862 crate::profile::increment_search_count("ranked_chunks", ranked_chunks.len() as u64);
863
864 if ranked_chunks.is_empty() {
865 return Ok(SearchResponse {
866 results: Vec::new(),
867 query: req.query,
868 requested_mode: requested_mode.clone(),
869 effective_mode: self.effective_search_mode(&requested_mode, &pipeline),
870 pipeline,
871 empty_reason: None,
872 staleness_hint,
873 elapsed_ms: started.elapsed().as_millis() as u64,
874 });
875 }
876
877 let ranked_len = ranked_chunks.len();
878 let candidates = crate::profile::timed_search_stage("assemble_candidates", || {
879 self.search_candidates_from_ranked_chunks(ranked_chunks, &collections_by_id)
880 })?;
881 crate::profile::increment_search_count("assembled_candidates", candidates.len() as u64);
882
883 if candidates.len() >= req.limit
884 || ranked_len < retrieval_limit
885 || retrieval_limit >= max_candidates
886 {
887 break candidates;
888 }
889
890 let next_limit = retrieval_limit.saturating_mul(2).min(max_candidates);
891 if next_limit <= retrieval_limit {
892 break candidates;
893 }
894 retrieval_limit = next_limit;
895 };
896 let results = crate::profile::timed_search_stage("assemble_results", || {
897 self.assemble_search_candidates(
898 query,
899 &requested_mode,
900 final_candidates,
901 req.debug,
902 rerank_enabled,
903 &mut pipeline,
904 req.limit,
905 )
906 })?;
907 crate::profile::increment_search_count("assembled_results", results.len() as u64);
908
909 Ok(SearchResponse {
910 results,
911 query: req.query,
912 requested_mode: requested_mode.clone(),
913 effective_mode: self.effective_search_mode(&requested_mode, &pipeline),
914 pipeline,
915 empty_reason: None,
916 staleness_hint,
917 elapsed_ms: started.elapsed().as_millis() as u64,
918 })
919 }
920
921 pub fn resolve_space(&self, explicit: Option<&str>) -> Result<String> {
922 let resolved = self.resolve_space_row(explicit, None)?;
923 Ok(resolved.name)
924 }
925
926 pub fn current_space(&self, explicit: Option<&str>) -> Result<Option<ActiveSpace>> {
927 let resolved = self.resolve_preferred_space(explicit)?;
928 Ok(resolved.map(|(space, source)| ActiveSpace {
929 name: space.name,
930 source,
931 }))
932 }
933
934 pub fn status(&self, space: Option<&str>) -> Result<StatusResponse> {
935 let _lock = self.acquire_operation_lock(LockMode::Shared)?;
936 let (spaces, totals_scope) = if let Some(space_name) = space {
937 let resolved = self.resolve_space_row(Some(space_name), None)?;
938 (vec![resolved.clone()], Some(resolved.id))
939 } else {
940 (self.storage.list_spaces()?, None)
941 };
942
943 let mut space_statuses = Vec::with_capacity(spaces.len());
944 for space_row in spaces {
945 let collections = self.storage.list_collections(Some(space_row.id))?;
946 let mut collection_statuses = Vec::with_capacity(collections.len());
947 let mut last_updated: Option<String> = None;
948
949 for collection in collections {
950 if last_updated
951 .as_ref()
952 .map(|existing| collection.updated > *existing)
953 .unwrap_or(true)
954 {
955 last_updated = Some(collection.updated.clone());
956 }
957
958 let documents = self
959 .storage
960 .count_documents_in_collection(collection.id, false)?;
961 let active_documents = self
962 .storage
963 .count_documents_in_collection(collection.id, true)?;
964 let chunks = self.storage.count_chunks_in_collection(collection.id)?;
965 let embedded_chunks = self
966 .storage
967 .count_embedded_chunks_in_collection(collection.id)?;
968
969 collection_statuses.push(CollectionStatus {
970 name: collection.name,
971 path: collection.path,
972 documents,
973 active_documents,
974 chunks,
975 embedded_chunks,
976 last_updated: collection.updated,
977 });
978 }
979
980 space_statuses.push(SpaceStatus {
981 name: space_row.name,
982 description: space_row.description,
983 collections: collection_statuses,
984 last_updated,
985 });
986 }
987
988 let models = self.model_status_unlocked()?;
989
990 Ok(StatusResponse {
991 spaces: space_statuses,
992 models,
993 cache_dir: self.config.cache_dir.clone(),
994 config_dir: self.config.config_dir.clone(),
995 total_documents: self.storage.count_documents(totals_scope)?,
996 total_chunks: self.storage.count_chunks(totals_scope)?,
997 total_embedded: self.storage.count_embedded_chunks(totals_scope)?,
998 disk_usage: self.storage.disk_usage()?,
999 })
1000 }
1001
1002 pub fn model_status(&self) -> Result<ModelStatus> {
1003 self.model_status_unlocked()
1004 }
1005
1006 fn model_status_unlocked(&self) -> Result<ModelStatus> {
1007 models::status(&self.config)
1008 }
1009
1010 pub fn config(&self) -> &Config {
1011 &self.config
1012 }
1013
1014 pub fn storage(&self) -> &Storage {
1015 &self.storage
1016 }
1017
1018 pub fn shortest_unambiguous_docid_prefix(
1019 &self,
1020 hash: &str,
1021 space: &str,
1022 min_len: usize,
1023 ) -> Result<String> {
1024 let normalized = normalize_docid(hash)?;
1025 let min_len = min_len.clamp(1, normalized.len());
1026
1027 for len in min_len..=normalized.len() {
1028 let prefix = &normalized[..len];
1029 let matches = self
1030 .storage
1031 .document_hashes_by_prefix_in_space(prefix, space)?;
1032 if matches.len() == 1 && matches[0] == normalized {
1033 return Ok(format!("#{prefix}"));
1034 }
1035 }
1036
1037 Ok(format!("#{normalized}"))
1038 }
1039
1040 fn embedding_model_name(&self) -> &str {
1041 self.config
1042 .roles
1043 .embedder
1044 .as_ref()
1045 .and_then(|role| self.config.providers.get(&role.provider))
1046 .map(|profile| profile.model())
1047 .unwrap_or("embedder")
1048 }
1049
1050 fn embedding_model_key(&self) -> String {
1051 let formatter = self.embedding_input_formatter();
1052 format!(
1053 "{}|{}|{}",
1054 self.embedding_model_name(),
1055 DENSE_DOCUMENT_RENDER_IDENTITY,
1056 formatter.format_identity()
1057 )
1058 }
1059
1060 fn embedding_input_formatter(&self) -> models::EmbeddingInputFormatter {
1061 models::embedding_input_formatter_for_config(&self.config)
1062 }
1063
1064 fn format_embedding_query(&self, query: &str) -> String {
1065 self.embedding_input_formatter()
1066 .format_query(query)
1067 .into_owned()
1068 }
1069
1070 fn format_embedding_queries(&self, queries: &[String]) -> Vec<String> {
1071 let formatter = self.embedding_input_formatter();
1072 queries
1073 .iter()
1074 .map(|query| formatter.format_query(query).into_owned())
1075 .collect()
1076 }
1077
1078 fn format_embedding_document(&self, input: &EmbeddingDocumentInput<'_>) -> String {
1079 self.embedding_input_formatter().format_document(input)
1080 }
1081
1082 fn acquire_operation_lock(&self, mode: LockMode) -> Result<OperationLock> {
1083 OperationLock::acquire(&self.config.cache_dir, mode)
1084 }
1085
1086 fn collect_document_chunk_ids(&self, doc_ids: &[i64]) -> Result<Vec<i64>> {
1087 let mut chunk_ids = Vec::new();
1088 for doc_id in doc_ids {
1089 let chunks = self.storage.get_chunks_for_document(*doc_id)?;
1090 chunk_ids.extend(chunks.into_iter().map(|chunk| chunk.id));
1091 }
1092 Ok(chunk_ids)
1093 }
1094
1095 fn purge_space_chunks(&self, space: &str, chunk_ids: &[i64]) -> Result<()> {
1096 if chunk_ids.is_empty() {
1097 return Ok(());
1098 }
1099
1100 self.storage.delete_tantivy(space, chunk_ids)?;
1101 self.storage.commit_tantivy(space)?;
1102 self.storage.delete_usearch(space, chunk_ids)?;
1103 Ok(())
1104 }
1105
1106 fn build_space_info(&self, space: &crate::storage::SpaceRow) -> Result<SpaceInfo> {
1107 let collection_count = self.storage.list_collections(Some(space.id))?.len();
1108 let document_count = self.storage.count_documents(Some(space.id))?;
1109 let chunk_count = self.storage.count_chunks(Some(space.id))?;
1110
1111 Ok(SpaceInfo {
1112 name: space.name.clone(),
1113 description: space.description.clone(),
1114 collection_count,
1115 document_count,
1116 chunk_count,
1117 created: space.created.clone(),
1118 })
1119 }
1120
1121 fn build_collection_info(
1122 &self,
1123 space_name: &str,
1124 collection: &CollectionRow,
1125 ) -> Result<CollectionInfo> {
1126 let document_count = self
1127 .storage
1128 .count_documents_in_collection(collection.id, false)?;
1129 let active_document_count = self
1130 .storage
1131 .count_documents_in_collection(collection.id, true)?;
1132 let chunk_count = self.storage.count_chunks_in_collection(collection.id)?;
1133 let embedded_chunk_count = self
1134 .storage
1135 .count_embedded_chunks_in_collection(collection.id)?;
1136
1137 Ok(CollectionInfo {
1138 name: collection.name.clone(),
1139 space: space_name.to_string(),
1140 path: collection.path.clone(),
1141 description: collection.description.clone(),
1142 extensions: collection.extensions.clone(),
1143 document_count,
1144 active_document_count,
1145 chunk_count,
1146 embedded_chunk_count,
1147 created: collection.created.clone(),
1148 updated: collection.updated.clone(),
1149 })
1150 }
1151
1152 fn resolve_space_row(
1153 &self,
1154 explicit: Option<&str>,
1155 collection_for_lookup: Option<&str>,
1156 ) -> Result<crate::storage::SpaceRow> {
1157 if let Some((space, _source)) = self.resolve_preferred_space(explicit)? {
1158 return Ok(space);
1159 }
1160
1161 if let Some(collection) = collection_for_lookup {
1162 return match self.storage.find_space_for_collection(collection)? {
1163 SpaceResolution::Found(space) => Ok(space),
1164 SpaceResolution::Ambiguous(spaces) => Err(KboltError::AmbiguousSpace {
1165 collection: collection.to_string(),
1166 spaces,
1167 }
1168 .into()),
1169 SpaceResolution::NotFound => Err(KboltError::CollectionNotFound {
1170 name: collection.to_string(),
1171 }
1172 .into()),
1173 };
1174 }
1175
1176 Err(KboltError::NoActiveSpace.into())
1177 }
1178
1179 fn resolve_preferred_space(
1180 &self,
1181 explicit: Option<&str>,
1182 ) -> Result<Option<(crate::storage::SpaceRow, ActiveSpaceSource)>> {
1183 if let Some(space_name) = explicit {
1184 let space = self.storage.get_space(space_name)?;
1185 return Ok(Some((space, ActiveSpaceSource::Flag)));
1186 }
1187
1188 if let Ok(space_name) = std::env::var("KBOLT_SPACE") {
1189 let trimmed = space_name.trim();
1190 if !trimmed.is_empty() {
1191 let space = self.storage.get_space(trimmed)?;
1192 return Ok(Some((space, ActiveSpaceSource::EnvVar)));
1193 }
1194 }
1195
1196 if let Some(space_name) = self.config.default_space.as_deref() {
1197 let space = self.storage.get_space(space_name)?;
1198 return Ok(Some((space, ActiveSpaceSource::ConfigDefault)));
1199 }
1200
1201 Ok(None)
1202 }
1203}
1204
1205fn multi_get_item_content_len(item: &MultiGetItem) -> usize {
1206 match item {
1207 MultiGetItem::Document(document) => document.content.len(),
1208 MultiGetItem::Chunk(chunk) => chunk.content.len(),
1209 }
1210}
1211
1212fn multi_get_item_kind(item: &MultiGetItem) -> MultiGetItemKind {
1213 match item {
1214 MultiGetItem::Document(_) => MultiGetItemKind::Document,
1215 MultiGetItem::Chunk(_) => MultiGetItemKind::Chunk,
1216 }
1217}
1218
1219fn multi_get_item_locator(item: &MultiGetItem) -> &str {
1220 match item {
1221 MultiGetItem::Document(document) => &document.docid,
1222 MultiGetItem::Chunk(chunk) => &chunk.locator,
1223 }
1224}
1225
1226fn multi_get_item_path(item: &MultiGetItem) -> &str {
1227 match item {
1228 MultiGetItem::Document(document) => &document.path,
1229 MultiGetItem::Chunk(chunk) => &chunk.path,
1230 }
1231}
1232
1233fn multi_get_item_docid(item: &MultiGetItem) -> &str {
1234 match item {
1235 MultiGetItem::Document(document) => &document.docid,
1236 MultiGetItem::Chunk(chunk) => &chunk.docid,
1237 }
1238}
1239
1240fn multi_get_item_space(item: &MultiGetItem) -> &str {
1241 match item {
1242 MultiGetItem::Document(document) => &document.space,
1243 MultiGetItem::Chunk(chunk) => &chunk.space,
1244 }
1245}
1246
1247fn slice_lines(text: &str, offset: Option<usize>, limit: Option<usize>) -> (String, usize, usize) {
1248 let lines = text.lines().collect::<Vec<_>>();
1249 let total_lines = lines.len();
1250 let start = offset.unwrap_or(0).min(total_lines);
1251 let requested = limit.unwrap_or(total_lines.saturating_sub(start));
1252 let end = start.saturating_add(requested).min(total_lines);
1253 let returned_lines = end.saturating_sub(start);
1254 let content = if returned_lines == 0 {
1255 String::new()
1256 } else {
1257 lines[start..end].join("\n")
1258 };
1259
1260 (content, total_lines, returned_lines)
1261}
1262
1263fn slice_lines_exact_when_unsliced(
1264 text: &str,
1265 offset: Option<usize>,
1266 limit: Option<usize>,
1267) -> (String, usize, usize) {
1268 if offset.is_none() && limit.is_none() {
1269 let total_lines = text.lines().count();
1270 return (text.to_string(), total_lines, total_lines);
1271 }
1272
1273 slice_lines(text, offset, limit)
1274}
1275
1276fn source_is_stale(path: &Path, indexed_hash: &str) -> Result<bool> {
1277 match std::fs::read(path) {
1278 Ok(bytes) => Ok(sha256_hex(&bytes) != indexed_hash),
1279 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(true),
1280 Err(err) => Err(std::io::Error::new(
1281 err.kind(),
1282 format!(
1283 "failed to read indexed source '{}' for stale check: {err}",
1284 path.display()
1285 ),
1286 )
1287 .into()),
1288 }
1289}
1290
1291#[cfg(test)]
1292mod tests;
1293
1294#[cfg(test)]
1295mod lock_relief_tests {
1296 use fs2::FileExt;
1297 use std::collections::HashMap;
1298 use std::fs::OpenOptions;
1299 use std::mem;
1300
1301 use tempfile::tempdir;
1302
1303 use super::Engine;
1304 use crate::config::{ChunkingConfig, Config, RankingConfig, ReapingConfig};
1305 use crate::storage::Storage;
1306 use kbolt_types::{
1307 AddCollectionRequest, GetRequest, Locator, MultiGetItem, MultiGetRequest, ReadLocator,
1308 UpdateOptions,
1309 };
1310
1311 #[test]
1312 fn metadata_reads_succeed_while_global_lock_is_held() {
1313 let engine = test_engine_with_indexed_collection();
1314 let _holder = hold_global_lock(&engine);
1315
1316 let spaces = engine.list_spaces().expect("list spaces");
1317 assert_eq!(
1318 spaces
1319 .iter()
1320 .map(|space| space.name.as_str())
1321 .collect::<Vec<_>>(),
1322 vec!["default", "work"]
1323 );
1324
1325 let space = engine.space_info("work").expect("load space info");
1326 assert_eq!(space.name, "work");
1327 assert_eq!(space.collection_count, 1);
1328
1329 let collections = engine
1330 .list_collections(Some("work"))
1331 .expect("list collections");
1332 assert_eq!(collections.len(), 1);
1333 assert_eq!(collections[0].name, "api");
1334
1335 let collection = engine
1336 .collection_info(Some("work"), "api")
1337 .expect("load collection info");
1338 assert_eq!(collection.name, "api");
1339 assert_eq!(collection.document_count, 1);
1340
1341 let models = engine.model_status().expect("read model status");
1342 assert!(!models.embedder.configured);
1343 assert!(!models.reranker.configured);
1344 assert!(!models.expander.configured);
1345 }
1346
1347 #[test]
1348 fn document_reads_succeed_while_global_lock_is_held() {
1349 let engine = test_engine_with_indexed_collection();
1350 let _holder = hold_global_lock(&engine);
1351
1352 let files = engine
1353 .list_files(Some("work"), "api", None)
1354 .expect("list files");
1355 assert_eq!(files.len(), 1);
1356 assert_eq!(files[0].path, "guide.md");
1357
1358 let document = engine
1359 .get_document(GetRequest {
1360 locator: Locator::Path("api/guide.md".to_string()),
1361 space: Some("work".to_string()),
1362 offset: None,
1363 limit: None,
1364 })
1365 .expect("read document");
1366 assert_eq!(document.path, "api/guide.md");
1367 assert!(document.content.contains("hello from kbolt"));
1368
1369 let response = engine
1370 .multi_get(MultiGetRequest {
1371 locators: vec![ReadLocator::Document(Locator::Path(
1372 "api/guide.md".to_string(),
1373 ))],
1374 space: Some("work".to_string()),
1375 max_files: 4,
1376 max_bytes: 4_096,
1377 })
1378 .expect("read multiple documents");
1379 assert_eq!(response.resolved_count, 1);
1380 assert_eq!(response.items.len(), 1);
1381 assert!(matches!(response.items[0], MultiGetItem::Document(_)));
1382 assert!(response.warnings.is_empty());
1383 }
1384
1385 fn test_engine_with_indexed_collection() -> Engine {
1386 let root = tempdir().expect("create temp root");
1387 let root_path = root.path().to_path_buf();
1388 mem::forget(root);
1389
1390 let config_dir = root_path.join("config");
1391 let cache_dir = root_path.join("cache");
1392 let docs_dir = root_path.join("docs");
1393 std::fs::create_dir_all(&docs_dir).expect("create docs dir");
1394 std::fs::write(docs_dir.join("guide.md"), "# Hello\n\nhello from kbolt\n")
1395 .expect("write document");
1396
1397 let storage = Storage::new(&cache_dir).expect("create storage");
1398 let config = Config {
1399 config_dir,
1400 cache_dir,
1401 default_space: None,
1402 providers: HashMap::new(),
1403 roles: crate::config::RoleBindingsConfig::default(),
1404 reaping: ReapingConfig { days: 7 },
1405 chunking: ChunkingConfig::default(),
1406 ranking: RankingConfig::default(),
1407 };
1408 let engine = Engine::from_parts(storage, config);
1409
1410 engine
1411 .add_collection(AddCollectionRequest {
1412 path: docs_dir,
1413 space: Some("work".to_string()),
1414 name: Some("api".to_string()),
1415 description: None,
1416 extensions: None,
1417 no_index: true,
1418 })
1419 .expect("add collection");
1420 engine
1421 .update(UpdateOptions {
1422 space: Some("work".to_string()),
1423 collections: vec!["api".to_string()],
1424 no_embed: true,
1425 dry_run: false,
1426 verbose: false,
1427 })
1428 .expect("index collection");
1429
1430 engine
1431 }
1432
1433 fn hold_global_lock(engine: &Engine) -> std::fs::File {
1434 std::fs::create_dir_all(&engine.config().cache_dir).expect("create cache dir");
1435 let holder = OpenOptions::new()
1436 .read(true)
1437 .write(true)
1438 .create(true)
1439 .truncate(false)
1440 .open(engine.config().cache_dir.join("kbolt.lock"))
1441 .expect("open lock file");
1442 FileExt::try_lock_exclusive(&holder).expect("acquire global lock");
1443 holder
1444 }
1445}