orbok-workers 0.9.6

orbok pipeline workers: extraction → chunking → keyword indexing (M5/M6)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

//! Chunk-and-index worker (RFC-006 §12): loads an extraction result
//! from the cache, chunks it, and atomically inserts chunks + FTS index
//! into the catalog (one transaction).

use orbok_cache::{CacheService, EngineOptions, OrbokCacheNamespace};
use orbok_core::{ErrorCategory, ExtractionId, FileId, OrbokError, OrbokResult};
use orbok_db::Catalog;
use orbok_db::repo::{ChunkRepository, FileRepository, SourceRepository};
use orbok_extract::{ExtractOutput, chunk};
use orbok_fs::{GuardedSource, PathGuard};
use rusqlite::params;
use std::path::Path;

/// Chunk-and-index worker.
pub struct ChunkAndIndexWorker<'a> {
    catalog: &'a Catalog,
    cache: &'a CacheService,
}

impl<'a> ChunkAndIndexWorker<'a> {
    pub fn new(catalog: &'a Catalog, cache: &'a CacheService) -> Self {
        Self { catalog, cache }
    }

    /// Load the extraction cache for a file, chunk, and index.
    pub fn run(&self, file_id: &FileId) -> OrbokResult<()> {
        let files = FileRepository::new(self.catalog);
        let record = files
            .get_by_id(file_id)?
            .ok_or(OrbokError::FileNotFound)?;
        let sources = SourceRepository::new(self.catalog);
        let source = sources
            .get(&record.source_id)?
            .ok_or(OrbokError::SourceNotFound)?;

        let guard = PathGuard::new(vec![GuardedSource::from_record(&source)]);
        let validated = guard.validate(Path::new(&record.canonical_path))?;

        let engine = self.cache.engine::<ExtractOutput>(
            self.catalog,
            &OrbokCacheNamespace::ExtractSegments,
            EngineOptions::default(),
        )?;
        let output = CacheService::get_fresh(&engine, &validated)?.ok_or_else(|| {
            OrbokError::Extraction {
                category: ErrorCategory::ParserError,
                message: "extraction cache miss: run extraction first".into(),
            }
        })?;

        // Find the most recent succeeded extraction record for this file.
        let extraction_id = self.latest_extraction_id(file_id)?;

        let file_name = Path::new(&record.display_path)
            .file_name()
            .map(|n| n.to_string_lossy().into_owned())
            .unwrap_or_else(|| record.display_path.clone());

        let specs = chunk(&output, &file_name);
        if specs.is_empty() || (specs.len() == 1 && specs[0].normalized_text.is_empty()) {
            return Ok(());
        }

        ChunkRepository::new(self.catalog).insert_bundle(file_id, &extraction_id, &specs)?;
        Ok(())
    }

    fn latest_extraction_id(&self, file_id: &FileId) -> OrbokResult<ExtractionId> {
        let conn = self.catalog.lock();
        let id: String = conn
            .query_row(
                "SELECT extraction_id FROM extraction_records \
                 WHERE file_id = ?1 AND status = 'succeeded' \
                 ORDER BY completed_at DESC LIMIT 1",
                params![file_id.as_str()],
                |row| row.get(0),
            )
            .map_err(|e| OrbokError::Database(format!("no extraction record: {e}")))?;
        Ok(ExtractionId::from_string(id))
    }
}