episteme-local 0.1.1

Local-first knowledge ingestion and intelligence workflows
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

//! Format routing and deterministic OCR fallback.

use std::fs;

use async_trait::async_trait;

use crate::domain::{DocumentKind, ExtractedDocument, ExtractionMethod, StagedSource};
use crate::ports::{DocumentExtractor, DocumentTools, ExtractionError};

/// Routes supported document formats through configured conversion tools.
#[derive(Debug)]
pub struct DocumentCliExtractor<T> {
    tools: T,
    minimum_text_characters: usize,
}

impl<T> DocumentCliExtractor<T> {
    /// Creates an extractor with a deterministic direct-text quality threshold.
    #[must_use]
    pub const fn new(tools: T, minimum_text_characters: usize) -> Self {
        Self {
            tools,
            minimum_text_characters,
        }
    }
}

#[async_trait]
impl<T> DocumentExtractor for DocumentCliExtractor<T>
where
    T: DocumentTools,
{
    async fn extract(&self, source: &StagedSource) -> Result<ExtractedDocument, ExtractionError> {
        verify_source(source)?;
        let (text, method) = match source.kind() {
            DocumentKind::Pdf => {
                let direct_text = self.tools.pdf_text(source.path()).await?;
                if direct_text
                    .chars()
                    .filter(|character| !character.is_whitespace())
                    .count()
                    >= self.minimum_text_characters
                {
                    (direct_text, ExtractionMethod::PdfText)
                } else {
                    (
                        self.tools.pdf_ocr(source.path()).await?,
                        ExtractionMethod::PdfOcr,
                    )
                }
            }
            DocumentKind::Html => (
                self.tools.html_text(source.path()).await?,
                ExtractionMethod::HtmlPandoc,
            ),
            DocumentKind::Image => (
                self.tools.image_text(source.path()).await?,
                ExtractionMethod::ImageOcr,
            ),
        };
        verify_source(source)?;

        ExtractedDocument::new(source.clone(), text, method)
            .map_err(|error| ExtractionError::Invalid(error.to_string()))
    }
}

fn verify_source(source: &StagedSource) -> Result<(), ExtractionError> {
    let bytes =
        fs::read(source.path()).map_err(|error| ExtractionError::Tool(error.to_string()))?;
    if &crate::domain::SourceDigest::from_bytes(&bytes) != source.digest() {
        return Err(ExtractionError::SourceChanged);
    }
    Ok(())
}