poet 0.7.0

Static site generator with optional MCP server and LLM SEO optimizations
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

use std::collections::BTreeMap;
use std::sync::Arc;

use anyhow::Result;
use anyhow::anyhow;
use tantivy::Index;
use tantivy::IndexReader;
use tantivy::TantivyDocument;
use tantivy::collector::TopDocs;
use tantivy::query::QueryParser;
use tantivy::schema::Value as _;

use crate::content_document_basename::ContentDocumentBasename;
use crate::content_document_source::ContentDocumentSource;
use crate::mcp::list_resources_cursor::ListResourcesCursor;
use crate::search_index_fields::SearchIndexFields;
use crate::search_index_found_document::SearchIndexFoundDocument;
use crate::search_index_query_params::SearchIndexQueryParams;

pub struct SearchIndexReader {
    pub content_document_sources: Arc<BTreeMap<ContentDocumentBasename, ContentDocumentSource>>,
    pub fields: Arc<SearchIndexFields>,
    pub index: Index,
    pub index_reader: IndexReader,
}

impl SearchIndexReader {
    pub fn query(
        &self,
        SearchIndexQueryParams {
            cursor: ListResourcesCursor { offset, per_page },
            query,
        }: SearchIndexQueryParams,
    ) -> Result<Vec<SearchIndexFoundDocument>> {
        let mut query_parser = QueryParser::for_index(
            &self.index,
            vec![self.fields.title, self.fields.header, self.fields.paragraph],
        );

        query_parser.set_field_boost(self.fields.title, 4.0);
        query_parser.set_field_boost(self.fields.description, 3.0);
        query_parser.set_field_boost(self.fields.header, 2.0);

        let query = query_parser.parse_query(&query)?;

        let searcher = self.index_reader.searcher();
        let results = searcher.search(&query, &TopDocs::with_limit(per_page).and_offset(offset))?;

        let mut ret = Vec::new();

        for (_score, doc_address) in results {
            let tantivy_document: TantivyDocument = searcher.doc::<TantivyDocument>(doc_address)?;

            let basename: ContentDocumentBasename = tantivy_document
                .get_first(self.fields.basename)
                .ok_or_else(|| anyhow!("Document does not have a stored basename"))?
                .as_str()
                .ok_or_else(|| anyhow!("Unable to convert Tantivy Value to string slice"))?
                .to_string()
                .into();

            let ContentDocumentSource { reference, .. }: &ContentDocumentSource = self
                .content_document_sources
                .get(&basename)
                .ok_or_else(|| anyhow!("There is no document with basename: '{basename}'"))?;

            ret.push(SearchIndexFoundDocument {
                content_document_reference: reference.clone(),
            });
        }

        Ok(ret)
    }
}