codanna 0.9.19

Code Intelligence for Large Language Models
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

//! Tantivy schema for document chunk storage.
//!
//! This module defines the index schema for storing document chunks
//! with metadata for filtering and navigation.

use tantivy::schema::{
    FAST, Field, IndexRecordOption, NumericOptions, STORED, STRING, Schema, SchemaBuilder,
    TextFieldIndexing, TextOptions,
};

/// Schema fields for document chunk storage.
#[derive(Debug)]
pub struct DocumentSchema {
    /// Document type discriminator (always "chunk" for this index).
    pub doc_type: Field,

    /// Unique identifier for this chunk.
    pub chunk_id: Field,

    /// Collection name for filtering (e.g., "rust-book", "project-docs").
    pub collection_name: Field,

    /// Source file path relative to collection root.
    pub source_path: Field,

    /// Heading hierarchy as JSON array (e.g., ["Chapter 1", "Section 1.2"]).
    pub heading_context: Field,

    /// Full chunk content for search and embedding.
    pub content: Field,

    /// Content preview (first ~200 chars) for display.
    pub content_preview: Field,

    /// Start byte offset in source file.
    pub byte_start: Field,

    /// End byte offset in source file.
    pub byte_end: Field,

    /// Character count for the chunk.
    pub char_count: Field,

    /// File content hash for change detection.
    pub file_hash: Field,

    /// Timestamp when indexed (UTC seconds).
    pub indexed_at: Field,

    /// Metadata key field (for counters, etc.).
    pub meta_key: Field,

    /// Metadata value field.
    pub meta_value: Field,
}

impl DocumentSchema {
    /// Build the schema for document chunk storage.
    pub fn build() -> (Schema, Self) {
        let mut builder = SchemaBuilder::default();

        // Document type discriminator
        let doc_type = builder.add_text_field("doc_type", STRING | STORED | FAST);

        // Numeric options for indexed u64 fields
        let indexed_u64 = NumericOptions::default()
            .set_indexed()
            .set_stored()
            .set_fast();

        // Chunk ID
        let chunk_id = builder.add_u64_field("chunk_id", indexed_u64.clone());

        // Collection name - STRING for exact filtering
        let collection_name = builder.add_text_field("collection_name", STRING | STORED | FAST);

        // Source path - STRING for exact matching
        let source_path = builder.add_text_field("source_path", STRING | STORED);

        // Heading context - TEXT for search, STORED for retrieval
        let text_options = TextOptions::default()
            .set_indexing_options(
                TextFieldIndexing::default()
                    .set_tokenizer("default")
                    .set_index_option(IndexRecordOption::WithFreqsAndPositions),
            )
            .set_stored();
        let heading_context = builder.add_text_field("heading_context", text_options.clone());

        // Full content - TEXT for full-text search
        let content = builder.add_text_field("content", text_options.clone());

        // Content preview - STORED only, not indexed
        let content_preview = builder.add_text_field("content_preview", STORED);

        // Byte offsets
        let byte_start = builder.add_u64_field("byte_start", STORED);
        let byte_end = builder.add_u64_field("byte_end", STORED);

        // Character count
        let char_count = builder.add_u64_field("char_count", STORED);

        // File hash for change detection
        let file_hash = builder.add_text_field("file_hash", STRING | STORED);

        // Indexed timestamp
        let indexed_at = builder.add_u64_field("indexed_at", STORED | FAST);

        // Metadata fields (for counters like chunk_counter)
        let meta_key = builder.add_text_field("meta_key", STRING | STORED | FAST);
        let meta_value = builder.add_u64_field("meta_value", STORED | FAST);

        let schema = builder.build();

        let document_schema = Self {
            doc_type,
            chunk_id,
            collection_name,
            source_path,
            heading_context,
            content,
            content_preview,
            byte_start,
            byte_end,
            char_count,
            file_hash,
            indexed_at,
            meta_key,
            meta_value,
        };

        (schema, document_schema)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_schema_build() {
        let (schema, _fields) = DocumentSchema::build();

        // Verify all fields exist
        assert!(schema.get_field("doc_type").is_ok());
        assert!(schema.get_field("chunk_id").is_ok());
        assert!(schema.get_field("collection_name").is_ok());
        assert!(schema.get_field("source_path").is_ok());
        assert!(schema.get_field("heading_context").is_ok());
        assert!(schema.get_field("content").is_ok());
        assert!(schema.get_field("content_preview").is_ok());
        assert!(schema.get_field("byte_start").is_ok());
        assert!(schema.get_field("byte_end").is_ok());

        // Verify field count matches
        assert_eq!(schema.fields().count(), 14);
    }
}