basemind 0.17.0

Full AI context layer over MCP — tree-sitter code-map, document RAG (PDF/Office/HTML/email + OCR + reranker), shared agent memory, on-demand web crawl, git history + blame + per-symbol diff. 300+ languages, 10+ coding-agent harnesses, content-addressed Fjall + LanceDB.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

//! Arrow schemas for the LanceDB-backed `documents` and `memory` tables.
//!
//! The vector dimension is fixed once at table-creation time; mismatched dims
//! trigger a wipe-and-rebuild (see [`crate::lance::LanceStore::open`]).

use std::sync::Arc;

use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};

/// Build the schema for the per-document-chunk `documents` table.
///
/// Columns:
/// - `scope`     UTF-8     repo identity (normalised git remote URL or workdir path)
/// - `path`      UTF-8     repo-relative path of the source file
/// - `chunk_idx` UInt32    0-based index of this chunk within the file
/// - `mime_type` UTF-8     IANA MIME type xberg detected
/// - `text`      UTF-8     the chunk text (snippet returned by search results)
/// - `byte_start` UInt32   chunk start byte offset in the original document
/// - `byte_end`  UInt32    chunk end byte offset
/// - `embedding` FixedSizeList<Float32, DIM>  the embedding vector
pub fn documents_schema(dim: u16) -> SchemaRef {
    Arc::new(Schema::new(vec![
        Field::new("scope", DataType::Utf8, false),
        Field::new("path", DataType::Utf8, false),
        Field::new("chunk_idx", DataType::UInt32, false),
        Field::new("mime_type", DataType::Utf8, false),
        Field::new("text", DataType::Utf8, false),
        Field::new("byte_start", DataType::UInt32, false),
        Field::new("byte_end", DataType::UInt32, false),
        Field::new(
            "embedding",
            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), i32::from(dim)),
            false,
        ),
    ]))
}

/// Build the schema for the `memory` table.
///
/// Columns:
/// - `scope`       UTF-8     repo identity
/// - `key`         UTF-8     primary lookup key (unique within `(scope, visibility, agent_id)`)
/// - `value`       UTF-8     the stored value text
/// - `tags`        `List<UTF-8>`  optional tags
/// - `visibility`  UTF-8     memory tier: `"group"` (shared) or `"individual"` (per-agent)
/// - `agent_id`    UTF-8     owner of an individual-tier row (empty for the group tier)
/// - `embedding`   FixedSizeList<Float32, DIM>
/// - `created_at`  TimestampMicros
/// - `updated_at`  TimestampMicros
pub fn memory_schema(dim: u16) -> SchemaRef {
    let tags_inner = Arc::new(Field::new("item", DataType::Utf8, true));
    Arc::new(Schema::new(vec![
        Field::new("scope", DataType::Utf8, false),
        Field::new("key", DataType::Utf8, false),
        Field::new("value", DataType::Utf8, false),
        Field::new("tags", DataType::List(tags_inner), true),
        Field::new("visibility", DataType::Utf8, false),
        Field::new("agent_id", DataType::Utf8, false),
        Field::new(
            "embedding",
            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), i32::from(dim)),
            false,
        ),
        Field::new("created_at", DataType::Timestamp(TimeUnit::Microsecond, None), false),
        Field::new("updated_at", DataType::Timestamp(TimeUnit::Microsecond, None), false),
    ]))
}

/// Build the schema for the per-chunk `code_chunks` table (semantic code search).
///
/// Columns mirror the pointer fields `search_code` returns plus the chunk `text` and the
/// embedding. Shares the single `meta.json` dim/model with the `documents` / `memory` tables.
///
/// - `scope`      UTF-8    repo identity
/// - `path`       UTF-8    repo-relative source path
/// - `chunk_id`   UTF-8    content-addressed `<hash>:<ordinal>`
/// - `symbol`     UTF-8    symbol name (empty for a module-level gap chunk)
/// - `kind`       UTF-8    symbol kind (`function`, `method`, `module`, …)
/// - `lang`       UTF-8    tree-sitter language pack name
/// - `line_start` UInt32   1-based inclusive start line
/// - `line_end`   UInt32   1-based inclusive end line
/// - `byte_start` UInt32   chunk start byte offset
/// - `byte_end`   UInt32   chunk end byte offset
/// - `text`       UTF-8    the chunk text
/// - `embedding`  FixedSizeList<Float32, DIM>
#[cfg(feature = "code-search")]
pub fn code_chunks_schema(dim: u16) -> SchemaRef {
    Arc::new(Schema::new(vec![
        Field::new("scope", DataType::Utf8, false),
        Field::new("path", DataType::Utf8, false),
        Field::new("chunk_id", DataType::Utf8, false),
        Field::new("symbol", DataType::Utf8, false),
        Field::new("kind", DataType::Utf8, false),
        Field::new("lang", DataType::Utf8, false),
        Field::new("line_start", DataType::UInt32, false),
        Field::new("line_end", DataType::UInt32, false),
        Field::new("byte_start", DataType::UInt32, false),
        Field::new("byte_end", DataType::UInt32, false),
        Field::new("text", DataType::Utf8, false),
        Field::new(
            "embedding",
            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), i32::from(dim)),
            false,
        ),
    ]))
}

/// Table names — small constants in one place so the `LanceStore` impl and any
/// future migration code agree on what's where.
pub const DOCUMENTS_TABLE: &str = "documents";
pub const MEMORY_TABLE: &str = "memory";
#[cfg(feature = "code-search")]
pub const CODE_CHUNKS_TABLE: &str = "code_chunks";