oxibrain-core 0.5.0

Core domain types, temporal fold, extraction, ranking, and packing for oxibrain
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

//! Context assembly types (DESIGN §9.5).

use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ContextBudget {
    pub max_tokens: usize,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ContextLayer {
    pub kind: LayerKind,
    pub text: String,
    pub estimated_tokens: usize,
    pub provenance: Vec<String>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LayerKind {
    Profile,
    PinnedFacts,
    HighSalienceBeliefs,
    QueryNeighborhood,
    Summaries,
    RecentEpisodes,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ContextResult {
    pub layers: Vec<ContextLayer>,
    pub total_tokens: usize,
    pub budget: ContextBudget,
    pub truncated: bool,
}

/// Pre-load fallback token estimate (§7.5). Uses chars/4 — off by roughly
/// fivefold on CJK (F27). Replaced by `TokenizerPort::count()` once the model
/// tokenizer is available.
pub fn estimate_tokens_rough(text: &str) -> usize {
    (text.chars().count() / 4).max(1)
}