remem/retrieval/
embedding.rs

1use std::time::Duration;
2
3use anyhow::{bail, Context, Result};
4use serde::{Deserialize, Serialize};
5use sha2::{Digest, Sha256};
6
7mod config;
8mod fallback;
9mod index_text;
10mod local_semantic;
11mod network_policy;
12#[cfg(test)]
13mod network_policy_tests;
14mod status;
15
16use config::env_value;
17pub(crate) use config::resolve_embedding_config;
18pub use fallback::EmbeddingExecutionMetadata;
19pub(crate) use index_text::embed_memory_index_with_fallback_cache;
20pub use index_text::{embed_memory_index, memory_index_hash};
21pub(crate) use local_semantic::with_configured_model_read_lock;
22use local_semantic::LocalEmbeddingInputKind;
23pub use local_semantic::{
24    LocalEmbeddingDownloadReport, LocalEmbeddingInventoryReport, LocalEmbeddingModelInventory,
25};
26pub(crate) use network_policy::{
27    embed_query_if_enabled, embed_query_local_only_if_enabled,
28    local_only_embedding_profile_fingerprint,
29};
30pub(crate) use status::is_embedding_provider_off_error;
31
32pub const FEATURE_HASH_EMBEDDING_DIMENSIONS: usize = 768;
33pub const FEATURE_HASH_EMBEDDING_MODEL: &str = "remem-local-feature-hash-v1";
34pub const LOCAL_EMBEDDING_DIMENSIONS: usize = FEATURE_HASH_EMBEDDING_DIMENSIONS;
35pub const LOCAL_EMBEDDING_MODEL: &str = FEATURE_HASH_EMBEDDING_MODEL;
36
37const DEFAULT_PROVIDER: EmbeddingProvider = EmbeddingProvider::Auto;
38const OPENAI_DEFAULT_BASE_URL: &str = "https://api.openai.com/v1";
39const OPENAI_DEFAULT_MODEL: &str = "text-embedding-3-small";
40const DEFAULT_API_KEY_ENV: &str = "OPENAI_API_KEY";
41const DEFAULT_TIMEOUT_SECS: u64 = 30;
42
43const ENV_PROVIDER: &str = "REMEM_EMBEDDINGS_PROVIDER";
44const ENV_PROVIDER_LEGACY: &str = "REMEM_EMBEDDING_PROVIDER";
45const ENV_MODEL: &str = "REMEM_EMBEDDINGS_MODEL";
46const ENV_MODEL_LEGACY: &str = "REMEM_EMBEDDING_MODEL";
47const ENV_BASE_URL: &str = "REMEM_EMBEDDINGS_BASE_URL";
48const ENV_BASE_URL_LEGACY: &str = "REMEM_EMBEDDING_BASE_URL";
49const ENV_DIMENSIONS: &str = "REMEM_EMBEDDINGS_DIMENSIONS";
50const ENV_DIMENSIONS_LEGACY: &str = "REMEM_EMBEDDING_DIMENSIONS";
51const ENV_API_KEY: &str = "REMEM_EMBEDDINGS_API_KEY";
52const ENV_API_KEY_LEGACY: &str = "REMEM_EMBEDDING_API_KEY";
53const ENV_API_KEY_ENV: &str = "REMEM_EMBEDDINGS_API_KEY_ENV";
54const ENV_TIMEOUT_SECS: &str = "REMEM_EMBEDDINGS_TIMEOUT_SECS";
55const ENV_FALLBACK: &str = "REMEM_EMBEDDINGS_FALLBACK";
56const ENV_MODEL_DIR: &str = "REMEM_EMBEDDINGS_MODEL_DIR";
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub enum EmbeddingProvider {
60    Auto,
61    Local,
62    FeatureHash,
63    OpenAi,
64    Off,
65}
66
67impl EmbeddingProvider {
68    fn parse(raw: &str) -> Result<Self> {
69        match raw.trim().to_ascii_lowercase().as_str() {
70            "auto" => Ok(Self::Auto),
71            "local" => Ok(Self::Local),
72            "feature-hash" | "feature_hash" | "offline" => Ok(Self::FeatureHash),
73            "api" | "openai" | "openai-compatible" | "openai_compatible" => Ok(Self::OpenAi),
74            "off" | "disabled" | "none" => Ok(Self::Off),
75            other => bail!("unknown embeddings.provider: {other}"),
76        }
77    }
78
79    pub fn label(self) -> &'static str {
80        match self {
81            Self::Auto => "auto",
82            Self::Local => "local",
83            Self::FeatureHash => "feature-hash",
84            Self::OpenAi => "api",
85            Self::Off => "off",
86        }
87    }
88}
89
90#[derive(Debug, Clone, PartialEq, Eq)]
91pub struct EmbeddingConfig {
92    pub provider: EmbeddingProvider,
93    pub fallback: Option<EmbeddingProvider>,
94    pub model: String,
95    pub base_url: String,
96    pub dimensions: Option<usize>,
97    pub api_key_env: String,
98    pub model_dir: Option<String>,
99    pub timeout_secs: u64,
100}
101
102impl Default for EmbeddingConfig {
103    fn default() -> Self {
104        Self {
105            provider: DEFAULT_PROVIDER,
106            fallback: None,
107            model: OPENAI_DEFAULT_MODEL.to_string(),
108            base_url: OPENAI_DEFAULT_BASE_URL.to_string(),
109            dimensions: None,
110            api_key_env: DEFAULT_API_KEY_ENV.to_string(),
111            model_dir: None,
112            timeout_secs: DEFAULT_TIMEOUT_SECS,
113        }
114    }
115}
116
117#[derive(Debug, Clone, PartialEq, Eq)]
118pub struct EmbeddingProviderStatus {
119    pub configured_provider: String,
120    pub fallback_provider: Option<String>,
121    pub active_provider: String,
122    pub active_model_id: Option<String>,
123    pub active_dimensions: Option<usize>,
124    pub degraded: bool,
125    pub disabled: bool,
126    pub unavailable_reason: Option<String>,
127    pub degradation_reason: Option<String>,
128    pub model_dir: Option<String>,
129}
130
131#[derive(Debug, Clone, PartialEq)]
132pub struct TextEmbedding {
133    model: String,
134    values: Vec<f32>,
135}
136
137impl TextEmbedding {
138    pub fn new(model: impl Into<String>, values: Vec<f32>) -> Result<Self> {
139        let model = model.into();
140        if model.trim().is_empty() {
141            bail!("embedding model must not be empty");
142        }
143        validate_embedding_values(&values)?;
144        Ok(Self { model, values })
145    }
146
147    pub fn model(&self) -> &str {
148        &self.model
149    }
150
151    pub fn values(&self) -> &[f32] {
152        &self.values
153    }
154
155    pub fn dimensions(&self) -> usize {
156        self.values.len()
157    }
158
159    pub fn profile(&self) -> EmbeddingProfile<'_> {
160        EmbeddingProfile {
161            model: &self.model,
162            dimensions: self.values.len(),
163        }
164    }
165}
166
167#[derive(Debug, Clone, Copy, PartialEq, Eq)]
168pub struct EmbeddingProfile<'a> {
169    pub model: &'a str,
170    pub dimensions: usize,
171}
172
173#[derive(Debug, Clone, PartialEq, Eq)]
174pub struct EmbeddingBackfillTarget {
175    pub model: String,
176    pub dimensions: usize,
177}
178
179#[derive(Debug, Default)]
180pub(crate) struct EmbeddingFallbackCache {
181    call_failure_fallback: Option<EmbeddingProvider>,
182    call_failure_fallback_target: Option<EmbeddingBackfillTarget>,
183    execution_provider: Option<EmbeddingProvider>,
184    degradation_reason: Option<String>,
185}
186
187impl EmbeddingFallbackCache {
188    pub(crate) fn call_failure_fallback_target(&self) -> Option<EmbeddingBackfillTarget> {
189        match self.call_failure_fallback {
190            Some(EmbeddingProvider::Local) => self.call_failure_fallback_target.clone(),
191            Some(EmbeddingProvider::FeatureHash) => Some(EmbeddingBackfillTarget {
192                model: FEATURE_HASH_EMBEDDING_MODEL.to_string(),
193                dimensions: FEATURE_HASH_EMBEDDING_DIMENSIONS,
194            }),
195            Some(EmbeddingProvider::Auto)
196            | Some(EmbeddingProvider::OpenAi)
197            | Some(EmbeddingProvider::Off) => None,
198            None => None,
199        }
200    }
201}
202
203#[derive(Debug)]
204pub(crate) struct QueryEmbeddingExecution {
205    pub(crate) embedding: TextEmbedding,
206    pub(crate) metadata: EmbeddingExecutionMetadata,
207}
208
209pub fn embed_query(query: &str) -> Result<TextEmbedding> {
210    embed_text(query, LocalEmbeddingInputKind::Query)
211}
212
213pub(crate) fn embed_query_with_fallback_cache(
214    query: &str,
215    cache: &mut EmbeddingFallbackCache,
216) -> Result<TextEmbedding> {
217    embed_text_with_fallback_cache(query, LocalEmbeddingInputKind::Query, cache)
218}
219
220pub(crate) fn embed_query_with_execution_if_enabled(
221    query: &str,
222) -> Result<Option<QueryEmbeddingExecution>> {
223    #[cfg(test)]
224    let _test_env_guard = config::lock_test_env();
225    let config = resolve_embedding_config()?;
226    let status_before = status::resolve_provider_status(&config);
227    if let Some(error) = disabled_provider_status_error(&status_before) {
228        return if is_embedding_provider_off_error(&error) {
229            Ok(None)
230        } else {
231            Err(error)
232        };
233    }
234    let mut cache = EmbeddingFallbackCache::default();
235    let embedding = embed_text_with_resolved_config(
236        query,
237        LocalEmbeddingInputKind::Query,
238        &config,
239        &mut cache,
240    )?;
241    let status_after = status::resolve_provider_status(&config);
242    let metadata =
243        fallback::embedding_execution_metadata(&status_before, &status_after, &cache, &embedding)?;
244    Ok(Some(QueryEmbeddingExecution {
245        embedding,
246        metadata,
247    }))
248}
249
250pub fn embed_memory(
251    title: &str,
252    content: &str,
253    memory_type: &str,
254    topic_key: Option<&str>,
255) -> Result<TextEmbedding> {
256    let text = memory_embedding_text(title, content, memory_type, topic_key);
257    embed_text(&text, LocalEmbeddingInputKind::Passage)
258}
259
260pub fn embed_query_text_local(query: &str) -> Vec<f32> {
261    embed_text_local(query)
262}
263
264pub fn embed_memory_text_local(
265    title: &str,
266    content: &str,
267    memory_type: &str,
268    topic_key: Option<&str>,
269) -> Vec<f32> {
270    embed_text_local(&memory_embedding_text(
271        title,
272        content,
273        memory_type,
274        topic_key,
275    ))
276}
277
278pub fn embedding_content_hash(
279    title: &str,
280    content: &str,
281    memory_type: &str,
282    topic_key: Option<&str>,
283) -> String {
284    let mut hasher = Sha256::new();
285    hasher.update(memory_type.as_bytes());
286    hasher.update([0]);
287    if let Some(topic_key) = topic_key {
288        hasher.update(topic_key.as_bytes());
289    }
290    hasher.update([0]);
291    hasher.update(title.as_bytes());
292    hasher.update([0]);
293    hasher.update(content.as_bytes());
294    let digest = hasher.finalize();
295    digest.iter().map(|byte| format!("{byte:02x}")).collect()
296}
297
298pub(crate) fn configured_backfill_target() -> Result<EmbeddingBackfillTarget> {
299    let mut cache = EmbeddingFallbackCache::default();
300    configured_backfill_target_with_fallback_cache(&mut cache)
301}
302
303pub(crate) fn configured_backfill_target_with_fallback_cache(
304    cache: &mut EmbeddingFallbackCache,
305) -> Result<EmbeddingBackfillTarget> {
306    #[cfg(test)]
307    let _test_env_guard = config::lock_test_env();
308    let status = embedding_provider_status_without_probe()?;
309    if let Some(error) = disabled_provider_status_error(&status) {
310        return Err(error);
311    }
312    let probe = embed_text_with_fallback_cache(
313        "remem embedding profile probe",
314        LocalEmbeddingInputKind::Generic,
315        cache,
316    )?;
317    Ok(EmbeddingBackfillTarget {
318        model: probe.model().to_string(),
319        dimensions: probe.dimensions(),
320    })
321}
322
323pub fn embedding_provider_status() -> Result<EmbeddingProviderStatus> {
324    #[cfg(test)]
325    let _test_env_guard = config::lock_test_env();
326    let config = resolve_embedding_config()?;
327    let mut status = status::resolve_provider_status(&config);
328    status::probe_active_api_profile(&config, &mut status);
329    Ok(status)
330}
331
332pub(crate) fn embedding_provider_status_without_probe() -> Result<EmbeddingProviderStatus> {
333    #[cfg(test)]
334    let _test_env_guard = config::lock_test_env();
335    let config = resolve_embedding_config()?;
336    Ok(status::resolve_provider_status(&config))
337}
338
339pub(crate) fn disabled_provider_status_error(
340    status: &EmbeddingProviderStatus,
341) -> Option<anyhow::Error> {
342    if !status.disabled {
343        return None;
344    }
345    status
346        .degradation_reason
347        .clone()
348        .or_else(|| status.unavailable_reason.clone())
349        .map(status::embedding_provider_off_error_with_cause)
350        .or_else(|| Some(status::embedding_provider_off_error()))
351}
352
353pub(crate) fn provider_disabled_or_error() -> Result<bool> {
354    let status = embedding_provider_status_without_probe()?;
355    match disabled_provider_status_error(&status) {
356        Some(error) if is_embedding_provider_off_error(&error) => Ok(true),
357        Some(error) => Err(error),
358        None => Ok(false),
359    }
360}
361
362pub(crate) fn configured_local_embedding_model_id(config: &EmbeddingConfig) -> Result<String> {
363    local_semantic::configured_model_id(config)
364}
365
366pub(crate) fn configured_local_embedding_model_root(
367    config: &EmbeddingConfig,
368) -> Result<std::path::PathBuf> {
369    local_semantic::model_root(config)
370}
371
372pub(crate) fn configured_local_embedding_artifact_sha256(
373    config: &EmbeddingConfig,
374) -> Result<String> {
375    Ok(local_semantic::installed_model_profile(config)?.artifact_sha256)
376}
377
378pub fn download_local_embedding_model(model: Option<&str>) -> Result<LocalEmbeddingDownloadReport> {
379    local_semantic::download_model(model)
380}
381
382pub fn local_embedding_inventory() -> Result<LocalEmbeddingInventoryReport> {
383    local_semantic::inventory()
384}
385
386#[cfg(test)]
387pub(crate) use local_semantic::install_test_model as install_test_local_embedding_model;
388#[cfg(all(test, feature = "local-onnx"))]
389pub(crate) use local_semantic::{
390    fail_next_test_model_embed_generic, fail_next_test_model_embed_unavailable,
391};
392#[cfg(test)]
393pub(crate) const TEST_LOCAL_SEMANTIC_MODEL: &str = local_semantic::DEFAULT_LOCAL_SEMANTIC_MODEL;
394
395pub(crate) fn is_local_embedding_model_unavailable_error(error: &anyhow::Error) -> bool {
396    local_semantic::is_model_unavailable_error(error)
397}
398
399fn embed_text(text: &str, kind: LocalEmbeddingInputKind) -> Result<TextEmbedding> {
400    let mut cache = EmbeddingFallbackCache::default();
401    embed_text_with_fallback_cache(text, kind, &mut cache)
402}
403
404fn embed_text_with_fallback_cache(
405    text: &str,
406    kind: LocalEmbeddingInputKind,
407    cache: &mut EmbeddingFallbackCache,
408) -> Result<TextEmbedding> {
409    #[cfg(test)]
410    let _test_env_guard = config::lock_test_env();
411    let config = resolve_embedding_config()?;
412    embed_text_with_resolved_config(text, kind, &config, cache)
413}
414
415fn embed_text_with_resolved_config(
416    text: &str,
417    kind: LocalEmbeddingInputKind,
418    config: &EmbeddingConfig,
419    cache: &mut EmbeddingFallbackCache,
420) -> Result<TextEmbedding> {
421    if let Some(fallback) = cache.call_failure_fallback {
422        let embedding =
423            fallback::embed_with_cached_call_failure_fallback(text, kind, config, fallback)?;
424        cache.execution_provider = Some(fallback);
425        return Ok(embedding);
426    }
427    match active_provider(config)? {
428        ActiveEmbeddingProvider::Local => {
429            fallback::embed_local_with_auto_race_fallback(text, kind, config, cache)
430        }
431        ActiveEmbeddingProvider::FeatureHash => {
432            let embedding =
433                TextEmbedding::new(FEATURE_HASH_EMBEDDING_MODEL, embed_text_local(text))?;
434            cache.execution_provider = Some(EmbeddingProvider::FeatureHash);
435            Ok(embedding)
436        }
437        ActiveEmbeddingProvider::OpenAi { api_key } => match embed_openai(text, config, &api_key) {
438            Ok(embedding) => {
439                cache.execution_provider = Some(EmbeddingProvider::OpenAi);
440                Ok(embedding)
441            }
442            Err(error) => {
443                fallback::embed_with_call_failure_fallback(text, kind, config, error, cache)
444            }
445        },
446        ActiveEmbeddingProvider::Off => Err(status::embedding_provider_off_error()),
447    }
448}
449
450fn memory_embedding_text(
451    title: &str,
452    content: &str,
453    memory_type: &str,
454    topic_key: Option<&str>,
455) -> String {
456    let mut text = String::new();
457    text.push_str(memory_type);
458    text.push('\n');
459    if let Some(topic_key) = topic_key {
460        text.push_str(topic_key);
461        text.push('\n');
462    }
463    text.push_str(title);
464    text.push('\n');
465    text.push_str(content);
466    text
467}
468
469#[derive(Debug, Clone, PartialEq, Eq)]
470enum ActiveEmbeddingProvider {
471    Local,
472    FeatureHash,
473    OpenAi { api_key: String },
474    Off,
475}
476
477fn active_provider(config: &EmbeddingConfig) -> Result<ActiveEmbeddingProvider> {
478    let status = status::resolve_provider_status(config);
479    if status.active_provider == EmbeddingProvider::Off.label() && status.degraded {
480        return Err(status::embedding_provider_off_error_with_cause(
481            status
482                .degradation_reason
483                .unwrap_or_else(|| "embedding provider fallback is off".to_string()),
484        ));
485    }
486    if let Some(reason) = status.unavailable_reason {
487        if status.active_provider == EmbeddingProvider::Off.label() {
488            return Err(status::embedding_provider_off_error_with_cause(reason));
489        }
490        if status.active_provider == EmbeddingProvider::Local.label() {
491            return Err(local_semantic::model_unavailable_error(reason));
492        }
493        bail!("{reason}");
494    }
495    match EmbeddingProvider::parse(&status.active_provider)? {
496        EmbeddingProvider::Local => Ok(ActiveEmbeddingProvider::Local),
497        EmbeddingProvider::FeatureHash => Ok(ActiveEmbeddingProvider::FeatureHash),
498        EmbeddingProvider::OpenAi => Ok(ActiveEmbeddingProvider::OpenAi {
499            api_key: configured_api_key(config)?.with_context(|| {
500                format!(
501                    "embedding provider api requires {ENV_API_KEY} or {}",
502                    config.api_key_env
503                )
504            })?,
505        }),
506        EmbeddingProvider::Off => Ok(ActiveEmbeddingProvider::Off),
507        EmbeddingProvider::Auto => bail!("auto must resolve to a concrete embedding provider"),
508    }
509}
510
511fn auto_api_key(config: &EmbeddingConfig) -> Result<Option<String>> {
512    if let Some(value) = env_value(ENV_API_KEY).or_else(|| env_value(ENV_API_KEY_LEGACY)) {
513        return Ok(Some(value));
514    }
515    if config.api_key_env != DEFAULT_API_KEY_ENV {
516        configured_api_key(config)
517    } else {
518        Ok(None)
519    }
520}
521
522fn configured_api_key(config: &EmbeddingConfig) -> Result<Option<String>> {
523    if let Some(value) = env_value(ENV_API_KEY).or_else(|| env_value(ENV_API_KEY_LEGACY)) {
524        return Ok(Some(value));
525    }
526    Ok(std::env::var(&config.api_key_env)
527        .ok()
528        .map(|value| value.trim().to_string())
529        .filter(|value| !value.is_empty()))
530}
531
532#[derive(Debug, Serialize)]
533struct OpenAiEmbeddingRequest<'a> {
534    input: &'a str,
535    model: &'a str,
536    encoding_format: &'static str,
537    #[serde(skip_serializing_if = "Option::is_none")]
538    dimensions: Option<usize>,
539}
540
541#[derive(Debug, Deserialize)]
542struct OpenAiEmbeddingResponse {
543    data: Vec<OpenAiEmbeddingData>,
544    model: Option<String>,
545}
546
547#[derive(Debug, Deserialize)]
548struct OpenAiEmbeddingData {
549    embedding: Vec<f32>,
550}
551
552fn embed_openai(text: &str, config: &EmbeddingConfig, api_key: &str) -> Result<TextEmbedding> {
553    if text.trim().is_empty() {
554        bail!("embedding input must not be empty");
555    }
556    let client = reqwest::blocking::Client::builder()
557        .timeout(Duration::from_secs(config.timeout_secs))
558        .build()
559        .context("build embedding HTTP client")?;
560    let request = OpenAiEmbeddingRequest {
561        input: text,
562        model: &config.model,
563        encoding_format: "float",
564        dimensions: config.dimensions,
565    };
566    let url = format!("{}/embeddings", config.base_url.trim_end_matches('/'));
567    let response = client
568        .post(&url)
569        .bearer_auth(api_key)
570        .json(&request)
571        .send()
572        .with_context(|| format!("call embedding provider at {url}"))?;
573    let status = response.status();
574    let body = response
575        .text()
576        .context("read embedding provider response body")?;
577    if !status.is_success() {
578        bail!(
579            "embedding provider returned HTTP {status}: {}",
580            truncate_error_body(&body)
581        );
582    }
583    parse_openai_embedding_response(&body, &config.model)
584}
585
586fn parse_openai_embedding_response(body: &str, fallback_model: &str) -> Result<TextEmbedding> {
587    let response: OpenAiEmbeddingResponse =
588        serde_json::from_str(body).context("parse embedding provider response")?;
589    let mut data = response.data.into_iter();
590    let first = data
591        .next()
592        .context("embedding provider response did not include data[0]")?;
593    if data.next().is_some() {
594        bail!("embedding provider returned multiple embeddings for single input");
595    }
596    TextEmbedding::new(
597        response.model.unwrap_or_else(|| fallback_model.to_string()),
598        first.embedding,
599    )
600}
601
602fn truncate_error_body(body: &str) -> String {
603    const MAX: usize = 500;
604    if body.len() <= MAX {
605        body.to_string()
606    } else {
607        let mut end = MAX;
608        while !body.is_char_boundary(end) {
609            end -= 1;
610        }
611        format!("{}...", &body[..end])
612    }
613}
614
615fn validate_embedding_values(values: &[f32]) -> Result<()> {
616    if values.is_empty() {
617        bail!("embedding vector must not be empty");
618    }
619    if values.iter().any(|value| !value.is_finite()) {
620        bail!("embedding vector contains non-finite values");
621    }
622    Ok(())
623}
624
625fn embed_text_local(text: &str) -> Vec<f32> {
626    let normalized = text.to_lowercase();
627    let mut vector = vec![0.0f32; LOCAL_EMBEDDING_DIMENSIONS];
628    for token in semantic_tokens(&normalized) {
629        add_feature(&mut vector, &format!("token:{token}"), 1.0);
630    }
631    for ngram in char_ngrams(&normalized) {
632        add_feature(&mut vector, &format!("ngram:{ngram}"), 0.35);
633    }
634    for (concept, phrases) in semantic_concepts() {
635        if phrases.iter().any(|phrase| normalized.contains(phrase)) {
636            add_feature(&mut vector, &format!("concept:{concept}"), 4.0);
637        }
638    }
639    normalize(&mut vector);
640    vector
641}
642
643fn semantic_tokens(text: &str) -> Vec<String> {
644    let mut tokens = Vec::new();
645    let mut current = String::new();
646    for ch in text.chars() {
647        if ch.is_alphanumeric() || is_cjk(ch) {
648            current.push(ch);
649        } else if !current.is_empty() {
650            tokens.push(std::mem::take(&mut current));
651        }
652    }
653    if !current.is_empty() {
654        tokens.push(current);
655    }
656    tokens
657}
658
659fn char_ngrams(text: &str) -> Vec<String> {
660    let chars: Vec<char> = text
661        .chars()
662        .filter(|ch| ch.is_alphanumeric() || is_cjk(*ch))
663        .collect();
664    let mut grams = Vec::new();
665    for width in [2usize, 3] {
666        if chars.len() < width {
667            continue;
668        }
669        grams.extend(
670            chars
671                .windows(width)
672                .map(|window| window.iter().collect::<String>()),
673        );
674    }
675    grams
676}
677
678fn add_feature(vector: &mut [f32], feature: &str, weight: f32) {
679    let digest = Sha256::digest(feature.as_bytes());
680    for offset in [0usize, 8, 16] {
681        let raw = u64::from_le_bytes([
682            digest[offset],
683            digest[offset + 1],
684            digest[offset + 2],
685            digest[offset + 3],
686            digest[offset + 4],
687            digest[offset + 5],
688            digest[offset + 6],
689            digest[offset + 7],
690        ]);
691        let idx = raw as usize % vector.len();
692        let sign = if raw & 1 == 0 { 1.0 } else { -1.0 };
693        vector[idx] += weight * sign;
694    }
695}
696
697fn normalize(vector: &mut [f32]) {
698    let norm = vector.iter().map(|value| value * value).sum::<f32>().sqrt();
699    if norm == 0.0 {
700        return;
701    }
702    for value in vector {
703        *value /= norm;
704    }
705}
706
707fn is_cjk(ch: char) -> bool {
708    matches!(
709        ch,
710        '\u{4E00}'..='\u{9FFF}' |
711        '\u{3400}'..='\u{4DBF}' |
712        '\u{F900}'..='\u{FAFF}'
713    )
714}
715
716fn semantic_concepts() -> &'static [(&'static str, &'static [&'static str])] {
717    &[
718        (
719            "data-security",
720            &[
721                "sqlcipher",
722                "encrypt",
723                "encrypted",
724                "encryption",
725                "secret",
726                "secrets",
727                "credential",
728                "credentials",
729                "private",
730                "confidential",
731                "protect",
732                "protected",
733                "at rest",
734                "persisted data",
735                "加密",
736                "密钥",
737            ],
738        ),
739        (
740            "transcript-capture",
741            &[
742                "transcript",
743                "raw archive",
744                "raw message",
745                "hook fallback",
746                "assistant message",
747                "conversation capture",
748                "jsonl",
749                "会话",
750                "原始消息",
751            ],
752        ),
753        (
754            "retrieval-quality",
755            &[
756                "semantic",
757                "embedding",
758                "vector",
759                "recall",
760                "search quality",
761                "paraphrase",
762                "检索",
763                "语义",
764                "召回",
765                "向量",
766            ],
767        ),
768        (
769            "current-state",
770            &[
771                "current decision",
772                "current state",
773                "supersede",
774                "supersedes",
775                "stale",
776                "replacement",
777                "现在",
778                "当前",
779                "替代",
780            ],
781        ),
782        (
783            "compression",
784            &[
785                "compress",
786                "compression",
787                "compaction",
788                "summarize",
789                "compressed",
790                "压缩",
791                "摘要",
792                "总结",
793            ],
794        ),
795    ]
796}
797
798#[cfg(test)]
799mod tests;
remem/retrieval/embedding.rs

remem/retrieval/
embedding.rs
