1use std::{
9 collections::BTreeMap,
10 path::{Path, PathBuf},
11};
12
13use anyhow::{Context, Result, anyhow, bail};
14use lance_io::object_store::uri_to_url;
15use serde::{Deserialize, Deserializer, Serialize, de};
16use serde_json::Value;
17use url::Url;
18
19fn parse_byte_size(raw: &str) -> Result<usize, String> {
24 let trimmed = raw.trim();
25 if trimmed.is_empty() {
26 return Err("byte-size value is empty".to_owned());
27 }
28 let split = trimmed
29 .find(|c: char| c.is_ascii_alphabetic())
30 .unwrap_or(trimmed.len());
31 let (number, unit) = trimmed.split_at(split);
32 let number: f64 = number
33 .trim()
34 .parse()
35 .map_err(|_| format!("byte-size value {raw:?} is not a number"))?;
36 if !number.is_finite() || number < 0.0 {
37 return Err(format!("byte-size value {raw:?} must be non-negative"));
38 }
39 let multiplier: f64 = match unit.trim().to_ascii_lowercase().as_str() {
40 "" | "b" => 1.0,
41 "k" | "kb" => 1_000.0,
42 "kib" => 1_024.0,
43 "m" | "mb" => 1_000_000.0,
44 "mib" => 1_048_576.0,
45 "g" | "gb" => 1_000_000_000.0,
46 "gib" => 1_073_741_824.0,
47 "tib" => 1_099_511_627_776.0,
48 other => {
49 return Err(format!(
50 "byte-size unit {other:?} not recognized (try MiB / GiB)"
51 ));
52 }
53 };
54 let bytes = number * multiplier;
55 if !bytes.is_finite() || bytes > usize::MAX as f64 {
56 return Err(format!("byte-size value {raw:?} overflows usize"));
57 }
58 Ok(bytes as usize)
59}
60
61fn deserialize_byte_size_opt<'de, D>(deserializer: D) -> Result<Option<usize>, D::Error>
62where
63 D: Deserializer<'de>,
64{
65 #[derive(Deserialize)]
66 #[serde(untagged)]
67 enum Repr {
68 Bytes(u64),
69 Text(String),
70 }
71 let repr: Option<Repr> = Option::deserialize(deserializer)?;
72 match repr {
73 None => Ok(None),
74 Some(Repr::Bytes(value)) => usize::try_from(value).map(Some).map_err(de::Error::custom),
75 Some(Repr::Text(value)) => parse_byte_size(&value).map(Some).map_err(de::Error::custom),
76 }
77}
78
79pub fn parse_data_dir(input: &str) -> Result<Url> {
90 uri_to_url(input).with_context(|| format!("invalid --data-dir {input:?}"))
91}
92
93pub fn is_local(url: &Url) -> bool {
97 matches!(url.scheme(), "file" | "file+uring")
98}
99
100pub fn local_path(url: &Url) -> Option<PathBuf> {
102 if is_local(url) {
103 url.to_file_path().ok()
104 } else {
105 None
106 }
107}
108
109pub fn child_uri(base: &Url, suffix: &str) -> String {
115 if let Some(path) = local_path(base) {
119 return path.join(suffix).display().to_string();
120 }
121 format!("{}/{suffix}", base.as_str().trim_end_matches('/'))
122}
123
124pub fn display(url: &Url) -> String {
127 if let Some(path) = local_path(url) {
128 path.display().to_string()
129 } else {
130 url.to_string()
131 }
132}
133
134pub fn url_for_path(path: impl AsRef<Path>) -> Result<Url> {
139 let path = path.as_ref();
140 let absolute = if path.is_absolute() {
141 path.to_path_buf()
142 } else {
143 std::path::absolute(path)
144 .with_context(|| format!("failed to absolutize {}", path.display()))?
145 };
146 Url::from_file_path(&absolute).map_err(|()| {
147 anyhow!(
148 "failed to convert path {} into a file:// URL",
149 absolute.display()
150 )
151 })
152}
153
154pub const DEFAULT_CONFIG_TOML: &str = "\
158# pond configuration.
159#
160# pond ships built-in defaults, so every setting here is optional - delete this
161# file and pond still works. Uncomment and edit to override.
162
163# Where pond looks for source data to import. One entry per adapter type
164# (`claude-code`, `codex-cli`, ...). `pond sync` with no arguments syncs every
165# entry; `pond sync <adapter>` syncs just one. With an empty `[sources]`,
166# `pond sync` runs an interactive discovery against the known default paths
167# and writes the picks back here.
168#
169# Future wrap: pond is single-namespace in v1 (spec.md#wire-namespace-resolution); `[sources]` is
170# flat here. When multi-namespace pond lands, source registration becomes
171# per-tenant under `[namespaces.<ns>.sources.<adapter>]`. Pre-v1 the schema
172# is breakable; the rename is operationally free until a real second tenant
173# exists.
174#
175# [sources.claude-code]
176# path = \"~/.claude/projects\"
177#
178# [sources.codex-cli]
179# path = \"~/.codex/sessions\"
180
181# Embeddings. Search runs hybrid (vector + FTS) whenever the store has any
182# vectors, and FTS-only otherwise - the model loads lazily on the first hybrid
183# query, so there's no cost on FTS-only corpora. `model` selects the
184# HuggingFace XLM-RoBERTa model; `dim` declares its output width and is baked
185# into the messages.vector schema on table creation - it must equal the
186# model's hidden_size and be a multiple of 8 (IVF_PQ subspace stride).
187#
188# Common pairings:
189# model = \"intfloat/multilingual-e5-small\" dim = 384 (default)
190# model = \"intfloat/multilingual-e5-base\" dim = 768
191# model = \"intfloat/multilingual-e5-large\" dim = 1024
192#
193# A different-dim model needs a fresh data dir; pond enforces this at the
194# schema boundary.
195#
196# [embeddings]
197# model = \"intfloat/multilingual-e5-small\"
198# dim = 384
199
200# Search tuning. Leave unset for Lance defaults; set when tuning IVF_PQ recall
201# against a corpus.
202#
203# `index_lag_threshold` is the minimum unindexed-fragment count before a
204# per-intent append/rebuild runs in `pond index optimize`; the brute-force
205# fallback keeps queries correct while fragments accumulate. Defaults to 4.
206#
207# [search]
208# nprobes = 16
209# refine_factor = 2
210# index_lag_threshold = 4
211
212# Long-running process caps. Both accept either a plain byte count or a
213# humansize-style suffix (\"128 MiB\", \"1 GiB\"). Both are optional - leave
214# unset to let pond pick the backend-aware default:
215# local FS : index_cache = 256 MiB, metadata_cache = 128 MiB
216# remote : index_cache = 2 GiB, metadata_cache = 512 MiB
217# Lance's library defaults (6 GiB / 1 GiB) are too generous for a per-session
218# `pond mcp` process; tightening them is what keeps RSS under the 500 MiB target
219# without measurable latency regressions on typical agent-history corpora.
220#
221# [runtime]
222# index_cache_bytes = \"256 MiB\"
223# metadata_cache_bytes = \"128 MiB\"
224
225# Object-store credentials and tuning, passed verbatim to Lance's
226# `DatasetBuilder::with_storage_options`. Required only when `--data-dir` is
227# an `s3://` / `gs://` / `az://` URI that needs auth or a non-default region.
228# Keys follow the `object_store` crate's standard names. Environment
229# variables of the same name are read by `object_store` automatically;
230# values in this block override them. pond does not parse these.
231#
232# Future wrap: pond is single-namespace in v1 (spec.md#wire-namespace-resolution); `[storage]` is
233# flat here on the assumption of one bucket per pond. When multi-namespace
234# pond lands and tenants need separate buckets/regions, this becomes
235# `[namespaces.<ns>.storage]`. Pre-v1 the schema is breakable; the rename is
236# operationally free until a real second tenant exists.
237#
238# [storage]
239# AWS_ACCESS_KEY_ID = \"...\"
240# AWS_SECRET_ACCESS_KEY = \"...\"
241# AWS_REGION = \"us-east-1\"
242# AWS_ENDPOINT = \"https://minio.example.com\" # for self-hosted MinIO
243# allow_http = \"true\" # only for non-TLS endpoints
244";
245
246#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
248#[serde(deny_unknown_fields)]
249pub struct Config {
250 #[serde(default)]
251 pub embeddings: EmbeddingsConfig,
252 #[serde(default)]
253 pub search: SearchConfig,
254 #[serde(default)]
255 pub runtime: RuntimeConfig,
256 #[serde(default)]
262 pub sources: BTreeMap<String, Value>,
263 #[serde(default)]
273 pub storage: BTreeMap<String, String>,
274}
275
276#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
281#[serde(deny_unknown_fields, default)]
282pub struct RuntimeConfig {
283 #[serde(default, deserialize_with = "deserialize_byte_size_opt")]
284 pub index_cache_bytes: Option<usize>,
285 #[serde(default, deserialize_with = "deserialize_byte_size_opt")]
286 pub metadata_cache_bytes: Option<usize>,
287}
288
289#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
291#[serde(deny_unknown_fields)]
292pub struct SearchConfig {
293 #[serde(default)]
294 pub nprobes: Option<usize>,
295 #[serde(default)]
296 pub refine_factor: Option<u32>,
297 #[serde(default)]
303 pub index_lag_threshold: Option<usize>,
304}
305
306#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
315#[serde(deny_unknown_fields, default)]
316pub struct EmbeddingsConfig {
317 pub model: String,
320 pub dim: usize,
324}
325
326impl Default for EmbeddingsConfig {
327 fn default() -> Self {
328 Self {
329 model: crate::embed::DEFAULT_MODEL_ID.to_owned(),
330 dim: crate::sessions::DEFAULT_EMBEDDING_DIM,
331 }
332 }
333}
334
335pub fn resolve_data_dir(
341 explicit: Option<Url>,
342 xdg_data_home: Option<PathBuf>,
343 home: Option<PathBuf>,
344) -> Result<Url> {
345 if let Some(location) = explicit {
346 return Ok(location);
347 }
348 if let Some(xdg) = xdg_data_home.filter(|path| path.is_absolute()) {
349 return url_for_path(xdg.join("pond"));
350 }
351 if let Some(home) = home {
352 return url_for_path(home.join(".local").join("share").join("pond"));
353 }
354 url_for_path(PathBuf::from(".pond"))
356}
357
358pub fn default_config_path(xdg_config_home: Option<PathBuf>, home: Option<PathBuf>) -> PathBuf {
363 if let Some(xdg) = xdg_config_home.filter(|path| path.is_absolute()) {
364 return xdg.join("pond").join("config.toml");
365 }
366 if let Some(home) = home {
367 return home.join(".config").join("pond").join("config.toml");
368 }
369 PathBuf::from(".pond.toml")
370}
371
372impl Config {
373 pub fn load(path: impl AsRef<Path>) -> Result<Self> {
379 let path = path.as_ref();
380 let config = if path.exists() {
381 let text = std::fs::read_to_string(path)
382 .with_context(|| format!("failed to read config {}", path.display()))?;
383 toml::from_str::<Self>(&text)
384 .with_context(|| format!("failed to parse config {}", path.display()))?
385 } else {
386 Self::default()
387 };
388 config.embeddings.validate()?;
389 config.embeddings.install_runtime();
390 if let Some(threshold) = config.search.index_lag_threshold {
391 crate::substrate::init_index_lag_threshold(threshold);
392 }
393 Ok(config)
397 }
398
399 pub fn resolve_sources(&self, adapter: Option<&str>) -> Result<Vec<(String, Value)>> {
406 match adapter {
407 None => Ok(self
408 .sources
409 .iter()
410 .map(|(name, blob)| (name.clone(), blob.clone()))
411 .collect()),
412 Some(name) => {
413 let blob = self
414 .sources
415 .get(name)
416 .ok_or_else(|| anyhow!("no [sources.{name}] entry in config"))?;
417 Ok(vec![(name.to_owned(), blob.clone())])
418 }
419 }
420 }
421}
422
423pub fn expand_home_under(path: &Path, home: &Path) -> PathBuf {
429 let Some(text) = path.to_str() else {
430 return path.to_path_buf();
431 };
432 if text == "~" {
433 return home.to_path_buf();
434 }
435 if let Some(rest) = text.strip_prefix("~/") {
436 return home.join(rest);
437 }
438 path.to_path_buf()
439}
440
441impl EmbeddingsConfig {
442 pub fn validate(&self) -> Result<()> {
447 if self.model.trim().is_empty() {
448 bail!("embeddings.model must be a non-empty HuggingFace model id");
449 }
450 if self.dim == 0 || !self.dim.is_multiple_of(8) {
451 bail!(
452 "embeddings.dim = {} must be a positive multiple of 8 (IVF_PQ subspace stride)",
453 self.dim,
454 );
455 }
456 Ok(())
457 }
458
459 pub fn install_runtime(&self) {
463 crate::embed::init_model_id(self.model.clone());
464 crate::sessions::init_embedding_dim(self.dim);
465 }
466}
467
468#[cfg(test)]
469mod tests {
470 #![allow(clippy::expect_used, clippy::unwrap_used)]
471
472 use super::*;
473 use serde_json::Value;
474 use tempfile::TempDir;
475
476 #[test]
477 fn validate_catches_empty_model_and_bad_dim() {
478 assert!(EmbeddingsConfig::default().validate().is_ok());
479 let bad_model = EmbeddingsConfig {
482 model: " ".to_owned(),
483 dim: 768,
484 };
485 assert!(bad_model.validate().is_err());
486 let bad_dim = EmbeddingsConfig {
488 model: "intfloat/multilingual-e5-base".to_owned(),
489 dim: 100,
490 };
491 assert!(bad_dim.validate().is_err());
492 let zero_dim = EmbeddingsConfig {
494 model: "intfloat/multilingual-e5-base".to_owned(),
495 dim: 0,
496 };
497 assert!(zero_dim.validate().is_err());
498 }
499
500 #[test]
501 fn config_load_missing_file_falls_back_to_builtin() {
502 let config = Config::load("/nonexistent/pond-config-xyz.toml").unwrap();
503 assert_eq!(config.embeddings, EmbeddingsConfig::default());
504 }
505
506 #[test]
507 fn default_config_toml_loads_to_the_builtin_defaults() {
508 let dir = TempDir::new().unwrap();
509 let path = dir.path().join("config.toml");
510 std::fs::write(&path, DEFAULT_CONFIG_TOML).unwrap();
511 let config = Config::load(&path).unwrap();
514 assert_eq!(config.embeddings, EmbeddingsConfig::default());
515 assert_eq!(config.embeddings.model, crate::embed::DEFAULT_MODEL_ID);
516 assert_eq!(
517 config.embeddings.dim,
518 crate::sessions::DEFAULT_EMBEDDING_DIM
519 );
520 }
521
522 #[test]
523 fn resolve_data_dir_follows_explicit_then_xdg_then_home() {
524 let explicit = parse_data_dir("/explicit").unwrap();
528 let resolved = resolve_data_dir(
529 Some(explicit.clone()),
530 Some(PathBuf::from("/xdg")),
531 Some(PathBuf::from("/home")),
532 )
533 .unwrap();
534 assert_eq!(resolved, explicit);
535
536 let resolved = resolve_data_dir(
538 None,
539 Some(PathBuf::from("/xdg")),
540 Some(PathBuf::from("/home")),
541 )
542 .unwrap();
543 assert!(is_local(&resolved));
544 assert_eq!(local_path(&resolved).unwrap(), PathBuf::from("/xdg/pond"));
545
546 let resolved = resolve_data_dir(
548 None,
549 Some(PathBuf::from("relative")),
550 Some(PathBuf::from("/home")),
551 )
552 .unwrap();
553 assert_eq!(
554 local_path(&resolved).unwrap(),
555 PathBuf::from("/home/.local/share/pond"),
556 );
557
558 let resolved = resolve_data_dir(None, None, None).unwrap();
562 assert!(is_local(&resolved));
563 assert!(
564 local_path(&resolved).unwrap().ends_with(".pond"),
565 "fallback path should end with .pond: {resolved}",
566 );
567 }
568
569 #[test]
570 fn expand_home_under_handles_tilde_forms() {
571 let home = Path::new("/srv/me");
572 assert_eq!(
573 expand_home_under(Path::new("~"), home),
574 PathBuf::from("/srv/me")
575 );
576 assert_eq!(
577 expand_home_under(Path::new("~/.codex/sessions"), home),
578 PathBuf::from("/srv/me/.codex/sessions"),
579 );
580 assert_eq!(
582 expand_home_under(Path::new("/etc/passwd"), home),
583 PathBuf::from("/etc/passwd"),
584 );
585 assert_eq!(
587 expand_home_under(Path::new("~user/elsewhere"), home),
588 PathBuf::from("~user/elsewhere"),
589 );
590 }
591
592 #[test]
593 fn resolve_sources_returns_one_or_all_or_errors() {
594 let temp = TempDir::new().unwrap();
595 let body = "\
596[sources.claude-code]
597path = \"/srv/claude\"
598
599[sources.codex-cli]
600path = \"/srv/codex\"
601";
602 let path = temp.path().join("config.toml");
603 std::fs::write(&path, body).expect("write config");
604 let config = Config::load(&path).unwrap();
605
606 let all = config.resolve_sources(None).unwrap();
608 assert_eq!(all.len(), 2);
609 let names: Vec<_> = all.iter().map(|(n, _)| n.as_str()).collect();
610 assert!(names.contains(&"claude-code"));
611 assert!(names.contains(&"codex-cli"));
612
613 let one = config.resolve_sources(Some("codex-cli")).unwrap();
615 assert_eq!(one.len(), 1);
616 assert_eq!(one[0].0, "codex-cli");
617 assert_eq!(
618 one[0].1.get("path").and_then(Value::as_str),
619 Some("/srv/codex"),
620 );
621
622 assert!(config.resolve_sources(Some("nope")).is_err());
624 }
625
626 #[test]
627 fn memory_uri_is_classified_as_remote() {
628 let url = parse_data_dir("memory:///pond-remote-test").expect("memory uri parses");
629 assert!(
630 !is_local(&url),
631 "memory:// is not a local-filesystem URL: {url}",
632 );
633 assert!(
634 local_path(&url).is_none(),
635 "local_path must return None for non-file schemes",
636 );
637 }
638}