1use std::{
9 collections::BTreeMap,
10 path::{Path, PathBuf},
11};
12
13use anyhow::{Context, Result, anyhow, bail};
14use lance_io::object_store::uri_to_url;
15use serde::{Deserialize, Deserializer, Serialize, de};
16use serde_json::Value;
17use url::Url;
18
19fn parse_byte_size(raw: &str) -> Result<usize, String> {
24 let trimmed = raw.trim();
25 if trimmed.is_empty() {
26 return Err("byte-size value is empty".to_owned());
27 }
28 let split = trimmed
29 .find(|c: char| c.is_ascii_alphabetic())
30 .unwrap_or(trimmed.len());
31 let (number, unit) = trimmed.split_at(split);
32 let number: f64 = number
33 .trim()
34 .parse()
35 .map_err(|_| format!("byte-size value {raw:?} is not a number"))?;
36 if !number.is_finite() || number < 0.0 {
37 return Err(format!("byte-size value {raw:?} must be non-negative"));
38 }
39 let multiplier: f64 = match unit.trim().to_ascii_lowercase().as_str() {
40 "" | "b" => 1.0,
41 "k" | "kb" => 1_000.0,
42 "kib" => 1_024.0,
43 "m" | "mb" => 1_000_000.0,
44 "mib" => 1_048_576.0,
45 "g" | "gb" => 1_000_000_000.0,
46 "gib" => 1_073_741_824.0,
47 "tib" => 1_099_511_627_776.0,
48 other => {
49 return Err(format!(
50 "byte-size unit {other:?} not recognized (try MiB / GiB)"
51 ));
52 }
53 };
54 let bytes = number * multiplier;
55 if !bytes.is_finite() || bytes > usize::MAX as f64 {
56 return Err(format!("byte-size value {raw:?} overflows usize"));
57 }
58 Ok(bytes as usize)
59}
60
61fn deserialize_byte_size_opt<'de, D>(deserializer: D) -> Result<Option<usize>, D::Error>
62where
63 D: Deserializer<'de>,
64{
65 #[derive(Deserialize)]
66 #[serde(untagged)]
67 enum Repr {
68 Bytes(u64),
69 Text(String),
70 }
71 let repr: Option<Repr> = Option::deserialize(deserializer)?;
72 match repr {
73 None => Ok(None),
74 Some(Repr::Bytes(value)) => usize::try_from(value).map(Some).map_err(de::Error::custom),
75 Some(Repr::Text(value)) => parse_byte_size(&value).map(Some).map_err(de::Error::custom),
76 }
77}
78
79pub fn parse_data_dir(input: &str) -> Result<Url> {
90 uri_to_url(input).with_context(|| format!("invalid --data-dir {input:?}"))
91}
92
93pub fn is_local(url: &Url) -> bool {
97 matches!(url.scheme(), "file" | "file+uring")
98}
99
100pub fn local_path(url: &Url) -> Option<PathBuf> {
102 if is_local(url) {
103 url.to_file_path().ok()
104 } else {
105 None
106 }
107}
108
109pub fn child_uri(base: &Url, suffix: &str) -> String {
115 if let Some(path) = local_path(base) {
119 return path.join(suffix).display().to_string();
120 }
121 format!("{}/{suffix}", base.as_str().trim_end_matches('/'))
122}
123
124pub fn display(url: &Url) -> String {
127 if let Some(path) = local_path(url) {
128 path.display().to_string()
129 } else {
130 url.to_string()
131 }
132}
133
134pub fn url_for_path(path: impl AsRef<Path>) -> Result<Url> {
139 let path = path.as_ref();
140 let absolute = if path.is_absolute() {
141 path.to_path_buf()
142 } else {
143 std::path::absolute(path)
144 .with_context(|| format!("failed to absolutize {}", path.display()))?
145 };
146 Url::from_file_path(&absolute).map_err(|()| {
147 anyhow!(
148 "failed to convert path {} into a file:// URL",
149 absolute.display()
150 )
151 })
152}
153
154pub const DEFAULT_CONFIG_TOML: &str = "\
158# pond configuration.
159#
160# pond ships built-in defaults, so every setting here is optional - delete this
161# file and pond still works. Uncomment and edit to override.
162
163# Where pond looks for source data to import. One entry per adapter type
164# (`claude-code`, `codex-cli`, ...). `pond sync` with no arguments syncs every
165# entry; `pond sync <adapter>` syncs just one. With an empty `[sources]`,
166# `pond sync` runs an interactive discovery against the known default paths
167# and writes the picks back here.
168#
169# Future wrap: pond is single-namespace in v1 (spec.md#wire-namespace-resolution); `[sources]` is
170# flat here. When multi-namespace pond lands, source registration becomes
171# per-tenant under `[namespaces.<ns>.sources.<adapter>]`. Pre-v1 the schema
172# is breakable; the rename is operationally free until a real second tenant
173# exists.
174#
175# [sources.claude-code]
176# enabled = true
177# path = \"~/.claude/projects\"
178#
179# [sources.codex-cli]
180# enabled = true
181# path = \"~/.codex/sessions\"
182#
183# Set `enabled = false` to keep the section but skip it on `pond sync`;
184# re-enable via `pond sync <adapter>`.
185
186# Embeddings. Search runs hybrid (vector + FTS) whenever the store has any
187# vectors, and FTS-only otherwise - the model loads lazily on the first hybrid
188# query, so there's no cost on FTS-only corpora. `model` selects the
189# HuggingFace XLM-RoBERTa model; `dim` declares its output width and is baked
190# into the messages.vector schema on table creation - it must equal the
191# model's hidden_size and be a multiple of 8 (IVF_PQ subspace stride).
192#
193# Common pairings:
194# model = \"intfloat/multilingual-e5-small\" dim = 384 (default)
195# model = \"intfloat/multilingual-e5-base\" dim = 768
196# model = \"intfloat/multilingual-e5-large\" dim = 1024
197#
198# A different-dim model needs a fresh data dir; pond enforces this at the
199# schema boundary.
200#
201# [embeddings]
202# model = \"intfloat/multilingual-e5-small\"
203# dim = 384
204
205# Search tuning. Leave unset for Lance defaults; set when tuning IVF_PQ recall
206# against a corpus.
207#
208# `index_lag_threshold` is the minimum unindexed-fragment count before a
209# per-intent append/rebuild runs in `pond index optimize`; the brute-force
210# fallback keeps queries correct while fragments accumulate. Defaults to 4.
211#
212# [search]
213# nprobes = 16
214# refine_factor = 2
215# index_lag_threshold = 4
216
217# Long-running process caps. Both accept either a plain byte count or a
218# humansize-style suffix (\"128 MiB\", \"1 GiB\"). Both are optional - leave
219# unset to let pond pick the backend-aware default:
220# local FS : index_cache = 256 MiB, metadata_cache = 128 MiB
221# remote : index_cache = 2 GiB, metadata_cache = 512 MiB
222# Lance's library defaults (6 GiB / 1 GiB) are too generous for a per-session
223# `pond mcp` process; tightening them is what keeps RSS under the 500 MiB target
224# without measurable latency regressions on typical agent-history corpora.
225#
226# [runtime]
227# index_cache_bytes = \"256 MiB\"
228# metadata_cache_bytes = \"128 MiB\"
229
230# Object-store credentials and tuning, passed verbatim to Lance's
231# `DatasetBuilder::with_storage_options`. Required only when `--data-dir` is
232# an `s3://` / `gs://` / `az://` URI that needs auth or a non-default region.
233# Keys follow the `object_store` crate's standard names. Environment
234# variables of the same name are read by `object_store` automatically;
235# values in this block override them. pond does not parse these.
236#
237# Future wrap: pond is single-namespace in v1 (spec.md#wire-namespace-resolution); `[storage]` is
238# flat here on the assumption of one bucket per pond. When multi-namespace
239# pond lands and tenants need separate buckets/regions, this becomes
240# `[namespaces.<ns>.storage]`. Pre-v1 the schema is breakable; the rename is
241# operationally free until a real second tenant exists.
242#
243# [storage]
244# AWS_ACCESS_KEY_ID = \"...\"
245# AWS_SECRET_ACCESS_KEY = \"...\"
246# AWS_REGION = \"us-east-1\"
247# AWS_ENDPOINT = \"https://minio.example.com\" # for self-hosted MinIO
248# allow_http = \"true\" # only for non-TLS endpoints
249";
250
251#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
253#[serde(deny_unknown_fields)]
254pub struct Config {
255 #[serde(default)]
256 pub embeddings: EmbeddingsConfig,
257 #[serde(default)]
258 pub search: SearchConfig,
259 #[serde(default)]
260 pub runtime: RuntimeConfig,
261 #[serde(default)]
267 pub sources: BTreeMap<String, Value>,
268 #[serde(default)]
278 pub storage: BTreeMap<String, String>,
279}
280
281#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
286#[serde(deny_unknown_fields, default)]
287pub struct RuntimeConfig {
288 #[serde(default, deserialize_with = "deserialize_byte_size_opt")]
289 pub index_cache_bytes: Option<usize>,
290 #[serde(default, deserialize_with = "deserialize_byte_size_opt")]
291 pub metadata_cache_bytes: Option<usize>,
292}
293
294#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
296#[serde(deny_unknown_fields)]
297pub struct SearchConfig {
298 #[serde(default)]
299 pub nprobes: Option<usize>,
300 #[serde(default)]
301 pub refine_factor: Option<u32>,
302 #[serde(default)]
308 pub index_lag_threshold: Option<usize>,
309}
310
311#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
320#[serde(deny_unknown_fields, default)]
321pub struct EmbeddingsConfig {
322 pub model: String,
325 pub dim: usize,
329}
330
331impl Default for EmbeddingsConfig {
332 fn default() -> Self {
333 Self {
334 model: crate::embed::DEFAULT_MODEL_ID.to_owned(),
335 dim: crate::sessions::DEFAULT_EMBEDDING_DIM,
336 }
337 }
338}
339
340pub fn resolve_data_dir(
346 explicit: Option<Url>,
347 xdg_data_home: Option<PathBuf>,
348 home: Option<PathBuf>,
349) -> Result<Url> {
350 if let Some(location) = explicit {
351 return Ok(location);
352 }
353 if let Some(xdg) = xdg_data_home.filter(|path| path.is_absolute()) {
354 return url_for_path(xdg.join("pond"));
355 }
356 if let Some(home) = home {
357 return url_for_path(home.join(".local").join("share").join("pond"));
358 }
359 url_for_path(PathBuf::from(".pond"))
361}
362
363pub fn default_config_path(xdg_config_home: Option<PathBuf>, home: Option<PathBuf>) -> PathBuf {
368 if let Some(xdg) = xdg_config_home.filter(|path| path.is_absolute()) {
369 return xdg.join("pond").join("config.toml");
370 }
371 if let Some(home) = home {
372 return home.join(".config").join("pond").join("config.toml");
373 }
374 PathBuf::from(".pond.toml")
375}
376
377impl Config {
378 pub fn load(path: impl AsRef<Path>) -> Result<Self> {
384 let path = path.as_ref();
385 let config = if path.exists() {
386 let text = std::fs::read_to_string(path)
387 .with_context(|| format!("failed to read config {}", path.display()))?;
388 toml::from_str::<Self>(&text)
389 .with_context(|| format!("failed to parse config {}", path.display()))?
390 } else {
391 Self::default()
392 };
393 config.embeddings.validate()?;
394 config.embeddings.install_runtime();
395 if let Some(threshold) = config.search.index_lag_threshold {
396 crate::substrate::init_index_lag_threshold(threshold);
397 }
398 Ok(config)
402 }
403
404 pub fn resolve_sources(&self, adapter: Option<&str>) -> Result<Vec<(String, Value)>> {
413 match adapter {
414 None => Ok(self
415 .sources
416 .iter()
417 .filter_map(|(name, blob)| take_enabled(name, blob))
418 .collect()),
419 Some(name) => {
420 let blob = self
421 .sources
422 .get(name)
423 .ok_or_else(|| anyhow!("no [sources.{name}] entry in config"))?;
424 take_enabled(name, blob).map(|entry| vec![entry]).ok_or_else(|| {
425 anyhow!(
426 "source [{name}] is disabled (enabled = false); run `pond sync {name}` to re-enable"
427 )
428 })
429 }
430 }
431 }
432
433 pub fn disabled_source_names(&self) -> Vec<&str> {
438 self.sources
439 .iter()
440 .filter_map(|(name, blob)| {
441 let enabled = blob
442 .get("enabled")
443 .and_then(Value::as_bool)
444 .unwrap_or(false);
445 if enabled { None } else { Some(name.as_str()) }
446 })
447 .collect()
448 }
449}
450
451fn take_enabled(name: &str, blob: &Value) -> Option<(String, Value)> {
455 let enabled = blob
456 .get("enabled")
457 .and_then(Value::as_bool)
458 .unwrap_or(false);
459 if !enabled {
460 return None;
461 }
462 let mut clean = blob.clone();
463 if let Some(obj) = clean.as_object_mut() {
464 obj.remove("enabled");
465 }
466 Some((name.to_owned(), clean))
467}
468
469pub fn expand_home_under(path: &Path, home: &Path) -> PathBuf {
475 let Some(text) = path.to_str() else {
476 return path.to_path_buf();
477 };
478 if text == "~" {
479 return home.to_path_buf();
480 }
481 if let Some(rest) = text.strip_prefix("~/") {
482 return home.join(rest);
483 }
484 path.to_path_buf()
485}
486
487impl EmbeddingsConfig {
488 pub fn validate(&self) -> Result<()> {
493 if self.model.trim().is_empty() {
494 bail!("embeddings.model must be a non-empty HuggingFace model id");
495 }
496 if self.dim == 0 || !self.dim.is_multiple_of(8) {
497 bail!(
498 "embeddings.dim = {} must be a positive multiple of 8 (IVF_PQ subspace stride)",
499 self.dim,
500 );
501 }
502 Ok(())
503 }
504
505 pub fn install_runtime(&self) {
509 crate::embed::init_model_id(self.model.clone());
510 crate::sessions::init_embedding_dim(self.dim);
511 }
512}
513
514#[cfg(test)]
515mod tests {
516 #![allow(clippy::expect_used, clippy::unwrap_used)]
517
518 use super::*;
519 use serde_json::Value;
520 use tempfile::TempDir;
521
522 #[test]
523 fn validate_catches_empty_model_and_bad_dim() {
524 assert!(EmbeddingsConfig::default().validate().is_ok());
525 let bad_model = EmbeddingsConfig {
528 model: " ".to_owned(),
529 dim: 768,
530 };
531 assert!(bad_model.validate().is_err());
532 let bad_dim = EmbeddingsConfig {
534 model: "intfloat/multilingual-e5-base".to_owned(),
535 dim: 100,
536 };
537 assert!(bad_dim.validate().is_err());
538 let zero_dim = EmbeddingsConfig {
540 model: "intfloat/multilingual-e5-base".to_owned(),
541 dim: 0,
542 };
543 assert!(zero_dim.validate().is_err());
544 }
545
546 #[test]
547 fn config_load_missing_file_falls_back_to_builtin() {
548 let config = Config::load("/nonexistent/pond-config-xyz.toml").unwrap();
549 assert_eq!(config.embeddings, EmbeddingsConfig::default());
550 }
551
552 #[test]
553 fn default_config_toml_loads_to_the_builtin_defaults() {
554 let dir = TempDir::new().unwrap();
555 let path = dir.path().join("config.toml");
556 std::fs::write(&path, DEFAULT_CONFIG_TOML).unwrap();
557 let config = Config::load(&path).unwrap();
560 assert_eq!(config.embeddings, EmbeddingsConfig::default());
561 assert_eq!(config.embeddings.model, crate::embed::DEFAULT_MODEL_ID);
562 assert_eq!(
563 config.embeddings.dim,
564 crate::sessions::DEFAULT_EMBEDDING_DIM
565 );
566 }
567
568 #[test]
569 fn resolve_data_dir_follows_explicit_then_xdg_then_home() {
570 let explicit = parse_data_dir("/explicit").unwrap();
574 let resolved = resolve_data_dir(
575 Some(explicit.clone()),
576 Some(PathBuf::from("/xdg")),
577 Some(PathBuf::from("/home")),
578 )
579 .unwrap();
580 assert_eq!(resolved, explicit);
581
582 let resolved = resolve_data_dir(
584 None,
585 Some(PathBuf::from("/xdg")),
586 Some(PathBuf::from("/home")),
587 )
588 .unwrap();
589 assert!(is_local(&resolved));
590 assert_eq!(local_path(&resolved).unwrap(), PathBuf::from("/xdg/pond"));
591
592 let resolved = resolve_data_dir(
594 None,
595 Some(PathBuf::from("relative")),
596 Some(PathBuf::from("/home")),
597 )
598 .unwrap();
599 assert_eq!(
600 local_path(&resolved).unwrap(),
601 PathBuf::from("/home/.local/share/pond"),
602 );
603
604 let resolved = resolve_data_dir(None, None, None).unwrap();
608 assert!(is_local(&resolved));
609 assert!(
610 local_path(&resolved).unwrap().ends_with(".pond"),
611 "fallback path should end with .pond: {resolved}",
612 );
613 }
614
615 #[test]
616 fn expand_home_under_handles_tilde_forms() {
617 let home = Path::new("/srv/me");
618 assert_eq!(
619 expand_home_under(Path::new("~"), home),
620 PathBuf::from("/srv/me")
621 );
622 assert_eq!(
623 expand_home_under(Path::new("~/.codex/sessions"), home),
624 PathBuf::from("/srv/me/.codex/sessions"),
625 );
626 assert_eq!(
628 expand_home_under(Path::new("/etc/passwd"), home),
629 PathBuf::from("/etc/passwd"),
630 );
631 assert_eq!(
633 expand_home_under(Path::new("~user/elsewhere"), home),
634 PathBuf::from("~user/elsewhere"),
635 );
636 }
637
638 #[test]
639 fn resolve_sources_returns_one_or_all_or_errors() {
640 let temp = TempDir::new().unwrap();
641 let body = "\
642[sources.claude-code]
643enabled = true
644path = \"/srv/claude\"
645
646[sources.codex-cli]
647enabled = true
648path = \"/srv/codex\"
649
650[sources.opencode]
651enabled = false
652";
653 let path = temp.path().join("config.toml");
654 std::fs::write(&path, body).expect("write config");
655 let config = Config::load(&path).unwrap();
656
657 let all = config.resolve_sources(None).unwrap();
659 assert_eq!(all.len(), 2);
660 let names: Vec<_> = all.iter().map(|(n, _)| n.as_str()).collect();
661 assert!(names.contains(&"claude-code"));
662 assert!(names.contains(&"codex-cli"));
663 for (_, blob) in &all {
665 assert!(blob.get("enabled").is_none(), "enabled should be stripped");
666 }
667
668 let one = config.resolve_sources(Some("codex-cli")).unwrap();
670 assert_eq!(one.len(), 1);
671 assert_eq!(one[0].0, "codex-cli");
672 assert_eq!(
673 one[0].1.get("path").and_then(Value::as_str),
674 Some("/srv/codex"),
675 );
676
677 let disabled = config.resolve_sources(Some("opencode"));
679 let err = disabled
680 .expect_err("disabled adapter must error")
681 .to_string();
682 assert!(err.contains("enabled = false"), "got: {err}");
683 assert!(err.contains("pond sync opencode"), "got: {err}");
684
685 assert!(config.resolve_sources(Some("nope")).is_err());
687
688 assert_eq!(config.disabled_source_names(), vec!["opencode"]);
690 }
691
692 #[test]
693 fn memory_uri_is_classified_as_remote() {
694 let url = parse_data_dir("memory:///pond-remote-test").expect("memory uri parses");
695 assert!(
696 !is_local(&url),
697 "memory:// is not a local-filesystem URL: {url}",
698 );
699 assert!(
700 local_path(&url).is_none(),
701 "local_path must return None for non-file schemes",
702 );
703 }
704}