Skip to main content

shaul_core/
config.rs

1use std::collections::BTreeMap;
2use std::path::{Path, PathBuf};
3use std::str::FromStr;
4use std::time::Duration;
5
6use directories::{BaseDirs, ProjectDirs};
7use serde::{Deserialize, Serialize};
8
9use crate::error::{Error, Result};
10use crate::header::{Header, RequestOptions};
11use crate::task::{TaskOptions, VerifyMode};
12
13/// Effective configuration. Precedence is CLI flags > config file >
14/// environment variables > built-in defaults; see `Config::load`.
15#[derive(Clone, Debug, Serialize, Deserialize)]
16#[serde(default)]
17#[derive(Default)]
18#[non_exhaustive]
19pub struct Config {
20    /// Download behavior and defaults.
21    pub download: DownloadConfig,
22    /// HTTP timeout, retry and user-agent settings.
23    pub network: NetworkConfig,
24    /// Hub provider cache and endpoint settings.
25    pub hub: HubConfig,
26    /// BitTorrent backend settings.
27    pub bt: BtConfig,
28    /// Directory for task sidecars; defaults to the platform data directory.
29    pub state_dir: Option<PathBuf>,
30    /// Default headers applied to every request; values may use `${env:NAME}`
31    /// placeholders.
32    pub headers: BTreeMap<String, String>,
33}
34
35/// Defaults for how files are downloaded, resumed, verified and extracted.
36#[derive(Clone, Debug, Serialize, Deserialize)]
37#[serde(default)]
38#[non_exhaustive]
39pub struct DownloadConfig {
40    /// Default destination directory.
41    pub dir: PathBuf,
42    /// Default number of parallel connections per task.
43    pub connections: usize,
44    /// Maximum number of idle connections kept per host in the HTTP pool.
45    pub per_host: usize,
46    /// Maximum number of active HTTP file-transfer requests per host across
47    /// all tasks.
48    pub per_host_active: usize,
49    /// Maximum number of active HTTP file-transfer requests across all tasks.
50    pub global: usize,
51    /// Global bandwidth limit in bytes per second. `None` or `0` means
52    /// unlimited. Direct HTTP and Hub file transfers share this limit.
53    pub rate_limit: Option<u64>,
54    /// Additional bandwidth cap per destination host in bytes per second.
55    /// `None` or `0` means unlimited.
56    pub per_host_rate_limit: Option<u64>,
57    /// Number of files downloaded concurrently within one multi-file task.
58    pub file_jobs: usize,
59    /// Reuse verified downloads through the content-addressed cache.
60    pub content_cache: bool,
61    /// Override for the content-addressed cache root.
62    pub content_cache_dir: Option<PathBuf>,
63    /// Refuse a task before transfer when the destination filesystem has less
64    /// free space than the known remaining size.
65    pub disk_space_check: bool,
66    /// Size in bytes of each ranged HTTP segment.
67    pub segment_size: u64,
68    /// Resume interrupted downloads from checkpoints by default.
69    pub resume: bool,
70    /// Default checksum verification policy.
71    pub verify: VerifyMode,
72    /// Extract archives after download by default.
73    pub extract: bool,
74    /// Default extraction directory; `None` extracts under the destination.
75    pub extract_dir: Option<PathBuf>,
76    /// Delete archives after successful extraction by default.
77    pub delete_archive: bool,
78}
79
80/// HTTP networking defaults: timeouts, retries, user agent and auth policy.
81#[derive(Clone, Debug, Serialize, Deserialize)]
82#[serde(default)]
83#[non_exhaustive]
84pub struct NetworkConfig {
85    /// Total request timeout in seconds.
86    pub timeout_secs: u64,
87    /// Connection timeout in seconds.
88    pub connect_timeout_secs: u64,
89    /// Number of retry attempts for failed transfers.
90    pub retries: u32,
91    /// `User-Agent` header sent with every request.
92    pub user_agent: String,
93    /// Allow authentication headers to follow cross-origin redirects.
94    pub allow_cross_origin_auth: bool,
95    /// Explicit HTTP/HTTPS/SOCKS proxy URL. When unset, the HTTP client keeps
96    /// reqwest's ambient behavior.
97    pub proxy: Option<String>,
98    /// Comma-separated hosts that bypass the configured proxy.
99    pub no_proxy: Option<String>,
100}
101
102/// Hugging Face and ModelScope cache and endpoint settings.
103#[derive(Clone, Debug, Serialize, Deserialize)]
104#[serde(default)]
105#[derive(Default)]
106#[non_exhaustive]
107pub struct HubConfig {
108    /// Populate a provider cache while downloading. HF uses the Python
109    /// compatible `models--org--repo` layout.
110    pub use_cache: bool,
111    /// Override for the Hugging Face cache root.
112    pub hf_cache: Option<PathBuf>,
113    /// Override for the ModelScope cache root.
114    pub modelscope_cache: Option<PathBuf>,
115    /// Override for the Hugging Face endpoint; defaults to the public site.
116    pub hf_endpoint: Option<String>,
117    /// Override for the ModelScope endpoint; defaults to the public site.
118    pub modelscope_endpoint: Option<String>,
119}
120
121/// BitTorrent backend settings.
122#[derive(Clone, Debug, Serialize, Deserialize)]
123#[serde(default)]
124#[non_exhaustive]
125pub struct BtConfig {
126    /// Enable DHT peer discovery.
127    pub dht: bool,
128    /// Use the trackers announced by the torrent.
129    pub trackers: bool,
130    /// Listen for incoming peer connections.
131    pub listen: bool,
132    /// Persist session state and fastresume data under the state directory.
133    pub persistence: bool,
134    /// Optional limit on connected peers.
135    pub peer_limit: Option<usize>,
136    /// `ip:port` endpoints used directly as initial peers.
137    pub initial_peers: Vec<String>,
138}
139
140impl Default for BtConfig {
141    fn default() -> Self {
142        Self {
143            dht: true,
144            trackers: true,
145            listen: true,
146            persistence: true,
147            peer_limit: None,
148            initial_peers: Vec::new(),
149        }
150    }
151}
152
153impl Default for DownloadConfig {
154    fn default() -> Self {
155        Self {
156            dir: PathBuf::from("."),
157            connections: 8,
158            per_host: 16,
159            per_host_active: 16,
160            global: 64,
161            rate_limit: None,
162            per_host_rate_limit: None,
163            file_jobs: 1,
164            content_cache: false,
165            content_cache_dir: None,
166            disk_space_check: true,
167            segment_size: 8 * 1024 * 1024,
168            resume: true,
169            verify: VerifyMode::Auto,
170            extract: false,
171            extract_dir: None,
172            delete_archive: false,
173        }
174    }
175}
176
177impl Default for NetworkConfig {
178    fn default() -> Self {
179        Self {
180            timeout_secs: 60,
181            connect_timeout_secs: 15,
182            retries: 5,
183            user_agent: format!("shaul/{}", env!("CARGO_PKG_VERSION")),
184            allow_cross_origin_auth: false,
185            proxy: None,
186            no_proxy: None,
187        }
188    }
189}
190
191/// Options after task overrides and config defaults have been resolved.
192#[derive(Clone, Debug)]
193#[non_exhaustive]
194pub struct ResolvedOptions {
195    /// Root directory for the task's output files.
196    pub destination: PathBuf,
197    /// Explicit output path, if one was requested.
198    pub output: Option<PathBuf>,
199    /// Effective connection count for this task.
200    pub connections: usize,
201    /// Effective HTTP idle-connection limit per host.
202    pub per_host: usize,
203    /// Maximum active HTTP file-transfer requests per host carried from
204    /// configuration.
205    pub per_host_active: usize,
206    /// Global active HTTP file-transfer request budget carried from
207    /// configuration.
208    pub global: usize,
209    /// Global bandwidth limit in bytes per second, if configured.
210    pub rate_limit: Option<u64>,
211    /// Per-host bandwidth limit in bytes per second, if configured.
212    pub per_host_rate_limit: Option<u64>,
213    /// Additional per-task bandwidth limit in bytes per second, if configured.
214    pub task_rate_limit: Option<u64>,
215    /// Number of files downloaded concurrently within this task.
216    pub file_jobs: usize,
217    /// Whether verified files are reused through the content-addressed cache.
218    pub content_cache: bool,
219    /// Root of the content-addressed cache.
220    pub content_cache_dir: PathBuf,
221    /// Whether to preflight the destination filesystem's free space.
222    pub disk_space_check: bool,
223    /// Alternate HTTP(S) URLs for the same file.
224    pub mirrors: Vec<url::Url>,
225    /// Byte size of each ranged HTTP segment.
226    pub segment_size: u64,
227    /// Whether resume is enabled for this task.
228    pub resume: bool,
229    /// Effective checksum verification policy.
230    pub verify: VerifyMode,
231    /// Expected checksum supplied by the task, if any.
232    pub checksum: Option<crate::checksum::Checksum>,
233    /// Whether archives should be extracted after download.
234    pub extract: bool,
235    /// Directory to extract archives into; `None` means the destination.
236    pub extract_dir: Option<PathBuf>,
237    /// Whether to delete the archive after extraction.
238    pub delete_archive: bool,
239    /// Whether to populate the provider cache for Hub tasks.
240    pub cache: bool,
241    /// Include glob filters passed from the task.
242    pub include: Vec<String>,
243    /// Exclude glob filters passed from the task.
244    pub exclude: Vec<String>,
245    /// Merged request options: config headers plus task overrides.
246    pub request: RequestOptions,
247    /// Whether existing destination files are replaced.
248    pub overwrite: bool,
249    /// Whether conflicting destinations are renamed.
250    pub rename_on_conflict: bool,
251    /// Effective retry count for failed transfers.
252    pub retries: u32,
253    /// Effective total request timeout.
254    pub timeout: Duration,
255    /// Effective connection timeout.
256    pub connect_timeout: Duration,
257    /// Effective `User-Agent` header.
258    pub user_agent: String,
259}
260
261impl Config {
262    /// Platform default config file path (`config.toml` in the user config
263    /// directory).
264    pub fn default_config_path() -> Result<PathBuf> {
265        let dirs = project_dirs()?;
266        Ok(dirs.config_dir().join("config.toml"))
267    }
268
269    /// Platform default directory for task sidecars.
270    pub fn default_state_dir() -> Result<PathBuf> {
271        let dirs = project_dirs()?;
272        Ok(dirs.data_dir().join("tasks"))
273    }
274
275    /// Configured state directory, or the platform default.
276    pub fn state_dir(&self) -> Result<PathBuf> {
277        match &self.state_dir {
278            Some(dir) => Ok(dir.clone()),
279            None => Self::default_state_dir(),
280        }
281    }
282
283    /// Resolved Hugging Face cache root: configured override or
284    /// `~/.cache/huggingface/hub`.
285    pub fn hf_cache_dir(&self) -> Result<PathBuf> {
286        if let Some(dir) = &self.hub.hf_cache {
287            return Ok(dir.clone());
288        }
289        let base = BaseDirs::new()
290            .ok_or_else(|| Error::Config("cannot determine home directory".to_string()))?;
291        Ok(base
292            .home_dir()
293            .join(".cache")
294            .join("huggingface")
295            .join("hub"))
296    }
297
298    /// Resolved ModelScope cache root: configured override or the platform
299    /// cache directory.
300    pub fn modelscope_cache_dir(&self) -> Result<PathBuf> {
301        if let Some(dir) = &self.hub.modelscope_cache {
302            return Ok(dir.clone());
303        }
304        Ok(project_dirs()?.cache_dir().join("modelscope"))
305    }
306
307    /// Resolved content-addressed cache root: configured override or the
308    /// platform cache directory under `shaul/content`.
309    pub fn content_cache_dir(&self) -> Result<PathBuf> {
310        if let Some(dir) = &self.download.content_cache_dir
311            && !dir.as_os_str().is_empty()
312        {
313            return Ok(dir.clone());
314        }
315        Ok(project_dirs()?.cache_dir().join("content"))
316    }
317
318    /// Configured Hugging Face endpoint or the public default.
319    pub fn hf_endpoint(&self) -> String {
320        self.hub
321            .hf_endpoint
322            .clone()
323            .unwrap_or_else(|| "https://huggingface.co".to_string())
324    }
325
326    /// Configured ModelScope endpoint or the public default.
327    pub fn modelscope_endpoint(&self) -> String {
328        self.hub
329            .modelscope_endpoint
330            .clone()
331            .unwrap_or_else(|| "https://modelscope.cn".to_string())
332    }
333
334    /// Resolve the config path from an explicit argument, `SHAUL_CONFIG`, or
335    /// the platform default.
336    pub fn resolve_path(explicit: Option<&Path>) -> Result<PathBuf> {
337        if let Some(path) = explicit {
338            return Ok(path.to_path_buf());
339        }
340        if let Ok(path) = std::env::var("SHAUL_CONFIG")
341            && !path.trim().is_empty()
342        {
343            return Ok(PathBuf::from(path));
344        }
345        Self::default_config_path()
346    }
347
348    /// Load defaults, then environment variables, then the config file. CLI
349    /// flags are applied by the caller afterwards.
350    pub fn load(explicit: Option<&Path>) -> Result<(Self, PathBuf)> {
351        let path = Self::resolve_path(explicit)?;
352        let mut config = Self::default();
353        config.apply_env()?;
354        if path.exists() {
355            let text = std::fs::read_to_string(&path)?;
356            let partial: PartialConfig = toml::from_str(&text)?;
357            partial.apply(&mut config)?;
358        }
359        Ok((config, path))
360    }
361
362    /// Load only defaults + file. Used by `config set` so environment values
363    /// are not baked into the file.
364    pub fn load_file_only(path: &Path) -> Result<Self> {
365        let mut config = Self::default();
366        if path.exists() {
367            let text = std::fs::read_to_string(path)?;
368            let partial: PartialConfig = toml::from_str(&text)?;
369            partial.apply(&mut config)?;
370        }
371        Ok(config)
372    }
373
374    /// Serialize the config as TOML, creating parent directories as needed.
375    pub fn save(&self, path: &Path) -> Result<()> {
376        if let Some(parent) = path.parent()
377            && !parent.as_os_str().is_empty()
378        {
379            std::fs::create_dir_all(parent)?;
380        }
381        let text = toml::to_string_pretty(self)?;
382        std::fs::write(path, text)?;
383        Ok(())
384    }
385
386    /// Apply `SHAUL_*` and provider environment variables on top of the
387    /// current values.
388    pub fn apply_env(&mut self) -> Result<()> {
389        if let Some(value) = env("SHAUL_DIR") {
390            self.download.dir = PathBuf::from(value);
391        }
392        if let Some(value) = env_parse::<usize>("SHAUL_CONNECTIONS")? {
393            self.download.connections = value;
394        }
395        if let Some(value) = env_parse::<usize>("SHAUL_PER_HOST")? {
396            self.download.per_host = value;
397        }
398        if let Some(value) = env_parse::<usize>("SHAUL_PER_HOST_ACTIVE")? {
399            self.download.per_host_active = value.max(1);
400        }
401        if let Some(value) = env_parse::<usize>("SHAUL_GLOBAL")? {
402            self.download.global = value;
403        }
404        if let Some(value) = env("SHAUL_RATE_LIMIT") {
405            self.download.rate_limit = Some(parse_bytes(&value)?).filter(|value| *value > 0);
406        }
407        if let Some(value) = env("SHAUL_PER_HOST_RATE_LIMIT") {
408            self.download.per_host_rate_limit =
409                Some(parse_bytes(&value)?).filter(|value| *value > 0);
410        }
411        if let Some(value) = env_parse::<usize>("SHAUL_FILE_JOBS")? {
412            self.download.file_jobs = value.max(1);
413        }
414        if let Some(value) = env_parse::<bool>("SHAUL_CONTENT_CACHE")? {
415            self.download.content_cache = value;
416        }
417        if let Some(value) = env("SHAUL_CONTENT_CACHE_DIR") {
418            self.download.content_cache_dir = Some(PathBuf::from(value));
419        }
420        if let Some(value) = env_parse::<bool>("SHAUL_DISK_SPACE_CHECK")? {
421            self.download.disk_space_check = value;
422        }
423        if let Some(value) = env("SHAUL_SEGMENT_SIZE") {
424            self.download.segment_size = parse_bytes(&value)?;
425        }
426        if let Some(value) = env_parse::<bool>("SHAUL_RESUME")? {
427            self.download.resume = value;
428        }
429        if let Some(value) = env("SHAUL_VERIFY") {
430            self.download.verify = parse_verify_mode(&value)?;
431        }
432        if let Some(value) = env_parse::<bool>("SHAUL_EXTRACT")? {
433            self.download.extract = value;
434        }
435        if let Some(value) = env("SHAUL_EXTRACT_DIR") {
436            self.download.extract_dir = Some(PathBuf::from(value));
437        }
438        if let Some(value) = env_parse::<bool>("SHAUL_DELETE_ARCHIVE")? {
439            self.download.delete_archive = value;
440        }
441        if let Some(value) = env("SHAUL_STATE_DIR") {
442            self.state_dir = Some(PathBuf::from(value));
443        }
444        if let Some(value) = env_parse::<u64>("SHAUL_TIMEOUT_SECS")? {
445            self.network.timeout_secs = value;
446        }
447        if let Some(value) = env_parse::<u64>("SHAUL_CONNECT_TIMEOUT_SECS")? {
448            self.network.connect_timeout_secs = value;
449        }
450        if let Some(value) = env_parse::<u32>("SHAUL_RETRIES")? {
451            self.network.retries = value;
452        }
453        if let Some(value) = env("SHAUL_USER_AGENT") {
454            self.network.user_agent = value;
455        }
456        if let Some(value) = env_parse::<bool>("SHAUL_ALLOW_CROSS_ORIGIN_AUTH")? {
457            self.network.allow_cross_origin_auth = value;
458        }
459        if let Some(value) = env("SHAUL_PROXY").or_else(|| {
460            env("HTTPS_PROXY")
461                .or_else(|| env("https_proxy"))
462                .or_else(|| env("HTTP_PROXY"))
463                .or_else(|| env("http_proxy"))
464                .or_else(|| env("ALL_PROXY"))
465                .or_else(|| env("all_proxy"))
466        }) {
467            self.network.proxy = Some(value);
468        }
469        if let Some(value) = env("SHAUL_NO_PROXY")
470            .or_else(|| env("NO_PROXY"))
471            .or_else(|| env("no_proxy"))
472        {
473            self.network.no_proxy = Some(value);
474        }
475
476        if let Some(value) = env("HF_HOME") {
477            self.hub.hf_cache = Some(PathBuf::from(value).join("hub"));
478        }
479        if let Some(value) = env("HF_HUB_CACHE") {
480            self.hub.hf_cache = Some(PathBuf::from(value));
481        }
482        if let Some(value) = env("SHAUL_HF_CACHE") {
483            self.hub.hf_cache = Some(PathBuf::from(value));
484        }
485        if let Some(value) = env("HF_ENDPOINT") {
486            self.hub.hf_endpoint = Some(value);
487        }
488        if let Some(value) = env("SHAUL_HF_ENDPOINT") {
489            self.hub.hf_endpoint = Some(value);
490        }
491        if let Some(value) = env("MODELSCOPE_CACHE") {
492            self.hub.modelscope_cache = Some(PathBuf::from(value));
493        }
494        if let Some(value) = env("SHAUL_MODELSCOPE_CACHE") {
495            self.hub.modelscope_cache = Some(PathBuf::from(value));
496        }
497        if let Some(value) = env("MODELSCOPE_ENDPOINT").or_else(|| env("MODELSCOPE_DOMAIN")) {
498            self.hub.modelscope_endpoint = Some(value);
499        }
500        if let Some(value) = env("SHAUL_MODELSCOPE_ENDPOINT") {
501            self.hub.modelscope_endpoint = Some(value);
502        }
503        if let Some(value) = env_parse::<bool>("SHAUL_HUB_USE_CACHE")? {
504            self.hub.use_cache = value;
505        }
506
507        if let Some(value) = env_parse::<bool>("SHAUL_BT_DHT")? {
508            self.bt.dht = value;
509        }
510        if let Some(value) = env_parse::<bool>("SHAUL_BT_TRACKERS")? {
511            self.bt.trackers = value;
512        }
513        if let Some(value) = env_parse::<bool>("SHAUL_BT_LISTEN")? {
514            self.bt.listen = value;
515        }
516        if let Some(value) = env_parse::<bool>("SHAUL_BT_PERSISTENCE")? {
517            self.bt.persistence = value;
518        }
519        if let Some(value) = env_parse::<usize>("SHAUL_BT_PEER_LIMIT")? {
520            self.bt.peer_limit = Some(value);
521        }
522        if let Some(value) = env("SHAUL_BT_INITIAL_PEERS") {
523            self.bt.initial_peers = value
524                .split(',')
525                .map(str::trim)
526                .filter(|value| !value.is_empty())
527                .map(str::to_string)
528                .collect();
529        }
530        Ok(())
531    }
532
533    /// Parse the configured default headers into validated [`Header`] values.
534    pub fn header_defaults(&self) -> Result<Vec<Header>> {
535        self.headers
536            .iter()
537            .map(|(name, value)| Header::new(name, value))
538            .collect()
539    }
540
541    /// Combine config defaults with per-task options into fully resolved
542    /// settings.
543    pub fn resolve(&self, destination: &Path, options: &TaskOptions) -> Result<ResolvedOptions> {
544        let mut request = options.request.merged(&self.header_defaults()?);
545        request.allow_cross_origin_auth |= self.network.allow_cross_origin_auth;
546        Ok(ResolvedOptions {
547            destination: destination.to_path_buf(),
548            output: options.output.clone(),
549            connections: options
550                .connections
551                .unwrap_or(self.download.connections)
552                .max(1),
553            per_host: self.download.per_host.max(1),
554            per_host_active: self.download.per_host_active.max(1),
555            global: self.download.global.max(1),
556            rate_limit: self.download.rate_limit.filter(|value| *value > 0),
557            per_host_rate_limit: self.download.per_host_rate_limit.filter(|value| *value > 0),
558            task_rate_limit: options.rate_limit.filter(|value| *value > 0),
559            file_jobs: options.file_jobs.unwrap_or(self.download.file_jobs).max(1),
560            content_cache: options.content_cache.unwrap_or(self.download.content_cache),
561            content_cache_dir: self.content_cache_dir()?,
562            disk_space_check: self.download.disk_space_check,
563            mirrors: options.mirrors.clone(),
564            segment_size: self.download.segment_size.max(64 * 1024),
565            resume: options.resume.unwrap_or(self.download.resume),
566            verify: options.verify.unwrap_or(self.download.verify),
567            checksum: options.checksum.clone(),
568            extract: options.extract.unwrap_or(self.download.extract),
569            extract_dir: options
570                .extract_dir
571                .clone()
572                .or_else(|| self.download.extract_dir.clone()),
573            delete_archive: options
574                .delete_archive
575                .unwrap_or(self.download.delete_archive),
576            cache: options.cache.unwrap_or(self.hub.use_cache),
577            include: options.include.clone(),
578            exclude: options.exclude.clone(),
579            request,
580            overwrite: options.overwrite,
581            rename_on_conflict: options.rename_on_conflict,
582            retries: self.network.retries,
583            timeout: Duration::from_secs(self.network.timeout_secs.max(1)),
584            connect_timeout: Duration::from_secs(self.network.connect_timeout_secs.max(1)),
585            user_agent: self.network.user_agent.clone(),
586        })
587    }
588
589    /// Read a config value by its dotted key, as used by `config get`.
590    pub fn get_value(&self, key: &str) -> Result<String> {
591        let value = match key {
592            "download.dir" => self.download.dir.display().to_string(),
593            "download.connections" => self.download.connections.to_string(),
594            "download.per_host" => self.download.per_host.to_string(),
595            "download.per_host_active" => self.download.per_host_active.to_string(),
596            "download.global" => self.download.global.to_string(),
597            "download.rate_limit" => self
598                .download
599                .rate_limit
600                .map(|value| value.to_string())
601                .unwrap_or_default(),
602            "download.per_host_rate_limit" => self
603                .download
604                .per_host_rate_limit
605                .map(|value| value.to_string())
606                .unwrap_or_default(),
607            "download.file_jobs" => self.download.file_jobs.to_string(),
608            "download.content_cache" => self.download.content_cache.to_string(),
609            "download.content_cache_dir" => self.content_cache_dir()?.display().to_string(),
610            "download.disk_space_check" => self.download.disk_space_check.to_string(),
611            "download.segment_size" => self.download.segment_size.to_string(),
612            "download.resume" => self.download.resume.to_string(),
613            "download.verify" => verify_mode_str(self.download.verify).to_string(),
614            "download.extract" => self.download.extract.to_string(),
615            "download.extract_dir" => self
616                .download
617                .extract_dir
618                .as_ref()
619                .map(|path| path.display().to_string())
620                .unwrap_or_default(),
621            "download.delete_archive" => self.download.delete_archive.to_string(),
622            "network.timeout_secs" => self.network.timeout_secs.to_string(),
623            "network.connect_timeout_secs" => self.network.connect_timeout_secs.to_string(),
624            "network.retries" => self.network.retries.to_string(),
625            "network.user_agent" => self.network.user_agent.clone(),
626            "network.allow_cross_origin_auth" => self.network.allow_cross_origin_auth.to_string(),
627            "network.proxy" => self.network.proxy.clone().unwrap_or_default(),
628            "network.no_proxy" => self.network.no_proxy.clone().unwrap_or_default(),
629            "hub.use_cache" => self.hub.use_cache.to_string(),
630            "hub.hf_cache" => self.hf_cache_dir()?.display().to_string(),
631            "hub.modelscope_cache" => self.modelscope_cache_dir()?.display().to_string(),
632            "hub.hf_endpoint" => self.hf_endpoint(),
633            "hub.modelscope_endpoint" => self.modelscope_endpoint(),
634            "bt.dht" => self.bt.dht.to_string(),
635            "bt.trackers" => self.bt.trackers.to_string(),
636            "bt.listen" => self.bt.listen.to_string(),
637            "bt.persistence" => self.bt.persistence.to_string(),
638            "bt.peer_limit" => self
639                .bt
640                .peer_limit
641                .map(|value| value.to_string())
642                .unwrap_or_default(),
643            "bt.initial_peers" => self.bt.initial_peers.join(","),
644            "state_dir" => self
645                .state_dir
646                .as_ref()
647                .map(|path| path.display().to_string())
648                .unwrap_or_default(),
649            _ if key.starts_with("headers.") => {
650                let name = &key["headers.".len()..];
651                self.headers.get(name).cloned().unwrap_or_default()
652            }
653            _ => return Err(Error::Config(format!("unknown config key: {key}"))),
654        };
655        Ok(value)
656    }
657
658    /// Set a config value by its dotted key, as used by `config set`.
659    pub fn set_value(&mut self, key: &str, value: &str) -> Result<()> {
660        match key {
661            "download.dir" => self.download.dir = PathBuf::from(value),
662            "download.connections" => self.download.connections = parse_value(value)?,
663            "download.per_host" => self.download.per_host = parse_value(value)?,
664            "download.per_host_active" => {
665                self.download.per_host_active = parse_value::<usize>(value)?.max(1)
666            }
667            "download.global" => self.download.global = parse_value(value)?,
668            "download.rate_limit" => {
669                self.download.rate_limit = if value.trim().is_empty() {
670                    None
671                } else {
672                    Some(parse_bytes(value)?).filter(|value| *value > 0)
673                }
674            }
675            "download.per_host_rate_limit" => {
676                self.download.per_host_rate_limit = if value.trim().is_empty() {
677                    None
678                } else {
679                    Some(parse_bytes(value)?).filter(|value| *value > 0)
680                }
681            }
682            "download.file_jobs" => self.download.file_jobs = parse_value::<usize>(value)?.max(1),
683            "download.content_cache" => self.download.content_cache = parse_value(value)?,
684            "download.content_cache_dir" => {
685                self.download.content_cache_dir = if value.trim().is_empty() {
686                    None
687                } else {
688                    Some(PathBuf::from(value))
689                }
690            }
691            "download.disk_space_check" => self.download.disk_space_check = parse_value(value)?,
692            "download.segment_size" => self.download.segment_size = parse_bytes(value)?,
693            "download.resume" => self.download.resume = parse_value(value)?,
694            "download.verify" => self.download.verify = parse_verify_mode(value)?,
695            "download.extract" => self.download.extract = parse_value(value)?,
696            "download.extract_dir" => {
697                self.download.extract_dir = if value.trim().is_empty() {
698                    None
699                } else {
700                    Some(PathBuf::from(value))
701                }
702            }
703            "download.delete_archive" => self.download.delete_archive = parse_value(value)?,
704            "network.timeout_secs" => self.network.timeout_secs = parse_value(value)?,
705            "network.connect_timeout_secs" => {
706                self.network.connect_timeout_secs = parse_value(value)?
707            }
708            "network.retries" => self.network.retries = parse_value(value)?,
709            "network.user_agent" => self.network.user_agent = value.to_string(),
710            "network.allow_cross_origin_auth" => {
711                self.network.allow_cross_origin_auth = parse_value(value)?
712            }
713            "network.proxy" => {
714                self.network.proxy = if value.trim().is_empty() {
715                    None
716                } else {
717                    Some(value.to_string())
718                }
719            }
720            "network.no_proxy" => {
721                self.network.no_proxy = if value.trim().is_empty() {
722                    None
723                } else {
724                    Some(value.to_string())
725                }
726            }
727            "hub.use_cache" => self.hub.use_cache = parse_value(value)?,
728            "hub.hf_cache" => {
729                self.hub.hf_cache = if value.trim().is_empty() {
730                    None
731                } else {
732                    Some(PathBuf::from(value))
733                }
734            }
735            "hub.modelscope_cache" => {
736                self.hub.modelscope_cache = if value.trim().is_empty() {
737                    None
738                } else {
739                    Some(PathBuf::from(value))
740                }
741            }
742            "hub.hf_endpoint" => {
743                self.hub.hf_endpoint = if value.trim().is_empty() {
744                    None
745                } else {
746                    Some(value.to_string())
747                }
748            }
749            "hub.modelscope_endpoint" => {
750                self.hub.modelscope_endpoint = if value.trim().is_empty() {
751                    None
752                } else {
753                    Some(value.to_string())
754                }
755            }
756            "bt.dht" => self.bt.dht = parse_value(value)?,
757            "bt.trackers" => self.bt.trackers = parse_value(value)?,
758            "bt.listen" => self.bt.listen = parse_value(value)?,
759            "bt.persistence" => self.bt.persistence = parse_value(value)?,
760            "bt.peer_limit" => {
761                self.bt.peer_limit = if value.trim().is_empty() {
762                    None
763                } else {
764                    Some(parse_value(value)?)
765                }
766            }
767            "bt.initial_peers" => {
768                self.bt.initial_peers = value
769                    .split(',')
770                    .map(str::trim)
771                    .filter(|value| !value.is_empty())
772                    .map(str::to_string)
773                    .collect();
774            }
775            "state_dir" => {
776                self.state_dir = if value.trim().is_empty() {
777                    None
778                } else {
779                    Some(PathBuf::from(value))
780                }
781            }
782            _ if key.starts_with("headers.") => {
783                let name = key["headers.".len()..].to_string();
784                Header::new(&name, value)?;
785                if value.trim().is_empty() {
786                    self.headers.remove(&name);
787                } else {
788                    self.headers.insert(name, value.to_string());
789                }
790            }
791            _ => return Err(Error::Config(format!("unknown config key: {key}"))),
792        }
793        Ok(())
794    }
795}
796
797fn project_dirs() -> Result<ProjectDirs> {
798    ProjectDirs::from("", "", "shaul")
799        .ok_or_else(|| Error::Config("cannot determine platform config directory".to_string()))
800}
801
802fn env(name: &str) -> Option<String> {
803    std::env::var(name)
804        .ok()
805        .map(|value| value.trim().to_string())
806        .filter(|value| !value.is_empty())
807}
808
809fn env_parse<T: FromStr>(name: &str) -> Result<Option<T>> {
810    match env(name) {
811        Some(value) => value
812            .parse::<T>()
813            .map(Some)
814            .map_err(|_| Error::Config(format!("invalid value for {name}: {value}"))),
815        None => Ok(None),
816    }
817}
818
819fn parse_value<T: FromStr>(value: &str) -> Result<T> {
820    value
821        .parse::<T>()
822        .map_err(|_| Error::Config(format!("invalid value: {value}")))
823}
824
825/// Parse a verify mode name or one of its boolean-like aliases.
826pub fn parse_verify_mode(value: &str) -> Result<VerifyMode> {
827    match value.trim().to_ascii_lowercase().as_str() {
828        "auto" => Ok(VerifyMode::Auto),
829        "require" | "on" | "true" | "yes" | "1" => Ok(VerifyMode::Require),
830        "disabled" | "off" | "false" | "no" | "0" | "none" => Ok(VerifyMode::Disabled),
831        _ => Err(Error::Config(format!("invalid verify mode: {value}"))),
832    }
833}
834
835/// Canonical string form of a verify mode.
836pub fn verify_mode_str(mode: VerifyMode) -> &'static str {
837    match mode {
838        VerifyMode::Auto => "auto",
839        VerifyMode::Require => "require",
840        VerifyMode::Disabled => "disabled",
841    }
842}
843
844/// Parse byte sizes such as `8MiB`, `256K`, `1G` or a plain byte count.
845pub fn parse_bytes(value: &str) -> Result<u64> {
846    let value = value.trim();
847    let lower = value.to_ascii_lowercase();
848    let (number, multiplier) = if let Some(number) = lower.strip_suffix("kib") {
849        (number, 1024)
850    } else if let Some(number) = lower.strip_suffix("mib") {
851        (number, 1024 * 1024)
852    } else if let Some(number) = lower.strip_suffix("gib") {
853        (number, 1024 * 1024 * 1024)
854    } else if let Some(number) = lower.strip_suffix("kb") {
855        (number, 1000)
856    } else if let Some(number) = lower.strip_suffix("mb") {
857        (number, 1000 * 1000)
858    } else if let Some(number) = lower.strip_suffix("gb") {
859        (number, 1000 * 1000 * 1000)
860    } else if let Some(number) = lower.strip_suffix('k') {
861        (number, 1024)
862    } else if let Some(number) = lower.strip_suffix('m') {
863        (number, 1024 * 1024)
864    } else if let Some(number) = lower.strip_suffix('g') {
865        (number, 1024 * 1024 * 1024)
866    } else {
867        (lower.as_str(), 1)
868    };
869    let number = number.trim();
870    let parsed = number
871        .parse::<f64>()
872        .map_err(|_| Error::Config(format!("invalid byte size: {value}")))?;
873    if !parsed.is_finite() || parsed < 0.0 {
874        return Err(Error::Config(format!("invalid byte size: {value}")));
875    }
876    Ok((parsed * multiplier as f64) as u64)
877}
878
879#[derive(Debug, Default, Deserialize)]
880struct PartialConfig {
881    #[serde(default)]
882    download: Option<PartialDownload>,
883    #[serde(default)]
884    network: Option<PartialNetwork>,
885    #[serde(default)]
886    hub: Option<PartialHub>,
887    #[serde(default)]
888    bt: Option<PartialBt>,
889    #[serde(default)]
890    state_dir: Option<PathBuf>,
891    #[serde(default)]
892    headers: Option<BTreeMap<String, String>>,
893}
894
895#[derive(Debug, Default, Deserialize)]
896struct PartialDownload {
897    #[serde(default)]
898    dir: Option<PathBuf>,
899    #[serde(default)]
900    connections: Option<usize>,
901    #[serde(default)]
902    per_host: Option<usize>,
903    #[serde(default)]
904    per_host_active: Option<usize>,
905    #[serde(default)]
906    global: Option<usize>,
907    #[serde(default)]
908    rate_limit: Option<u64>,
909    #[serde(default)]
910    per_host_rate_limit: Option<u64>,
911    #[serde(default)]
912    file_jobs: Option<usize>,
913    #[serde(default)]
914    content_cache: Option<bool>,
915    #[serde(default)]
916    content_cache_dir: Option<PathBuf>,
917    #[serde(default)]
918    disk_space_check: Option<bool>,
919    #[serde(default)]
920    segment_size: Option<u64>,
921    #[serde(default)]
922    resume: Option<bool>,
923    #[serde(default)]
924    verify: Option<VerifyMode>,
925    #[serde(default)]
926    extract: Option<bool>,
927    #[serde(default)]
928    extract_dir: Option<PathBuf>,
929    #[serde(default)]
930    delete_archive: Option<bool>,
931}
932
933#[derive(Debug, Default, Deserialize)]
934struct PartialNetwork {
935    #[serde(default)]
936    timeout_secs: Option<u64>,
937    #[serde(default)]
938    connect_timeout_secs: Option<u64>,
939    #[serde(default)]
940    retries: Option<u32>,
941    #[serde(default)]
942    user_agent: Option<String>,
943    #[serde(default)]
944    allow_cross_origin_auth: Option<bool>,
945    #[serde(default)]
946    proxy: Option<String>,
947    #[serde(default)]
948    no_proxy: Option<String>,
949}
950
951#[derive(Debug, Default, Deserialize)]
952struct PartialHub {
953    #[serde(default)]
954    use_cache: Option<bool>,
955    #[serde(default)]
956    hf_cache: Option<PathBuf>,
957    #[serde(default)]
958    modelscope_cache: Option<PathBuf>,
959    #[serde(default)]
960    hf_endpoint: Option<String>,
961    #[serde(default)]
962    modelscope_endpoint: Option<String>,
963}
964
965#[derive(Debug, Default, Deserialize)]
966struct PartialBt {
967    #[serde(default)]
968    dht: Option<bool>,
969    #[serde(default)]
970    trackers: Option<bool>,
971    #[serde(default)]
972    listen: Option<bool>,
973    #[serde(default)]
974    persistence: Option<bool>,
975    #[serde(default)]
976    peer_limit: Option<usize>,
977    #[serde(default)]
978    initial_peers: Option<Vec<String>>,
979}
980
981impl PartialConfig {
982    fn apply(self, config: &mut Config) -> Result<()> {
983        if let Some(download) = self.download {
984            if let Some(value) = download.dir {
985                config.download.dir = value;
986            }
987            if let Some(value) = download.connections {
988                config.download.connections = value;
989            }
990            if let Some(value) = download.per_host {
991                config.download.per_host = value;
992            }
993            if let Some(value) = download.per_host_active {
994                config.download.per_host_active = value.max(1);
995            }
996            if let Some(value) = download.global {
997                config.download.global = value;
998            }
999            if let Some(value) = download.rate_limit {
1000                config.download.rate_limit = (value > 0).then_some(value);
1001            }
1002            if let Some(value) = download.per_host_rate_limit {
1003                config.download.per_host_rate_limit = (value > 0).then_some(value);
1004            }
1005            if let Some(value) = download.file_jobs {
1006                config.download.file_jobs = value.max(1);
1007            }
1008            if let Some(value) = download.content_cache {
1009                config.download.content_cache = value;
1010            }
1011            if let Some(value) = download.content_cache_dir {
1012                config.download.content_cache_dir =
1013                    (!value.as_os_str().is_empty()).then_some(value);
1014            }
1015            if let Some(value) = download.disk_space_check {
1016                config.download.disk_space_check = value;
1017            }
1018            if let Some(value) = download.segment_size {
1019                config.download.segment_size = value;
1020            }
1021            if let Some(value) = download.resume {
1022                config.download.resume = value;
1023            }
1024            if let Some(value) = download.verify {
1025                config.download.verify = value;
1026            }
1027            if let Some(value) = download.extract {
1028                config.download.extract = value;
1029            }
1030            if let Some(value) = download.extract_dir {
1031                config.download.extract_dir = Some(value);
1032            }
1033            if let Some(value) = download.delete_archive {
1034                config.download.delete_archive = value;
1035            }
1036        }
1037        if let Some(network) = self.network {
1038            if let Some(value) = network.timeout_secs {
1039                config.network.timeout_secs = value;
1040            }
1041            if let Some(value) = network.connect_timeout_secs {
1042                config.network.connect_timeout_secs = value;
1043            }
1044            if let Some(value) = network.retries {
1045                config.network.retries = value;
1046            }
1047            if let Some(value) = network.user_agent {
1048                config.network.user_agent = value;
1049            }
1050            if let Some(value) = network.allow_cross_origin_auth {
1051                config.network.allow_cross_origin_auth = value;
1052            }
1053            if let Some(value) = network.proxy {
1054                config.network.proxy = (!value.trim().is_empty()).then_some(value);
1055            }
1056            if let Some(value) = network.no_proxy {
1057                config.network.no_proxy = (!value.trim().is_empty()).then_some(value);
1058            }
1059        }
1060        if let Some(hub) = self.hub {
1061            if let Some(value) = hub.use_cache {
1062                config.hub.use_cache = value;
1063            }
1064            if let Some(value) = hub.hf_cache {
1065                config.hub.hf_cache = Some(value);
1066            }
1067            if let Some(value) = hub.modelscope_cache {
1068                config.hub.modelscope_cache = Some(value);
1069            }
1070            if let Some(value) = hub.hf_endpoint {
1071                config.hub.hf_endpoint = Some(value);
1072            }
1073            if let Some(value) = hub.modelscope_endpoint {
1074                config.hub.modelscope_endpoint = Some(value);
1075            }
1076        }
1077        if let Some(bt) = self.bt {
1078            if let Some(value) = bt.dht {
1079                config.bt.dht = value;
1080            }
1081            if let Some(value) = bt.trackers {
1082                config.bt.trackers = value;
1083            }
1084            if let Some(value) = bt.listen {
1085                config.bt.listen = value;
1086            }
1087            if let Some(value) = bt.persistence {
1088                config.bt.persistence = value;
1089            }
1090            if let Some(value) = bt.peer_limit {
1091                config.bt.peer_limit = Some(value);
1092            }
1093            if let Some(value) = bt.initial_peers {
1094                config.bt.initial_peers = value;
1095            }
1096        }
1097        if let Some(value) = self.state_dir {
1098            config.state_dir = Some(value);
1099        }
1100        if let Some(headers) = self.headers {
1101            for (name, value) in headers {
1102                Header::new(&name, &value)?;
1103                config.headers.insert(name, value);
1104            }
1105        }
1106        Ok(())
1107    }
1108}
1109
1110#[cfg(test)]
1111mod tests {
1112    use super::*;
1113
1114    #[test]
1115    fn file_overrides_environment_layer() {
1116        let mut config = Config::default();
1117        config.download.connections = 3; // pretend env set this
1118        let partial: PartialConfig = toml::from_str("[download]\nconnections = 11\n").unwrap();
1119        partial.apply(&mut config).unwrap();
1120        assert_eq!(config.download.connections, 11);
1121    }
1122
1123    #[test]
1124    fn parses_byte_sizes() {
1125        assert_eq!(parse_bytes("8MiB").unwrap(), 8 * 1024 * 1024);
1126        assert_eq!(parse_bytes("1.5G").unwrap(), 1_610_612_736);
1127    }
1128
1129    #[test]
1130    fn gets_and_sets_values() {
1131        let mut config = Config::default();
1132        config.set_value("download.connections", "12").unwrap();
1133        assert_eq!(config.get_value("download.connections").unwrap(), "12");
1134        config.set_value("download.rate_limit", "2MiB").unwrap();
1135        assert_eq!(
1136            config.get_value("download.rate_limit").unwrap(),
1137            (2 * 1024 * 1024).to_string()
1138        );
1139        config
1140            .set_value("download.per_host_rate_limit", "1MiB")
1141            .unwrap();
1142        assert_eq!(
1143            config.get_value("download.per_host_rate_limit").unwrap(),
1144            (1024 * 1024).to_string()
1145        );
1146        config.set_value("download.file_jobs", "3").unwrap();
1147        assert_eq!(config.get_value("download.file_jobs").unwrap(), "3");
1148        config.set_value("download.content_cache", "true").unwrap();
1149        assert_eq!(config.get_value("download.content_cache").unwrap(), "true");
1150        config
1151            .set_value("download.content_cache_dir", "/tmp/shaul-content")
1152            .unwrap();
1153        assert_eq!(
1154            config.get_value("download.content_cache_dir").unwrap(),
1155            Path::new("/tmp/shaul-content").display().to_string()
1156        );
1157        config.set_value("download.per_host_active", "7").unwrap();
1158        assert_eq!(config.get_value("download.per_host_active").unwrap(), "7");
1159        config
1160            .set_value("network.proxy", "http://127.0.0.1:8080")
1161            .unwrap();
1162        assert_eq!(
1163            config.get_value("network.proxy").unwrap(),
1164            "http://127.0.0.1:8080"
1165        );
1166        config
1167            .set_value("network.no_proxy", "localhost,127.0.0.1")
1168            .unwrap();
1169        assert_eq!(
1170            config.get_value("network.no_proxy").unwrap(),
1171            "localhost,127.0.0.1"
1172        );
1173        config.set_value("headers.X-Test", "yes").unwrap();
1174        assert_eq!(config.get_value("headers.X-Test").unwrap(), "yes");
1175    }
1176
1177    #[test]
1178    fn resolves_proxy_and_rate_limits() {
1179        let mut config = Config::default();
1180        config.download.rate_limit = Some(1024);
1181        config.download.per_host_rate_limit = Some(512);
1182        config.network.proxy = Some("http://127.0.0.1:8080".to_string());
1183        config.network.no_proxy = Some("localhost".to_string());
1184        config.download.file_jobs = 4;
1185        config.download.content_cache = true;
1186        let options = TaskOptions {
1187            rate_limit: Some(2048),
1188            file_jobs: Some(2),
1189            content_cache: Some(false),
1190            ..TaskOptions::default()
1191        };
1192        let resolved = config.resolve(Path::new("."), &options).unwrap();
1193        assert_eq!(resolved.rate_limit, Some(1024));
1194        assert_eq!(resolved.per_host_rate_limit, Some(512));
1195        assert_eq!(resolved.task_rate_limit, Some(2048));
1196        assert_eq!(resolved.file_jobs, 2);
1197        assert_eq!(resolved.per_host_active, 16);
1198        assert!(!resolved.content_cache);
1199        assert!(resolved.content_cache_dir.ends_with("content"));
1200
1201        let partial: PartialConfig = toml::from_str(
1202            "[download]\nrate_limit = 0\n[network]\nproxy = \"\"\nno_proxy = \"\"\n",
1203        )
1204        .unwrap();
1205        partial.apply(&mut config).unwrap();
1206        assert_eq!(config.download.rate_limit, None);
1207        assert_eq!(config.network.proxy, None);
1208        assert_eq!(config.network.no_proxy, None);
1209    }
1210}