1use std::collections::BTreeMap;
2use std::path::{Path, PathBuf};
3use std::str::FromStr;
4use std::time::Duration;
5
6use directories::{BaseDirs, ProjectDirs};
7use serde::{Deserialize, Serialize};
8
9use crate::error::{Error, Result};
10use crate::header::{Header, RequestOptions};
11use crate::task::{TaskOptions, VerifyMode};
12
13#[derive(Clone, Debug, Serialize, Deserialize)]
16#[serde(default)]
17#[derive(Default)]
18#[non_exhaustive]
19pub struct Config {
20 pub download: DownloadConfig,
22 pub network: NetworkConfig,
24 pub hub: HubConfig,
26 pub bt: BtConfig,
28 pub state_dir: Option<PathBuf>,
30 pub headers: BTreeMap<String, String>,
33}
34
35#[derive(Clone, Debug, Serialize, Deserialize)]
37#[serde(default)]
38#[non_exhaustive]
39pub struct DownloadConfig {
40 pub dir: PathBuf,
42 pub connections: usize,
44 pub per_host: usize,
46 pub per_host_active: usize,
49 pub global: usize,
51 pub rate_limit: Option<u64>,
54 pub per_host_rate_limit: Option<u64>,
57 pub file_jobs: usize,
59 pub content_cache: bool,
61 pub content_cache_dir: Option<PathBuf>,
63 pub disk_space_check: bool,
66 pub segment_size: u64,
68 pub resume: bool,
70 pub verify: VerifyMode,
72 pub extract: bool,
74 pub extract_dir: Option<PathBuf>,
76 pub delete_archive: bool,
78}
79
80#[derive(Clone, Debug, Serialize, Deserialize)]
82#[serde(default)]
83#[non_exhaustive]
84pub struct NetworkConfig {
85 pub timeout_secs: u64,
87 pub connect_timeout_secs: u64,
89 pub retries: u32,
91 pub user_agent: String,
93 pub allow_cross_origin_auth: bool,
95 pub proxy: Option<String>,
98 pub no_proxy: Option<String>,
100}
101
102#[derive(Clone, Debug, Serialize, Deserialize)]
104#[serde(default)]
105#[derive(Default)]
106#[non_exhaustive]
107pub struct HubConfig {
108 pub use_cache: bool,
111 pub hf_cache: Option<PathBuf>,
113 pub modelscope_cache: Option<PathBuf>,
115 pub hf_endpoint: Option<String>,
117 pub modelscope_endpoint: Option<String>,
119}
120
121#[derive(Clone, Debug, Serialize, Deserialize)]
123#[serde(default)]
124#[non_exhaustive]
125pub struct BtConfig {
126 pub dht: bool,
128 pub trackers: bool,
130 pub listen: bool,
132 pub persistence: bool,
134 pub peer_limit: Option<usize>,
136 pub initial_peers: Vec<String>,
138}
139
140impl Default for BtConfig {
141 fn default() -> Self {
142 Self {
143 dht: true,
144 trackers: true,
145 listen: true,
146 persistence: true,
147 peer_limit: None,
148 initial_peers: Vec::new(),
149 }
150 }
151}
152
153impl Default for DownloadConfig {
154 fn default() -> Self {
155 Self {
156 dir: PathBuf::from("."),
157 connections: 8,
158 per_host: 16,
159 per_host_active: 16,
160 global: 64,
161 rate_limit: None,
162 per_host_rate_limit: None,
163 file_jobs: 1,
164 content_cache: false,
165 content_cache_dir: None,
166 disk_space_check: true,
167 segment_size: 8 * 1024 * 1024,
168 resume: true,
169 verify: VerifyMode::Auto,
170 extract: false,
171 extract_dir: None,
172 delete_archive: false,
173 }
174 }
175}
176
177impl Default for NetworkConfig {
178 fn default() -> Self {
179 Self {
180 timeout_secs: 60,
181 connect_timeout_secs: 15,
182 retries: 5,
183 user_agent: format!("shaul/{}", env!("CARGO_PKG_VERSION")),
184 allow_cross_origin_auth: false,
185 proxy: None,
186 no_proxy: None,
187 }
188 }
189}
190
191#[derive(Clone, Debug)]
193#[non_exhaustive]
194pub struct ResolvedOptions {
195 pub destination: PathBuf,
197 pub output: Option<PathBuf>,
199 pub connections: usize,
201 pub per_host: usize,
203 pub per_host_active: usize,
206 pub global: usize,
209 pub rate_limit: Option<u64>,
211 pub per_host_rate_limit: Option<u64>,
213 pub task_rate_limit: Option<u64>,
215 pub file_jobs: usize,
217 pub content_cache: bool,
219 pub content_cache_dir: PathBuf,
221 pub disk_space_check: bool,
223 pub mirrors: Vec<url::Url>,
225 pub segment_size: u64,
227 pub resume: bool,
229 pub verify: VerifyMode,
231 pub checksum: Option<crate::checksum::Checksum>,
233 pub extract: bool,
235 pub extract_dir: Option<PathBuf>,
237 pub delete_archive: bool,
239 pub cache: bool,
241 pub include: Vec<String>,
243 pub exclude: Vec<String>,
245 pub request: RequestOptions,
247 pub overwrite: bool,
249 pub rename_on_conflict: bool,
251 pub retries: u32,
253 pub timeout: Duration,
255 pub connect_timeout: Duration,
257 pub user_agent: String,
259}
260
261impl Config {
262 pub fn default_config_path() -> Result<PathBuf> {
265 let dirs = project_dirs()?;
266 Ok(dirs.config_dir().join("config.toml"))
267 }
268
269 pub fn default_state_dir() -> Result<PathBuf> {
271 let dirs = project_dirs()?;
272 Ok(dirs.data_dir().join("tasks"))
273 }
274
275 pub fn state_dir(&self) -> Result<PathBuf> {
277 match &self.state_dir {
278 Some(dir) => Ok(dir.clone()),
279 None => Self::default_state_dir(),
280 }
281 }
282
283 pub fn hf_cache_dir(&self) -> Result<PathBuf> {
286 if let Some(dir) = &self.hub.hf_cache {
287 return Ok(dir.clone());
288 }
289 let base = BaseDirs::new()
290 .ok_or_else(|| Error::Config("cannot determine home directory".to_string()))?;
291 Ok(base
292 .home_dir()
293 .join(".cache")
294 .join("huggingface")
295 .join("hub"))
296 }
297
298 pub fn modelscope_cache_dir(&self) -> Result<PathBuf> {
301 if let Some(dir) = &self.hub.modelscope_cache {
302 return Ok(dir.clone());
303 }
304 Ok(project_dirs()?.cache_dir().join("modelscope"))
305 }
306
307 pub fn content_cache_dir(&self) -> Result<PathBuf> {
310 if let Some(dir) = &self.download.content_cache_dir
311 && !dir.as_os_str().is_empty()
312 {
313 return Ok(dir.clone());
314 }
315 Ok(project_dirs()?.cache_dir().join("content"))
316 }
317
318 pub fn hf_endpoint(&self) -> String {
320 self.hub
321 .hf_endpoint
322 .clone()
323 .unwrap_or_else(|| "https://huggingface.co".to_string())
324 }
325
326 pub fn modelscope_endpoint(&self) -> String {
328 self.hub
329 .modelscope_endpoint
330 .clone()
331 .unwrap_or_else(|| "https://modelscope.cn".to_string())
332 }
333
334 pub fn resolve_path(explicit: Option<&Path>) -> Result<PathBuf> {
337 if let Some(path) = explicit {
338 return Ok(path.to_path_buf());
339 }
340 if let Ok(path) = std::env::var("SHAUL_CONFIG")
341 && !path.trim().is_empty()
342 {
343 return Ok(PathBuf::from(path));
344 }
345 Self::default_config_path()
346 }
347
348 pub fn load(explicit: Option<&Path>) -> Result<(Self, PathBuf)> {
351 let path = Self::resolve_path(explicit)?;
352 let mut config = Self::default();
353 config.apply_env()?;
354 if path.exists() {
355 let text = std::fs::read_to_string(&path)?;
356 let partial: PartialConfig = toml::from_str(&text)?;
357 partial.apply(&mut config)?;
358 }
359 Ok((config, path))
360 }
361
362 pub fn load_file_only(path: &Path) -> Result<Self> {
365 let mut config = Self::default();
366 if path.exists() {
367 let text = std::fs::read_to_string(path)?;
368 let partial: PartialConfig = toml::from_str(&text)?;
369 partial.apply(&mut config)?;
370 }
371 Ok(config)
372 }
373
374 pub fn save(&self, path: &Path) -> Result<()> {
376 if let Some(parent) = path.parent()
377 && !parent.as_os_str().is_empty()
378 {
379 std::fs::create_dir_all(parent)?;
380 }
381 let text = toml::to_string_pretty(self)?;
382 std::fs::write(path, text)?;
383 Ok(())
384 }
385
386 pub fn apply_env(&mut self) -> Result<()> {
389 if let Some(value) = env("SHAUL_DIR") {
390 self.download.dir = PathBuf::from(value);
391 }
392 if let Some(value) = env_parse::<usize>("SHAUL_CONNECTIONS")? {
393 self.download.connections = value;
394 }
395 if let Some(value) = env_parse::<usize>("SHAUL_PER_HOST")? {
396 self.download.per_host = value;
397 }
398 if let Some(value) = env_parse::<usize>("SHAUL_PER_HOST_ACTIVE")? {
399 self.download.per_host_active = value.max(1);
400 }
401 if let Some(value) = env_parse::<usize>("SHAUL_GLOBAL")? {
402 self.download.global = value;
403 }
404 if let Some(value) = env("SHAUL_RATE_LIMIT") {
405 self.download.rate_limit = Some(parse_bytes(&value)?).filter(|value| *value > 0);
406 }
407 if let Some(value) = env("SHAUL_PER_HOST_RATE_LIMIT") {
408 self.download.per_host_rate_limit =
409 Some(parse_bytes(&value)?).filter(|value| *value > 0);
410 }
411 if let Some(value) = env_parse::<usize>("SHAUL_FILE_JOBS")? {
412 self.download.file_jobs = value.max(1);
413 }
414 if let Some(value) = env_parse::<bool>("SHAUL_CONTENT_CACHE")? {
415 self.download.content_cache = value;
416 }
417 if let Some(value) = env("SHAUL_CONTENT_CACHE_DIR") {
418 self.download.content_cache_dir = Some(PathBuf::from(value));
419 }
420 if let Some(value) = env_parse::<bool>("SHAUL_DISK_SPACE_CHECK")? {
421 self.download.disk_space_check = value;
422 }
423 if let Some(value) = env("SHAUL_SEGMENT_SIZE") {
424 self.download.segment_size = parse_bytes(&value)?;
425 }
426 if let Some(value) = env_parse::<bool>("SHAUL_RESUME")? {
427 self.download.resume = value;
428 }
429 if let Some(value) = env("SHAUL_VERIFY") {
430 self.download.verify = parse_verify_mode(&value)?;
431 }
432 if let Some(value) = env_parse::<bool>("SHAUL_EXTRACT")? {
433 self.download.extract = value;
434 }
435 if let Some(value) = env("SHAUL_EXTRACT_DIR") {
436 self.download.extract_dir = Some(PathBuf::from(value));
437 }
438 if let Some(value) = env_parse::<bool>("SHAUL_DELETE_ARCHIVE")? {
439 self.download.delete_archive = value;
440 }
441 if let Some(value) = env("SHAUL_STATE_DIR") {
442 self.state_dir = Some(PathBuf::from(value));
443 }
444 if let Some(value) = env_parse::<u64>("SHAUL_TIMEOUT_SECS")? {
445 self.network.timeout_secs = value;
446 }
447 if let Some(value) = env_parse::<u64>("SHAUL_CONNECT_TIMEOUT_SECS")? {
448 self.network.connect_timeout_secs = value;
449 }
450 if let Some(value) = env_parse::<u32>("SHAUL_RETRIES")? {
451 self.network.retries = value;
452 }
453 if let Some(value) = env("SHAUL_USER_AGENT") {
454 self.network.user_agent = value;
455 }
456 if let Some(value) = env_parse::<bool>("SHAUL_ALLOW_CROSS_ORIGIN_AUTH")? {
457 self.network.allow_cross_origin_auth = value;
458 }
459 if let Some(value) = env("SHAUL_PROXY").or_else(|| {
460 env("HTTPS_PROXY")
461 .or_else(|| env("https_proxy"))
462 .or_else(|| env("HTTP_PROXY"))
463 .or_else(|| env("http_proxy"))
464 .or_else(|| env("ALL_PROXY"))
465 .or_else(|| env("all_proxy"))
466 }) {
467 self.network.proxy = Some(value);
468 }
469 if let Some(value) = env("SHAUL_NO_PROXY")
470 .or_else(|| env("NO_PROXY"))
471 .or_else(|| env("no_proxy"))
472 {
473 self.network.no_proxy = Some(value);
474 }
475
476 if let Some(value) = env("HF_HOME") {
477 self.hub.hf_cache = Some(PathBuf::from(value).join("hub"));
478 }
479 if let Some(value) = env("HF_HUB_CACHE") {
480 self.hub.hf_cache = Some(PathBuf::from(value));
481 }
482 if let Some(value) = env("SHAUL_HF_CACHE") {
483 self.hub.hf_cache = Some(PathBuf::from(value));
484 }
485 if let Some(value) = env("HF_ENDPOINT") {
486 self.hub.hf_endpoint = Some(value);
487 }
488 if let Some(value) = env("SHAUL_HF_ENDPOINT") {
489 self.hub.hf_endpoint = Some(value);
490 }
491 if let Some(value) = env("MODELSCOPE_CACHE") {
492 self.hub.modelscope_cache = Some(PathBuf::from(value));
493 }
494 if let Some(value) = env("SHAUL_MODELSCOPE_CACHE") {
495 self.hub.modelscope_cache = Some(PathBuf::from(value));
496 }
497 if let Some(value) = env("MODELSCOPE_ENDPOINT").or_else(|| env("MODELSCOPE_DOMAIN")) {
498 self.hub.modelscope_endpoint = Some(value);
499 }
500 if let Some(value) = env("SHAUL_MODELSCOPE_ENDPOINT") {
501 self.hub.modelscope_endpoint = Some(value);
502 }
503 if let Some(value) = env_parse::<bool>("SHAUL_HUB_USE_CACHE")? {
504 self.hub.use_cache = value;
505 }
506
507 if let Some(value) = env_parse::<bool>("SHAUL_BT_DHT")? {
508 self.bt.dht = value;
509 }
510 if let Some(value) = env_parse::<bool>("SHAUL_BT_TRACKERS")? {
511 self.bt.trackers = value;
512 }
513 if let Some(value) = env_parse::<bool>("SHAUL_BT_LISTEN")? {
514 self.bt.listen = value;
515 }
516 if let Some(value) = env_parse::<bool>("SHAUL_BT_PERSISTENCE")? {
517 self.bt.persistence = value;
518 }
519 if let Some(value) = env_parse::<usize>("SHAUL_BT_PEER_LIMIT")? {
520 self.bt.peer_limit = Some(value);
521 }
522 if let Some(value) = env("SHAUL_BT_INITIAL_PEERS") {
523 self.bt.initial_peers = value
524 .split(',')
525 .map(str::trim)
526 .filter(|value| !value.is_empty())
527 .map(str::to_string)
528 .collect();
529 }
530 Ok(())
531 }
532
533 pub fn header_defaults(&self) -> Result<Vec<Header>> {
535 self.headers
536 .iter()
537 .map(|(name, value)| Header::new(name, value))
538 .collect()
539 }
540
541 pub fn resolve(&self, destination: &Path, options: &TaskOptions) -> Result<ResolvedOptions> {
544 let mut request = options.request.merged(&self.header_defaults()?);
545 request.allow_cross_origin_auth |= self.network.allow_cross_origin_auth;
546 Ok(ResolvedOptions {
547 destination: destination.to_path_buf(),
548 output: options.output.clone(),
549 connections: options
550 .connections
551 .unwrap_or(self.download.connections)
552 .max(1),
553 per_host: self.download.per_host.max(1),
554 per_host_active: self.download.per_host_active.max(1),
555 global: self.download.global.max(1),
556 rate_limit: self.download.rate_limit.filter(|value| *value > 0),
557 per_host_rate_limit: self.download.per_host_rate_limit.filter(|value| *value > 0),
558 task_rate_limit: options.rate_limit.filter(|value| *value > 0),
559 file_jobs: options.file_jobs.unwrap_or(self.download.file_jobs).max(1),
560 content_cache: options.content_cache.unwrap_or(self.download.content_cache),
561 content_cache_dir: self.content_cache_dir()?,
562 disk_space_check: self.download.disk_space_check,
563 mirrors: options.mirrors.clone(),
564 segment_size: self.download.segment_size.max(64 * 1024),
565 resume: options.resume.unwrap_or(self.download.resume),
566 verify: options.verify.unwrap_or(self.download.verify),
567 checksum: options.checksum.clone(),
568 extract: options.extract.unwrap_or(self.download.extract),
569 extract_dir: options
570 .extract_dir
571 .clone()
572 .or_else(|| self.download.extract_dir.clone()),
573 delete_archive: options
574 .delete_archive
575 .unwrap_or(self.download.delete_archive),
576 cache: options.cache.unwrap_or(self.hub.use_cache),
577 include: options.include.clone(),
578 exclude: options.exclude.clone(),
579 request,
580 overwrite: options.overwrite,
581 rename_on_conflict: options.rename_on_conflict,
582 retries: self.network.retries,
583 timeout: Duration::from_secs(self.network.timeout_secs.max(1)),
584 connect_timeout: Duration::from_secs(self.network.connect_timeout_secs.max(1)),
585 user_agent: self.network.user_agent.clone(),
586 })
587 }
588
589 pub fn get_value(&self, key: &str) -> Result<String> {
591 let value = match key {
592 "download.dir" => self.download.dir.display().to_string(),
593 "download.connections" => self.download.connections.to_string(),
594 "download.per_host" => self.download.per_host.to_string(),
595 "download.per_host_active" => self.download.per_host_active.to_string(),
596 "download.global" => self.download.global.to_string(),
597 "download.rate_limit" => self
598 .download
599 .rate_limit
600 .map(|value| value.to_string())
601 .unwrap_or_default(),
602 "download.per_host_rate_limit" => self
603 .download
604 .per_host_rate_limit
605 .map(|value| value.to_string())
606 .unwrap_or_default(),
607 "download.file_jobs" => self.download.file_jobs.to_string(),
608 "download.content_cache" => self.download.content_cache.to_string(),
609 "download.content_cache_dir" => self.content_cache_dir()?.display().to_string(),
610 "download.disk_space_check" => self.download.disk_space_check.to_string(),
611 "download.segment_size" => self.download.segment_size.to_string(),
612 "download.resume" => self.download.resume.to_string(),
613 "download.verify" => verify_mode_str(self.download.verify).to_string(),
614 "download.extract" => self.download.extract.to_string(),
615 "download.extract_dir" => self
616 .download
617 .extract_dir
618 .as_ref()
619 .map(|path| path.display().to_string())
620 .unwrap_or_default(),
621 "download.delete_archive" => self.download.delete_archive.to_string(),
622 "network.timeout_secs" => self.network.timeout_secs.to_string(),
623 "network.connect_timeout_secs" => self.network.connect_timeout_secs.to_string(),
624 "network.retries" => self.network.retries.to_string(),
625 "network.user_agent" => self.network.user_agent.clone(),
626 "network.allow_cross_origin_auth" => self.network.allow_cross_origin_auth.to_string(),
627 "network.proxy" => self.network.proxy.clone().unwrap_or_default(),
628 "network.no_proxy" => self.network.no_proxy.clone().unwrap_or_default(),
629 "hub.use_cache" => self.hub.use_cache.to_string(),
630 "hub.hf_cache" => self.hf_cache_dir()?.display().to_string(),
631 "hub.modelscope_cache" => self.modelscope_cache_dir()?.display().to_string(),
632 "hub.hf_endpoint" => self.hf_endpoint(),
633 "hub.modelscope_endpoint" => self.modelscope_endpoint(),
634 "bt.dht" => self.bt.dht.to_string(),
635 "bt.trackers" => self.bt.trackers.to_string(),
636 "bt.listen" => self.bt.listen.to_string(),
637 "bt.persistence" => self.bt.persistence.to_string(),
638 "bt.peer_limit" => self
639 .bt
640 .peer_limit
641 .map(|value| value.to_string())
642 .unwrap_or_default(),
643 "bt.initial_peers" => self.bt.initial_peers.join(","),
644 "state_dir" => self
645 .state_dir
646 .as_ref()
647 .map(|path| path.display().to_string())
648 .unwrap_or_default(),
649 _ if key.starts_with("headers.") => {
650 let name = &key["headers.".len()..];
651 self.headers.get(name).cloned().unwrap_or_default()
652 }
653 _ => return Err(Error::Config(format!("unknown config key: {key}"))),
654 };
655 Ok(value)
656 }
657
658 pub fn set_value(&mut self, key: &str, value: &str) -> Result<()> {
660 match key {
661 "download.dir" => self.download.dir = PathBuf::from(value),
662 "download.connections" => self.download.connections = parse_value(value)?,
663 "download.per_host" => self.download.per_host = parse_value(value)?,
664 "download.per_host_active" => {
665 self.download.per_host_active = parse_value::<usize>(value)?.max(1)
666 }
667 "download.global" => self.download.global = parse_value(value)?,
668 "download.rate_limit" => {
669 self.download.rate_limit = if value.trim().is_empty() {
670 None
671 } else {
672 Some(parse_bytes(value)?).filter(|value| *value > 0)
673 }
674 }
675 "download.per_host_rate_limit" => {
676 self.download.per_host_rate_limit = if value.trim().is_empty() {
677 None
678 } else {
679 Some(parse_bytes(value)?).filter(|value| *value > 0)
680 }
681 }
682 "download.file_jobs" => self.download.file_jobs = parse_value::<usize>(value)?.max(1),
683 "download.content_cache" => self.download.content_cache = parse_value(value)?,
684 "download.content_cache_dir" => {
685 self.download.content_cache_dir = if value.trim().is_empty() {
686 None
687 } else {
688 Some(PathBuf::from(value))
689 }
690 }
691 "download.disk_space_check" => self.download.disk_space_check = parse_value(value)?,
692 "download.segment_size" => self.download.segment_size = parse_bytes(value)?,
693 "download.resume" => self.download.resume = parse_value(value)?,
694 "download.verify" => self.download.verify = parse_verify_mode(value)?,
695 "download.extract" => self.download.extract = parse_value(value)?,
696 "download.extract_dir" => {
697 self.download.extract_dir = if value.trim().is_empty() {
698 None
699 } else {
700 Some(PathBuf::from(value))
701 }
702 }
703 "download.delete_archive" => self.download.delete_archive = parse_value(value)?,
704 "network.timeout_secs" => self.network.timeout_secs = parse_value(value)?,
705 "network.connect_timeout_secs" => {
706 self.network.connect_timeout_secs = parse_value(value)?
707 }
708 "network.retries" => self.network.retries = parse_value(value)?,
709 "network.user_agent" => self.network.user_agent = value.to_string(),
710 "network.allow_cross_origin_auth" => {
711 self.network.allow_cross_origin_auth = parse_value(value)?
712 }
713 "network.proxy" => {
714 self.network.proxy = if value.trim().is_empty() {
715 None
716 } else {
717 Some(value.to_string())
718 }
719 }
720 "network.no_proxy" => {
721 self.network.no_proxy = if value.trim().is_empty() {
722 None
723 } else {
724 Some(value.to_string())
725 }
726 }
727 "hub.use_cache" => self.hub.use_cache = parse_value(value)?,
728 "hub.hf_cache" => {
729 self.hub.hf_cache = if value.trim().is_empty() {
730 None
731 } else {
732 Some(PathBuf::from(value))
733 }
734 }
735 "hub.modelscope_cache" => {
736 self.hub.modelscope_cache = if value.trim().is_empty() {
737 None
738 } else {
739 Some(PathBuf::from(value))
740 }
741 }
742 "hub.hf_endpoint" => {
743 self.hub.hf_endpoint = if value.trim().is_empty() {
744 None
745 } else {
746 Some(value.to_string())
747 }
748 }
749 "hub.modelscope_endpoint" => {
750 self.hub.modelscope_endpoint = if value.trim().is_empty() {
751 None
752 } else {
753 Some(value.to_string())
754 }
755 }
756 "bt.dht" => self.bt.dht = parse_value(value)?,
757 "bt.trackers" => self.bt.trackers = parse_value(value)?,
758 "bt.listen" => self.bt.listen = parse_value(value)?,
759 "bt.persistence" => self.bt.persistence = parse_value(value)?,
760 "bt.peer_limit" => {
761 self.bt.peer_limit = if value.trim().is_empty() {
762 None
763 } else {
764 Some(parse_value(value)?)
765 }
766 }
767 "bt.initial_peers" => {
768 self.bt.initial_peers = value
769 .split(',')
770 .map(str::trim)
771 .filter(|value| !value.is_empty())
772 .map(str::to_string)
773 .collect();
774 }
775 "state_dir" => {
776 self.state_dir = if value.trim().is_empty() {
777 None
778 } else {
779 Some(PathBuf::from(value))
780 }
781 }
782 _ if key.starts_with("headers.") => {
783 let name = key["headers.".len()..].to_string();
784 Header::new(&name, value)?;
785 if value.trim().is_empty() {
786 self.headers.remove(&name);
787 } else {
788 self.headers.insert(name, value.to_string());
789 }
790 }
791 _ => return Err(Error::Config(format!("unknown config key: {key}"))),
792 }
793 Ok(())
794 }
795}
796
797fn project_dirs() -> Result<ProjectDirs> {
798 ProjectDirs::from("", "", "shaul")
799 .ok_or_else(|| Error::Config("cannot determine platform config directory".to_string()))
800}
801
802fn env(name: &str) -> Option<String> {
803 std::env::var(name)
804 .ok()
805 .map(|value| value.trim().to_string())
806 .filter(|value| !value.is_empty())
807}
808
809fn env_parse<T: FromStr>(name: &str) -> Result<Option<T>> {
810 match env(name) {
811 Some(value) => value
812 .parse::<T>()
813 .map(Some)
814 .map_err(|_| Error::Config(format!("invalid value for {name}: {value}"))),
815 None => Ok(None),
816 }
817}
818
819fn parse_value<T: FromStr>(value: &str) -> Result<T> {
820 value
821 .parse::<T>()
822 .map_err(|_| Error::Config(format!("invalid value: {value}")))
823}
824
825pub fn parse_verify_mode(value: &str) -> Result<VerifyMode> {
827 match value.trim().to_ascii_lowercase().as_str() {
828 "auto" => Ok(VerifyMode::Auto),
829 "require" | "on" | "true" | "yes" | "1" => Ok(VerifyMode::Require),
830 "disabled" | "off" | "false" | "no" | "0" | "none" => Ok(VerifyMode::Disabled),
831 _ => Err(Error::Config(format!("invalid verify mode: {value}"))),
832 }
833}
834
835pub fn verify_mode_str(mode: VerifyMode) -> &'static str {
837 match mode {
838 VerifyMode::Auto => "auto",
839 VerifyMode::Require => "require",
840 VerifyMode::Disabled => "disabled",
841 }
842}
843
844pub fn parse_bytes(value: &str) -> Result<u64> {
846 let value = value.trim();
847 let lower = value.to_ascii_lowercase();
848 let (number, multiplier) = if let Some(number) = lower.strip_suffix("kib") {
849 (number, 1024)
850 } else if let Some(number) = lower.strip_suffix("mib") {
851 (number, 1024 * 1024)
852 } else if let Some(number) = lower.strip_suffix("gib") {
853 (number, 1024 * 1024 * 1024)
854 } else if let Some(number) = lower.strip_suffix("kb") {
855 (number, 1000)
856 } else if let Some(number) = lower.strip_suffix("mb") {
857 (number, 1000 * 1000)
858 } else if let Some(number) = lower.strip_suffix("gb") {
859 (number, 1000 * 1000 * 1000)
860 } else if let Some(number) = lower.strip_suffix('k') {
861 (number, 1024)
862 } else if let Some(number) = lower.strip_suffix('m') {
863 (number, 1024 * 1024)
864 } else if let Some(number) = lower.strip_suffix('g') {
865 (number, 1024 * 1024 * 1024)
866 } else {
867 (lower.as_str(), 1)
868 };
869 let number = number.trim();
870 let parsed = number
871 .parse::<f64>()
872 .map_err(|_| Error::Config(format!("invalid byte size: {value}")))?;
873 if !parsed.is_finite() || parsed < 0.0 {
874 return Err(Error::Config(format!("invalid byte size: {value}")));
875 }
876 Ok((parsed * multiplier as f64) as u64)
877}
878
879#[derive(Debug, Default, Deserialize)]
880struct PartialConfig {
881 #[serde(default)]
882 download: Option<PartialDownload>,
883 #[serde(default)]
884 network: Option<PartialNetwork>,
885 #[serde(default)]
886 hub: Option<PartialHub>,
887 #[serde(default)]
888 bt: Option<PartialBt>,
889 #[serde(default)]
890 state_dir: Option<PathBuf>,
891 #[serde(default)]
892 headers: Option<BTreeMap<String, String>>,
893}
894
895#[derive(Debug, Default, Deserialize)]
896struct PartialDownload {
897 #[serde(default)]
898 dir: Option<PathBuf>,
899 #[serde(default)]
900 connections: Option<usize>,
901 #[serde(default)]
902 per_host: Option<usize>,
903 #[serde(default)]
904 per_host_active: Option<usize>,
905 #[serde(default)]
906 global: Option<usize>,
907 #[serde(default)]
908 rate_limit: Option<u64>,
909 #[serde(default)]
910 per_host_rate_limit: Option<u64>,
911 #[serde(default)]
912 file_jobs: Option<usize>,
913 #[serde(default)]
914 content_cache: Option<bool>,
915 #[serde(default)]
916 content_cache_dir: Option<PathBuf>,
917 #[serde(default)]
918 disk_space_check: Option<bool>,
919 #[serde(default)]
920 segment_size: Option<u64>,
921 #[serde(default)]
922 resume: Option<bool>,
923 #[serde(default)]
924 verify: Option<VerifyMode>,
925 #[serde(default)]
926 extract: Option<bool>,
927 #[serde(default)]
928 extract_dir: Option<PathBuf>,
929 #[serde(default)]
930 delete_archive: Option<bool>,
931}
932
933#[derive(Debug, Default, Deserialize)]
934struct PartialNetwork {
935 #[serde(default)]
936 timeout_secs: Option<u64>,
937 #[serde(default)]
938 connect_timeout_secs: Option<u64>,
939 #[serde(default)]
940 retries: Option<u32>,
941 #[serde(default)]
942 user_agent: Option<String>,
943 #[serde(default)]
944 allow_cross_origin_auth: Option<bool>,
945 #[serde(default)]
946 proxy: Option<String>,
947 #[serde(default)]
948 no_proxy: Option<String>,
949}
950
951#[derive(Debug, Default, Deserialize)]
952struct PartialHub {
953 #[serde(default)]
954 use_cache: Option<bool>,
955 #[serde(default)]
956 hf_cache: Option<PathBuf>,
957 #[serde(default)]
958 modelscope_cache: Option<PathBuf>,
959 #[serde(default)]
960 hf_endpoint: Option<String>,
961 #[serde(default)]
962 modelscope_endpoint: Option<String>,
963}
964
965#[derive(Debug, Default, Deserialize)]
966struct PartialBt {
967 #[serde(default)]
968 dht: Option<bool>,
969 #[serde(default)]
970 trackers: Option<bool>,
971 #[serde(default)]
972 listen: Option<bool>,
973 #[serde(default)]
974 persistence: Option<bool>,
975 #[serde(default)]
976 peer_limit: Option<usize>,
977 #[serde(default)]
978 initial_peers: Option<Vec<String>>,
979}
980
981impl PartialConfig {
982 fn apply(self, config: &mut Config) -> Result<()> {
983 if let Some(download) = self.download {
984 if let Some(value) = download.dir {
985 config.download.dir = value;
986 }
987 if let Some(value) = download.connections {
988 config.download.connections = value;
989 }
990 if let Some(value) = download.per_host {
991 config.download.per_host = value;
992 }
993 if let Some(value) = download.per_host_active {
994 config.download.per_host_active = value.max(1);
995 }
996 if let Some(value) = download.global {
997 config.download.global = value;
998 }
999 if let Some(value) = download.rate_limit {
1000 config.download.rate_limit = (value > 0).then_some(value);
1001 }
1002 if let Some(value) = download.per_host_rate_limit {
1003 config.download.per_host_rate_limit = (value > 0).then_some(value);
1004 }
1005 if let Some(value) = download.file_jobs {
1006 config.download.file_jobs = value.max(1);
1007 }
1008 if let Some(value) = download.content_cache {
1009 config.download.content_cache = value;
1010 }
1011 if let Some(value) = download.content_cache_dir {
1012 config.download.content_cache_dir =
1013 (!value.as_os_str().is_empty()).then_some(value);
1014 }
1015 if let Some(value) = download.disk_space_check {
1016 config.download.disk_space_check = value;
1017 }
1018 if let Some(value) = download.segment_size {
1019 config.download.segment_size = value;
1020 }
1021 if let Some(value) = download.resume {
1022 config.download.resume = value;
1023 }
1024 if let Some(value) = download.verify {
1025 config.download.verify = value;
1026 }
1027 if let Some(value) = download.extract {
1028 config.download.extract = value;
1029 }
1030 if let Some(value) = download.extract_dir {
1031 config.download.extract_dir = Some(value);
1032 }
1033 if let Some(value) = download.delete_archive {
1034 config.download.delete_archive = value;
1035 }
1036 }
1037 if let Some(network) = self.network {
1038 if let Some(value) = network.timeout_secs {
1039 config.network.timeout_secs = value;
1040 }
1041 if let Some(value) = network.connect_timeout_secs {
1042 config.network.connect_timeout_secs = value;
1043 }
1044 if let Some(value) = network.retries {
1045 config.network.retries = value;
1046 }
1047 if let Some(value) = network.user_agent {
1048 config.network.user_agent = value;
1049 }
1050 if let Some(value) = network.allow_cross_origin_auth {
1051 config.network.allow_cross_origin_auth = value;
1052 }
1053 if let Some(value) = network.proxy {
1054 config.network.proxy = (!value.trim().is_empty()).then_some(value);
1055 }
1056 if let Some(value) = network.no_proxy {
1057 config.network.no_proxy = (!value.trim().is_empty()).then_some(value);
1058 }
1059 }
1060 if let Some(hub) = self.hub {
1061 if let Some(value) = hub.use_cache {
1062 config.hub.use_cache = value;
1063 }
1064 if let Some(value) = hub.hf_cache {
1065 config.hub.hf_cache = Some(value);
1066 }
1067 if let Some(value) = hub.modelscope_cache {
1068 config.hub.modelscope_cache = Some(value);
1069 }
1070 if let Some(value) = hub.hf_endpoint {
1071 config.hub.hf_endpoint = Some(value);
1072 }
1073 if let Some(value) = hub.modelscope_endpoint {
1074 config.hub.modelscope_endpoint = Some(value);
1075 }
1076 }
1077 if let Some(bt) = self.bt {
1078 if let Some(value) = bt.dht {
1079 config.bt.dht = value;
1080 }
1081 if let Some(value) = bt.trackers {
1082 config.bt.trackers = value;
1083 }
1084 if let Some(value) = bt.listen {
1085 config.bt.listen = value;
1086 }
1087 if let Some(value) = bt.persistence {
1088 config.bt.persistence = value;
1089 }
1090 if let Some(value) = bt.peer_limit {
1091 config.bt.peer_limit = Some(value);
1092 }
1093 if let Some(value) = bt.initial_peers {
1094 config.bt.initial_peers = value;
1095 }
1096 }
1097 if let Some(value) = self.state_dir {
1098 config.state_dir = Some(value);
1099 }
1100 if let Some(headers) = self.headers {
1101 for (name, value) in headers {
1102 Header::new(&name, &value)?;
1103 config.headers.insert(name, value);
1104 }
1105 }
1106 Ok(())
1107 }
1108}
1109
1110#[cfg(test)]
1111mod tests {
1112 use super::*;
1113
1114 #[test]
1115 fn file_overrides_environment_layer() {
1116 let mut config = Config::default();
1117 config.download.connections = 3; let partial: PartialConfig = toml::from_str("[download]\nconnections = 11\n").unwrap();
1119 partial.apply(&mut config).unwrap();
1120 assert_eq!(config.download.connections, 11);
1121 }
1122
1123 #[test]
1124 fn parses_byte_sizes() {
1125 assert_eq!(parse_bytes("8MiB").unwrap(), 8 * 1024 * 1024);
1126 assert_eq!(parse_bytes("1.5G").unwrap(), 1_610_612_736);
1127 }
1128
1129 #[test]
1130 fn gets_and_sets_values() {
1131 let mut config = Config::default();
1132 config.set_value("download.connections", "12").unwrap();
1133 assert_eq!(config.get_value("download.connections").unwrap(), "12");
1134 config.set_value("download.rate_limit", "2MiB").unwrap();
1135 assert_eq!(
1136 config.get_value("download.rate_limit").unwrap(),
1137 (2 * 1024 * 1024).to_string()
1138 );
1139 config
1140 .set_value("download.per_host_rate_limit", "1MiB")
1141 .unwrap();
1142 assert_eq!(
1143 config.get_value("download.per_host_rate_limit").unwrap(),
1144 (1024 * 1024).to_string()
1145 );
1146 config.set_value("download.file_jobs", "3").unwrap();
1147 assert_eq!(config.get_value("download.file_jobs").unwrap(), "3");
1148 config.set_value("download.content_cache", "true").unwrap();
1149 assert_eq!(config.get_value("download.content_cache").unwrap(), "true");
1150 config
1151 .set_value("download.content_cache_dir", "/tmp/shaul-content")
1152 .unwrap();
1153 assert_eq!(
1154 config.get_value("download.content_cache_dir").unwrap(),
1155 Path::new("/tmp/shaul-content").display().to_string()
1156 );
1157 config.set_value("download.per_host_active", "7").unwrap();
1158 assert_eq!(config.get_value("download.per_host_active").unwrap(), "7");
1159 config
1160 .set_value("network.proxy", "http://127.0.0.1:8080")
1161 .unwrap();
1162 assert_eq!(
1163 config.get_value("network.proxy").unwrap(),
1164 "http://127.0.0.1:8080"
1165 );
1166 config
1167 .set_value("network.no_proxy", "localhost,127.0.0.1")
1168 .unwrap();
1169 assert_eq!(
1170 config.get_value("network.no_proxy").unwrap(),
1171 "localhost,127.0.0.1"
1172 );
1173 config.set_value("headers.X-Test", "yes").unwrap();
1174 assert_eq!(config.get_value("headers.X-Test").unwrap(), "yes");
1175 }
1176
1177 #[test]
1178 fn resolves_proxy_and_rate_limits() {
1179 let mut config = Config::default();
1180 config.download.rate_limit = Some(1024);
1181 config.download.per_host_rate_limit = Some(512);
1182 config.network.proxy = Some("http://127.0.0.1:8080".to_string());
1183 config.network.no_proxy = Some("localhost".to_string());
1184 config.download.file_jobs = 4;
1185 config.download.content_cache = true;
1186 let options = TaskOptions {
1187 rate_limit: Some(2048),
1188 file_jobs: Some(2),
1189 content_cache: Some(false),
1190 ..TaskOptions::default()
1191 };
1192 let resolved = config.resolve(Path::new("."), &options).unwrap();
1193 assert_eq!(resolved.rate_limit, Some(1024));
1194 assert_eq!(resolved.per_host_rate_limit, Some(512));
1195 assert_eq!(resolved.task_rate_limit, Some(2048));
1196 assert_eq!(resolved.file_jobs, 2);
1197 assert_eq!(resolved.per_host_active, 16);
1198 assert!(!resolved.content_cache);
1199 assert!(resolved.content_cache_dir.ends_with("content"));
1200
1201 let partial: PartialConfig = toml::from_str(
1202 "[download]\nrate_limit = 0\n[network]\nproxy = \"\"\nno_proxy = \"\"\n",
1203 )
1204 .unwrap();
1205 partial.apply(&mut config).unwrap();
1206 assert_eq!(config.download.rate_limit, None);
1207 assert_eq!(config.network.proxy, None);
1208 assert_eq!(config.network.no_proxy, None);
1209 }
1210}