1use async_trait::async_trait;
2use futures_util::TryStreamExt;
3use serde::{Deserialize, Serialize};
4use sha2::{Digest, Sha256};
5use std::{
6 collections::HashSet,
7 path::{Path, PathBuf},
8 pin::Pin,
9 time::Duration,
10};
11use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
12use tokio_util::{io::StreamReader, sync::CancellationToken};
13
14#[derive(Debug, Clone, Deserialize, Serialize)]
15#[serde(deny_unknown_fields)]
16pub struct Artifact {
17 pub backend: String,
18 pub repo: String,
19 pub revision: String,
20 pub file: String,
21 pub sha256: String,
22 pub bytes: u64,
23}
24pub struct DownloadResponse {
25 pub status: u16,
26 pub commit: Option<String>,
27 pub length: Option<u64>,
28 pub body: Pin<Box<dyn AsyncRead + Send>>,
29}
30#[async_trait]
31pub trait DownloadTransport: Send + Sync {
32 async fn get(&self, url: &str) -> Result<DownloadResponse, String>;
33 async fn get_from(&self, url: &str, offset: u64) -> Result<DownloadResponse, String> {
35 if offset == 0 {
36 self.get(url).await
37 } else {
38 Err("download resume unsupported".into())
39 }
40 }
41}
42const MAX_RESUMES: u32 = 8;
43pub struct HfTransport {
44 client: reqwest::Client,
45}
46fn retain_commit(evidence: &mut Option<String>, observed: Option<&str>) -> Result<(), String> {
47 if let Some(observed) = observed {
48 if observed.len() != 40 || !observed.bytes().all(|byte| byte.is_ascii_hexdigit()) {
49 return Err("invalid repository commit evidence".into());
50 }
51 if evidence
52 .as_ref()
53 .is_some_and(|previous| !previous.eq_ignore_ascii_case(observed))
54 {
55 return Err("conflicting repository commit evidence".into());
56 }
57 *evidence = Some(observed.to_owned());
58 }
59 Ok(())
60}
61#[test]
62fn commit_evidence_survives_cdn_redirects_and_rejects_conflicts() {
63 let revision = "a".repeat(40);
64 let mut evidence = None;
65 retain_commit(&mut evidence, Some(&revision)).unwrap();
66 retain_commit(&mut evidence, None).unwrap();
67 assert_eq!(evidence.as_deref(), Some(revision.as_str()));
68 assert!(retain_commit(&mut evidence, Some(&"b".repeat(40))).is_err());
69 assert!(retain_commit(&mut None, Some("not-a-commit")).is_err());
70 let mut absent = None;
71 retain_commit(&mut absent, None).unwrap();
72 assert!(absent.is_none());
73}
74impl HfTransport {
75 pub fn new() -> Result<Self, String> {
76 Ok(Self {
77 client: reqwest::Client::builder()
78 .redirect(reqwest::redirect::Policy::none())
79 .no_proxy()
80 .dns_resolver(std::sync::Arc::new(PublicResolver))
81 .connect_timeout(Duration::from_secs(10))
82 .build()
83 .map_err(|error| error.to_string())?,
84 })
85 }
86}
87fn safe_url(raw: &str) -> Result<url::Url, String> {
88 let url = url::Url::parse(raw).map_err(|error| error.to_string())?;
89 let host = url.host_str().ok_or("missing download host")?;
90 if url.scheme() != "https"
91 || !url.username().is_empty()
92 || url.password().is_some()
93 || url.port().is_some()
94 || !["huggingface.co", "hf.co"]
95 .iter()
96 .any(|allowed| host == *allowed || host.ends_with(&format!(".{allowed}")))
97 {
98 return Err("download URL is not approved HTTPS source".into());
99 }
100 Ok(url)
101}
102fn content_range_starts_at(value: Option<&str>, offset: u64) -> bool {
104 value
105 .and_then(|value| value.strip_prefix("bytes "))
106 .and_then(|range| range.split_once('-'))
107 .and_then(|(start, _)| start.parse::<u64>().ok())
108 == Some(offset)
109}
110#[test]
111fn resume_requires_matching_content_range() {
112 assert!(content_range_starts_at(Some("bytes 40-99/100"), 40));
113 assert!(!content_range_starts_at(Some("bytes 0-99/100"), 40));
114 assert!(!content_range_starts_at(Some("items 40-99/100"), 40));
115 assert!(!content_range_starts_at(None, 40));
116}
117#[async_trait]
118impl DownloadTransport for HfTransport {
119 async fn get(&self, raw: &str) -> Result<DownloadResponse, String> {
120 self.get_from(raw, 0).await
121 }
122 async fn get_from(&self, raw: &str, offset: u64) -> Result<DownloadResponse, String> {
123 let mut url = safe_url(raw)?;
124 let mut commit = None;
125 for redirect in 0..=5 {
126 let mut request = self.client.get(url.clone());
127 if offset > 0 {
128 request = request.header("range", format!("bytes={offset}-"));
129 }
130 let response = request
131 .send()
132 .await
133 .map_err(|_| "download transport failed".to_owned())?;
134 retain_commit(
135 &mut commit,
136 response
137 .headers()
138 .get("x-repo-commit")
139 .map(|value| {
140 value
141 .to_str()
142 .map_err(|_| "invalid repository commit header".to_owned())
143 })
144 .transpose()?,
145 )?;
146 if response.status().is_redirection() {
147 if redirect == 5 {
148 return Err("too many download redirects".into());
149 }
150 let location = response
151 .headers()
152 .get("location")
153 .and_then(|value| value.to_str().ok())
154 .ok_or("redirect has no location")?;
155 url = safe_url(
156 url.join(location)
157 .map_err(|error| error.to_string())?
158 .as_str(),
159 )?;
160 continue;
161 }
162 let length = response.content_length();
163 let status = response.status().as_u16();
164 if offset > 0
165 && (status != 206
166 || !content_range_starts_at(
167 response
168 .headers()
169 .get("content-range")
170 .and_then(|value| value.to_str().ok()),
171 offset,
172 ))
173 {
174 return Err("download resume was not honored".into());
175 }
176 let stream = response.bytes_stream().map_err(std::io::Error::other);
177 return Ok(DownloadResponse {
178 status,
179 commit,
180 length,
181 body: Box::pin(StreamReader::new(stream)),
182 });
183 }
184 Err("download redirect limit".into())
185 }
186}
187
188pub fn public_addresses(
189 addresses: Vec<std::net::SocketAddr>,
190) -> Result<Vec<std::net::SocketAddr>, String> {
191 fn public(address: std::net::IpAddr) -> bool {
192 match address {
193 std::net::IpAddr::V4(address) => {
194 let octets = address.octets();
195 !address.is_private()
196 && !address.is_loopback()
197 && !address.is_link_local()
198 && !address.is_multicast()
199 && !address.is_unspecified()
200 && !address.is_broadcast()
201 && !address.is_documentation()
202 && octets[0] != 0
203 && octets[0] < 224
204 && !(octets[0] == 100 && (64..=127).contains(&octets[1]))
205 && !(octets[0] == 198 && (18..=19).contains(&octets[1]))
206 && !(octets[0] == 192 && octets[1] == 0 && octets[2] == 0)
207 }
208 std::net::IpAddr::V6(address) => {
209 if let Some(mapped) = address.to_ipv4_mapped() {
210 return public(mapped.into());
211 }
212 let segments = address.segments();
213 segments[0] & 0xe000 == 0x2000
214 && segments[0] != 0x2002
215 && !(segments[0] == 0x2001 && (segments[1] < 0x200 || segments[1] == 0xdb8))
216 }
217 }
218 }
219 if addresses.is_empty()
220 || addresses.len() > 64
221 || addresses.iter().any(|address| !public(address.ip()))
222 {
223 return Err("download DNS contains private or unsupported addresses".into());
224 }
225 Ok(addresses)
226}
227struct PublicResolver;
228impl reqwest::dns::Resolve for PublicResolver {
229 fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
230 Box::pin(async move {
231 let addresses = tokio::time::timeout(
232 Duration::from_secs(5),
233 tokio::net::lookup_host((name.as_str(), 0)),
234 )
235 .await
236 .map_err(|_| std::io::Error::other("download DNS timed out"))??;
237 let addresses =
238 public_addresses(addresses.take(65).collect()).map_err(std::io::Error::other)?;
239 Ok(Box::new(addresses.into_iter()) as reqwest::dns::Addrs)
240 })
241 }
242}
243
244#[derive(Debug)]
245pub struct Acquired {
246 pub path: PathBuf,
247 pub bytes: u64,
248 pub cached: bool,
249}
250#[derive(Debug, Clone, Copy)]
251pub enum SizePolicy {
252 Exact,
253 Ceiling,
254}
255impl SizePolicy {
256 fn accepts(self, actual: u64, expected: u64) -> bool {
257 actual > 0
258 && match self {
259 Self::Exact => actual == expected,
260 Self::Ceiling => actual <= expected,
261 }
262 }
263}
264pub struct Acquisition {
265 root: PathBuf,
266 progress: Option<std::sync::Arc<Progress>>,
267 timeout: Duration,
268}
269type Progress = dyn Fn(u64, u64, &str) + Send + Sync;
270
271struct ArtifactLock {
272 guard: crate::state::LockGuard,
273}
274impl ArtifactLock {
275 fn acquire(path: PathBuf) -> Result<Self, String> {
276 let mut config =
277 crate::state::Config::from_home(path.parent().ok_or("missing lock parent")?)
278 .map_err(|error| error.to_string())?;
279 config.lock = path;
280 let guard = crate::state::StateStore::new(config)
281 .lock(Duration::ZERO)
282 .map_err(|error| error.to_string())?;
283 Ok(Self { guard })
284 }
285 fn check(&self) -> Result<(), String> {
286 self.guard.assert_owned().map_err(|error| error.to_string())
287 }
288}
289fn suffix(path: &Path, suffix: &str) -> PathBuf {
290 let mut value = path.as_os_str().to_owned();
291 value.push(suffix);
292 PathBuf::from(value)
293}
294pub fn cleanup_partials(parent: &Path, alive: impl Fn(u32) -> bool) -> Result<(), String> {
295 for (index, entry) in std::fs::read_dir(parent)
296 .map_err(|error| error.to_string())?
297 .enumerate()
298 {
299 if index >= 10000 {
300 return Err("cache directory exceeds cleanup entry limit".into());
301 }
302 let entry = entry.map_err(|error| error.to_string())?;
303 let name = entry.file_name();
304 let Some(name) = name.to_str() else {
305 continue;
306 };
307 let Some(rest) = name
308 .strip_prefix(".llmup-download.")
309 .and_then(|value| value.strip_suffix(".part"))
310 else {
311 continue;
312 };
313 let Some((pid, unique)) = rest.split_once('.') else {
314 continue;
315 };
316 if unique.is_empty()
317 || unique.contains('.')
318 || !pid.bytes().all(|byte| byte.is_ascii_digit())
319 || pid.starts_with('0')
320 {
321 continue;
322 }
323 let Some(pid) = pid
324 .parse::<u32>()
325 .ok()
326 .filter(|pid| *pid > 0 && *pid <= i32::MAX as u32)
327 else {
328 continue;
329 };
330 if entry
331 .file_type()
332 .map_err(|error| error.to_string())?
333 .is_file()
334 && !alive(pid)
335 {
336 std::fs::remove_file(entry.path()).map_err(|error| error.to_string())?;
337 }
338 }
339 Ok(())
340}
341fn segment(value: &str) -> bool {
342 !value.is_empty()
343 && portable_component(value)
344 && value.as_bytes()[0].is_ascii_alphanumeric()
345 && value
346 .bytes()
347 .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
348}
349fn portable_component(value: &str) -> bool {
350 let base = value.split('.').next().unwrap_or("").to_ascii_uppercase();
351 !value.is_empty()
352 && value != "."
353 && value != ".."
354 && !value.ends_with('.')
355 && value.trim() == value
356 && !["CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$"].contains(&base.as_str())
357 && !(base.starts_with("COM") || base.starts_with("LPT"))
358 .then(|| &base[3..])
359 .is_some_and(|suffix| {
360 [
361 "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u{b9}", "\u{b2}", "\u{b3}",
362 ]
363 .contains(&suffix)
364 })
365}
366pub fn safe_file(value: &str) -> bool {
367 !value.is_empty()
368 && value.len() <= 512
369 && !value
370 .chars()
371 .any(|ch| ch.is_control() || "\\%*?[]{}:".contains(ch))
372 && value.split('/').all(portable_component)
373}
374fn digest(value: &str, length: usize) -> bool {
375 value.len() == length && value.bytes().all(|byte| byte.is_ascii_hexdigit())
376}
377impl Artifact {
378 pub fn validate(&self) -> Result<(), String> {
379 let coordinates: Vec<_> = self.repo.split('/').collect();
380 if !["llamacpp", "mlx", "comfyui"].contains(&self.backend.as_str())
381 || coordinates.len() != 2
382 || !coordinates.iter().all(|part| segment(part))
383 || !digest(&self.revision, 40)
384 || !digest(&self.sha256, 64)
385 || !safe_file(&self.file)
386 || self.bytes == 0
387 || self.bytes > 9007199254740991
388 {
389 return Err("invalid pinned artifact".into());
390 }
391 Ok(())
392 }
393}
394fn private_directory(path: &Path) -> Result<(), String> {
395 match std::fs::symlink_metadata(path) {
396 Ok(metadata) => {
397 if !metadata.is_dir() || metadata.file_type().is_symlink() {
398 return Err("unsafe cache directory".into());
399 }
400 }
401 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
402 if let Some(parent) = path.parent() {
403 private_directory(parent)?;
404 }
405 let builder = std::fs::DirBuilder::new();
406 #[cfg(unix)]
407 let mut builder = builder;
408 #[cfg(unix)]
409 {
410 use std::os::unix::fs::DirBuilderExt;
411 builder.mode(0o700);
412 }
413 match builder.create(path) {
414 Ok(()) => (),
415 Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
416 return private_directory(path);
417 }
418 Err(error) => return Err(error.to_string()),
419 }
420 }
421 Err(error) => return Err(error.to_string()),
422 }
423 Ok(())
424}
425pub async fn hash_file(path: &Path, cancel: &CancellationToken) -> Result<(String, u64), String> {
426 let mut options = std::fs::OpenOptions::new();
427 options.read(true);
428 #[cfg(unix)]
429 {
430 use std::os::unix::fs::OpenOptionsExt;
431 options.custom_flags(
432 (rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits() as i32,
433 );
434 }
435 let file = options.open(path).map_err(|error| error.to_string())?;
436 if !file
437 .metadata()
438 .map_err(|error| error.to_string())?
439 .is_file()
440 || std::fs::symlink_metadata(path)
441 .map_err(|error| error.to_string())?
442 .file_type()
443 .is_symlink()
444 {
445 return Err("artifact is not a regular file".into());
446 }
447 let mut file = tokio::fs::File::from_std(file);
448 let mut hash = Sha256::new();
449 let mut total = 0u64;
450 let mut buffer = vec![0u8; 65536];
451 loop {
452 let count = tokio::select! { _=cancel.cancelled()=>return Err("cancelled".into()), result=file.read(&mut buffer)=>result.map_err(|error|error.to_string())? };
453 if count == 0 {
454 break;
455 }
456 total = total
457 .checked_add(count as u64)
458 .ok_or("artifact size overflow")?;
459 hash.update(&buffer[..count]);
460 }
461 Ok((format!("{:x}", hash.finalize()), total))
462}
463impl Acquisition {
464 fn cache_directory(&self, path: &Path) -> Result<(), String> {
465 let relative = path
466 .strip_prefix(&self.root)
467 .map_err(|_| "cache path escaped root")?;
468 let mut current = self.root.clone();
469 private_directory(¤t)?;
470 for component in relative.components() {
471 let std::path::Component::Normal(name) = component else {
472 return Err("invalid cache path component".into());
473 };
474 current.push(name);
475 private_directory(¤t)?;
476 if !current
477 .canonicalize()
478 .map_err(|error| error.to_string())?
479 .starts_with(&self.root)
480 {
481 return Err("cache path escaped root".into());
482 }
483 }
484 Ok(())
485 }
486 pub fn new(root: impl AsRef<Path>) -> Result<Self, String> {
487 let root = std::path::absolute(root).map_err(|error| error.to_string())?;
488 private_directory(&root)?;
489 #[cfg(unix)]
490 {
491 use std::os::unix::fs::PermissionsExt;
492 std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))
493 .map_err(|error| error.to_string())?;
494 }
495 Ok(Self {
496 root: root.canonicalize().map_err(|error| error.to_string())?,
497 progress: None,
498 timeout: Duration::from_secs(1800),
499 })
500 }
501 pub fn with_progress(
502 mut self,
503 progress: impl Fn(u64, u64, &str) + Send + Sync + 'static,
504 ) -> Self {
505 self.progress = Some(std::sync::Arc::new(progress));
506 self
507 }
508 pub fn with_timeout(mut self, timeout: Duration) -> Self {
510 self.timeout = timeout;
511 self
512 }
513 fn emit(&self, completed: u64, artifact: &Artifact) {
514 if let Some(progress) = &self.progress {
515 progress(completed, artifact.bytes, &artifact.file);
516 }
517 }
518 pub async fn acquire(
519 &self,
520 artifact: &Artifact,
521 transport: &dyn DownloadTransport,
522 cancel: &CancellationToken,
523 ) -> Result<Acquired, String> {
524 self.acquire_sized(artifact, SizePolicy::Exact, transport, cancel)
525 .await
526 }
527 pub async fn acquire_sized(
528 &self,
529 artifact: &Artifact,
530 size: SizePolicy,
531 transport: &dyn DownloadTransport,
532 cancel: &CancellationToken,
533 ) -> Result<Acquired, String> {
534 let operation = async {
535 artifact.validate()?;
536 let coordinates: Vec<_> = artifact.repo.split('/').collect();
537 let owner = self.root.join(&artifact.backend).join(coordinates[0]);
538 self.cache_directory(&owner)?;
539 let repository = owner.join(format!("{}@{}", coordinates[1], artifact.revision));
540 let lock = ArtifactLock::acquire(suffix(&repository, ".repository.lock"))?;
541 let acquired = self
542 .acquire_inner(artifact, size, transport, cancel)
543 .await?;
544 lock.check()?;
545 Ok(acquired)
546 };
547 tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(self.timeout,operation)=>result.map_err(|_|"acquisition timed out".to_owned())?}
548 }
549 async fn acquire_inner(
550 &self,
551 artifact: &Artifact,
552 size: SizePolicy,
553 transport: &dyn DownloadTransport,
554 cancel: &CancellationToken,
555 ) -> Result<Acquired, String> {
556 artifact.validate()?;
557 if cancel.is_cancelled() {
558 return Err("cancelled".into());
559 }
560 let coordinates: Vec<_> = artifact.repo.split('/').collect();
561 let directory = self
562 .root
563 .join(&artifact.backend)
564 .join(coordinates[0])
565 .join(format!("{}@{}", coordinates[1], artifact.revision));
566 let target = directory.join(&artifact.file);
567 let parent = target.parent().ok_or("missing artifact parent")?;
568 self.cache_directory(parent)?;
569 if !parent
570 .canonicalize()
571 .map_err(|error| error.to_string())?
572 .starts_with(&self.root)
573 {
574 return Err("cache escaped root".into());
575 }
576 let lock = ArtifactLock::acquire(suffix(&target, ".lock"))?;
577 cleanup_partials(parent, crate::state::process_alive)?;
578 self.emit(0, artifact);
579 if let Ok(metadata) = std::fs::symlink_metadata(&target) {
580 if !metadata.is_file() || metadata.file_type().is_symlink() {
581 return Err("unsafe cached artifact".into());
582 }
583 let (sha, bytes) = hash_file(&target, cancel).await?;
584 if sha.eq_ignore_ascii_case(&artifact.sha256) && size.accepts(bytes, artifact.bytes) {
585 lock.check()?;
586 self.emit(bytes, artifact);
587 return Ok(Acquired {
588 path: target,
589 bytes,
590 cached: true,
591 });
592 }
593 }
594 let mut url =
595 url::Url::parse("https://huggingface.co").map_err(|error| error.to_string())?;
596 {
597 let mut path = url.path_segments_mut().map_err(|_| "invalid URL base")?;
598 path.extend(coordinates)
599 .push("resolve")
600 .push(&artifact.revision)
601 .extend(artifact.file.split('/'));
602 }
603 url.set_query(Some("download=true"));
604 let operation = async {
605 let mut response = transport.get(url.as_str()).await?;
606 if !(200..300).contains(&response.status) {
607 return Err(format!("download HTTP {}", response.status));
608 }
609 if response
610 .commit
611 .as_ref()
612 .is_none_or(|commit| !commit.eq_ignore_ascii_case(&artifact.revision))
613 {
614 return Err("resolved commit mismatch".into());
615 }
616 if response
617 .length
618 .is_some_and(|length| length > artifact.bytes)
619 {
620 return Err("download exceeds byte limit".into());
621 }
622 let temporary = tempfile::Builder::new()
623 .prefix(&format!(".llmup-download.{}.", std::process::id()))
624 .suffix(".part")
625 .tempfile_in(parent)
626 .map_err(|error| error.to_string())?;
627 let mut output =
628 tokio::fs::File::from_std(temporary.reopen().map_err(|error| error.to_string())?);
629 let mut hash = Sha256::new();
630 let mut total = 0u64;
631 let mut reported = 0u64;
632 let mut buffer = vec![0u8; 65536];
633 let mut resumes = 0u32;
634 loop {
635 let count = match response.body.read(&mut buffer).await {
636 Ok(count) => count,
637 Err(error) => {
638 resumes += 1;
639 if resumes > MAX_RESUMES {
640 return Err(error.to_string());
641 }
642 tokio::time::sleep(Duration::from_secs(u64::from(resumes.min(5)))).await;
643 response = transport
644 .get_from(url.as_str(), total)
645 .await
646 .map_err(|resume| format!("{error}; {resume}"))?;
647 if response.status != 206
648 || response.commit.as_ref().is_none_or(|commit| {
649 !commit.eq_ignore_ascii_case(&artifact.revision)
650 })
651 || response
652 .length
653 .is_some_and(|length| length > artifact.bytes - total)
654 {
655 return Err("download resume response mismatch".into());
656 }
657 continue;
658 }
659 };
660 if count == 0 {
661 break;
662 }
663 total = total
664 .checked_add(count as u64)
665 .ok_or("download size overflow")?;
666 if total > artifact.bytes {
667 return Err("download exceeds byte limit".into());
668 }
669 hash.update(&buffer[..count]);
670 output
671 .write_all(&buffer[..count])
672 .await
673 .map_err(|error| error.to_string())?;
674 if total - reported >= 8 * 1024 * 1024 {
675 self.emit(total, artifact);
676 reported = total;
677 }
678 }
679 if !size.accepts(total, artifact.bytes)
680 || !format!("{:x}", hash.finalize()).eq_ignore_ascii_case(&artifact.sha256)
681 {
682 return Err("artifact integrity mismatch".into());
683 }
684 output.sync_all().await.map_err(|error| error.to_string())?;
685 drop(output);
686 lock.check()?;
687 if cancel.is_cancelled() {
688 return Err("cancelled".into());
689 }
690 temporary
691 .persist(&target)
692 .map_err(|error| error.to_string())?;
693 self.emit(total, artifact);
694 Ok(Acquired {
695 path: target.clone(),
696 bytes: total,
697 cached: false,
698 })
699 };
700 tokio::select! { _=cancel.cancelled()=>Err("cancelled".into()), result=tokio::time::timeout(self.timeout,operation)=>result.map_err(|_|"download timed out".to_owned())? }
701 }
702 pub async fn repository(
703 &self,
704 artifacts: &[Artifact],
705 transport: &dyn DownloadTransport,
706 cancel: &CancellationToken,
707 ) -> Result<PathBuf, String> {
708 tokio::select! {biased;_=cancel.cancelled()=>Err("cancelled".into()),result=tokio::time::timeout(Duration::from_secs(1800),self.repository_inner(artifacts,transport,cancel))=>result.map_err(|_|"repository acquisition timed out".to_owned())?}
709 }
710 async fn repository_inner(
711 &self,
712 artifacts: &[Artifact],
713 transport: &dyn DownloadTransport,
714 cancel: &CancellationToken,
715 ) -> Result<PathBuf, String> {
716 let first = artifacts.first().ok_or("empty repository manifest")?;
717 if artifacts.len() > 256 {
718 return Err("repository file count exceeds limit".into());
719 }
720 let mut names = HashSet::new();
721 let mut root = None;
722 for artifact in artifacts {
723 artifact.validate()?;
724 if ["py", "pyc", "pyo", "so", "dylib", "dll", "bundle"]
725 .iter()
726 .any(|extension| {
727 artifact
728 .file
729 .to_ascii_lowercase()
730 .ends_with(&format!(".{extension}"))
731 })
732 {
733 return Err("executable repository artifact forbidden".into());
734 }
735 if artifact.repo != first.repo
736 || artifact.revision != first.revision
737 || artifact.backend != first.backend
738 || !names.insert(artifact.file.clone())
739 {
740 return Err("inconsistent repository manifest".into());
741 }
742 }
743 let folded: HashSet<_> = names.iter().map(|name| name.to_lowercase()).collect();
744 if folded.len() != names.len()
745 || folded.iter().any(|name| {
746 name.match_indices('/')
747 .any(|(index, _)| folded.contains(&name[..index]))
748 })
749 {
750 return Err("repository contains path aliases or file/directory collisions".into());
751 }
752 if first.backend == "mlx"
753 && (artifacts.len() < 3
754 || !names.contains("config.json")
755 || !names.contains("tokenizer_config.json")
756 || !names.iter().any(|name| name.ends_with(".safetensors")))
757 {
758 return Err("incomplete MLX repository manifest".into());
759 }
760 let total = artifacts
761 .iter()
762 .try_fold(0u64, |total, artifact| total.checked_add(artifact.bytes))
763 .ok_or("repository byte total overflow")?;
764 if total > 9007199254740991 {
765 return Err("repository byte total exceeds safe integer".into());
766 }
767 let coordinates: Vec<_> = first.repo.split('/').collect();
768 let owner = self.root.join(&first.backend).join(coordinates[0]);
769 self.cache_directory(&owner)?;
770 let repository = owner.join(format!("{}@{}", coordinates[1], first.revision));
771 let lock = ArtifactLock::acquire(suffix(&repository, ".repository.lock"))?;
772 for artifact in artifacts {
773 let acquired = self
774 .acquire_inner(artifact, SizePolicy::Exact, transport, cancel)
775 .await?;
776 let mut directory = acquired.path;
777 for _ in artifact.file.split('/') {
778 directory.pop();
779 }
780 if root.as_ref().is_some_and(|prior| prior != &directory) {
781 return Err("repository root mismatch".into());
782 }
783 root = Some(directory);
784 }
785 let root = root.ok_or("missing repository root")?;
786 fn list(root: &Path, path: &Path, files: &mut HashSet<String>) -> Result<(), String> {
787 for entry in std::fs::read_dir(path).map_err(|error| error.to_string())? {
788 let entry = entry.map_err(|error| error.to_string())?;
789 let kind = entry.file_type().map_err(|error| error.to_string())?;
790 if kind.is_symlink() {
791 return Err("symlink in repository".into());
792 }
793 if kind.is_dir() {
794 list(root, &entry.path(), files)?;
795 } else if kind.is_file() {
796 files.insert(
797 entry
798 .path()
799 .strip_prefix(root)
800 .map_err(|error| error.to_string())?
801 .to_string_lossy()
802 .replace('\\', "/"),
803 );
804 } else {
805 return Err("special file in repository".into());
806 }
807 }
808 Ok(())
809 }
810 let mut actual = HashSet::new();
811 list(&root, &root, &mut actual)?;
812 if actual != names {
813 return Err("repository content differs from manifest".into());
814 }
815 lock.check()?;
816 Ok(root)
817 }
818}