1mod data;
17#[macro_use]
18extern crate lazy_static;
19use data::PackageFiles;
20pub use data::{
21 Dependency, DependencyConstraints, DependencyConstraintsParseError, DependencyVersion,
22 DependencyVersionParseError, Package,
23};
24use flate2::read::GzDecoder;
25use reqwest::{StatusCode, Url};
26use serde::__private::Formatter;
27use std::collections::HashMap;
28use std::error::Error;
29use std::fmt::Display;
30use std::io::{Cursor, Read, Write};
31use std::ops::Index;
32use std::sync::Arc;
33use tar::Archive;
34
35#[derive(Clone, Debug, PartialEq)]
36pub struct HttpError {
37 status: StatusCode,
38}
39
40impl Display for HttpError {
41 fn fmt(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
42 write!(formatter, "Server returned {} status", self.status.as_u16())
43 }
44}
45
46impl std::error::Error for HttpError {}
47
48pub enum Progress {
50 LoadingDb,
52 LoadingDbChunk(u64, Option<u64>),
54 ReadingDbFile(String),
56 ReadingDbDone,
58 LoadingFilesMetadata,
60 LoadingFilesMetadataChunk(u64, Option<u64>),
62 ReadingFilesMetadataFile(String),
64 ReadingFilesDone,
66}
67
68impl Display for Progress {
69 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
70 match self {
71 Progress::LoadingDb => write!(f, "Loading repository database"),
72 Progress::LoadingDbChunk(current, max) => {
73 if let Some(m) = max {
74 write!(f, "Loading repository: {} of {} bytes", current, m)
75 } else {
76 write!(f, "Loading repository: {} bytes", current)
77 }
78 }
79 Progress::ReadingDbFile(name) => write!(f, "Loading repository file: {}", name),
80 Progress::LoadingFilesMetadata => write!(f, "Loading files metadata"),
81 Progress::LoadingFilesMetadataChunk(current, max) => {
82 if let Some(m) = max {
83 write!(f, "Loading files metadata: {} of {} bytes", current, m)
84 } else {
85 write!(f, "Loading files metadata: {} bytes", current)
86 }
87 }
88 Progress::ReadingFilesMetadataFile(name) => {
89 write!(f, "Loading files metadata file: {}", name)
90 }
91 Progress::ReadingDbDone => write!(f, "Database loaded"),
92 Progress::ReadingFilesDone => write!(f, "Files metadata loaded"),
93 }
94 }
95}
96
97lazy_static! {
98 static ref SUFFIXES: Vec<&'static str> = vec!["-cvs", "-svn", "-hg", "-darcs", "-bzr", "-git"];
99}
100
101#[derive(Default)]
102struct Inner {
103 packages: Vec<Arc<Package>>,
104 package_base: HashMap<String, Arc<Package>>,
105 package_name: HashMap<String, Arc<Package>>,
106 package_version: HashMap<String, Arc<Package>>,
107 package_files: HashMap<String, PackageFiles>,
108}
109
110impl Inner {
111 async fn load<P>(
112 url: &str,
113 name: &str,
114 load_files_meta: bool,
115 progress: P,
116 ) -> Result<Self, Box<dyn Error>>
117 where
118 P: Fn(Progress),
119 {
120 let mut inner = Inner::default();
121 inner.load_db(url, name, &progress).await?;
122 if load_files_meta {
123 inner.load_files(url, name, &progress).await?;
124 }
125 Ok(inner)
126 }
127
128 async fn load_db<P>(&mut self, url: &str, name: &str, progress: P) -> Result<(), Box<dyn Error>>
129 where
130 P: Fn(Progress),
131 {
132 let db_url = format!("{}/{}.db.tar.gz", url, name);
133 progress(Progress::LoadingDb);
134 let mut db_archive =
135 Inner::load_archive(&db_url, |r, a| progress(Progress::LoadingDbChunk(r, a))).await?;
136 for entry_result in db_archive.entries()? {
137 let mut entry = entry_result?;
138 let path = entry.path()?.to_str().unwrap().to_owned();
139 if path.ends_with("/desc") {
140 progress(Progress::ReadingDbFile(path));
141 let mut contents = String::new();
142 entry.read_to_string(&mut contents)?;
143 let package: Package = archlinux_repo_parser::from_str(&contents)?;
144 self.insert(package);
145 }
146 }
147 progress(Progress::ReadingDbDone);
148 Ok(())
149 }
150
151 async fn load_files<P>(
152 &mut self,
153 url: &str,
154 name: &str,
155 progress: P,
156 ) -> Result<(), Box<dyn Error>>
157 where
158 P: Fn(Progress),
159 {
160 let db_url = format!("{}/{}.files.tar.gz", url, name);
161 progress(Progress::LoadingFilesMetadata);
162 let mut db_archive = Inner::load_archive(&db_url, |r, a| {
163 progress(Progress::LoadingFilesMetadataChunk(r, a))
164 })
165 .await?;
166 for entry_result in db_archive.entries()? {
167 let mut entry = entry_result?;
168 let path = entry.path()?.to_str().unwrap().to_owned();
169 if path.ends_with("/files") {
170 progress(Progress::ReadingFilesMetadataFile(path.clone()));
171 let mut contents = String::new();
172 entry.read_to_string(&mut contents)?;
173 let files: PackageFiles = archlinux_repo_parser::from_str(&contents)?;
174 let name = path.replace("/files", "").replace("/", "");
175 let package = &self.package_version[&name];
176 self.package_files.insert(package.name.to_owned(), files);
177 }
178 }
179 progress(Progress::ReadingFilesDone);
180 Ok(())
181 }
182
183 fn insert(&mut self, package: Package) {
184 let package_ref = self.insert_into_maps(package);
185 for suffix in SUFFIXES.iter() {
186 if package_ref.name.ends_with(suffix) {
187 let base_name = package_ref.name.replace(suffix, "");
188 let mut base_package = self
189 .package_name
190 .get(&base_name)
191 .map(|p| p.as_ref().clone())
192 .unwrap_or_else(|| Package::base_package_for_csv(package_ref.as_ref(), suffix));
193 base_package.linked_sources.push(package_ref.clone());
194 self.insert_into_maps(base_package);
195 }
196 }
197 }
198
199 fn insert_into_maps(&mut self, package: Package) -> Arc<Package> {
200 let package_ref = Arc::new(package);
201 if let Some(base) = package_ref.base.as_ref() {
202 if let std::collections::hash_map::Entry::Vacant(e) =
203 self.package_base.entry(base.to_owned())
204 {
205 e.insert(package_ref.clone());
206 } else {
207 log::warn!("[archlinux-repo-rs] Found package {} with already registered base name! Ignoring...", &package_ref.name)
208 }
209 }
210 self.package_name
211 .insert(package_ref.name.to_owned(), package_ref.clone());
212 self.package_version.insert(
213 package_ref.name.to_owned() + "-" + &package_ref.version,
214 package_ref.clone(),
215 );
216 self.packages.push(package_ref.clone());
217 package_ref
218 }
219
220 async fn load_archive<P>(
221 url: &str,
222 progress: P,
223 ) -> Result<Archive<Cursor<Vec<u8>>>, Box<dyn Error>>
224 where
225 P: Fn(u64, Option<u64>),
226 {
227 let mut enc_buf = Vec::new();
228 let mut response = reqwest::get(Url::parse(url)?).await?;
229 if !response.status().is_success() {
230 return Err(Box::new(HttpError {
231 status: response.status(),
232 }));
233 }
234 let mut bytes_read: u64 = 0;
235 let length = response.content_length();
236 while let Some(chunk) = response.chunk().await? {
237 enc_buf.write_all(&chunk[..])?;
238 bytes_read += chunk.len() as u64;
239 progress(bytes_read, length);
240 }
241 let mut decoder = GzDecoder::new(&enc_buf[..]);
242 let mut buf = Vec::new();
243 decoder.read_to_end(&mut buf)?;
244 Ok(Archive::new(Cursor::new(buf)))
245 }
246}
247
248pub struct Repository {
250 inner: Inner,
251 url: String,
252 name: String,
253 load_files_meta: bool,
254 progress_listener: Option<Box<dyn Fn(Progress)>>,
255}
256
257impl Repository {
258 async fn new(
259 url: String,
260 name: String,
261 load_files_meta: bool,
262 progress_listener: Option<Box<dyn Fn(Progress)>>,
263 ) -> Result<Self, Box<dyn Error>> {
264 let listener = progress_listener.as_ref();
265 let inner = Inner::load(&url, &name, load_files_meta, |progress| {
266 if let Some(l) = listener {
267 l(progress)
268 }
269 })
270 .await?;
271 Ok(Repository {
272 inner,
273 url,
274 name,
275 load_files_meta,
276 progress_listener,
277 })
278 }
279 pub async fn load(name: &str, url: &str) -> Result<Repository, Box<dyn Error>> {
288 RepositoryBuilder::new(name, url).load().await
289 }
290
291 pub fn get_package_by_name(&self, name: &str) -> Option<&Package> {
301 self.inner.package_name.get(name).map(|p| p as &Package)
302 }
303
304 pub fn get_package_by_name_and_version(&self, name: &str) -> Option<&Package> {
314 self.inner.package_version.get(name).map(|p| p as &Package)
315 }
316
317 pub fn get_package_by_base(&self, name: &str) -> Option<&Package> {
329 self.inner.package_base.get(name).map(|p| p as &Package)
330 }
331
332 pub fn get_package_files(&self, name: &str) -> Option<&Vec<String>> {
349 self.inner.package_files.get(name).map(|m| &m.files)
350 }
351
352 pub async fn request_package(&self, name: &str) -> Result<reqwest::Response, Box<dyn Error>> {
363 let package = self.index(name);
364 let url = format!("{}/{}", self.url, package.file_name);
365 Ok(reqwest::get(Url::parse(&url)?).await?)
366 }
367
368 pub async fn reload(&mut self) -> Result<(), Box<dyn Error>> {
371 let listener = self.progress_listener.as_ref();
372 self.inner = Inner::load(&self.url, &self.name, self.load_files_meta, |progress| {
373 if let Some(l) = listener {
374 l(progress)
375 }
376 })
377 .await?;
378 Ok(())
379 }
380}
381
382impl Index<&str> for Repository {
383 type Output = Package;
384
385 #[inline]
386 fn index(&self, index: &str) -> &Self::Output {
387 self.get_package_by_base(index)
388 .or_else(|| self.get_package_by_name(index))
389 .or_else(|| self.get_package_by_name_and_version(index))
390 .expect("package not found")
391 }
392}
393
394impl<'a> IntoIterator for &'a Repository {
395 type Item = &'a Package;
396 type IntoIter = Box<(dyn Iterator<Item = Self::Item> + 'a)>;
397
398 #[inline]
399 fn into_iter(self) -> Self::IntoIter {
400 Box::new(self.inner.packages.iter().map(|v| &**v))
401 }
402}
403
404pub struct RepositoryBuilder {
417 name: String,
418 url: String,
419 files_meta: bool,
420 progress_listener: Option<Box<dyn Fn(Progress)>>,
421}
422
423impl RepositoryBuilder {
424 pub fn new(name: &str, url: &str) -> Self {
426 RepositoryBuilder {
427 name: name.to_owned(),
428 url: url.to_owned(),
429 files_meta: false,
430 progress_listener: None,
431 }
432 }
433
434 pub fn files_metadata(mut self, load: bool) -> Self {
436 self.files_meta = load;
437 self
438 }
439
440 pub fn progress_listener(mut self, listener: Box<dyn Fn(Progress)>) -> Self {
442 self.progress_listener = Some(listener);
443 self
444 }
445
446 pub async fn load(self) -> Result<Repository, Box<dyn Error>> {
448 Ok(Repository::new(self.url, self.name, self.files_meta, self.progress_listener).await?)
449 }
450}
451
452#[cfg(test)]
453mod test {
454 use crate::data::PackageFiles;
455 use crate::{Package, Repository, RepositoryBuilder};
456
457 #[tokio::test]
458 async fn repo_loads_msys2_mingw_repo() {
459 Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
460 .await
461 .unwrap();
462 }
463
464 #[tokio::test]
465 async fn get_gtk_by_name() {
466 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
467 .await
468 .unwrap();
469 let gtk = repo.get_package_by_name("mingw-w64-x86_64-gtk3").unwrap();
470 assert_eq!("mingw-w64-gtk3", gtk.base.as_ref().unwrap())
471 }
472
473 #[tokio::test]
474 async fn get_none_from_not_existing_name() {
475 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
476 .await
477 .unwrap();
478 let package = repo.get_package_by_name("not_exist");
479 assert!(package.is_none())
480 }
481
482 #[tokio::test]
483 async fn get_gtk_by_base() {
484 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
485 .await
486 .unwrap();
487 let gtk = repo.get_package_by_base("mingw-w64-gtk3").unwrap();
488 assert_eq!("mingw-w64-x86_64-gtk3", >k.name)
489 }
490
491 #[tokio::test]
492 async fn get_none_from_not_existing_base() {
493 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
494 .await
495 .unwrap();
496 let package = repo.get_package_by_base("not_exist");
497 assert!(package.is_none());
498 }
499
500 #[tokio::test]
501 async fn get_gtk_by_name_and_version() {
502 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
503 .await
504 .unwrap();
505 let gtk = repo.get_package_by_name("mingw-w64-x86_64-gtk3").unwrap();
506 assert_eq!("mingw-w64-gtk3", gtk.base.as_ref().unwrap());
507 let gtk_name_and_version = format!("mingw-w64-x86_64-gtk3-{}", >k.version);
508 let gtk_from_ver = repo
509 .get_package_by_name_and_version(>k_name_and_version)
510 .unwrap();
511 assert_eq!(gtk, gtk_from_ver);
512 }
513
514 #[tokio::test]
515 async fn get_none_from_not_existing_name_and_version() {
516 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
517 .await
518 .unwrap();
519 let package = repo.get_package_by_name_and_version("not_exist-1.0.0");
520 assert!(package.is_none());
521 }
522
523 #[tokio::test]
524 async fn get_gtk_files_with_file_metadata_enabled() {
525 let repo = RepositoryBuilder::new("mingw64", "http://repo.msys2.org/mingw/x86_64")
526 .files_metadata(true)
527 .load()
528 .await
529 .unwrap();
530 assert!(!repo
531 .get_package_files("mingw-w64-x86_64-gtk3")
532 .unwrap()
533 .is_empty());
534 }
535
536 #[tokio::test]
537 async fn get_none_with_file_metadata_disabled() {
538 let repo = RepositoryBuilder::new("mingw64", "http://repo.msys2.org/mingw/x86_64")
539 .files_metadata(false)
540 .load()
541 .await
542 .unwrap();
543 assert!(repo.get_package_files("mingw-w64-x86_64-gtk3").is_none());
544 }
545
546 #[tokio::test]
547 async fn get_none_with_default() {
548 let repo = RepositoryBuilder::new("mingw64", "http://repo.msys2.org/mingw/x86_64")
549 .load()
550 .await
551 .unwrap();
552 assert!(repo.get_package_files("mingw-w64-x86_64-gtk3").is_none());
553 }
554
555 #[tokio::test]
556 async fn get_gtk_by_index_and_full_name() {
557 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
558 .await
559 .unwrap();
560 let gtk = &repo["mingw-w64-x86_64-gtk3"];
561 assert_eq!("mingw-w64-gtk3", gtk.base.as_ref().unwrap());
562 }
563
564 #[tokio::test]
565 async fn get_libwinpthread_by_csv_and_base_names() {
566 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
567 .await
568 .unwrap();
569 let a = repo
570 .get_package_by_name("mingw-w64-x86_64-libwinpthread-git")
571 .unwrap();
572 let b = repo
573 .get_package_by_name("mingw-w64-x86_64-libwinpthread")
574 .unwrap();
575 assert_eq!(1, b.linked_sources.len());
576 assert_eq!(a, b.linked_sources[0].as_ref())
577 }
578
579 #[tokio::test]
580 async fn get_gtk_by_index_and_base_name() {
581 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
582 .await
583 .unwrap();
584 let gtk = &repo["mingw-w64-gtk3"];
585 assert_eq!("mingw-w64-x86_64-gtk3", >k.name);
586 }
587
588 #[tokio::test]
589 async fn get_gtk_by_index_and_full_name_and_version() {
590 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
591 .await
592 .unwrap();
593 let gtk = repo.get_package_by_name("mingw-w64-x86_64-gtk3").unwrap();
594 assert_eq!("mingw-w64-gtk3", gtk.base.as_ref().unwrap());
595 let gtk_name_and_version = format!("mingw-w64-x86_64-gtk3-{}", >k.version);
596 let gtk_package = &repo[>k_name_and_version];
597 assert_eq!("mingw-w64-x86_64-gtk3", >k_package.name);
598 }
599
600 #[tokio::test]
601 async fn request_gtk_by_full_name() {
602 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
603 .await
604 .unwrap();
605 let bytes = repo
606 .request_package("mingw-w64-x86_64-gtk3")
607 .await
608 .unwrap()
609 .bytes()
610 .await
611 .unwrap();
612 assert!(!&bytes[..].is_empty());
613 }
614
615 #[tokio::test]
616 async fn request_gtk_by_full_name_and_version() {
617 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
618 .await
619 .unwrap();
620 let gtk = repo.get_package_by_name("mingw-w64-x86_64-gtk3").unwrap();
621 assert_eq!("mingw-w64-gtk3", gtk.base.as_ref().unwrap());
622 let gtk_name_and_version = format!("mingw-w64-x86_64-gtk3-{}", >k.version);
623 let bytes = repo
624 .request_package(>k_name_and_version)
625 .await
626 .unwrap()
627 .bytes()
628 .await
629 .unwrap();
630 assert!(!&bytes[..].is_empty());
631 }
632
633 #[tokio::test]
634 async fn request_gtk_by_base_name() {
635 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
636 .await
637 .unwrap();
638 let bytes = repo
639 .request_package("mingw-w64-gtk3")
640 .await
641 .unwrap()
642 .bytes()
643 .await
644 .unwrap();
645 assert!(!&bytes[..].is_empty());
646 }
647
648 #[tokio::test]
649 async fn iterator_should_have_gtk() {
650 let repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
651 .await
652 .unwrap();
653 for package in &repo {
654 if package.name == "mingw-w64-x86_64-gtk3" {
655 return;
656 }
657 }
658 unreachable!();
659 }
660
661 #[tokio::test]
662 async fn reload_should_not_fail() {
663 let mut repo = Repository::load("mingw64", "http://repo.msys2.org/mingw/x86_64")
664 .await
665 .unwrap();
666 repo.reload().await.unwrap();
667 }
668
669 #[tokio::test]
670 async fn should_report_progress() {
671 RepositoryBuilder::new("mingw64", "http://repo.msys2.org/mingw/x86_64")
672 .files_metadata(true)
673 .progress_listener(Box::new(|p| println!("{}", p)))
674 .load()
675 .await
676 .unwrap();
677 }
678
679 #[tokio::test]
680 #[should_panic]
681 async fn should_not_load_bad_repo() {
682 Repository::load("bad", "http://repo.msys2.org/mingw/x86_64")
683 .await
684 .unwrap();
685 }
686
687 #[test]
688 fn test_send() {
689 fn assert_send<T: Send>() {}
690 assert_send::<Package>();
691 assert_send::<PackageFiles>();
692 }
693
694 #[test]
695 fn test_sync() {
696 fn assert_sync<T: Sync>() {}
697 assert_sync::<Package>();
698 assert_sync::<PackageFiles>();
699 }
700}