normalize_package_index/index/
ubuntu.rs

1//! Ubuntu package index fetcher.
2//!
3//! Fetches package metadata from Ubuntu repositories by parsing
4//! Packages files from mirror indices.
5//!
6//! ## API Strategy
7//! - **fetch**: Uses Launchpad API
8//! - **fetch_versions**: Launchpad API
9//! - **search**: Filters cached Packages entries
10//! - **fetch_all**: Returns all packages from configured repos (cached 1 hour)
11//!
12//! ## Multi-repo Support
13//! ```rust,ignore
14//! use normalize_packages::index::ubuntu::{Ubuntu, UbuntuRepo};
15//!
16//! // All repos (default)
17//! let all = Ubuntu::all();
18//!
19//! // Noble (24.04 LTS) only
20//! let noble = Ubuntu::noble();
21//!
22//! // Jammy (22.04 LTS) only
23//! let jammy = Ubuntu::jammy();
24//!
25//! // Custom selection
26//! let custom = Ubuntu::with_repos(&[UbuntuRepo::NobleMain, UbuntuRepo::NobleUniverse]);
27//! ```
28
29use super::{IndexError, PackageIndex, PackageMeta, VersionMeta};
30use crate::cache;
31use flate2::read::GzDecoder;
32use rayon::prelude::*;
33use std::collections::HashMap;
34use std::io::{BufRead, BufReader, Cursor, Read};
35use std::time::Duration;
36
37/// Default cache TTL for package indices (1 hour).
38const INDEX_CACHE_TTL: Duration = Duration::from_secs(60 * 60);
39
40/// Ubuntu mirror URL.
41const UBUNTU_MIRROR: &str = "https://archive.ubuntu.com/ubuntu";
42
43/// Available Ubuntu repositories.
44#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
45pub enum UbuntuRepo {
46    // === Noble Numbat (24.04 LTS) ===
47    /// Noble main repository
48    NobleMain,
49    /// Noble restricted repository
50    NobleRestricted,
51    /// Noble universe repository
52    NobleUniverse,
53    /// Noble multiverse repository
54    NobleMultiverse,
55    /// Noble updates main
56    NobleUpdatesMain,
57    /// Noble updates universe
58    NobleUpdatesUniverse,
59    /// Noble security main
60    NobleSecurityMain,
61    /// Noble security universe
62    NobleSecurityUniverse,
63    /// Noble backports main
64    NobleBackportsMain,
65    /// Noble backports universe
66    NobleBackportsUniverse,
67
68    // === Jammy Jellyfish (22.04 LTS) ===
69    /// Jammy main repository
70    JammyMain,
71    /// Jammy restricted repository
72    JammyRestricted,
73    /// Jammy universe repository
74    JammyUniverse,
75    /// Jammy multiverse repository
76    JammyMultiverse,
77    /// Jammy updates main
78    JammyUpdatesMain,
79    /// Jammy updates universe
80    JammyUpdatesUniverse,
81    /// Jammy security main
82    JammySecurityMain,
83    /// Jammy security universe
84    JammySecurityUniverse,
85    /// Jammy backports main
86    JammyBackportsMain,
87    /// Jammy backports universe
88    JammyBackportsUniverse,
89
90    // === Oracular Oriole (24.10) ===
91    /// Oracular main repository
92    OracularMain,
93    /// Oracular universe repository
94    OracularUniverse,
95}
96
97struct DistComponent {
98    dist: &'static str,
99    component: &'static str,
100}
101
102impl UbuntuRepo {
103    /// Get the distribution and component parts.
104    fn parts(&self) -> DistComponent {
105        let (dist, component) = match self {
106            // Noble 24.04 LTS
107            Self::NobleMain => ("noble", "main"),
108            Self::NobleRestricted => ("noble", "restricted"),
109            Self::NobleUniverse => ("noble", "universe"),
110            Self::NobleMultiverse => ("noble", "multiverse"),
111            Self::NobleUpdatesMain => ("noble-updates", "main"),
112            Self::NobleUpdatesUniverse => ("noble-updates", "universe"),
113            Self::NobleSecurityMain => ("noble-security", "main"),
114            Self::NobleSecurityUniverse => ("noble-security", "universe"),
115            Self::NobleBackportsMain => ("noble-backports", "main"),
116            Self::NobleBackportsUniverse => ("noble-backports", "universe"),
117
118            // Jammy 22.04 LTS
119            Self::JammyMain => ("jammy", "main"),
120            Self::JammyRestricted => ("jammy", "restricted"),
121            Self::JammyUniverse => ("jammy", "universe"),
122            Self::JammyMultiverse => ("jammy", "multiverse"),
123            Self::JammyUpdatesMain => ("jammy-updates", "main"),
124            Self::JammyUpdatesUniverse => ("jammy-updates", "universe"),
125            Self::JammySecurityMain => ("jammy-security", "main"),
126            Self::JammySecurityUniverse => ("jammy-security", "universe"),
127            Self::JammyBackportsMain => ("jammy-backports", "main"),
128            Self::JammyBackportsUniverse => ("jammy-backports", "universe"),
129
130            // Oracular 24.10
131            Self::OracularMain => ("oracular", "main"),
132            Self::OracularUniverse => ("oracular", "universe"),
133        };
134        DistComponent { dist, component }
135    }
136
137    /// Get the Packages.gz URL for this repository.
138    fn packages_url(&self) -> String {
139        let parts = self.parts();
140        format!(
141            "{}/dists/{}/{}/binary-amd64/Packages.gz",
142            UBUNTU_MIRROR, parts.dist, parts.component
143        )
144    }
145
146    /// Get the repository name for tagging.
147    pub fn name(&self) -> &'static str {
148        match self {
149            Self::NobleMain => "noble-main",
150            Self::NobleRestricted => "noble-restricted",
151            Self::NobleUniverse => "noble-universe",
152            Self::NobleMultiverse => "noble-multiverse",
153            Self::NobleUpdatesMain => "noble-updates-main",
154            Self::NobleUpdatesUniverse => "noble-updates-universe",
155            Self::NobleSecurityMain => "noble-security-main",
156            Self::NobleSecurityUniverse => "noble-security-universe",
157            Self::NobleBackportsMain => "noble-backports-main",
158            Self::NobleBackportsUniverse => "noble-backports-universe",
159
160            Self::JammyMain => "jammy-main",
161            Self::JammyRestricted => "jammy-restricted",
162            Self::JammyUniverse => "jammy-universe",
163            Self::JammyMultiverse => "jammy-multiverse",
164            Self::JammyUpdatesMain => "jammy-updates-main",
165            Self::JammyUpdatesUniverse => "jammy-updates-universe",
166            Self::JammySecurityMain => "jammy-security-main",
167            Self::JammySecurityUniverse => "jammy-security-universe",
168            Self::JammyBackportsMain => "jammy-backports-main",
169            Self::JammyBackportsUniverse => "jammy-backports-universe",
170
171            Self::OracularMain => "oracular-main",
172            Self::OracularUniverse => "oracular-universe",
173        }
174    }
175
176    /// All available repositories.
177    pub fn all() -> &'static [UbuntuRepo] {
178        &[
179            Self::NobleMain,
180            Self::NobleRestricted,
181            Self::NobleUniverse,
182            Self::NobleMultiverse,
183            Self::NobleUpdatesMain,
184            Self::NobleUpdatesUniverse,
185            Self::NobleSecurityMain,
186            Self::NobleSecurityUniverse,
187            Self::NobleBackportsMain,
188            Self::NobleBackportsUniverse,
189            Self::JammyMain,
190            Self::JammyRestricted,
191            Self::JammyUniverse,
192            Self::JammyMultiverse,
193            Self::JammyUpdatesMain,
194            Self::JammyUpdatesUniverse,
195            Self::JammySecurityMain,
196            Self::JammySecurityUniverse,
197            Self::JammyBackportsMain,
198            Self::JammyBackportsUniverse,
199            Self::OracularMain,
200            Self::OracularUniverse,
201        ]
202    }
203
204    /// Noble (24.04 LTS) repositories only.
205    pub fn noble() -> &'static [UbuntuRepo] {
206        &[
207            Self::NobleMain,
208            Self::NobleRestricted,
209            Self::NobleUniverse,
210            Self::NobleMultiverse,
211            Self::NobleUpdatesMain,
212            Self::NobleUpdatesUniverse,
213            Self::NobleSecurityMain,
214            Self::NobleSecurityUniverse,
215            Self::NobleBackportsMain,
216            Self::NobleBackportsUniverse,
217        ]
218    }
219
220    /// Jammy (22.04 LTS) repositories only.
221    pub fn jammy() -> &'static [UbuntuRepo] {
222        &[
223            Self::JammyMain,
224            Self::JammyRestricted,
225            Self::JammyUniverse,
226            Self::JammyMultiverse,
227            Self::JammyUpdatesMain,
228            Self::JammyUpdatesUniverse,
229            Self::JammySecurityMain,
230            Self::JammySecurityUniverse,
231            Self::JammyBackportsMain,
232            Self::JammyBackportsUniverse,
233        ]
234    }
235
236    /// LTS releases only (Noble + Jammy).
237    pub fn lts() -> &'static [UbuntuRepo] {
238        &[
239            Self::NobleMain,
240            Self::NobleUniverse,
241            Self::NobleUpdatesMain,
242            Self::NobleUpdatesUniverse,
243            Self::JammyMain,
244            Self::JammyUniverse,
245            Self::JammyUpdatesMain,
246            Self::JammyUpdatesUniverse,
247        ]
248    }
249
250    /// Main repositories only (no universe/multiverse).
251    pub fn main_only() -> &'static [UbuntuRepo] {
252        &[
253            Self::NobleMain,
254            Self::NobleUpdatesMain,
255            Self::NobleSecurityMain,
256            Self::JammyMain,
257            Self::JammyUpdatesMain,
258            Self::JammySecurityMain,
259            Self::OracularMain,
260        ]
261    }
262}
263
264/// Ubuntu package index fetcher with configurable repositories.
265pub struct Ubuntu {
266    repos: Vec<UbuntuRepo>,
267}
268
269impl Ubuntu {
270    /// Create a fetcher with all repositories.
271    pub fn all() -> Self {
272        Self {
273            repos: UbuntuRepo::all().to_vec(),
274        }
275    }
276
277    /// Create a fetcher with Noble (24.04 LTS) repositories.
278    pub fn noble() -> Self {
279        Self {
280            repos: UbuntuRepo::noble().to_vec(),
281        }
282    }
283
284    /// Create a fetcher with Jammy (22.04 LTS) repositories.
285    pub fn jammy() -> Self {
286        Self {
287            repos: UbuntuRepo::jammy().to_vec(),
288        }
289    }
290
291    /// Create a fetcher with LTS releases only.
292    pub fn lts() -> Self {
293        Self {
294            repos: UbuntuRepo::lts().to_vec(),
295        }
296    }
297
298    /// Create a fetcher with main repositories only.
299    pub fn main_only() -> Self {
300        Self {
301            repos: UbuntuRepo::main_only().to_vec(),
302        }
303    }
304
305    /// Create a fetcher with custom repository selection.
306    pub fn with_repos(repos: &[UbuntuRepo]) -> Self {
307        Self {
308            repos: repos.to_vec(),
309        }
310    }
311
312    /// Parse a Packages file in Debian control format.
313    fn parse_control<R: Read>(reader: R, repo: UbuntuRepo) -> Vec<PackageMeta> {
314        let reader = BufReader::new(reader);
315        let mut packages = Vec::new();
316        let mut current: Option<PackageBuilder> = None;
317
318        for line in reader.lines().map_while(Result::ok) {
319            if line.is_empty() {
320                if let Some(builder) = current.take()
321                    && let Some(pkg) = builder.build(repo)
322                {
323                    packages.push(pkg);
324                }
325                continue;
326            }
327
328            if line.starts_with(' ') || line.starts_with('\t') {
329                continue;
330            }
331
332            if let Some((key, value)) = line.split_once(':') {
333                let key = key.trim();
334                let value = value.trim();
335
336                let builder = current.get_or_insert_with(PackageBuilder::new);
337
338                match key {
339                    "Package" => builder.name = Some(value.to_string()),
340                    "Version" => builder.version = Some(value.to_string()),
341                    "Description" => builder.description = Some(value.to_string()),
342                    "Homepage" => builder.homepage = Some(value.to_string()),
343                    "Vcs-Git" | "Vcs-Browser" => {
344                        if builder.repository.is_none() {
345                            builder.repository = Some(value.to_string());
346                        }
347                    }
348                    "Filename" => builder.filename = Some(value.to_string()),
349                    "SHA256" => builder.sha256 = Some(value.to_string()),
350                    "Depends" => builder.depends = Some(value.to_string()),
351                    "Size" => builder.size = value.parse().ok(),
352                    _ => {}
353                }
354            }
355        }
356
357        if let Some(builder) = current
358            && let Some(pkg) = builder.build(repo)
359        {
360            packages.push(pkg);
361        }
362
363        packages
364    }
365
366    /// Load packages from a single repository.
367    fn load_repo(repo: UbuntuRepo) -> Result<Vec<PackageMeta>, IndexError> {
368        let url = repo.packages_url();
369
370        let (data, _was_cached) = cache::fetch_with_cache(
371            "ubuntu",
372            &format!("packages-{}", repo.name()),
373            &url,
374            INDEX_CACHE_TTL,
375        )
376        .map_err(IndexError::Network)?;
377
378        let reader: Box<dyn Read> = if data.len() >= 2 && data[0] == 0x1f && data[1] == 0x8b {
379            Box::new(GzDecoder::new(Cursor::new(data)))
380        } else {
381            Box::new(Cursor::new(data))
382        };
383
384        Ok(Self::parse_control(reader, repo))
385    }
386
387    /// Load packages from all configured repositories in parallel.
388    fn load_packages(&self) -> Result<Vec<PackageMeta>, IndexError> {
389        let results: Vec<_> = self
390            .repos
391            .par_iter()
392            .map(|&repo| Self::load_repo(repo))
393            .collect();
394
395        let mut packages = Vec::new();
396        for result in results {
397            match result {
398                Ok(pkgs) => packages.extend(pkgs),
399                Err(e) => {
400                    tracing::warn!("failed to load Ubuntu repo: {}", e);
401                }
402            }
403        }
404
405        Ok(packages)
406    }
407}
408
409impl PackageIndex for Ubuntu {
410    fn ecosystem(&self) -> &'static str {
411        "ubuntu"
412    }
413
414    fn display_name(&self) -> &'static str {
415        "Ubuntu"
416    }
417
418    fn fetch(&self, name: &str) -> Result<PackageMeta, IndexError> {
419        // Use Launchpad API for single package lookup
420        let url = format!(
421            "https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name={}&exact_match=true",
422            urlencoding::encode(name)
423        );
424
425        let response: serde_json::Value = ureq::get(&url).call()?.into_json()?;
426
427        let entries = response["entries"]
428            .as_array()
429            .ok_or_else(|| IndexError::NotFound(name.to_string()))?;
430
431        let latest = entries
432            .first()
433            .ok_or_else(|| IndexError::NotFound(name.to_string()))?;
434
435        Ok(PackageMeta {
436            name: latest["source_package_name"]
437                .as_str()
438                .unwrap_or(name)
439                .to_string(),
440            version: latest["source_package_version"]
441                .as_str()
442                .unwrap_or("unknown")
443                .to_string(),
444            description: None,
445            homepage: None,
446            repository: None,
447            license: None,
448            binaries: Vec::new(),
449            keywords: Vec::new(),
450            maintainers: Vec::new(),
451            published: None,
452            downloads: None,
453            archive_url: None,
454            checksum: None,
455            extra: Default::default(),
456        })
457    }
458
459    fn fetch_versions(&self, name: &str) -> Result<Vec<VersionMeta>, IndexError> {
460        let url = format!(
461            "https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name={}&exact_match=true",
462            urlencoding::encode(name)
463        );
464
465        let response: serde_json::Value = ureq::get(&url).call()?.into_json()?;
466
467        let entries = response["entries"]
468            .as_array()
469            .ok_or_else(|| IndexError::NotFound(name.to_string()))?;
470
471        if entries.is_empty() {
472            return Err(IndexError::NotFound(name.to_string()));
473        }
474
475        Ok(entries
476            .iter()
477            .filter_map(|e| {
478                Some(VersionMeta {
479                    version: e["source_package_version"].as_str()?.to_string(),
480                    released: None,
481                    yanked: false,
482                })
483            })
484            .collect())
485    }
486
487    fn supports_fetch_all(&self) -> bool {
488        true
489    }
490
491    fn fetch_all(&self) -> Result<Vec<PackageMeta>, IndexError> {
492        self.load_packages()
493    }
494
495    fn search(&self, query: &str) -> Result<Vec<PackageMeta>, IndexError> {
496        let packages = self.load_packages()?;
497        let query_lower = query.to_lowercase();
498
499        Ok(packages
500            .into_iter()
501            .filter(|pkg| {
502                pkg.name.to_lowercase().contains(&query_lower)
503                    || pkg
504                        .description
505                        .as_ref()
506                        .map(|d| d.to_lowercase().contains(&query_lower))
507                        .unwrap_or(false)
508            })
509            .collect())
510    }
511}
512
513#[derive(Default)]
514struct PackageBuilder {
515    name: Option<String>,
516    version: Option<String>,
517    description: Option<String>,
518    homepage: Option<String>,
519    repository: Option<String>,
520    filename: Option<String>,
521    sha256: Option<String>,
522    depends: Option<String>,
523    size: Option<u64>,
524}
525
526impl PackageBuilder {
527    fn new() -> Self {
528        Self::default()
529    }
530
531    fn build(self, repo: UbuntuRepo) -> Option<PackageMeta> {
532        let mut extra = HashMap::new();
533
534        if let Some(deps) = self.depends {
535            let parsed_deps: Vec<String> = deps
536                .split(',')
537                .map(|d| {
538                    d.trim()
539                        .split_once(' ')
540                        .map(|(name, _)| name)
541                        .unwrap_or(d.trim())
542                        .to_string()
543                })
544                .filter(|d| !d.is_empty())
545                .collect();
546            extra.insert(
547                "depends".to_string(),
548                serde_json::Value::Array(
549                    parsed_deps
550                        .into_iter()
551                        .map(serde_json::Value::String)
552                        .collect(),
553                ),
554            );
555        }
556
557        if let Some(size) = self.size {
558            extra.insert("size".to_string(), serde_json::Value::Number(size.into()));
559        }
560
561        extra.insert(
562            "source_repo".to_string(),
563            serde_json::Value::String(repo.name().to_string()),
564        );
565
566        Some(PackageMeta {
567            name: self.name?,
568            version: self.version?,
569            description: self.description,
570            homepage: self.homepage,
571            repository: self.repository,
572            license: None,
573            binaries: Vec::new(),
574            archive_url: self.filename.map(|f| format!("{}/{}", UBUNTU_MIRROR, f)),
575            checksum: self.sha256.map(|h| format!("sha256:{}", h)),
576            keywords: Vec::new(),
577            maintainers: Vec::new(),
578            published: None,
579            downloads: None,
580            extra,
581        })
582    }
583}
584
585mod urlencoding {
586    pub fn encode(s: &str) -> String {
587        let mut result = String::with_capacity(s.len() * 3);
588        for c in s.chars() {
589            match c {
590                'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => result.push(c),
591                _ => {
592                    for b in c.to_string().bytes() {
593                        result.push_str(&format!("%{:02X}", b));
594                    }
595                }
596            }
597        }
598        result
599    }
600}
normalize_package_index/index/ubuntu.rs

normalize_package_index/index/
ubuntu.rs
