normalize_package_index/index/
ubuntu.rs

1//! Ubuntu package index fetcher.
2//!
3//! Fetches package metadata from Ubuntu repositories by parsing
4//! Packages files from mirror indices.
5//!
6//! ## API Strategy
7//! - **fetch**: Uses Launchpad API
8//! - **fetch_versions**: Launchpad API
9//! - **search**: Filters cached Packages entries
10//! - **fetch_all**: Returns all packages from configured repos (cached 1 hour)
11//!
12//! ## Multi-repo Support
13//! ```rust,ignore
14//! use normalize_packages::index::ubuntu::{Ubuntu, UbuntuRepo};
15//!
16//! // All repos (default)
17//! let all = Ubuntu::all();
18//!
19//! // Noble (24.04 LTS) only
20//! let noble = Ubuntu::noble();
21//!
22//! // Jammy (22.04 LTS) only
23//! let jammy = Ubuntu::jammy();
24//!
25//! // Custom selection
26//! let custom = Ubuntu::with_repos(&[UbuntuRepo::NobleMain, UbuntuRepo::NobleUniverse]);
27//! ```
28
29use super::{IndexError, PackageIndex, PackageMeta, VersionMeta};
30use crate::cache;
31use flate2::read::GzDecoder;
32use rayon::prelude::*;
33use std::collections::HashMap;
34use std::io::{BufRead, BufReader, Cursor, Read};
35use std::time::Duration;
36
37/// Default cache TTL for package indices (1 hour).
38const INDEX_CACHE_TTL: Duration = Duration::from_secs(60 * 60);
39
40/// Ubuntu mirror URL.
41const UBUNTU_MIRROR: &str = "https://archive.ubuntu.com/ubuntu";
42
43/// Available Ubuntu repositories.
44#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
45pub enum UbuntuRepo {
46    // === Noble Numbat (24.04 LTS) ===
47    /// Noble main repository
48    NobleMain,
49    /// Noble restricted repository
50    NobleRestricted,
51    /// Noble universe repository
52    NobleUniverse,
53    /// Noble multiverse repository
54    NobleMultiverse,
55    /// Noble updates main
56    NobleUpdatesMain,
57    /// Noble updates universe
58    NobleUpdatesUniverse,
59    /// Noble security main
60    NobleSecurityMain,
61    /// Noble security universe
62    NobleSecurityUniverse,
63    /// Noble backports main
64    NobleBackportsMain,
65    /// Noble backports universe
66    NobleBackportsUniverse,
67
68    // === Jammy Jellyfish (22.04 LTS) ===
69    /// Jammy main repository
70    JammyMain,
71    /// Jammy restricted repository
72    JammyRestricted,
73    /// Jammy universe repository
74    JammyUniverse,
75    /// Jammy multiverse repository
76    JammyMultiverse,
77    /// Jammy updates main
78    JammyUpdatesMain,
79    /// Jammy updates universe
80    JammyUpdatesUniverse,
81    /// Jammy security main
82    JammySecurityMain,
83    /// Jammy security universe
84    JammySecurityUniverse,
85    /// Jammy backports main
86    JammyBackportsMain,
87    /// Jammy backports universe
88    JammyBackportsUniverse,
89
90    // === Oracular Oriole (24.10) ===
91    /// Oracular main repository
92    OracularMain,
93    /// Oracular universe repository
94    OracularUniverse,
95}
96
97impl UbuntuRepo {
98    /// Get the distribution and component parts.
99    fn parts(&self) -> (&'static str, &'static str) {
100        match self {
101            // Noble 24.04 LTS
102            Self::NobleMain => ("noble", "main"),
103            Self::NobleRestricted => ("noble", "restricted"),
104            Self::NobleUniverse => ("noble", "universe"),
105            Self::NobleMultiverse => ("noble", "multiverse"),
106            Self::NobleUpdatesMain => ("noble-updates", "main"),
107            Self::NobleUpdatesUniverse => ("noble-updates", "universe"),
108            Self::NobleSecurityMain => ("noble-security", "main"),
109            Self::NobleSecurityUniverse => ("noble-security", "universe"),
110            Self::NobleBackportsMain => ("noble-backports", "main"),
111            Self::NobleBackportsUniverse => ("noble-backports", "universe"),
112
113            // Jammy 22.04 LTS
114            Self::JammyMain => ("jammy", "main"),
115            Self::JammyRestricted => ("jammy", "restricted"),
116            Self::JammyUniverse => ("jammy", "universe"),
117            Self::JammyMultiverse => ("jammy", "multiverse"),
118            Self::JammyUpdatesMain => ("jammy-updates", "main"),
119            Self::JammyUpdatesUniverse => ("jammy-updates", "universe"),
120            Self::JammySecurityMain => ("jammy-security", "main"),
121            Self::JammySecurityUniverse => ("jammy-security", "universe"),
122            Self::JammyBackportsMain => ("jammy-backports", "main"),
123            Self::JammyBackportsUniverse => ("jammy-backports", "universe"),
124
125            // Oracular 24.10
126            Self::OracularMain => ("oracular", "main"),
127            Self::OracularUniverse => ("oracular", "universe"),
128        }
129    }
130
131    /// Get the Packages.gz URL for this repository.
132    fn packages_url(&self) -> String {
133        let (dist, component) = self.parts();
134        format!(
135            "{}/dists/{}/{}/binary-amd64/Packages.gz",
136            UBUNTU_MIRROR, dist, component
137        )
138    }
139
140    /// Get the repository name for tagging.
141    pub fn name(&self) -> &'static str {
142        match self {
143            Self::NobleMain => "noble-main",
144            Self::NobleRestricted => "noble-restricted",
145            Self::NobleUniverse => "noble-universe",
146            Self::NobleMultiverse => "noble-multiverse",
147            Self::NobleUpdatesMain => "noble-updates-main",
148            Self::NobleUpdatesUniverse => "noble-updates-universe",
149            Self::NobleSecurityMain => "noble-security-main",
150            Self::NobleSecurityUniverse => "noble-security-universe",
151            Self::NobleBackportsMain => "noble-backports-main",
152            Self::NobleBackportsUniverse => "noble-backports-universe",
153
154            Self::JammyMain => "jammy-main",
155            Self::JammyRestricted => "jammy-restricted",
156            Self::JammyUniverse => "jammy-universe",
157            Self::JammyMultiverse => "jammy-multiverse",
158            Self::JammyUpdatesMain => "jammy-updates-main",
159            Self::JammyUpdatesUniverse => "jammy-updates-universe",
160            Self::JammySecurityMain => "jammy-security-main",
161            Self::JammySecurityUniverse => "jammy-security-universe",
162            Self::JammyBackportsMain => "jammy-backports-main",
163            Self::JammyBackportsUniverse => "jammy-backports-universe",
164
165            Self::OracularMain => "oracular-main",
166            Self::OracularUniverse => "oracular-universe",
167        }
168    }
169
170    /// All available repositories.
171    pub fn all() -> &'static [UbuntuRepo] {
172        &[
173            Self::NobleMain,
174            Self::NobleRestricted,
175            Self::NobleUniverse,
176            Self::NobleMultiverse,
177            Self::NobleUpdatesMain,
178            Self::NobleUpdatesUniverse,
179            Self::NobleSecurityMain,
180            Self::NobleSecurityUniverse,
181            Self::NobleBackportsMain,
182            Self::NobleBackportsUniverse,
183            Self::JammyMain,
184            Self::JammyRestricted,
185            Self::JammyUniverse,
186            Self::JammyMultiverse,
187            Self::JammyUpdatesMain,
188            Self::JammyUpdatesUniverse,
189            Self::JammySecurityMain,
190            Self::JammySecurityUniverse,
191            Self::JammyBackportsMain,
192            Self::JammyBackportsUniverse,
193            Self::OracularMain,
194            Self::OracularUniverse,
195        ]
196    }
197
198    /// Noble (24.04 LTS) repositories only.
199    pub fn noble() -> &'static [UbuntuRepo] {
200        &[
201            Self::NobleMain,
202            Self::NobleRestricted,
203            Self::NobleUniverse,
204            Self::NobleMultiverse,
205            Self::NobleUpdatesMain,
206            Self::NobleUpdatesUniverse,
207            Self::NobleSecurityMain,
208            Self::NobleSecurityUniverse,
209            Self::NobleBackportsMain,
210            Self::NobleBackportsUniverse,
211        ]
212    }
213
214    /// Jammy (22.04 LTS) repositories only.
215    pub fn jammy() -> &'static [UbuntuRepo] {
216        &[
217            Self::JammyMain,
218            Self::JammyRestricted,
219            Self::JammyUniverse,
220            Self::JammyMultiverse,
221            Self::JammyUpdatesMain,
222            Self::JammyUpdatesUniverse,
223            Self::JammySecurityMain,
224            Self::JammySecurityUniverse,
225            Self::JammyBackportsMain,
226            Self::JammyBackportsUniverse,
227        ]
228    }
229
230    /// LTS releases only (Noble + Jammy).
231    pub fn lts() -> &'static [UbuntuRepo] {
232        &[
233            Self::NobleMain,
234            Self::NobleUniverse,
235            Self::NobleUpdatesMain,
236            Self::NobleUpdatesUniverse,
237            Self::JammyMain,
238            Self::JammyUniverse,
239            Self::JammyUpdatesMain,
240            Self::JammyUpdatesUniverse,
241        ]
242    }
243
244    /// Main repositories only (no universe/multiverse).
245    pub fn main_only() -> &'static [UbuntuRepo] {
246        &[
247            Self::NobleMain,
248            Self::NobleUpdatesMain,
249            Self::NobleSecurityMain,
250            Self::JammyMain,
251            Self::JammyUpdatesMain,
252            Self::JammySecurityMain,
253            Self::OracularMain,
254        ]
255    }
256}
257
258/// Ubuntu package index fetcher with configurable repositories.
259pub struct Ubuntu {
260    repos: Vec<UbuntuRepo>,
261}
262
263impl Ubuntu {
264    /// Create a fetcher with all repositories.
265    pub fn all() -> Self {
266        Self {
267            repos: UbuntuRepo::all().to_vec(),
268        }
269    }
270
271    /// Create a fetcher with Noble (24.04 LTS) repositories.
272    pub fn noble() -> Self {
273        Self {
274            repos: UbuntuRepo::noble().to_vec(),
275        }
276    }
277
278    /// Create a fetcher with Jammy (22.04 LTS) repositories.
279    pub fn jammy() -> Self {
280        Self {
281            repos: UbuntuRepo::jammy().to_vec(),
282        }
283    }
284
285    /// Create a fetcher with LTS releases only.
286    pub fn lts() -> Self {
287        Self {
288            repos: UbuntuRepo::lts().to_vec(),
289        }
290    }
291
292    /// Create a fetcher with main repositories only.
293    pub fn main_only() -> Self {
294        Self {
295            repos: UbuntuRepo::main_only().to_vec(),
296        }
297    }
298
299    /// Create a fetcher with custom repository selection.
300    pub fn with_repos(repos: &[UbuntuRepo]) -> Self {
301        Self {
302            repos: repos.to_vec(),
303        }
304    }
305
306    /// Parse a Packages file in Debian control format.
307    fn parse_control<R: Read>(reader: R, repo: UbuntuRepo) -> Vec<PackageMeta> {
308        let reader = BufReader::new(reader);
309        let mut packages = Vec::new();
310        let mut current: Option<PackageBuilder> = None;
311
312        for line in reader.lines().map_while(Result::ok) {
313            if line.is_empty() {
314                if let Some(builder) = current.take() {
315                    if let Some(pkg) = builder.build(repo) {
316                        packages.push(pkg);
317                    }
318                }
319                continue;
320            }
321
322            if line.starts_with(' ') || line.starts_with('\t') {
323                continue;
324            }
325
326            if let Some((key, value)) = line.split_once(':') {
327                let key = key.trim();
328                let value = value.trim();
329
330                let builder = current.get_or_insert_with(PackageBuilder::new);
331
332                match key {
333                    "Package" => builder.name = Some(value.to_string()),
334                    "Version" => builder.version = Some(value.to_string()),
335                    "Description" => builder.description = Some(value.to_string()),
336                    "Homepage" => builder.homepage = Some(value.to_string()),
337                    "Vcs-Git" | "Vcs-Browser" => {
338                        if builder.repository.is_none() {
339                            builder.repository = Some(value.to_string());
340                        }
341                    }
342                    "Filename" => builder.filename = Some(value.to_string()),
343                    "SHA256" => builder.sha256 = Some(value.to_string()),
344                    "Depends" => builder.depends = Some(value.to_string()),
345                    "Size" => builder.size = value.parse().ok(),
346                    _ => {}
347                }
348            }
349        }
350
351        if let Some(builder) = current {
352            if let Some(pkg) = builder.build(repo) {
353                packages.push(pkg);
354            }
355        }
356
357        packages
358    }
359
360    /// Load packages from a single repository.
361    fn load_repo(repo: UbuntuRepo) -> Result<Vec<PackageMeta>, IndexError> {
362        let url = repo.packages_url();
363
364        let (data, _was_cached) = cache::fetch_with_cache(
365            "ubuntu",
366            &format!("packages-{}", repo.name()),
367            &url,
368            INDEX_CACHE_TTL,
369        )
370        .map_err(IndexError::Network)?;
371
372        let reader: Box<dyn Read> = if data.len() >= 2 && data[0] == 0x1f && data[1] == 0x8b {
373            Box::new(GzDecoder::new(Cursor::new(data)))
374        } else {
375            Box::new(Cursor::new(data))
376        };
377
378        Ok(Self::parse_control(reader, repo))
379    }
380
381    /// Load packages from all configured repositories in parallel.
382    fn load_packages(&self) -> Result<Vec<PackageMeta>, IndexError> {
383        let results: Vec<_> = self
384            .repos
385            .par_iter()
386            .map(|&repo| Self::load_repo(repo))
387            .collect();
388
389        let mut packages = Vec::new();
390        for result in results {
391            match result {
392                Ok(pkgs) => packages.extend(pkgs),
393                Err(e) => {
394                    eprintln!("Warning: failed to load Ubuntu repo: {}", e);
395                }
396            }
397        }
398
399        Ok(packages)
400    }
401}
402
403impl PackageIndex for Ubuntu {
404    fn ecosystem(&self) -> &'static str {
405        "ubuntu"
406    }
407
408    fn display_name(&self) -> &'static str {
409        "Ubuntu"
410    }
411
412    fn fetch(&self, name: &str) -> Result<PackageMeta, IndexError> {
413        // Use Launchpad API for single package lookup
414        let url = format!(
415            "https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name={}&exact_match=true",
416            urlencoding::encode(name)
417        );
418
419        let response: serde_json::Value = ureq::get(&url).call()?.into_json()?;
420
421        let entries = response["entries"]
422            .as_array()
423            .ok_or_else(|| IndexError::NotFound(name.to_string()))?;
424
425        let latest = entries
426            .first()
427            .ok_or_else(|| IndexError::NotFound(name.to_string()))?;
428
429        Ok(PackageMeta {
430            name: latest["source_package_name"]
431                .as_str()
432                .unwrap_or(name)
433                .to_string(),
434            version: latest["source_package_version"]
435                .as_str()
436                .unwrap_or("unknown")
437                .to_string(),
438            description: None,
439            homepage: None,
440            repository: None,
441            license: None,
442            binaries: Vec::new(),
443            keywords: Vec::new(),
444            maintainers: Vec::new(),
445            published: None,
446            downloads: None,
447            archive_url: None,
448            checksum: None,
449            extra: Default::default(),
450        })
451    }
452
453    fn fetch_versions(&self, name: &str) -> Result<Vec<VersionMeta>, IndexError> {
454        let url = format!(
455            "https://api.launchpad.net/1.0/ubuntu/+archive/primary?ws.op=getPublishedSources&source_name={}&exact_match=true",
456            urlencoding::encode(name)
457        );
458
459        let response: serde_json::Value = ureq::get(&url).call()?.into_json()?;
460
461        let entries = response["entries"]
462            .as_array()
463            .ok_or_else(|| IndexError::NotFound(name.to_string()))?;
464
465        if entries.is_empty() {
466            return Err(IndexError::NotFound(name.to_string()));
467        }
468
469        Ok(entries
470            .iter()
471            .filter_map(|e| {
472                Some(VersionMeta {
473                    version: e["source_package_version"].as_str()?.to_string(),
474                    released: None,
475                    yanked: false,
476                })
477            })
478            .collect())
479    }
480
481    fn supports_fetch_all(&self) -> bool {
482        true
483    }
484
485    fn fetch_all(&self) -> Result<Vec<PackageMeta>, IndexError> {
486        self.load_packages()
487    }
488
489    fn search(&self, query: &str) -> Result<Vec<PackageMeta>, IndexError> {
490        let packages = self.load_packages()?;
491        let query_lower = query.to_lowercase();
492
493        Ok(packages
494            .into_iter()
495            .filter(|pkg| {
496                pkg.name.to_lowercase().contains(&query_lower)
497                    || pkg
498                        .description
499                        .as_ref()
500                        .map(|d| d.to_lowercase().contains(&query_lower))
501                        .unwrap_or(false)
502            })
503            .collect())
504    }
505}
506
507#[derive(Default)]
508struct PackageBuilder {
509    name: Option<String>,
510    version: Option<String>,
511    description: Option<String>,
512    homepage: Option<String>,
513    repository: Option<String>,
514    filename: Option<String>,
515    sha256: Option<String>,
516    depends: Option<String>,
517    size: Option<u64>,
518}
519
520impl PackageBuilder {
521    fn new() -> Self {
522        Self::default()
523    }
524
525    fn build(self, repo: UbuntuRepo) -> Option<PackageMeta> {
526        let mut extra = HashMap::new();
527
528        if let Some(deps) = self.depends {
529            let parsed_deps: Vec<String> = deps
530                .split(',')
531                .map(|d| {
532                    d.trim()
533                        .split_once(' ')
534                        .map(|(name, _)| name)
535                        .unwrap_or(d.trim())
536                        .to_string()
537                })
538                .filter(|d| !d.is_empty())
539                .collect();
540            extra.insert(
541                "depends".to_string(),
542                serde_json::Value::Array(
543                    parsed_deps
544                        .into_iter()
545                        .map(serde_json::Value::String)
546                        .collect(),
547                ),
548            );
549        }
550
551        if let Some(size) = self.size {
552            extra.insert("size".to_string(), serde_json::Value::Number(size.into()));
553        }
554
555        extra.insert(
556            "source_repo".to_string(),
557            serde_json::Value::String(repo.name().to_string()),
558        );
559
560        Some(PackageMeta {
561            name: self.name?,
562            version: self.version?,
563            description: self.description,
564            homepage: self.homepage,
565            repository: self.repository,
566            license: None,
567            binaries: Vec::new(),
568            archive_url: self.filename.map(|f| format!("{}/{}", UBUNTU_MIRROR, f)),
569            checksum: self.sha256.map(|h| format!("sha256:{}", h)),
570            keywords: Vec::new(),
571            maintainers: Vec::new(),
572            published: None,
573            downloads: None,
574            extra,
575        })
576    }
577}
578
579mod urlencoding {
580    pub fn encode(s: &str) -> String {
581        let mut result = String::with_capacity(s.len() * 3);
582        for c in s.chars() {
583            match c {
584                'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => result.push(c),
585                _ => {
586                    for b in c.to_string().bytes() {
587                        result.push_str(&format!("%{:02X}", b));
588                    }
589                }
590            }
591        }
592        result
593    }
594}
normalize_package_index/index/ubuntu.rs

normalize_package_index/index/
ubuntu.rs
