1use crate::config::PkgsrcEnv;
41use crate::sandbox::{SandboxScope, wait_output_with_shutdown};
42use crate::tui::{MultiProgress, REFRESH_INTERVAL, format_duration};
43use crate::{Config, Interrupted, Sandbox};
44use anyhow::{Context, Result, bail};
45use crossterm::event;
46use indexmap::IndexMap;
47use petgraph::graphmap::DiGraphMap;
48use pkgsrc::{Depend, PkgName, PkgPath, ScanIndex};
49use rayon::prelude::*;
50use std::collections::{HashMap, HashSet};
51use std::io::BufReader;
52use std::sync::atomic::{AtomicBool, Ordering};
53use std::sync::{Arc, Mutex};
54use tracing::{debug, error, info, info_span, trace, warn};
55
56#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
58pub enum SkipReason {
59 PkgSkip(String),
61 PkgFail(String),
63 IndirectSkip(String),
65 IndirectFail(String),
67 UnresolvedDep(String),
69}
70
71impl SkipReason {
72 pub fn status(&self) -> &'static str {
74 match self {
75 SkipReason::PkgSkip(_) => "pre-skipped",
76 SkipReason::PkgFail(_) => "pre-failed",
77 SkipReason::IndirectSkip(_) => "indirect-skipped",
78 SkipReason::IndirectFail(_) => "indirect-failed",
79 SkipReason::UnresolvedDep(_) => "unresolved",
80 }
81 }
82
83 pub fn is_direct(&self) -> bool {
85 matches!(
86 self,
87 SkipReason::PkgSkip(_) | SkipReason::PkgFail(_) | SkipReason::UnresolvedDep(_)
88 )
89 }
90}
91
92impl std::fmt::Display for SkipReason {
93 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
94 match self {
95 SkipReason::PkgSkip(r)
96 | SkipReason::PkgFail(r)
97 | SkipReason::IndirectSkip(r)
98 | SkipReason::IndirectFail(r) => write!(f, "{}", r),
99 SkipReason::UnresolvedDep(p) => {
100 write!(f, "Could not resolve: {}", p)
101 }
102 }
103 }
104}
105
106#[derive(Clone, Debug, Default)]
108pub struct SkippedCounts {
109 pub pkg_skip: usize,
111 pub pkg_fail: usize,
113 pub unresolved: usize,
115 pub indirect_skip: usize,
117 pub indirect_fail: usize,
119}
120
121#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
123pub struct ResolvedPackage {
124 pub index: ScanIndex,
126 pub pkgpath: PkgPath,
128}
129
130impl ResolvedPackage {
131 pub fn pkgname(&self) -> &PkgName {
133 &self.index.pkgname
134 }
135
136 pub fn depends(&self) -> &[PkgName] {
138 self.index.resolved_depends.as_deref().unwrap_or(&[])
139 }
140
141 pub fn bootstrap_pkg(&self) -> Option<&str> {
143 self.index.bootstrap_pkg.as_deref()
144 }
145
146 pub fn usergroup_phase(&self) -> Option<&str> {
148 self.index.usergroup_phase.as_deref()
149 }
150
151 pub fn multi_version(&self) -> Option<&[String]> {
153 self.index.multi_version.as_deref()
154 }
155
156 pub fn pbulk_weight(&self) -> Option<&str> {
158 self.index.pbulk_weight.as_deref()
159 }
160}
161
162impl std::fmt::Display for ResolvedPackage {
163 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
164 write!(f, "{}", self.index)
165 }
166}
167
168#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
170pub enum ScanResult {
171 Buildable(ResolvedPackage),
173 Skipped {
175 pkgpath: PkgPath,
177 reason: SkipReason,
179 index: Option<ScanIndex>,
181 resolved_depends: Vec<PkgName>,
183 },
184 ScanFail {
186 pkgpath: PkgPath,
188 error: String,
190 },
191}
192
193impl ScanResult {
194 pub fn pkgpath(&self) -> &PkgPath {
196 match self {
197 ScanResult::Buildable(pkg) => &pkg.pkgpath,
198 ScanResult::Skipped { pkgpath, .. } => pkgpath,
199 ScanResult::ScanFail { pkgpath, .. } => pkgpath,
200 }
201 }
202
203 pub fn pkgname(&self) -> Option<&PkgName> {
205 match self {
206 ScanResult::Buildable(pkg) => Some(pkg.pkgname()),
207 ScanResult::Skipped { index, .. } => index.as_ref().map(|i| &i.pkgname),
208 ScanResult::ScanFail { .. } => None,
209 }
210 }
211
212 pub fn is_buildable(&self) -> bool {
214 matches!(self, ScanResult::Buildable(_))
215 }
216
217 pub fn as_buildable(&self) -> Option<&ResolvedPackage> {
219 match self {
220 ScanResult::Buildable(pkg) => Some(pkg),
221 _ => None,
222 }
223 }
224
225 pub fn depends(&self) -> &[PkgName] {
227 match self {
228 ScanResult::Buildable(pkg) => pkg.depends(),
229 ScanResult::Skipped {
230 resolved_depends, ..
231 } => resolved_depends,
232 ScanResult::ScanFail { .. } => &[],
233 }
234 }
235}
236
237impl std::fmt::Display for ScanResult {
238 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
239 match self {
240 ScanResult::Buildable(pkg) => write!(f, "{}", pkg),
241 ScanResult::Skipped {
242 index,
243 pkgpath,
244 reason,
245 resolved_depends,
246 } => {
247 if let Some(idx) = index {
248 write!(f, "{}", idx)?;
249 if !matches!(reason, SkipReason::UnresolvedDep(_))
251 && !resolved_depends.is_empty()
252 {
253 write!(f, "DEPENDS=")?;
254 for (i, d) in resolved_depends.iter().enumerate() {
255 if i > 0 {
256 write!(f, " ")?;
257 }
258 write!(f, "{d}")?;
259 }
260 writeln!(f)?;
261 }
262 } else {
263 writeln!(f, "PKGPATH={}", pkgpath)?;
264 }
265 Ok(())
266 }
267 ScanResult::ScanFail { pkgpath, .. } => {
268 writeln!(f, "PKGPATH={}", pkgpath)
269 }
270 }
271 }
272}
273
274#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
278pub struct ScanSummary {
279 pub pkgpaths: usize,
281 pub packages: Vec<ScanResult>,
283}
284
285#[derive(Clone, Debug, Default)]
287pub struct ScanCounts {
288 pub buildable: usize,
290 pub skipped: SkippedCounts,
292 pub scanfail: usize,
294}
295
296impl ScanSummary {
297 pub fn counts(&self) -> ScanCounts {
299 let mut c = ScanCounts::default();
300 for p in &self.packages {
301 match p {
302 ScanResult::Buildable(_) => c.buildable += 1,
303 ScanResult::Skipped {
304 reason: SkipReason::PkgSkip(_),
305 ..
306 } => c.skipped.pkg_skip += 1,
307 ScanResult::Skipped {
308 reason: SkipReason::PkgFail(_),
309 ..
310 } => c.skipped.pkg_fail += 1,
311 ScanResult::Skipped {
312 reason: SkipReason::IndirectSkip(_),
313 ..
314 } => c.skipped.indirect_skip += 1,
315 ScanResult::Skipped {
316 reason: SkipReason::IndirectFail(_),
317 ..
318 } => c.skipped.indirect_fail += 1,
319 ScanResult::Skipped {
320 reason: SkipReason::UnresolvedDep(_),
321 ..
322 } => c.skipped.unresolved += 1,
323 ScanResult::ScanFail { .. } => c.scanfail += 1,
324 }
325 }
326 c
327 }
328
329 pub fn buildable(&self) -> impl Iterator<Item = &ResolvedPackage> {
331 self.packages.iter().filter_map(|p| p.as_buildable())
332 }
333
334 pub fn failed(&self) -> impl Iterator<Item = &ScanResult> {
336 self.packages.iter().filter(|p| !p.is_buildable())
337 }
338
339 pub fn count_buildable(&self) -> usize {
341 self.packages.iter().filter(|p| p.is_buildable()).count()
342 }
343
344 pub fn errors(&self) -> impl Iterator<Item = &str> {
346 self.packages.iter().filter_map(|p| match p {
347 ScanResult::ScanFail { error, .. } => Some(error.as_str()),
348 ScanResult::Skipped {
349 reason: SkipReason::UnresolvedDep(e),
350 ..
351 } => Some(e.as_str()),
352 _ => None,
353 })
354 }
355}
356
357impl std::fmt::Display for ScanSummary {
358 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
359 let c = self.counts();
360 let s = &c.skipped;
361 write!(
362 f,
363 "Resolved {} total packages from {} package paths\n\
364 {} buildable, {} prefailed, {} indirect-prefailed, {} unresolved",
365 self.packages.len(),
366 self.pkgpaths,
367 c.buildable,
368 s.pkg_skip + s.pkg_fail,
369 s.indirect_skip + s.indirect_fail,
370 s.unresolved
371 )
372 }
373}
374
375#[derive(Debug, Default)]
377pub struct Scan {
378 config: Config,
379 sandbox: Sandbox,
380 incoming: HashSet<PkgPath>,
381 done: HashSet<PkgPath>,
383 initial_cached: usize,
385 discovered_cached: usize,
387 packages: IndexMap<PkgName, ScanIndex>,
389 full_tree: bool,
392 full_scan_complete: bool,
394 scan_failures: Vec<(PkgPath, String)>,
396 pkgsrc_env: Option<PkgsrcEnv>,
398 initial_pkgpaths: HashSet<PkgPath>,
401}
402
403impl Scan {
404 pub fn new(config: &Config) -> Scan {
405 let sandbox = Sandbox::new(config);
406 debug!(pkgsrc = %config.pkgsrc().display(),
407 make = %config.make().display(),
408 scan_threads = config.scan_threads(),
409 "Created new Scan instance"
410 );
411 Scan {
412 config: config.clone(),
413 sandbox,
414 incoming: HashSet::new(),
415 done: HashSet::new(),
416 initial_cached: 0,
417 discovered_cached: 0,
418 packages: IndexMap::new(),
419 full_tree: true,
420 full_scan_complete: false,
421 scan_failures: Vec::new(),
422 pkgsrc_env: None,
423 initial_pkgpaths: HashSet::new(),
424 }
425 }
426
427 pub fn add(&mut self, pkgpath: &PkgPath) {
428 info!(pkgpath = %pkgpath.as_path().display(), "Adding package to scan queue");
429 self.full_tree = false;
430 self.incoming.insert(pkgpath.clone());
431 self.initial_pkgpaths.insert(pkgpath.clone());
432 }
433
434 pub fn is_full_tree(&self) -> bool {
436 self.full_tree
437 }
438
439 pub fn set_full_scan_complete(&mut self) {
441 self.full_scan_complete = true;
442 }
443
444 pub fn init_from_db(&mut self, db: &crate::db::Database) -> Result<(usize, usize)> {
448 let scanned = db.get_scanned_pkgpaths()?;
449 let cached_count = scanned.len();
450 let mut pending_count = 0;
451
452 if cached_count > 0 {
453 info!(cached_count, "Found cached scan results in database");
454
455 if !self.full_tree {
458 self.incoming.retain(|p| !scanned.contains(&p.to_string()));
459 }
460
461 for pkgpath_str in &scanned {
463 if let Ok(pkgpath) = PkgPath::new(pkgpath_str) {
464 self.done.insert(pkgpath);
465 }
466 }
467
468 if self.full_tree {
478 let unscanned = db.get_unscanned_dependencies()?;
479 if !unscanned.is_empty() {
480 info!(
481 unscanned_count = unscanned.len(),
482 "Found unscanned dependencies from interrupted scan"
483 );
484 for pkgpath_str in unscanned {
485 if let Ok(pkgpath) = PkgPath::new(&pkgpath_str) {
486 if !self.done.contains(&pkgpath) {
487 self.incoming.insert(pkgpath);
488 pending_count += 1;
489 }
490 }
491 }
492 }
493 }
494 }
495
496 Ok((cached_count, pending_count))
497 }
498
499 fn discover_packages(
501 &mut self,
502 pool: &rayon::ThreadPool,
503 shutdown: &AtomicBool,
504 ) -> anyhow::Result<()> {
505 println!("Discovering packages...");
506 let pkgsrc = self.config.pkgsrc().display().to_string();
507
508 let child = self.sandbox.execute_command(
510 0,
511 self.config.make(),
512 ["-C", &pkgsrc, "show-subdir-var", "VARNAME=SUBDIR"],
513 vec![],
514 )?;
515 let output =
516 wait_output_with_shutdown(child, shutdown).context("Failed to run show-subdir-var")?;
517
518 if !output.status.success() {
519 let stderr = String::from_utf8_lossy(&output.stderr);
520 bail!("Failed to get categories: {}", stderr);
521 }
522
523 let stdout = String::from_utf8_lossy(&output.stdout);
524 let entries: Vec<&str> = stdout.split_whitespace().collect();
525
526 let mut categories: Vec<&str> = Vec::new();
528 for entry in entries {
529 if entry.contains('/') {
530 if let Ok(pkgpath) = PkgPath::new(entry) {
531 self.incoming.insert(pkgpath);
532 }
533 } else {
534 categories.push(entry);
535 }
536 }
537
538 let make = self.config.make();
540 let sandbox = &self.sandbox;
541 let discovered: Vec<PkgPath> = pool.install(|| {
542 categories
543 .par_iter()
544 .flat_map(|category| {
545 let workdir = format!("{}/{}", pkgsrc, category);
546 let result = sandbox
547 .execute_command(
548 0,
549 make,
550 [
551 "-C",
552 &workdir,
553 "show-subdir-var",
554 "VARNAME=SUBDIR",
555 ],
556 vec![],
557 )
558 .and_then(|c| wait_output_with_shutdown(c, shutdown));
559
560 match result {
561 Ok(o) if o.status.success() => {
562 let pkgs = String::from_utf8_lossy(&o.stdout);
563 pkgs.split_whitespace()
564 .filter_map(|pkg| {
565 let path = format!("{}/{}", category, pkg);
566 PkgPath::new(&path).ok()
567 })
568 .collect::<Vec<_>>()
569 }
570 Ok(o) => {
571 let stderr = String::from_utf8_lossy(&o.stderr);
572 debug!(category = *category, %stderr, "Failed to get packages for category");
573 vec![]
574 }
575 Err(e) => {
576 debug!(category = *category, error = %e, "Failed to run make in category");
577 vec![]
578 }
579 }
580 })
581 .collect()
582 });
583
584 self.incoming.extend(discovered);
585
586 info!(
587 discovered = self.incoming.len(),
588 "Package discovery complete"
589 );
590 println!("Discovered {} package paths", self.incoming.len());
591
592 Ok(())
593 }
594
595 pub fn start(
596 &mut self,
597 db: &crate::db::Database,
598 scope: &mut SandboxScope,
599 ) -> anyhow::Result<()> {
600 info!(
601 incoming_count = self.incoming.len(),
602 sandbox_enabled = self.sandbox.enabled(),
603 "Starting package scan"
604 );
605
606 let pool = rayon::ThreadPoolBuilder::new()
607 .num_threads(self.config.scan_threads())
608 .build()
609 .context("Failed to build scan thread pool")?;
610
611 let shutdown_flag = Arc::clone(scope.shutdown());
612
613 if self.full_tree && self.full_scan_complete && !self.done.is_empty() {
616 println!("All {} package paths already scanned", self.done.len());
617 return Ok(());
618 }
619
620 if !self.full_tree {
628 self.incoming.retain(|p| !self.done.contains(p));
629 if self.incoming.is_empty() {
630 if let Ok(deps) = self.unscanned_deps(db) {
631 self.incoming = deps;
632 }
633 if self.incoming.is_empty() {
634 if !self.done.is_empty() {
635 println!("All {} package paths already scanned", self.done.len());
636 }
637 return Ok(());
638 }
639 }
640 }
641
642 scope.ensure(1)?;
649
650 if scope.enabled() {
651 if !self
653 .sandbox
654 .run_pre_build(0, &self.config, self.config.script_env(None))?
655 {
656 warn!("pre-build script failed");
657 }
658 }
659
660 let env = match db.load_pkgsrc_env() {
661 Ok(env) => env,
662 Err(_) => {
663 let env = PkgsrcEnv::fetch(&self.config, &self.sandbox)?;
664 db.store_pkgsrc_env(&env)?;
665 env
666 }
667 };
668 self.pkgsrc_env = Some(env);
669
670 if self.full_tree {
672 self.discover_packages(&pool, &shutdown_flag)?;
673 self.incoming.retain(|p| !self.done.contains(p));
674 }
675
676 if self.incoming.is_empty() {
678 if !self.done.is_empty() {
679 println!("All {} package paths already scanned", self.done.len());
680 }
681
682 if scope.enabled() {
683 self.run_post_build()?;
684 }
685 return Ok(());
686 }
687
688 db.clear_resolved_depends()?;
690
691 println!("Scanning packages...");
692
693 self.initial_cached = self.done.len();
695
696 let total_count = self.initial_cached + self.incoming.len();
699 let progress = Arc::new(Mutex::new(
700 MultiProgress::new("Scanning", "", total_count, self.config.scan_threads())
701 .expect("Failed to initialize progress display"),
702 ));
703
704 if self.initial_cached > 0 {
706 if let Ok(mut p) = progress.lock() {
707 p.state_mut().cached = self.initial_cached;
708 }
709 }
710
711 let stop_refresh = Arc::new(AtomicBool::new(false));
713
714 let progress_refresh = Arc::clone(&progress);
716 let stop_flag = Arc::clone(&stop_refresh);
717 let shutdown_for_refresh = Arc::clone(&shutdown_flag);
718 let refresh_thread = std::thread::spawn(move || {
719 while !stop_flag.load(Ordering::Relaxed) && !shutdown_for_refresh.load(Ordering::SeqCst)
720 {
721 let has_event = event::poll(REFRESH_INTERVAL).unwrap_or(false);
723
724 if let Ok(mut p) = progress_refresh.lock() {
725 if has_event {
726 let _ = p.handle_event();
727 }
728 let _ = p.render();
729 }
730 }
731 });
732
733 db.begin_transaction()?;
735
736 let config = &self.config;
739 let sandbox = &self.sandbox;
740 let scan_env = self.scan_env();
741
742 if !self.full_tree && self.incoming.is_empty() {
748 if let Ok(deps) = self.unscanned_deps(db) {
749 for pkgpath in deps {
750 self.incoming.insert(pkgpath);
751 if let Ok(mut p) = progress.lock() {
752 p.state_mut().total += 1;
753 }
754 }
755 }
756 }
757
758 loop {
764 if shutdown_flag.load(Ordering::SeqCst) {
768 stop_refresh.store(true, Ordering::Relaxed);
769 if let Ok(mut p) = progress.lock() {
770 let _ = p.finish_interrupted();
771 }
772 break;
773 }
774
775 let pkgpaths: Vec<PkgPath> = self.incoming.drain().collect();
779 if pkgpaths.is_empty() {
780 break;
781 }
782
783 const CHANNEL_BUFFER_SIZE: usize = 128;
785 let (tx, rx) = std::sync::mpsc::sync_channel::<(PkgPath, Result<Vec<ScanIndex>>)>(
786 CHANNEL_BUFFER_SIZE,
787 );
788
789 let mut new_incoming: HashSet<PkgPath> = HashSet::new();
790
791 std::thread::scope(|s| {
792 let progress_clone = Arc::clone(&progress);
794 let shutdown_clone = Arc::clone(&shutdown_flag);
795 let pool_ref = &pool;
796 let scan_env_ref = &scan_env;
797
798 s.spawn(move || {
799 pool_ref.install(|| {
800 pkgpaths.par_iter().for_each(|pkgpath| {
801 if shutdown_clone.load(Ordering::SeqCst) {
803 return;
804 }
805
806 let pathname = pkgpath.as_path().to_string_lossy().to_string();
807 let thread_id = rayon::current_thread_index().unwrap_or(0);
808
809 if let Ok(mut p) = progress_clone.lock() {
811 p.state_mut().set_worker_active(thread_id, &pathname);
812 }
813
814 let result = Self::scan_pkgpath_with(
815 config,
816 sandbox,
817 pkgpath,
818 scan_env_ref,
819 &shutdown_clone,
820 );
821
822 if let Ok(mut p) = progress_clone.lock() {
824 p.state_mut().set_worker_idle(thread_id);
825 if result.is_ok() {
826 p.state_mut().increment_completed();
827 } else {
828 p.state_mut().increment_failed();
829 }
830 }
831
832 let _ = tx.send((pkgpath.clone(), result));
834 });
835 });
836 drop(tx);
837 });
838
839 for (pkgpath, result) in rx {
843 let scanpkgs = match result {
844 Ok(pkgs) => pkgs,
845 Err(e) => {
846 self.scan_failures.push((pkgpath.clone(), e.to_string()));
847 self.done.insert(pkgpath);
848 continue;
849 }
850 };
851 self.done.insert(pkgpath.clone());
852
853 if !scanpkgs.is_empty() {
855 if let Err(e) = db.store_scan_pkgpath(&pkgpath.to_string(), &scanpkgs) {
856 error!(error = %e, "Failed to store scan results");
857 }
858 }
859 }
860 });
861
862 new_incoming.retain(|p| !self.done.contains(p));
872
873 if !self.full_tree && new_incoming.is_empty() {
879 match self.unscanned_deps(db) {
880 Ok(deps) if !deps.is_empty() => {
881 let count = deps.len();
882 for pkgpath in deps {
883 new_incoming.insert(pkgpath);
884 if let Ok(mut p) = progress.lock() {
885 p.state_mut().total += 1;
886 }
887 }
888 debug!(
889 missing_count = count,
890 "Discovered missing dependency pkgpaths"
891 );
892 }
893 Err(e) => {
894 warn!(error = %e, "Failed to find missing pkgpaths");
895 }
896 _ => {}
897 }
898 }
899
900 self.incoming = new_incoming;
901 }
902
903 db.commit()?;
905
906 stop_refresh.store(true, Ordering::Relaxed);
908 let _ = refresh_thread.join();
909
910 if !shutdown_flag.load(Ordering::SeqCst) {
913 let elapsed = if let Ok(mut p) = progress.lock() {
915 p.finish_silent().ok()
916 } else {
917 None
918 };
919
920 let total = self.done.len();
924 let cached = self.initial_cached + self.discovered_cached;
925 let failed = self.scan_failures.len();
926 let succeeded = total.saturating_sub(cached).saturating_sub(failed);
927
928 let elapsed_str = elapsed
929 .map(format_duration)
930 .unwrap_or_else(|| "?".to_string());
931
932 if cached > 0 {
933 println!(
934 "Scanned {} package paths in {} ({} scanned, {} cached, {} failed)",
935 total, elapsed_str, succeeded, cached, failed
936 );
937 } else {
938 println!(
939 "Scanned {} package paths in {} ({} succeeded, {} failed)",
940 total, elapsed_str, succeeded, failed
941 );
942 }
943 }
944
945 if scope.enabled() {
946 self.run_post_build()?;
947 }
948
949 if shutdown_flag.load(Ordering::SeqCst) {
950 return Err(Interrupted.into());
951 }
952
953 Ok(())
954 }
955
956 fn run_post_build(&self) -> anyhow::Result<()> {
958 if !self.sandbox.run_post_build(
959 0,
960 &self.config,
961 self.config.script_env(self.pkgsrc_env.as_ref()),
962 )? {
963 warn!("post-build script failed");
964 }
965 Ok(())
966 }
967
968 pub fn scan_errors(&self) -> impl Iterator<Item = &str> {
970 self.scan_failures.iter().map(|(_, e)| e.as_str())
971 }
972
973 fn scan_env(&self) -> Vec<(String, String)> {
974 self.pkgsrc_env
975 .as_ref()
976 .map(|e| {
977 e.cachevars
978 .iter()
979 .map(|(k, v)| (k.clone(), v.clone()))
980 .collect()
981 })
982 .unwrap_or_default()
983 }
984
985 fn unscanned_deps(&self, db: &crate::db::Database) -> Result<HashSet<PkgPath>> {
986 let missing = self.find_missing_pkgpaths(db)?;
987 Ok(missing
988 .into_iter()
989 .filter(|p| !self.done.contains(p))
990 .collect())
991 }
992
993 fn scan_pkgpath_with(
998 config: &Config,
999 sandbox: &Sandbox,
1000 pkgpath: &PkgPath,
1001 scan_env: &[(String, String)],
1002 shutdown: &AtomicBool,
1003 ) -> anyhow::Result<Vec<ScanIndex>> {
1004 let pkgpath_str = pkgpath.as_path().display().to_string();
1005 let span = info_span!("scan", pkgpath = %pkgpath_str);
1006 let _guard = span.enter();
1007 debug!("Scanning package");
1008
1009 let pkgsrcdir = config.pkgsrc().display().to_string();
1010 let workdir = format!("{}/{}", pkgsrcdir, pkgpath_str);
1011
1012 trace!(%workdir, ?scan_env, "Executing pkg-scan");
1013 let child = sandbox.execute_command(
1014 0,
1015 config.make(),
1016 ["-C", &workdir, "pbulk-index"],
1017 scan_env.to_vec(),
1018 )?;
1019 let output = wait_output_with_shutdown(child, shutdown)?;
1020
1021 if !output.status.success() {
1022 let stderr = String::from_utf8_lossy(&output.stderr);
1023 error!(exit_code = ?output.status.code(), %stderr, "pkg-scan script failed");
1024 let stderr = stderr.trim();
1025 let msg = if stderr.is_empty() {
1026 format!("Scan failed for {}", pkgpath_str)
1027 } else {
1028 format!("Scan failed for {}: {}", pkgpath_str, stderr)
1029 };
1030 bail!(msg);
1031 }
1032
1033 let stdout = String::from_utf8_lossy(&output.stdout);
1034 trace!(stdout_len = stdout.len(), %stdout, "pkg-scan script output");
1035
1036 let reader = BufReader::new(&output.stdout[..]);
1037 let all_results: Vec<ScanIndex> =
1038 ScanIndex::from_reader(reader).collect::<Result<_, _>>()?;
1039
1040 let mut seen_pkgnames = HashSet::new();
1046 let mut index: Vec<ScanIndex> = Vec::new();
1047 for pkg in all_results {
1048 if seen_pkgnames.insert(pkg.pkgname.clone()) {
1049 index.push(pkg);
1050 }
1051 }
1052
1053 info!(packages_found = index.len(), "Scan complete");
1054
1055 for pkg in &mut index {
1059 pkg.pkg_location = Some(pkgpath.clone());
1060 debug!(
1061 pkgname = %pkg.pkgname.pkgname(),
1062 skip_reason = ?pkg.pkg_skip_reason,
1063 fail_reason = ?pkg.pkg_fail_reason,
1064 depends_count = pkg.all_depends.as_ref().map_or(0, |v| v.len()),
1065 "Found package in scan"
1066 );
1067 }
1068
1069 Ok(index)
1070 }
1071
1072 fn find_missing_pkgpaths(&self, db: &crate::db::Database) -> Result<HashSet<PkgPath>> {
1084 let all_scan_data = db.get_all_scan_data()?;
1093
1094 let mut available_pkgnames: HashSet<PkgName> = HashSet::new();
1095 let mut packages: IndexMap<PkgName, ScanIndex> = IndexMap::new();
1096
1097 for pkg in all_scan_data {
1098 if !packages.contains_key(&pkg.pkgname) {
1099 available_pkgnames.insert(pkg.pkgname.clone());
1100 packages.insert(pkg.pkgname.clone(), pkg);
1101 }
1102 }
1103
1104 let pkgbase_map = Self::build_pkgbase_map(&available_pkgnames);
1105
1106 let mut active_pkgnames: HashSet<PkgName> = HashSet::new();
1107 for pkg in packages.values() {
1108 if let Some(ref loc) = pkg.pkg_location {
1109 if self.initial_pkgpaths.contains(loc) {
1110 active_pkgnames.insert(pkg.pkgname.clone());
1111 }
1112 }
1113 }
1114
1115 let mut missing_pkgpaths: HashSet<PkgPath> = HashSet::new();
1116 let mut changed = true;
1117
1118 while changed {
1119 changed = false;
1120 let current_active: Vec<PkgName> = active_pkgnames.iter().cloned().collect();
1121
1122 for active_pkgname in current_active {
1123 let Some(pkg) = packages.get(&active_pkgname) else {
1124 continue;
1125 };
1126 let Some(ref all_deps) = pkg.all_depends else {
1127 continue;
1128 };
1129
1130 for depend in all_deps {
1131 let candidates =
1132 Self::find_candidates(depend, &pkgbase_map, available_pkgnames.iter());
1133
1134 if candidates.is_empty() {
1135 let dep_path = depend.pkgpath();
1136 if !self.done.contains(dep_path) {
1137 missing_pkgpaths.insert(dep_path.clone());
1138 }
1139 } else {
1140 for candidate in &candidates {
1141 if !active_pkgnames.contains(*candidate) {
1142 active_pkgnames.insert((*candidate).clone());
1143 changed = true;
1144 }
1145 }
1146 }
1147 }
1148 }
1149 }
1150
1151 debug!(
1152 missing_count = missing_pkgpaths.len(),
1153 active_count = active_pkgnames.len(),
1154 "Found missing dependency pkgpaths"
1155 );
1156
1157 Ok(missing_pkgpaths)
1158 }
1159
1160 fn build_pkgbase_map<'a>(
1164 pkgnames: impl IntoIterator<Item = &'a PkgName>,
1165 ) -> HashMap<&'a str, Vec<&'a PkgName>> {
1166 let mut map: HashMap<&str, Vec<&PkgName>> = HashMap::new();
1167 for pkgname in pkgnames {
1168 map.entry(pkgname.pkgbase()).or_default().push(pkgname);
1169 }
1170 map
1171 }
1172
1173 fn find_candidates<'a>(
1180 depend: &Depend,
1181 pkgbase_map: &HashMap<&str, Vec<&'a PkgName>>,
1182 all_pkgnames: impl Iterator<Item = &'a PkgName>,
1183 ) -> Vec<&'a PkgName> {
1184 if let Some(base) = depend.pattern().pkgbase() {
1185 pkgbase_map.get(base).map_or(Vec::new(), |v| {
1186 v.iter()
1187 .filter(|c| depend.pattern().matches(c.pkgname()))
1188 .copied()
1189 .collect()
1190 })
1191 } else {
1192 all_pkgnames
1193 .filter(|c| depend.pattern().matches(c.pkgname()))
1194 .collect()
1195 }
1196 }
1197
1198 fn find_best_match<'a>(
1210 depend: &Depend,
1211 pkgbase_map: &HashMap<&str, Vec<&'a PkgName>>,
1212 pkgnames: &'a [PkgName],
1213 ) -> Result<Option<&'a PkgName>, pkgsrc::PatternError> {
1214 let candidates = Self::find_candidates(depend, pkgbase_map, pkgnames.iter());
1215
1216 let mut best: Option<&PkgName> = None;
1217 for candidate in candidates {
1218 best = match best {
1219 None => Some(candidate),
1220 Some(current) => {
1221 match depend
1222 .pattern()
1223 .best_match_pbulk(current.pkgname(), candidate.pkgname())
1224 {
1225 Ok(Some(m)) if m == candidate.pkgname() => Some(candidate),
1226 Ok(_) => Some(current),
1227 Err(e) => return Err(e),
1228 }
1229 }
1230 };
1231 }
1232
1233 Ok(best)
1234 }
1235
1236 fn propagate_failures(
1243 depends: &HashMap<PkgName, Vec<PkgName>>,
1244 skip_reasons: &mut HashMap<PkgName, SkipReason>,
1245 ) {
1246 loop {
1247 let mut new_skip_reasons: Vec<(PkgName, SkipReason)> = Vec::new();
1248 for (pkgname, pkg_depends) in depends {
1249 if skip_reasons.contains_key(pkgname) {
1250 continue;
1251 }
1252 for dep in pkg_depends {
1253 if let Some(dep_reason) = skip_reasons.get(dep) {
1254 let reason = match dep_reason {
1255 SkipReason::PkgSkip(_) | SkipReason::IndirectSkip(_) => {
1256 SkipReason::IndirectSkip(format!(
1257 "dependency {} skipped",
1258 dep.pkgname()
1259 ))
1260 }
1261 _ => SkipReason::IndirectFail(format!(
1262 "dependency {} failed",
1263 dep.pkgname()
1264 )),
1265 };
1266 new_skip_reasons.push((pkgname.clone(), reason));
1267 break;
1268 }
1269 }
1270 }
1271 if new_skip_reasons.is_empty() {
1272 break;
1273 }
1274 for (pkgname, reason) in new_skip_reasons {
1275 skip_reasons.insert(pkgname, reason);
1276 }
1277 }
1278 }
1279
1280 fn check_circular_deps(packages: &[ScanResult]) -> Result<()> {
1286 let mut graph = DiGraphMap::new();
1287 for pkg in packages {
1288 if let ScanResult::Buildable(resolved) = pkg {
1289 for dep in resolved.depends() {
1290 graph.add_edge(dep.pkgname(), resolved.pkgname().pkgname(), ());
1291 }
1292 }
1293 }
1294 if let Some(cycle) = find_cycle(&graph) {
1295 let mut err = "Circular dependencies detected:\n".to_string();
1296 for n in cycle.iter().rev() {
1297 err.push_str(&format!("\t{}\n", n));
1298 }
1299 err.push_str(&format!("\t{}", cycle.last().unwrap()));
1300 error!(?cycle, "Circular dependency detected");
1301 bail!(err);
1302 }
1303 Ok(())
1304 }
1305
1306 pub fn resolve(&mut self, scan_data: Vec<ScanIndex>) -> Result<ScanSummary> {
1352 info!(
1353 done_pkgpaths = self.done.len(),
1354 "Starting dependency resolution"
1355 );
1356
1357 let mut skip_reasons: HashMap<PkgName, SkipReason> = HashMap::new();
1358 let mut depends: HashMap<PkgName, Vec<PkgName>> = HashMap::new();
1359 let mut active: HashSet<PkgName> = HashSet::new();
1360 let use_active_filter = !self.full_tree && !self.initial_pkgpaths.is_empty();
1361
1362 for pkg in scan_data {
1363 if self.packages.contains_key(&pkg.pkgname) {
1364 debug!(pkgname = %pkg.pkgname.pkgname(), "Skipping duplicate PKGNAME");
1365 continue;
1366 }
1367
1368 if let Some(reason) = &pkg.pkg_skip_reason {
1369 if !reason.is_empty() {
1370 info!(pkgname = %pkg.pkgname.pkgname(), %reason, "PKG_SKIP_REASON");
1371 skip_reasons.insert(pkg.pkgname.clone(), SkipReason::PkgSkip(reason.clone()));
1372 }
1373 }
1374
1375 if use_active_filter {
1376 if let Some(ref loc) = pkg.pkg_location {
1377 if self.initial_pkgpaths.contains(loc) {
1378 active.insert(pkg.pkgname.clone());
1379 }
1380 }
1381 }
1382
1383 if let Some(reason) = &pkg.pkg_fail_reason {
1384 if !reason.is_empty() && !skip_reasons.contains_key(&pkg.pkgname) {
1385 info!(pkgname = %pkg.pkgname.pkgname(), %reason, "PKG_FAIL_REASON");
1386 skip_reasons.insert(pkg.pkgname.clone(), SkipReason::PkgFail(reason.clone()));
1387 }
1388 }
1389
1390 depends.insert(pkg.pkgname.clone(), Vec::new());
1391 self.packages.insert(pkg.pkgname.clone(), pkg);
1392 }
1393
1394 info!(packages = self.packages.len(), "Loaded packages");
1395
1396 let pkgnames: Vec<PkgName> = self.packages.keys().cloned().collect();
1397 let pkgbase_map = Self::build_pkgbase_map(&pkgnames);
1398 let mut match_cache: HashMap<Depend, PkgName> = HashMap::new();
1399 let is_satisfied = |deps: &[PkgName], pattern: &pkgsrc::Pattern| {
1400 deps.iter()
1401 .any(|existing| pattern.matches(existing.pkgname()))
1402 };
1403
1404 let mut resolved: HashSet<PkgName> = HashSet::new();
1405 loop {
1406 let mut new_active = false;
1407 for pkg in self.packages.values_mut() {
1408 if use_active_filter && !active.contains(&pkg.pkgname) {
1409 continue;
1410 }
1411 if resolved.contains(&pkg.pkgname) {
1412 continue;
1413 }
1414 resolved.insert(pkg.pkgname.clone());
1415
1416 let all_deps = match pkg.all_depends.take() {
1417 Some(deps) => deps,
1418 None => continue,
1419 };
1420 let pkg_depends = depends.get_mut(&pkg.pkgname).unwrap();
1421
1422 for depend in all_deps.iter() {
1423 if let Some(pkgname) = match_cache.get(depend) {
1424 if !is_satisfied(pkg_depends, depend.pattern())
1425 && !pkg_depends.contains(pkgname)
1426 {
1427 pkg_depends.push(pkgname.clone());
1428 }
1429 continue;
1430 }
1431
1432 match Self::find_best_match(depend, &pkgbase_map, &pkgnames) {
1433 Err(e) => {
1434 let reason = format!(
1435 "{}: pattern error for {}: {}",
1436 pkg.pkgname.pkgname(),
1437 depend.pattern().pattern(),
1438 e
1439 );
1440 if !skip_reasons.contains_key(&pkg.pkgname) {
1441 skip_reasons
1442 .insert(pkg.pkgname.clone(), SkipReason::PkgFail(reason));
1443 }
1444 }
1445 Ok(Some(pkgname)) => {
1446 if !is_satisfied(pkg_depends, depend.pattern())
1447 && !pkg_depends.contains(pkgname)
1448 {
1449 pkg_depends.push(pkgname.clone());
1450 }
1451 match_cache.insert(depend.clone(), pkgname.clone());
1452 if use_active_filter && !active.contains(pkgname) {
1453 active.insert(pkgname.clone());
1454 new_active = true;
1455 }
1456 }
1457 Ok(None) => {
1458 let pattern = depend.pattern().pattern();
1459 let fail_reason =
1460 format!("\"could not resolve dependency \"{}\"\"", pattern);
1461 pkg.pkg_fail_reason = Some(fail_reason);
1462 let msg = format!(
1463 "No match found for dependency {} of package {}",
1464 pattern,
1465 pkg.pkgname.pkgname()
1466 );
1467 match skip_reasons.get_mut(&pkg.pkgname) {
1468 Some(SkipReason::UnresolvedDep(existing)) => {
1469 existing.push('\n');
1470 existing.push_str(&msg);
1471 }
1472 None => {
1473 skip_reasons.insert(
1474 pkg.pkgname.clone(),
1475 SkipReason::UnresolvedDep(msg),
1476 );
1477 }
1478 _ => {}
1479 }
1480 }
1481 }
1482 }
1483 pkg.all_depends = Some(all_deps);
1484 }
1485 if !use_active_filter || !new_active {
1486 break;
1487 }
1488 }
1489
1490 Self::propagate_failures(&depends, &mut skip_reasons);
1491
1492 let mut packages: Vec<ScanResult> = Vec::new();
1493 let mut count_buildable = 0;
1494 let mut count_filtered = 0;
1495
1496 for (pkgname, mut index) in std::mem::take(&mut self.packages) {
1497 if use_active_filter && !active.contains(&pkgname) {
1498 count_filtered += 1;
1499 continue;
1500 }
1501
1502 let Some(pkgpath) = index.pkg_location.clone() else {
1503 error!(%pkgname, "Package missing PKG_LOCATION, skipping");
1504 continue;
1505 };
1506 let resolved_depends = depends.remove(&pkgname).unwrap_or_default();
1507 let result = match skip_reasons.remove(&pkgname) {
1508 Some(reason) => ScanResult::Skipped {
1509 pkgpath,
1510 reason,
1511 index: Some(index),
1512 resolved_depends,
1513 },
1514 None => {
1515 count_buildable += 1;
1516 index.resolved_depends = Some(resolved_depends);
1517 ScanResult::Buildable(ResolvedPackage { index, pkgpath })
1518 }
1519 };
1520 packages.push(result);
1521 }
1522
1523 if count_filtered > 0 {
1524 debug!(
1525 count_filtered,
1526 "Filtered inactive packages (not needed for resolution)"
1527 );
1528 }
1529
1530 for (pkgpath, error) in &self.scan_failures {
1531 packages.push(ScanResult::ScanFail {
1532 pkgpath: pkgpath.clone(),
1533 error: error.clone(),
1534 });
1535 }
1536
1537 debug!(count_buildable, "Checking for circular dependencies");
1538 Self::check_circular_deps(&packages)?;
1539
1540 let pkgpaths = packages
1541 .iter()
1542 .map(|p| p.pkgpath())
1543 .collect::<HashSet<_>>()
1544 .len();
1545 let summary = ScanSummary { pkgpaths, packages };
1546
1547 let c = summary.counts();
1548 info!(
1549 buildable = c.buildable,
1550 preskip = c.skipped.pkg_skip,
1551 prefail = c.skipped.pkg_fail,
1552 unresolved = c.skipped.unresolved,
1553 "Resolution complete"
1554 );
1555
1556 Ok(summary)
1557 }
1558
1559 pub fn resolve_with_report(
1567 &mut self,
1568 db: &crate::db::Database,
1569 strict: bool,
1570 ) -> Result<ScanSummary> {
1571 print!("Resolving dependencies...");
1572 std::io::Write::flush(&mut std::io::stdout())?;
1573
1574 let start = std::time::Instant::now();
1575 let scan_data = db.get_all_scan_data()?;
1576 let result = self.resolve(scan_data)?;
1577 db.store_resolved_deps(&result)?;
1578 db.store_scan_skipped(&result)?;
1579 println!(" done ({:.1}s)", start.elapsed().as_secs_f32());
1580
1581 let errors: Vec<_> = result.errors().collect();
1582 if !errors.is_empty() {
1583 eprintln!("Unresolved dependencies:\n {}", errors.join("\n "));
1584 if strict {
1585 bail!("Aborting due to unresolved dependencies (strict_scan enabled)");
1586 }
1587 }
1588
1589 Ok(result)
1590 }
1591}
1592
1593fn find_cycle<'a>(graph: &'a DiGraphMap<&'a str, ()>) -> Option<Vec<&'a str>> {
1594 let mut visited = HashSet::new();
1595 let mut in_stack = HashSet::new();
1596 let mut stack = Vec::new();
1597
1598 for node in graph.nodes() {
1599 if visited.contains(&node) {
1600 continue;
1601 }
1602 if let Some(cycle) = dfs(graph, node, &mut visited, &mut stack, &mut in_stack) {
1603 return Some(cycle);
1604 }
1605 }
1606 None
1607}
1608
1609fn dfs<'a>(
1610 graph: &'a DiGraphMap<&'a str, ()>,
1611 node: &'a str,
1612 visited: &mut HashSet<&'a str>,
1613 stack: &mut Vec<&'a str>,
1614 in_stack: &mut HashSet<&'a str>,
1615) -> Option<Vec<&'a str>> {
1616 visited.insert(node);
1617 stack.push(node);
1618 in_stack.insert(node);
1619 for neighbor in graph.neighbors(node) {
1620 if in_stack.contains(neighbor) {
1621 if let Some(pos) = stack.iter().position(|&n| n == neighbor) {
1622 return Some(stack[pos..].to_vec());
1623 }
1624 } else if !visited.contains(neighbor) {
1625 let cycle = dfs(graph, neighbor, visited, stack, in_stack);
1626 if cycle.is_some() {
1627 return cycle;
1628 }
1629 }
1630 }
1631 stack.pop();
1632 in_stack.remove(node);
1633 None
1634}