1pub mod http_server;
4pub mod mcp;
5mod sync;
6mod work_state;
7mod worker;
8
9use std::collections::{BTreeMap, BTreeSet};
10use std::ffi::OsStr;
11use std::fs::{self, OpenOptions};
12use std::io::{Read, Write};
13use std::path::{Path, PathBuf};
14use std::sync::Arc;
15use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
16
17use atomic_write_file::AtomicWriteFile;
18use code_system_graph_core::{
19 AffectedTestsRequest, AnalyzerVersions, ArtifactKey, BatchAction, BatchPlanError, BitbucketProvider, ChangeAnalysisError, ChangeAnalysisOptions, ChangeError, ChangeImpactReport, ChangeProvider, ChangeRequest, ChangeScope, ChangeSet, CodeGraphConfig, CodeGraphProvider, CommunityError, ConfigDoctorInput, ConfigError, ConfigExtractionError, ContractReport, ContractRequest, CorroborationReport, DataDocument, DataExtractionError, DeclaredImplementation, DeclaredTestCase, DoctorReport, DoctorRequest, DocumentationDocument, DocumentationExtractionError, EXTRACTION_CONTRACT_VERSION, EffectiveRepositoryConfig, EventDocument, EventExtractionError, EventGraphFacts, ExecutionPolicy, ExitCode, ExportReport, ExportRequest, ExtractionBudgets, ExtractionGraphFacts, ExtractionLimitExceeded, ExtractionTracker, ExtractorBatch, ExtractorBatchPlan, FederatedGraph, FreshnessDoctorInput, GeneratedClientError, GeneratedClientMetadata, GitCliChangeProvider, GitHubProvider, GraphqlDocument, GraphqlExtractionError, GraphqlGraphFacts, HttpBoundary, HttpExtractionError, ImpactContext, ImpactError, ImpactReport, ImpactRequest, ImpactTarget, IncrementalPlan, InfrastructureDocument, InfrastructureExtractionError, IntegrityDoctorInput, InterfaceError, LinkError, LocalCodeIntelligenceProvider, LocalContextRequest, LocalContextResult, LocalEnrichmentInput, LocalEnrichmentStatus, LocalImpactItem, LocalImpactRequest, ManifestEdit, ManifestEditError, ManifestError, ManualLinkConfig, ManualLinkError, PackageGraphFacts, PackageManifest, PackageManifestError, PrAuthToken, ProtobufDocument, ProtobufExtractionError, ProtobufGraphFacts, ProviderBudget, ProviderCapability, ProviderDoctorInput, ProviderDoctorStatus, ProviderError, ProviderRequest, ProviderStatus, PullRequestCoordinates, PullRequestError, PullRequestInspectRequest, PullRequestInspection, PullRequestListPage, PullRequestListRequest, PullRequestListState, PullRequestProvider, PullRequestProviderConfig, PullRequestProviderKind, QueryError, RecommendedCommand, RegisteredWorkspace, RegistryError, ReqwestPrHttpTransport, SafeConfigDocument, SchemaDoctorInput, SearchFilters, SearchReport, SearchRequest, SourceEpistemicStatus, SourceGraphFacts, SourceLanguage, SourceObservation, SourceRole, SourceSyntaxError, SourceSyntaxLanguage, SourceWarning, SymbolAnchor, SymbolCorroboration, TraceError, TraversalReport, TraversalRequest, WorkspaceManifest, affected_link_keys, analyze_changes, analyze_communities_with_progress, analyze_impact, apply_openapi_override, classify_interface_error, commit_manifest_edit, compare_community_snapshots, corroborate_repository, declared_implementation, declared_test_case, doctor, documents_to_graph, encode_native_path, event_documents_to_graph, export_graph, extract_asyncapi, extract_codeowners, extract_data_artifact, extract_docker_compose, extract_generated_client_metadata, extract_graphql_document_with_tracker, extract_graphql_persisted_operations_with_tracker, extract_helm, extract_kubernetes, extract_markdown, extract_openapi_with_tracker, extract_package_manifest_with_tracker, extract_protobuf_with_tracker, extract_safe_config, extract_service_catalog, extract_terraform, graphql_documents_to_graph, inspect_contracts, inspect_source_syntax, link_declared_implementations, link_declared_tests, link_http_boundaries, link_registered_package_owners, load_extractor_batch_with_budgets, merge_affected_link_neighborhoods, package_manifest_to_graph, parse_event_source, parse_go_source_with_tracker, parse_graphql_source_with_tracker, parse_java_source_with_tracker, parse_javascript_source_at_path_with_tracker, parse_literal_sql_source_at_root, parse_manifest, parse_protobuf_generated_source, parse_python_source_with_tracker, parse_rust_source_with_tracker, parse_typescript_source_at_path_with_tracker, plan_extractor_batches, plan_incremental_scan, precheck_focused_source_values, preview_add_manual_link, preview_add_repository, preview_remove_repository, protobuf_documents_to_graph, register_workspace, resolve_manual_links, resolve_repository_config, search, source_observations_to_graph, store_extractor_batch, traverse
20};
21pub use code_system_graph_core::{
22 ConfigSource, DEFAULT_EXCLUDES, IgnorePolicy, PROTECTED_EXCLUDES
23};
24use code_system_graph_model::{
25 ArtifactFingerprint, CheckoutId, Community, CommunityAlgorithm, CommunityConfig, CommunityDelta, CommunityId, CommunityScope, Edge, EdgeId, EdgeKind, EpistemicStatus, Evidence, EvidenceId, ExtractorRun, ExtractorRunStatus, FreshnessSummary, LinkDecision, LinkStatus, Node, NodeId, NodeKind, OverallFreshness, Provenance, RepoFreshness, RepoFreshnessState, RepoId, RepositoryRecord, StoredExtractorBatch, ToolEnvelope, ToolStatus, TraceReport, WorkspaceRecord, stable_id, stable_id_bytes
26};
27use code_system_graph_store_sqlite::{
28 ManualLinkDisposition, ManualLinkRecord, ProviderCapabilityRecord, QueryCacheRecord, SnapshotBatch, SqliteStore, StoreError, StoreLock, latest_schema_version
29};
30use schemars::JsonSchema;
31use serde::{Deserialize, Serialize};
32pub use sync::{
33 CodeGraphRepositorySync, CodeGraphSyncState, CodeGraphSyncSummary, SyncSummary, SyncTarget, load_persisted_watch_targets, sync_workspace_with_overrides, sync_workspace_with_wall_time_cap, sync_workspace_with_worker_executable, workspace_sync_targets
34};
35use thiserror::Error;
36use tokio_util::sync::CancellationToken;
37#[doc(hidden)]
38pub use worker::{
39 SupervisedProcessGroup, configure_supervised_process_group, run_worker_from_stdio, terminate_process_tree, terminate_supervised_process
40};
41
42const MAX_TRACE_DEPTH: usize = 32;
43const MAX_SCAN_DEGRADATIONS: usize = 25;
44const GENERATED_STATE_IGNORE_RULE: &[u8] = b".code-system-graph/";
45pub(crate) const CODEGRAPH_DISABLED_CODE: &str = "codegraph_disabled";
46pub(crate) const CODEGRAPH_DISABLED_MESSAGE: &str =
47 "CodeGraph is disabled by the trusted process policy.";
48
49#[derive(Debug, Error)]
51pub enum ApplicationError {
52 #[error("failed to read `{path}`: {source}")]
54 ReadFile {
55 path: PathBuf,
57 source: std::io::Error,
59 },
60 #[error("failed to create `{path}`: {source}")]
62 WriteFile {
63 path: PathBuf,
65 source: std::io::Error,
67 },
68 #[error(transparent)]
70 Manifest(#[from] ManifestError),
71 #[error(
73 "advanced global resource policy in `{config}` is inside analyzed repository `{repository}`; move the workspace manifest outside every analyzed checkout"
74 )]
75 UntrustedGlobalPolicySource {
76 config: PathBuf,
78 repository: String,
80 },
81 #[error(transparent)]
83 ExtractionLimit(#[from] ExtractionLimitExceeded),
84 #[error(transparent)]
86 ExecutionLimit(#[from] code_system_graph_core::ExecutionLimitExceeded),
87 #[error(transparent)]
89 HttpExtraction(#[from] HttpExtractionError),
90 #[error(transparent)]
92 PackageManifest(#[from] PackageManifestError),
93 #[error(transparent)]
95 GeneratedClient(#[from] GeneratedClientError),
96 #[error(transparent)]
98 Graphql(#[from] GraphqlExtractionError),
99 #[error(transparent)]
101 Event(#[from] EventExtractionError),
102 #[error(transparent)]
104 Protobuf(#[from] ProtobufExtractionError),
105 #[error(transparent)]
107 Data(#[from] DataExtractionError),
108 #[error(transparent)]
110 Infrastructure(#[from] InfrastructureExtractionError),
111 #[error(transparent)]
113 Documentation(#[from] DocumentationExtractionError),
114 #[error(transparent)]
116 ConfigExtraction(#[from] ConfigExtractionError),
117 #[error(transparent)]
119 SourceSyntax(#[from] SourceSyntaxError),
120 #[error("invalid focused source observation: {0}")]
122 InvalidSourceObservation(String),
123 #[error(transparent)]
125 BatchPlan(#[from] BatchPlanError),
126 #[error(transparent)]
128 Link(#[from] LinkError),
129 #[error(transparent)]
131 ManualLink(#[from] ManualLinkError),
132 #[error(transparent)]
134 Registry(#[from] RegistryError),
135 #[error(transparent)]
137 Config(#[from] ConfigError),
138 #[error(transparent)]
140 ManifestEdit(#[from] ManifestEditError),
141 #[error("validated registry omitted repository alias `{0}`")]
143 RegistryAliasMissing(String),
144 #[error("CLI override references unknown repository alias `{0}`")]
146 UnknownOverrideRepository(String),
147 #[error("workspace name `{requested}` does not match manifest name `{manifest}`")]
149 WorkspaceNameMismatch {
150 requested: String,
152 manifest: String,
154 },
155 #[error("workspace `{0}` is already registered")]
157 WorkspaceAlreadyExists(String),
158 #[error("watcher for workspace `{0}` is already active")]
160 WatcherAlreadyActive(String),
161 #[error("workspace `{0}` is not registered")]
163 WorkspaceNotFound(String),
164 #[error(
166 "extraction budgets changed since the current snapshot; run a full scan without `--repo`"
167 )]
168 PartialScanBudgetChanged,
169 #[error("artifact `{path}` resolves outside checkout `{checkout}`")]
171 ArtifactOutsideCheckout {
172 path: PathBuf,
174 checkout: PathBuf,
176 },
177 #[error("artifact path contains unsafe control or bidirectional characters")]
179 UnsafeArtifactPath,
180 #[error(transparent)]
182 Store(#[from] StoreError),
183 #[error(transparent)]
185 Trace(#[from] TraceError),
186 #[error(transparent)]
188 Community(#[from] CommunityError),
189 #[error(transparent)]
191 Query(#[from] QueryError),
192 #[error(transparent)]
194 Impact(#[from] ImpactError),
195 #[error(transparent)]
197 Change(#[from] ChangeError),
198 #[error(transparent)]
200 ChangeAnalysis(#[from] ChangeAnalysisError),
201 #[error(transparent)]
203 Interface(#[from] InterfaceError),
204 #[error(transparent)]
206 PullRequest(#[from] PullRequestError),
207 #[error("workspace initialization failed: {0}")]
209 Initialization(String),
210 #[error("transient supervised execution failure: {0}")]
212 TransientExecution(String),
213 #[error("supervised execution failed: {message}")]
215 SupervisedApplication {
216 exit_code: ExitCode,
218 message: String,
220 },
221 #[error("trace max_depth must be between 1 and {maximum}; received {found}")]
223 InvalidTraceDepth {
224 found: usize,
226 maximum: usize,
228 },
229}
230
231#[must_use]
233pub const fn application_exit_code(error: &ApplicationError) -> ExitCode {
234 match error {
235 ApplicationError::SupervisedApplication { exit_code, .. } => *exit_code,
236 ApplicationError::Interface(source) => classify_interface_error(source),
237 ApplicationError::ExecutionLimit(code_system_graph_core::ExecutionLimitExceeded {
238 resource: code_system_graph_core::ExecutionResource::Cancellation,
239 ..
240 })
241 | ApplicationError::Change(ChangeError::Cancelled)
242 | ApplicationError::PullRequest(PullRequestError::Cancelled) => ExitCode::Cancelled,
243 ApplicationError::Change(ChangeError::Timeout { .. })
244 | ApplicationError::PullRequest(PullRequestError::Timeout) => ExitCode::Timeout,
245 ApplicationError::WorkspaceNotFound(_) => ExitCode::NotFound,
246 ApplicationError::WorkspaceAlreadyExists(_) | ApplicationError::WatcherAlreadyActive(_) => {
247 ExitCode::Conflict
248 }
249 ApplicationError::Manifest(_)
250 | ApplicationError::UntrustedGlobalPolicySource { .. }
251 | ApplicationError::ExtractionLimit(_)
252 | ApplicationError::ExecutionLimit(_)
253 | ApplicationError::HttpExtraction(_)
254 | ApplicationError::PackageManifest(_)
255 | ApplicationError::GeneratedClient(_)
256 | ApplicationError::Graphql(_)
257 | ApplicationError::Event(_)
258 | ApplicationError::Protobuf(_)
259 | ApplicationError::Data(_)
260 | ApplicationError::Infrastructure(_)
261 | ApplicationError::Documentation(_)
262 | ApplicationError::ConfigExtraction(_)
263 | ApplicationError::SourceSyntax(_)
264 | ApplicationError::InvalidSourceObservation(_)
265 | ApplicationError::BatchPlan(_)
266 | ApplicationError::Link(_)
267 | ApplicationError::ManualLink(_)
268 | ApplicationError::Registry(_)
269 | ApplicationError::Config(_)
270 | ApplicationError::ManifestEdit(_)
271 | ApplicationError::UnknownOverrideRepository(_)
272 | ApplicationError::WorkspaceNameMismatch { .. }
273 | ApplicationError::ArtifactOutsideCheckout { .. }
274 | ApplicationError::UnsafeArtifactPath
275 | ApplicationError::PartialScanBudgetChanged
276 | ApplicationError::Trace(_)
277 | ApplicationError::Community(_)
278 | ApplicationError::Query(_)
279 | ApplicationError::Impact(_)
280 | ApplicationError::Change(_)
281 | ApplicationError::ChangeAnalysis(_)
282 | ApplicationError::PullRequest(_)
283 | ApplicationError::InvalidTraceDepth { .. } => ExitCode::InvalidInput,
284 ApplicationError::ReadFile { .. }
285 | ApplicationError::WriteFile { .. }
286 | ApplicationError::RegistryAliasMissing(_)
287 | ApplicationError::Store(_)
288 | ApplicationError::Initialization(_)
289 | ApplicationError::TransientExecution(_) => ExitCode::Internal,
290 }
291}
292
293pub(crate) fn load_execution_policy(
294 config_path: &Path,
295) -> Result<ExecutionPolicy, ApplicationError> {
296 let manifest_source = read_file(config_path)?;
297 let manifest = parse_manifest(&manifest_source)?;
298 validate_global_policy_source(config_path, &manifest)?;
299 ExecutionPolicy::resolve(manifest.execution_policy.as_ref())
300 .map_err(ManifestError::from)
301 .map_err(Into::into)
302}
303
304#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
306pub struct ScanSummary {
307 pub execution: code_system_graph_core::ExecutionSummary,
309 pub workspace: String,
311 pub snapshot_id: String,
313 pub node_count: usize,
315 pub edge_count: usize,
317 pub evidence_count: usize,
319 pub community_count: usize,
321 pub community_delta_count: usize,
323 pub discovered_input_count: usize,
325 pub changed_input_count: usize,
327 pub reused_snapshot: bool,
329 pub corroborated_symbol_count: usize,
331 pub affected_test_count: usize,
333 pub degradation_count: usize,
335 pub degradations: Vec<String>,
337}
338
339#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
341pub struct ScanOverrides {
342 pub workspace: Option<String>,
344 pub repo_openapi: BTreeMap<String, String>,
346 pub codegraph: bool,
348 pub codegraph_binary: Option<PathBuf>,
350 pub repository: Option<String>,
352 pub force: bool,
354}
355
356#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
358#[serde(rename_all = "snake_case")]
359pub enum IgnoreRuleAction {
360 Exclude,
362 Include,
364}
365
366#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
368#[serde(rename_all = "snake_case")]
369pub enum IgnoreRuleSource {
370 Protected,
372 BuiltInDefault,
374 WorkspaceManifest,
376 RepositoryLocal,
378}
379
380#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
382pub struct EffectiveIgnoreRule {
383 pub action: IgnoreRuleAction,
385 pub source: IgnoreRuleSource,
387 pub pattern: String,
389}
390
391#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
393pub struct ConfiguredPatterns {
394 pub source: ConfigSource,
396 pub patterns: Vec<String>,
398}
399
400#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
402pub struct IgnorePolicyReport {
403 pub protected_excludes: Vec<String>,
405 pub default_excludes: Vec<String>,
407 pub configured_excludes: ConfiguredPatterns,
409 pub include_defaults: ConfiguredPatterns,
411 pub effective_rules: Vec<EffectiveIgnoreRule>,
413}
414
415#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
417pub struct RepositoryConfigReport {
418 pub alias: String,
420 pub path: String,
422 pub ignore_policy: IgnorePolicyReport,
424}
425
426#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
428pub struct ConfigReport {
429 pub schema_version: u8,
431 pub workspace: String,
433 pub extraction_budgets: ExtractionBudgets,
435 pub execution_policy: ExecutionPolicy,
437 pub execution_policy_fingerprint: String,
439 pub repositories: Vec<RepositoryConfigReport>,
441}
442
443#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
445pub struct WorkspaceStatus {
446 pub workspace: String,
448 pub schema_version: i64,
450 pub integrity_ok: bool,
452 pub freshness: FreshnessSummary,
454 pub repositories: Vec<RepoFreshness>,
456 pub watcher: WatcherStatus,
458}
459
460#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
462#[serde(rename_all = "snake_case")]
463pub enum WatcherState {
464 Active,
466 ExpiredIdle,
468 ExpiredSession,
470 FailedLimit,
472 Stale,
474 NeverStarted,
476}
477
478#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
480pub struct WatcherStatus {
481 pub state: WatcherState,
483 pub detail: Option<String>,
485}
486
487#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
489pub struct BackupSummary {
490 pub database: String,
492 pub backup: String,
494}
495
496#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
498pub struct RestoreSummary {
499 pub source: String,
501 pub safety_backup: Option<String>,
503 pub schema_version: i64,
505}
506
507#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
509pub struct WorkspaceRegistryItem {
510 pub id: String,
512 pub name: String,
514 pub manifest_hash: String,
516 pub config_path: Option<String>,
518 pub repository_count: usize,
520}
521
522#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
524pub struct ManifestMutationSummary {
525 pub summary: String,
527 pub manifest: String,
529 pub backup: Option<String>,
531 pub applied: bool,
533 pub rendered_manifest: Option<String>,
535}
536
537#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
539pub struct WorkspaceMutationSummary {
540 pub workspace: String,
542 pub config_path: Option<String>,
544 pub repository_count: usize,
546 pub operation: String,
548}
549
550#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
552pub struct TraceInput {
553 pub from: String,
555 pub to: String,
557 #[serde(default = "default_max_depth")]
559 #[schemars(range(min = 1, max = 32))]
560 pub max_depth: usize,
561}
562
563#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
565pub struct SearchInput {
566 pub query: String,
568 #[serde(default)]
570 pub node_kinds: Vec<NodeKind>,
571 #[serde(default)]
573 pub repo_ids: Vec<RepoId>,
574 #[serde(default)]
576 pub service_ids: Vec<NodeId>,
577 #[serde(default)]
579 pub community_ids: Vec<CommunityId>,
580 #[serde(default)]
582 pub offset: usize,
583 #[serde(default = "default_search_limit")]
585 #[schemars(range(min = 1, max = 100))]
586 pub limit: usize,
587}
588
589#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
591pub struct CommunityInput {
592 pub community_id: Option<CommunityId>,
594 pub compare_snapshot_id: Option<String>,
596 #[serde(default)]
598 pub offset: usize,
599 #[serde(default = "default_community_limit")]
601 #[schemars(range(min = 1, max = 100))]
602 pub limit: usize,
603}
604
605#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
607pub struct CommunityReport {
608 pub snapshot_id: String,
610 pub engine_version: String,
612 pub config: CommunityConfig,
614 pub communities: Vec<Community>,
616 pub total_communities: usize,
618 pub offset: usize,
620 pub limit: usize,
622 pub truncated: bool,
624 pub delta: Option<CommunityDelta>,
626}
627
628#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
630pub struct ChangesInput {
631 pub repository: String,
633 pub scope: ChangeScope,
635}
636
637#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
639#[serde(deny_unknown_fields)]
640pub struct ExploreInput {
641 pub workspace: String,
643 #[serde(default)]
645 pub repository: Option<String>,
646 pub query: String,
648 #[serde(default = "default_explore_max_files")]
650 #[schemars(range(min = 1, max = 25))]
651 pub max_files: usize,
652}
653
654const fn default_explore_max_files() -> usize {
655 12
656}
657
658#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
660pub struct PullRequestInput {
661 pub provider: PullRequestProviderKind,
663 pub owner: String,
665 pub repository: String,
667 pub number: u64,
669 pub consent_to_remote_access: bool,
671}
672
673#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
675pub struct PullRequestListInput {
676 pub provider: PullRequestProviderKind,
678 pub owner: String,
680 pub repository: String,
682 pub state: PullRequestListState,
684 pub cursor: Option<String>,
686 pub limit: usize,
688 pub consent_to_remote_access: bool,
690}
691
692#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
694pub struct InitReport {
695 pub manifest_path: PathBuf,
697 pub gitignore_path: Option<PathBuf>,
699 pub gitignore_updated: bool,
701 pub workspace: String,
703 pub manifest_version: u32,
705}
706
707#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
709pub struct DiagnosticBundle {
710 pub schema_version: u32,
712 pub binary_version: String,
714 pub operating_system: String,
716 pub architecture: String,
718 pub generated_at_unix_ms: u128,
720 pub debug_requested: bool,
722 pub doctor: DoctorReport,
724}
725
726struct WorkspaceContext {
727 manifest: WorkspaceManifest,
728 registry: RegisteredWorkspace,
729 repository_configs: BTreeMap<String, EffectiveRepositoryConfig>,
730 extraction_budgets: ExtractionBudgets,
731 execution_policy: ExecutionPolicy,
732}
733
734pub fn show_config(
741 config_path: &Path,
742 selected_repository: Option<&str>,
743) -> Result<ConfigReport, ApplicationError> {
744 let context = load_workspace_context(config_path, &ScanOverrides::default())?;
745 if let Some(selected) = selected_repository
746 && !context.manifest.repos.contains_key(selected)
747 {
748 return Err(ApplicationError::UnknownOverrideRepository(
749 selected.to_owned(),
750 ));
751 }
752 let repositories = context
753 .manifest
754 .repos
755 .keys()
756 .filter(|alias| selected_repository.is_none_or(|selected| selected == alias.as_str()))
757 .map(|alias| {
758 let path = context
759 .registry
760 .checkout_path(alias)
761 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
762 let effective = context
763 .repository_configs
764 .get(alias)
765 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
766 Ok(RepositoryConfigReport {
767 alias: alias.clone(),
768 path: path.display().to_string(),
769 ignore_policy: ignore_policy_report(&effective.ignore_policy),
770 })
771 })
772 .collect::<Result<Vec<_>, ApplicationError>>()?;
773 Ok(ConfigReport {
774 schema_version: 1,
775 workspace: context.manifest.name,
776 extraction_budgets: context.extraction_budgets,
777 execution_policy_fingerprint: context.execution_policy.fingerprint(),
778 execution_policy: context.execution_policy,
779 repositories,
780 })
781}
782
783fn ignore_policy_report(policy: &IgnorePolicy) -> IgnorePolicyReport {
784 let configured_source = rule_source(policy.configured_excludes_source());
785 let include_source = rule_source(policy.include_defaults_source());
786 let mut effective_rules = DEFAULT_EXCLUDES
787 .iter()
788 .map(|pattern| EffectiveIgnoreRule {
789 action: IgnoreRuleAction::Exclude,
790 source: IgnoreRuleSource::BuiltInDefault,
791 pattern: (*pattern).to_owned(),
792 })
793 .chain(
794 policy
795 .include_defaults()
796 .iter()
797 .map(|pattern| EffectiveIgnoreRule {
798 action: IgnoreRuleAction::Include,
799 source: include_source,
800 pattern: pattern.clone(),
801 }),
802 )
803 .chain(
804 policy
805 .configured_excludes()
806 .iter()
807 .map(|pattern| EffectiveIgnoreRule {
808 action: IgnoreRuleAction::Exclude,
809 source: configured_source,
810 pattern: pattern.clone(),
811 }),
812 )
813 .collect::<Vec<_>>();
814 effective_rules.extend(
815 PROTECTED_EXCLUDES
816 .iter()
817 .map(|pattern| EffectiveIgnoreRule {
818 action: IgnoreRuleAction::Exclude,
819 source: IgnoreRuleSource::Protected,
820 pattern: (*pattern).to_owned(),
821 }),
822 );
823 IgnorePolicyReport {
824 protected_excludes: PROTECTED_EXCLUDES
825 .iter()
826 .map(|pattern| (*pattern).to_owned())
827 .collect(),
828 default_excludes: DEFAULT_EXCLUDES
829 .iter()
830 .map(|pattern| (*pattern).to_owned())
831 .collect(),
832 configured_excludes: ConfiguredPatterns {
833 source: policy.configured_excludes_source(),
834 patterns: policy.configured_excludes().to_vec(),
835 },
836 include_defaults: ConfiguredPatterns {
837 source: policy.include_defaults_source(),
838 patterns: policy.include_defaults().to_vec(),
839 },
840 effective_rules,
841 }
842}
843
844const fn rule_source(source: ConfigSource) -> IgnoreRuleSource {
845 match source {
846 ConfigSource::WorkspaceManifest => IgnoreRuleSource::WorkspaceManifest,
847 ConfigSource::RepositoryLocal => IgnoreRuleSource::RepositoryLocal,
848 ConfigSource::CliOverride | ConfigSource::AutoDetected | ConfigSource::Default => {
849 IgnoreRuleSource::BuiltInDefault
850 }
851 }
852}
853
854struct GraphAssembly {
855 nodes: Vec<Node>,
856 edges: Vec<Edge>,
857 evidence: Vec<Evidence>,
858 link_decisions: Vec<LinkDecision>,
859 link_node_keys: BTreeMap<NodeId, String>,
860}
861
862struct FocusedBatchState {
863 source_batches: Vec<ExtractorBatch<SourceObservation>>,
864 previous_source_batches: Vec<ExtractorBatch<SourceObservation>>,
865 package_batches: Vec<ExtractorBatch<PackageManifest>>,
866 generated_client_batches: Vec<ExtractorBatch<GeneratedClientMetadata>>,
867 graphql_batches: Vec<ExtractorBatch<GraphqlDocument>>,
868 event_batches: Vec<ExtractorBatch<EventDocument>>,
869 protobuf_batches: Vec<ExtractorBatch<ProtobufDocument>>,
870 data_batches: Vec<ExtractorBatch<DataDocument>>,
871 infrastructure_batches: Vec<ExtractorBatch<InfrastructureDocument>>,
872 documentation_batches: Vec<ExtractorBatch<DocumentationDocument>>,
873 config_batches: Vec<ExtractorBatch<SafeConfigDocument>>,
874 stored_batches: Vec<StoredExtractorBatch>,
875 degradations: Vec<String>,
876 force_relink: bool,
877 checkpoint_writes: u64,
878 artifact_durations_ms: Vec<u64>,
879}
880
881struct RepositoryCorroboration {
882 repo_id: RepoId,
883 report: CorroborationReport,
884}
885
886#[derive(Default)]
887struct CorroborationSummary {
888 reports: Vec<RepositoryCorroboration>,
889 confirmed_symbols: usize,
890 affected_tests: usize,
891 degradations: Vec<String>,
892}
893
894fn default_max_depth() -> usize {
895 8
896}
897
898fn current_unix_millis() -> u64 {
899 SystemTime::now()
900 .duration_since(UNIX_EPOCH)
901 .ok()
902 .and_then(|duration| u64::try_from(duration.as_millis()).ok())
903 .unwrap_or(u64::MAX)
904}
905
906fn duration_millis(duration: Duration) -> u64 {
907 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
908}
909
910fn artifact_execution_summary(durations_ms: &[u64]) -> code_system_graph_core::ExecutionSummary {
911 let mut ordered = durations_ms.to_vec();
912 ordered.sort_unstable();
913 code_system_graph_core::ExecutionSummary {
914 measured_artifacts: u64::try_from(ordered.len()).unwrap_or(u64::MAX),
915 artifact_duration_p50_ms: duration_percentile(&ordered, 50),
916 artifact_duration_p95_ms: duration_percentile(&ordered, 95),
917 artifact_duration_p99_ms: duration_percentile(&ordered, 99),
918 ..code_system_graph_core::ExecutionSummary::default()
919 }
920}
921
922fn duration_percentile(ordered: &[u64], percentile: usize) -> u64 {
923 if ordered.is_empty() {
924 return 0;
925 }
926 let index = ordered
927 .len()
928 .saturating_mul(percentile)
929 .div_ceil(100)
930 .saturating_sub(1)
931 .min(ordered.len() - 1);
932 ordered[index]
933}
934
935fn default_search_limit() -> usize {
936 20
937}
938
939fn default_community_limit() -> usize {
940 20
941}
942
943fn default_community_config() -> CommunityConfig {
944 CommunityConfig {
945 algorithm: CommunityAlgorithm::Louvain,
946 scope: CommunityScope::Federated,
947 seed: 0,
948 resolution: 1.0,
949 minimum_confidence: 0.5,
950 edge_weights: Vec::new(),
951 max_iterations: 100,
952 }
953}
954
955pub fn initialize_workspace(
966 directory: &Path,
967 requested_name: Option<&str>,
968) -> Result<InitReport, ApplicationError> {
969 fs::create_dir_all(directory).map_err(|error| {
970 ApplicationError::Initialization(format!(
971 "cannot create `{}`: {error}",
972 directory.display()
973 ))
974 })?;
975 let workspace = requested_name.map_or_else(
976 || {
977 directory
978 .file_name()
979 .and_then(OsStr::to_str)
980 .filter(|name| !name.is_empty())
981 .unwrap_or("workspace")
982 .to_owned()
983 },
984 str::to_owned,
985 );
986 let encoded_name = serde_json::to_string(&workspace)
987 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
988 let source = format!("version: 1\nname: {encoded_name}\nrepos:\n root:\n path: .\n");
989 let manifest = parse_manifest(&source)?;
990 let manifest_path = directory.join("code-system-graph.yaml");
991 let mut destination = fs::OpenOptions::new()
992 .write(true)
993 .create_new(true)
994 .open(&manifest_path)
995 .map_err(|error| {
996 ApplicationError::Initialization(format!(
997 "cannot create `{}` without overwriting: {error}",
998 manifest_path.display()
999 ))
1000 })?;
1001 destination.write_all(source.as_bytes()).map_err(|error| {
1002 ApplicationError::Initialization(format!(
1003 "cannot write `{}`: {error}",
1004 manifest_path.display()
1005 ))
1006 })?;
1007 destination.sync_all().map_err(|error| {
1008 ApplicationError::Initialization(format!(
1009 "cannot synchronize `{}`: {error}",
1010 manifest_path.display()
1011 ))
1012 })?;
1013 drop(destination);
1014 let (gitignore_path, gitignore_updated) = configure_generated_state_ignore(directory)
1015 .inspect_err(|_| {
1016 let _ = fs::remove_file(&manifest_path);
1017 })?;
1018 Ok(InitReport {
1019 manifest_path,
1020 gitignore_path,
1021 gitignore_updated,
1022 workspace: manifest.name,
1023 manifest_version: manifest.version,
1024 })
1025}
1026
1027fn configure_generated_state_ignore(
1028 directory: &Path,
1029) -> Result<(Option<PathBuf>, bool), ApplicationError> {
1030 let canonical_directory = fs::canonicalize(directory).map_err(|error| {
1031 ApplicationError::Initialization(format!(
1032 "cannot resolve workspace directory `{}`: {error}",
1033 directory.display()
1034 ))
1035 })?;
1036 if !belongs_to_git_worktree(&canonical_directory) {
1037 return Ok((None, false));
1038 }
1039 let (path, updated) = ensure_generated_state_ignored(directory)?;
1040 Ok((Some(path), updated))
1041}
1042
1043fn belongs_to_git_worktree(directory: &Path) -> bool {
1044 directory
1045 .ancestors()
1046 .any(|ancestor| valid_git_worktree_marker(&ancestor.join(".git")))
1047}
1048
1049fn valid_git_worktree_marker(marker: &Path) -> bool {
1050 let Ok(metadata) = fs::symlink_metadata(marker) else {
1051 return false;
1052 };
1053 if metadata.file_type().is_dir() {
1054 return fs::symlink_metadata(marker.join("HEAD"))
1055 .is_ok_and(|head| head.file_type().is_file());
1056 }
1057 if !metadata.file_type().is_file() || metadata.len() > 4_096 {
1058 return false;
1059 }
1060 let Ok(source) = fs::read_to_string(marker) else {
1061 return false;
1062 };
1063 source
1064 .lines()
1065 .next()
1066 .is_some_and(|line| line.trim_start().starts_with("gitdir:"))
1067}
1068
1069fn ensure_generated_state_ignored(directory: &Path) -> Result<(PathBuf, bool), ApplicationError> {
1070 let gitignore_path = directory.join(".gitignore");
1071 let mut content = match fs::read(&gitignore_path) {
1072 Ok(content) => content,
1073 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
1074 Err(error) => {
1075 return Err(ApplicationError::Initialization(format!(
1076 "cannot read `{}` before updating generated-state rules: {error}",
1077 gitignore_path.display()
1078 )));
1079 }
1080 };
1081 if generated_state_is_ignored(&content) {
1082 return Ok((gitignore_path, false));
1083 }
1084 if !content.is_empty() && !content.ends_with(b"\n") {
1085 content.push(b'\n');
1086 }
1087 content.extend_from_slice(GENERATED_STATE_IGNORE_RULE);
1088 content.push(b'\n');
1089
1090 let mut destination = AtomicWriteFile::open(&gitignore_path).map_err(|error| {
1091 ApplicationError::Initialization(format!(
1092 "cannot prepare `{}` for generated-state rules: {error}",
1093 gitignore_path.display()
1094 ))
1095 })?;
1096 destination
1097 .write_all(&content)
1098 .and_then(|()| destination.sync_all())
1099 .map_err(|error| {
1100 ApplicationError::Initialization(format!(
1101 "cannot write `{}`: {error}",
1102 gitignore_path.display()
1103 ))
1104 })?;
1105 destination.commit().map_err(|error| {
1106 ApplicationError::Initialization(format!(
1107 "cannot commit `{}`: {error}",
1108 gitignore_path.display()
1109 ))
1110 })?;
1111 Ok((gitignore_path, true))
1112}
1113
1114fn generated_state_is_ignored(content: &[u8]) -> bool {
1115 content
1116 .split(|byte| *byte == b'\n')
1117 .map(|line| line.strip_suffix(b"\r").map_or(line, |trimmed| trimmed))
1118 .fold(None, |state, line| match line {
1119 b".code-system-graph/"
1120 | b"/.code-system-graph/"
1121 | b".code-system-graph"
1122 | b"/.code-system-graph" => Some(true),
1123 b"!.code-system-graph/"
1124 | b"!/.code-system-graph/"
1125 | b"!.code-system-graph"
1126 | b"!/.code-system-graph" => Some(false),
1127 _ => state,
1128 })
1129 .unwrap_or(false)
1130}
1131
1132pub fn scan_workspace(
1139 config_path: &Path,
1140 database_path: &Path,
1141) -> Result<ScanSummary, ApplicationError> {
1142 scan_workspace_with_overrides(config_path, database_path, &ScanOverrides::default())
1143}
1144
1145pub fn scan_workspace_with_overrides(
1152 config_path: &Path,
1153 database_path: &Path,
1154 overrides: &ScanOverrides,
1155) -> Result<ScanSummary, ApplicationError> {
1156 worker::supervise_scan(config_path, database_path, overrides)
1157}
1158
1159pub fn scan_workspace_with_worker_executable(
1169 config_path: &Path,
1170 database_path: &Path,
1171 overrides: &ScanOverrides,
1172 worker_executable: &Path,
1173) -> Result<ScanSummary, ApplicationError> {
1174 worker::supervise_scan_with_executable(config_path, database_path, overrides, worker_executable)
1175}
1176
1177#[doc(hidden)]
1178#[expect(
1179 clippy::too_many_lines,
1180 reason = "Atomic scan orchestration keeps lock, resume, publication, and summary sequencing visible"
1181)]
1182pub(crate) fn scan_workspace_direct(
1183 config_path: &Path,
1184 database_path: &Path,
1185 overrides: &ScanOverrides,
1186) -> Result<ScanSummary, ApplicationError> {
1187 let context = load_workspace_context(config_path, overrides)?;
1188 worker::report_progress(code_system_graph_core::JobPhase::Configuration, 1);
1189 if let Some(requested) = &overrides.workspace
1190 && requested != &context.manifest.name
1191 {
1192 return Err(ApplicationError::WorkspaceNameMismatch {
1193 requested: requested.clone(),
1194 manifest: context.manifest.name,
1195 });
1196 }
1197 let mut fingerprints = discover_artifact_fingerprints(&context)?;
1198 worker::report_progress(
1199 code_system_graph_core::JobPhase::Fingerprinting,
1200 u64::try_from(fingerprints.len()).unwrap_or(u64::MAX),
1201 );
1202 let _writer_lock = StoreLock::acquire(database_path, Duration::from_mins(5))?;
1203 let mut store = SqliteStore::open(database_path)?;
1204 let database_instance_id = store.database_instance_id()?;
1205 let mut work_state = work_state::WorkState::open(database_path, &database_instance_id)
1206 .map_err(ApplicationError::Initialization)?;
1207 let mut previous_fingerprints =
1208 match store.load_current_artifact_fingerprints(&context.manifest.name) {
1209 Ok(previous) => previous,
1210 Err(StoreError::CurrentSnapshotMissing(_)) => Vec::new(),
1211 Err(error) => return Err(error.into()),
1212 };
1213 if let Some(alias) = &overrides.repository {
1214 let selected = context
1215 .registry
1216 .record
1217 .repositories
1218 .iter()
1219 .find(|repository| &repository.alias == alias)
1220 .ok_or_else(|| ApplicationError::UnknownOverrideRepository(alias.clone()))?
1221 .id
1222 .clone();
1223 fingerprints.retain(|fingerprint| fingerprint.repo_id == selected);
1224 fingerprints.extend(
1225 previous_fingerprints
1226 .iter()
1227 .filter(|fingerprint| fingerprint.repo_id != selected)
1228 .cloned(),
1229 );
1230 fingerprints.sort_by_key(|fingerprint| ArtifactKey::from(fingerprint));
1231 if overrides.force {
1232 previous_fingerprints.retain(|fingerprint| fingerprint.repo_id != selected);
1233 }
1234 } else if overrides.force {
1235 previous_fingerprints.clear();
1236 }
1237 let fingerprint_hashes = fingerprints
1238 .iter()
1239 .map(|fingerprint| fingerprint.content_hash.as_str())
1240 .collect::<Vec<_>>()
1241 .join(":");
1242 let snapshot_id = stable_id(
1243 "snapshot",
1244 &format!(
1245 "{}:{fingerprint_hashes}:community-engine-v1",
1246 context.registry.record.manifest_hash
1247 ),
1248 );
1249 let mut previous_extractor_batches = match store.load_current_extractor_batches_with_limit(
1250 &context.manifest.name,
1251 context
1252 .extraction_budgets
1253 .max_serialized_output_bytes_per_artifact,
1254 ) {
1255 Ok(previous) => previous,
1256 Err(StoreError::CurrentSnapshotMissing(_)) => Vec::new(),
1257 Err(error) => return Err(error.into()),
1258 };
1259 let budget_fingerprint = context.extraction_budgets.fingerprint();
1260 let candidate_material = serde_json::to_vec(&(
1261 &context.registry.record.manifest_hash,
1262 &fingerprints,
1263 &budget_fingerprint,
1264 EXTRACTION_CONTRACT_VERSION,
1265 ))
1266 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
1267 let candidate_fingerprint = stable_id_bytes("work-candidate-v1", &candidate_material);
1268 let _resumed_candidate = work_state
1269 .begin_candidate(
1270 &context.manifest.name,
1271 &candidate_fingerprint,
1272 current_unix_millis(),
1273 )
1274 .map_err(ApplicationError::Initialization)?;
1275 let cached_batches = if overrides.force {
1276 Vec::new()
1277 } else {
1278 work_state
1279 .load_batches(
1280 &fingerprints,
1281 &budget_fingerprint,
1282 EXTRACTION_CONTRACT_VERSION,
1283 context
1284 .extraction_budgets
1285 .max_serialized_output_bytes_per_artifact,
1286 current_unix_millis(),
1287 )
1288 .map_err(ApplicationError::Initialization)?
1289 };
1290 let cached_keys = cached_batches
1291 .iter()
1292 .map(|batch| ArtifactKey::from(&batch.source))
1293 .collect::<BTreeSet<_>>();
1294 for cached in cached_batches {
1295 let key = ArtifactKey::from(&cached.source);
1296 if !previous_extractor_batches
1297 .iter()
1298 .any(|batch| ArtifactKey::from(&batch.source) == key && batch.source == cached.source)
1299 {
1300 previous_extractor_batches.push(cached);
1301 }
1302 }
1303 let checkpoint_hits = u64::try_from(cached_keys.len()).unwrap_or(u64::MAX);
1304 if overrides.repository.is_some()
1305 && previous_extractor_batches
1306 .iter()
1307 .any(|batch| batch.budget_fingerprint != budget_fingerprint)
1308 {
1309 return Err(ApplicationError::PartialScanBudgetChanged);
1310 }
1311 let previous_graph = match store.load_current_graph(&context.manifest.name) {
1312 Ok(graph) => graph,
1313 Err(StoreError::CurrentSnapshotMissing(_)) => (Vec::new(), Vec::new()),
1314 Err(error) => return Err(error.into()),
1315 };
1316 let previous_communities = match store.load_current_community_snapshot(&context.manifest.name) {
1317 Ok(snapshot) => Some(snapshot),
1318 Err(StoreError::CurrentSnapshotMissing(_) | StoreError::CommunitySnapshotMissing(_)) => {
1319 None
1320 }
1321 Err(error) => return Err(error.into()),
1322 };
1323 let previous_manifest_matches = match store.load_workspace_registry(&context.manifest.name) {
1324 Ok(previous) => previous.manifest_hash == context.registry.record.manifest_hash,
1325 Err(StoreError::RegistryIncomplete(_)) => false,
1326 Err(error) => return Err(error.into()),
1327 };
1328 let plan = plan_incremental_scan(&previous_fingerprints, &fingerprints);
1329 let staged_snapshot = if let Ok(candidate) =
1330 work_state.load_candidate_snapshot(&context.manifest.name, &candidate_fingerprint)
1331 {
1332 candidate
1333 } else {
1334 work_state
1335 .complete_candidate(&context.manifest.name)
1336 .map_err(ApplicationError::Initialization)?;
1337 work_state
1338 .begin_candidate(
1339 &context.manifest.name,
1340 &candidate_fingerprint,
1341 current_unix_millis(),
1342 )
1343 .map_err(ApplicationError::Initialization)?;
1344 None
1345 };
1346 if let Some(candidate) = staged_snapshot {
1347 publish_snapshot_candidate(
1348 &mut store,
1349 SnapshotBatch {
1350 workspace: &context.registry.record,
1351 snapshot_id: &candidate.snapshot_id,
1352 nodes: &candidate.nodes,
1353 edges: &candidate.edges,
1354 evidence: &candidate.evidence,
1355 fingerprints: &candidate.fingerprints,
1356 extractor_batches: &candidate.extractor_batches,
1357 extractor_runs: &candidate.extractor_runs,
1358 manual_links: &candidate.manual_links,
1359 community_snapshot: Some(&candidate.community_snapshot),
1360 },
1361 )?;
1362 work_state
1363 .complete_candidate(&context.manifest.name)
1364 .map_err(ApplicationError::Initialization)?;
1365 let mut execution = candidate.execution;
1366 execution.checkpoint_hits = checkpoint_hits;
1367 return Ok(ScanSummary {
1368 execution,
1369 workspace: context.manifest.name,
1370 snapshot_id: candidate.snapshot_id,
1371 node_count: candidate.nodes.len(),
1372 edge_count: candidate.edges.len(),
1373 evidence_count: candidate.evidence.len(),
1374 community_count: candidate.community_snapshot.communities.len(),
1375 community_delta_count: candidate.community_delta_count,
1376 discovered_input_count: candidate.fingerprints.len(),
1377 changed_input_count: plan.changed_count(),
1378 reused_snapshot: false,
1379 corroborated_symbol_count: candidate.corroborated_symbol_count,
1380 affected_test_count: candidate.affected_test_count,
1381 degradation_count: candidate.degradations.len(),
1382 degradations: candidate.degradations,
1383 });
1384 }
1385 if previous_manifest_matches
1386 && !plan.has_changes()
1387 && focused_batch_cache_complete(
1388 &fingerprints,
1389 &previous_extractor_batches,
1390 &context.extraction_budgets,
1391 )
1392 && previous_communities.is_some()
1393 && !overrides.codegraph
1394 {
1395 work_state
1396 .complete_candidate(&context.manifest.name)
1397 .map_err(ApplicationError::Initialization)?;
1398 let current = store.current_snapshot_summary(&context.manifest.name)?;
1399 let (degradation_count, degradations) = finalize_scan_degradations(
1400 stored_batch_degradations(&previous_extractor_batches, &context.extraction_budgets)?,
1401 );
1402 return Ok(ScanSummary {
1403 execution: code_system_graph_core::ExecutionSummary {
1404 checkpoint_hits,
1405 ..code_system_graph_core::ExecutionSummary::default()
1406 },
1407 workspace: context.manifest.name,
1408 snapshot_id: current.snapshot_id,
1409 node_count: current.node_count,
1410 edge_count: current.edge_count,
1411 evidence_count: current.evidence_count,
1412 community_count: previous_communities
1413 .as_ref()
1414 .map_or(0, |snapshot| snapshot.communities.len()),
1415 community_delta_count: 0,
1416 discovered_input_count: fingerprints.len(),
1417 changed_input_count: 0,
1418 reused_snapshot: true,
1419 corroborated_symbol_count: 0,
1420 affected_test_count: 0,
1421 degradation_count,
1422 degradations,
1423 });
1424 }
1425
1426 let batch_plan = plan_extractor_batches(&plan);
1427 let focused_batches = assemble_focused_batches(
1428 &context,
1429 &fingerprints,
1430 &previous_extractor_batches,
1431 &batch_plan,
1432 &cached_keys,
1433 &mut work_state,
1434 )?;
1435 work_state
1436 .set_candidate_phase(&context.manifest.name, "extracted", current_unix_millis())
1437 .map_err(ApplicationError::Initialization)?;
1438 worker::report_progress(
1439 code_system_graph_core::JobPhase::Extraction,
1440 u64::try_from(focused_batches.stored_batches.len()).unwrap_or(u64::MAX),
1441 );
1442 let mut graph = assemble_graph(&context, &fingerprints, &focused_batches)?;
1443 relink_affected_graph(
1444 &mut graph,
1445 &plan,
1446 &batch_plan,
1447 &focused_batches,
1448 &previous_graph.0,
1449 &previous_graph.1,
1450 )?;
1451 let corroboration = if overrides.codegraph {
1452 run_codegraph_corroboration(
1453 &context,
1454 &focused_batches,
1455 &plan,
1456 overrides.codegraph_binary.clone(),
1457 )
1458 } else {
1459 CorroborationSummary::default()
1460 };
1461 apply_codegraph_corroboration(&mut graph, &corroboration.reports);
1462 let manual_links =
1463 resolve_manual_links(&context.manifest.manual_links, &graph.nodes, &graph.edges)?;
1464 graph.edges = manual_links.edges;
1465 graph.evidence.extend(manual_links.evidence);
1466 graph.evidence.sort_by(|left, right| left.id.cmp(&right.id));
1467 graph.evidence.dedup_by(|left, right| left.id == right.id);
1468 graph.link_decisions = manual_links.decisions;
1469 worker::report_progress(
1470 code_system_graph_core::JobPhase::GraphAssembly,
1471 u64::try_from(graph.nodes.len().saturating_add(graph.edges.len())).unwrap_or(u64::MAX),
1472 );
1473 let GraphAssembly {
1474 nodes,
1475 edges,
1476 evidence,
1477 link_decisions,
1478 ..
1479 } = graph;
1480 let community_config = default_community_config();
1481 let community_snapshot = previous_communities
1482 .as_ref()
1483 .filter(|previous| {
1484 previous.snapshot_id == snapshot_id
1485 && previous.config == community_config
1486 && community_topology_unchanged(
1487 &previous_graph.0,
1488 &previous_graph.1,
1489 &nodes,
1490 &edges,
1491 )
1492 })
1493 .cloned()
1494 .map_or_else(
1495 || {
1496 analyze_communities_with_progress(
1497 &snapshot_id,
1498 &nodes,
1499 &edges,
1500 community_config.clone(),
1501 |completed| {
1502 worker::report_progress(
1503 code_system_graph_core::JobPhase::Communities,
1504 completed,
1505 );
1506 },
1507 )
1508 },
1509 Ok,
1510 )?;
1511 let community_delta_count = previous_communities.as_ref().map_or(0, |previous| {
1512 compare_community_snapshots(previous, &community_snapshot)
1513 .changes
1514 .len()
1515 });
1516 worker::report_progress(
1517 code_system_graph_core::JobPhase::Communities,
1518 u64::try_from(community_snapshot.communities.len()).unwrap_or(u64::MAX),
1519 );
1520 let extractor_runs = extractor_runs(&snapshot_id, &fingerprints, &plan);
1521 let manual_link_records = persisted_manual_link_records(&snapshot_id, &link_decisions)?;
1522 let mut staged_degradations = focused_batches.degradations.clone();
1523 staged_degradations.extend(corroboration.degradations.clone());
1524 staged_degradations.extend(stored_batch_degradations(
1525 &focused_batches.stored_batches,
1526 &context.extraction_budgets,
1527 )?);
1528 let (_, staged_degradations) = finalize_scan_degradations(staged_degradations);
1529 let mut artifact_execution = artifact_execution_summary(&focused_batches.artifact_durations_ms);
1530 artifact_execution.checkpoint_hits = checkpoint_hits;
1531 artifact_execution.checkpoints_written = focused_batches.checkpoint_writes;
1532 let candidate = work_state::StagedSnapshot {
1533 snapshot_id,
1534 nodes,
1535 edges,
1536 evidence,
1537 fingerprints,
1538 extractor_batches: focused_batches.stored_batches,
1539 extractor_runs,
1540 manual_links: manual_link_records,
1541 community_snapshot,
1542 community_delta_count,
1543 corroborated_symbol_count: corroboration.confirmed_symbols,
1544 affected_test_count: corroboration.affected_tests,
1545 execution: artifact_execution,
1546 degradations: staged_degradations,
1547 };
1548 work_state
1549 .store_candidate_snapshot(
1550 &context.manifest.name,
1551 &candidate_fingerprint,
1552 &candidate,
1553 current_unix_millis(),
1554 )
1555 .map_err(ApplicationError::Initialization)?;
1556 publish_snapshot_candidate(
1557 &mut store,
1558 SnapshotBatch {
1559 workspace: &context.registry.record,
1560 snapshot_id: &candidate.snapshot_id,
1561 nodes: &candidate.nodes,
1562 edges: &candidate.edges,
1563 evidence: &candidate.evidence,
1564 fingerprints: &candidate.fingerprints,
1565 extractor_batches: &candidate.extractor_batches,
1566 extractor_runs: &candidate.extractor_runs,
1567 manual_links: &candidate.manual_links,
1568 community_snapshot: Some(&candidate.community_snapshot),
1569 },
1570 )?;
1571 let mut degradations = candidate.degradations.clone();
1572 for item in &corroboration.reports {
1573 let Some(capability) = &item.report.capability else {
1574 continue;
1575 };
1576 let record = provider_capability_record(&context.manifest.name, &item.repo_id, capability);
1577 if let Err(error) = store.upsert_provider_capabilities(&record) {
1578 degradations.push(format!(
1579 "CodeGraph capability persistence for `{}` degraded: {error}",
1580 item.repo_id.as_str()
1581 ));
1582 }
1583 }
1584 let (degradation_count, degradations) = finalize_scan_degradations(degradations);
1585 let _ = work_state.complete_candidate(&context.manifest.name);
1586 Ok(ScanSummary {
1587 execution: candidate.execution,
1588 workspace: context.manifest.name,
1589 snapshot_id: candidate.snapshot_id,
1590 node_count: candidate.nodes.len(),
1591 edge_count: candidate.edges.len(),
1592 evidence_count: candidate.evidence.len(),
1593 community_count: candidate.community_snapshot.communities.len(),
1594 community_delta_count: candidate.community_delta_count,
1595 discovered_input_count: candidate.fingerprints.len(),
1596 changed_input_count: plan.changed_count(),
1597 reused_snapshot: false,
1598 corroborated_symbol_count: candidate.corroborated_symbol_count,
1599 affected_test_count: candidate.affected_test_count,
1600 degradation_count,
1601 degradations,
1602 })
1603}
1604
1605fn publish_snapshot_candidate(
1606 store: &mut SqliteStore,
1607 batch: SnapshotBatch<'_>,
1608) -> Result<(), StoreError> {
1609 const REPORT_INTERVAL: u64 = 1_024;
1610 let mut pending = 0_u64;
1611 worker::report_progress(code_system_graph_core::JobPhase::Publication, 1);
1612 store.publish_snapshot_with_progress(batch, |rows| {
1613 worker::check_time(code_system_graph_core::JobPhase::Publication);
1614 pending = pending.saturating_add(rows);
1615 if pending >= REPORT_INTERVAL {
1616 worker::report_progress(code_system_graph_core::JobPhase::Publication, pending);
1617 pending = 0;
1618 }
1619 })?;
1620 Ok(())
1621}
1622
1623pub fn trace_workspace(
1629 database_path: &Path,
1630 workspace: &str,
1631 input: &TraceInput,
1632) -> Result<ToolEnvelope<TraceReport>, ApplicationError> {
1633 if !(1..=MAX_TRACE_DEPTH).contains(&input.max_depth) {
1634 return Err(ApplicationError::InvalidTraceDepth {
1635 found: input.max_depth,
1636 maximum: MAX_TRACE_DEPTH,
1637 });
1638 }
1639 let store = SqliteStore::open_read_only(database_path)?;
1640 let (nodes, edges) = store.load_current_graph(workspace)?;
1641 let freshness = freshness_summary(&store.load_current_freshness(workspace)?);
1642 let graph = FederatedGraph::new(nodes, edges)?;
1643 let report = graph.trace(
1644 &NodeId::new(&input.from),
1645 &NodeId::new(&input.to),
1646 input.max_depth,
1647 )?;
1648 let status = if report.coverage_gaps.is_empty() && freshness.overall == OverallFreshness::Fresh
1649 {
1650 ToolStatus::Ok
1651 } else {
1652 ToolStatus::Degraded
1653 };
1654 Ok(ToolEnvelope {
1655 schema_version: 1,
1656 status,
1657 data: Some(report),
1658 freshness,
1659 warnings: Vec::new(),
1660 })
1661}
1662
1663pub fn search_workspace(
1670 database_path: &Path,
1671 workspace: &str,
1672 input: &SearchInput,
1673) -> Result<ToolEnvelope<SearchReport>, ApplicationError> {
1674 let store = SqliteStore::open_read_only(database_path)?;
1675 let snapshot = store.current_snapshot_summary(workspace)?;
1676 let input_json = serde_json::to_vec(input)
1677 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
1678 let input_fingerprint = stable_id_bytes("query-cache", &input_json);
1679 let now_unix_ms = current_unix_millis();
1680 if let Some(cached) = store.load_query_cache(
1681 workspace,
1682 &snapshot.snapshot_id,
1683 &input_fingerprint,
1684 now_unix_ms,
1685 )? && let Ok(envelope) =
1686 serde_json::from_slice::<ToolEnvelope<SearchReport>>(&cached.result_summary_json)
1687 {
1688 return Ok(envelope);
1689 }
1690 let (nodes, edges) = store.load_current_graph(workspace)?;
1691 let repository_freshness = store.load_current_freshness(workspace)?;
1692 let freshness = freshness_summary(&repository_freshness);
1693 let fts_hits = if input.query.trim().is_empty() {
1694 Vec::new()
1695 } else {
1696 store.search_current_nodes_ranked(workspace, &input.query, 500)?
1697 };
1698 let community_snapshot = store.load_current_community_snapshot(workspace)?;
1699 let evidence = store.load_current_evidence(workspace)?;
1700 let request = SearchRequest {
1701 query: input.query.clone(),
1702 filters: SearchFilters {
1703 node_kinds: input.node_kinds.clone(),
1704 repo_ids: input.repo_ids.clone(),
1705 workspace_nodes: Vec::new(),
1706 service_ids: input.service_ids.clone(),
1707 community_ids: input.community_ids.clone(),
1708 },
1709 fts_scores: fts_search_scores(&fts_hits),
1710 centrality_scores: community_centrality_scores(&community_snapshot.communities),
1711 service_memberships: service_memberships(&nodes, &edges),
1712 community_memberships: community_memberships(&community_snapshot.communities),
1713 evidence: node_evidence(&edges, &evidence),
1714 freshness: repository_freshness
1715 .iter()
1716 .map(|item| (item.repo_id.clone(), item.state))
1717 .collect(),
1718 offset: input.offset,
1719 limit: input.limit,
1720 };
1721 let report = search(&nodes, &request)?;
1722 let status = if report.coverage.gaps.is_empty() && freshness.overall == OverallFreshness::Fresh
1723 {
1724 ToolStatus::Ok
1725 } else {
1726 ToolStatus::Degraded
1727 };
1728 let envelope = ToolEnvelope {
1729 schema_version: 1,
1730 status,
1731 data: Some(report),
1732 freshness,
1733 warnings: Vec::new(),
1734 };
1735 let encoded = serde_json::to_vec(&envelope)
1736 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
1737 let envelope = serde_json::from_slice(&encoded)
1738 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
1739 drop(store);
1740 if let Ok(_lock) = StoreLock::acquire(database_path, Duration::from_secs(5))
1741 && let Ok(mut writable) = SqliteStore::open(database_path)
1742 {
1743 let _ = writable.put_query_cache(&QueryCacheRecord {
1744 workspace_name: workspace.to_owned(),
1745 snapshot_id: snapshot.snapshot_id,
1746 input_fingerprint,
1747 result_summary_json: encoded,
1748 stored_at_unix_ms: now_unix_ms,
1749 expires_at_unix_ms: None,
1750 });
1751 }
1752 Ok(envelope)
1753}
1754
1755pub fn traverse_workspace(
1762 database_path: &Path,
1763 workspace: &str,
1764 input: &TraversalRequest,
1765) -> Result<ToolEnvelope<TraversalReport>, ApplicationError> {
1766 if input.options.max_depth > 32
1767 || input.options.max_cross_repo_hops > 16
1768 || input.options.node_limit > 50_000
1769 || input.options.edge_limit > 250_000
1770 || input.options.timeout_ms > 5_000
1771 || input.options.k > 8
1772 {
1773 return Err(QueryError::InvalidTraversalLimits.into());
1774 }
1775 let store = SqliteStore::open_read_only(database_path)?;
1776 let (nodes, edges) = store.load_current_graph(workspace)?;
1777 let freshness = freshness_summary(&store.load_current_freshness(workspace)?);
1778 let report = traverse(&nodes, &edges, input)?;
1779 let status = if report.coverage_gaps.is_empty()
1780 && !report.truncated
1781 && freshness.overall == OverallFreshness::Fresh
1782 {
1783 ToolStatus::Ok
1784 } else {
1785 ToolStatus::Degraded
1786 };
1787 Ok(ToolEnvelope {
1788 schema_version: 1,
1789 status,
1790 data: Some(report),
1791 freshness,
1792 warnings: Vec::new(),
1793 })
1794}
1795
1796pub fn communities_workspace(
1802 database_path: &Path,
1803 workspace: &str,
1804 input: &CommunityInput,
1805) -> Result<ToolEnvelope<CommunityReport>, ApplicationError> {
1806 if !(1..=100).contains(&input.limit) || input.offset > 1_000_000 {
1807 return Err(QueryError::InvalidPagination.into());
1808 }
1809 let store = SqliteStore::open_read_only(database_path)?;
1810 let current = store.load_current_community_snapshot(workspace)?;
1811 let freshness = freshness_summary(&store.load_current_freshness(workspace)?);
1812 let mut selected = current
1813 .communities
1814 .iter()
1815 .filter(|community| {
1816 input
1817 .community_id
1818 .as_ref()
1819 .is_none_or(|id| community.id == *id)
1820 })
1821 .cloned()
1822 .collect::<Vec<_>>();
1823 selected.sort_by(|left, right| {
1824 right
1825 .metrics
1826 .size
1827 .cmp(&left.metrics.size)
1828 .then_with(|| right.metrics.cohesion.total_cmp(&left.metrics.cohesion))
1829 .then_with(|| left.id.cmp(&right.id))
1830 });
1831 let total_communities = selected.len();
1832 let communities = selected
1833 .into_iter()
1834 .skip(input.offset)
1835 .take(input.limit)
1836 .collect();
1837 let delta = input
1838 .compare_snapshot_id
1839 .as_ref()
1840 .map(|snapshot_id| store.load_workspace_community_snapshot(workspace, snapshot_id))
1841 .transpose()?
1842 .map(|before| compare_community_snapshots(&before, ¤t));
1843 let exact_missing = input.community_id.is_some() && total_communities == 0;
1844 let truncated = input.offset.saturating_add(input.limit) < total_communities;
1845 let report = CommunityReport {
1846 snapshot_id: current.snapshot_id,
1847 engine_version: current.engine_version,
1848 config: current.config,
1849 communities,
1850 total_communities,
1851 offset: input.offset,
1852 limit: input.limit,
1853 truncated,
1854 delta,
1855 };
1856 let status = if !exact_missing && freshness.overall == OverallFreshness::Fresh {
1857 ToolStatus::Ok
1858 } else {
1859 ToolStatus::Degraded
1860 };
1861 Ok(ToolEnvelope {
1862 schema_version: 1,
1863 status,
1864 data: Some(report),
1865 freshness,
1866 warnings: exact_missing
1867 .then(|| "Requested community was not observed in the selected snapshot.".to_owned())
1868 .into_iter()
1869 .collect(),
1870 })
1871}
1872
1873pub async fn explore_repository(
1878 database_path: &Path,
1879 workspace: &str,
1880 input: &ExploreInput,
1881 binary: Option<std::ffi::OsString>,
1882) -> ToolEnvelope<LocalContextResult> {
1883 if input.workspace != workspace {
1884 return explore_error_envelope(format!(
1885 "workspace `{}` is outside this server's configured workspace `{workspace}`",
1886 input.workspace
1887 ));
1888 }
1889 if !(1..=25).contains(&input.max_files) {
1890 return explore_error_envelope("max_files must be between 1 and 25".to_owned());
1891 }
1892
1893 let store = match SqliteStore::open_read_only(database_path) {
1894 Ok(store) => store,
1895 Err(error) => return explore_error_envelope(error.to_string()),
1896 };
1897 let registry = match store.load_workspace_registry(workspace) {
1898 Ok(registry) => registry,
1899 Err(error) => return explore_error_envelope(error.to_string()),
1900 };
1901 let persisted_freshness = match store.load_current_freshness(workspace) {
1902 Ok(freshness) => freshness,
1903 Err(error) => return explore_error_envelope(error.to_string()),
1904 };
1905 let freshness = freshness_summary(&persisted_freshness);
1906 let requested_alias = input
1907 .repository
1908 .as_deref()
1909 .map(str::trim)
1910 .filter(|alias| !alias.is_empty());
1911 let repository = match select_explore_repository(®istry, requested_alias, workspace) {
1912 Ok(repository) => repository,
1913 Err(message) => {
1914 return explore_scoped_error_envelope(freshness, ToolStatus::Error, message);
1915 }
1916 };
1917
1918 let mut config = CodeGraphConfig::default();
1919 if let Some(binary) = binary {
1920 config.binary = binary;
1921 }
1922 let provider = match CodeGraphProvider::new(config) {
1923 Ok(provider) => provider,
1924 Err(error) => {
1925 return explore_scoped_error_envelope(
1926 freshness,
1927 ToolStatus::Degraded,
1928 error.to_string(),
1929 );
1930 }
1931 };
1932 let result = provider
1933 .build_local_context(LocalContextRequest {
1934 request: ProviderRequest {
1935 repo_id: repository.id.clone(),
1936 project_path: native_relative_path(&repository.canonical_path),
1937 budget: ProviderBudget {
1938 timeout: Duration::from_secs(5),
1939 max_output_bytes: 256 * 1024,
1940 max_items: 25,
1941 },
1942 cancellation: CancellationToken::new(),
1943 },
1944 query: input.query.clone(),
1945 max_files: input.max_files,
1946 })
1947 .await;
1948 let shutdown = provider.shutdown().await;
1949 explore_result_envelope(result, shutdown, freshness)
1950}
1951
1952fn select_explore_repository<'a>(
1953 registry: &'a WorkspaceRecord,
1954 requested_alias: Option<&str>,
1955 workspace: &str,
1956) -> Result<&'a RepositoryRecord, String> {
1957 if let Some(alias) = requested_alias {
1958 return registry
1959 .repositories
1960 .iter()
1961 .find(|repository| repository.alias == alias)
1962 .ok_or_else(|| {
1963 format!("repository alias `{alias}` is not registered in workspace `{workspace}`")
1964 });
1965 }
1966 if registry.repositories.len() == 1 {
1967 return registry.repositories.first().ok_or_else(|| {
1968 format!("workspace `{workspace}` unexpectedly has no registered repository")
1969 });
1970 }
1971 Err(format!(
1972 "repository is required because workspace `{workspace}` contains {} repositories",
1973 registry.repositories.len()
1974 ))
1975}
1976
1977fn explore_result_envelope(
1978 result: Result<LocalContextResult, ProviderError>,
1979 shutdown: Result<(), ProviderError>,
1980 freshness: FreshnessSummary,
1981) -> ToolEnvelope<LocalContextResult> {
1982 match result {
1983 Ok(context) => {
1984 let mut warnings = context
1985 .execution
1986 .degradations
1987 .iter()
1988 .map(|degradation| degradation.message.clone())
1989 .collect::<Vec<_>>();
1990 if context.execution.truncated {
1991 warnings.push("Local context was truncated by the configured budget.".to_owned());
1992 }
1993 if let Err(error) = shutdown {
1994 warnings.push(format!("CodeGraph shutdown degraded: {error}"));
1995 }
1996 let status = if warnings.is_empty() && freshness.overall == OverallFreshness::Fresh {
1997 ToolStatus::Ok
1998 } else {
1999 ToolStatus::Degraded
2000 };
2001 ToolEnvelope {
2002 schema_version: 1,
2003 status,
2004 data: Some(context),
2005 freshness,
2006 warnings,
2007 }
2008 }
2009 Err(error) => ToolEnvelope {
2010 schema_version: 1,
2011 status: if matches!(&error, ProviderError::InvalidRequest(_)) {
2012 ToolStatus::Error
2013 } else {
2014 ToolStatus::Degraded
2015 },
2016 data: None,
2017 freshness,
2018 warnings: vec![error.to_string()],
2019 },
2020 }
2021}
2022
2023fn explore_scoped_error_envelope(
2024 freshness: FreshnessSummary,
2025 status: ToolStatus,
2026 message: String,
2027) -> ToolEnvelope<LocalContextResult> {
2028 ToolEnvelope {
2029 schema_version: 1,
2030 status,
2031 data: None,
2032 freshness,
2033 warnings: vec![message],
2034 }
2035}
2036
2037fn explore_error_envelope(message: String) -> ToolEnvelope<LocalContextResult> {
2038 ToolEnvelope {
2039 schema_version: 1,
2040 status: ToolStatus::Error,
2041 data: None,
2042 freshness: FreshnessSummary {
2043 overall: OverallFreshness::Unknown,
2044 stale_repositories: Vec::new(),
2045 reasons: vec!["Local repository exploration could not start.".to_owned()],
2046 },
2047 warnings: vec![message],
2048 }
2049}
2050
2051pub fn impact_workspace(
2062 database_path: &Path,
2063 workspace: &str,
2064 request: &ImpactRequest,
2065) -> Result<ToolEnvelope<ImpactReport>, ApplicationError> {
2066 let loaded = load_impact_context(database_path, workspace)?;
2067 impact_envelope(request, &loaded.context, loaded.freshness)
2068}
2069
2070pub async fn impact_workspace_with_codegraph(
2080 database_path: &Path,
2081 workspace: &str,
2082 request: &ImpactRequest,
2083 binary: Option<std::ffi::OsString>,
2084) -> Result<ToolEnvelope<ImpactReport>, ApplicationError> {
2085 let mut loaded = load_impact_context(database_path, workspace)?;
2086 let mut config = CodeGraphConfig::default();
2087 if let Some(binary) = binary {
2088 config.binary = binary;
2089 }
2090 match CodeGraphProvider::new(config) {
2091 Ok(provider) => {
2092 enrich_impact_context(
2093 &mut loaded.context,
2094 request,
2095 &loaded.registry,
2096 &loaded.node_evidence,
2097 &provider,
2098 )
2099 .await;
2100 let _ = provider.shutdown().await;
2101 }
2102 Err(error) => loaded.context.local_enrichment.push(LocalEnrichmentInput {
2103 repo_id: impact_target_repo(&loaded.context, request)
2104 .unwrap_or_else(|| RepoId::new("repo:unknown")),
2105 anchor: impact_target_label(&loaded.context, request)
2106 .unwrap_or_else(|| "unknown impact target".to_owned()),
2107 status: LocalEnrichmentStatus::Unavailable,
2108 affected: Vec::new(),
2109 affected_tests: Vec::new(),
2110 truncated: false,
2111 degradations: vec![error.to_string()],
2112 }),
2113 }
2114 impact_envelope(request, &loaded.context, loaded.freshness)
2115}
2116
2117pub async fn collect_workspace_changes(
2126 database_path: &Path,
2127 workspace: &str,
2128 input: &ChangesInput,
2129 git_binary: Option<std::ffi::OsString>,
2130) -> Result<ToolEnvelope<ChangeSet>, ApplicationError> {
2131 collect_workspace_changes_with_cancellation(
2132 database_path,
2133 workspace,
2134 input,
2135 git_binary,
2136 &CancellationToken::new(),
2137 )
2138 .await
2139}
2140
2141pub async fn collect_workspace_changes_with_cancellation(
2147 database_path: &Path,
2148 workspace: &str,
2149 input: &ChangesInput,
2150 git_binary: Option<std::ffi::OsString>,
2151 cancellation: &CancellationToken,
2152) -> Result<ToolEnvelope<ChangeSet>, ApplicationError> {
2153 let store = SqliteStore::open_read_only(database_path)?;
2154 let registry = store.load_workspace_registry(workspace)?;
2155 let snapshot = store.current_snapshot_summary(workspace)?;
2156 let repository = registry
2157 .repositories
2158 .iter()
2159 .find(|repository| repository.alias == input.repository)
2160 .ok_or_else(|| ApplicationError::RegistryAliasMissing(input.repository.clone()))?;
2161 let persisted_freshness = store.load_current_freshness(workspace)?;
2162 let freshness = freshness_summary(&persisted_freshness);
2163 let provider = git_binary.map_or_else(GitCliChangeProvider::new, |binary| {
2164 GitCliChangeProvider::with_limits(
2165 binary,
2166 Duration::from_secs(30),
2167 8 * 1024 * 1024,
2168 64 * 1024,
2169 )
2170 });
2171 let mut analyzer_versions = AnalyzerVersions::new();
2172 analyzer_versions.insert(
2173 "code-system-graph.changes".to_owned(),
2174 env!("CARGO_PKG_VERSION").to_owned(),
2175 );
2176 let change_set = provider
2177 .changes(
2178 &ChangeRequest {
2179 repo_id: repository.id.clone(),
2180 checkout_id: repository.checkout_id.clone(),
2181 worktree: native_relative_path(&repository.canonical_path),
2182 scope: input.scope.clone(),
2183 workspace_manifest_hash: registry.manifest_hash.clone(),
2184 contract_registry_hash: snapshot.snapshot_id,
2185 analyzer_versions,
2186 },
2187 cancellation,
2188 )
2189 .await?;
2190 let repository_is_fresh = persisted_freshness.iter().any(|item| {
2191 item.checkout_id == repository.checkout_id && item.state == RepoFreshnessState::Fresh
2192 });
2193 Ok(ToolEnvelope {
2194 schema_version: 1,
2195 status: if repository_is_fresh {
2196 ToolStatus::Ok
2197 } else {
2198 ToolStatus::Degraded
2199 },
2200 data: Some(change_set),
2201 freshness,
2202 warnings: Vec::new(),
2203 })
2204}
2205
2206pub async fn analyze_workspace_changes(
2213 database_path: &Path,
2214 workspace: &str,
2215 input: &ChangesInput,
2216 options: &ChangeAnalysisOptions,
2217 git_binary: Option<std::ffi::OsString>,
2218) -> Result<ToolEnvelope<ChangeImpactReport>, ApplicationError> {
2219 analyze_workspace_changes_with_cancellation(
2220 database_path,
2221 workspace,
2222 input,
2223 options,
2224 git_binary,
2225 &CancellationToken::new(),
2226 )
2227 .await
2228}
2229
2230pub async fn analyze_workspace_changes_with_cancellation(
2236 database_path: &Path,
2237 workspace: &str,
2238 input: &ChangesInput,
2239 options: &ChangeAnalysisOptions,
2240 git_binary: Option<std::ffi::OsString>,
2241 cancellation: &CancellationToken,
2242) -> Result<ToolEnvelope<ChangeImpactReport>, ApplicationError> {
2243 let collected = collect_workspace_changes_with_cancellation(
2244 database_path,
2245 workspace,
2246 input,
2247 git_binary,
2248 cancellation,
2249 )
2250 .await?;
2251 let change_set = collected.data.ok_or_else(|| {
2252 ApplicationError::InvalidSourceObservation(
2253 "local change collection returned no change set".to_owned(),
2254 )
2255 })?;
2256 let store = SqliteStore::open_read_only(database_path)?;
2257 let (nodes, edges) = store.load_current_graph(workspace)?;
2258 let evidence = store.load_current_evidence(workspace)?;
2259 let community = store.load_current_community_snapshot(workspace)?;
2260 let freshness = store.load_current_freshness(workspace)?;
2261 let report = analyze_changes(
2262 &change_set,
2263 &nodes,
2264 &edges,
2265 &evidence,
2266 Some(&community),
2267 &freshness,
2268 &[],
2269 options,
2270 )?;
2271 let status = if collected.status == ToolStatus::Ok && report.coverage.complete {
2272 ToolStatus::Ok
2273 } else {
2274 ToolStatus::Degraded
2275 };
2276 Ok(ToolEnvelope {
2277 schema_version: 1,
2278 status,
2279 data: Some(report),
2280 freshness: collected.freshness,
2281 warnings: collected.warnings,
2282 })
2283}
2284
2285fn pull_request_provider(
2286 kind: PullRequestProviderKind,
2287 enabled: bool,
2288 token: Option<String>,
2289 basic_auth_username: Option<String>,
2290) -> Result<Box<dyn PullRequestProvider>, ApplicationError> {
2291 if !enabled {
2292 return Err(PullRequestError::Disabled.into());
2293 }
2294 let mut config = match kind {
2295 PullRequestProviderKind::GitHub => PullRequestProviderConfig::default(),
2296 PullRequestProviderKind::BitbucketCloud => PullRequestProviderConfig::bitbucket_cloud(),
2297 PullRequestProviderKind::BitbucketDataCenter => {
2298 return Err(PullRequestError::InvalidConfiguration(
2299 "Bitbucket Data Center is not implemented in Code System Graph 1.0.0".to_owned(),
2300 )
2301 .into());
2302 }
2303 };
2304 config.enabled = true;
2305 config.auth_token = PrAuthToken::new(token.unwrap_or_default());
2306 if kind == PullRequestProviderKind::BitbucketCloud {
2307 config.basic_auth_username = basic_auth_username;
2308 }
2309 let transport = Arc::new(
2310 ReqwestPrHttpTransport::new()
2311 .map_err(|error| PullRequestError::Transport(error.to_string()))?,
2312 );
2313 match kind {
2314 PullRequestProviderKind::GitHub => Ok(Box::new(GitHubProvider::new(config, transport)?)),
2315 PullRequestProviderKind::BitbucketCloud => {
2316 Ok(Box::new(BitbucketProvider::new(config, transport)?))
2317 }
2318 PullRequestProviderKind::BitbucketDataCenter => unreachable!("rejected above"),
2319 }
2320}
2321
2322pub async fn inspect_pull_request(
2333 input: &PullRequestInput,
2334 enabled: bool,
2335 token: Option<String>,
2336 basic_auth_username: Option<String>,
2337) -> Result<ToolEnvelope<PullRequestInspection>, ApplicationError> {
2338 inspect_pull_request_with_cancellation(
2339 input,
2340 enabled,
2341 token,
2342 basic_auth_username,
2343 CancellationToken::new(),
2344 )
2345 .await
2346}
2347
2348pub async fn inspect_pull_request_with_cancellation(
2354 input: &PullRequestInput,
2355 enabled: bool,
2356 token: Option<String>,
2357 basic_auth_username: Option<String>,
2358 cancellation: CancellationToken,
2359) -> Result<ToolEnvelope<PullRequestInspection>, ApplicationError> {
2360 if !input.consent_to_remote_access {
2361 return Err(PullRequestError::ConsentRequired.into());
2362 }
2363 let provider = pull_request_provider(input.provider, enabled, token, basic_auth_username)?;
2364 let inspection = provider
2365 .inspect(PullRequestInspectRequest {
2366 coordinates: PullRequestCoordinates {
2367 provider: input.provider,
2368 owner: input.owner.clone(),
2369 repository: input.repository.clone(),
2370 number: input.number,
2371 },
2372 consent_to_remote_access: input.consent_to_remote_access,
2373 cancellation,
2374 })
2375 .await?;
2376 let status = if inspection.warnings.is_empty() {
2377 ToolStatus::Ok
2378 } else {
2379 ToolStatus::Degraded
2380 };
2381 Ok(ToolEnvelope {
2382 schema_version: 1,
2383 status,
2384 data: Some(inspection),
2385 freshness: FreshnessSummary {
2386 overall: OverallFreshness::Unknown,
2387 stale_repositories: Vec::new(),
2388 reasons: vec![
2389 "Remote pull-request inspection does not establish local graph freshness."
2390 .to_owned(),
2391 ],
2392 },
2393 warnings: Vec::new(),
2394 })
2395}
2396
2397pub async fn list_pull_requests(
2404 input: &PullRequestListInput,
2405 enabled: bool,
2406 token: Option<String>,
2407 basic_auth_username: Option<String>,
2408 cancellation: CancellationToken,
2409) -> Result<ToolEnvelope<PullRequestListPage>, ApplicationError> {
2410 if !input.consent_to_remote_access {
2411 return Err(PullRequestError::ConsentRequired.into());
2412 }
2413 let provider = pull_request_provider(input.provider, enabled, token, basic_auth_username)?;
2414 let page = provider
2415 .list(PullRequestListRequest {
2416 provider: input.provider,
2417 owner: input.owner.clone(),
2418 repository: input.repository.clone(),
2419 state: input.state,
2420 cursor: input.cursor.clone(),
2421 limit: input.limit,
2422 consent_to_remote_access: input.consent_to_remote_access,
2423 cancellation,
2424 })
2425 .await?;
2426 let status = if page.warnings.is_empty() && !page.truncated {
2427 ToolStatus::Ok
2428 } else {
2429 ToolStatus::Degraded
2430 };
2431 Ok(ToolEnvelope {
2432 schema_version: 1,
2433 status,
2434 data: Some(page),
2435 freshness: FreshnessSummary {
2436 overall: OverallFreshness::Unknown,
2437 stale_repositories: Vec::new(),
2438 reasons: vec![
2439 "Remote pull-request listing does not establish local graph freshness.".to_owned(),
2440 ],
2441 },
2442 warnings: Vec::new(),
2443 })
2444}
2445
2446struct LoadedImpactContext {
2447 context: ImpactContext,
2448 freshness: FreshnessSummary,
2449 registry: WorkspaceRecord,
2450 node_evidence: BTreeMap<NodeId, Vec<Evidence>>,
2451}
2452
2453fn load_impact_context(
2454 database_path: &Path,
2455 workspace: &str,
2456) -> Result<LoadedImpactContext, ApplicationError> {
2457 let store = SqliteStore::open_read_only(database_path)?;
2458 let (nodes, edges) = store.load_current_graph(workspace)?;
2459 let persisted_freshness = store.load_current_freshness(workspace)?;
2460 let freshness = freshness_summary(&persisted_freshness);
2461 let communities = store.load_current_community_snapshot(workspace)?;
2462 let evidence = store.load_current_evidence(workspace)?;
2463 let registry = store.load_workspace_registry(workspace)?;
2464 let node_evidence = node_evidence(&edges, &evidence);
2465 let context = ImpactContext {
2466 centrality: impact_centrality_scores(&communities.communities),
2467 service_memberships: service_memberships(&nodes, &edges),
2468 recommended_commands: recommended_test_commands(&nodes, &node_evidence),
2469 nodes,
2470 edges,
2471 communities: Some(communities),
2472 freshness: conservative_repo_freshness(&persisted_freshness),
2473 compatibility: Vec::new(),
2474 local_enrichment: Vec::new(),
2475 public_contracts: Vec::new(),
2476 criticality: Vec::new(),
2477 environments: Vec::new(),
2478 graph_complete: true,
2479 coverage_gaps: Vec::new(),
2480 };
2481 Ok(LoadedImpactContext {
2482 context,
2483 freshness,
2484 registry,
2485 node_evidence,
2486 })
2487}
2488
2489fn impact_envelope(
2490 request: &ImpactRequest,
2491 context: &ImpactContext,
2492 freshness: FreshnessSummary,
2493) -> Result<ToolEnvelope<ImpactReport>, ApplicationError> {
2494 let report = analyze_impact(request, context)?;
2495 let status = if report.risk == code_system_graph_core::RiskLevel::Unknown
2496 || freshness.overall != OverallFreshness::Fresh
2497 {
2498 ToolStatus::Degraded
2499 } else {
2500 ToolStatus::Ok
2501 };
2502 Ok(ToolEnvelope {
2503 schema_version: 1,
2504 status,
2505 data: Some(report),
2506 freshness,
2507 warnings: Vec::new(),
2508 })
2509}
2510
2511#[expect(
2512 clippy::too_many_lines,
2513 reason = "Provider probing, impact, affected tests, and degradation form one bounded lifecycle"
2514)]
2515async fn enrich_impact_context(
2516 context: &mut ImpactContext,
2517 request: &ImpactRequest,
2518 registry: &WorkspaceRecord,
2519 evidence: &BTreeMap<NodeId, Vec<Evidence>>,
2520 provider: &impl LocalCodeIntelligenceProvider,
2521) {
2522 let Some(target) = impact_target_node(context, request).cloned() else {
2523 return;
2524 };
2525 let by_id = context
2526 .nodes
2527 .iter()
2528 .map(|node| (node.id.clone(), node))
2529 .collect::<BTreeMap<_, _>>();
2530 let mut anchors = context
2531 .edges
2532 .iter()
2533 .filter_map(|edge| {
2534 let adjacent = if edge.source == target.id {
2535 by_id.get(&edge.target)
2536 } else if edge.target == target.id {
2537 by_id.get(&edge.source)
2538 } else {
2539 None
2540 };
2541 adjacent
2542 .copied()
2543 .filter(|node| node.kind == NodeKind::SymbolRef)
2544 .cloned()
2545 })
2546 .chain((target.kind == NodeKind::SymbolRef).then_some(target.clone()))
2547 .collect::<Vec<_>>();
2548 anchors.sort_by(|left, right| left.id.cmp(&right.id));
2549 anchors.dedup_by(|left, right| left.id == right.id);
2550 anchors.truncate(3);
2551
2552 for anchor in anchors {
2553 let Some(repo_id) = anchor.repo_id.clone() else {
2554 continue;
2555 };
2556 let Some(repository) = registry
2557 .repositories
2558 .iter()
2559 .find(|repository| repository.id == repo_id)
2560 else {
2561 context.local_enrichment.push(unavailable_local_enrichment(
2562 repo_id,
2563 &anchor.label,
2564 "Registered checkout metadata was unavailable for local enrichment.",
2565 ));
2566 continue;
2567 };
2568 let project_path = native_relative_path(&repository.canonical_path);
2569 let provider_request = || ProviderRequest {
2570 repo_id: repo_id.clone(),
2571 project_path: project_path.clone(),
2572 budget: ProviderBudget {
2573 timeout: Duration::from_secs(2),
2574 max_output_bytes: 256 * 1024,
2575 max_items: 25,
2576 },
2577 cancellation: CancellationToken::new(),
2578 };
2579 let capability = match provider.probe(provider_request()).await {
2580 Ok(capability) => capability,
2581 Err(error) => {
2582 context.local_enrichment.push(unavailable_local_enrichment(
2583 repo_id,
2584 &anchor.label,
2585 &error.to_string(),
2586 ));
2587 continue;
2588 }
2589 };
2590 if capability.status != ProviderStatus::Available {
2591 let status = if capability.status == ProviderStatus::Stale {
2592 LocalEnrichmentStatus::Stale
2593 } else {
2594 LocalEnrichmentStatus::Unavailable
2595 };
2596 let mut degradations = capability
2597 .degradations
2598 .into_iter()
2599 .map(|degradation| degradation.message)
2600 .collect::<Vec<_>>();
2601 degradations.extend(capability.remediation);
2602 context.local_enrichment.push(LocalEnrichmentInput {
2603 repo_id,
2604 anchor: anchor.label.clone(),
2605 status,
2606 affected: Vec::new(),
2607 affected_tests: Vec::new(),
2608 truncated: false,
2609 degradations,
2610 });
2611 continue;
2612 }
2613 let result = match provider
2614 .get_local_impact(LocalImpactRequest {
2615 request: provider_request(),
2616 symbol: anchor.label.clone(),
2617 max_depth: request.options.max_depth.min(8),
2618 })
2619 .await
2620 {
2621 Ok(result) => result,
2622 Err(error) => {
2623 context.local_enrichment.push(unavailable_local_enrichment(
2624 repo_id,
2625 &anchor.label,
2626 &error.to_string(),
2627 ));
2628 continue;
2629 }
2630 };
2631 let changed_files = evidence
2632 .get(&anchor.id)
2633 .into_iter()
2634 .flatten()
2635 .filter_map(|item| item.file_path.clone())
2636 .collect::<BTreeSet<_>>()
2637 .into_iter()
2638 .collect::<Vec<_>>();
2639 let affected_tests = if changed_files.is_empty() {
2640 None
2641 } else {
2642 provider
2643 .get_affected_tests(AffectedTestsRequest {
2644 request: provider_request(),
2645 changed_files,
2646 max_depth: request.options.max_depth.min(8),
2647 })
2648 .await
2649 .ok()
2650 .flatten()
2651 };
2652 let mut degradations = result
2653 .execution
2654 .degradations
2655 .iter()
2656 .map(|degradation| degradation.message.clone())
2657 .collect::<Vec<_>>();
2658 if affected_tests.is_none() {
2659 degradations.push(
2660 "CodeGraph affected-test capability was unavailable or no changed file was known."
2661 .to_owned(),
2662 );
2663 }
2664 let tests_truncated = affected_tests
2665 .as_ref()
2666 .is_some_and(|tests| tests.execution.truncated);
2667 let status = if result.execution.truncated || tests_truncated || !degradations.is_empty() {
2668 LocalEnrichmentStatus::Partial
2669 } else {
2670 LocalEnrichmentStatus::Available
2671 };
2672 context.local_enrichment.push(LocalEnrichmentInput {
2673 repo_id,
2674 anchor: result.symbol,
2675 status,
2676 affected: result
2677 .affected
2678 .into_iter()
2679 .map(|item| LocalImpactItem {
2680 symbol: item.name,
2681 file_path: item.file_path,
2682 start_line: Some(item.start_line),
2683 depth: result.depth,
2684 })
2685 .collect(),
2686 affected_tests: affected_tests
2687 .map(|tests| tests.affected_tests)
2688 .unwrap_or_default(),
2689 truncated: result.execution.truncated || tests_truncated,
2690 degradations,
2691 });
2692 }
2693}
2694
2695fn unavailable_local_enrichment(
2696 repo_id: RepoId,
2697 anchor: &str,
2698 message: &str,
2699) -> LocalEnrichmentInput {
2700 LocalEnrichmentInput {
2701 repo_id,
2702 anchor: anchor.to_owned(),
2703 status: LocalEnrichmentStatus::Unavailable,
2704 affected: Vec::new(),
2705 affected_tests: Vec::new(),
2706 truncated: false,
2707 degradations: vec![message.to_owned()],
2708 }
2709}
2710
2711fn impact_target_node<'a>(context: &'a ImpactContext, request: &ImpactRequest) -> Option<&'a Node> {
2712 context.nodes.iter().find(|node| match &request.target {
2713 ImpactTarget::NodeId(node_id) => node.id == *node_id,
2714 ImpactTarget::StableKey(stable_key) => node.stable_key == *stable_key,
2715 })
2716}
2717
2718fn impact_target_repo(context: &ImpactContext, request: &ImpactRequest) -> Option<RepoId> {
2719 impact_target_node(context, request).and_then(|node| node.repo_id.clone())
2720}
2721
2722fn impact_target_label(context: &ImpactContext, request: &ImpactRequest) -> Option<String> {
2723 impact_target_node(context, request).map(|node| node.label.clone())
2724}
2725
2726fn fts_search_scores(
2727 hits: &[code_system_graph_store_sqlite::StoredNodeSearchHit],
2728) -> BTreeMap<NodeId, f64> {
2729 hits.iter()
2730 .enumerate()
2731 .map(|(index, hit)| {
2732 (
2733 hit.node.id.clone(),
2734 1.0 / f64::from(u32::try_from(index + 1).unwrap_or(u32::MAX)),
2735 )
2736 })
2737 .collect()
2738}
2739
2740fn impact_centrality_scores(communities: &[Community]) -> BTreeMap<NodeId, f32> {
2741 let mut scores = BTreeMap::<NodeId, f32>::new();
2742 for community in communities {
2743 for (index, node_id) in community.central_nodes.iter().enumerate() {
2744 let denominator = u16::try_from(index + 1).unwrap_or(u16::MAX);
2745 let score = 1.0 / f32::from(denominator);
2746 scores
2747 .entry(node_id.clone())
2748 .and_modify(|current| *current = current.max(score))
2749 .or_insert(score);
2750 }
2751 }
2752 scores
2753}
2754
2755fn conservative_repo_freshness(items: &[RepoFreshness]) -> Vec<RepoFreshness> {
2756 let mut by_repository = BTreeMap::<RepoId, RepoFreshness>::new();
2757 for item in items {
2758 by_repository
2759 .entry(item.repo_id.clone())
2760 .and_modify(|current| {
2761 if freshness_severity(item.state) > freshness_severity(current.state) {
2762 *current = item.clone();
2763 }
2764 })
2765 .or_insert_with(|| item.clone());
2766 }
2767 by_repository.into_values().collect()
2768}
2769
2770fn freshness_severity(state: RepoFreshnessState) -> u8 {
2771 match state {
2772 RepoFreshnessState::Fresh => 0,
2773 RepoFreshnessState::WorkingTreeChanged
2774 | RepoFreshnessState::CommitsBehind
2775 | RepoFreshnessState::ConfigChanged
2776 | RepoFreshnessState::ExtractorChanged
2777 | RepoFreshnessState::CodegraphPending => 1,
2778 RepoFreshnessState::Partial | RepoFreshnessState::Unknown => 2,
2779 RepoFreshnessState::Unavailable => 3,
2780 RepoFreshnessState::Corrupt => 4,
2781 }
2782}
2783
2784fn recommended_test_commands(
2785 nodes: &[Node],
2786 evidence: &BTreeMap<NodeId, Vec<Evidence>>,
2787) -> Vec<RecommendedCommand> {
2788 let mut commands = BTreeMap::<(RepoId, String), RecommendedCommand>::new();
2789 for test in nodes.iter().filter(|node| node.kind == NodeKind::TestCase) {
2790 let Some(repo_id) = test.repo_id.clone() else {
2791 continue;
2792 };
2793 for item in evidence.get(&test.id).into_iter().flatten() {
2794 let Some(path) = item.file_path.as_deref() else {
2795 continue;
2796 };
2797 let command = match Path::new(path).extension().and_then(|value| value.to_str()) {
2798 Some(extension) if extension.eq_ignore_ascii_case("py") => "python -m pytest",
2799 Some(extension) if extension.eq_ignore_ascii_case("rs") => "cargo test",
2800 _ => continue,
2801 };
2802 commands.entry((repo_id.clone(), command.to_owned())).or_insert_with(|| RecommendedCommand {
2803 repo_id: repo_id.clone(),
2804 command: command.to_owned(),
2805 description: format!("Run the repository test suite covering `{}`; Code System Graph does not execute it.", test.label),
2806 });
2807 }
2808 }
2809 commands.into_values().collect()
2810}
2811
2812fn community_centrality_scores(communities: &[Community]) -> BTreeMap<NodeId, f64> {
2813 let mut scores = BTreeMap::<NodeId, f64>::new();
2814 for community in communities {
2815 for (index, node_id) in community.central_nodes.iter().enumerate() {
2816 let score = 1.0 / f64::from(u32::try_from(index + 1).unwrap_or(u32::MAX));
2817 scores
2818 .entry(node_id.clone())
2819 .and_modify(|current| *current = current.max(score))
2820 .or_insert(score);
2821 }
2822 }
2823 scores
2824}
2825
2826fn community_memberships(communities: &[Community]) -> BTreeMap<NodeId, Vec<CommunityId>> {
2827 let mut memberships = BTreeMap::<NodeId, Vec<CommunityId>>::new();
2828 for community in communities {
2829 for member in &community.members {
2830 memberships
2831 .entry(member.clone())
2832 .or_default()
2833 .push(community.id.clone());
2834 }
2835 }
2836 memberships
2837}
2838
2839fn service_memberships(nodes: &[Node], edges: &[Edge]) -> BTreeMap<NodeId, Vec<NodeId>> {
2840 let services = nodes
2841 .iter()
2842 .filter(|node| node.kind == NodeKind::Service)
2843 .map(|node| node.id.clone())
2844 .collect::<BTreeSet<_>>();
2845 let mut memberships = services
2846 .iter()
2847 .map(|service| (service.clone(), vec![service.clone()]))
2848 .collect::<BTreeMap<_, _>>();
2849 for edge in edges {
2850 if services.contains(&edge.source) {
2851 memberships
2852 .entry(edge.target.clone())
2853 .or_default()
2854 .push(edge.source.clone());
2855 }
2856 if services.contains(&edge.target) {
2857 memberships
2858 .entry(edge.source.clone())
2859 .or_default()
2860 .push(edge.target.clone());
2861 }
2862 }
2863 for values in memberships.values_mut() {
2864 values.sort();
2865 values.dedup();
2866 }
2867 memberships
2868}
2869
2870fn node_evidence(edges: &[Edge], evidence: &[Evidence]) -> BTreeMap<NodeId, Vec<Evidence>> {
2871 let by_id = evidence
2872 .iter()
2873 .map(|item| (item.id.clone(), item))
2874 .collect::<BTreeMap<_, _>>();
2875 let mut result = BTreeMap::<NodeId, Vec<Evidence>>::new();
2876 for edge in edges {
2877 for node_id in [&edge.source, &edge.target] {
2878 let values = result.entry(node_id.clone()).or_default();
2879 values.extend(
2880 edge.evidence
2881 .iter()
2882 .filter_map(|evidence_id| by_id.get(evidence_id))
2883 .map(|item| (*item).clone()),
2884 );
2885 }
2886 }
2887 for values in result.values_mut() {
2888 values.sort_by(|left, right| left.id.cmp(&right.id));
2889 values.dedup_by(|left, right| left.id == right.id);
2890 }
2891 result
2892}
2893
2894pub fn status_workspace(
2901 config_path: &Path,
2902 database_path: &Path,
2903) -> Result<WorkspaceStatus, ApplicationError> {
2904 let current = load_workspace_context(config_path, &ScanOverrides::default())?;
2905 let store = SqliteStore::open_read_only(database_path)?;
2906 let previous = store.load_current_freshness(¤t.manifest.name)?;
2907 let previous_by_checkout = previous
2908 .iter()
2909 .map(|freshness| (freshness.checkout_id.clone(), freshness))
2910 .collect::<BTreeMap<CheckoutId, _>>();
2911 let current_checkouts = current
2912 .registry
2913 .record
2914 .repositories
2915 .iter()
2916 .map(|repository| repository.checkout_id.clone())
2917 .collect::<std::collections::BTreeSet<_>>();
2918 let mut repositories = current
2919 .registry
2920 .record
2921 .repositories
2922 .iter()
2923 .map(|repository| {
2924 let previous = previous_by_checkout.get(&repository.checkout_id);
2925 let (state, reason) = evaluate_repository_freshness(
2926 repository,
2927 previous.copied(),
2928 ¤t.registry.record.manifest_hash,
2929 );
2930 RepoFreshness {
2931 repo_id: repository.id.clone(),
2932 checkout_id: repository.checkout_id.clone(),
2933 head_commit: repository.head_commit.clone(),
2934 manifest_hash: current.registry.record.manifest_hash.clone(),
2935 state,
2936 reason,
2937 }
2938 })
2939 .collect::<Vec<_>>();
2940 repositories.extend(
2941 previous
2942 .into_iter()
2943 .filter(|freshness| !current_checkouts.contains(&freshness.checkout_id))
2944 .map(|mut freshness| {
2945 freshness.state = RepoFreshnessState::Unavailable;
2946 freshness.reason = Some(
2947 "Checkout from the published snapshot is absent from the current manifest."
2948 .to_owned(),
2949 );
2950 freshness
2951 }),
2952 );
2953 repositories.sort_by(|left, right| {
2954 (&left.repo_id, &left.checkout_id).cmp(&(&right.repo_id, &right.checkout_id))
2955 });
2956 let watcher = watcher_status(
2957 database_path,
2958 ¤t.manifest.name,
2959 current.execution_policy.max_no_progress_time_ms,
2960 &store.database_instance_id()?,
2961 );
2962 Ok(WorkspaceStatus {
2963 workspace: current.manifest.name,
2964 schema_version: store.schema_version()?,
2965 integrity_ok: store.integrity_check()?,
2966 freshness: freshness_summary(&repositories),
2967 repositories,
2968 watcher,
2969 })
2970}
2971
2972fn watcher_status(
2973 database: &Path,
2974 workspace: &str,
2975 stale_after_ms: u64,
2976 database_instance_id: &str,
2977) -> WatcherStatus {
2978 if !work_state::work_path(database).exists() {
2979 return WatcherStatus {
2980 state: WatcherState::NeverStarted,
2981 detail: None,
2982 };
2983 }
2984 let lease = work_state::WorkState::open(database, database_instance_id)
2985 .and_then(|state| state.watcher_lease(workspace));
2986 let Ok(Some(lease)) = lease else {
2987 return WatcherStatus {
2988 state: WatcherState::NeverStarted,
2989 detail: None,
2990 };
2991 };
2992 let persisted = match lease.state.as_str() {
2993 "expired_idle" => Some(WatcherState::ExpiredIdle),
2994 "expired_session" => Some(WatcherState::ExpiredSession),
2995 "failed_limit" => Some(WatcherState::FailedLimit),
2996 "active" => None,
2997 _ => Some(WatcherState::Stale),
2998 };
2999 if let Some(state) = persisted {
3000 return WatcherStatus {
3001 state,
3002 detail: lease.detail,
3003 };
3004 }
3005 let identity_matches = lease
3006 .pid
3007 .zip(lease.process_start_identity.as_deref())
3008 .is_some_and(|(pid, expected)| worker::process_identity(pid).as_deref() == Some(expected));
3009 let heartbeat_fresh = lease.heartbeat_unix_ms.is_some_and(|heartbeat| {
3010 current_unix_millis().saturating_sub(heartbeat) <= stale_after_ms.saturating_mul(2)
3011 });
3012 if identity_matches && heartbeat_fresh {
3013 WatcherStatus {
3014 state: WatcherState::Active,
3015 detail: lease.detail,
3016 }
3017 } else {
3018 WatcherStatus {
3019 state: WatcherState::Stale,
3020 detail: Some("watcher heartbeat or process start identity is stale".to_owned()),
3021 }
3022 }
3023}
3024
3025#[doc(hidden)]
3027pub fn start_watcher_lease(
3028 config: &Path,
3029 database: &Path,
3030) -> Result<(String, String, ExecutionPolicy), ApplicationError> {
3031 let manifest_source = read_file(config)?;
3032 let manifest = parse_manifest(&manifest_source)?;
3033 validate_global_policy_source(config, &manifest)?;
3034 let execution_policy = ExecutionPolicy::resolve(manifest.execution_policy.as_ref())
3035 .map_err(ManifestError::from)?;
3036 let identity = worker::process_identity(std::process::id()).ok_or_else(|| {
3037 ApplicationError::Initialization("failed to resolve watcher process identity".to_owned())
3038 })?;
3039 let now = current_unix_millis();
3040 let owner_token = stable_id(
3041 "watcher-lease",
3042 &format!("{}:{}:{identity}:{now}", manifest.name, std::process::id()),
3043 );
3044 let database_instance_id = SqliteStore::open(database)?.database_instance_id()?;
3045 let result =
3046 work_state::WorkState::open(database, &database_instance_id).and_then(|mut state| {
3047 state.start_watcher(
3048 &manifest.name,
3049 &owner_token,
3050 std::process::id(),
3051 &identity,
3052 now,
3053 execution_policy.max_no_progress_time_ms,
3054 )
3055 });
3056 if let Err(message) = result {
3057 if message
3058 == format!(
3059 "watcher for workspace `{}` is already active",
3060 manifest.name
3061 )
3062 {
3063 return Err(ApplicationError::WatcherAlreadyActive(manifest.name));
3064 }
3065 return Err(ApplicationError::Initialization(message));
3066 }
3067 Ok((manifest.name, owner_token, execution_policy))
3068}
3069
3070#[doc(hidden)]
3072pub fn heartbeat_watcher_lease(
3073 database: &Path,
3074 workspace: &str,
3075 owner_token: &str,
3076 successful_activity: bool,
3077) -> Result<(), ApplicationError> {
3078 let database_instance_id = work_database_instance_id(database)?;
3079 work_state::WorkState::open(database, &database_instance_id)
3080 .and_then(|state| {
3081 state.heartbeat_watcher(
3082 workspace,
3083 owner_token,
3084 current_unix_millis(),
3085 successful_activity,
3086 )
3087 })
3088 .map_err(ApplicationError::Initialization)
3089}
3090
3091#[doc(hidden)]
3093pub fn finish_watcher_lease(
3094 database: &Path,
3095 workspace: &str,
3096 owner_token: &str,
3097 state: &str,
3098 detail: Option<&str>,
3099) -> Result<(), ApplicationError> {
3100 let database_instance_id = work_database_instance_id(database)?;
3101 work_state::WorkState::open(database, &database_instance_id)
3102 .and_then(|work| {
3103 work.finish_watcher(workspace, owner_token, state, detail, current_unix_millis())
3104 })
3105 .map_err(ApplicationError::Initialization)
3106}
3107
3108fn work_database_instance_id(database: &Path) -> Result<String, ApplicationError> {
3109 Ok(SqliteStore::open_read_only(database)?.database_instance_id()?)
3110}
3111
3112pub fn contracts_workspace(
3119 database_path: &Path,
3120 workspace: &str,
3121 request: &ContractRequest,
3122) -> Result<ContractReport, ApplicationError> {
3123 let store = SqliteStore::open_read_only(database_path)?;
3124 let (nodes, edges) = store.load_current_graph(workspace)?;
3125 let evidence = store.load_current_evidence(workspace)?;
3126 Ok(inspect_contracts(&nodes, &edges, &evidence, &[], request)?)
3127}
3128
3129pub fn export_workspace(
3135 database_path: &Path,
3136 workspace: &str,
3137 request: &ExportRequest,
3138) -> Result<ExportReport, ApplicationError> {
3139 let store = SqliteStore::open_read_only(database_path)?;
3140 let (nodes, edges) = store.load_current_graph(workspace)?;
3141 let evidence = store.load_current_evidence(workspace)?;
3142 Ok(export_graph(&nodes, &edges, &evidence, request)?)
3143}
3144
3145#[expect(
3151 clippy::too_many_lines,
3152 reason = "Doctor keeps every explicit safety observation visible in one conservative report"
3153)]
3154pub fn doctor_workspace(
3155 config_path: &Path,
3156 database_path: &Path,
3157) -> Result<DoctorReport, ApplicationError> {
3158 let status = status_workspace(config_path, database_path)?;
3159 let store = SqliteStore::open_read_only(database_path)?;
3160 let diagnostics = store.diagnostics()?;
3161 let context = load_workspace_context(config_path, &ScanOverrides::default())?;
3162 let expected_version = u32::try_from(latest_schema_version()).map_err(|error| {
3163 ApplicationError::Initialization(format!("unsupported schema version: {error}"))
3164 })?;
3165 let actual_version = u32::try_from(status.schema_version).ok();
3166 let freshness = status
3167 .repositories
3168 .iter()
3169 .map(|repository| FreshnessDoctorInput {
3170 repo_id: repository.repo_id.clone(),
3171 state: repository.state,
3172 detail: repository.reason.clone(),
3173 })
3174 .collect();
3175 let repository_paths_valid = context
3176 .registry
3177 .record
3178 .repositories
3179 .iter()
3180 .all(|repository| {
3181 context
3182 .registry
3183 .checkout_path(&repository.alias)
3184 .is_some_and(Path::is_dir)
3185 });
3186 let git_available = std::process::Command::new("git")
3187 .arg("--version")
3188 .stdin(std::process::Stdio::null())
3189 .stdout(std::process::Stdio::null())
3190 .stderr(std::process::Stdio::null())
3191 .status()
3192 .is_ok_and(|status| status.success());
3193 Ok(doctor(&DoctorRequest {
3194 schema: vec![SchemaDoctorInput {
3195 name: "sqlite".to_owned(),
3196 expected_version,
3197 actual_version,
3198 metadata_consistent: Some(actual_version == Some(expected_version)),
3199 }],
3200 integrity: vec![
3201 IntegrityDoctorInput {
3202 name: "binary-version".to_owned(),
3203 passed: Some(true),
3204 detail: Some(env!("CARGO_PKG_VERSION").to_owned()),
3205 },
3206 IntegrityDoctorInput {
3207 name: "sqlite".to_owned(),
3208 passed: Some(status.integrity_ok),
3209 detail: None,
3210 },
3211 IntegrityDoctorInput {
3212 name: "sqlite-wal".to_owned(),
3213 passed: Some(diagnostics.journal_mode.eq_ignore_ascii_case("wal")),
3214 detail: Some(diagnostics.journal_mode),
3215 },
3216 IntegrityDoctorInput {
3217 name: "sqlite-foreign-keys".to_owned(),
3218 passed: Some(diagnostics.foreign_keys_enabled),
3219 detail: None,
3220 },
3221 IntegrityDoctorInput {
3222 name: "sqlite-fts5".to_owned(),
3223 passed: Some(diagnostics.fts5_index_available),
3224 detail: None,
3225 },
3226 IntegrityDoctorInput {
3227 name: "repository-paths".to_owned(),
3228 passed: Some(repository_paths_valid),
3229 detail: None,
3230 },
3231 IntegrityDoctorInput {
3232 name: "git".to_owned(),
3233 passed: Some(git_available),
3234 detail: None,
3235 },
3236 IntegrityDoctorInput {
3237 name: "extractor-matrix".to_owned(),
3238 passed: Some(true),
3239 detail: Some(
3240 "Built-in extractor inventory is compiled into this binary.".to_owned(),
3241 ),
3242 },
3243 IntegrityDoctorInput {
3244 name: "network-policy".to_owned(),
3245 passed: Some(true),
3246 detail: Some(
3247 "Remote providers and non-loopback HTTP remain explicitly gated.".to_owned(),
3248 ),
3249 },
3250 IntegrityDoctorInput {
3251 name: "writer-lock".to_owned(),
3252 passed: None,
3253 detail: Some(
3254 "Read-only doctor does not acquire or reclaim the workspace writer lock."
3255 .to_owned(),
3256 ),
3257 },
3258 IntegrityDoctorInput {
3259 name: "hook-installation".to_owned(),
3260 passed: None,
3261 detail: Some(
3262 "Use `csgraph hooks status` with an explicit host and repository root."
3263 .to_owned(),
3264 ),
3265 },
3266 IntegrityDoctorInput {
3267 name: "release-files".to_owned(),
3268 passed: None,
3269 detail: Some(
3270 "Release artifact notices and checksums require an explicit installation path."
3271 .to_owned(),
3272 ),
3273 },
3274 ],
3275 freshness,
3276 providers: vec![ProviderDoctorInput {
3277 name: "codegraph".to_owned(),
3278 status: ProviderDoctorStatus::Unavailable,
3279 detail: Some(
3280 "CodeGraph probing is opt-in and was not requested by this doctor invocation."
3281 .to_owned(),
3282 ),
3283 }],
3284 config: vec![ConfigDoctorInput {
3285 name: status.workspace,
3286 valid: Some(true),
3287 detail: None,
3288 }],
3289 }))
3290}
3291
3292pub fn create_diagnostic_bundle(
3302 config_path: &Path,
3303 database_path: &Path,
3304 output_path: &Path,
3305) -> Result<DiagnosticBundle, ApplicationError> {
3306 let generated_at_unix_ms = SystemTime::now()
3307 .duration_since(UNIX_EPOCH)
3308 .map_err(|error| ApplicationError::Initialization(error.to_string()))?
3309 .as_millis();
3310 let bundle = DiagnosticBundle {
3311 schema_version: 1,
3312 binary_version: env!("CARGO_PKG_VERSION").to_owned(),
3313 operating_system: std::env::consts::OS.to_owned(),
3314 architecture: std::env::consts::ARCH.to_owned(),
3315 generated_at_unix_ms,
3316 debug_requested: std::env::var("CODE_SYSTEM_GRAPH_DEBUG")
3317 .is_ok_and(|value| value.trim() == "1"),
3318 doctor: doctor_workspace(config_path, database_path)?,
3319 };
3320
3321 let mut options = OpenOptions::new();
3322 options.write(true).create_new(true);
3323 #[cfg(unix)]
3324 {
3325 use std::os::unix::fs::OpenOptionsExt;
3326
3327 options.mode(0o600);
3328 }
3329 let mut output = options
3330 .open(output_path)
3331 .map_err(|source| ApplicationError::WriteFile {
3332 path: output_path.to_path_buf(),
3333 source,
3334 })?;
3335 serde_json::to_writer_pretty(&mut output, &bundle)
3336 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
3337 output
3338 .write_all(b"\n")
3339 .and_then(|()| output.sync_all())
3340 .map_err(|source| ApplicationError::WriteFile {
3341 path: output_path.to_path_buf(),
3342 source,
3343 })?;
3344 Ok(bundle)
3345}
3346
3347pub fn backup_database(
3353 database_path: &Path,
3354 backup_path: &Path,
3355) -> Result<BackupSummary, ApplicationError> {
3356 SqliteStore::backup_file(database_path, backup_path)?;
3357 Ok(BackupSummary {
3358 database: database_path.to_string_lossy().into_owned(),
3359 backup: backup_path.to_string_lossy().into_owned(),
3360 })
3361}
3362
3363pub fn restore_database(
3369 database_path: &Path,
3370 backup_path: &Path,
3371) -> Result<RestoreSummary, ApplicationError> {
3372 let report = SqliteStore::restore_from(database_path, backup_path)?;
3373 Ok(RestoreSummary {
3374 source: report.source_path.to_string_lossy().into_owned(),
3375 safety_backup: report
3376 .safety_backup_path
3377 .map(|path| path.to_string_lossy().into_owned()),
3378 schema_version: report.schema_version,
3379 })
3380}
3381
3382pub fn list_workspace_registry(
3388 database_path: &Path,
3389) -> Result<Vec<WorkspaceRegistryItem>, ApplicationError> {
3390 let store = SqliteStore::open_read_only(database_path)?;
3391 Ok(store
3392 .list_workspaces()?
3393 .into_iter()
3394 .map(|workspace| WorkspaceRegistryItem {
3395 id: workspace.id.as_str().to_owned(),
3396 name: workspace.name,
3397 manifest_hash: workspace.manifest_hash,
3398 config_path: workspace.config_path.map(|path| path.display),
3399 repository_count: workspace.repository_count,
3400 })
3401 .collect())
3402}
3403
3404pub fn add_workspace_to_registry(
3411 database_path: &Path,
3412 workspace_name: &str,
3413 config_path: &Path,
3414) -> Result<WorkspaceMutationSummary, ApplicationError> {
3415 let context = load_workspace_context(config_path, &ScanOverrides::default())?;
3416 if context.manifest.name != workspace_name {
3417 return Err(ApplicationError::WorkspaceNameMismatch {
3418 requested: workspace_name.to_owned(),
3419 manifest: context.manifest.name,
3420 });
3421 }
3422 let _lock = StoreLock::acquire(database_path, Duration::from_mins(5))?;
3423 let mut store = SqliteStore::open(database_path)?;
3424 if store.workspace_exists(workspace_name)? {
3425 return Err(ApplicationError::WorkspaceAlreadyExists(
3426 workspace_name.to_owned(),
3427 ));
3428 }
3429 store.save_workspace_registry(&context.registry.record)?;
3430 Ok(WorkspaceMutationSummary {
3431 workspace: workspace_name.to_owned(),
3432 config_path: context.registry.record.config_path.map(|path| path.display),
3433 repository_count: context.registry.record.repositories.len(),
3434 operation: "add".to_owned(),
3435 })
3436}
3437
3438pub fn remove_workspace_from_registry(
3444 database_path: &Path,
3445 workspace_name: &str,
3446) -> Result<WorkspaceMutationSummary, ApplicationError> {
3447 let _lock = StoreLock::acquire(database_path, Duration::from_mins(5))?;
3448 let mut store = SqliteStore::open(database_path)?;
3449 let repository_count = store
3450 .load_workspace_registry(workspace_name)
3451 .map_err(|error| match error {
3452 StoreError::RegistryIncomplete(_) => {
3453 ApplicationError::WorkspaceNotFound(workspace_name.to_owned())
3454 }
3455 other => ApplicationError::Store(other),
3456 })?
3457 .repositories
3458 .len();
3459 if !store.remove_workspace(workspace_name)? {
3460 return Err(ApplicationError::WorkspaceNotFound(
3461 workspace_name.to_owned(),
3462 ));
3463 }
3464 Ok(WorkspaceMutationSummary {
3465 workspace: workspace_name.to_owned(),
3466 config_path: None,
3467 repository_count,
3468 operation: "remove".to_owned(),
3469 })
3470}
3471
3472pub fn list_repository_registry(
3478 database_path: &Path,
3479 workspace: &str,
3480) -> Result<Vec<RepositoryRecord>, ApplicationError> {
3481 let store = SqliteStore::open_read_only(database_path)?;
3482 Ok(store.load_workspace_registry(workspace)?.repositories)
3483}
3484
3485pub fn add_repository_to_manifest(
3491 manifest_path: &Path,
3492 alias: &str,
3493 repository_path: &str,
3494 dry_run: bool,
3495) -> Result<ManifestMutationSummary, ApplicationError> {
3496 let source = read_file(manifest_path)?;
3497 let edit = preview_add_repository(&source, alias, repository_path)?;
3498 validate_manifest_edit(manifest_path, &edit)?;
3499 finish_manifest_edit(manifest_path, edit, dry_run)
3500}
3501
3502pub fn remove_repository_from_manifest(
3508 manifest_path: &Path,
3509 alias: &str,
3510 dry_run: bool,
3511) -> Result<ManifestMutationSummary, ApplicationError> {
3512 let source = read_file(manifest_path)?;
3513 let edit = preview_remove_repository(&source, alias)?;
3514 validate_manifest_edit(manifest_path, &edit)?;
3515 finish_manifest_edit(manifest_path, edit, dry_run)
3516}
3517
3518pub fn add_manual_link_to_manifest(
3525 manifest_path: &Path,
3526 link: &ManualLinkConfig,
3527 dry_run: bool,
3528) -> Result<ManifestMutationSummary, ApplicationError> {
3529 let source = read_file(manifest_path)?;
3530 let edit = preview_add_manual_link(&source, link)?;
3531 validate_manifest_edit(manifest_path, &edit)?;
3532 finish_manifest_edit(manifest_path, edit, dry_run)
3533}
3534
3535fn validate_manifest_edit(
3536 manifest_path: &Path,
3537 edit: &ManifestEdit,
3538) -> Result<(), ApplicationError> {
3539 let manifest = parse_manifest(&edit.updated_source)?;
3540 register_workspace(manifest_path, &edit.updated_source, &manifest)?;
3541 Ok(())
3542}
3543
3544fn finish_manifest_edit(
3545 manifest_path: &Path,
3546 edit: ManifestEdit,
3547 dry_run: bool,
3548) -> Result<ManifestMutationSummary, ApplicationError> {
3549 if dry_run {
3550 return Ok(ManifestMutationSummary {
3551 summary: edit.summary,
3552 manifest: manifest_path.to_string_lossy().into_owned(),
3553 backup: None,
3554 applied: false,
3555 rendered_manifest: Some(edit.updated_source),
3556 });
3557 }
3558 let report = commit_manifest_edit(manifest_path, &edit)?;
3559 Ok(ManifestMutationSummary {
3560 summary: edit.summary,
3561 manifest: report.manifest_path.to_string_lossy().into_owned(),
3562 backup: Some(report.backup_path.to_string_lossy().into_owned()),
3563 applied: true,
3564 rendered_manifest: None,
3565 })
3566}
3567
3568fn evaluate_repository_freshness(
3569 repository: &code_system_graph_model::RepositoryRecord,
3570 previous: Option<&RepoFreshness>,
3571 current_manifest_hash: &str,
3572) -> (RepoFreshnessState, Option<String>) {
3573 if repository.working_tree_dirty {
3574 return (
3575 RepoFreshnessState::WorkingTreeChanged,
3576 Some("Git working tree has tracked or untracked changes.".to_owned()),
3577 );
3578 }
3579 let Some(previous) = previous else {
3580 return (
3581 RepoFreshnessState::Unknown,
3582 Some("Checkout was not observed in the published snapshot.".to_owned()),
3583 );
3584 };
3585 if previous.manifest_hash != current_manifest_hash {
3586 return (
3587 RepoFreshnessState::ConfigChanged,
3588 Some("Workspace manifest changed after the published snapshot.".to_owned()),
3589 );
3590 }
3591 if previous.head_commit != repository.head_commit {
3592 return (
3593 RepoFreshnessState::CommitsBehind,
3594 Some("Repository HEAD changed after the published snapshot.".to_owned()),
3595 );
3596 }
3597 (previous.state, previous.reason.clone())
3598}
3599
3600fn freshness_summary(repositories: &[RepoFreshness]) -> FreshnessSummary {
3601 let overall = if repositories
3602 .iter()
3603 .all(|freshness| freshness.state == RepoFreshnessState::Fresh)
3604 {
3605 OverallFreshness::Fresh
3606 } else if repositories.iter().any(|freshness| {
3607 matches!(
3608 freshness.state,
3609 RepoFreshnessState::Unknown
3610 | RepoFreshnessState::Unavailable
3611 | RepoFreshnessState::Corrupt
3612 )
3613 }) {
3614 OverallFreshness::Unknown
3615 } else if repositories
3616 .iter()
3617 .any(|freshness| freshness.state == RepoFreshnessState::Partial)
3618 {
3619 OverallFreshness::Partial
3620 } else {
3621 OverallFreshness::Stale
3622 };
3623 FreshnessSummary {
3624 overall,
3625 stale_repositories: repositories
3626 .iter()
3627 .filter(|freshness| freshness.state != RepoFreshnessState::Fresh)
3628 .map(|freshness| freshness.repo_id.clone())
3629 .collect(),
3630 reasons: repositories
3631 .iter()
3632 .filter_map(|freshness| freshness.reason.clone())
3633 .collect(),
3634 }
3635}
3636
3637fn load_workspace_context(
3638 config_path: &Path,
3639 overrides: &ScanOverrides,
3640) -> Result<WorkspaceContext, ApplicationError> {
3641 let manifest_source = read_file(config_path)?;
3642 let manifest = parse_manifest(&manifest_source)?;
3643 validate_global_policy_source(config_path, &manifest)?;
3644 let extraction_budgets = ExtractionBudgets::resolve(manifest.extraction_budgets.as_ref())
3645 .map_err(ManifestError::from)?;
3646 let execution_policy = ExecutionPolicy::resolve(manifest.execution_policy.as_ref())
3647 .map_err(ManifestError::from)?;
3648 for alias in overrides.repo_openapi.keys() {
3649 if !manifest.repos.contains_key(alias) {
3650 return Err(ApplicationError::UnknownOverrideRepository(alias.clone()));
3651 }
3652 }
3653 let mut registry = register_workspace(config_path, &manifest_source, &manifest)?;
3654 let mut repository_configs = BTreeMap::new();
3655 let mut semantic_manifest = manifest.clone();
3656 semantic_manifest.execution_policy = None;
3657 let mut fingerprint_material = serde_json::to_string(&semantic_manifest)
3658 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
3659 for (alias, repository) in &manifest.repos {
3660 let checkout_path = registry
3661 .checkout_path(alias)
3662 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
3663 let mut effective = resolve_repository_config(checkout_path, repository)?;
3664 if let Some(openapi) = overrides.repo_openapi.get(alias) {
3665 apply_openapi_override(&mut effective, openapi)?;
3666 }
3667 fingerprint_material.push('\n');
3668 fingerprint_material.push_str(alias);
3669 fingerprint_material.push(':');
3670 fingerprint_material.push_str(&effective.fingerprint);
3671 repository_configs.insert(alias.clone(), effective);
3672 }
3673 registry.record.manifest_hash = stable_id("manifest", &fingerprint_material);
3674 Ok(WorkspaceContext {
3675 manifest,
3676 registry,
3677 repository_configs,
3678 extraction_budgets,
3679 execution_policy,
3680 })
3681}
3682
3683fn validate_global_policy_source(
3684 config_path: &Path,
3685 manifest: &WorkspaceManifest,
3686) -> Result<(), ApplicationError> {
3687 if manifest.extraction_budgets.is_none() && manifest.execution_policy.is_none() {
3688 return Ok(());
3689 }
3690 let canonical_config =
3691 fs::canonicalize(config_path).map_err(|source| ApplicationError::ReadFile {
3692 path: config_path.to_path_buf(),
3693 source,
3694 })?;
3695 let base = canonical_config.parent().ok_or_else(|| {
3696 ApplicationError::Initialization(format!(
3697 "workspace manifest `{}` has no parent directory",
3698 canonical_config.display()
3699 ))
3700 })?;
3701 for (alias, repository) in &manifest.repos {
3702 let configured = Path::new(&repository.path);
3703 let candidate = if configured.is_absolute() {
3704 configured.to_path_buf()
3705 } else {
3706 base.join(configured)
3707 };
3708 let checkout =
3709 fs::canonicalize(&candidate).map_err(|source| ApplicationError::ReadFile {
3710 path: candidate.clone(),
3711 source,
3712 })?;
3713 if canonical_config.starts_with(&checkout) {
3714 return Err(ApplicationError::UntrustedGlobalPolicySource {
3715 config: canonical_config,
3716 repository: alias.clone(),
3717 });
3718 }
3719 }
3720 Ok(())
3721}
3722
3723fn focused_batch_cache_complete(
3724 fingerprints: &[ArtifactFingerprint],
3725 stored: &[StoredExtractorBatch],
3726 budgets: &ExtractionBudgets,
3727) -> bool {
3728 let budget_fingerprint = budgets.fingerprint();
3729 fingerprints
3730 .iter()
3731 .filter(|fingerprint| focused_extractor(&fingerprint.extractor))
3732 .all(|fingerprint| {
3733 stored.iter().any(|batch| {
3734 batch.source == *fingerprint
3735 && batch.extractor_version == EXTRACTION_CONTRACT_VERSION
3736 && batch.budget_fingerprint == budget_fingerprint
3737 })
3738 })
3739}
3740
3741fn stored_batch_degradations(
3742 stored: &[StoredExtractorBatch],
3743 budgets: &ExtractionBudgets,
3744) -> Result<Vec<String>, ApplicationError> {
3745 let mut degradations = Vec::new();
3746 for batch in stored {
3747 if batch.source_was_lossy {
3748 degradations.push(format!("{} contains invalid UTF-8 and was decoded lossily; extracted evidence is incomplete", batch.source.path.display));
3749 }
3750 if batch.source.extractor == "code-system-graph.data.artifact" {
3751 let decoded: ExtractorBatch<DataDocument> =
3752 load_extractor_batch_with_budgets(batch, budgets)?;
3753 for document in decoded.outputs {
3754 if document.incomplete {
3755 degradations.push(format!(
3756 "{} data extraction is incomplete: {:?}",
3757 batch.source.path.display, document.warnings
3758 ));
3759 }
3760 }
3761 }
3762 }
3763 Ok(degradations)
3764}
3765
3766fn finalize_scan_degradations(mut degradations: Vec<String>) -> (usize, Vec<String>) {
3767 degradations.sort();
3768 degradations.dedup();
3769 let count = degradations.len();
3770 if count > MAX_SCAN_DEGRADATIONS {
3771 degradations.truncate(MAX_SCAN_DEGRADATIONS - 1);
3772 degradations.push(format!(
3773 "{} additional degradations omitted",
3774 count - (MAX_SCAN_DEGRADATIONS - 1)
3775 ));
3776 }
3777 (count, degradations)
3778}
3779
3780#[expect(
3781 clippy::too_many_lines,
3782 reason = "Batch decode, reuse, extraction, and persistence share one fail-closed type boundary"
3783)]
3784fn assemble_focused_batches(
3785 context: &WorkspaceContext,
3786 fingerprints: &[ArtifactFingerprint],
3787 previous: &[StoredExtractorBatch],
3788 plan: &ExtractorBatchPlan,
3789 cached_keys: &BTreeSet<ArtifactKey>,
3790 work_state: &mut work_state::WorkState,
3791) -> Result<FocusedBatchState, ApplicationError> {
3792 let previous_by_key = previous
3793 .iter()
3794 .map(|batch| (ArtifactKey::from(&batch.source), batch))
3795 .collect::<BTreeMap<_, _>>();
3796 let planned_actions = plan
3797 .batches
3798 .iter()
3799 .map(|batch| (batch.key.clone(), batch.action))
3800 .collect::<BTreeMap<_, _>>();
3801 let mut previous_source_batches = previous
3802 .iter()
3803 .filter(|batch| source_extractor(&batch.source.extractor))
3804 .map(|batch| load_extractor_batch_with_budgets(batch, &context.extraction_budgets))
3805 .collect::<Result<Vec<ExtractorBatch<SourceObservation>>, _>>()?;
3806 previous_source_batches.sort_by_key(ExtractorBatch::key);
3807
3808 let checkouts = context
3809 .registry
3810 .record
3811 .repositories
3812 .iter()
3813 .filter_map(|repository| {
3814 context
3815 .registry
3816 .checkout_path(&repository.alias)
3817 .map(|path| (repository.id.clone(), path))
3818 })
3819 .collect::<BTreeMap<_, _>>();
3820 let mut source_batches = Vec::new();
3821 let mut package_batches = Vec::new();
3822 let mut generated_client_batches = Vec::new();
3823 let mut graphql_batches = Vec::new();
3824 let mut event_batches = Vec::new();
3825 let mut protobuf_batches = Vec::new();
3826 let mut data_batches = Vec::new();
3827 let mut infrastructure_batches = Vec::new();
3828 let mut documentation_batches = Vec::new();
3829 let mut config_batches = Vec::new();
3830 let mut stored_batches = Vec::new();
3831 let mut degradations = Vec::new();
3832 let mut force_relink = false;
3833 let mut checkpoint_writes = 0_u64;
3834 let mut artifact_durations_ms = Vec::new();
3835 for fingerprint in fingerprints
3836 .iter()
3837 .filter(|fingerprint| focused_extractor(&fingerprint.extractor))
3838 {
3839 let artifact_started = Instant::now();
3840 let key = ArtifactKey::from(fingerprint);
3841 let reusable = previous_by_key.get(&key).copied().filter(|batch| {
3842 (planned_actions.get(&key) == Some(&BatchAction::Reuse) || cached_keys.contains(&key))
3843 && batch.source.content_hash == fingerprint.content_hash
3844 && batch.extractor_version == EXTRACTION_CONTRACT_VERSION
3845 && batch.budget_fingerprint == context.extraction_budgets.fingerprint()
3846 });
3847 if let Some(stored) = reusable {
3848 stored_batches.push(stored.clone());
3849 if source_extractor(&fingerprint.extractor) {
3850 source_batches.push(load_extractor_batch_with_budgets(
3851 stored,
3852 &context.extraction_budgets,
3853 )?);
3854 } else if fingerprint.extractor == "code-system-graph.packages" {
3855 package_batches.push(load_extractor_batch_with_budgets(
3856 stored,
3857 &context.extraction_budgets,
3858 )?);
3859 } else if graphql_extractor(&fingerprint.extractor) {
3860 graphql_batches.push(load_extractor_batch_with_budgets(
3861 stored,
3862 &context.extraction_budgets,
3863 )?);
3864 } else if event_extractor(&fingerprint.extractor) {
3865 event_batches.push(load_extractor_batch_with_budgets(
3866 stored,
3867 &context.extraction_budgets,
3868 )?);
3869 } else if protobuf_extractor(&fingerprint.extractor) {
3870 protobuf_batches.push(load_extractor_batch_with_budgets(
3871 stored,
3872 &context.extraction_budgets,
3873 )?);
3874 } else if data_extractor(&fingerprint.extractor) {
3875 data_batches.push(load_extractor_batch_with_budgets(
3876 stored,
3877 &context.extraction_budgets,
3878 )?);
3879 } else if infrastructure_extractor(&fingerprint.extractor) {
3880 infrastructure_batches.push(load_extractor_batch_with_budgets(
3881 stored,
3882 &context.extraction_budgets,
3883 )?);
3884 } else if documentation_extractor(&fingerprint.extractor) {
3885 documentation_batches.push(load_extractor_batch_with_budgets(
3886 stored,
3887 &context.extraction_budgets,
3888 )?);
3889 } else if fingerprint.extractor == "code-system-graph.config.safe" {
3890 config_batches.push(load_extractor_batch_with_budgets(
3891 stored,
3892 &context.extraction_budgets,
3893 )?);
3894 } else {
3895 generated_client_batches.push(load_extractor_batch_with_budgets(
3896 stored,
3897 &context.extraction_budgets,
3898 )?);
3899 }
3900 worker::report_progress(code_system_graph_core::JobPhase::Extraction, 1);
3901 artifact_durations_ms.push(duration_millis(artifact_started.elapsed()));
3902 continue;
3903 }
3904 if previous_by_key.contains_key(&key) {
3905 force_relink = true;
3906 }
3907 let checkout = checkouts.get(&fingerprint.repo_id).ok_or_else(|| {
3908 ApplicationError::RegistryAliasMissing(fingerprint.repo_id.as_str().to_owned())
3909 })?;
3910 let relative_path = native_relative_path(&fingerprint.path);
3911 let artifact_path = checkout.join(&relative_path);
3912 let mut tracker = ExtractionTracker::new(
3913 &fingerprint.path.display,
3914 &fingerprint.extractor,
3915 &context.extraction_budgets,
3916 );
3917 let (source, source_was_lossy) = read_source_file(&artifact_path, &mut tracker)?;
3918 if source_was_lossy {
3919 degradations.push(format!("{} contains invalid UTF-8 and was decoded lossily; extracted evidence is incomplete", fingerprint.path.display));
3920 }
3921 let mut persist = |stored: StoredExtractorBatch| -> Result<(), ApplicationError> {
3922 if work_state
3923 .put_batch(
3924 &stored,
3925 context.execution_policy.max_checkpoint_cache_bytes,
3926 current_unix_millis(),
3927 )
3928 .map_err(ApplicationError::Initialization)?
3929 {
3930 checkpoint_writes = checkpoint_writes.checked_add(1).ok_or_else(|| {
3931 ApplicationError::Initialization(
3932 "checkpoint write counter overflowed".to_owned(),
3933 )
3934 })?;
3935 }
3936 stored_batches.push(stored);
3937 Ok(())
3938 };
3939 if source_extractor(&fingerprint.extractor) {
3940 let syntax = inspect_source_syntax(
3941 source_syntax_language(&fingerprint.extractor),
3942 &portable_path(&fingerprint.path.display),
3943 &source,
3944 &mut tracker,
3945 )?;
3946 let reserved_observations =
3947 u64::try_from(syntax.boundary_candidate_count).map_err(|_| {
3948 ApplicationError::InvalidSourceObservation(format!(
3949 "{} contains too many syntax candidates",
3950 fingerprint.path.display
3951 ))
3952 })?;
3953 tracker.charge_work(reserved_observations)?;
3954 precheck_focused_source_values(&source, &mut tracker)?;
3955 let mut observations = match fingerprint.extractor.as_str() {
3956 "code-system-graph.source.javascript" => {
3957 parse_javascript_source_at_path_with_tracker(
3958 &portable_path(&fingerprint.path.display),
3959 &source,
3960 &mut tracker,
3961 )?
3962 }
3963 "code-system-graph.source.typescript" => {
3964 parse_typescript_source_at_path_with_tracker(
3965 &portable_path(&fingerprint.path.display),
3966 &source,
3967 &mut tracker,
3968 )?
3969 }
3970 "code-system-graph.source.rust" => {
3971 parse_rust_source_with_tracker(&source, &mut tracker)?
3972 }
3973 "code-system-graph.source.python" => {
3974 parse_python_source_with_tracker(&source, &mut tracker)?
3975 }
3976 "code-system-graph.source.go" => {
3977 parse_go_source_with_tracker(&source, &mut tracker)?
3978 }
3979 "code-system-graph.source.java" => {
3980 parse_java_source_with_tracker(&source, &mut tracker)?
3981 }
3982 _ => Vec::new(),
3983 };
3984 if observations
3985 .iter()
3986 .any(|observation| observation.role != SourceRole::Test)
3987 && syntax.boundary_candidate_count == 0
3988 {
3989 return Err(ApplicationError::InvalidSourceObservation(format!(
3990 "{} produced framework facts without a Tree-sitter boundary candidate",
3991 fingerprint.path.display
3992 )));
3993 }
3994 if syntax.has_error {
3995 for observation in &mut observations {
3996 observation.status = SourceEpistemicStatus::Incomplete;
3997 if !observation
3998 .warnings
3999 .contains(&SourceWarning::SyntaxErrorRecovery)
4000 {
4001 observation
4002 .warnings
4003 .push(SourceWarning::SyntaxErrorRecovery);
4004 }
4005 }
4006 }
4007 let batch = ExtractorBatch::new(fingerprint.clone(), observations);
4008 persist(store_extractor_batch(
4009 &batch,
4010 &mut tracker,
4011 source_was_lossy,
4012 )?)?;
4013 source_batches.push(batch);
4014 } else if fingerprint.extractor == "code-system-graph.packages" {
4015 let portable_path = portable_path(&fingerprint.path.display);
4016 let manifest =
4017 extract_package_manifest_with_tracker(&portable_path, &source, &mut tracker)?;
4018 let batch = ExtractorBatch::new(fingerprint.clone(), vec![manifest]);
4019 persist(store_extractor_batch(
4020 &batch,
4021 &mut tracker,
4022 source_was_lossy,
4023 )?)?;
4024 package_batches.push(batch);
4025 } else if graphql_extractor(&fingerprint.extractor) {
4026 let portable_path = portable_path(&fingerprint.path.display);
4027 let document = match fingerprint.extractor.as_str() {
4028 "code-system-graph.graphql.document" => {
4029 extract_graphql_document_with_tracker(&portable_path, &source, &mut tracker)?
4030 }
4031 "code-system-graph.graphql.persisted" => GraphqlDocument {
4032 source_path: portable_path.clone(),
4033 types: Vec::new(),
4034 operations: Vec::new(),
4035 fragments: Vec::new(),
4036 persisted_operations: extract_graphql_persisted_operations_with_tracker(
4037 &portable_path,
4038 &source,
4039 &mut tracker,
4040 )?,
4041 resolvers: Vec::new(),
4042 federation: Vec::new(),
4043 complete: true,
4044 warnings: Vec::new(),
4045 },
4046 "code-system-graph.graphql.source" => {
4047 let language = source_language_for_path(&relative_path).ok_or_else(|| {
4048 ApplicationError::InvalidSourceObservation(format!(
4049 "unsupported GraphQL source language for `{portable_path}`"
4050 ))
4051 })?;
4052 let mut document =
4053 parse_graphql_source_with_tracker(language, &source, &mut tracker)?;
4054 document.source_path = portable_path;
4055 document
4056 }
4057 _ => unreachable!("graphql extractor classification must be exhaustive"),
4058 };
4059 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4060 persist(store_extractor_batch(
4061 &batch,
4062 &mut tracker,
4063 source_was_lossy,
4064 )?)?;
4065 graphql_batches.push(batch);
4066 } else if event_extractor(&fingerprint.extractor) {
4067 let portable_path = portable_path(&fingerprint.path.display);
4068 let document = if fingerprint.extractor == "code-system-graph.events.asyncapi" {
4069 extract_asyncapi(&portable_path, &source)?
4070 } else {
4071 let language = source_language_for_path(&relative_path).ok_or_else(|| {
4072 ApplicationError::InvalidSourceObservation(format!(
4073 "unsupported event source language for `{portable_path}`"
4074 ))
4075 })?;
4076 let mut document = parse_event_source(language, &source);
4077 document.source_path = Some(portable_path);
4078 document
4079 };
4080 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4081 persist(store_extractor_batch(
4082 &batch,
4083 &mut tracker,
4084 source_was_lossy,
4085 )?)?;
4086 event_batches.push(batch);
4087 } else if protobuf_extractor(&fingerprint.extractor) {
4088 let portable_path = portable_path(&fingerprint.path.display);
4089 let document = if fingerprint.extractor == "code-system-graph.protobuf" {
4090 ProtobufDocument::File(Box::new(extract_protobuf_with_tracker(
4091 &portable_path,
4092 &source,
4093 &mut tracker,
4094 )?))
4095 } else {
4096 let language = source_language_for_path(&relative_path).ok_or_else(|| {
4097 ApplicationError::InvalidSourceObservation(format!(
4098 "unsupported generated protobuf source language for `{portable_path}`"
4099 ))
4100 })?;
4101 ProtobufDocument::Generated(parse_protobuf_generated_source(
4102 language,
4103 &portable_path,
4104 &source,
4105 ))
4106 };
4107 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4108 persist(store_extractor_batch(
4109 &batch,
4110 &mut tracker,
4111 source_was_lossy,
4112 )?)?;
4113 protobuf_batches.push(batch);
4114 } else if data_extractor(&fingerprint.extractor) {
4115 let portable_path = portable_path(&fingerprint.path.display);
4116 let document = if fingerprint.extractor == "code-system-graph.data.source" {
4117 let language = source_language_for_path(&relative_path).ok_or_else(|| {
4118 ApplicationError::InvalidSourceObservation(format!(
4119 "unsupported data source language for `{portable_path}`"
4120 ))
4121 })?;
4122 let crate_root = cargo_crate_root(checkout, &artifact_path);
4123 parse_literal_sql_source_at_root(language, &portable_path, &crate_root, &source)
4124 } else {
4125 extract_data_artifact(&portable_path, &source)?
4126 };
4127 if fingerprint.extractor == "code-system-graph.data.artifact" && document.incomplete {
4128 degradations.push(format!(
4129 "{} data extraction is incomplete: {:?}",
4130 fingerprint.path.display, document.warnings
4131 ));
4132 }
4133 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4134 persist(store_extractor_batch(
4135 &batch,
4136 &mut tracker,
4137 source_was_lossy,
4138 )?)?;
4139 data_batches.push(batch);
4140 } else if infrastructure_extractor(&fingerprint.extractor) {
4141 let portable_path = portable_path(&fingerprint.path.display);
4142 let document = match fingerprint.extractor.as_str() {
4143 "code-system-graph.infrastructure.compose" => {
4144 extract_docker_compose(&portable_path, &source)?
4145 }
4146 "code-system-graph.infrastructure.kubernetes" => {
4147 extract_kubernetes(&portable_path, &source)?
4148 }
4149 "code-system-graph.infrastructure.helm" => extract_helm(&portable_path, &source)?,
4150 "code-system-graph.infrastructure.terraform" => {
4151 extract_terraform(&portable_path, &source)?
4152 }
4153 _ => unreachable!("infrastructure extractor classification must be exhaustive"),
4154 };
4155 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4156 persist(store_extractor_batch(
4157 &batch,
4158 &mut tracker,
4159 source_was_lossy,
4160 )?)?;
4161 infrastructure_batches.push(batch);
4162 } else if documentation_extractor(&fingerprint.extractor) {
4163 let portable_path = portable_path(&fingerprint.path.display);
4164 let document = match fingerprint.extractor.as_str() {
4165 "code-system-graph.documents.markdown" => {
4166 extract_markdown(&portable_path, &source)?
4167 }
4168 "code-system-graph.documents.codeowners" => {
4169 extract_codeowners(&portable_path, &source)?
4170 }
4171 "code-system-graph.documents.catalog" => {
4172 extract_service_catalog(&portable_path, &source)?
4173 }
4174 _ => unreachable!("documentation extractor classification must be exhaustive"),
4175 };
4176 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4177 persist(store_extractor_batch(
4178 &batch,
4179 &mut tracker,
4180 source_was_lossy,
4181 )?)?;
4182 documentation_batches.push(batch);
4183 } else if fingerprint.extractor == "code-system-graph.config.safe" {
4184 let portable_path = portable_path(&fingerprint.path.display);
4185 let document = extract_safe_config(&portable_path, &source)?;
4186 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4187 persist(store_extractor_batch(
4188 &batch,
4189 &mut tracker,
4190 source_was_lossy,
4191 )?)?;
4192 config_batches.push(batch);
4193 } else {
4194 let portable_path = portable_path(&fingerprint.path.display);
4195 let metadata =
4196 extract_generated_client_metadata(&portable_path, &source, &mut tracker)?;
4197 let batch = ExtractorBatch::new(fingerprint.clone(), metadata);
4198 persist(store_extractor_batch(
4199 &batch,
4200 &mut tracker,
4201 source_was_lossy,
4202 )?)?;
4203 generated_client_batches.push(batch);
4204 }
4205 worker::report_progress(code_system_graph_core::JobPhase::Extraction, 1);
4206 artifact_durations_ms.push(duration_millis(artifact_started.elapsed()));
4207 }
4208 source_batches.sort_by_key(ExtractorBatch::key);
4209 package_batches.sort_by_key(ExtractorBatch::key);
4210 generated_client_batches.sort_by_key(ExtractorBatch::key);
4211 graphql_batches.sort_by_key(ExtractorBatch::key);
4212 event_batches.sort_by_key(ExtractorBatch::key);
4213 protobuf_batches.sort_by_key(ExtractorBatch::key);
4214 data_batches.sort_by_key(ExtractorBatch::key);
4215 infrastructure_batches.sort_by_key(ExtractorBatch::key);
4216 documentation_batches.sort_by_key(ExtractorBatch::key);
4217 config_batches.sort_by_key(ExtractorBatch::key);
4218 stored_batches.sort_by(|left, right| {
4219 ArtifactKey::from(&left.source).cmp(&ArtifactKey::from(&right.source))
4220 });
4221 Ok(FocusedBatchState {
4222 source_batches,
4223 previous_source_batches,
4224 package_batches,
4225 generated_client_batches,
4226 graphql_batches,
4227 event_batches,
4228 protobuf_batches,
4229 data_batches,
4230 infrastructure_batches,
4231 documentation_batches,
4232 config_batches,
4233 stored_batches,
4234 degradations,
4235 force_relink,
4236 checkpoint_writes,
4237 artifact_durations_ms,
4238 })
4239}
4240
4241fn focused_extractor(extractor: &str) -> bool {
4242 source_extractor(extractor)
4243 || matches!(
4244 extractor,
4245 "code-system-graph.packages" | "code-system-graph.http.generated-client"
4246 )
4247 || graphql_extractor(extractor)
4248 || event_extractor(extractor)
4249 || protobuf_extractor(extractor)
4250 || data_extractor(extractor)
4251 || infrastructure_extractor(extractor)
4252 || documentation_extractor(extractor)
4253 || extractor == "code-system-graph.config.safe"
4254}
4255
4256fn graphql_extractor(extractor: &str) -> bool {
4257 matches!(
4258 extractor,
4259 "code-system-graph.graphql.document"
4260 | "code-system-graph.graphql.persisted"
4261 | "code-system-graph.graphql.source"
4262 )
4263}
4264
4265fn event_extractor(extractor: &str) -> bool {
4266 matches!(
4267 extractor,
4268 "code-system-graph.events.asyncapi" | "code-system-graph.events.source"
4269 )
4270}
4271
4272fn protobuf_extractor(extractor: &str) -> bool {
4273 matches!(
4274 extractor,
4275 "code-system-graph.protobuf" | "code-system-graph.protobuf.generated"
4276 )
4277}
4278
4279fn data_extractor(extractor: &str) -> bool {
4280 matches!(
4281 extractor,
4282 "code-system-graph.data.artifact" | "code-system-graph.data.source"
4283 )
4284}
4285
4286fn infrastructure_extractor(extractor: &str) -> bool {
4287 matches!(
4288 extractor,
4289 "code-system-graph.infrastructure.compose"
4290 | "code-system-graph.infrastructure.kubernetes"
4291 | "code-system-graph.infrastructure.helm"
4292 | "code-system-graph.infrastructure.terraform"
4293 )
4294}
4295
4296fn documentation_extractor(extractor: &str) -> bool {
4297 matches!(
4298 extractor,
4299 "code-system-graph.documents.markdown"
4300 | "code-system-graph.documents.codeowners"
4301 | "code-system-graph.documents.catalog"
4302 )
4303}
4304
4305fn source_extractor(extractor: &str) -> bool {
4306 matches!(
4307 extractor,
4308 "code-system-graph.source.javascript"
4309 | "code-system-graph.source.typescript"
4310 | "code-system-graph.source.rust"
4311 | "code-system-graph.source.python"
4312 | "code-system-graph.source.go"
4313 | "code-system-graph.source.java"
4314 )
4315}
4316
4317fn source_syntax_language(extractor: &str) -> SourceSyntaxLanguage {
4318 match extractor {
4319 "code-system-graph.source.javascript" => SourceSyntaxLanguage::JavaScript,
4320 "code-system-graph.source.typescript" => SourceSyntaxLanguage::TypeScript,
4321 "code-system-graph.source.rust" => SourceSyntaxLanguage::Rust,
4322 "code-system-graph.source.python" => SourceSyntaxLanguage::Python,
4323 "code-system-graph.source.go" => SourceSyntaxLanguage::Go,
4324 "code-system-graph.source.java" => SourceSyntaxLanguage::Java,
4325 _ => unreachable!("source_syntax_language requires a focused source extractor"),
4326 }
4327}
4328
4329fn source_language_for_path(path: &Path) -> Option<SourceLanguage> {
4330 let extension = path.extension()?.to_str()?;
4331 match extension {
4332 "js" | "jsx" => Some(SourceLanguage::JavaScript),
4333 "ts" | "tsx" => Some(SourceLanguage::TypeScript),
4334 "rs" => Some(SourceLanguage::Rust),
4335 "py" => Some(SourceLanguage::Python),
4336 "go" => Some(SourceLanguage::Go),
4337 "java" => Some(SourceLanguage::Java),
4338 _ => None,
4339 }
4340}
4341
4342#[cfg(unix)]
4343fn native_relative_path(path: &code_system_graph_model::NativePath) -> PathBuf {
4344 use std::os::unix::ffi::OsStringExt;
4345
4346 PathBuf::from(std::ffi::OsString::from_vec(path.bytes.clone()))
4347}
4348
4349#[cfg(windows)]
4350fn native_relative_path(path: &code_system_graph_model::NativePath) -> PathBuf {
4351 use std::os::windows::ffi::OsStringExt;
4352
4353 let wide = path
4354 .bytes
4355 .chunks_exact(2)
4356 .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
4357 .collect::<Vec<_>>();
4358 PathBuf::from(std::ffi::OsString::from_wide(&wide))
4359}
4360
4361#[cfg(not(any(unix, windows)))]
4362fn native_relative_path(path: &code_system_graph_model::NativePath) -> PathBuf {
4363 PathBuf::from(String::from_utf8_lossy(&path.bytes).into_owned())
4364}
4365
4366fn generated_client_graph(
4367 batches: &[ExtractorBatch<GeneratedClientMetadata>],
4368 boundaries: &[HttpBoundary],
4369) -> (Vec<Node>, Vec<Edge>, Vec<Evidence>) {
4370 let mut nodes = Vec::new();
4371 let mut edges = Vec::new();
4372 let mut evidence = Vec::new();
4373 for batch in batches {
4374 let source_path = portable_path(&batch.source.path.display);
4375 for metadata in &batch.outputs {
4376 let name = metadata.name.as_deref().unwrap_or("metadata");
4377 let stable_key = format!(
4378 "generated-client:{}:{source_path}:{name}:{}",
4379 batch.source.repo_id.as_str(),
4380 metadata.generated_file.as_deref().unwrap_or("")
4381 );
4382 let node = Node {
4383 id: NodeId::new(stable_id("node", &stable_key)),
4384 kind: NodeKind::Artifact,
4385 repo_id: Some(batch.source.repo_id.clone()),
4386 stable_key: stable_key.clone(),
4387 label: metadata
4388 .generator_name
4389 .as_ref()
4390 .map_or_else(|| format!("{} {name}", metadata.tool), Clone::clone),
4391 };
4392 let item_evidence = Evidence {
4393 id: EvidenceId::new(stable_id("evidence", &stable_key)),
4394 repo_id: Some(batch.source.repo_id.clone()),
4395 file_path: Some(source_path.clone()),
4396 start_line: Some(metadata.line),
4397 end_line: Some(metadata.line),
4398 extractor: "code-system-graph.http.generated-client".to_owned(),
4399 extractor_version: "1.0.0".to_owned(),
4400 provenance: Provenance::Extracted,
4401 confidence: 1.0,
4402 observed_at_commit: None,
4403 content_hash: Some(batch.source.content_hash.clone()),
4404 note: Some("explicit generated-client metadata".to_owned()),
4405 };
4406 if let Some(input_spec) = &metadata.input_spec {
4407 edges.extend(
4408 boundaries
4409 .iter()
4410 .filter(|boundary| {
4411 boundary.node.repo_id.as_ref() == Some(&batch.source.repo_id)
4412 && boundary.evidence.file_path.as_ref() == Some(input_spec)
4413 })
4414 .map(|boundary| {
4415 let key = format!(
4416 "{}:{:?}:{}",
4417 node.id.as_str(),
4418 EdgeKind::Consumes,
4419 boundary.node.id.as_str()
4420 );
4421 Edge {
4422 id: EdgeId::new(stable_id("edge", &key)),
4423 source: node.id.clone(),
4424 target: boundary.node.id.clone(),
4425 kind: EdgeKind::Consumes,
4426 confidence: 1.0,
4427 status: EpistemicStatus::Confirmed,
4428 evidence: vec![item_evidence.id.clone()],
4429 }
4430 }),
4431 );
4432 }
4433 nodes.push(node);
4434 evidence.push(item_evidence);
4435 }
4436 }
4437 nodes.sort_by(|left, right| left.id.cmp(&right.id));
4438 nodes.dedup_by(|left, right| left.id == right.id);
4439 edges.sort_by(|left, right| left.id.cmp(&right.id));
4440 edges.dedup_by(|left, right| left.id == right.id);
4441 evidence.sort_by(|left, right| left.id.cmp(&right.id));
4442 evidence.dedup_by(|left, right| left.id == right.id);
4443 (nodes, edges, evidence)
4444}
4445
4446#[expect(
4447 clippy::too_many_lines,
4448 reason = "Graph assembly keeps every source family in one deterministic merge boundary"
4449)]
4450fn assemble_graph(
4451 context: &WorkspaceContext,
4452 fingerprints: &[ArtifactFingerprint],
4453 focused: &FocusedBatchState,
4454) -> Result<GraphAssembly, ApplicationError> {
4455 let mut boundaries = extract_boundaries(context)?;
4456 let mut tests = extract_declared_tests(context, fingerprints)?;
4457 let mut implementations = extract_declared_implementations(context, fingerprints)?;
4458 let source_facts = focused
4459 .source_batches
4460 .iter()
4461 .map(|batch| {
4462 source_observations_to_graph(
4463 &batch.source.repo_id,
4464 &portable_path(&batch.source.path.display),
4465 &batch.source.content_hash,
4466 &batch.outputs,
4467 )
4468 })
4469 .collect::<Vec<SourceGraphFacts>>();
4470 for facts in &source_facts {
4471 boundaries.extend(facts.boundaries.clone());
4472 tests.extend(facts.tests.clone());
4473 implementations.extend(facts.implementations.clone());
4474 }
4475 let mut package_facts = focused
4476 .package_batches
4477 .iter()
4478 .flat_map(|batch| {
4479 batch.outputs.iter().map(|manifest| {
4480 package_manifest_to_graph(
4481 &batch.source.repo_id,
4482 &portable_path(&batch.source.path.display),
4483 &batch.source.content_hash,
4484 manifest,
4485 )
4486 })
4487 })
4488 .collect::<Vec<PackageGraphFacts>>();
4489 link_registered_package_owners(&mut package_facts);
4490 let (generated_nodes, generated_edges, generated_evidence) =
4491 generated_client_graph(&focused.generated_client_batches, &boundaries);
4492 let graphql_sources = focused
4493 .graphql_batches
4494 .iter()
4495 .flat_map(|batch| {
4496 batch.outputs.iter().map(|document| {
4497 (
4498 batch.source.repo_id.clone(),
4499 portable_path(&batch.source.path.display),
4500 batch.source.content_hash.clone(),
4501 document,
4502 )
4503 })
4504 })
4505 .collect::<Vec<_>>();
4506 let graphql_inputs = graphql_sources
4507 .iter()
4508 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4509 .collect::<Vec<_>>();
4510 let GraphqlGraphFacts {
4511 nodes: graphql_nodes,
4512 edges: graphql_edges,
4513 evidence: graphql_evidence,
4514 } = graphql_documents_to_graph(&graphql_inputs);
4515 let event_sources = focused
4516 .event_batches
4517 .iter()
4518 .flat_map(|batch| {
4519 batch.outputs.iter().map(|document| {
4520 (
4521 batch.source.repo_id.clone(),
4522 portable_path(&batch.source.path.display),
4523 batch.source.content_hash.clone(),
4524 document,
4525 )
4526 })
4527 })
4528 .collect::<Vec<_>>();
4529 let event_inputs = event_sources
4530 .iter()
4531 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4532 .collect::<Vec<_>>();
4533 let EventGraphFacts {
4534 nodes: event_nodes,
4535 edges: event_edges,
4536 evidence: event_evidence,
4537 } = event_documents_to_graph(&event_inputs);
4538 let protobuf_sources = focused
4539 .protobuf_batches
4540 .iter()
4541 .flat_map(|batch| {
4542 batch.outputs.iter().map(|document| {
4543 (
4544 batch.source.repo_id.clone(),
4545 portable_path(&batch.source.path.display),
4546 batch.source.content_hash.clone(),
4547 document,
4548 )
4549 })
4550 })
4551 .collect::<Vec<_>>();
4552 let protobuf_inputs = protobuf_sources
4553 .iter()
4554 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4555 .collect::<Vec<_>>();
4556 let ProtobufGraphFacts {
4557 nodes: protobuf_nodes,
4558 edges: protobuf_edges,
4559 evidence: protobuf_evidence,
4560 } = protobuf_documents_to_graph(&protobuf_inputs);
4561 let data_sources = focused
4562 .data_batches
4563 .iter()
4564 .flat_map(|batch| {
4565 batch.outputs.iter().map(|document| {
4566 (
4567 batch.source.repo_id.clone(),
4568 portable_path(&batch.source.path.display),
4569 batch.source.content_hash.clone(),
4570 document,
4571 )
4572 })
4573 })
4574 .collect::<Vec<_>>();
4575 let data_inputs = data_sources
4576 .iter()
4577 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4578 .collect::<Vec<_>>();
4579 let infrastructure_sources = focused
4580 .infrastructure_batches
4581 .iter()
4582 .flat_map(|batch| {
4583 batch.outputs.iter().map(|document| {
4584 (
4585 batch.source.repo_id.clone(),
4586 portable_path(&batch.source.path.display),
4587 batch.source.content_hash.clone(),
4588 document,
4589 )
4590 })
4591 })
4592 .collect::<Vec<_>>();
4593 let infrastructure_inputs = infrastructure_sources
4594 .iter()
4595 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4596 .collect::<Vec<_>>();
4597 let documentation_sources = focused
4598 .documentation_batches
4599 .iter()
4600 .flat_map(|batch| {
4601 batch.outputs.iter().map(|document| {
4602 (
4603 batch.source.repo_id.clone(),
4604 portable_path(&batch.source.path.display),
4605 batch.source.content_hash.clone(),
4606 document,
4607 )
4608 })
4609 })
4610 .collect::<Vec<_>>();
4611 let documentation_inputs = documentation_sources
4612 .iter()
4613 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4614 .collect::<Vec<_>>();
4615 let config_sources = focused
4616 .config_batches
4617 .iter()
4618 .flat_map(|batch| {
4619 batch.outputs.iter().map(|document| {
4620 (
4621 batch.source.repo_id.clone(),
4622 portable_path(&batch.source.path.display),
4623 batch.source.content_hash.clone(),
4624 document,
4625 )
4626 })
4627 })
4628 .collect::<Vec<_>>();
4629 let config_inputs = config_sources
4630 .iter()
4631 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4632 .collect::<Vec<_>>();
4633 let mut known_nodes = boundaries
4634 .iter()
4635 .map(|boundary| boundary.node.clone())
4636 .chain(tests.iter().map(|test| test.node.clone()))
4637 .chain(
4638 implementations
4639 .iter()
4640 .map(|implementation| implementation.node.clone()),
4641 )
4642 .chain(
4643 source_facts
4644 .iter()
4645 .flat_map(|facts| facts.standalone_test_nodes.iter().cloned()),
4646 )
4647 .chain(
4648 source_facts
4649 .iter()
4650 .flat_map(|facts| facts.relation_nodes.iter().cloned()),
4651 )
4652 .chain(
4653 package_facts
4654 .iter()
4655 .flat_map(|facts| facts.nodes.iter().cloned()),
4656 )
4657 .chain(generated_nodes.iter().cloned())
4658 .chain(graphql_nodes.iter().cloned())
4659 .chain(event_nodes.iter().cloned())
4660 .chain(protobuf_nodes.iter().cloned())
4661 .collect::<Vec<_>>();
4662 known_nodes.sort_by(|left, right| left.id.cmp(&right.id));
4663 known_nodes.dedup_by(|left, right| left.id == right.id);
4664 let repository_aliases = context
4665 .registry
4666 .record
4667 .repositories
4668 .iter()
4669 .map(|repository| (repository.alias.as_str(), &repository.id))
4670 .collect::<Vec<_>>();
4671 let ExtractionGraphFacts {
4672 nodes: extraction_nodes,
4673 edges: extraction_edges,
4674 evidence: extraction_evidence,
4675 } = documents_to_graph(
4676 &data_inputs,
4677 &infrastructure_inputs,
4678 &documentation_inputs,
4679 &config_inputs,
4680 &known_nodes,
4681 &repository_aliases,
4682 );
4683
4684 let mut edges = link_http_boundaries(&boundaries)?;
4685 edges.extend(link_declared_tests(&tests, &boundaries)?);
4686 edges.extend(link_declared_implementations(
4687 &implementations,
4688 &boundaries,
4689 )?);
4690 edges.extend(
4691 source_facts
4692 .iter()
4693 .flat_map(|facts| facts.relation_edges.iter().cloned()),
4694 );
4695 edges.extend(
4696 package_facts
4697 .iter()
4698 .flat_map(|facts| facts.edges.iter().cloned()),
4699 );
4700 edges.extend(generated_edges);
4701 edges.extend(graphql_edges);
4702 edges.extend(event_edges);
4703 edges.extend(protobuf_edges);
4704 edges.extend(extraction_edges);
4705 edges.sort_by(|left, right| left.id.cmp(&right.id));
4706 edges.dedup_by(|left, right| left.id == right.id);
4707
4708 let mut nodes = boundaries
4709 .iter()
4710 .map(|boundary| (boundary.node.id.clone(), boundary.node.clone()))
4711 .collect::<BTreeMap<_, _>>();
4712 nodes.extend(
4713 tests
4714 .iter()
4715 .map(|test| (test.node.id.clone(), test.node.clone())),
4716 );
4717 nodes.extend(
4718 implementations
4719 .iter()
4720 .map(|implementation| (implementation.node.id.clone(), implementation.node.clone())),
4721 );
4722 nodes.extend(source_facts.iter().flat_map(|facts| {
4723 facts
4724 .standalone_test_nodes
4725 .iter()
4726 .map(|node| (node.id.clone(), node.clone()))
4727 }));
4728 nodes.extend(source_facts.iter().flat_map(|facts| {
4729 facts
4730 .relation_nodes
4731 .iter()
4732 .map(|node| (node.id.clone(), node.clone()))
4733 }));
4734 nodes.extend(package_facts.iter().flat_map(|facts| {
4735 facts
4736 .nodes
4737 .iter()
4738 .map(|node| (node.id.clone(), node.clone()))
4739 }));
4740 nodes.extend(
4741 generated_nodes
4742 .into_iter()
4743 .map(|node| (node.id.clone(), node)),
4744 );
4745 nodes.extend(
4746 graphql_nodes
4747 .into_iter()
4748 .map(|node| (node.id.clone(), node)),
4749 );
4750 nodes.extend(event_nodes.into_iter().map(|node| (node.id.clone(), node)));
4751 nodes.extend(
4752 protobuf_nodes
4753 .into_iter()
4754 .map(|node| (node.id.clone(), node)),
4755 );
4756 nodes.extend(
4757 extraction_nodes
4758 .into_iter()
4759 .map(|node| (node.id.clone(), node)),
4760 );
4761 let mut evidence = boundaries
4762 .iter()
4763 .map(|boundary| (boundary.evidence.id.clone(), boundary.evidence.clone()))
4764 .collect::<BTreeMap<_, _>>();
4765 evidence.extend(
4766 tests
4767 .iter()
4768 .map(|test| (test.evidence.id.clone(), test.evidence.clone())),
4769 );
4770 evidence.extend(implementations.iter().map(|implementation| {
4771 (
4772 implementation.evidence.id.clone(),
4773 implementation.evidence.clone(),
4774 )
4775 }));
4776 evidence.extend(source_facts.iter().flat_map(|facts| {
4777 facts
4778 .standalone_test_evidence
4779 .iter()
4780 .map(|evidence| (evidence.id.clone(), evidence.clone()))
4781 }));
4782 evidence.extend(source_facts.iter().flat_map(|facts| {
4783 facts
4784 .relation_evidence
4785 .iter()
4786 .map(|evidence| (evidence.id.clone(), evidence.clone()))
4787 }));
4788 evidence.extend(package_facts.iter().flat_map(|facts| {
4789 facts
4790 .evidence
4791 .iter()
4792 .map(|evidence| (evidence.id.clone(), evidence.clone()))
4793 }));
4794 evidence.extend(
4795 generated_evidence
4796 .into_iter()
4797 .map(|item| (item.id.clone(), item)),
4798 );
4799 evidence.extend(
4800 graphql_evidence
4801 .into_iter()
4802 .map(|item| (item.id.clone(), item)),
4803 );
4804 evidence.extend(
4805 event_evidence
4806 .into_iter()
4807 .map(|item| (item.id.clone(), item)),
4808 );
4809 evidence.extend(
4810 protobuf_evidence
4811 .into_iter()
4812 .map(|item| (item.id.clone(), item)),
4813 );
4814 evidence.extend(
4815 extraction_evidence
4816 .into_iter()
4817 .map(|item| (item.id.clone(), item)),
4818 );
4819 let mut link_node_keys = BTreeMap::new();
4820 link_node_keys.extend(boundaries.iter().map(|boundary| {
4821 (
4822 boundary.node.id.clone(),
4823 format!("{}:{}", boundary.method, boundary.path),
4824 )
4825 }));
4826 link_node_keys.extend(tests.iter().map(|test| {
4827 (
4828 test.node.id.clone(),
4829 format!("{}:{}", test.method, test.path),
4830 )
4831 }));
4832 link_node_keys.extend(implementations.iter().map(|implementation| {
4833 (
4834 implementation.node.id.clone(),
4835 format!("{}:{}", implementation.method, implementation.path),
4836 )
4837 }));
4838 Ok(GraphAssembly {
4839 nodes: nodes.into_values().collect(),
4840 edges,
4841 evidence: evidence.into_values().collect(),
4842 link_decisions: Vec::new(),
4843 link_node_keys,
4844 })
4845}
4846
4847#[expect(
4848 clippy::too_many_lines,
4849 reason = "Fail-closed relinking keeps old/new neighborhood handling in one audit boundary"
4850)]
4851fn relink_affected_graph(
4852 graph: &mut GraphAssembly,
4853 plan: &IncrementalPlan,
4854 batch_plan: &ExtractorBatchPlan,
4855 focused: &FocusedBatchState,
4856 _previous_nodes: &[Node],
4857 previous_edges: &[Edge],
4858) -> Result<(), ApplicationError> {
4859 let full_relink = focused.force_relink
4860 || plan.changes.iter().any(|change| {
4861 change.kind != code_system_graph_model::ArtifactChangeKind::Unchanged
4862 && change.extractor != "code-system-graph.packages"
4863 && !source_extractor(&change.extractor)
4864 });
4865 if full_relink || previous_edges.is_empty() {
4866 return Ok(());
4867 }
4868 let source_plan = ExtractorBatchPlan {
4869 batches: batch_plan
4870 .batches
4871 .iter()
4872 .filter(|batch| source_extractor(&batch.key.extractor))
4873 .cloned()
4874 .collect(),
4875 };
4876 let affected = affected_link_keys(
4877 &source_plan,
4878 &focused.previous_source_batches,
4879 &focused.source_batches,
4880 |observation| {
4881 observation
4882 .method
4883 .as_ref()
4884 .zip(observation.path.as_ref())
4885 .map(|(method, path)| format!("{method}:{path}"))
4886 },
4887 )?
4888 .into_iter()
4889 .flatten()
4890 .collect::<BTreeSet<_>>();
4891 if affected.is_empty() {
4892 return Ok(());
4893 }
4894
4895 let mut previous_node_keys = graph.link_node_keys.clone();
4896 for batch in &focused.previous_source_batches {
4897 let facts = source_observations_to_graph(
4898 &batch.source.repo_id,
4899 &portable_path(&batch.source.path.display),
4900 &batch.source.content_hash,
4901 &batch.outputs,
4902 );
4903 previous_node_keys.extend(facts.boundaries.iter().map(|boundary| {
4904 (
4905 boundary.node.id.clone(),
4906 format!("{}:{}", boundary.method, boundary.path),
4907 )
4908 }));
4909 previous_node_keys.extend(facts.tests.iter().map(|test| {
4910 (
4911 test.node.id.clone(),
4912 format!("{}:{}", test.method, test.path),
4913 )
4914 }));
4915 previous_node_keys.extend(facts.implementations.iter().map(|implementation| {
4916 (
4917 implementation.node.id.clone(),
4918 format!("{}:{}", implementation.method, implementation.path),
4919 )
4920 }));
4921 }
4922 let current_node_ids = graph
4923 .nodes
4924 .iter()
4925 .map(|node| node.id.clone())
4926 .collect::<BTreeSet<_>>();
4927 let is_http_link = |edge: &Edge| {
4928 matches!(
4929 edge.kind,
4930 code_system_graph_model::EdgeKind::CallsRemote
4931 | code_system_graph_model::EdgeKind::Validates
4932 | code_system_graph_model::EdgeKind::ImplementedBy
4933 )
4934 };
4935 let recomputed_http = graph
4936 .edges
4937 .iter()
4938 .filter(|edge| is_http_link(edge))
4939 .cloned()
4940 .collect::<Vec<_>>();
4941 let previous_http = previous_edges
4942 .iter()
4943 .filter(|edge| is_http_link(edge))
4944 .cloned()
4945 .collect::<Vec<_>>();
4946 let mut edges = graph
4947 .edges
4948 .iter()
4949 .filter(|edge| !is_http_link(edge))
4950 .cloned()
4951 .collect::<Vec<_>>();
4952 edges.extend(merge_affected_link_neighborhoods(
4953 &previous_http,
4954 &recomputed_http,
4955 &affected,
4956 &previous_node_keys,
4957 &graph.link_node_keys,
4958 ¤t_node_ids,
4959 ));
4960 edges.sort_by(|left, right| left.id.cmp(&right.id));
4961 edges.dedup_by(|left, right| left.id == right.id);
4962 graph.edges = edges;
4963 Ok(())
4964}
4965
4966#[expect(
4967 clippy::too_many_lines,
4968 reason = "Optional provider orchestration keeps budgets, isolation, and degradation accounting together"
4969)]
4970fn run_codegraph_corroboration(
4971 context: &WorkspaceContext,
4972 focused: &FocusedBatchState,
4973 plan: &IncrementalPlan,
4974 codegraph_binary: Option<PathBuf>,
4975) -> CorroborationSummary {
4976 let mut jobs = Vec::new();
4977 let mut setup_degradations = Vec::new();
4978 for repository in &context.registry.record.repositories {
4979 let Some(project_path) = context.registry.checkout_path(&repository.alias) else {
4980 setup_degradations.push(format!(
4981 "CodeGraph skipped `{}` because its checkout is unavailable",
4982 repository.alias
4983 ));
4984 continue;
4985 };
4986 let mut anchors = focused
4987 .source_batches
4988 .iter()
4989 .filter(|batch| batch.source.repo_id == repository.id)
4990 .flat_map(|batch| {
4991 let source_path = portable_path(&batch.source.path.display);
4992 batch.outputs.iter().filter_map(move |observation| {
4993 if observation.role != SourceRole::Provider {
4994 return None;
4995 }
4996 Some(SymbolAnchor {
4997 symbol: observation.symbol_name.clone()?,
4998 source_path: source_path.clone(),
4999 start_line: usize::try_from(observation.lines.start).ok()?,
5000 })
5001 })
5002 })
5003 .collect::<Vec<_>>();
5004 anchors.sort_by(|left, right| {
5005 (&left.source_path, left.start_line, &left.symbol).cmp(&(
5006 &right.source_path,
5007 right.start_line,
5008 &right.symbol,
5009 ))
5010 });
5011 anchors.dedup();
5012 if anchors.len() > 50 {
5013 anchors.truncate(50);
5014 setup_degradations.push(format!(
5015 "CodeGraph symbol corroboration for `{}` was limited to 50 anchors",
5016 repository.alias
5017 ));
5018 }
5019 let mut changed_files = plan
5020 .changes
5021 .iter()
5022 .filter(|change| repository.id == change.repo_id)
5023 .filter(|change| change.kind != code_system_graph_model::ArtifactChangeKind::Unchanged)
5024 .map(|change| portable_path(&change.path.display))
5025 .collect::<Vec<_>>();
5026 changed_files.sort();
5027 changed_files.dedup();
5028 if changed_files.len() > 1_024 {
5029 changed_files.truncate(1_024);
5030 setup_degradations.push(format!(
5031 "CodeGraph affected-test corroboration for `{}` was limited to 1,024 changed files",
5032 repository.alias
5033 ));
5034 }
5035 if !anchors.is_empty() || !changed_files.is_empty() {
5036 jobs.push((
5037 repository.id.clone(),
5038 project_path.to_path_buf(),
5039 anchors,
5040 changed_files,
5041 ));
5042 }
5043 }
5044 let worker = std::thread::spawn(move || -> Result<Vec<RepositoryCorroboration>, String> {
5045 let runtime = tokio::runtime::Builder::new_current_thread()
5046 .enable_all()
5047 .build()
5048 .map_err(|error| format!("cannot start CodeGraph corroboration runtime: {error}"))?;
5049 runtime.block_on(async move {
5050 let mut config = CodeGraphConfig::default();
5051 if let Some(binary) = codegraph_binary {
5052 config.binary = binary.into_os_string();
5053 }
5054 let provider = CodeGraphProvider::new(config).map_err(|error| error.to_string())?;
5055 let mut reports = Vec::new();
5056 for (repo_id, project_path, anchors, changed_files) in jobs {
5057 let report = corroborate_repository(
5058 &provider,
5059 repo_id.clone(),
5060 project_path,
5061 &anchors,
5062 &changed_files,
5063 ProviderBudget {
5064 timeout: Duration::from_secs(2),
5065 max_output_bytes: 256 * 1024,
5066 max_items: 20,
5067 },
5068 CancellationToken::new(),
5069 )
5070 .await;
5071 reports.push(RepositoryCorroboration { repo_id, report });
5072 }
5073 provider
5074 .shutdown()
5075 .await
5076 .map_err(|error| error.to_string())?;
5077 Ok(reports)
5078 })
5079 });
5080 let mut summary = CorroborationSummary {
5081 degradations: setup_degradations,
5082 ..CorroborationSummary::default()
5083 };
5084 match worker.join() {
5085 Ok(Ok(reports)) => {
5086 summary.confirmed_symbols = reports
5087 .iter()
5088 .flat_map(|item| &item.report.symbols)
5089 .filter(|symbol| matches!(symbol, SymbolCorroboration::Confirmed { .. }))
5090 .count();
5091 summary.affected_tests = reports
5092 .iter()
5093 .map(|item| item.report.affected_tests.len())
5094 .sum();
5095 summary.degradations.extend(
5096 reports
5097 .iter()
5098 .flat_map(|item| item.report.degradations.iter().cloned()),
5099 );
5100 summary.reports = reports;
5101 }
5102 Ok(Err(error)) => summary.degradations.push(error),
5103 Err(_) => summary
5104 .degradations
5105 .push("CodeGraph corroboration worker terminated unexpectedly".to_owned()),
5106 }
5107 summary.degradations.sort();
5108 summary.degradations.dedup();
5109 summary
5110}
5111
5112fn apply_codegraph_corroboration(graph: &mut GraphAssembly, reports: &[RepositoryCorroboration]) {
5113 for item in reports {
5114 for outcome in &item.report.symbols {
5115 let SymbolCorroboration::Confirmed {
5116 symbol,
5117 source_path,
5118 local_id,
5119 } = outcome
5120 else {
5121 continue;
5122 };
5123 let implementation_ids = ["javascript", "typescript", "rust", "python", "go", "java"]
5124 .map(|language| {
5125 NodeId::new(stable_id(
5126 "node",
5127 &format!(
5128 "symbol:{}:{language}:{source_path}:{symbol}",
5129 item.repo_id.as_str()
5130 ),
5131 ))
5132 })
5133 .into_iter()
5134 .filter(|candidate| {
5135 graph
5136 .nodes
5137 .iter()
5138 .any(|node| node.id == *candidate && node.kind == NodeKind::SymbolRef)
5139 })
5140 .collect::<BTreeSet<_>>();
5141 if implementation_ids.is_empty() {
5142 continue;
5143 }
5144 let source_evidence = graph.evidence.iter().find(|evidence| {
5145 evidence.repo_id.as_ref() == Some(&item.repo_id)
5146 && evidence.file_path.as_deref() == Some(source_path)
5147 && evidence.extractor.starts_with("code-system-graph.source.")
5148 });
5149 let (start_line, end_line, content_hash) =
5150 source_evidence.map_or((None, None, None), |evidence| {
5151 (
5152 evidence.start_line,
5153 evidence.end_line,
5154 evidence.content_hash.clone(),
5155 )
5156 });
5157 let evidence_key = format!(
5158 "codegraph:{}:{source_path}:{symbol}:{}",
5159 item.repo_id.as_str(),
5160 local_id.as_deref().unwrap_or("anonymous")
5161 );
5162 let evidence = Evidence {
5163 id: EvidenceId::new(stable_id("evidence", &evidence_key)),
5164 repo_id: Some(item.repo_id.clone()),
5165 file_path: Some(source_path.clone()),
5166 start_line,
5167 end_line,
5168 extractor: "code-system-graph.codegraph.corroboration".to_owned(),
5169 extractor_version: "1.0.0".to_owned(),
5170 provenance: Provenance::CodeGraph,
5171 confidence: 1.0,
5172 observed_at_commit: None,
5173 content_hash,
5174 note: Some("exact provider symbol name, path, and line match".to_owned()),
5175 };
5176 for edge in graph.edges.iter_mut().filter(|edge| {
5177 edge.kind == code_system_graph_model::EdgeKind::ImplementedBy
5178 && implementation_ids.contains(&edge.target)
5179 }) {
5180 if !edge.evidence.contains(&evidence.id) {
5181 edge.evidence.push(evidence.id.clone());
5182 edge.evidence.sort();
5183 }
5184 }
5185 graph.evidence.push(evidence);
5186 }
5187 }
5188 graph.evidence.sort_by(|left, right| left.id.cmp(&right.id));
5189 graph.evidence.dedup_by(|left, right| left.id == right.id);
5190}
5191
5192fn persisted_manual_link_records(
5193 snapshot_id: &str,
5194 decisions: &[LinkDecision],
5195) -> Result<Vec<ManualLinkRecord>, ApplicationError> {
5196 decisions
5197 .iter()
5198 .map(|decision| {
5199 let disposition = match decision.status {
5200 LinkStatus::Confirmed => ManualLinkDisposition::Active,
5201 LinkStatus::Suppressed => ManualLinkDisposition::Suppression,
5202 LinkStatus::Ambiguous | LinkStatus::Rejected => {
5203 return Err(ApplicationError::Initialization(
5204 "unapplied manual-link decision reached snapshot publication".to_owned(),
5205 ));
5206 }
5207 };
5208 let kind = serde_json::to_value(decision.relation)
5209 .ok()
5210 .and_then(|value| value.as_str().map(str::to_owned))
5211 .ok_or_else(|| {
5212 ApplicationError::Initialization(
5213 "manual-link relation could not be encoded".to_owned(),
5214 )
5215 })?;
5216 let reason = decision.reasons.first().cloned().ok_or_else(|| {
5217 ApplicationError::Initialization(
5218 "manual-link decision omitted its required rationale".to_owned(),
5219 )
5220 })?;
5221 let identity = format!(
5222 "{}:{kind}:{}:{}:{reason}",
5223 decision.source.as_str(),
5224 decision.target.as_str(),
5225 match disposition {
5226 ManualLinkDisposition::Active => "active",
5227 ManualLinkDisposition::Suppression => "suppression",
5228 }
5229 );
5230 Ok(ManualLinkRecord {
5231 id: stable_id("manual-link", &identity),
5232 snapshot_id: snapshot_id.to_owned(),
5233 source_node_id: decision.source.clone(),
5234 target_node_id: decision.target.clone(),
5235 kind,
5236 disposition,
5237 reason,
5238 decision: decision.clone(),
5239 config_version: 1,
5240 })
5241 })
5242 .collect()
5243}
5244
5245fn provider_capability_record(
5246 workspace: &str,
5247 repo_id: &RepoId,
5248 capability: &ProviderCapability,
5249) -> ProviderCapabilityRecord {
5250 let mut capabilities = Vec::new();
5251 if let Some(status) = serialized_enum_name(capability.status) {
5252 capabilities.push(format!("status:{status}"));
5253 }
5254 capabilities.extend(
5255 capability
5256 .tools
5257 .iter()
5258 .filter(|tool| tool.len() <= 240)
5259 .map(|tool| format!("tool:{tool}")),
5260 );
5261 for operation in &capability.operations {
5262 if let (Some(operation_name), Some(transport)) = (
5263 serialized_enum_name(operation.operation),
5264 serialized_enum_name(operation.transport),
5265 ) {
5266 capabilities.push(format!("operation:{operation_name}:{transport}"));
5267 }
5268 }
5269 capabilities.sort();
5270 capabilities.dedup();
5271 ProviderCapabilityRecord {
5272 workspace_name: workspace.to_owned(),
5273 repo_id: repo_id.clone(),
5274 provider: capability.provider.clone(),
5275 provider_version: capability
5276 .version
5277 .clone()
5278 .unwrap_or_else(|| "unknown".to_owned()),
5279 capabilities,
5280 observed_at_unix_ms: current_unix_millis(),
5281 }
5282}
5283
5284fn community_topology_unchanged(
5285 previous_nodes: &[Node],
5286 previous_edges: &[Edge],
5287 current_nodes: &[Node],
5288 current_edges: &[Edge],
5289) -> bool {
5290 previous_nodes == current_nodes
5291 && previous_edges.len() == current_edges.len()
5292 && previous_edges
5293 .iter()
5294 .zip(current_edges)
5295 .all(|(previous, current)| {
5296 previous.id == current.id
5297 && previous.source == current.source
5298 && previous.target == current.target
5299 && previous.kind == current.kind
5300 && previous.confidence.to_bits() == current.confidence.to_bits()
5301 && previous.status == current.status
5302 })
5303}
5304
5305fn serialized_enum_name<T: Serialize>(value: T) -> Option<String> {
5306 serde_json::to_value(value)
5307 .ok()
5308 .and_then(|value| value.as_str().map(str::to_owned))
5309}
5310
5311fn extract_boundaries(context: &WorkspaceContext) -> Result<Vec<HttpBoundary>, ApplicationError> {
5312 let repository_records = context
5313 .registry
5314 .record
5315 .repositories
5316 .iter()
5317 .map(|repository| (repository.alias.as_str(), repository))
5318 .collect::<BTreeMap<_, _>>();
5319 let mut boundaries = Vec::new();
5320 for alias in context.manifest.repos.keys() {
5321 let registered = repository_records
5322 .get(alias.as_str())
5323 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5324 let effective = context
5325 .repository_configs
5326 .get(alias)
5327 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5328 boundaries.extend(
5329 effective
5330 .http_consumers
5331 .iter()
5332 .map(|consumer| HttpBoundary::consumer(registered.id.clone(), consumer)),
5333 );
5334 let repository_path = context
5335 .registry
5336 .checkout_path(alias)
5337 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5338 for openapi in &effective.openapi {
5339 let openapi_path = repository_path.join(openapi);
5340 let mut tracker = ExtractionTracker::new(
5341 openapi,
5342 "code-system-graph.http.openapi",
5343 &context.extraction_budgets,
5344 );
5345 let (openapi_source, _) = read_source_file(&openapi_path, &mut tracker)?;
5346 boundaries.extend(extract_openapi_with_tracker(
5347 ®istered.id,
5348 openapi,
5349 &openapi_source,
5350 &mut tracker,
5351 )?);
5352 }
5353 }
5354 Ok(boundaries)
5355}
5356
5357fn extract_declared_tests(
5358 context: &WorkspaceContext,
5359 fingerprints: &[ArtifactFingerprint],
5360) -> Result<Vec<DeclaredTestCase>, ApplicationError> {
5361 let repository_records = context
5362 .registry
5363 .record
5364 .repositories
5365 .iter()
5366 .map(|repository| (repository.alias.as_str(), repository))
5367 .collect::<BTreeMap<_, _>>();
5368 let mut tests = Vec::new();
5369 for alias in context.manifest.repos.keys() {
5370 let repository = repository_records
5371 .get(alias.as_str())
5372 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5373 let effective = context
5374 .repository_configs
5375 .get(alias)
5376 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5377 tests.extend(effective.integration_tests.iter().map(|test| {
5378 let content_hash = artifact_content_hash(
5379 fingerprints,
5380 &repository.id,
5381 "code-system-graph.tests.declared",
5382 &test.path,
5383 );
5384 declared_test_case(repository.id.clone(), test, content_hash)
5385 }));
5386 }
5387 tests.sort_by(|left, right| left.node.id.cmp(&right.node.id));
5388 Ok(tests)
5389}
5390
5391fn extract_declared_implementations(
5392 context: &WorkspaceContext,
5393 fingerprints: &[ArtifactFingerprint],
5394) -> Result<Vec<DeclaredImplementation>, ApplicationError> {
5395 let repository_records = context
5396 .registry
5397 .record
5398 .repositories
5399 .iter()
5400 .map(|repository| (repository.alias.as_str(), repository))
5401 .collect::<BTreeMap<_, _>>();
5402 let mut implementations = Vec::new();
5403 for alias in context.manifest.repos.keys() {
5404 let repository = repository_records
5405 .get(alias.as_str())
5406 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5407 let effective = context
5408 .repository_configs
5409 .get(alias)
5410 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5411 implementations.extend(effective.implementations.iter().map(|implementation| {
5412 let content_hash = artifact_content_hash(
5413 fingerprints,
5414 &repository.id,
5415 "code-system-graph.implementations.declared",
5416 &implementation.path,
5417 );
5418 declared_implementation(repository.id.clone(), implementation, content_hash)
5419 }));
5420 }
5421 implementations.sort_by(|left, right| left.node.id.cmp(&right.node.id));
5422 Ok(implementations)
5423}
5424
5425fn artifact_content_hash(
5426 fingerprints: &[ArtifactFingerprint],
5427 repo_id: &code_system_graph_model::RepoId,
5428 extractor: &str,
5429 path: &str,
5430) -> Option<String> {
5431 fingerprints
5432 .iter()
5433 .find(|fingerprint| {
5434 &fingerprint.repo_id == repo_id
5435 && fingerprint.extractor == extractor
5436 && portable_path(&fingerprint.path.display) == portable_path(path)
5437 })
5438 .map(|fingerprint| fingerprint.content_hash.clone())
5439}
5440
5441fn portable_path(path: &str) -> String {
5442 path.replace('\\', "/")
5443}
5444
5445fn cargo_crate_root(checkout: &Path, source_path: &Path) -> String {
5446 source_path
5447 .parent()
5448 .into_iter()
5449 .flat_map(Path::ancestors)
5450 .take_while(|candidate| candidate.starts_with(checkout))
5451 .find(|candidate| candidate.join("Cargo.toml").is_file())
5452 .and_then(|candidate| candidate.strip_prefix(checkout).ok())
5453 .map(|relative| portable_path(&relative.to_string_lossy()))
5454 .unwrap_or_default()
5455}
5456
5457fn discover_artifact_fingerprints(
5458 context: &WorkspaceContext,
5459) -> Result<Vec<ArtifactFingerprint>, ApplicationError> {
5460 let repository_records = context
5461 .registry
5462 .record
5463 .repositories
5464 .iter()
5465 .map(|repository| (repository.alias.as_str(), repository))
5466 .collect::<BTreeMap<_, _>>();
5467 let mut fingerprints = BTreeMap::new();
5468 for alias in context.manifest.repos.keys() {
5469 let repository = repository_records
5470 .get(alias.as_str())
5471 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5472 let checkout_path = context
5473 .registry
5474 .checkout_path(alias)
5475 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5476 let effective = context
5477 .repository_configs
5478 .get(alias)
5479 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5480 for openapi in &effective.openapi {
5481 let fingerprint = fingerprint_artifact(
5482 repository,
5483 checkout_path,
5484 Path::new(openapi),
5485 "code-system-graph.http.openapi",
5486 &context.extraction_budgets,
5487 )?;
5488 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5489 }
5490 for consumer in &effective.http_consumers {
5491 let fingerprint = fingerprint_artifact(
5492 repository,
5493 checkout_path,
5494 Path::new(&consumer.source),
5495 "code-system-graph.http.declared",
5496 &context.extraction_budgets,
5497 )?;
5498 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5499 }
5500 for test in &effective.integration_tests {
5501 let fingerprint = fingerprint_artifact(
5502 repository,
5503 checkout_path,
5504 Path::new(&test.path),
5505 "code-system-graph.tests.declared",
5506 &context.extraction_budgets,
5507 )?;
5508 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5509 }
5510 for implementation in &effective.implementations {
5511 let fingerprint = fingerprint_artifact(
5512 repository,
5513 checkout_path,
5514 Path::new(&implementation.path),
5515 "code-system-graph.implementations.declared",
5516 &context.extraction_budgets,
5517 )?;
5518 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5519 }
5520 for (relative_path, extractor) in
5521 discover_focused_artifacts(checkout_path, &effective.ignore_policy)?
5522 {
5523 let fingerprint = fingerprint_artifact(
5524 repository,
5525 checkout_path,
5526 &relative_path,
5527 extractor,
5528 &context.extraction_budgets,
5529 )?;
5530 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5531 }
5532 }
5533 Ok(fingerprints.into_values().collect())
5534}
5535
5536fn discover_focused_artifacts(
5537 checkout_path: &Path,
5538 ignore_policy: &IgnorePolicy,
5539) -> Result<Vec<(PathBuf, &'static str)>, ApplicationError> {
5540 let mut pending = vec![checkout_path.to_path_buf()];
5541 let canonical_checkout =
5542 fs::canonicalize(checkout_path).map_err(|source| ApplicationError::ReadFile {
5543 path: checkout_path.to_path_buf(),
5544 source,
5545 })?;
5546 let mut visited = BTreeSet::new();
5547 let mut discovered = Vec::new();
5548 while let Some(directory) = pending.pop() {
5549 let canonical_directory =
5550 fs::canonicalize(&directory).map_err(|source| ApplicationError::ReadFile {
5551 path: directory.clone(),
5552 source,
5553 })?;
5554 if !canonical_directory.starts_with(&canonical_checkout) {
5555 return Err(ApplicationError::ArtifactOutsideCheckout {
5556 path: canonical_directory,
5557 checkout: canonical_checkout,
5558 });
5559 }
5560 if !visited.insert(canonical_directory) {
5561 continue;
5562 }
5563 worker::report_progress(code_system_graph_core::JobPhase::Discovery, 1);
5564 let entries = fs::read_dir(&directory).map_err(|source| ApplicationError::ReadFile {
5565 path: directory.clone(),
5566 source,
5567 })?;
5568 for entry in entries {
5569 let entry = entry.map_err(|source| ApplicationError::ReadFile {
5570 path: directory.clone(),
5571 source,
5572 })?;
5573 let file_type = entry
5574 .file_type()
5575 .map_err(|source| ApplicationError::ReadFile {
5576 path: entry.path(),
5577 source,
5578 })?;
5579 if file_type.is_symlink() {
5580 continue;
5581 }
5582 let path = entry.path();
5583 let relative = path.strip_prefix(checkout_path).map_err(|_| {
5584 ApplicationError::ArtifactOutsideCheckout {
5585 path: path.clone(),
5586 checkout: checkout_path.to_path_buf(),
5587 }
5588 })?;
5589 if file_type.is_dir() {
5590 if !ignore_policy.excludes(relative, true) {
5591 pending.push(path);
5592 }
5593 continue;
5594 }
5595 if !file_type.is_file() || ignore_policy.excludes(relative, false) {
5596 continue;
5597 }
5598 for extractor in focused_extractors_for_path(&path) {
5599 discovered.push((relative.to_path_buf(), extractor));
5600 }
5601 }
5602 }
5603 discovered.sort_by(|left, right| left.0.cmp(&right.0).then(left.1.cmp(right.1)));
5604 Ok(discovered)
5605}
5606
5607fn focused_extractors_for_path(path: &Path) -> Vec<&'static str> {
5608 let Some(name) = path.file_name().and_then(std::ffi::OsStr::to_str) else {
5609 return Vec::new();
5610 };
5611 let mut extractors = Vec::new();
5612 let is_generated_client_metadata = name == "openapitools.json"
5613 || (matches!(name, "FILES" | "VERSION")
5614 && path
5615 .parent()
5616 .and_then(Path::file_name)
5617 .is_some_and(|parent| parent == ".openapi-generator"));
5618 if is_generated_client_metadata {
5619 extractors.push("code-system-graph.http.generated-client");
5620 }
5621 let extension = path
5622 .extension()
5623 .and_then(std::ffi::OsStr::to_str)
5624 .map(str::to_ascii_lowercase);
5625 let source_extractor = match extension.as_deref() {
5626 Some("js" | "jsx") => Some("code-system-graph.source.javascript"),
5627 Some("ts" | "tsx") => Some("code-system-graph.source.typescript"),
5628 Some("rs") => Some("code-system-graph.source.rust"),
5629 Some("py") => Some("code-system-graph.source.python"),
5630 Some("go") => Some("code-system-graph.source.go"),
5631 Some("java") => Some("code-system-graph.source.java"),
5632 _ => None,
5633 };
5634 if let Some(extractor) = source_extractor {
5635 extractors.extend([
5636 extractor,
5637 "code-system-graph.events.source",
5638 "code-system-graph.graphql.source",
5639 "code-system-graph.protobuf.generated",
5640 "code-system-graph.data.source",
5641 ]);
5642 }
5643 if matches!(extension.as_deref(), Some("graphql" | "gql")) {
5644 extractors.push("code-system-graph.graphql.document");
5645 }
5646 if extension.as_deref() == Some("proto") {
5647 extractors.push("code-system-graph.protobuf");
5648 }
5649 let lower_name = name.to_ascii_lowercase();
5650 if matches!(
5651 lower_name.as_str(),
5652 "asyncapi.yaml" | "asyncapi.yml" | "asyncapi.json"
5653 ) {
5654 extractors.push("code-system-graph.events.asyncapi");
5655 }
5656 if matches!(
5657 lower_name.as_str(),
5658 "persisted-queries.json"
5659 | "persisted_queries.json"
5660 | "apollo-manifest.json"
5661 | "operation-manifest.json"
5662 ) {
5663 extractors.push("code-system-graph.graphql.persisted");
5664 }
5665 extractors.extend(document_extractors_for_path(
5666 path,
5667 name,
5668 extension.as_deref(),
5669 ));
5670 let is_package_artifact = matches!(
5671 name,
5672 "package.json"
5673 | "package-lock.json"
5674 | "pnpm-lock.yaml"
5675 | "yarn.lock"
5676 | "pyproject.toml"
5677 | "poetry.lock"
5678 | "Cargo.toml"
5679 | "Cargo.lock"
5680 | "go.mod"
5681 | "go.work"
5682 | "pom.xml"
5683 | "build.gradle"
5684 | "build.gradle.kts"
5685 | "packages.config"
5686 ) || extension.as_deref() == Some("csproj")
5687 || (name.starts_with("requirements")
5688 && Path::new(name)
5689 .extension()
5690 .is_some_and(|extension| extension.eq_ignore_ascii_case("txt")));
5691 if is_package_artifact {
5692 extractors.push("code-system-graph.packages");
5693 }
5694 extractors.sort_unstable();
5695 extractors.dedup();
5696 extractors
5697}
5698
5699fn document_extractors_for_path(
5700 path: &Path,
5701 name: &str,
5702 extension: Option<&str>,
5703) -> Vec<&'static str> {
5704 let mut extractors = Vec::new();
5705 let lower_name = name.to_ascii_lowercase();
5706 let lower_components = path
5707 .components()
5708 .filter_map(|component| component.as_os_str().to_str())
5709 .map(str::to_ascii_lowercase)
5710 .collect::<Vec<_>>();
5711 let in_database_migrations = lower_components
5712 .iter()
5713 .any(|component| matches!(component.as_str(), "alembic" | "migrations"));
5714 if matches!(extension, Some("sql" | "prisma"))
5715 || lower_name == "sqlx.toml"
5716 || lower_name == "schema.rs"
5717 || (extension == Some("py")
5718 && (in_database_migrations || matches!(lower_name.as_str(), "model.py" | "models.py")))
5719 {
5720 extractors.push("code-system-graph.data.artifact");
5721 }
5722
5723 let is_compose = matches!(
5724 lower_name.as_str(),
5725 "compose.yaml"
5726 | "compose.yml"
5727 | "compose.json"
5728 | "docker-compose.yaml"
5729 | "docker-compose.yml"
5730 | "docker-compose.json"
5731 );
5732 if is_compose {
5733 extractors.push("code-system-graph.infrastructure.compose");
5734 }
5735 if extension == Some("tf") {
5736 extractors.push("code-system-graph.infrastructure.terraform");
5737 }
5738 let in_helm_templates = lower_components
5739 .iter()
5740 .any(|component| component == "templates");
5741 let is_helm = in_helm_templates
5742 || matches!(
5743 lower_name.as_str(),
5744 "chart.yaml" | "chart.yml" | "values.yaml" | "values.yml"
5745 );
5746 if is_helm {
5747 extractors.push("code-system-graph.infrastructure.helm");
5748 } else if matches!(extension, Some("yaml" | "yml" | "json"))
5749 && !is_compose
5750 && (lower_components
5751 .iter()
5752 .any(|component| matches!(component.as_str(), "k8s" | "kubernetes" | "manifests"))
5753 || [
5754 "deployment",
5755 "service",
5756 "ingress",
5757 "statefulset",
5758 "daemonset",
5759 "cronjob",
5760 "job",
5761 "pod",
5762 ]
5763 .iter()
5764 .any(|kind| lower_name.starts_with(kind)))
5765 {
5766 extractors.push("code-system-graph.infrastructure.kubernetes");
5767 }
5768
5769 if extension == Some("md") || lower_name == "markdown" {
5770 extractors.push("code-system-graph.documents.markdown");
5771 }
5772 if lower_name == "codeowners" {
5773 extractors.push("code-system-graph.documents.codeowners");
5774 }
5775 if matches!(
5776 lower_name.as_str(),
5777 "catalog.yaml"
5778 | "catalog.yml"
5779 | "catalog.json"
5780 | "catalog-info.yaml"
5781 | "catalog-info.yml"
5782 | "service-catalog.yaml"
5783 | "service-catalog.yml"
5784 | "service-catalog.json"
5785 ) {
5786 extractors.push("code-system-graph.documents.catalog");
5787 }
5788
5789 let is_named_config = [
5790 "config.",
5791 "settings.",
5792 "application.",
5793 "values.",
5794 "secrets.",
5795 ]
5796 .iter()
5797 .any(|prefix| lower_name.starts_with(prefix));
5798 if lower_name == ".env"
5799 || lower_name.starts_with(".env.")
5800 || (is_named_config && matches!(extension, Some("yaml" | "yml" | "json" | "toml")))
5801 {
5802 extractors.push("code-system-graph.config.safe");
5803 }
5804 extractors
5805}
5806
5807fn fingerprint_artifact(
5808 repository: &RepositoryRecord,
5809 checkout_path: &Path,
5810 relative_path: &Path,
5811 extractor: &str,
5812 budgets: &ExtractionBudgets,
5813) -> Result<ArtifactFingerprint, ApplicationError> {
5814 let configured_path = checkout_path.join(relative_path);
5815 let canonical_path =
5816 std::fs::canonicalize(&configured_path).map_err(|source| ApplicationError::ReadFile {
5817 path: configured_path.clone(),
5818 source,
5819 })?;
5820 if !canonical_path.starts_with(checkout_path) {
5821 return Err(ApplicationError::ArtifactOutsideCheckout {
5822 path: configured_path,
5823 checkout: checkout_path.to_path_buf(),
5824 });
5825 }
5826 let metadata =
5827 std::fs::metadata(&canonical_path).map_err(|source| ApplicationError::ReadFile {
5828 path: canonical_path.clone(),
5829 source,
5830 })?;
5831 let relative = canonical_path.strip_prefix(checkout_path).map_err(|_| {
5832 ApplicationError::ArtifactOutsideCheckout {
5833 path: canonical_path.clone(),
5834 checkout: checkout_path.to_path_buf(),
5835 }
5836 })?;
5837 let path = encode_native_path(relative);
5838 code_system_graph_model::validate_safe_path_display(&path.display)
5839 .map_err(|_| ApplicationError::UnsafeArtifactPath)?;
5840 let mut tracker = ExtractionTracker::new(&path.display, extractor, budgets);
5841 let content = read_bounded_bytes(&canonical_path, &mut tracker)?;
5842 let fingerprint = ArtifactFingerprint {
5843 repo_id: repository.id.clone(),
5844 checkout_id: repository.checkout_id.clone(),
5845 path,
5846 extractor: extractor.to_owned(),
5847 content_hash: stable_id_bytes("artifact-content", &content),
5848 size_bytes: metadata.len(),
5849 };
5850 worker::report_progress(code_system_graph_core::JobPhase::Fingerprinting, 1);
5851 Ok(fingerprint)
5852}
5853
5854fn artifact_key(
5855 fingerprint: &ArtifactFingerprint,
5856) -> (CheckoutId, code_system_graph_model::NativePath, String) {
5857 (
5858 fingerprint.checkout_id.clone(),
5859 fingerprint.path.clone(),
5860 fingerprint.extractor.clone(),
5861 )
5862}
5863
5864fn extractor_runs(
5865 snapshot_id: &str,
5866 fingerprints: &[ArtifactFingerprint],
5867 plan: &IncrementalPlan,
5868) -> Vec<ExtractorRun> {
5869 let actions = plan
5870 .changes
5871 .iter()
5872 .map(|change| {
5873 (
5874 ArtifactKey {
5875 repo_id: change.repo_id.clone(),
5876 checkout_id: change.checkout_id.clone(),
5877 path: change.path.clone(),
5878 extractor: change.extractor.clone(),
5879 },
5880 change.kind,
5881 )
5882 })
5883 .collect::<BTreeMap<_, _>>();
5884 let mut groups = BTreeMap::<(String, String, String), Vec<&ArtifactFingerprint>>::new();
5885 for fingerprint in fingerprints {
5886 groups
5887 .entry((
5888 fingerprint.repo_id.as_str().to_owned(),
5889 fingerprint.checkout_id.as_str().to_owned(),
5890 fingerprint.extractor.clone(),
5891 ))
5892 .or_default()
5893 .push(fingerprint);
5894 }
5895 groups
5896 .into_iter()
5897 .map(|((repo_id, checkout_id, extractor), inputs)| {
5898 let changed = inputs
5899 .iter()
5900 .filter(|fingerprint| {
5901 actions.get(&ArtifactKey::from(**fingerprint))
5902 != Some(&code_system_graph_model::ArtifactChangeKind::Unchanged)
5903 })
5904 .count();
5905 let discovered_files = u64::try_from(inputs.len()).unwrap_or(u64::MAX);
5906 let parsed_files = u64::try_from(changed).unwrap_or(u64::MAX);
5907 let skipped_files = discovered_files.saturating_sub(parsed_files);
5908 let key = format!("{snapshot_id}:{repo_id}:{checkout_id}:{extractor}");
5909 ExtractorRun {
5910 id: stable_id("extractor-run", &key),
5911 snapshot_id: snapshot_id.to_owned(),
5912 repo_id: RepoId::new(repo_id),
5913 checkout_id: CheckoutId::new(checkout_id),
5914 extractor_version: if focused_extractor(&extractor) {
5915 EXTRACTION_CONTRACT_VERSION.to_owned()
5916 } else {
5917 env!("CARGO_PKG_VERSION").to_owned()
5918 },
5919 extractor,
5920 status: if changed == 0 {
5921 ExtractorRunStatus::SkippedUnchanged
5922 } else {
5923 ExtractorRunStatus::Success
5924 },
5925 discovered_files,
5926 parsed_files,
5927 skipped_files,
5928 elapsed_ms: 0,
5929 }
5930 })
5931 .collect()
5932}
5933
5934fn read_file(path: &Path) -> Result<String, ApplicationError> {
5935 std::fs::read_to_string(path).map_err(|source| ApplicationError::ReadFile {
5936 path: path.to_path_buf(),
5937 source,
5938 })
5939}
5940
5941fn read_source_file(
5942 path: &Path,
5943 tracker: &mut ExtractionTracker,
5944) -> Result<(String, bool), ApplicationError> {
5945 let bytes = read_bounded_bytes(path, tracker)?;
5946 match String::from_utf8(bytes) {
5947 Ok(source) => Ok((source, false)),
5948 Err(error) => Ok((String::from_utf8_lossy(error.as_bytes()).into_owned(), true)),
5949 }
5950}
5951
5952fn read_bounded_bytes(
5953 path: &Path,
5954 tracker: &mut ExtractionTracker,
5955) -> Result<Vec<u8>, ApplicationError> {
5956 let metadata = std::fs::metadata(path).map_err(|source| ApplicationError::ReadFile {
5957 path: path.to_path_buf(),
5958 source,
5959 })?;
5960 tracker.check_input_bytes(metadata.len())?;
5961 let file = std::fs::File::open(path).map_err(|source| ApplicationError::ReadFile {
5962 path: path.to_path_buf(),
5963 source,
5964 })?;
5965 let maximum = tracker.budgets().max_input_bytes_per_artifact;
5966 let mut bytes = Vec::new();
5967 file.take(maximum.saturating_add(1))
5968 .read_to_end(&mut bytes)
5969 .map_err(|source| ApplicationError::ReadFile {
5970 path: path.to_path_buf(),
5971 source,
5972 })?;
5973 tracker.check_input_bytes(u64::try_from(bytes.len()).unwrap_or(u64::MAX))?;
5974 Ok(bytes)
5975}
5976
5977#[cfg(test)]
5978mod budget_regression_tests {
5979 use super::*;
5980
5981 #[test]
5982 fn direct_scan_should_preserve_focused_source_value_limit() {
5983 let temporary = tempfile::tempdir().expect("temporary directory");
5984 let repository = temporary.path().join("api");
5985 fs::create_dir_all(repository.join("src")).expect("source directory");
5986 fs::write(
5987 repository.join("src/routes.rs"),
5988 "fn focused_source_boundary() {}",
5989 )
5990 .expect("source fixture");
5991 let config = temporary.path().join("code-system-graph.yaml");
5992 let database = temporary.path().join("code-system-graph.db");
5993 fs::write(
5994 &config,
5995 "version: 1\nname: source-budget-unit\nextractionBudgets:\n maxIdentifierBytesPerValue: 3\nrepos:\n api:\n path: api\n",
5996 )
5997 .expect("manifest fixture");
5998 let expected_artifact = encode_native_path(&Path::new("src").join("routes.rs")).display;
5999
6000 let result = scan_workspace_direct(&config, &database, &ScanOverrides::default());
6001 assert!(
6002 matches!(
6003 &result,
6004 Err(ApplicationError::ExtractionLimit(error))
6005 if error.resource == code_system_graph_core::ExtractionResource::IdentifierBytesPerValue
6006 && error.artifact == expected_artifact
6007 && error.extractor == "code-system-graph.source.rust"
6008 ),
6009 "unexpected direct source budget result: {result:?}"
6010 );
6011 }
6012
6013 #[test]
6014 fn direct_scan_should_charge_observations_independently_of_syntax_candidates() {
6015 let temporary = tempfile::tempdir().expect("temporary directory");
6016 let repository = temporary.path().join("tests");
6017 fs::create_dir_all(repository.join("tests")).expect("source directory");
6018 fs::write(
6019 repository.join("tests/test_api.py"),
6020 "import requests\ndef test_create_order():\n requests.post(\"https://api.test/v1/orders\")\n",
6021 )
6022 .expect("source fixture");
6023 let config = temporary.path().join("code-system-graph.yaml");
6024 let database = temporary.path().join("code-system-graph.db");
6025 let manifest = |maximum| {
6026 format!(
6027 "version: 1\nname: source-observation-unit\nextractionBudgets:\n maxObservationsPerArtifact: {maximum}\nrepos:\n tests:\n path: tests\n"
6028 )
6029 };
6030 fs::write(&config, manifest(1)).expect("limited manifest");
6031
6032 let rejected = scan_workspace_direct(&config, &database, &ScanOverrides::default());
6033 assert!(matches!(
6034 rejected,
6035 Err(ApplicationError::ExtractionLimit(error))
6036 if error.resource == code_system_graph_core::ExtractionResource::Observations
6037 && error.observed == 2
6038 && error.maximum == 1
6039 ));
6040
6041 fs::write(&config, manifest(2)).expect("exact manifest");
6042 let accepted = scan_workspace_direct(&config, &database, &ScanOverrides::default());
6043 assert!(
6044 accepted.is_ok(),
6045 "exact observation budget failed: {accepted:?}"
6046 );
6047 }
6048}
6049
6050#[cfg(test)]
6051mod codex_review_regression_tests {
6052 use code_system_graph_model::validate_safe_path_display;
6053
6054 use super::ApplicationError;
6055
6056 #[test]
6057 fn unsafe_artifact_path_error_must_not_echo_rejected_display() {
6058 const BIDI_PAYLOAD: &str = "invoice\u{202e}pay.pdf";
6059 assert!(validate_safe_path_display(BIDI_PAYLOAD).is_err());
6060 let rendered = ApplicationError::UnsafeArtifactPath.to_string();
6061 assert!(!rendered.contains(BIDI_PAYLOAD));
6062 assert!(!rendered.contains('\u{202e}'));
6063 assert_eq!(
6064 rendered,
6065 "artifact path contains unsafe control or bidirectional characters"
6066 );
6067 }
6068}