1pub mod http_server;
4pub mod mcp;
5mod sync;
6mod work_state;
7mod worker;
8
9use std::collections::{BTreeMap, BTreeSet};
10use std::ffi::OsStr;
11use std::fs::{self, OpenOptions};
12use std::io::{Read, Write};
13use std::path::{Path, PathBuf};
14use std::sync::Arc;
15use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
16
17use atomic_write_file::AtomicWriteFile;
18use code_system_graph_core::{
19 AffectedTestsRequest, AnalyzerVersions, ArtifactKey, BatchAction, BatchPlanError, BitbucketProvider, ChangeAnalysisError, ChangeAnalysisOptions, ChangeError, ChangeImpactReport, ChangeProvider, ChangeRequest, ChangeScope, ChangeSet, CodeGraphConfig, CodeGraphProvider, CommunityError, ConfigDoctorInput, ConfigError, ConfigExtractionError, ContractReport, ContractRequest, CorroborationReport, DataDocument, DataExtractionError, DeclaredImplementation, DeclaredTestCase, DoctorReport, DoctorRequest, DocumentationDocument, DocumentationExtractionError, EXTRACTION_CONTRACT_VERSION, EffectiveRepositoryConfig, EventDocument, EventExtractionError, EventGraphFacts, ExecutionPolicy, ExitCode, ExportReport, ExportRequest, ExtractionBudgets, ExtractionGraphFacts, ExtractionLimitExceeded, ExtractionTracker, ExtractorBatch, ExtractorBatchPlan, FederatedGraph, FreshnessDoctorInput, GeneratedClientError, GeneratedClientMetadata, GitCliChangeProvider, GitHubProvider, GraphqlDocument, GraphqlExtractionError, GraphqlGraphFacts, HttpBoundary, HttpExtractionError, ImpactContext, ImpactError, ImpactReport, ImpactRequest, ImpactTarget, IncrementalPlan, InfrastructureDocument, InfrastructureExtractionError, IntegrityDoctorInput, InterfaceError, LinkError, LocalCodeIntelligenceProvider, LocalContextRequest, LocalContextResult, LocalEnrichmentInput, LocalEnrichmentStatus, LocalImpactItem, LocalImpactRequest, ManifestEdit, ManifestEditError, ManifestError, ManualLinkConfig, ManualLinkError, PackageGraphFacts, PackageManifest, PackageManifestError, PrAuthToken, ProtobufDocument, ProtobufExtractionError, ProtobufGraphFacts, ProviderBudget, ProviderCapability, ProviderDoctorInput, ProviderDoctorStatus, ProviderError, ProviderRequest, ProviderStatus, PullRequestCoordinates, PullRequestError, PullRequestInspectRequest, PullRequestInspection, PullRequestListPage, PullRequestListRequest, PullRequestListState, PullRequestProvider, PullRequestProviderConfig, PullRequestProviderKind, QueryError, RecommendedCommand, RegisteredWorkspace, RegistryError, ReqwestPrHttpTransport, SafeConfigDocument, SchemaDoctorInput, SearchFilters, SearchReport, SearchRequest, SourceEpistemicStatus, SourceGraphFacts, SourceLanguage, SourceObservation, SourceRole, SourceSyntaxError, SourceSyntaxLanguage, SourceWarning, SymbolAnchor, SymbolCorroboration, TraceError, TraversalReport, TraversalRequest, WorkspaceManifest, affected_link_keys, analyze_changes, analyze_communities_with_progress, analyze_impact, apply_openapi_override, classify_interface_error, commit_manifest_edit, compare_community_snapshots, corroborate_repository, declared_implementation, declared_test_case, doctor, documents_to_graph, encode_native_path, event_documents_to_graph, export_graph, extract_asyncapi, extract_codeowners, extract_data_artifact, extract_docker_compose, extract_generated_client_metadata, extract_graphql_document_with_tracker, extract_graphql_persisted_operations_with_tracker, extract_helm, extract_kubernetes, extract_markdown, extract_openapi_with_tracker, extract_package_manifest_with_tracker, extract_protobuf_with_tracker, extract_safe_config, extract_service_catalog, extract_terraform, graphql_documents_to_graph, inspect_contracts, inspect_source_syntax, link_declared_implementations_with_ambiguities, link_declared_tests_with_ambiguities, link_http_boundaries_with_ambiguities, link_registered_package_owners, load_extractor_batch_with_budgets, merge_affected_link_neighborhoods, package_manifest_to_graph, parse_event_source, parse_go_source_with_tracker, parse_graphql_source_with_tracker, parse_java_source_with_tracker, parse_javascript_source_at_path_with_tracker, parse_literal_sql_source_at_root, parse_manifest, parse_protobuf_generated_source, parse_python_source_with_tracker, parse_rust_source_with_tracker, parse_typescript_source_at_path_with_tracker, plan_extractor_batches, plan_incremental_scan, precheck_focused_source_values, preview_add_manual_link, preview_add_repository, preview_remove_repository, protobuf_documents_to_graph, register_workspace, resolve_manual_links, resolve_repository_config, search, source_observations_to_graph, store_extractor_batch, traverse
20};
21pub use code_system_graph_core::{
22 ConfigSource, DEFAULT_EXCLUDES, IgnorePolicy, PROTECTED_EXCLUDES
23};
24use code_system_graph_model::{
25 ArtifactFingerprint, CheckoutId, Community, CommunityAlgorithm, CommunityConfig, CommunityDelta, CommunityId, CommunityScope, Edge, EdgeId, EdgeKind, EpistemicStatus, Evidence, EvidenceId, ExtractorRun, ExtractorRunStatus, FreshnessSummary, LinkDecision, LinkStatus, Node, NodeId, NodeKind, OverallFreshness, Provenance, RepoFreshness, RepoFreshnessState, RepoId, RepositoryRecord, StoredExtractorBatch, ToolEnvelope, ToolStatus, TraceReport, WorkspaceRecord, stable_id, stable_id_bytes
26};
27use code_system_graph_store_sqlite::{
28 ManualLinkDisposition, ManualLinkRecord, ProviderCapabilityRecord, QueryCacheRecord, SnapshotBatch, SqliteStore, StoreError, StoreLock, latest_schema_version
29};
30use schemars::JsonSchema;
31use serde::{Deserialize, Serialize};
32pub use sync::{
33 CodeGraphRepositorySync, CodeGraphSyncState, CodeGraphSyncSummary, SyncSummary, SyncTarget, load_persisted_watch_targets, sync_workspace_with_overrides, sync_workspace_with_wall_time_cap, sync_workspace_with_worker_executable, workspace_sync_targets
34};
35use thiserror::Error;
36use tokio_util::sync::CancellationToken;
37#[doc(hidden)]
38pub use worker::{
39 SupervisedProcessGroup, configure_supervised_process_group, run_worker_from_stdio, terminate_process_tree, terminate_supervised_process
40};
41
42const MAX_TRACE_DEPTH: usize = 32;
43const MAX_SCAN_DEGRADATIONS: usize = 25;
44const GENERATED_STATE_IGNORE_RULE: &[u8] = b".code-system-graph/";
45pub(crate) const CODEGRAPH_DISABLED_CODE: &str = "codegraph_disabled";
46pub(crate) const CODEGRAPH_DISABLED_MESSAGE: &str =
47 "CodeGraph is disabled by the trusted process policy.";
48
49#[derive(Debug, Error)]
51pub enum ApplicationError {
52 #[error("failed to read `{path}`: {source}")]
54 ReadFile {
55 path: PathBuf,
57 source: std::io::Error,
59 },
60 #[error("failed to create `{path}`: {source}")]
62 WriteFile {
63 path: PathBuf,
65 source: std::io::Error,
67 },
68 #[error(transparent)]
70 Manifest(#[from] ManifestError),
71 #[error(
73 "advanced global resource policy in `{config}` is inside analyzed repository `{repository}`; move the workspace manifest outside every analyzed checkout"
74 )]
75 UntrustedGlobalPolicySource {
76 config: PathBuf,
78 repository: String,
80 },
81 #[error(transparent)]
83 ExtractionLimit(#[from] ExtractionLimitExceeded),
84 #[error(transparent)]
86 ExecutionLimit(#[from] code_system_graph_core::ExecutionLimitExceeded),
87 #[error(transparent)]
89 HttpExtraction(#[from] HttpExtractionError),
90 #[error(transparent)]
92 PackageManifest(#[from] PackageManifestError),
93 #[error(transparent)]
95 GeneratedClient(#[from] GeneratedClientError),
96 #[error(transparent)]
98 Graphql(#[from] GraphqlExtractionError),
99 #[error(transparent)]
101 Event(#[from] EventExtractionError),
102 #[error(transparent)]
104 Protobuf(#[from] ProtobufExtractionError),
105 #[error(transparent)]
107 Data(#[from] DataExtractionError),
108 #[error(transparent)]
110 Infrastructure(#[from] InfrastructureExtractionError),
111 #[error(transparent)]
113 Documentation(#[from] DocumentationExtractionError),
114 #[error(transparent)]
116 ConfigExtraction(#[from] ConfigExtractionError),
117 #[error(transparent)]
119 SourceSyntax(#[from] SourceSyntaxError),
120 #[error("invalid focused source observation: {0}")]
122 InvalidSourceObservation(String),
123 #[error(transparent)]
125 BatchPlan(#[from] BatchPlanError),
126 #[error(transparent)]
128 Link(#[from] LinkError),
129 #[error(transparent)]
131 ManualLink(#[from] ManualLinkError),
132 #[error(transparent)]
134 Registry(#[from] RegistryError),
135 #[error(transparent)]
137 Config(#[from] ConfigError),
138 #[error(transparent)]
140 ManifestEdit(#[from] ManifestEditError),
141 #[error("validated registry omitted repository alias `{0}`")]
143 RegistryAliasMissing(String),
144 #[error("CLI override references unknown repository alias `{0}`")]
146 UnknownOverrideRepository(String),
147 #[error("workspace name `{requested}` does not match manifest name `{manifest}`")]
149 WorkspaceNameMismatch {
150 requested: String,
152 manifest: String,
154 },
155 #[error("workspace `{0}` is already registered")]
157 WorkspaceAlreadyExists(String),
158 #[error("watcher for workspace `{0}` is already active")]
160 WatcherAlreadyActive(String),
161 #[error("workspace `{0}` is not registered")]
163 WorkspaceNotFound(String),
164 #[error(
166 "extraction budgets changed since the current snapshot; run a full scan without `--repo`"
167 )]
168 PartialScanBudgetChanged,
169 #[error("artifact `{path}` resolves outside checkout `{checkout}`")]
171 ArtifactOutsideCheckout {
172 path: PathBuf,
174 checkout: PathBuf,
176 },
177 #[error("artifact path contains unsafe control or bidirectional characters")]
179 UnsafeArtifactPath,
180 #[error(transparent)]
182 Store(#[from] StoreError),
183 #[error(transparent)]
185 Trace(#[from] TraceError),
186 #[error(transparent)]
188 Community(#[from] CommunityError),
189 #[error(transparent)]
191 Query(#[from] QueryError),
192 #[error(transparent)]
194 Impact(#[from] ImpactError),
195 #[error(transparent)]
197 Change(#[from] ChangeError),
198 #[error(transparent)]
200 ChangeAnalysis(#[from] ChangeAnalysisError),
201 #[error(transparent)]
203 Interface(#[from] InterfaceError),
204 #[error(transparent)]
206 PullRequest(#[from] PullRequestError),
207 #[error("workspace initialization failed: {0}")]
209 Initialization(String),
210 #[error("transient supervised execution failure: {0}")]
212 TransientExecution(String),
213 #[error("supervised execution failed: {message}")]
215 SupervisedApplication {
216 exit_code: ExitCode,
218 message: String,
220 },
221 #[error("trace max_depth must be between 1 and {maximum}; received {found}")]
223 InvalidTraceDepth {
224 found: usize,
226 maximum: usize,
228 },
229}
230
231#[must_use]
233pub const fn application_exit_code(error: &ApplicationError) -> ExitCode {
234 match error {
235 ApplicationError::SupervisedApplication { exit_code, .. } => *exit_code,
236 ApplicationError::Interface(source) => classify_interface_error(source),
237 ApplicationError::ExecutionLimit(code_system_graph_core::ExecutionLimitExceeded {
238 resource: code_system_graph_core::ExecutionResource::Cancellation,
239 ..
240 })
241 | ApplicationError::Change(ChangeError::Cancelled)
242 | ApplicationError::PullRequest(PullRequestError::Cancelled) => ExitCode::Cancelled,
243 ApplicationError::Change(ChangeError::Timeout { .. })
244 | ApplicationError::PullRequest(PullRequestError::Timeout) => ExitCode::Timeout,
245 ApplicationError::WorkspaceNotFound(_) => ExitCode::NotFound,
246 ApplicationError::WorkspaceAlreadyExists(_) | ApplicationError::WatcherAlreadyActive(_) => {
247 ExitCode::Conflict
248 }
249 ApplicationError::Manifest(_)
250 | ApplicationError::UntrustedGlobalPolicySource { .. }
251 | ApplicationError::ExtractionLimit(_)
252 | ApplicationError::ExecutionLimit(_)
253 | ApplicationError::HttpExtraction(_)
254 | ApplicationError::PackageManifest(_)
255 | ApplicationError::GeneratedClient(_)
256 | ApplicationError::Graphql(_)
257 | ApplicationError::Event(_)
258 | ApplicationError::Protobuf(_)
259 | ApplicationError::Data(_)
260 | ApplicationError::Infrastructure(_)
261 | ApplicationError::Documentation(_)
262 | ApplicationError::ConfigExtraction(_)
263 | ApplicationError::SourceSyntax(_)
264 | ApplicationError::InvalidSourceObservation(_)
265 | ApplicationError::BatchPlan(_)
266 | ApplicationError::Link(_)
267 | ApplicationError::ManualLink(_)
268 | ApplicationError::Registry(_)
269 | ApplicationError::Config(_)
270 | ApplicationError::ManifestEdit(_)
271 | ApplicationError::UnknownOverrideRepository(_)
272 | ApplicationError::WorkspaceNameMismatch { .. }
273 | ApplicationError::ArtifactOutsideCheckout { .. }
274 | ApplicationError::UnsafeArtifactPath
275 | ApplicationError::PartialScanBudgetChanged
276 | ApplicationError::Trace(_)
277 | ApplicationError::Community(_)
278 | ApplicationError::Query(_)
279 | ApplicationError::Impact(_)
280 | ApplicationError::Change(_)
281 | ApplicationError::ChangeAnalysis(_)
282 | ApplicationError::PullRequest(_)
283 | ApplicationError::InvalidTraceDepth { .. } => ExitCode::InvalidInput,
284 ApplicationError::ReadFile { .. }
285 | ApplicationError::WriteFile { .. }
286 | ApplicationError::RegistryAliasMissing(_)
287 | ApplicationError::Store(_)
288 | ApplicationError::Initialization(_)
289 | ApplicationError::TransientExecution(_) => ExitCode::Internal,
290 }
291}
292
293pub(crate) fn load_execution_policy(
294 config_path: &Path,
295) -> Result<ExecutionPolicy, ApplicationError> {
296 let manifest_source = read_file(config_path)?;
297 let manifest = parse_manifest(&manifest_source)?;
298 validate_global_policy_source(config_path, &manifest)?;
299 ExecutionPolicy::resolve(manifest.execution_policy.as_ref())
300 .map_err(ManifestError::from)
301 .map_err(Into::into)
302}
303
304#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
306pub struct ScanSummary {
307 pub execution: code_system_graph_core::ExecutionSummary,
309 pub workspace: String,
311 pub snapshot_id: String,
313 pub node_count: usize,
315 pub edge_count: usize,
317 pub evidence_count: usize,
319 pub community_count: usize,
321 pub community_delta_count: usize,
323 pub discovered_input_count: usize,
325 pub changed_input_count: usize,
327 pub reused_snapshot: bool,
329 pub corroborated_symbol_count: usize,
331 pub affected_test_count: usize,
333 pub degradation_count: usize,
335 pub degradations: Vec<String>,
337}
338
339#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
341pub struct ScanOverrides {
342 pub workspace: Option<String>,
344 pub repo_openapi: BTreeMap<String, String>,
346 pub codegraph: bool,
348 pub codegraph_binary: Option<PathBuf>,
350 pub repository: Option<String>,
352 pub force: bool,
354}
355
356#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
358#[serde(rename_all = "snake_case")]
359pub enum IgnoreRuleAction {
360 Exclude,
362 Include,
364}
365
366#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
368#[serde(rename_all = "snake_case")]
369pub enum IgnoreRuleSource {
370 Protected,
372 BuiltInDefault,
374 WorkspaceManifest,
376 RepositoryLocal,
378}
379
380#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
382pub struct EffectiveIgnoreRule {
383 pub action: IgnoreRuleAction,
385 pub source: IgnoreRuleSource,
387 pub pattern: String,
389}
390
391#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
393pub struct ConfiguredPatterns {
394 pub source: ConfigSource,
396 pub patterns: Vec<String>,
398}
399
400#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
402pub struct IgnorePolicyReport {
403 pub protected_excludes: Vec<String>,
405 pub default_excludes: Vec<String>,
407 pub configured_excludes: ConfiguredPatterns,
409 pub include_defaults: ConfiguredPatterns,
411 pub effective_rules: Vec<EffectiveIgnoreRule>,
413}
414
415#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
417pub struct RepositoryConfigReport {
418 pub alias: String,
420 pub path: String,
422 pub ignore_policy: IgnorePolicyReport,
424}
425
426#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
428pub struct ConfigReport {
429 pub schema_version: u8,
431 pub workspace: String,
433 pub extraction_budgets: ExtractionBudgets,
435 pub execution_policy: ExecutionPolicy,
437 pub execution_policy_fingerprint: String,
439 pub repositories: Vec<RepositoryConfigReport>,
441}
442
443#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
445pub struct WorkspaceStatus {
446 pub workspace: String,
448 pub schema_version: i64,
450 pub integrity_ok: bool,
452 pub freshness: FreshnessSummary,
454 pub repositories: Vec<RepoFreshness>,
456 pub watcher: WatcherStatus,
458}
459
460#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
462#[serde(rename_all = "snake_case")]
463pub enum WatcherState {
464 Active,
466 ExpiredIdle,
468 ExpiredSession,
470 FailedLimit,
472 Stale,
474 NeverStarted,
476}
477
478#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
480pub struct WatcherStatus {
481 pub state: WatcherState,
483 pub detail: Option<String>,
485}
486
487#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
489pub struct BackupSummary {
490 pub database: String,
492 pub backup: String,
494}
495
496#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
498pub struct RestoreSummary {
499 pub source: String,
501 pub safety_backup: Option<String>,
503 pub schema_version: i64,
505}
506
507#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
509pub struct WorkspaceRegistryItem {
510 pub id: String,
512 pub name: String,
514 pub manifest_hash: String,
516 pub config_path: Option<String>,
518 pub repository_count: usize,
520}
521
522#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
524pub struct ManifestMutationSummary {
525 pub summary: String,
527 pub manifest: String,
529 pub backup: Option<String>,
531 pub applied: bool,
533 pub rendered_manifest: Option<String>,
535}
536
537#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
539pub struct WorkspaceMutationSummary {
540 pub workspace: String,
542 pub config_path: Option<String>,
544 pub repository_count: usize,
546 pub operation: String,
548}
549
550#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
552pub struct TraceInput {
553 pub from: String,
555 pub to: String,
557 #[serde(default = "default_max_depth")]
559 #[schemars(range(min = 1, max = 32))]
560 pub max_depth: usize,
561}
562
563#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
565pub struct SearchInput {
566 pub query: String,
568 #[serde(default)]
570 pub node_kinds: Vec<NodeKind>,
571 #[serde(default)]
573 pub repo_ids: Vec<RepoId>,
574 #[serde(default)]
576 pub service_ids: Vec<NodeId>,
577 #[serde(default)]
579 pub community_ids: Vec<CommunityId>,
580 #[serde(default)]
582 pub offset: usize,
583 #[serde(default = "default_search_limit")]
585 #[schemars(range(min = 1, max = 100))]
586 pub limit: usize,
587}
588
589#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
591pub struct CommunityInput {
592 pub community_id: Option<CommunityId>,
594 pub compare_snapshot_id: Option<String>,
596 #[serde(default)]
598 pub offset: usize,
599 #[serde(default = "default_community_limit")]
601 #[schemars(range(min = 1, max = 100))]
602 pub limit: usize,
603}
604
605#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
607pub struct CommunityReport {
608 pub snapshot_id: String,
610 pub engine_version: String,
612 pub config: CommunityConfig,
614 pub communities: Vec<Community>,
616 pub total_communities: usize,
618 pub offset: usize,
620 pub limit: usize,
622 pub truncated: bool,
624 pub delta: Option<CommunityDelta>,
626}
627
628#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
630pub struct ChangesInput {
631 pub repository: String,
633 pub scope: ChangeScope,
635}
636
637#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
639#[serde(deny_unknown_fields)]
640pub struct ExploreInput {
641 pub workspace: String,
643 #[serde(default)]
645 pub repository: Option<String>,
646 pub query: String,
648 #[serde(default = "default_explore_max_files")]
650 #[schemars(range(min = 1, max = 25))]
651 pub max_files: usize,
652}
653
654const fn default_explore_max_files() -> usize {
655 12
656}
657
658#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
660pub struct PullRequestInput {
661 pub provider: PullRequestProviderKind,
663 pub owner: String,
665 pub repository: String,
667 pub number: u64,
669 pub consent_to_remote_access: bool,
671}
672
673#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
675pub struct PullRequestListInput {
676 pub provider: PullRequestProviderKind,
678 pub owner: String,
680 pub repository: String,
682 pub state: PullRequestListState,
684 pub cursor: Option<String>,
686 pub limit: usize,
688 pub consent_to_remote_access: bool,
690}
691
692#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
694pub struct InitReport {
695 pub manifest_path: PathBuf,
697 pub gitignore_path: Option<PathBuf>,
699 pub gitignore_updated: bool,
701 pub workspace: String,
703 pub manifest_version: u32,
705}
706
707#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
709pub struct DiagnosticBundle {
710 pub schema_version: u32,
712 pub binary_version: String,
714 pub operating_system: String,
716 pub architecture: String,
718 pub generated_at_unix_ms: u128,
720 pub debug_requested: bool,
722 pub doctor: DoctorReport,
724}
725
726struct WorkspaceContext {
727 manifest: WorkspaceManifest,
728 registry: RegisteredWorkspace,
729 repository_configs: BTreeMap<String, EffectiveRepositoryConfig>,
730 extraction_budgets: ExtractionBudgets,
731 execution_policy: ExecutionPolicy,
732}
733
734pub fn show_config(
741 config_path: &Path,
742 selected_repository: Option<&str>,
743) -> Result<ConfigReport, ApplicationError> {
744 let context = load_workspace_context(config_path, &ScanOverrides::default())?;
745 if let Some(selected) = selected_repository
746 && !context.manifest.repos.contains_key(selected)
747 {
748 return Err(ApplicationError::UnknownOverrideRepository(
749 selected.to_owned(),
750 ));
751 }
752 let repositories = context
753 .manifest
754 .repos
755 .keys()
756 .filter(|alias| selected_repository.is_none_or(|selected| selected == alias.as_str()))
757 .map(|alias| {
758 let path = context
759 .registry
760 .checkout_path(alias)
761 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
762 let effective = context
763 .repository_configs
764 .get(alias)
765 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
766 Ok(RepositoryConfigReport {
767 alias: alias.clone(),
768 path: path.display().to_string(),
769 ignore_policy: ignore_policy_report(&effective.ignore_policy),
770 })
771 })
772 .collect::<Result<Vec<_>, ApplicationError>>()?;
773 Ok(ConfigReport {
774 schema_version: 1,
775 workspace: context.manifest.name,
776 extraction_budgets: context.extraction_budgets,
777 execution_policy_fingerprint: context.execution_policy.fingerprint(),
778 execution_policy: context.execution_policy,
779 repositories,
780 })
781}
782
783fn ignore_policy_report(policy: &IgnorePolicy) -> IgnorePolicyReport {
784 let configured_source = rule_source(policy.configured_excludes_source());
785 let include_source = rule_source(policy.include_defaults_source());
786 let mut effective_rules = DEFAULT_EXCLUDES
787 .iter()
788 .map(|pattern| EffectiveIgnoreRule {
789 action: IgnoreRuleAction::Exclude,
790 source: IgnoreRuleSource::BuiltInDefault,
791 pattern: (*pattern).to_owned(),
792 })
793 .chain(
794 policy
795 .include_defaults()
796 .iter()
797 .map(|pattern| EffectiveIgnoreRule {
798 action: IgnoreRuleAction::Include,
799 source: include_source,
800 pattern: pattern.clone(),
801 }),
802 )
803 .chain(
804 policy
805 .configured_excludes()
806 .iter()
807 .map(|pattern| EffectiveIgnoreRule {
808 action: IgnoreRuleAction::Exclude,
809 source: configured_source,
810 pattern: pattern.clone(),
811 }),
812 )
813 .collect::<Vec<_>>();
814 effective_rules.extend(
815 PROTECTED_EXCLUDES
816 .iter()
817 .map(|pattern| EffectiveIgnoreRule {
818 action: IgnoreRuleAction::Exclude,
819 source: IgnoreRuleSource::Protected,
820 pattern: (*pattern).to_owned(),
821 }),
822 );
823 IgnorePolicyReport {
824 protected_excludes: PROTECTED_EXCLUDES
825 .iter()
826 .map(|pattern| (*pattern).to_owned())
827 .collect(),
828 default_excludes: DEFAULT_EXCLUDES
829 .iter()
830 .map(|pattern| (*pattern).to_owned())
831 .collect(),
832 configured_excludes: ConfiguredPatterns {
833 source: policy.configured_excludes_source(),
834 patterns: policy.configured_excludes().to_vec(),
835 },
836 include_defaults: ConfiguredPatterns {
837 source: policy.include_defaults_source(),
838 patterns: policy.include_defaults().to_vec(),
839 },
840 effective_rules,
841 }
842}
843
844const fn rule_source(source: ConfigSource) -> IgnoreRuleSource {
845 match source {
846 ConfigSource::WorkspaceManifest => IgnoreRuleSource::WorkspaceManifest,
847 ConfigSource::RepositoryLocal => IgnoreRuleSource::RepositoryLocal,
848 ConfigSource::CliOverride | ConfigSource::AutoDetected | ConfigSource::Default => {
849 IgnoreRuleSource::BuiltInDefault
850 }
851 }
852}
853
854struct GraphAssembly {
855 nodes: Vec<Node>,
856 edges: Vec<Edge>,
857 evidence: Vec<Evidence>,
858 link_decisions: Vec<LinkDecision>,
859 link_node_keys: BTreeMap<NodeId, String>,
860 degradations: Vec<String>,
861}
862
863struct FocusedBatchState {
864 source_batches: Vec<ExtractorBatch<SourceObservation>>,
865 previous_source_batches: Vec<ExtractorBatch<SourceObservation>>,
866 package_batches: Vec<ExtractorBatch<PackageManifest>>,
867 generated_client_batches: Vec<ExtractorBatch<GeneratedClientMetadata>>,
868 graphql_batches: Vec<ExtractorBatch<GraphqlDocument>>,
869 event_batches: Vec<ExtractorBatch<EventDocument>>,
870 protobuf_batches: Vec<ExtractorBatch<ProtobufDocument>>,
871 data_batches: Vec<ExtractorBatch<DataDocument>>,
872 infrastructure_batches: Vec<ExtractorBatch<InfrastructureDocument>>,
873 documentation_batches: Vec<ExtractorBatch<DocumentationDocument>>,
874 config_batches: Vec<ExtractorBatch<SafeConfigDocument>>,
875 stored_batches: Vec<StoredExtractorBatch>,
876 degradations: Vec<String>,
877 force_relink: bool,
878 checkpoint_writes: u64,
879 artifact_durations_ms: Vec<u64>,
880}
881
882struct RepositoryCorroboration {
883 repo_id: RepoId,
884 report: CorroborationReport,
885}
886
887#[derive(Default)]
888struct CorroborationSummary {
889 reports: Vec<RepositoryCorroboration>,
890 confirmed_symbols: usize,
891 affected_tests: usize,
892 degradations: Vec<String>,
893}
894
895fn default_max_depth() -> usize {
896 8
897}
898
899fn current_unix_millis() -> u64 {
900 SystemTime::now()
901 .duration_since(UNIX_EPOCH)
902 .ok()
903 .and_then(|duration| u64::try_from(duration.as_millis()).ok())
904 .unwrap_or(u64::MAX)
905}
906
907fn duration_millis(duration: Duration) -> u64 {
908 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
909}
910
911fn artifact_execution_summary(durations_ms: &[u64]) -> code_system_graph_core::ExecutionSummary {
912 let mut ordered = durations_ms.to_vec();
913 ordered.sort_unstable();
914 code_system_graph_core::ExecutionSummary {
915 measured_artifacts: u64::try_from(ordered.len()).unwrap_or(u64::MAX),
916 artifact_duration_p50_ms: duration_percentile(&ordered, 50),
917 artifact_duration_p95_ms: duration_percentile(&ordered, 95),
918 artifact_duration_p99_ms: duration_percentile(&ordered, 99),
919 ..code_system_graph_core::ExecutionSummary::default()
920 }
921}
922
923fn duration_percentile(ordered: &[u64], percentile: usize) -> u64 {
924 if ordered.is_empty() {
925 return 0;
926 }
927 let index = ordered
928 .len()
929 .saturating_mul(percentile)
930 .div_ceil(100)
931 .saturating_sub(1)
932 .min(ordered.len() - 1);
933 ordered[index]
934}
935
936fn default_search_limit() -> usize {
937 20
938}
939
940fn default_community_limit() -> usize {
941 20
942}
943
944fn default_community_config() -> CommunityConfig {
945 CommunityConfig {
946 algorithm: CommunityAlgorithm::Louvain,
947 scope: CommunityScope::Federated,
948 seed: 0,
949 resolution: 1.0,
950 minimum_confidence: 0.5,
951 edge_weights: Vec::new(),
952 max_iterations: 100,
953 }
954}
955
956pub fn initialize_workspace(
967 directory: &Path,
968 requested_name: Option<&str>,
969) -> Result<InitReport, ApplicationError> {
970 fs::create_dir_all(directory).map_err(|error| {
971 ApplicationError::Initialization(format!(
972 "cannot create `{}`: {error}",
973 directory.display()
974 ))
975 })?;
976 let workspace = requested_name.map_or_else(
977 || {
978 directory
979 .file_name()
980 .and_then(OsStr::to_str)
981 .filter(|name| !name.is_empty())
982 .unwrap_or("workspace")
983 .to_owned()
984 },
985 str::to_owned,
986 );
987 let encoded_name = serde_json::to_string(&workspace)
988 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
989 let source = format!("version: 1\nname: {encoded_name}\nrepos:\n root:\n path: .\n");
990 let manifest = parse_manifest(&source)?;
991 let manifest_path = directory.join("code-system-graph.yaml");
992 let mut destination = fs::OpenOptions::new()
993 .write(true)
994 .create_new(true)
995 .open(&manifest_path)
996 .map_err(|error| {
997 ApplicationError::Initialization(format!(
998 "cannot create `{}` without overwriting: {error}",
999 manifest_path.display()
1000 ))
1001 })?;
1002 destination.write_all(source.as_bytes()).map_err(|error| {
1003 ApplicationError::Initialization(format!(
1004 "cannot write `{}`: {error}",
1005 manifest_path.display()
1006 ))
1007 })?;
1008 destination.sync_all().map_err(|error| {
1009 ApplicationError::Initialization(format!(
1010 "cannot synchronize `{}`: {error}",
1011 manifest_path.display()
1012 ))
1013 })?;
1014 drop(destination);
1015 let (gitignore_path, gitignore_updated) = configure_generated_state_ignore(directory)
1016 .inspect_err(|_| {
1017 let _ = fs::remove_file(&manifest_path);
1018 })?;
1019 Ok(InitReport {
1020 manifest_path,
1021 gitignore_path,
1022 gitignore_updated,
1023 workspace: manifest.name,
1024 manifest_version: manifest.version,
1025 })
1026}
1027
1028fn configure_generated_state_ignore(
1029 directory: &Path,
1030) -> Result<(Option<PathBuf>, bool), ApplicationError> {
1031 let canonical_directory = fs::canonicalize(directory).map_err(|error| {
1032 ApplicationError::Initialization(format!(
1033 "cannot resolve workspace directory `{}`: {error}",
1034 directory.display()
1035 ))
1036 })?;
1037 if !belongs_to_git_worktree(&canonical_directory) {
1038 return Ok((None, false));
1039 }
1040 let (path, updated) = ensure_generated_state_ignored(directory)?;
1041 Ok((Some(path), updated))
1042}
1043
1044fn belongs_to_git_worktree(directory: &Path) -> bool {
1045 directory
1046 .ancestors()
1047 .any(|ancestor| valid_git_worktree_marker(&ancestor.join(".git")))
1048}
1049
1050fn valid_git_worktree_marker(marker: &Path) -> bool {
1051 let Ok(metadata) = fs::symlink_metadata(marker) else {
1052 return false;
1053 };
1054 if metadata.file_type().is_dir() {
1055 return fs::symlink_metadata(marker.join("HEAD"))
1056 .is_ok_and(|head| head.file_type().is_file());
1057 }
1058 if !metadata.file_type().is_file() || metadata.len() > 4_096 {
1059 return false;
1060 }
1061 let Ok(source) = fs::read_to_string(marker) else {
1062 return false;
1063 };
1064 source
1065 .lines()
1066 .next()
1067 .is_some_and(|line| line.trim_start().starts_with("gitdir:"))
1068}
1069
1070fn ensure_generated_state_ignored(directory: &Path) -> Result<(PathBuf, bool), ApplicationError> {
1071 let gitignore_path = directory.join(".gitignore");
1072 let mut content = match fs::read(&gitignore_path) {
1073 Ok(content) => content,
1074 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
1075 Err(error) => {
1076 return Err(ApplicationError::Initialization(format!(
1077 "cannot read `{}` before updating generated-state rules: {error}",
1078 gitignore_path.display()
1079 )));
1080 }
1081 };
1082 if generated_state_is_ignored(&content) {
1083 return Ok((gitignore_path, false));
1084 }
1085 if !content.is_empty() && !content.ends_with(b"\n") {
1086 content.push(b'\n');
1087 }
1088 content.extend_from_slice(GENERATED_STATE_IGNORE_RULE);
1089 content.push(b'\n');
1090
1091 let mut destination = AtomicWriteFile::open(&gitignore_path).map_err(|error| {
1092 ApplicationError::Initialization(format!(
1093 "cannot prepare `{}` for generated-state rules: {error}",
1094 gitignore_path.display()
1095 ))
1096 })?;
1097 destination
1098 .write_all(&content)
1099 .and_then(|()| destination.sync_all())
1100 .map_err(|error| {
1101 ApplicationError::Initialization(format!(
1102 "cannot write `{}`: {error}",
1103 gitignore_path.display()
1104 ))
1105 })?;
1106 destination.commit().map_err(|error| {
1107 ApplicationError::Initialization(format!(
1108 "cannot commit `{}`: {error}",
1109 gitignore_path.display()
1110 ))
1111 })?;
1112 Ok((gitignore_path, true))
1113}
1114
1115fn generated_state_is_ignored(content: &[u8]) -> bool {
1116 content
1117 .split(|byte| *byte == b'\n')
1118 .map(|line| line.strip_suffix(b"\r").map_or(line, |trimmed| trimmed))
1119 .fold(None, |state, line| match line {
1120 b".code-system-graph/"
1121 | b"/.code-system-graph/"
1122 | b".code-system-graph"
1123 | b"/.code-system-graph" => Some(true),
1124 b"!.code-system-graph/"
1125 | b"!/.code-system-graph/"
1126 | b"!.code-system-graph"
1127 | b"!/.code-system-graph" => Some(false),
1128 _ => state,
1129 })
1130 .unwrap_or(false)
1131}
1132
1133pub fn scan_workspace(
1140 config_path: &Path,
1141 database_path: &Path,
1142) -> Result<ScanSummary, ApplicationError> {
1143 scan_workspace_with_overrides(config_path, database_path, &ScanOverrides::default())
1144}
1145
1146pub fn scan_workspace_with_overrides(
1154 config_path: &Path,
1155 database_path: &Path,
1156 overrides: &ScanOverrides,
1157) -> Result<ScanSummary, ApplicationError> {
1158 worker::supervise_scan(config_path, database_path, overrides)
1159}
1160
1161pub fn scan_workspace_with_worker_executable(
1171 config_path: &Path,
1172 database_path: &Path,
1173 overrides: &ScanOverrides,
1174 worker_executable: &Path,
1175) -> Result<ScanSummary, ApplicationError> {
1176 worker::supervise_scan_with_executable(config_path, database_path, overrides, worker_executable)
1177}
1178
1179#[doc(hidden)]
1180pub(crate) fn scan_workspace_direct(
1181 config_path: &Path,
1182 database_path: &Path,
1183 overrides: &ScanOverrides,
1184) -> Result<ScanSummary, ApplicationError> {
1185 scan_workspace_direct_with_mode(config_path, database_path, overrides, false)
1186}
1187
1188pub(crate) fn scan_workspace_direct_for_sync(
1189 config_path: &Path,
1190 database_path: &Path,
1191 overrides: &ScanOverrides,
1192 reuse_unchanged_with_codegraph: bool,
1193) -> Result<ScanSummary, ApplicationError> {
1194 scan_workspace_direct_with_mode(
1195 config_path,
1196 database_path,
1197 overrides,
1198 reuse_unchanged_with_codegraph,
1199 )
1200}
1201
1202#[expect(
1203 clippy::too_many_lines,
1204 reason = "Atomic scan orchestration keeps lock, resume, publication, and summary sequencing visible"
1205)]
1206fn scan_workspace_direct_with_mode(
1207 config_path: &Path,
1208 database_path: &Path,
1209 overrides: &ScanOverrides,
1210 reuse_unchanged_with_codegraph: bool,
1211) -> Result<ScanSummary, ApplicationError> {
1212 let context = load_workspace_context(config_path, overrides)?;
1213 worker::report_progress(code_system_graph_core::JobPhase::Configuration, 1);
1214 if let Some(requested) = &overrides.workspace
1215 && requested != &context.manifest.name
1216 {
1217 return Err(ApplicationError::WorkspaceNameMismatch {
1218 requested: requested.clone(),
1219 manifest: context.manifest.name,
1220 });
1221 }
1222 let mut fingerprints = discover_artifact_fingerprints(&context)?;
1223 worker::report_progress(
1224 code_system_graph_core::JobPhase::Fingerprinting,
1225 u64::try_from(fingerprints.len()).unwrap_or(u64::MAX),
1226 );
1227 let _writer_lock = StoreLock::acquire(database_path, Duration::from_mins(5))?;
1228 let mut store = SqliteStore::open(database_path)?;
1229 let database_instance_id = store.database_instance_id()?;
1230 let mut work_state = work_state::WorkState::open(database_path, &database_instance_id)
1231 .map_err(ApplicationError::Initialization)?;
1232 let mut previous_fingerprints =
1233 match store.load_current_artifact_fingerprints(&context.manifest.name) {
1234 Ok(previous) => previous,
1235 Err(StoreError::CurrentSnapshotMissing(_)) => Vec::new(),
1236 Err(error) => return Err(error.into()),
1237 };
1238 if let Some(alias) = &overrides.repository {
1239 let selected = context
1240 .registry
1241 .record
1242 .repositories
1243 .iter()
1244 .find(|repository| &repository.alias == alias)
1245 .ok_or_else(|| ApplicationError::UnknownOverrideRepository(alias.clone()))?
1246 .id
1247 .clone();
1248 fingerprints.retain(|fingerprint| fingerprint.repo_id == selected);
1249 fingerprints.extend(
1250 previous_fingerprints
1251 .iter()
1252 .filter(|fingerprint| fingerprint.repo_id != selected)
1253 .cloned(),
1254 );
1255 fingerprints.sort_by_key(|fingerprint| ArtifactKey::from(fingerprint));
1256 if overrides.force {
1257 previous_fingerprints.retain(|fingerprint| fingerprint.repo_id != selected);
1258 }
1259 } else if overrides.force {
1260 previous_fingerprints.clear();
1261 }
1262 let fingerprint_hashes = fingerprints
1263 .iter()
1264 .map(|fingerprint| fingerprint.content_hash.as_str())
1265 .collect::<Vec<_>>()
1266 .join(":");
1267 let snapshot_id = stable_id(
1268 "snapshot",
1269 &format!(
1270 "{}:{fingerprint_hashes}:community-engine-v1",
1271 context.registry.record.manifest_hash
1272 ),
1273 );
1274 let mut previous_extractor_batches = match store.load_current_extractor_batches_with_limit(
1275 &context.manifest.name,
1276 context
1277 .extraction_budgets
1278 .max_serialized_output_bytes_per_artifact,
1279 ) {
1280 Ok(previous) => previous,
1281 Err(StoreError::CurrentSnapshotMissing(_)) => Vec::new(),
1282 Err(error) => return Err(error.into()),
1283 };
1284 let budget_fingerprint = context.extraction_budgets.fingerprint();
1285 let candidate_material = serde_json::to_vec(&(
1286 &context.registry.record.manifest_hash,
1287 &fingerprints,
1288 &budget_fingerprint,
1289 EXTRACTION_CONTRACT_VERSION,
1290 ))
1291 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
1292 let candidate_fingerprint = stable_id_bytes("work-candidate-v1", &candidate_material);
1293 let _resumed_candidate = work_state
1294 .begin_candidate(
1295 &context.manifest.name,
1296 &candidate_fingerprint,
1297 current_unix_millis(),
1298 )
1299 .map_err(ApplicationError::Initialization)?;
1300 let cached_batches = if overrides.force {
1301 Vec::new()
1302 } else {
1303 work_state
1304 .load_batches(
1305 &fingerprints,
1306 &budget_fingerprint,
1307 EXTRACTION_CONTRACT_VERSION,
1308 context
1309 .extraction_budgets
1310 .max_serialized_output_bytes_per_artifact,
1311 current_unix_millis(),
1312 )
1313 .map_err(ApplicationError::Initialization)?
1314 };
1315 let cached_keys = cached_batches
1316 .iter()
1317 .map(|batch| ArtifactKey::from(&batch.source))
1318 .collect::<BTreeSet<_>>();
1319 for cached in cached_batches {
1320 let key = ArtifactKey::from(&cached.source);
1321 if !previous_extractor_batches
1322 .iter()
1323 .any(|batch| ArtifactKey::from(&batch.source) == key && batch.source == cached.source)
1324 {
1325 previous_extractor_batches.push(cached);
1326 }
1327 }
1328 let checkpoint_hits = u64::try_from(cached_keys.len()).unwrap_or(u64::MAX);
1329 if overrides.repository.is_some()
1330 && previous_extractor_batches
1331 .iter()
1332 .any(|batch| batch.budget_fingerprint != budget_fingerprint)
1333 {
1334 return Err(ApplicationError::PartialScanBudgetChanged);
1335 }
1336 let previous_graph = match store.load_current_graph(&context.manifest.name) {
1337 Ok(graph) => graph,
1338 Err(StoreError::CurrentSnapshotMissing(_)) => (Vec::new(), Vec::new()),
1339 Err(error) => return Err(error.into()),
1340 };
1341 let previous_communities = match store.load_current_community_snapshot(&context.manifest.name) {
1342 Ok(snapshot) => Some(snapshot),
1343 Err(StoreError::CurrentSnapshotMissing(_) | StoreError::CommunitySnapshotMissing(_)) => {
1344 None
1345 }
1346 Err(error) => return Err(error.into()),
1347 };
1348 let previous_manifest_matches = match store.load_workspace_registry(&context.manifest.name) {
1349 Ok(previous) => previous.manifest_hash == context.registry.record.manifest_hash,
1350 Err(StoreError::RegistryIncomplete(_)) => false,
1351 Err(error) => return Err(error.into()),
1352 };
1353 let plan = plan_incremental_scan(&previous_fingerprints, &fingerprints);
1354 let staged_snapshot = if let Ok(candidate) =
1355 work_state.load_candidate_snapshot(&context.manifest.name, &candidate_fingerprint)
1356 {
1357 candidate
1358 } else {
1359 work_state
1360 .complete_candidate(&context.manifest.name)
1361 .map_err(ApplicationError::Initialization)?;
1362 work_state
1363 .begin_candidate(
1364 &context.manifest.name,
1365 &candidate_fingerprint,
1366 current_unix_millis(),
1367 )
1368 .map_err(ApplicationError::Initialization)?;
1369 None
1370 };
1371 if let Some(candidate) = staged_snapshot {
1372 publish_snapshot_candidate(
1373 &mut store,
1374 SnapshotBatch {
1375 workspace: &context.registry.record,
1376 snapshot_id: &candidate.snapshot_id,
1377 nodes: &candidate.nodes,
1378 edges: &candidate.edges,
1379 evidence: &candidate.evidence,
1380 fingerprints: &candidate.fingerprints,
1381 extractor_batches: &candidate.extractor_batches,
1382 extractor_runs: &candidate.extractor_runs,
1383 manual_links: &candidate.manual_links,
1384 community_snapshot: Some(&candidate.community_snapshot),
1385 },
1386 )?;
1387 work_state
1388 .complete_candidate(&context.manifest.name)
1389 .map_err(ApplicationError::Initialization)?;
1390 let mut execution = candidate.execution;
1391 execution.checkpoint_hits = checkpoint_hits;
1392 return Ok(ScanSummary {
1393 execution,
1394 workspace: context.manifest.name,
1395 snapshot_id: candidate.snapshot_id,
1396 node_count: candidate.nodes.len(),
1397 edge_count: candidate.edges.len(),
1398 evidence_count: candidate.evidence.len(),
1399 community_count: candidate.community_snapshot.communities.len(),
1400 community_delta_count: candidate.community_delta_count,
1401 discovered_input_count: candidate.fingerprints.len(),
1402 changed_input_count: plan.changed_count(),
1403 reused_snapshot: false,
1404 corroborated_symbol_count: candidate.corroborated_symbol_count,
1405 affected_test_count: candidate.affected_test_count,
1406 degradation_count: candidate.degradations.len(),
1407 degradations: candidate.degradations,
1408 });
1409 }
1410 if previous_manifest_matches
1411 && !plan.has_changes()
1412 && focused_batch_cache_complete(
1413 &fingerprints,
1414 &previous_extractor_batches,
1415 &context.extraction_budgets,
1416 )
1417 && previous_communities.is_some()
1418 && (!overrides.codegraph || reuse_unchanged_with_codegraph)
1419 {
1420 work_state
1421 .complete_candidate(&context.manifest.name)
1422 .map_err(ApplicationError::Initialization)?;
1423 let current = store.current_snapshot_summary(&context.manifest.name)?;
1424 let (degradation_count, degradations) = finalize_scan_degradations(
1425 stored_batch_degradations(&previous_extractor_batches, &context.extraction_budgets)?,
1426 );
1427 return Ok(ScanSummary {
1428 execution: code_system_graph_core::ExecutionSummary {
1429 checkpoint_hits,
1430 ..code_system_graph_core::ExecutionSummary::default()
1431 },
1432 workspace: context.manifest.name,
1433 snapshot_id: current.snapshot_id,
1434 node_count: current.node_count,
1435 edge_count: current.edge_count,
1436 evidence_count: current.evidence_count,
1437 community_count: previous_communities
1438 .as_ref()
1439 .map_or(0, |snapshot| snapshot.communities.len()),
1440 community_delta_count: 0,
1441 discovered_input_count: fingerprints.len(),
1442 changed_input_count: 0,
1443 reused_snapshot: true,
1444 corroborated_symbol_count: 0,
1445 affected_test_count: 0,
1446 degradation_count,
1447 degradations,
1448 });
1449 }
1450
1451 let batch_plan = plan_extractor_batches(&plan);
1452 let focused_batches = assemble_focused_batches(
1453 &context,
1454 &fingerprints,
1455 &previous_extractor_batches,
1456 &batch_plan,
1457 &cached_keys,
1458 &mut work_state,
1459 )?;
1460 work_state
1461 .set_candidate_phase(&context.manifest.name, "extracted", current_unix_millis())
1462 .map_err(ApplicationError::Initialization)?;
1463 worker::report_progress(
1464 code_system_graph_core::JobPhase::Extraction,
1465 u64::try_from(focused_batches.stored_batches.len()).unwrap_or(u64::MAX),
1466 );
1467 let mut graph = assemble_graph(&context, &fingerprints, &focused_batches)?;
1468 relink_affected_graph(
1469 &mut graph,
1470 &plan,
1471 &batch_plan,
1472 &focused_batches,
1473 &previous_graph.0,
1474 &previous_graph.1,
1475 )?;
1476 let corroboration = if overrides.codegraph {
1477 run_codegraph_corroboration(
1478 &context,
1479 &focused_batches,
1480 &plan,
1481 overrides.repository.as_deref(),
1482 overrides.codegraph_binary.clone(),
1483 )
1484 } else {
1485 CorroborationSummary::default()
1486 };
1487 apply_codegraph_corroboration(&mut graph, &corroboration.reports);
1488 let manual_links =
1489 resolve_manual_links(&context.manifest.manual_links, &graph.nodes, &graph.edges)?;
1490 graph.edges = manual_links.edges;
1491 graph.evidence.extend(manual_links.evidence);
1492 graph.evidence.sort_by(|left, right| left.id.cmp(&right.id));
1493 graph.evidence.dedup_by(|left, right| left.id == right.id);
1494 graph.link_decisions = manual_links.decisions;
1495 worker::report_progress(
1496 code_system_graph_core::JobPhase::GraphAssembly,
1497 u64::try_from(graph.nodes.len().saturating_add(graph.edges.len())).unwrap_or(u64::MAX),
1498 );
1499 let GraphAssembly {
1500 nodes,
1501 edges,
1502 evidence,
1503 link_decisions,
1504 degradations: graph_degradations,
1505 ..
1506 } = graph;
1507 let community_config = default_community_config();
1508 let community_snapshot = previous_communities
1509 .as_ref()
1510 .filter(|previous| {
1511 previous.snapshot_id == snapshot_id
1512 && previous.config == community_config
1513 && community_topology_unchanged(
1514 &previous_graph.0,
1515 &previous_graph.1,
1516 &nodes,
1517 &edges,
1518 )
1519 })
1520 .cloned()
1521 .map_or_else(
1522 || {
1523 analyze_communities_with_progress(
1524 &snapshot_id,
1525 &nodes,
1526 &edges,
1527 community_config.clone(),
1528 |completed| {
1529 worker::report_progress(
1530 code_system_graph_core::JobPhase::Communities,
1531 completed,
1532 );
1533 },
1534 )
1535 },
1536 Ok,
1537 )?;
1538 drop(previous_graph);
1539 let community_delta_count = previous_communities.as_ref().map_or(0, |previous| {
1540 compare_community_snapshots(previous, &community_snapshot)
1541 .changes
1542 .len()
1543 });
1544 worker::report_progress(
1545 code_system_graph_core::JobPhase::Communities,
1546 u64::try_from(community_snapshot.communities.len()).unwrap_or(u64::MAX),
1547 );
1548 let extractor_runs = extractor_runs(&snapshot_id, &fingerprints, &plan);
1549 let manual_link_records = persisted_manual_link_records(&snapshot_id, &link_decisions)?;
1550 let mut staged_degradations = focused_batches.degradations.clone();
1551 staged_degradations.extend(graph_degradations);
1552 staged_degradations.extend(corroboration.degradations.clone());
1553 staged_degradations.extend(stored_batch_degradations(
1554 &focused_batches.stored_batches,
1555 &context.extraction_budgets,
1556 )?);
1557 let (_, staged_degradations) = finalize_scan_degradations(staged_degradations);
1558 let mut artifact_execution = artifact_execution_summary(&focused_batches.artifact_durations_ms);
1559 artifact_execution.checkpoint_hits = checkpoint_hits;
1560 artifact_execution.checkpoints_written = focused_batches.checkpoint_writes;
1561 let candidate = work_state::StagedSnapshot {
1562 snapshot_id,
1563 nodes,
1564 edges,
1565 evidence,
1566 fingerprints,
1567 extractor_batches: focused_batches.stored_batches,
1568 extractor_runs,
1569 manual_links: manual_link_records,
1570 community_snapshot,
1571 community_delta_count,
1572 corroborated_symbol_count: corroboration.confirmed_symbols,
1573 affected_test_count: corroboration.affected_tests,
1574 execution: artifact_execution,
1575 degradations: staged_degradations,
1576 };
1577 work_state
1578 .store_candidate_snapshot(
1579 &context.manifest.name,
1580 &candidate_fingerprint,
1581 &candidate,
1582 current_unix_millis(),
1583 )
1584 .map_err(ApplicationError::Initialization)?;
1585 publish_snapshot_candidate(
1586 &mut store,
1587 SnapshotBatch {
1588 workspace: &context.registry.record,
1589 snapshot_id: &candidate.snapshot_id,
1590 nodes: &candidate.nodes,
1591 edges: &candidate.edges,
1592 evidence: &candidate.evidence,
1593 fingerprints: &candidate.fingerprints,
1594 extractor_batches: &candidate.extractor_batches,
1595 extractor_runs: &candidate.extractor_runs,
1596 manual_links: &candidate.manual_links,
1597 community_snapshot: Some(&candidate.community_snapshot),
1598 },
1599 )?;
1600 let mut degradations = candidate.degradations.clone();
1601 for item in &corroboration.reports {
1602 let Some(capability) = &item.report.capability else {
1603 continue;
1604 };
1605 let record = provider_capability_record(&context.manifest.name, &item.repo_id, capability);
1606 if let Err(error) = store.upsert_provider_capabilities(&record) {
1607 degradations.push(format!(
1608 "CodeGraph capability persistence for `{}` degraded: {error}",
1609 item.repo_id.as_str()
1610 ));
1611 }
1612 }
1613 let (degradation_count, degradations) = finalize_scan_degradations(degradations);
1614 let _ = work_state.complete_candidate(&context.manifest.name);
1615 Ok(ScanSummary {
1616 execution: candidate.execution,
1617 workspace: context.manifest.name,
1618 snapshot_id: candidate.snapshot_id,
1619 node_count: candidate.nodes.len(),
1620 edge_count: candidate.edges.len(),
1621 evidence_count: candidate.evidence.len(),
1622 community_count: candidate.community_snapshot.communities.len(),
1623 community_delta_count: candidate.community_delta_count,
1624 discovered_input_count: candidate.fingerprints.len(),
1625 changed_input_count: plan.changed_count(),
1626 reused_snapshot: false,
1627 corroborated_symbol_count: candidate.corroborated_symbol_count,
1628 affected_test_count: candidate.affected_test_count,
1629 degradation_count,
1630 degradations,
1631 })
1632}
1633
1634fn publish_snapshot_candidate(
1635 store: &mut SqliteStore,
1636 batch: SnapshotBatch<'_>,
1637) -> Result<(), StoreError> {
1638 const REPORT_INTERVAL: u64 = 1_024;
1639 let mut pending = 0_u64;
1640 worker::report_progress(code_system_graph_core::JobPhase::Publication, 1);
1641 store.publish_snapshot_with_progress(batch, |rows| {
1642 worker::check_time(code_system_graph_core::JobPhase::Publication);
1643 pending = pending.saturating_add(rows);
1644 if pending >= REPORT_INTERVAL {
1645 worker::report_progress(code_system_graph_core::JobPhase::Publication, pending);
1646 pending = 0;
1647 }
1648 })?;
1649 Ok(())
1650}
1651
1652pub fn trace_workspace(
1658 database_path: &Path,
1659 workspace: &str,
1660 input: &TraceInput,
1661) -> Result<ToolEnvelope<TraceReport>, ApplicationError> {
1662 if !(1..=MAX_TRACE_DEPTH).contains(&input.max_depth) {
1663 return Err(ApplicationError::InvalidTraceDepth {
1664 found: input.max_depth,
1665 maximum: MAX_TRACE_DEPTH,
1666 });
1667 }
1668 let store = SqliteStore::open_read_only(database_path)?;
1669 let (nodes, edges) = store.load_current_graph(workspace)?;
1670 let freshness = freshness_summary(&store.load_current_freshness(workspace)?);
1671 let graph = FederatedGraph::new(nodes, edges)?;
1672 let report = graph.trace(
1673 &NodeId::new(&input.from),
1674 &NodeId::new(&input.to),
1675 input.max_depth,
1676 )?;
1677 let status = if report.coverage_gaps.is_empty() && freshness.overall == OverallFreshness::Fresh
1678 {
1679 ToolStatus::Ok
1680 } else {
1681 ToolStatus::Degraded
1682 };
1683 Ok(ToolEnvelope {
1684 schema_version: 1,
1685 status,
1686 data: Some(report),
1687 freshness,
1688 warnings: Vec::new(),
1689 })
1690}
1691
1692pub fn search_workspace(
1699 database_path: &Path,
1700 workspace: &str,
1701 input: &SearchInput,
1702) -> Result<ToolEnvelope<SearchReport>, ApplicationError> {
1703 let store = SqliteStore::open_read_only(database_path)?;
1704 let snapshot = store.current_snapshot_summary(workspace)?;
1705 let input_json = serde_json::to_vec(input)
1706 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
1707 let input_fingerprint = stable_id_bytes("query-cache", &input_json);
1708 let now_unix_ms = current_unix_millis();
1709 if let Some(cached) = store.load_query_cache(
1710 workspace,
1711 &snapshot.snapshot_id,
1712 &input_fingerprint,
1713 now_unix_ms,
1714 )? && let Ok(envelope) =
1715 serde_json::from_slice::<ToolEnvelope<SearchReport>>(&cached.result_summary_json)
1716 {
1717 return Ok(envelope);
1718 }
1719 let (nodes, edges) = store.load_current_graph(workspace)?;
1720 let repository_freshness = store.load_current_freshness(workspace)?;
1721 let freshness = freshness_summary(&repository_freshness);
1722 let fts_hits = if input.query.trim().is_empty() {
1723 Vec::new()
1724 } else {
1725 store.search_current_nodes_ranked(workspace, &input.query, 500)?
1726 };
1727 let community_snapshot = store.load_current_community_snapshot(workspace)?;
1728 let evidence = store.load_current_evidence(workspace)?;
1729 let request = SearchRequest {
1730 query: input.query.clone(),
1731 filters: SearchFilters {
1732 node_kinds: input.node_kinds.clone(),
1733 repo_ids: input.repo_ids.clone(),
1734 workspace_nodes: Vec::new(),
1735 service_ids: input.service_ids.clone(),
1736 community_ids: input.community_ids.clone(),
1737 },
1738 fts_scores: fts_search_scores(&fts_hits),
1739 centrality_scores: community_centrality_scores(&community_snapshot.communities),
1740 service_memberships: service_memberships(&nodes, &edges),
1741 community_memberships: community_memberships(&community_snapshot.communities),
1742 evidence: node_evidence(&edges, &evidence),
1743 freshness: repository_freshness
1744 .iter()
1745 .map(|item| (item.repo_id.clone(), item.state))
1746 .collect(),
1747 offset: input.offset,
1748 limit: input.limit,
1749 };
1750 let report = search(&nodes, &request)?;
1751 let status = if report.coverage.gaps.is_empty() && freshness.overall == OverallFreshness::Fresh
1752 {
1753 ToolStatus::Ok
1754 } else {
1755 ToolStatus::Degraded
1756 };
1757 let envelope = ToolEnvelope {
1758 schema_version: 1,
1759 status,
1760 data: Some(report),
1761 freshness,
1762 warnings: Vec::new(),
1763 };
1764 let encoded = serde_json::to_vec(&envelope)
1765 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
1766 let envelope = serde_json::from_slice(&encoded)
1767 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
1768 drop(store);
1769 if let Ok(_lock) = StoreLock::acquire(database_path, Duration::from_secs(5))
1770 && let Ok(mut writable) = SqliteStore::open(database_path)
1771 {
1772 let _ = writable.put_query_cache(&QueryCacheRecord {
1773 workspace_name: workspace.to_owned(),
1774 snapshot_id: snapshot.snapshot_id,
1775 input_fingerprint,
1776 result_summary_json: encoded,
1777 stored_at_unix_ms: now_unix_ms,
1778 expires_at_unix_ms: None,
1779 });
1780 }
1781 Ok(envelope)
1782}
1783
1784pub fn traverse_workspace(
1791 database_path: &Path,
1792 workspace: &str,
1793 input: &TraversalRequest,
1794) -> Result<ToolEnvelope<TraversalReport>, ApplicationError> {
1795 if input.options.max_depth > 32
1796 || input.options.max_cross_repo_hops > 16
1797 || input.options.node_limit > 50_000
1798 || input.options.edge_limit > 250_000
1799 || input.options.timeout_ms > 5_000
1800 || input.options.k > 8
1801 {
1802 return Err(QueryError::InvalidTraversalLimits.into());
1803 }
1804 let store = SqliteStore::open_read_only(database_path)?;
1805 let (nodes, edges) = store.load_current_graph(workspace)?;
1806 let freshness = freshness_summary(&store.load_current_freshness(workspace)?);
1807 let report = traverse(&nodes, &edges, input)?;
1808 let status = if report.coverage_gaps.is_empty()
1809 && !report.truncated
1810 && freshness.overall == OverallFreshness::Fresh
1811 {
1812 ToolStatus::Ok
1813 } else {
1814 ToolStatus::Degraded
1815 };
1816 Ok(ToolEnvelope {
1817 schema_version: 1,
1818 status,
1819 data: Some(report),
1820 freshness,
1821 warnings: Vec::new(),
1822 })
1823}
1824
1825pub fn communities_workspace(
1831 database_path: &Path,
1832 workspace: &str,
1833 input: &CommunityInput,
1834) -> Result<ToolEnvelope<CommunityReport>, ApplicationError> {
1835 if !(1..=100).contains(&input.limit) || input.offset > 1_000_000 {
1836 return Err(QueryError::InvalidPagination.into());
1837 }
1838 let store = SqliteStore::open_read_only(database_path)?;
1839 let current = store.load_current_community_snapshot(workspace)?;
1840 let freshness = freshness_summary(&store.load_current_freshness(workspace)?);
1841 let mut selected = current
1842 .communities
1843 .iter()
1844 .filter(|community| {
1845 input
1846 .community_id
1847 .as_ref()
1848 .is_none_or(|id| community.id == *id)
1849 })
1850 .cloned()
1851 .collect::<Vec<_>>();
1852 selected.sort_by(|left, right| {
1853 right
1854 .metrics
1855 .size
1856 .cmp(&left.metrics.size)
1857 .then_with(|| right.metrics.cohesion.total_cmp(&left.metrics.cohesion))
1858 .then_with(|| left.id.cmp(&right.id))
1859 });
1860 let total_communities = selected.len();
1861 let communities = selected
1862 .into_iter()
1863 .skip(input.offset)
1864 .take(input.limit)
1865 .collect();
1866 let delta = input
1867 .compare_snapshot_id
1868 .as_ref()
1869 .map(|snapshot_id| store.load_workspace_community_snapshot(workspace, snapshot_id))
1870 .transpose()?
1871 .map(|before| compare_community_snapshots(&before, ¤t));
1872 let exact_missing = input.community_id.is_some() && total_communities == 0;
1873 let truncated = input.offset.saturating_add(input.limit) < total_communities;
1874 let report = CommunityReport {
1875 snapshot_id: current.snapshot_id,
1876 engine_version: current.engine_version,
1877 config: current.config,
1878 communities,
1879 total_communities,
1880 offset: input.offset,
1881 limit: input.limit,
1882 truncated,
1883 delta,
1884 };
1885 let status = if !exact_missing && freshness.overall == OverallFreshness::Fresh {
1886 ToolStatus::Ok
1887 } else {
1888 ToolStatus::Degraded
1889 };
1890 Ok(ToolEnvelope {
1891 schema_version: 1,
1892 status,
1893 data: Some(report),
1894 freshness,
1895 warnings: exact_missing
1896 .then(|| "Requested community was not observed in the selected snapshot.".to_owned())
1897 .into_iter()
1898 .collect(),
1899 })
1900}
1901
1902pub async fn explore_repository(
1907 database_path: &Path,
1908 workspace: &str,
1909 input: &ExploreInput,
1910 binary: Option<std::ffi::OsString>,
1911) -> ToolEnvelope<LocalContextResult> {
1912 if input.workspace != workspace {
1913 return explore_error_envelope(format!(
1914 "workspace `{}` is outside this server's configured workspace `{workspace}`",
1915 input.workspace
1916 ));
1917 }
1918 if !(1..=25).contains(&input.max_files) {
1919 return explore_error_envelope("max_files must be between 1 and 25".to_owned());
1920 }
1921
1922 let store = match SqliteStore::open_read_only(database_path) {
1923 Ok(store) => store,
1924 Err(error) => return explore_error_envelope(error.to_string()),
1925 };
1926 let registry = match store.load_workspace_registry(workspace) {
1927 Ok(registry) => registry,
1928 Err(error) => return explore_error_envelope(error.to_string()),
1929 };
1930 let persisted_freshness = match store.load_current_freshness(workspace) {
1931 Ok(freshness) => freshness,
1932 Err(error) => return explore_error_envelope(error.to_string()),
1933 };
1934 let freshness = freshness_summary(&persisted_freshness);
1935 let requested_alias = input
1936 .repository
1937 .as_deref()
1938 .map(str::trim)
1939 .filter(|alias| !alias.is_empty());
1940 let repository = match select_explore_repository(®istry, requested_alias, workspace) {
1941 Ok(repository) => repository,
1942 Err(message) => {
1943 return explore_scoped_error_envelope(freshness, ToolStatus::Error, message);
1944 }
1945 };
1946
1947 let mut config = CodeGraphConfig::default();
1948 if let Some(binary) = binary {
1949 config.binary = binary;
1950 }
1951 let provider = match CodeGraphProvider::new(config) {
1952 Ok(provider) => provider,
1953 Err(error) => {
1954 return explore_scoped_error_envelope(
1955 freshness,
1956 ToolStatus::Degraded,
1957 error.to_string(),
1958 );
1959 }
1960 };
1961 let result = provider
1962 .build_local_context(LocalContextRequest {
1963 request: ProviderRequest {
1964 repo_id: repository.id.clone(),
1965 project_path: native_relative_path(&repository.canonical_path),
1966 budget: ProviderBudget {
1967 timeout: Duration::from_secs(5),
1968 max_output_bytes: 256 * 1024,
1969 max_items: 25,
1970 },
1971 cancellation: CancellationToken::new(),
1972 },
1973 query: input.query.clone(),
1974 max_files: input.max_files,
1975 })
1976 .await;
1977 let shutdown = provider.shutdown().await;
1978 explore_result_envelope(result, shutdown, freshness)
1979}
1980
1981fn select_explore_repository<'a>(
1982 registry: &'a WorkspaceRecord,
1983 requested_alias: Option<&str>,
1984 workspace: &str,
1985) -> Result<&'a RepositoryRecord, String> {
1986 if let Some(alias) = requested_alias {
1987 return registry
1988 .repositories
1989 .iter()
1990 .find(|repository| repository.alias == alias)
1991 .ok_or_else(|| {
1992 format!("repository alias `{alias}` is not registered in workspace `{workspace}`")
1993 });
1994 }
1995 if registry.repositories.len() == 1 {
1996 return registry.repositories.first().ok_or_else(|| {
1997 format!("workspace `{workspace}` unexpectedly has no registered repository")
1998 });
1999 }
2000 Err(format!(
2001 "repository is required because workspace `{workspace}` contains {} repositories",
2002 registry.repositories.len()
2003 ))
2004}
2005
2006fn explore_result_envelope(
2007 result: Result<LocalContextResult, ProviderError>,
2008 shutdown: Result<(), ProviderError>,
2009 freshness: FreshnessSummary,
2010) -> ToolEnvelope<LocalContextResult> {
2011 match result {
2012 Ok(context) => {
2013 let mut warnings = context
2014 .execution
2015 .degradations
2016 .iter()
2017 .map(|degradation| degradation.message.clone())
2018 .collect::<Vec<_>>();
2019 if context.execution.truncated {
2020 warnings.push("Local context was truncated by the configured budget.".to_owned());
2021 }
2022 if let Err(error) = shutdown {
2023 warnings.push(format!("CodeGraph shutdown degraded: {error}"));
2024 }
2025 let status = if warnings.is_empty() && freshness.overall == OverallFreshness::Fresh {
2026 ToolStatus::Ok
2027 } else {
2028 ToolStatus::Degraded
2029 };
2030 ToolEnvelope {
2031 schema_version: 1,
2032 status,
2033 data: Some(context),
2034 freshness,
2035 warnings,
2036 }
2037 }
2038 Err(error) => ToolEnvelope {
2039 schema_version: 1,
2040 status: if matches!(&error, ProviderError::InvalidRequest(_)) {
2041 ToolStatus::Error
2042 } else {
2043 ToolStatus::Degraded
2044 },
2045 data: None,
2046 freshness,
2047 warnings: vec![error.to_string()],
2048 },
2049 }
2050}
2051
2052fn explore_scoped_error_envelope(
2053 freshness: FreshnessSummary,
2054 status: ToolStatus,
2055 message: String,
2056) -> ToolEnvelope<LocalContextResult> {
2057 ToolEnvelope {
2058 schema_version: 1,
2059 status,
2060 data: None,
2061 freshness,
2062 warnings: vec![message],
2063 }
2064}
2065
2066fn explore_error_envelope(message: String) -> ToolEnvelope<LocalContextResult> {
2067 ToolEnvelope {
2068 schema_version: 1,
2069 status: ToolStatus::Error,
2070 data: None,
2071 freshness: FreshnessSummary {
2072 overall: OverallFreshness::Unknown,
2073 stale_repositories: Vec::new(),
2074 reasons: vec!["Local repository exploration could not start.".to_owned()],
2075 },
2076 warnings: vec![message],
2077 }
2078}
2079
2080pub fn impact_workspace(
2091 database_path: &Path,
2092 workspace: &str,
2093 request: &ImpactRequest,
2094) -> Result<ToolEnvelope<ImpactReport>, ApplicationError> {
2095 let loaded = load_impact_context(database_path, workspace)?;
2096 impact_envelope(request, &loaded.context, loaded.freshness)
2097}
2098
2099pub async fn impact_workspace_with_codegraph(
2109 database_path: &Path,
2110 workspace: &str,
2111 request: &ImpactRequest,
2112 binary: Option<std::ffi::OsString>,
2113) -> Result<ToolEnvelope<ImpactReport>, ApplicationError> {
2114 let mut loaded = load_impact_context(database_path, workspace)?;
2115 let mut config = CodeGraphConfig::default();
2116 if let Some(binary) = binary {
2117 config.binary = binary;
2118 }
2119 match CodeGraphProvider::new(config) {
2120 Ok(provider) => {
2121 enrich_impact_context(
2122 &mut loaded.context,
2123 request,
2124 &loaded.registry,
2125 &loaded.node_evidence,
2126 &provider,
2127 )
2128 .await;
2129 let _ = provider.shutdown().await;
2130 }
2131 Err(error) => loaded.context.local_enrichment.push(LocalEnrichmentInput {
2132 repo_id: impact_target_repo(&loaded.context, request)
2133 .unwrap_or_else(|| RepoId::new("repo:unknown")),
2134 anchor: impact_target_label(&loaded.context, request)
2135 .unwrap_or_else(|| "unknown impact target".to_owned()),
2136 status: LocalEnrichmentStatus::Unavailable,
2137 affected: Vec::new(),
2138 affected_tests: Vec::new(),
2139 truncated: false,
2140 degradations: vec![error.to_string()],
2141 }),
2142 }
2143 impact_envelope(request, &loaded.context, loaded.freshness)
2144}
2145
2146pub async fn collect_workspace_changes(
2155 database_path: &Path,
2156 workspace: &str,
2157 input: &ChangesInput,
2158 git_binary: Option<std::ffi::OsString>,
2159) -> Result<ToolEnvelope<ChangeSet>, ApplicationError> {
2160 collect_workspace_changes_with_cancellation(
2161 database_path,
2162 workspace,
2163 input,
2164 git_binary,
2165 &CancellationToken::new(),
2166 )
2167 .await
2168}
2169
2170pub async fn collect_workspace_changes_with_cancellation(
2176 database_path: &Path,
2177 workspace: &str,
2178 input: &ChangesInput,
2179 git_binary: Option<std::ffi::OsString>,
2180 cancellation: &CancellationToken,
2181) -> Result<ToolEnvelope<ChangeSet>, ApplicationError> {
2182 let store = SqliteStore::open_read_only(database_path)?;
2183 let registry = store.load_workspace_registry(workspace)?;
2184 let snapshot = store.current_snapshot_summary(workspace)?;
2185 let repository = registry
2186 .repositories
2187 .iter()
2188 .find(|repository| repository.alias == input.repository)
2189 .ok_or_else(|| ApplicationError::RegistryAliasMissing(input.repository.clone()))?;
2190 let persisted_freshness = store.load_current_freshness(workspace)?;
2191 let freshness = freshness_summary(&persisted_freshness);
2192 let provider = git_binary.map_or_else(GitCliChangeProvider::new, |binary| {
2193 GitCliChangeProvider::with_limits(
2194 binary,
2195 Duration::from_secs(30),
2196 8 * 1024 * 1024,
2197 64 * 1024,
2198 )
2199 });
2200 let mut analyzer_versions = AnalyzerVersions::new();
2201 analyzer_versions.insert(
2202 "code-system-graph.changes".to_owned(),
2203 env!("CARGO_PKG_VERSION").to_owned(),
2204 );
2205 let change_set = provider
2206 .changes(
2207 &ChangeRequest {
2208 repo_id: repository.id.clone(),
2209 checkout_id: repository.checkout_id.clone(),
2210 worktree: native_relative_path(&repository.canonical_path),
2211 scope: input.scope.clone(),
2212 workspace_manifest_hash: registry.manifest_hash.clone(),
2213 contract_registry_hash: snapshot.snapshot_id,
2214 analyzer_versions,
2215 },
2216 cancellation,
2217 )
2218 .await?;
2219 let repository_is_fresh = persisted_freshness.iter().any(|item| {
2220 item.checkout_id == repository.checkout_id && item.state == RepoFreshnessState::Fresh
2221 });
2222 Ok(ToolEnvelope {
2223 schema_version: 1,
2224 status: if repository_is_fresh {
2225 ToolStatus::Ok
2226 } else {
2227 ToolStatus::Degraded
2228 },
2229 data: Some(change_set),
2230 freshness,
2231 warnings: Vec::new(),
2232 })
2233}
2234
2235pub async fn analyze_workspace_changes(
2242 database_path: &Path,
2243 workspace: &str,
2244 input: &ChangesInput,
2245 options: &ChangeAnalysisOptions,
2246 git_binary: Option<std::ffi::OsString>,
2247) -> Result<ToolEnvelope<ChangeImpactReport>, ApplicationError> {
2248 analyze_workspace_changes_with_cancellation(
2249 database_path,
2250 workspace,
2251 input,
2252 options,
2253 git_binary,
2254 &CancellationToken::new(),
2255 )
2256 .await
2257}
2258
2259pub async fn analyze_workspace_changes_with_cancellation(
2265 database_path: &Path,
2266 workspace: &str,
2267 input: &ChangesInput,
2268 options: &ChangeAnalysisOptions,
2269 git_binary: Option<std::ffi::OsString>,
2270 cancellation: &CancellationToken,
2271) -> Result<ToolEnvelope<ChangeImpactReport>, ApplicationError> {
2272 let collected = collect_workspace_changes_with_cancellation(
2273 database_path,
2274 workspace,
2275 input,
2276 git_binary,
2277 cancellation,
2278 )
2279 .await?;
2280 let change_set = collected.data.ok_or_else(|| {
2281 ApplicationError::InvalidSourceObservation(
2282 "local change collection returned no change set".to_owned(),
2283 )
2284 })?;
2285 let store = SqliteStore::open_read_only(database_path)?;
2286 let (nodes, edges) = store.load_current_graph(workspace)?;
2287 let evidence = store.load_current_evidence(workspace)?;
2288 let community = store.load_current_community_snapshot(workspace)?;
2289 let freshness = store.load_current_freshness(workspace)?;
2290 let report = analyze_changes(
2291 &change_set,
2292 &nodes,
2293 &edges,
2294 &evidence,
2295 Some(&community),
2296 &freshness,
2297 &[],
2298 options,
2299 )?;
2300 let status = if collected.status == ToolStatus::Ok && report.coverage.complete {
2301 ToolStatus::Ok
2302 } else {
2303 ToolStatus::Degraded
2304 };
2305 Ok(ToolEnvelope {
2306 schema_version: 1,
2307 status,
2308 data: Some(report),
2309 freshness: collected.freshness,
2310 warnings: collected.warnings,
2311 })
2312}
2313
2314fn pull_request_provider(
2315 kind: PullRequestProviderKind,
2316 enabled: bool,
2317 token: Option<String>,
2318 basic_auth_username: Option<String>,
2319) -> Result<Box<dyn PullRequestProvider>, ApplicationError> {
2320 if !enabled {
2321 return Err(PullRequestError::Disabled.into());
2322 }
2323 let mut config = match kind {
2324 PullRequestProviderKind::GitHub => PullRequestProviderConfig::default(),
2325 PullRequestProviderKind::BitbucketCloud => PullRequestProviderConfig::bitbucket_cloud(),
2326 PullRequestProviderKind::BitbucketDataCenter => {
2327 return Err(PullRequestError::InvalidConfiguration(
2328 "Bitbucket Data Center is not implemented in Code System Graph 1.0.0".to_owned(),
2329 )
2330 .into());
2331 }
2332 };
2333 config.enabled = true;
2334 config.auth_token = PrAuthToken::new(token.unwrap_or_default());
2335 if kind == PullRequestProviderKind::BitbucketCloud {
2336 config.basic_auth_username = basic_auth_username;
2337 }
2338 let transport = Arc::new(
2339 ReqwestPrHttpTransport::new()
2340 .map_err(|error| PullRequestError::Transport(error.to_string()))?,
2341 );
2342 match kind {
2343 PullRequestProviderKind::GitHub => Ok(Box::new(GitHubProvider::new(config, transport)?)),
2344 PullRequestProviderKind::BitbucketCloud => {
2345 Ok(Box::new(BitbucketProvider::new(config, transport)?))
2346 }
2347 PullRequestProviderKind::BitbucketDataCenter => unreachable!("rejected above"),
2348 }
2349}
2350
2351pub async fn inspect_pull_request(
2362 input: &PullRequestInput,
2363 enabled: bool,
2364 token: Option<String>,
2365 basic_auth_username: Option<String>,
2366) -> Result<ToolEnvelope<PullRequestInspection>, ApplicationError> {
2367 inspect_pull_request_with_cancellation(
2368 input,
2369 enabled,
2370 token,
2371 basic_auth_username,
2372 CancellationToken::new(),
2373 )
2374 .await
2375}
2376
2377pub async fn inspect_pull_request_with_cancellation(
2383 input: &PullRequestInput,
2384 enabled: bool,
2385 token: Option<String>,
2386 basic_auth_username: Option<String>,
2387 cancellation: CancellationToken,
2388) -> Result<ToolEnvelope<PullRequestInspection>, ApplicationError> {
2389 if !input.consent_to_remote_access {
2390 return Err(PullRequestError::ConsentRequired.into());
2391 }
2392 let provider = pull_request_provider(input.provider, enabled, token, basic_auth_username)?;
2393 let inspection = provider
2394 .inspect(PullRequestInspectRequest {
2395 coordinates: PullRequestCoordinates {
2396 provider: input.provider,
2397 owner: input.owner.clone(),
2398 repository: input.repository.clone(),
2399 number: input.number,
2400 },
2401 consent_to_remote_access: input.consent_to_remote_access,
2402 cancellation,
2403 })
2404 .await?;
2405 let status = if inspection.warnings.is_empty() {
2406 ToolStatus::Ok
2407 } else {
2408 ToolStatus::Degraded
2409 };
2410 Ok(ToolEnvelope {
2411 schema_version: 1,
2412 status,
2413 data: Some(inspection),
2414 freshness: FreshnessSummary {
2415 overall: OverallFreshness::Unknown,
2416 stale_repositories: Vec::new(),
2417 reasons: vec![
2418 "Remote pull-request inspection does not establish local graph freshness."
2419 .to_owned(),
2420 ],
2421 },
2422 warnings: Vec::new(),
2423 })
2424}
2425
2426pub async fn list_pull_requests(
2433 input: &PullRequestListInput,
2434 enabled: bool,
2435 token: Option<String>,
2436 basic_auth_username: Option<String>,
2437 cancellation: CancellationToken,
2438) -> Result<ToolEnvelope<PullRequestListPage>, ApplicationError> {
2439 if !input.consent_to_remote_access {
2440 return Err(PullRequestError::ConsentRequired.into());
2441 }
2442 let provider = pull_request_provider(input.provider, enabled, token, basic_auth_username)?;
2443 let page = provider
2444 .list(PullRequestListRequest {
2445 provider: input.provider,
2446 owner: input.owner.clone(),
2447 repository: input.repository.clone(),
2448 state: input.state,
2449 cursor: input.cursor.clone(),
2450 limit: input.limit,
2451 consent_to_remote_access: input.consent_to_remote_access,
2452 cancellation,
2453 })
2454 .await?;
2455 let status = if page.warnings.is_empty() && !page.truncated {
2456 ToolStatus::Ok
2457 } else {
2458 ToolStatus::Degraded
2459 };
2460 Ok(ToolEnvelope {
2461 schema_version: 1,
2462 status,
2463 data: Some(page),
2464 freshness: FreshnessSummary {
2465 overall: OverallFreshness::Unknown,
2466 stale_repositories: Vec::new(),
2467 reasons: vec![
2468 "Remote pull-request listing does not establish local graph freshness.".to_owned(),
2469 ],
2470 },
2471 warnings: Vec::new(),
2472 })
2473}
2474
2475struct LoadedImpactContext {
2476 context: ImpactContext,
2477 freshness: FreshnessSummary,
2478 registry: WorkspaceRecord,
2479 node_evidence: BTreeMap<NodeId, Vec<Evidence>>,
2480}
2481
2482fn load_impact_context(
2483 database_path: &Path,
2484 workspace: &str,
2485) -> Result<LoadedImpactContext, ApplicationError> {
2486 let store = SqliteStore::open_read_only(database_path)?;
2487 let (nodes, edges) = store.load_current_graph(workspace)?;
2488 let persisted_freshness = store.load_current_freshness(workspace)?;
2489 let freshness = freshness_summary(&persisted_freshness);
2490 let communities = store.load_current_community_snapshot(workspace)?;
2491 let evidence = store.load_current_evidence(workspace)?;
2492 let registry = store.load_workspace_registry(workspace)?;
2493 let node_evidence = node_evidence(&edges, &evidence);
2494 let context = ImpactContext {
2495 centrality: impact_centrality_scores(&communities.communities),
2496 service_memberships: service_memberships(&nodes, &edges),
2497 recommended_commands: recommended_test_commands(&nodes, &node_evidence),
2498 nodes,
2499 edges,
2500 communities: Some(communities),
2501 freshness: conservative_repo_freshness(&persisted_freshness),
2502 compatibility: Vec::new(),
2503 local_enrichment: Vec::new(),
2504 public_contracts: Vec::new(),
2505 criticality: Vec::new(),
2506 environments: Vec::new(),
2507 graph_complete: true,
2508 coverage_gaps: Vec::new(),
2509 };
2510 Ok(LoadedImpactContext {
2511 context,
2512 freshness,
2513 registry,
2514 node_evidence,
2515 })
2516}
2517
2518fn impact_envelope(
2519 request: &ImpactRequest,
2520 context: &ImpactContext,
2521 freshness: FreshnessSummary,
2522) -> Result<ToolEnvelope<ImpactReport>, ApplicationError> {
2523 let report = analyze_impact(request, context)?;
2524 let status = if report.risk == code_system_graph_core::RiskLevel::Unknown
2525 || freshness.overall != OverallFreshness::Fresh
2526 {
2527 ToolStatus::Degraded
2528 } else {
2529 ToolStatus::Ok
2530 };
2531 Ok(ToolEnvelope {
2532 schema_version: 1,
2533 status,
2534 data: Some(report),
2535 freshness,
2536 warnings: Vec::new(),
2537 })
2538}
2539
2540#[expect(
2541 clippy::too_many_lines,
2542 reason = "Provider probing, impact, affected tests, and degradation form one bounded lifecycle"
2543)]
2544async fn enrich_impact_context(
2545 context: &mut ImpactContext,
2546 request: &ImpactRequest,
2547 registry: &WorkspaceRecord,
2548 evidence: &BTreeMap<NodeId, Vec<Evidence>>,
2549 provider: &impl LocalCodeIntelligenceProvider,
2550) {
2551 let Some(target) = impact_target_node(context, request).cloned() else {
2552 return;
2553 };
2554 let by_id = context
2555 .nodes
2556 .iter()
2557 .map(|node| (node.id.clone(), node))
2558 .collect::<BTreeMap<_, _>>();
2559 let mut anchors = context
2560 .edges
2561 .iter()
2562 .filter_map(|edge| {
2563 let adjacent = if edge.source == target.id {
2564 by_id.get(&edge.target)
2565 } else if edge.target == target.id {
2566 by_id.get(&edge.source)
2567 } else {
2568 None
2569 };
2570 adjacent
2571 .copied()
2572 .filter(|node| node.kind == NodeKind::SymbolRef)
2573 .cloned()
2574 })
2575 .chain((target.kind == NodeKind::SymbolRef).then_some(target.clone()))
2576 .collect::<Vec<_>>();
2577 anchors.sort_by(|left, right| left.id.cmp(&right.id));
2578 anchors.dedup_by(|left, right| left.id == right.id);
2579 anchors.truncate(3);
2580
2581 for anchor in anchors {
2582 let Some(repo_id) = anchor.repo_id.clone() else {
2583 continue;
2584 };
2585 let Some(repository) = registry
2586 .repositories
2587 .iter()
2588 .find(|repository| repository.id == repo_id)
2589 else {
2590 context.local_enrichment.push(unavailable_local_enrichment(
2591 repo_id,
2592 &anchor.label,
2593 "Registered checkout metadata was unavailable for local enrichment.",
2594 ));
2595 continue;
2596 };
2597 let project_path = native_relative_path(&repository.canonical_path);
2598 let provider_request = || ProviderRequest {
2599 repo_id: repo_id.clone(),
2600 project_path: project_path.clone(),
2601 budget: ProviderBudget {
2602 timeout: Duration::from_secs(2),
2603 max_output_bytes: 256 * 1024,
2604 max_items: 25,
2605 },
2606 cancellation: CancellationToken::new(),
2607 };
2608 let capability = match provider.probe(provider_request()).await {
2609 Ok(capability) => capability,
2610 Err(error) => {
2611 context.local_enrichment.push(unavailable_local_enrichment(
2612 repo_id,
2613 &anchor.label,
2614 &error.to_string(),
2615 ));
2616 continue;
2617 }
2618 };
2619 if capability.status != ProviderStatus::Available {
2620 let status = if capability.status == ProviderStatus::Stale {
2621 LocalEnrichmentStatus::Stale
2622 } else {
2623 LocalEnrichmentStatus::Unavailable
2624 };
2625 let mut degradations = capability
2626 .degradations
2627 .into_iter()
2628 .map(|degradation| degradation.message)
2629 .collect::<Vec<_>>();
2630 degradations.extend(capability.remediation);
2631 context.local_enrichment.push(LocalEnrichmentInput {
2632 repo_id,
2633 anchor: anchor.label.clone(),
2634 status,
2635 affected: Vec::new(),
2636 affected_tests: Vec::new(),
2637 truncated: false,
2638 degradations,
2639 });
2640 continue;
2641 }
2642 let result = match provider
2643 .get_local_impact(LocalImpactRequest {
2644 request: provider_request(),
2645 symbol: anchor.label.clone(),
2646 max_depth: request.options.max_depth.min(8),
2647 })
2648 .await
2649 {
2650 Ok(result) => result,
2651 Err(error) => {
2652 context.local_enrichment.push(unavailable_local_enrichment(
2653 repo_id,
2654 &anchor.label,
2655 &error.to_string(),
2656 ));
2657 continue;
2658 }
2659 };
2660 let changed_files = evidence
2661 .get(&anchor.id)
2662 .into_iter()
2663 .flatten()
2664 .filter_map(|item| item.file_path.clone())
2665 .collect::<BTreeSet<_>>()
2666 .into_iter()
2667 .collect::<Vec<_>>();
2668 let affected_tests = if changed_files.is_empty() {
2669 None
2670 } else {
2671 provider
2672 .get_affected_tests(AffectedTestsRequest {
2673 request: provider_request(),
2674 changed_files,
2675 max_depth: request.options.max_depth.min(8),
2676 })
2677 .await
2678 .ok()
2679 .flatten()
2680 };
2681 let mut degradations = result
2682 .execution
2683 .degradations
2684 .iter()
2685 .map(|degradation| degradation.message.clone())
2686 .collect::<Vec<_>>();
2687 if affected_tests.is_none() {
2688 degradations.push(
2689 "CodeGraph affected-test capability was unavailable or no changed file was known."
2690 .to_owned(),
2691 );
2692 }
2693 let tests_truncated = affected_tests
2694 .as_ref()
2695 .is_some_and(|tests| tests.execution.truncated);
2696 let status = if result.execution.truncated || tests_truncated || !degradations.is_empty() {
2697 LocalEnrichmentStatus::Partial
2698 } else {
2699 LocalEnrichmentStatus::Available
2700 };
2701 context.local_enrichment.push(LocalEnrichmentInput {
2702 repo_id,
2703 anchor: result.symbol,
2704 status,
2705 affected: result
2706 .affected
2707 .into_iter()
2708 .map(|item| LocalImpactItem {
2709 symbol: item.name,
2710 file_path: item.file_path,
2711 start_line: Some(item.start_line),
2712 depth: result.depth,
2713 })
2714 .collect(),
2715 affected_tests: affected_tests
2716 .map(|tests| tests.affected_tests)
2717 .unwrap_or_default(),
2718 truncated: result.execution.truncated || tests_truncated,
2719 degradations,
2720 });
2721 }
2722}
2723
2724fn unavailable_local_enrichment(
2725 repo_id: RepoId,
2726 anchor: &str,
2727 message: &str,
2728) -> LocalEnrichmentInput {
2729 LocalEnrichmentInput {
2730 repo_id,
2731 anchor: anchor.to_owned(),
2732 status: LocalEnrichmentStatus::Unavailable,
2733 affected: Vec::new(),
2734 affected_tests: Vec::new(),
2735 truncated: false,
2736 degradations: vec![message.to_owned()],
2737 }
2738}
2739
2740fn impact_target_node<'a>(context: &'a ImpactContext, request: &ImpactRequest) -> Option<&'a Node> {
2741 context.nodes.iter().find(|node| match &request.target {
2742 ImpactTarget::NodeId(node_id) => node.id == *node_id,
2743 ImpactTarget::StableKey(stable_key) => node.stable_key == *stable_key,
2744 })
2745}
2746
2747fn impact_target_repo(context: &ImpactContext, request: &ImpactRequest) -> Option<RepoId> {
2748 impact_target_node(context, request).and_then(|node| node.repo_id.clone())
2749}
2750
2751fn impact_target_label(context: &ImpactContext, request: &ImpactRequest) -> Option<String> {
2752 impact_target_node(context, request).map(|node| node.label.clone())
2753}
2754
2755fn fts_search_scores(
2756 hits: &[code_system_graph_store_sqlite::StoredNodeSearchHit],
2757) -> BTreeMap<NodeId, f64> {
2758 hits.iter()
2759 .enumerate()
2760 .map(|(index, hit)| {
2761 (
2762 hit.node.id.clone(),
2763 1.0 / f64::from(u32::try_from(index + 1).unwrap_or(u32::MAX)),
2764 )
2765 })
2766 .collect()
2767}
2768
2769fn impact_centrality_scores(communities: &[Community]) -> BTreeMap<NodeId, f32> {
2770 let mut scores = BTreeMap::<NodeId, f32>::new();
2771 for community in communities {
2772 for (index, node_id) in community.central_nodes.iter().enumerate() {
2773 let denominator = u16::try_from(index + 1).unwrap_or(u16::MAX);
2774 let score = 1.0 / f32::from(denominator);
2775 scores
2776 .entry(node_id.clone())
2777 .and_modify(|current| *current = current.max(score))
2778 .or_insert(score);
2779 }
2780 }
2781 scores
2782}
2783
2784fn conservative_repo_freshness(items: &[RepoFreshness]) -> Vec<RepoFreshness> {
2785 let mut by_repository = BTreeMap::<RepoId, RepoFreshness>::new();
2786 for item in items {
2787 by_repository
2788 .entry(item.repo_id.clone())
2789 .and_modify(|current| {
2790 if freshness_severity(item.state) > freshness_severity(current.state) {
2791 *current = item.clone();
2792 }
2793 })
2794 .or_insert_with(|| item.clone());
2795 }
2796 by_repository.into_values().collect()
2797}
2798
2799fn freshness_severity(state: RepoFreshnessState) -> u8 {
2800 match state {
2801 RepoFreshnessState::Fresh => 0,
2802 RepoFreshnessState::WorkingTreeChanged
2803 | RepoFreshnessState::CommitsBehind
2804 | RepoFreshnessState::ConfigChanged
2805 | RepoFreshnessState::ExtractorChanged
2806 | RepoFreshnessState::CodegraphPending => 1,
2807 RepoFreshnessState::Partial | RepoFreshnessState::Unknown => 2,
2808 RepoFreshnessState::Unavailable => 3,
2809 RepoFreshnessState::Corrupt => 4,
2810 }
2811}
2812
2813fn recommended_test_commands(
2814 nodes: &[Node],
2815 evidence: &BTreeMap<NodeId, Vec<Evidence>>,
2816) -> Vec<RecommendedCommand> {
2817 let mut commands = BTreeMap::<(RepoId, String), RecommendedCommand>::new();
2818 for test in nodes.iter().filter(|node| node.kind == NodeKind::TestCase) {
2819 let Some(repo_id) = test.repo_id.clone() else {
2820 continue;
2821 };
2822 for item in evidence.get(&test.id).into_iter().flatten() {
2823 let Some(path) = item.file_path.as_deref() else {
2824 continue;
2825 };
2826 let command = match Path::new(path).extension().and_then(|value| value.to_str()) {
2827 Some(extension) if extension.eq_ignore_ascii_case("py") => "python -m pytest",
2828 Some(extension) if extension.eq_ignore_ascii_case("rs") => "cargo test",
2829 _ => continue,
2830 };
2831 commands.entry((repo_id.clone(), command.to_owned())).or_insert_with(|| RecommendedCommand {
2832 repo_id: repo_id.clone(),
2833 command: command.to_owned(),
2834 description: format!("Run the repository test suite covering `{}`; Code System Graph does not execute it.", test.label),
2835 });
2836 }
2837 }
2838 commands.into_values().collect()
2839}
2840
2841fn community_centrality_scores(communities: &[Community]) -> BTreeMap<NodeId, f64> {
2842 let mut scores = BTreeMap::<NodeId, f64>::new();
2843 for community in communities {
2844 for (index, node_id) in community.central_nodes.iter().enumerate() {
2845 let score = 1.0 / f64::from(u32::try_from(index + 1).unwrap_or(u32::MAX));
2846 scores
2847 .entry(node_id.clone())
2848 .and_modify(|current| *current = current.max(score))
2849 .or_insert(score);
2850 }
2851 }
2852 scores
2853}
2854
2855fn community_memberships(communities: &[Community]) -> BTreeMap<NodeId, Vec<CommunityId>> {
2856 let mut memberships = BTreeMap::<NodeId, Vec<CommunityId>>::new();
2857 for community in communities {
2858 for member in &community.members {
2859 memberships
2860 .entry(member.clone())
2861 .or_default()
2862 .push(community.id.clone());
2863 }
2864 }
2865 memberships
2866}
2867
2868fn service_memberships(nodes: &[Node], edges: &[Edge]) -> BTreeMap<NodeId, Vec<NodeId>> {
2869 let services = nodes
2870 .iter()
2871 .filter(|node| node.kind == NodeKind::Service)
2872 .map(|node| node.id.clone())
2873 .collect::<BTreeSet<_>>();
2874 let mut memberships = services
2875 .iter()
2876 .map(|service| (service.clone(), vec![service.clone()]))
2877 .collect::<BTreeMap<_, _>>();
2878 for edge in edges {
2879 if services.contains(&edge.source) {
2880 memberships
2881 .entry(edge.target.clone())
2882 .or_default()
2883 .push(edge.source.clone());
2884 }
2885 if services.contains(&edge.target) {
2886 memberships
2887 .entry(edge.source.clone())
2888 .or_default()
2889 .push(edge.target.clone());
2890 }
2891 }
2892 for values in memberships.values_mut() {
2893 values.sort();
2894 values.dedup();
2895 }
2896 memberships
2897}
2898
2899fn node_evidence(edges: &[Edge], evidence: &[Evidence]) -> BTreeMap<NodeId, Vec<Evidence>> {
2900 let by_id = evidence
2901 .iter()
2902 .map(|item| (item.id.clone(), item))
2903 .collect::<BTreeMap<_, _>>();
2904 let mut result = BTreeMap::<NodeId, Vec<Evidence>>::new();
2905 for edge in edges {
2906 for node_id in [&edge.source, &edge.target] {
2907 let values = result.entry(node_id.clone()).or_default();
2908 values.extend(
2909 edge.evidence
2910 .iter()
2911 .filter_map(|evidence_id| by_id.get(evidence_id))
2912 .map(|item| (*item).clone()),
2913 );
2914 }
2915 }
2916 for values in result.values_mut() {
2917 values.sort_by(|left, right| left.id.cmp(&right.id));
2918 values.dedup_by(|left, right| left.id == right.id);
2919 }
2920 result
2921}
2922
2923pub fn status_workspace(
2930 config_path: &Path,
2931 database_path: &Path,
2932) -> Result<WorkspaceStatus, ApplicationError> {
2933 let current = load_workspace_context(config_path, &ScanOverrides::default())?;
2934 let store = SqliteStore::open_read_only(database_path)?;
2935 let previous = store.load_current_freshness(¤t.manifest.name)?;
2936 let previous_by_checkout = previous
2937 .iter()
2938 .map(|freshness| (freshness.checkout_id.clone(), freshness))
2939 .collect::<BTreeMap<CheckoutId, _>>();
2940 let current_checkouts = current
2941 .registry
2942 .record
2943 .repositories
2944 .iter()
2945 .map(|repository| repository.checkout_id.clone())
2946 .collect::<std::collections::BTreeSet<_>>();
2947 let mut repositories = current
2948 .registry
2949 .record
2950 .repositories
2951 .iter()
2952 .map(|repository| {
2953 let previous = previous_by_checkout.get(&repository.checkout_id);
2954 let (state, reason) = evaluate_repository_freshness(
2955 repository,
2956 previous.copied(),
2957 ¤t.registry.record.manifest_hash,
2958 );
2959 RepoFreshness {
2960 repo_id: repository.id.clone(),
2961 checkout_id: repository.checkout_id.clone(),
2962 head_commit: repository.head_commit.clone(),
2963 manifest_hash: current.registry.record.manifest_hash.clone(),
2964 state,
2965 reason,
2966 }
2967 })
2968 .collect::<Vec<_>>();
2969 repositories.extend(
2970 previous
2971 .into_iter()
2972 .filter(|freshness| !current_checkouts.contains(&freshness.checkout_id))
2973 .map(|mut freshness| {
2974 freshness.state = RepoFreshnessState::Unavailable;
2975 freshness.reason = Some(
2976 "Checkout from the published snapshot is absent from the current manifest."
2977 .to_owned(),
2978 );
2979 freshness
2980 }),
2981 );
2982 repositories.sort_by(|left, right| {
2983 (&left.repo_id, &left.checkout_id).cmp(&(&right.repo_id, &right.checkout_id))
2984 });
2985 let watcher = watcher_status(
2986 database_path,
2987 ¤t.manifest.name,
2988 current.execution_policy.max_no_progress_time_ms,
2989 &store.database_instance_id()?,
2990 );
2991 Ok(WorkspaceStatus {
2992 workspace: current.manifest.name,
2993 schema_version: store.schema_version()?,
2994 integrity_ok: store.integrity_check()?,
2995 freshness: freshness_summary(&repositories),
2996 repositories,
2997 watcher,
2998 })
2999}
3000
3001fn watcher_status(
3002 database: &Path,
3003 workspace: &str,
3004 stale_after_ms: u64,
3005 database_instance_id: &str,
3006) -> WatcherStatus {
3007 if !work_state::work_path(database).exists() {
3008 return WatcherStatus {
3009 state: WatcherState::NeverStarted,
3010 detail: None,
3011 };
3012 }
3013 let lease = work_state::WorkState::open(database, database_instance_id)
3014 .and_then(|state| state.watcher_lease(workspace));
3015 let Ok(Some(lease)) = lease else {
3016 return WatcherStatus {
3017 state: WatcherState::NeverStarted,
3018 detail: None,
3019 };
3020 };
3021 let persisted = match lease.state.as_str() {
3022 "expired_idle" => Some(WatcherState::ExpiredIdle),
3023 "expired_session" => Some(WatcherState::ExpiredSession),
3024 "failed_limit" => Some(WatcherState::FailedLimit),
3025 "active" => None,
3026 _ => Some(WatcherState::Stale),
3027 };
3028 if let Some(state) = persisted {
3029 return WatcherStatus {
3030 state,
3031 detail: lease.detail,
3032 };
3033 }
3034 let identity_matches = lease
3035 .pid
3036 .zip(lease.process_start_identity.as_deref())
3037 .is_some_and(|(pid, expected)| worker::process_identity(pid).as_deref() == Some(expected));
3038 let heartbeat_fresh = lease.heartbeat_unix_ms.is_some_and(|heartbeat| {
3039 current_unix_millis().saturating_sub(heartbeat) <= stale_after_ms.saturating_mul(2)
3040 });
3041 if identity_matches && heartbeat_fresh {
3042 WatcherStatus {
3043 state: WatcherState::Active,
3044 detail: lease.detail,
3045 }
3046 } else {
3047 WatcherStatus {
3048 state: WatcherState::Stale,
3049 detail: Some("watcher heartbeat or process start identity is stale".to_owned()),
3050 }
3051 }
3052}
3053
3054#[doc(hidden)]
3056pub fn start_watcher_lease(
3057 config: &Path,
3058 database: &Path,
3059) -> Result<(String, String, ExecutionPolicy), ApplicationError> {
3060 let manifest_source = read_file(config)?;
3061 let manifest = parse_manifest(&manifest_source)?;
3062 validate_global_policy_source(config, &manifest)?;
3063 let execution_policy = ExecutionPolicy::resolve(manifest.execution_policy.as_ref())
3064 .map_err(ManifestError::from)?;
3065 let identity = worker::process_identity(std::process::id()).ok_or_else(|| {
3066 ApplicationError::Initialization("failed to resolve watcher process identity".to_owned())
3067 })?;
3068 let now = current_unix_millis();
3069 let owner_token = stable_id(
3070 "watcher-lease",
3071 &format!("{}:{}:{identity}:{now}", manifest.name, std::process::id()),
3072 );
3073 let database_instance_id = SqliteStore::open(database)?.database_instance_id()?;
3074 let result =
3075 work_state::WorkState::open(database, &database_instance_id).and_then(|mut state| {
3076 state.start_watcher(
3077 &manifest.name,
3078 &owner_token,
3079 std::process::id(),
3080 &identity,
3081 now,
3082 execution_policy.max_no_progress_time_ms,
3083 )
3084 });
3085 if let Err(message) = result {
3086 if message
3087 == format!(
3088 "watcher for workspace `{}` is already active",
3089 manifest.name
3090 )
3091 {
3092 return Err(ApplicationError::WatcherAlreadyActive(manifest.name));
3093 }
3094 return Err(ApplicationError::Initialization(message));
3095 }
3096 Ok((manifest.name, owner_token, execution_policy))
3097}
3098
3099#[doc(hidden)]
3101pub fn heartbeat_watcher_lease(
3102 database: &Path,
3103 workspace: &str,
3104 owner_token: &str,
3105 successful_activity: bool,
3106) -> Result<(), ApplicationError> {
3107 let database_instance_id = work_database_instance_id(database)?;
3108 work_state::WorkState::open(database, &database_instance_id)
3109 .and_then(|state| {
3110 state.heartbeat_watcher(
3111 workspace,
3112 owner_token,
3113 current_unix_millis(),
3114 successful_activity,
3115 )
3116 })
3117 .map_err(ApplicationError::Initialization)
3118}
3119
3120#[doc(hidden)]
3122pub fn finish_watcher_lease(
3123 database: &Path,
3124 workspace: &str,
3125 owner_token: &str,
3126 state: &str,
3127 detail: Option<&str>,
3128) -> Result<(), ApplicationError> {
3129 let database_instance_id = work_database_instance_id(database)?;
3130 work_state::WorkState::open(database, &database_instance_id)
3131 .and_then(|work| {
3132 work.finish_watcher(workspace, owner_token, state, detail, current_unix_millis())
3133 })
3134 .map_err(ApplicationError::Initialization)
3135}
3136
3137fn work_database_instance_id(database: &Path) -> Result<String, ApplicationError> {
3138 Ok(SqliteStore::open_read_only(database)?.database_instance_id()?)
3139}
3140
3141pub fn contracts_workspace(
3148 database_path: &Path,
3149 workspace: &str,
3150 request: &ContractRequest,
3151) -> Result<ContractReport, ApplicationError> {
3152 let store = SqliteStore::open_read_only(database_path)?;
3153 let (nodes, edges) = store.load_current_graph(workspace)?;
3154 let evidence = store.load_current_evidence(workspace)?;
3155 Ok(inspect_contracts(&nodes, &edges, &evidence, &[], request)?)
3156}
3157
3158pub fn export_workspace(
3164 database_path: &Path,
3165 workspace: &str,
3166 request: &ExportRequest,
3167) -> Result<ExportReport, ApplicationError> {
3168 let store = SqliteStore::open_read_only(database_path)?;
3169 let (nodes, edges) = store.load_current_graph(workspace)?;
3170 let evidence = store.load_current_evidence(workspace)?;
3171 Ok(export_graph(&nodes, &edges, &evidence, request)?)
3172}
3173
3174#[expect(
3180 clippy::too_many_lines,
3181 reason = "Doctor keeps every explicit safety observation visible in one conservative report"
3182)]
3183pub fn doctor_workspace(
3184 config_path: &Path,
3185 database_path: &Path,
3186) -> Result<DoctorReport, ApplicationError> {
3187 let status = status_workspace(config_path, database_path)?;
3188 let store = SqliteStore::open_read_only(database_path)?;
3189 let diagnostics = store.diagnostics()?;
3190 let context = load_workspace_context(config_path, &ScanOverrides::default())?;
3191 let expected_version = u32::try_from(latest_schema_version()).map_err(|error| {
3192 ApplicationError::Initialization(format!("unsupported schema version: {error}"))
3193 })?;
3194 let actual_version = u32::try_from(status.schema_version).ok();
3195 let freshness = status
3196 .repositories
3197 .iter()
3198 .map(|repository| FreshnessDoctorInput {
3199 repo_id: repository.repo_id.clone(),
3200 state: repository.state,
3201 detail: repository.reason.clone(),
3202 })
3203 .collect();
3204 let repository_paths_valid = context
3205 .registry
3206 .record
3207 .repositories
3208 .iter()
3209 .all(|repository| {
3210 context
3211 .registry
3212 .checkout_path(&repository.alias)
3213 .is_some_and(Path::is_dir)
3214 });
3215 let git_available = std::process::Command::new("git")
3216 .arg("--version")
3217 .stdin(std::process::Stdio::null())
3218 .stdout(std::process::Stdio::null())
3219 .stderr(std::process::Stdio::null())
3220 .status()
3221 .is_ok_and(|status| status.success());
3222 Ok(doctor(&DoctorRequest {
3223 schema: vec![SchemaDoctorInput {
3224 name: "sqlite".to_owned(),
3225 expected_version,
3226 actual_version,
3227 metadata_consistent: Some(actual_version == Some(expected_version)),
3228 }],
3229 integrity: vec![
3230 IntegrityDoctorInput {
3231 name: "binary-version".to_owned(),
3232 passed: Some(true),
3233 detail: Some(env!("CARGO_PKG_VERSION").to_owned()),
3234 },
3235 IntegrityDoctorInput {
3236 name: "sqlite".to_owned(),
3237 passed: Some(status.integrity_ok),
3238 detail: None,
3239 },
3240 IntegrityDoctorInput {
3241 name: "sqlite-wal".to_owned(),
3242 passed: Some(diagnostics.journal_mode.eq_ignore_ascii_case("wal")),
3243 detail: Some(diagnostics.journal_mode),
3244 },
3245 IntegrityDoctorInput {
3246 name: "sqlite-foreign-keys".to_owned(),
3247 passed: Some(diagnostics.foreign_keys_enabled),
3248 detail: None,
3249 },
3250 IntegrityDoctorInput {
3251 name: "sqlite-fts5".to_owned(),
3252 passed: Some(diagnostics.fts5_index_available),
3253 detail: None,
3254 },
3255 IntegrityDoctorInput {
3256 name: "repository-paths".to_owned(),
3257 passed: Some(repository_paths_valid),
3258 detail: None,
3259 },
3260 IntegrityDoctorInput {
3261 name: "git".to_owned(),
3262 passed: Some(git_available),
3263 detail: None,
3264 },
3265 IntegrityDoctorInput {
3266 name: "extractor-matrix".to_owned(),
3267 passed: Some(true),
3268 detail: Some(
3269 "Built-in extractor inventory is compiled into this binary.".to_owned(),
3270 ),
3271 },
3272 IntegrityDoctorInput {
3273 name: "network-policy".to_owned(),
3274 passed: Some(true),
3275 detail: Some(
3276 "Remote providers and non-loopback HTTP remain explicitly gated.".to_owned(),
3277 ),
3278 },
3279 IntegrityDoctorInput {
3280 name: "writer-lock".to_owned(),
3281 passed: None,
3282 detail: Some(
3283 "Read-only doctor does not acquire or reclaim the workspace writer lock."
3284 .to_owned(),
3285 ),
3286 },
3287 IntegrityDoctorInput {
3288 name: "hook-installation".to_owned(),
3289 passed: None,
3290 detail: Some(
3291 "Use `csgraph hooks status` with an explicit host and repository root."
3292 .to_owned(),
3293 ),
3294 },
3295 IntegrityDoctorInput {
3296 name: "release-files".to_owned(),
3297 passed: None,
3298 detail: Some(
3299 "Release artifact notices and checksums require an explicit installation path."
3300 .to_owned(),
3301 ),
3302 },
3303 ],
3304 freshness,
3305 providers: vec![ProviderDoctorInput {
3306 name: "codegraph".to_owned(),
3307 status: ProviderDoctorStatus::Unavailable,
3308 detail: Some(
3309 "CodeGraph probing is opt-in and was not requested by this doctor invocation."
3310 .to_owned(),
3311 ),
3312 }],
3313 config: vec![ConfigDoctorInput {
3314 name: status.workspace,
3315 valid: Some(true),
3316 detail: None,
3317 }],
3318 }))
3319}
3320
3321pub fn create_diagnostic_bundle(
3331 config_path: &Path,
3332 database_path: &Path,
3333 output_path: &Path,
3334) -> Result<DiagnosticBundle, ApplicationError> {
3335 let generated_at_unix_ms = SystemTime::now()
3336 .duration_since(UNIX_EPOCH)
3337 .map_err(|error| ApplicationError::Initialization(error.to_string()))?
3338 .as_millis();
3339 let bundle = DiagnosticBundle {
3340 schema_version: 1,
3341 binary_version: env!("CARGO_PKG_VERSION").to_owned(),
3342 operating_system: std::env::consts::OS.to_owned(),
3343 architecture: std::env::consts::ARCH.to_owned(),
3344 generated_at_unix_ms,
3345 debug_requested: std::env::var("CODE_SYSTEM_GRAPH_DEBUG")
3346 .is_ok_and(|value| value.trim() == "1"),
3347 doctor: doctor_workspace(config_path, database_path)?,
3348 };
3349
3350 let mut options = OpenOptions::new();
3351 options.write(true).create_new(true);
3352 #[cfg(unix)]
3353 {
3354 use std::os::unix::fs::OpenOptionsExt;
3355
3356 options.mode(0o600);
3357 }
3358 let mut output = options
3359 .open(output_path)
3360 .map_err(|source| ApplicationError::WriteFile {
3361 path: output_path.to_path_buf(),
3362 source,
3363 })?;
3364 serde_json::to_writer_pretty(&mut output, &bundle)
3365 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
3366 output
3367 .write_all(b"\n")
3368 .and_then(|()| output.sync_all())
3369 .map_err(|source| ApplicationError::WriteFile {
3370 path: output_path.to_path_buf(),
3371 source,
3372 })?;
3373 Ok(bundle)
3374}
3375
3376pub fn backup_database(
3382 database_path: &Path,
3383 backup_path: &Path,
3384) -> Result<BackupSummary, ApplicationError> {
3385 SqliteStore::backup_file(database_path, backup_path)?;
3386 Ok(BackupSummary {
3387 database: database_path.to_string_lossy().into_owned(),
3388 backup: backup_path.to_string_lossy().into_owned(),
3389 })
3390}
3391
3392pub fn restore_database(
3398 database_path: &Path,
3399 backup_path: &Path,
3400) -> Result<RestoreSummary, ApplicationError> {
3401 let report = SqliteStore::restore_from(database_path, backup_path)?;
3402 Ok(RestoreSummary {
3403 source: report.source_path.to_string_lossy().into_owned(),
3404 safety_backup: report
3405 .safety_backup_path
3406 .map(|path| path.to_string_lossy().into_owned()),
3407 schema_version: report.schema_version,
3408 })
3409}
3410
3411pub fn list_workspace_registry(
3417 database_path: &Path,
3418) -> Result<Vec<WorkspaceRegistryItem>, ApplicationError> {
3419 let store = SqliteStore::open_read_only(database_path)?;
3420 Ok(store
3421 .list_workspaces()?
3422 .into_iter()
3423 .map(|workspace| WorkspaceRegistryItem {
3424 id: workspace.id.as_str().to_owned(),
3425 name: workspace.name,
3426 manifest_hash: workspace.manifest_hash,
3427 config_path: workspace.config_path.map(|path| path.display),
3428 repository_count: workspace.repository_count,
3429 })
3430 .collect())
3431}
3432
3433pub fn add_workspace_to_registry(
3440 database_path: &Path,
3441 workspace_name: &str,
3442 config_path: &Path,
3443) -> Result<WorkspaceMutationSummary, ApplicationError> {
3444 let context = load_workspace_context(config_path, &ScanOverrides::default())?;
3445 if context.manifest.name != workspace_name {
3446 return Err(ApplicationError::WorkspaceNameMismatch {
3447 requested: workspace_name.to_owned(),
3448 manifest: context.manifest.name,
3449 });
3450 }
3451 let _lock = StoreLock::acquire(database_path, Duration::from_mins(5))?;
3452 let mut store = SqliteStore::open(database_path)?;
3453 if store.workspace_exists(workspace_name)? {
3454 return Err(ApplicationError::WorkspaceAlreadyExists(
3455 workspace_name.to_owned(),
3456 ));
3457 }
3458 store.save_workspace_registry(&context.registry.record)?;
3459 Ok(WorkspaceMutationSummary {
3460 workspace: workspace_name.to_owned(),
3461 config_path: context.registry.record.config_path.map(|path| path.display),
3462 repository_count: context.registry.record.repositories.len(),
3463 operation: "add".to_owned(),
3464 })
3465}
3466
3467pub fn remove_workspace_from_registry(
3473 database_path: &Path,
3474 workspace_name: &str,
3475) -> Result<WorkspaceMutationSummary, ApplicationError> {
3476 let _lock = StoreLock::acquire(database_path, Duration::from_mins(5))?;
3477 let mut store = SqliteStore::open(database_path)?;
3478 let repository_count = store
3479 .load_workspace_registry(workspace_name)
3480 .map_err(|error| match error {
3481 StoreError::RegistryIncomplete(_) => {
3482 ApplicationError::WorkspaceNotFound(workspace_name.to_owned())
3483 }
3484 other => ApplicationError::Store(other),
3485 })?
3486 .repositories
3487 .len();
3488 if !store.remove_workspace(workspace_name)? {
3489 return Err(ApplicationError::WorkspaceNotFound(
3490 workspace_name.to_owned(),
3491 ));
3492 }
3493 Ok(WorkspaceMutationSummary {
3494 workspace: workspace_name.to_owned(),
3495 config_path: None,
3496 repository_count,
3497 operation: "remove".to_owned(),
3498 })
3499}
3500
3501pub fn list_repository_registry(
3507 database_path: &Path,
3508 workspace: &str,
3509) -> Result<Vec<RepositoryRecord>, ApplicationError> {
3510 let store = SqliteStore::open_read_only(database_path)?;
3511 Ok(store.load_workspace_registry(workspace)?.repositories)
3512}
3513
3514pub fn add_repository_to_manifest(
3520 manifest_path: &Path,
3521 alias: &str,
3522 repository_path: &str,
3523 dry_run: bool,
3524) -> Result<ManifestMutationSummary, ApplicationError> {
3525 let source = read_file(manifest_path)?;
3526 let edit = preview_add_repository(&source, alias, repository_path)?;
3527 validate_manifest_edit(manifest_path, &edit)?;
3528 finish_manifest_edit(manifest_path, edit, dry_run)
3529}
3530
3531pub fn remove_repository_from_manifest(
3537 manifest_path: &Path,
3538 alias: &str,
3539 dry_run: bool,
3540) -> Result<ManifestMutationSummary, ApplicationError> {
3541 let source = read_file(manifest_path)?;
3542 let edit = preview_remove_repository(&source, alias)?;
3543 validate_manifest_edit(manifest_path, &edit)?;
3544 finish_manifest_edit(manifest_path, edit, dry_run)
3545}
3546
3547pub fn add_manual_link_to_manifest(
3554 manifest_path: &Path,
3555 link: &ManualLinkConfig,
3556 dry_run: bool,
3557) -> Result<ManifestMutationSummary, ApplicationError> {
3558 let source = read_file(manifest_path)?;
3559 let edit = preview_add_manual_link(&source, link)?;
3560 validate_manifest_edit(manifest_path, &edit)?;
3561 finish_manifest_edit(manifest_path, edit, dry_run)
3562}
3563
3564fn validate_manifest_edit(
3565 manifest_path: &Path,
3566 edit: &ManifestEdit,
3567) -> Result<(), ApplicationError> {
3568 let manifest = parse_manifest(&edit.updated_source)?;
3569 register_workspace(manifest_path, &edit.updated_source, &manifest)?;
3570 Ok(())
3571}
3572
3573fn finish_manifest_edit(
3574 manifest_path: &Path,
3575 edit: ManifestEdit,
3576 dry_run: bool,
3577) -> Result<ManifestMutationSummary, ApplicationError> {
3578 if dry_run {
3579 return Ok(ManifestMutationSummary {
3580 summary: edit.summary,
3581 manifest: manifest_path.to_string_lossy().into_owned(),
3582 backup: None,
3583 applied: false,
3584 rendered_manifest: Some(edit.updated_source),
3585 });
3586 }
3587 let report = commit_manifest_edit(manifest_path, &edit)?;
3588 Ok(ManifestMutationSummary {
3589 summary: edit.summary,
3590 manifest: report.manifest_path.to_string_lossy().into_owned(),
3591 backup: Some(report.backup_path.to_string_lossy().into_owned()),
3592 applied: true,
3593 rendered_manifest: None,
3594 })
3595}
3596
3597fn evaluate_repository_freshness(
3598 repository: &code_system_graph_model::RepositoryRecord,
3599 previous: Option<&RepoFreshness>,
3600 current_manifest_hash: &str,
3601) -> (RepoFreshnessState, Option<String>) {
3602 if repository.working_tree_dirty {
3603 return (
3604 RepoFreshnessState::WorkingTreeChanged,
3605 Some("Git working tree has tracked or untracked changes.".to_owned()),
3606 );
3607 }
3608 let Some(previous) = previous else {
3609 return (
3610 RepoFreshnessState::Unknown,
3611 Some("Checkout was not observed in the published snapshot.".to_owned()),
3612 );
3613 };
3614 if previous.manifest_hash != current_manifest_hash {
3615 return (
3616 RepoFreshnessState::ConfigChanged,
3617 Some("Workspace manifest changed after the published snapshot.".to_owned()),
3618 );
3619 }
3620 if previous.head_commit != repository.head_commit {
3621 return (
3622 RepoFreshnessState::CommitsBehind,
3623 Some("Repository HEAD changed after the published snapshot.".to_owned()),
3624 );
3625 }
3626 (previous.state, previous.reason.clone())
3627}
3628
3629fn freshness_summary(repositories: &[RepoFreshness]) -> FreshnessSummary {
3630 let overall = if repositories
3631 .iter()
3632 .all(|freshness| freshness.state == RepoFreshnessState::Fresh)
3633 {
3634 OverallFreshness::Fresh
3635 } else if repositories.iter().any(|freshness| {
3636 matches!(
3637 freshness.state,
3638 RepoFreshnessState::Unknown
3639 | RepoFreshnessState::Unavailable
3640 | RepoFreshnessState::Corrupt
3641 )
3642 }) {
3643 OverallFreshness::Unknown
3644 } else if repositories
3645 .iter()
3646 .any(|freshness| freshness.state == RepoFreshnessState::Partial)
3647 {
3648 OverallFreshness::Partial
3649 } else {
3650 OverallFreshness::Stale
3651 };
3652 FreshnessSummary {
3653 overall,
3654 stale_repositories: repositories
3655 .iter()
3656 .filter(|freshness| freshness.state != RepoFreshnessState::Fresh)
3657 .map(|freshness| freshness.repo_id.clone())
3658 .collect(),
3659 reasons: repositories
3660 .iter()
3661 .filter_map(|freshness| freshness.reason.clone())
3662 .collect(),
3663 }
3664}
3665
3666fn load_workspace_context(
3667 config_path: &Path,
3668 overrides: &ScanOverrides,
3669) -> Result<WorkspaceContext, ApplicationError> {
3670 let manifest_source = read_file(config_path)?;
3671 let manifest = parse_manifest(&manifest_source)?;
3672 validate_global_policy_source(config_path, &manifest)?;
3673 let extraction_budgets = ExtractionBudgets::resolve(manifest.extraction_budgets.as_ref())
3674 .map_err(ManifestError::from)?;
3675 let execution_policy = ExecutionPolicy::resolve(manifest.execution_policy.as_ref())
3676 .map_err(ManifestError::from)?;
3677 for alias in overrides.repo_openapi.keys() {
3678 if !manifest.repos.contains_key(alias) {
3679 return Err(ApplicationError::UnknownOverrideRepository(alias.clone()));
3680 }
3681 }
3682 let mut registry = register_workspace(config_path, &manifest_source, &manifest)?;
3683 let mut repository_configs = BTreeMap::new();
3684 let mut semantic_manifest = manifest.clone();
3685 semantic_manifest.execution_policy = None;
3686 let mut fingerprint_material = serde_json::to_string(&semantic_manifest)
3687 .map_err(|error| ApplicationError::Initialization(error.to_string()))?;
3688 for (alias, repository) in &manifest.repos {
3689 let checkout_path = registry
3690 .checkout_path(alias)
3691 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
3692 let mut effective = resolve_repository_config(checkout_path, repository)?;
3693 if let Some(openapi) = overrides.repo_openapi.get(alias) {
3694 apply_openapi_override(&mut effective, openapi)?;
3695 }
3696 fingerprint_material.push('\n');
3697 fingerprint_material.push_str(alias);
3698 fingerprint_material.push(':');
3699 fingerprint_material.push_str(&effective.fingerprint);
3700 repository_configs.insert(alias.clone(), effective);
3701 }
3702 registry.record.manifest_hash = stable_id("manifest", &fingerprint_material);
3703 Ok(WorkspaceContext {
3704 manifest,
3705 registry,
3706 repository_configs,
3707 extraction_budgets,
3708 execution_policy,
3709 })
3710}
3711
3712fn validate_global_policy_source(
3713 config_path: &Path,
3714 manifest: &WorkspaceManifest,
3715) -> Result<(), ApplicationError> {
3716 if manifest.extraction_budgets.is_none() && manifest.execution_policy.is_none() {
3717 return Ok(());
3718 }
3719 let canonical_config =
3720 fs::canonicalize(config_path).map_err(|source| ApplicationError::ReadFile {
3721 path: config_path.to_path_buf(),
3722 source,
3723 })?;
3724 let base = canonical_config.parent().ok_or_else(|| {
3725 ApplicationError::Initialization(format!(
3726 "workspace manifest `{}` has no parent directory",
3727 canonical_config.display()
3728 ))
3729 })?;
3730 for (alias, repository) in &manifest.repos {
3731 let configured = Path::new(&repository.path);
3732 let candidate = if configured.is_absolute() {
3733 configured.to_path_buf()
3734 } else {
3735 base.join(configured)
3736 };
3737 let checkout =
3738 fs::canonicalize(&candidate).map_err(|source| ApplicationError::ReadFile {
3739 path: candidate.clone(),
3740 source,
3741 })?;
3742 if canonical_config.starts_with(&checkout) {
3743 return Err(ApplicationError::UntrustedGlobalPolicySource {
3744 config: canonical_config,
3745 repository: alias.clone(),
3746 });
3747 }
3748 }
3749 Ok(())
3750}
3751
3752fn focused_batch_cache_complete(
3753 fingerprints: &[ArtifactFingerprint],
3754 stored: &[StoredExtractorBatch],
3755 budgets: &ExtractionBudgets,
3756) -> bool {
3757 let budget_fingerprint = budgets.fingerprint();
3758 fingerprints
3759 .iter()
3760 .filter(|fingerprint| focused_extractor(&fingerprint.extractor))
3761 .all(|fingerprint| {
3762 stored.iter().any(|batch| {
3763 batch.source == *fingerprint
3764 && batch.extractor_version == EXTRACTION_CONTRACT_VERSION
3765 && batch.budget_fingerprint == budget_fingerprint
3766 })
3767 })
3768}
3769
3770fn stored_batch_degradations(
3771 stored: &[StoredExtractorBatch],
3772 budgets: &ExtractionBudgets,
3773) -> Result<Vec<String>, ApplicationError> {
3774 let mut degradations = Vec::new();
3775 for batch in stored {
3776 if batch.source_was_lossy {
3777 degradations.push(format!("{} contains invalid UTF-8 and was decoded lossily; extracted evidence is incomplete", batch.source.path.display));
3778 }
3779 if batch.source.extractor == "code-system-graph.data.artifact" {
3780 let decoded: ExtractorBatch<DataDocument> =
3781 load_extractor_batch_with_budgets(batch, budgets)?;
3782 for document in decoded.outputs {
3783 if document.incomplete {
3784 degradations.push(format!(
3785 "{} data extraction is incomplete: {:?}",
3786 batch.source.path.display, document.warnings
3787 ));
3788 }
3789 }
3790 }
3791 }
3792 Ok(degradations)
3793}
3794
3795fn finalize_scan_degradations(mut degradations: Vec<String>) -> (usize, Vec<String>) {
3796 degradations.sort();
3797 degradations.dedup();
3798 let count = degradations.len();
3799 if count > MAX_SCAN_DEGRADATIONS {
3800 degradations.truncate(MAX_SCAN_DEGRADATIONS - 1);
3801 degradations.push(format!(
3802 "{} additional degradations omitted",
3803 count - (MAX_SCAN_DEGRADATIONS - 1)
3804 ));
3805 }
3806 (count, degradations)
3807}
3808
3809#[expect(
3810 clippy::too_many_lines,
3811 reason = "Batch decode, reuse, extraction, and persistence share one fail-closed type boundary"
3812)]
3813fn assemble_focused_batches(
3814 context: &WorkspaceContext,
3815 fingerprints: &[ArtifactFingerprint],
3816 previous: &[StoredExtractorBatch],
3817 plan: &ExtractorBatchPlan,
3818 cached_keys: &BTreeSet<ArtifactKey>,
3819 work_state: &mut work_state::WorkState,
3820) -> Result<FocusedBatchState, ApplicationError> {
3821 let previous_by_key = previous
3822 .iter()
3823 .map(|batch| (ArtifactKey::from(&batch.source), batch))
3824 .collect::<BTreeMap<_, _>>();
3825 let planned_actions = plan
3826 .batches
3827 .iter()
3828 .map(|batch| (batch.key.clone(), batch.action))
3829 .collect::<BTreeMap<_, _>>();
3830 let mut previous_source_batches = previous
3831 .iter()
3832 .filter(|batch| source_extractor(&batch.source.extractor))
3833 .map(|batch| load_extractor_batch_with_budgets(batch, &context.extraction_budgets))
3834 .collect::<Result<Vec<ExtractorBatch<SourceObservation>>, _>>()?;
3835 previous_source_batches.sort_by_key(ExtractorBatch::key);
3836
3837 let checkouts = context
3838 .registry
3839 .record
3840 .repositories
3841 .iter()
3842 .filter_map(|repository| {
3843 context
3844 .registry
3845 .checkout_path(&repository.alias)
3846 .map(|path| (repository.id.clone(), path))
3847 })
3848 .collect::<BTreeMap<_, _>>();
3849 let mut source_batches = Vec::new();
3850 let mut package_batches = Vec::new();
3851 let mut generated_client_batches = Vec::new();
3852 let mut graphql_batches = Vec::new();
3853 let mut event_batches = Vec::new();
3854 let mut protobuf_batches = Vec::new();
3855 let mut data_batches = Vec::new();
3856 let mut infrastructure_batches = Vec::new();
3857 let mut documentation_batches = Vec::new();
3858 let mut config_batches = Vec::new();
3859 let mut stored_batches = Vec::new();
3860 let mut degradations = Vec::new();
3861 let mut force_relink = false;
3862 let mut checkpoint_writes = 0_u64;
3863 let mut artifact_durations_ms = Vec::new();
3864 for fingerprint in fingerprints
3865 .iter()
3866 .filter(|fingerprint| focused_extractor(&fingerprint.extractor))
3867 {
3868 let artifact_started = Instant::now();
3869 let key = ArtifactKey::from(fingerprint);
3870 let reusable = previous_by_key.get(&key).copied().filter(|batch| {
3871 (planned_actions.get(&key) == Some(&BatchAction::Reuse) || cached_keys.contains(&key))
3872 && batch.source.content_hash == fingerprint.content_hash
3873 && batch.extractor_version == EXTRACTION_CONTRACT_VERSION
3874 && batch.budget_fingerprint == context.extraction_budgets.fingerprint()
3875 });
3876 if let Some(stored) = reusable {
3877 stored_batches.push(stored.clone());
3878 if source_extractor(&fingerprint.extractor) {
3879 source_batches.push(load_extractor_batch_with_budgets(
3880 stored,
3881 &context.extraction_budgets,
3882 )?);
3883 } else if fingerprint.extractor == "code-system-graph.packages" {
3884 package_batches.push(load_extractor_batch_with_budgets(
3885 stored,
3886 &context.extraction_budgets,
3887 )?);
3888 } else if graphql_extractor(&fingerprint.extractor) {
3889 graphql_batches.push(load_extractor_batch_with_budgets(
3890 stored,
3891 &context.extraction_budgets,
3892 )?);
3893 } else if event_extractor(&fingerprint.extractor) {
3894 event_batches.push(load_extractor_batch_with_budgets(
3895 stored,
3896 &context.extraction_budgets,
3897 )?);
3898 } else if protobuf_extractor(&fingerprint.extractor) {
3899 protobuf_batches.push(load_extractor_batch_with_budgets(
3900 stored,
3901 &context.extraction_budgets,
3902 )?);
3903 } else if data_extractor(&fingerprint.extractor) {
3904 data_batches.push(load_extractor_batch_with_budgets(
3905 stored,
3906 &context.extraction_budgets,
3907 )?);
3908 } else if infrastructure_extractor(&fingerprint.extractor) {
3909 infrastructure_batches.push(load_extractor_batch_with_budgets(
3910 stored,
3911 &context.extraction_budgets,
3912 )?);
3913 } else if documentation_extractor(&fingerprint.extractor) {
3914 documentation_batches.push(load_extractor_batch_with_budgets(
3915 stored,
3916 &context.extraction_budgets,
3917 )?);
3918 } else if fingerprint.extractor == "code-system-graph.config.safe" {
3919 config_batches.push(load_extractor_batch_with_budgets(
3920 stored,
3921 &context.extraction_budgets,
3922 )?);
3923 } else {
3924 generated_client_batches.push(load_extractor_batch_with_budgets(
3925 stored,
3926 &context.extraction_budgets,
3927 )?);
3928 }
3929 worker::report_progress(code_system_graph_core::JobPhase::Extraction, 1);
3930 artifact_durations_ms.push(duration_millis(artifact_started.elapsed()));
3931 continue;
3932 }
3933 if previous_by_key.contains_key(&key) {
3934 force_relink = true;
3935 }
3936 let checkout = checkouts.get(&fingerprint.repo_id).ok_or_else(|| {
3937 ApplicationError::RegistryAliasMissing(fingerprint.repo_id.as_str().to_owned())
3938 })?;
3939 let relative_path = native_relative_path(&fingerprint.path);
3940 let artifact_path = checkout.join(&relative_path);
3941 let mut tracker = ExtractionTracker::new(
3942 &fingerprint.path.display,
3943 &fingerprint.extractor,
3944 &context.extraction_budgets,
3945 );
3946 let (source, source_was_lossy) = read_source_file(&artifact_path, &mut tracker)?;
3947 if source_was_lossy {
3948 degradations.push(format!("{} contains invalid UTF-8 and was decoded lossily; extracted evidence is incomplete", fingerprint.path.display));
3949 }
3950 let mut persist = |stored: StoredExtractorBatch| -> Result<(), ApplicationError> {
3951 if work_state
3952 .put_batch(
3953 &stored,
3954 context.execution_policy.max_checkpoint_cache_bytes,
3955 current_unix_millis(),
3956 )
3957 .map_err(ApplicationError::Initialization)?
3958 {
3959 checkpoint_writes = checkpoint_writes.checked_add(1).ok_or_else(|| {
3960 ApplicationError::Initialization(
3961 "checkpoint write counter overflowed".to_owned(),
3962 )
3963 })?;
3964 }
3965 stored_batches.push(stored);
3966 Ok(())
3967 };
3968 if source_extractor(&fingerprint.extractor) {
3969 let syntax = inspect_source_syntax(
3970 source_syntax_language(&fingerprint.extractor),
3971 &portable_path(&fingerprint.path.display),
3972 &source,
3973 &mut tracker,
3974 )?;
3975 let reserved_observations =
3976 u64::try_from(syntax.boundary_candidate_count).map_err(|_| {
3977 ApplicationError::InvalidSourceObservation(format!(
3978 "{} contains too many syntax candidates",
3979 fingerprint.path.display
3980 ))
3981 })?;
3982 tracker.charge_work(reserved_observations)?;
3983 precheck_focused_source_values(&source, &mut tracker)?;
3984 let mut observations = match fingerprint.extractor.as_str() {
3985 "code-system-graph.source.javascript" => {
3986 parse_javascript_source_at_path_with_tracker(
3987 &portable_path(&fingerprint.path.display),
3988 &source,
3989 &mut tracker,
3990 )?
3991 }
3992 "code-system-graph.source.typescript" => {
3993 parse_typescript_source_at_path_with_tracker(
3994 &portable_path(&fingerprint.path.display),
3995 &source,
3996 &mut tracker,
3997 )?
3998 }
3999 "code-system-graph.source.rust" => {
4000 parse_rust_source_with_tracker(&source, &mut tracker)?
4001 }
4002 "code-system-graph.source.python" => {
4003 parse_python_source_with_tracker(&source, &mut tracker)?
4004 }
4005 "code-system-graph.source.go" => {
4006 parse_go_source_with_tracker(&source, &mut tracker)?
4007 }
4008 "code-system-graph.source.java" => {
4009 parse_java_source_with_tracker(&source, &mut tracker)?
4010 }
4011 _ => Vec::new(),
4012 };
4013 if observations
4014 .iter()
4015 .any(|observation| observation.role != SourceRole::Test)
4016 && syntax.boundary_candidate_count == 0
4017 {
4018 return Err(ApplicationError::InvalidSourceObservation(format!(
4019 "{} produced framework facts without a Tree-sitter boundary candidate",
4020 fingerprint.path.display
4021 )));
4022 }
4023 if syntax.has_error {
4024 for observation in &mut observations {
4025 observation.status = SourceEpistemicStatus::Incomplete;
4026 if !observation
4027 .warnings
4028 .contains(&SourceWarning::SyntaxErrorRecovery)
4029 {
4030 observation
4031 .warnings
4032 .push(SourceWarning::SyntaxErrorRecovery);
4033 }
4034 }
4035 }
4036 let batch = ExtractorBatch::new(fingerprint.clone(), observations);
4037 persist(store_extractor_batch(
4038 &batch,
4039 &mut tracker,
4040 source_was_lossy,
4041 )?)?;
4042 source_batches.push(batch);
4043 } else if fingerprint.extractor == "code-system-graph.packages" {
4044 let portable_path = portable_path(&fingerprint.path.display);
4045 let manifest =
4046 extract_package_manifest_with_tracker(&portable_path, &source, &mut tracker)?;
4047 let batch = ExtractorBatch::new(fingerprint.clone(), vec![manifest]);
4048 persist(store_extractor_batch(
4049 &batch,
4050 &mut tracker,
4051 source_was_lossy,
4052 )?)?;
4053 package_batches.push(batch);
4054 } else if graphql_extractor(&fingerprint.extractor) {
4055 let portable_path = portable_path(&fingerprint.path.display);
4056 let document = match fingerprint.extractor.as_str() {
4057 "code-system-graph.graphql.document" => {
4058 extract_graphql_document_with_tracker(&portable_path, &source, &mut tracker)?
4059 }
4060 "code-system-graph.graphql.persisted" => GraphqlDocument {
4061 source_path: portable_path.clone(),
4062 types: Vec::new(),
4063 operations: Vec::new(),
4064 fragments: Vec::new(),
4065 persisted_operations: extract_graphql_persisted_operations_with_tracker(
4066 &portable_path,
4067 &source,
4068 &mut tracker,
4069 )?,
4070 resolvers: Vec::new(),
4071 federation: Vec::new(),
4072 complete: true,
4073 warnings: Vec::new(),
4074 },
4075 "code-system-graph.graphql.source" => {
4076 let language = source_language_for_path(&relative_path).ok_or_else(|| {
4077 ApplicationError::InvalidSourceObservation(format!(
4078 "unsupported GraphQL source language for `{portable_path}`"
4079 ))
4080 })?;
4081 let mut document =
4082 parse_graphql_source_with_tracker(language, &source, &mut tracker)?;
4083 document.source_path = portable_path;
4084 document
4085 }
4086 _ => unreachable!("graphql extractor classification must be exhaustive"),
4087 };
4088 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4089 persist(store_extractor_batch(
4090 &batch,
4091 &mut tracker,
4092 source_was_lossy,
4093 )?)?;
4094 graphql_batches.push(batch);
4095 } else if event_extractor(&fingerprint.extractor) {
4096 let portable_path = portable_path(&fingerprint.path.display);
4097 let document = if fingerprint.extractor == "code-system-graph.events.asyncapi" {
4098 extract_asyncapi(&portable_path, &source)?
4099 } else {
4100 let language = source_language_for_path(&relative_path).ok_or_else(|| {
4101 ApplicationError::InvalidSourceObservation(format!(
4102 "unsupported event source language for `{portable_path}`"
4103 ))
4104 })?;
4105 let mut document = parse_event_source(language, &source);
4106 document.source_path = Some(portable_path);
4107 document
4108 };
4109 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4110 persist(store_extractor_batch(
4111 &batch,
4112 &mut tracker,
4113 source_was_lossy,
4114 )?)?;
4115 event_batches.push(batch);
4116 } else if protobuf_extractor(&fingerprint.extractor) {
4117 let portable_path = portable_path(&fingerprint.path.display);
4118 let document = if fingerprint.extractor == "code-system-graph.protobuf" {
4119 ProtobufDocument::File(Box::new(extract_protobuf_with_tracker(
4120 &portable_path,
4121 &source,
4122 &mut tracker,
4123 )?))
4124 } else {
4125 let language = source_language_for_path(&relative_path).ok_or_else(|| {
4126 ApplicationError::InvalidSourceObservation(format!(
4127 "unsupported generated protobuf source language for `{portable_path}`"
4128 ))
4129 })?;
4130 ProtobufDocument::Generated(parse_protobuf_generated_source(
4131 language,
4132 &portable_path,
4133 &source,
4134 ))
4135 };
4136 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4137 persist(store_extractor_batch(
4138 &batch,
4139 &mut tracker,
4140 source_was_lossy,
4141 )?)?;
4142 protobuf_batches.push(batch);
4143 } else if data_extractor(&fingerprint.extractor) {
4144 let portable_path = portable_path(&fingerprint.path.display);
4145 let document = if fingerprint.extractor == "code-system-graph.data.source" {
4146 let language = source_language_for_path(&relative_path).ok_or_else(|| {
4147 ApplicationError::InvalidSourceObservation(format!(
4148 "unsupported data source language for `{portable_path}`"
4149 ))
4150 })?;
4151 let crate_root = cargo_crate_root(checkout, &artifact_path);
4152 parse_literal_sql_source_at_root(language, &portable_path, &crate_root, &source)
4153 } else {
4154 extract_data_artifact(&portable_path, &source)?
4155 };
4156 if fingerprint.extractor == "code-system-graph.data.artifact" && document.incomplete {
4157 degradations.push(format!(
4158 "{} data extraction is incomplete: {:?}",
4159 fingerprint.path.display, document.warnings
4160 ));
4161 }
4162 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4163 persist(store_extractor_batch(
4164 &batch,
4165 &mut tracker,
4166 source_was_lossy,
4167 )?)?;
4168 data_batches.push(batch);
4169 } else if infrastructure_extractor(&fingerprint.extractor) {
4170 let portable_path = portable_path(&fingerprint.path.display);
4171 let document = match fingerprint.extractor.as_str() {
4172 "code-system-graph.infrastructure.compose" => {
4173 extract_docker_compose(&portable_path, &source)?
4174 }
4175 "code-system-graph.infrastructure.kubernetes" => {
4176 extract_kubernetes(&portable_path, &source)?
4177 }
4178 "code-system-graph.infrastructure.helm" => extract_helm(&portable_path, &source)?,
4179 "code-system-graph.infrastructure.terraform" => {
4180 extract_terraform(&portable_path, &source)?
4181 }
4182 _ => unreachable!("infrastructure extractor classification must be exhaustive"),
4183 };
4184 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4185 persist(store_extractor_batch(
4186 &batch,
4187 &mut tracker,
4188 source_was_lossy,
4189 )?)?;
4190 infrastructure_batches.push(batch);
4191 } else if documentation_extractor(&fingerprint.extractor) {
4192 let portable_path = portable_path(&fingerprint.path.display);
4193 let document = match fingerprint.extractor.as_str() {
4194 "code-system-graph.documents.markdown" => {
4195 extract_markdown(&portable_path, &source)?
4196 }
4197 "code-system-graph.documents.codeowners" => {
4198 extract_codeowners(&portable_path, &source)?
4199 }
4200 "code-system-graph.documents.catalog" => {
4201 extract_service_catalog(&portable_path, &source)?
4202 }
4203 _ => unreachable!("documentation extractor classification must be exhaustive"),
4204 };
4205 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4206 persist(store_extractor_batch(
4207 &batch,
4208 &mut tracker,
4209 source_was_lossy,
4210 )?)?;
4211 documentation_batches.push(batch);
4212 } else if fingerprint.extractor == "code-system-graph.config.safe" {
4213 let portable_path = portable_path(&fingerprint.path.display);
4214 let document = extract_safe_config(&portable_path, &source)?;
4215 let batch = ExtractorBatch::new(fingerprint.clone(), vec![document]);
4216 persist(store_extractor_batch(
4217 &batch,
4218 &mut tracker,
4219 source_was_lossy,
4220 )?)?;
4221 config_batches.push(batch);
4222 } else {
4223 let portable_path = portable_path(&fingerprint.path.display);
4224 let metadata =
4225 extract_generated_client_metadata(&portable_path, &source, &mut tracker)?;
4226 let batch = ExtractorBatch::new(fingerprint.clone(), metadata);
4227 persist(store_extractor_batch(
4228 &batch,
4229 &mut tracker,
4230 source_was_lossy,
4231 )?)?;
4232 generated_client_batches.push(batch);
4233 }
4234 worker::report_progress(code_system_graph_core::JobPhase::Extraction, 1);
4235 artifact_durations_ms.push(duration_millis(artifact_started.elapsed()));
4236 }
4237 source_batches.sort_by_key(ExtractorBatch::key);
4238 package_batches.sort_by_key(ExtractorBatch::key);
4239 generated_client_batches.sort_by_key(ExtractorBatch::key);
4240 graphql_batches.sort_by_key(ExtractorBatch::key);
4241 event_batches.sort_by_key(ExtractorBatch::key);
4242 protobuf_batches.sort_by_key(ExtractorBatch::key);
4243 data_batches.sort_by_key(ExtractorBatch::key);
4244 infrastructure_batches.sort_by_key(ExtractorBatch::key);
4245 documentation_batches.sort_by_key(ExtractorBatch::key);
4246 config_batches.sort_by_key(ExtractorBatch::key);
4247 stored_batches.sort_by(|left, right| {
4248 ArtifactKey::from(&left.source).cmp(&ArtifactKey::from(&right.source))
4249 });
4250 Ok(FocusedBatchState {
4251 source_batches,
4252 previous_source_batches,
4253 package_batches,
4254 generated_client_batches,
4255 graphql_batches,
4256 event_batches,
4257 protobuf_batches,
4258 data_batches,
4259 infrastructure_batches,
4260 documentation_batches,
4261 config_batches,
4262 stored_batches,
4263 degradations,
4264 force_relink,
4265 checkpoint_writes,
4266 artifact_durations_ms,
4267 })
4268}
4269
4270fn focused_extractor(extractor: &str) -> bool {
4271 source_extractor(extractor)
4272 || matches!(
4273 extractor,
4274 "code-system-graph.packages" | "code-system-graph.http.generated-client"
4275 )
4276 || graphql_extractor(extractor)
4277 || event_extractor(extractor)
4278 || protobuf_extractor(extractor)
4279 || data_extractor(extractor)
4280 || infrastructure_extractor(extractor)
4281 || documentation_extractor(extractor)
4282 || extractor == "code-system-graph.config.safe"
4283}
4284
4285fn graphql_extractor(extractor: &str) -> bool {
4286 matches!(
4287 extractor,
4288 "code-system-graph.graphql.document"
4289 | "code-system-graph.graphql.persisted"
4290 | "code-system-graph.graphql.source"
4291 )
4292}
4293
4294fn event_extractor(extractor: &str) -> bool {
4295 matches!(
4296 extractor,
4297 "code-system-graph.events.asyncapi" | "code-system-graph.events.source"
4298 )
4299}
4300
4301fn protobuf_extractor(extractor: &str) -> bool {
4302 matches!(
4303 extractor,
4304 "code-system-graph.protobuf" | "code-system-graph.protobuf.generated"
4305 )
4306}
4307
4308fn data_extractor(extractor: &str) -> bool {
4309 matches!(
4310 extractor,
4311 "code-system-graph.data.artifact" | "code-system-graph.data.source"
4312 )
4313}
4314
4315fn infrastructure_extractor(extractor: &str) -> bool {
4316 matches!(
4317 extractor,
4318 "code-system-graph.infrastructure.compose"
4319 | "code-system-graph.infrastructure.kubernetes"
4320 | "code-system-graph.infrastructure.helm"
4321 | "code-system-graph.infrastructure.terraform"
4322 )
4323}
4324
4325fn documentation_extractor(extractor: &str) -> bool {
4326 matches!(
4327 extractor,
4328 "code-system-graph.documents.markdown"
4329 | "code-system-graph.documents.codeowners"
4330 | "code-system-graph.documents.catalog"
4331 )
4332}
4333
4334fn source_extractor(extractor: &str) -> bool {
4335 matches!(
4336 extractor,
4337 "code-system-graph.source.javascript"
4338 | "code-system-graph.source.typescript"
4339 | "code-system-graph.source.rust"
4340 | "code-system-graph.source.python"
4341 | "code-system-graph.source.go"
4342 | "code-system-graph.source.java"
4343 )
4344}
4345
4346fn source_syntax_language(extractor: &str) -> SourceSyntaxLanguage {
4347 match extractor {
4348 "code-system-graph.source.javascript" => SourceSyntaxLanguage::JavaScript,
4349 "code-system-graph.source.typescript" => SourceSyntaxLanguage::TypeScript,
4350 "code-system-graph.source.rust" => SourceSyntaxLanguage::Rust,
4351 "code-system-graph.source.python" => SourceSyntaxLanguage::Python,
4352 "code-system-graph.source.go" => SourceSyntaxLanguage::Go,
4353 "code-system-graph.source.java" => SourceSyntaxLanguage::Java,
4354 _ => unreachable!("source_syntax_language requires a focused source extractor"),
4355 }
4356}
4357
4358fn source_language_for_path(path: &Path) -> Option<SourceLanguage> {
4359 let extension = path.extension()?.to_str()?;
4360 match extension {
4361 "js" | "jsx" => Some(SourceLanguage::JavaScript),
4362 "ts" | "tsx" => Some(SourceLanguage::TypeScript),
4363 "rs" => Some(SourceLanguage::Rust),
4364 "py" => Some(SourceLanguage::Python),
4365 "go" => Some(SourceLanguage::Go),
4366 "java" => Some(SourceLanguage::Java),
4367 _ => None,
4368 }
4369}
4370
4371#[cfg(unix)]
4372fn native_relative_path(path: &code_system_graph_model::NativePath) -> PathBuf {
4373 use std::os::unix::ffi::OsStringExt;
4374
4375 PathBuf::from(std::ffi::OsString::from_vec(path.bytes.clone()))
4376}
4377
4378#[cfg(windows)]
4379fn native_relative_path(path: &code_system_graph_model::NativePath) -> PathBuf {
4380 use std::os::windows::ffi::OsStringExt;
4381
4382 let wide = path
4383 .bytes
4384 .chunks_exact(2)
4385 .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
4386 .collect::<Vec<_>>();
4387 PathBuf::from(std::ffi::OsString::from_wide(&wide))
4388}
4389
4390#[cfg(not(any(unix, windows)))]
4391fn native_relative_path(path: &code_system_graph_model::NativePath) -> PathBuf {
4392 PathBuf::from(String::from_utf8_lossy(&path.bytes).into_owned())
4393}
4394
4395fn generated_client_graph(
4396 batches: &[ExtractorBatch<GeneratedClientMetadata>],
4397 boundaries: &[HttpBoundary],
4398) -> (Vec<Node>, Vec<Edge>, Vec<Evidence>) {
4399 let mut nodes = Vec::new();
4400 let mut edges = Vec::new();
4401 let mut evidence = Vec::new();
4402 for batch in batches {
4403 let source_path = portable_path(&batch.source.path.display);
4404 for metadata in &batch.outputs {
4405 let name = metadata.name.as_deref().unwrap_or("metadata");
4406 let stable_key = format!(
4407 "generated-client:{}:{source_path}:{name}:{}",
4408 batch.source.repo_id.as_str(),
4409 metadata.generated_file.as_deref().unwrap_or("")
4410 );
4411 let node = Node {
4412 id: NodeId::new(stable_id("node", &stable_key)),
4413 kind: NodeKind::Artifact,
4414 repo_id: Some(batch.source.repo_id.clone()),
4415 stable_key: stable_key.clone(),
4416 label: metadata
4417 .generator_name
4418 .as_ref()
4419 .map_or_else(|| format!("{} {name}", metadata.tool), Clone::clone),
4420 };
4421 let item_evidence = Evidence {
4422 id: EvidenceId::new(stable_id("evidence", &stable_key)),
4423 repo_id: Some(batch.source.repo_id.clone()),
4424 file_path: Some(source_path.clone()),
4425 start_line: Some(metadata.line),
4426 end_line: Some(metadata.line),
4427 extractor: "code-system-graph.http.generated-client".to_owned(),
4428 extractor_version: "1.0.0".to_owned(),
4429 provenance: Provenance::Extracted,
4430 confidence: 1.0,
4431 observed_at_commit: None,
4432 content_hash: Some(batch.source.content_hash.clone()),
4433 note: Some("explicit generated-client metadata".to_owned()),
4434 };
4435 if let Some(input_spec) = &metadata.input_spec {
4436 edges.extend(
4437 boundaries
4438 .iter()
4439 .filter(|boundary| {
4440 boundary.node.repo_id.as_ref() == Some(&batch.source.repo_id)
4441 && boundary.evidence.file_path.as_ref() == Some(input_spec)
4442 })
4443 .map(|boundary| {
4444 let key = format!(
4445 "{}:{:?}:{}",
4446 node.id.as_str(),
4447 EdgeKind::Consumes,
4448 boundary.node.id.as_str()
4449 );
4450 Edge {
4451 id: EdgeId::new(stable_id("edge", &key)),
4452 source: node.id.clone(),
4453 target: boundary.node.id.clone(),
4454 kind: EdgeKind::Consumes,
4455 confidence: 1.0,
4456 status: EpistemicStatus::Confirmed,
4457 evidence: vec![item_evidence.id.clone()],
4458 }
4459 }),
4460 );
4461 }
4462 nodes.push(node);
4463 evidence.push(item_evidence);
4464 }
4465 }
4466 nodes.sort_by(|left, right| left.id.cmp(&right.id));
4467 nodes.dedup_by(|left, right| left.id == right.id);
4468 edges.sort_by(|left, right| left.id.cmp(&right.id));
4469 edges.dedup_by(|left, right| left.id == right.id);
4470 evidence.sort_by(|left, right| left.id.cmp(&right.id));
4471 evidence.dedup_by(|left, right| left.id == right.id);
4472 (nodes, edges, evidence)
4473}
4474
4475#[expect(
4476 clippy::too_many_lines,
4477 reason = "Graph assembly keeps every source family in one deterministic merge boundary"
4478)]
4479fn assemble_graph(
4480 context: &WorkspaceContext,
4481 fingerprints: &[ArtifactFingerprint],
4482 focused: &FocusedBatchState,
4483) -> Result<GraphAssembly, ApplicationError> {
4484 let mut boundaries = extract_boundaries(context)?;
4485 let mut tests = extract_declared_tests(context, fingerprints)?;
4486 let mut implementations = extract_declared_implementations(context, fingerprints)?;
4487 let source_facts = focused
4488 .source_batches
4489 .iter()
4490 .map(|batch| {
4491 source_observations_to_graph(
4492 &batch.source.repo_id,
4493 &portable_path(&batch.source.path.display),
4494 &batch.source.content_hash,
4495 &batch.outputs,
4496 )
4497 })
4498 .collect::<Vec<SourceGraphFacts>>();
4499 for facts in &source_facts {
4500 boundaries.extend(facts.boundaries.clone());
4501 tests.extend(facts.tests.clone());
4502 implementations.extend(facts.implementations.clone());
4503 }
4504 let mut package_facts = focused
4505 .package_batches
4506 .iter()
4507 .flat_map(|batch| {
4508 batch.outputs.iter().map(|manifest| {
4509 package_manifest_to_graph(
4510 &batch.source.repo_id,
4511 &portable_path(&batch.source.path.display),
4512 &batch.source.content_hash,
4513 manifest,
4514 )
4515 })
4516 })
4517 .collect::<Vec<PackageGraphFacts>>();
4518 link_registered_package_owners(&mut package_facts);
4519 let (generated_nodes, generated_edges, generated_evidence) =
4520 generated_client_graph(&focused.generated_client_batches, &boundaries);
4521 let graphql_sources = focused
4522 .graphql_batches
4523 .iter()
4524 .flat_map(|batch| {
4525 batch.outputs.iter().map(|document| {
4526 (
4527 batch.source.repo_id.clone(),
4528 portable_path(&batch.source.path.display),
4529 batch.source.content_hash.clone(),
4530 document,
4531 )
4532 })
4533 })
4534 .collect::<Vec<_>>();
4535 let graphql_inputs = graphql_sources
4536 .iter()
4537 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4538 .collect::<Vec<_>>();
4539 let GraphqlGraphFacts {
4540 nodes: graphql_nodes,
4541 edges: graphql_edges,
4542 evidence: graphql_evidence,
4543 } = graphql_documents_to_graph(&graphql_inputs);
4544 let event_sources = focused
4545 .event_batches
4546 .iter()
4547 .flat_map(|batch| {
4548 batch.outputs.iter().map(|document| {
4549 (
4550 batch.source.repo_id.clone(),
4551 portable_path(&batch.source.path.display),
4552 batch.source.content_hash.clone(),
4553 document,
4554 )
4555 })
4556 })
4557 .collect::<Vec<_>>();
4558 let event_inputs = event_sources
4559 .iter()
4560 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4561 .collect::<Vec<_>>();
4562 let EventGraphFacts {
4563 nodes: event_nodes,
4564 edges: event_edges,
4565 evidence: event_evidence,
4566 } = event_documents_to_graph(&event_inputs);
4567 let protobuf_sources = focused
4568 .protobuf_batches
4569 .iter()
4570 .flat_map(|batch| {
4571 batch.outputs.iter().map(|document| {
4572 (
4573 batch.source.repo_id.clone(),
4574 portable_path(&batch.source.path.display),
4575 batch.source.content_hash.clone(),
4576 document,
4577 )
4578 })
4579 })
4580 .collect::<Vec<_>>();
4581 let protobuf_inputs = protobuf_sources
4582 .iter()
4583 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4584 .collect::<Vec<_>>();
4585 let ProtobufGraphFacts {
4586 nodes: protobuf_nodes,
4587 edges: protobuf_edges,
4588 evidence: protobuf_evidence,
4589 } = protobuf_documents_to_graph(&protobuf_inputs);
4590 let data_sources = focused
4591 .data_batches
4592 .iter()
4593 .flat_map(|batch| {
4594 batch.outputs.iter().map(|document| {
4595 (
4596 batch.source.repo_id.clone(),
4597 portable_path(&batch.source.path.display),
4598 batch.source.content_hash.clone(),
4599 document,
4600 )
4601 })
4602 })
4603 .collect::<Vec<_>>();
4604 let data_inputs = data_sources
4605 .iter()
4606 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4607 .collect::<Vec<_>>();
4608 let infrastructure_sources = focused
4609 .infrastructure_batches
4610 .iter()
4611 .flat_map(|batch| {
4612 batch.outputs.iter().map(|document| {
4613 (
4614 batch.source.repo_id.clone(),
4615 portable_path(&batch.source.path.display),
4616 batch.source.content_hash.clone(),
4617 document,
4618 )
4619 })
4620 })
4621 .collect::<Vec<_>>();
4622 let infrastructure_inputs = infrastructure_sources
4623 .iter()
4624 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4625 .collect::<Vec<_>>();
4626 let documentation_sources = focused
4627 .documentation_batches
4628 .iter()
4629 .flat_map(|batch| {
4630 batch.outputs.iter().map(|document| {
4631 (
4632 batch.source.repo_id.clone(),
4633 portable_path(&batch.source.path.display),
4634 batch.source.content_hash.clone(),
4635 document,
4636 )
4637 })
4638 })
4639 .collect::<Vec<_>>();
4640 let documentation_inputs = documentation_sources
4641 .iter()
4642 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4643 .collect::<Vec<_>>();
4644 let config_sources = focused
4645 .config_batches
4646 .iter()
4647 .flat_map(|batch| {
4648 batch.outputs.iter().map(|document| {
4649 (
4650 batch.source.repo_id.clone(),
4651 portable_path(&batch.source.path.display),
4652 batch.source.content_hash.clone(),
4653 document,
4654 )
4655 })
4656 })
4657 .collect::<Vec<_>>();
4658 let config_inputs = config_sources
4659 .iter()
4660 .map(|(repo_id, path, hash, document)| (repo_id, path.as_str(), hash.as_str(), *document))
4661 .collect::<Vec<_>>();
4662 let mut known_nodes = boundaries
4663 .iter()
4664 .map(|boundary| boundary.node.clone())
4665 .chain(tests.iter().map(|test| test.node.clone()))
4666 .chain(
4667 implementations
4668 .iter()
4669 .map(|implementation| implementation.node.clone()),
4670 )
4671 .chain(
4672 source_facts
4673 .iter()
4674 .flat_map(|facts| facts.standalone_test_nodes.iter().cloned()),
4675 )
4676 .chain(
4677 source_facts
4678 .iter()
4679 .flat_map(|facts| facts.relation_nodes.iter().cloned()),
4680 )
4681 .chain(
4682 package_facts
4683 .iter()
4684 .flat_map(|facts| facts.nodes.iter().cloned()),
4685 )
4686 .chain(generated_nodes.iter().cloned())
4687 .chain(graphql_nodes.iter().cloned())
4688 .chain(event_nodes.iter().cloned())
4689 .chain(protobuf_nodes.iter().cloned())
4690 .collect::<Vec<_>>();
4691 known_nodes.sort_by(|left, right| left.id.cmp(&right.id));
4692 known_nodes.dedup_by(|left, right| left.id == right.id);
4693 let repository_aliases = context
4694 .registry
4695 .record
4696 .repositories
4697 .iter()
4698 .map(|repository| (repository.alias.as_str(), &repository.id))
4699 .collect::<Vec<_>>();
4700 let ExtractionGraphFacts {
4701 nodes: extraction_nodes,
4702 edges: extraction_edges,
4703 evidence: extraction_evidence,
4704 } = documents_to_graph(
4705 &data_inputs,
4706 &infrastructure_inputs,
4707 &documentation_inputs,
4708 &config_inputs,
4709 &known_nodes,
4710 &repository_aliases,
4711 );
4712
4713 let mut http_links = link_http_boundaries_with_ambiguities(&boundaries);
4714 let test_links = link_declared_tests_with_ambiguities(&tests, &boundaries);
4715 let implementation_links =
4716 link_declared_implementations_with_ambiguities(&implementations, &boundaries);
4717 http_links.ambiguities.extend(test_links.ambiguities);
4718 http_links
4719 .ambiguities
4720 .extend(implementation_links.ambiguities);
4721 http_links.ambiguities.sort_by(|left, right| {
4722 (&left.method, &left.path, &left.candidates).cmp(&(
4723 &right.method,
4724 &right.path,
4725 &right.candidates,
4726 ))
4727 });
4728 http_links.ambiguities.dedup();
4729 let degradations = http_links
4730 .ambiguities
4731 .iter()
4732 .map(|ambiguity| {
4733 format!(
4734 "ambiguous HTTP provider for {} {}; stable candidates: {}",
4735 ambiguity.method,
4736 ambiguity.path,
4737 ambiguity.candidates.join(", ")
4738 )
4739 })
4740 .collect::<Vec<_>>();
4741 let mut edges = http_links.edges;
4742 edges.extend(test_links.edges);
4743 edges.extend(implementation_links.edges);
4744 edges.extend(
4745 source_facts
4746 .iter()
4747 .flat_map(|facts| facts.relation_edges.iter().cloned()),
4748 );
4749 edges.extend(
4750 package_facts
4751 .iter()
4752 .flat_map(|facts| facts.edges.iter().cloned()),
4753 );
4754 edges.extend(generated_edges);
4755 edges.extend(graphql_edges);
4756 edges.extend(event_edges);
4757 edges.extend(protobuf_edges);
4758 edges.extend(extraction_edges);
4759 edges.sort_by(|left, right| left.id.cmp(&right.id));
4760 edges.dedup_by(|left, right| left.id == right.id);
4761
4762 let mut nodes = boundaries
4763 .iter()
4764 .map(|boundary| (boundary.node.id.clone(), boundary.node.clone()))
4765 .collect::<BTreeMap<_, _>>();
4766 nodes.extend(
4767 tests
4768 .iter()
4769 .map(|test| (test.node.id.clone(), test.node.clone())),
4770 );
4771 nodes.extend(
4772 implementations
4773 .iter()
4774 .map(|implementation| (implementation.node.id.clone(), implementation.node.clone())),
4775 );
4776 nodes.extend(source_facts.iter().flat_map(|facts| {
4777 facts
4778 .standalone_test_nodes
4779 .iter()
4780 .map(|node| (node.id.clone(), node.clone()))
4781 }));
4782 nodes.extend(source_facts.iter().flat_map(|facts| {
4783 facts
4784 .relation_nodes
4785 .iter()
4786 .map(|node| (node.id.clone(), node.clone()))
4787 }));
4788 nodes.extend(package_facts.iter().flat_map(|facts| {
4789 facts
4790 .nodes
4791 .iter()
4792 .map(|node| (node.id.clone(), node.clone()))
4793 }));
4794 nodes.extend(
4795 generated_nodes
4796 .into_iter()
4797 .map(|node| (node.id.clone(), node)),
4798 );
4799 nodes.extend(
4800 graphql_nodes
4801 .into_iter()
4802 .map(|node| (node.id.clone(), node)),
4803 );
4804 nodes.extend(event_nodes.into_iter().map(|node| (node.id.clone(), node)));
4805 nodes.extend(
4806 protobuf_nodes
4807 .into_iter()
4808 .map(|node| (node.id.clone(), node)),
4809 );
4810 nodes.extend(
4811 extraction_nodes
4812 .into_iter()
4813 .map(|node| (node.id.clone(), node)),
4814 );
4815 let mut evidence = boundaries
4816 .iter()
4817 .map(|boundary| (boundary.evidence.id.clone(), boundary.evidence.clone()))
4818 .collect::<BTreeMap<_, _>>();
4819 evidence.extend(
4820 tests
4821 .iter()
4822 .map(|test| (test.evidence.id.clone(), test.evidence.clone())),
4823 );
4824 evidence.extend(implementations.iter().map(|implementation| {
4825 (
4826 implementation.evidence.id.clone(),
4827 implementation.evidence.clone(),
4828 )
4829 }));
4830 evidence.extend(source_facts.iter().flat_map(|facts| {
4831 facts
4832 .standalone_test_evidence
4833 .iter()
4834 .map(|evidence| (evidence.id.clone(), evidence.clone()))
4835 }));
4836 evidence.extend(source_facts.iter().flat_map(|facts| {
4837 facts
4838 .relation_evidence
4839 .iter()
4840 .map(|evidence| (evidence.id.clone(), evidence.clone()))
4841 }));
4842 evidence.extend(package_facts.iter().flat_map(|facts| {
4843 facts
4844 .evidence
4845 .iter()
4846 .map(|evidence| (evidence.id.clone(), evidence.clone()))
4847 }));
4848 evidence.extend(
4849 generated_evidence
4850 .into_iter()
4851 .map(|item| (item.id.clone(), item)),
4852 );
4853 evidence.extend(
4854 graphql_evidence
4855 .into_iter()
4856 .map(|item| (item.id.clone(), item)),
4857 );
4858 evidence.extend(
4859 event_evidence
4860 .into_iter()
4861 .map(|item| (item.id.clone(), item)),
4862 );
4863 evidence.extend(
4864 protobuf_evidence
4865 .into_iter()
4866 .map(|item| (item.id.clone(), item)),
4867 );
4868 evidence.extend(
4869 extraction_evidence
4870 .into_iter()
4871 .map(|item| (item.id.clone(), item)),
4872 );
4873 let mut link_node_keys = BTreeMap::new();
4874 link_node_keys.extend(boundaries.iter().map(|boundary| {
4875 (
4876 boundary.node.id.clone(),
4877 format!("{}:{}", boundary.method, boundary.path),
4878 )
4879 }));
4880 link_node_keys.extend(tests.iter().map(|test| {
4881 (
4882 test.node.id.clone(),
4883 format!("{}:{}", test.method, test.path),
4884 )
4885 }));
4886 link_node_keys.extend(implementations.iter().map(|implementation| {
4887 (
4888 implementation.node.id.clone(),
4889 format!("{}:{}", implementation.method, implementation.path),
4890 )
4891 }));
4892 Ok(GraphAssembly {
4893 nodes: nodes.into_values().collect(),
4894 edges,
4895 evidence: evidence.into_values().collect(),
4896 link_decisions: Vec::new(),
4897 link_node_keys,
4898 degradations,
4899 })
4900}
4901
4902#[expect(
4903 clippy::too_many_lines,
4904 reason = "Fail-closed relinking keeps old/new neighborhood handling in one audit boundary"
4905)]
4906fn relink_affected_graph(
4907 graph: &mut GraphAssembly,
4908 plan: &IncrementalPlan,
4909 batch_plan: &ExtractorBatchPlan,
4910 focused: &FocusedBatchState,
4911 _previous_nodes: &[Node],
4912 previous_edges: &[Edge],
4913) -> Result<(), ApplicationError> {
4914 let full_relink = focused.force_relink
4915 || plan.changes.iter().any(|change| {
4916 change.kind != code_system_graph_model::ArtifactChangeKind::Unchanged
4917 && change.extractor != "code-system-graph.packages"
4918 && !source_extractor(&change.extractor)
4919 });
4920 if full_relink || previous_edges.is_empty() {
4921 return Ok(());
4922 }
4923 let source_plan = ExtractorBatchPlan {
4924 batches: batch_plan
4925 .batches
4926 .iter()
4927 .filter(|batch| source_extractor(&batch.key.extractor))
4928 .cloned()
4929 .collect(),
4930 };
4931 let affected = affected_link_keys(
4932 &source_plan,
4933 &focused.previous_source_batches,
4934 &focused.source_batches,
4935 |observation| {
4936 observation
4937 .method
4938 .as_ref()
4939 .zip(observation.path.as_ref())
4940 .map(|(method, path)| format!("{method}:{path}"))
4941 },
4942 )?
4943 .into_iter()
4944 .flatten()
4945 .collect::<BTreeSet<_>>();
4946 if affected.is_empty() {
4947 return Ok(());
4948 }
4949
4950 let mut previous_node_keys = graph.link_node_keys.clone();
4951 for batch in &focused.previous_source_batches {
4952 let facts = source_observations_to_graph(
4953 &batch.source.repo_id,
4954 &portable_path(&batch.source.path.display),
4955 &batch.source.content_hash,
4956 &batch.outputs,
4957 );
4958 previous_node_keys.extend(facts.boundaries.iter().map(|boundary| {
4959 (
4960 boundary.node.id.clone(),
4961 format!("{}:{}", boundary.method, boundary.path),
4962 )
4963 }));
4964 previous_node_keys.extend(facts.tests.iter().map(|test| {
4965 (
4966 test.node.id.clone(),
4967 format!("{}:{}", test.method, test.path),
4968 )
4969 }));
4970 previous_node_keys.extend(facts.implementations.iter().map(|implementation| {
4971 (
4972 implementation.node.id.clone(),
4973 format!("{}:{}", implementation.method, implementation.path),
4974 )
4975 }));
4976 }
4977 let current_node_ids = graph
4978 .nodes
4979 .iter()
4980 .map(|node| node.id.clone())
4981 .collect::<BTreeSet<_>>();
4982 let is_http_link = |edge: &Edge| {
4983 matches!(
4984 edge.kind,
4985 code_system_graph_model::EdgeKind::CallsRemote
4986 | code_system_graph_model::EdgeKind::Validates
4987 | code_system_graph_model::EdgeKind::ImplementedBy
4988 )
4989 };
4990 let recomputed_http = graph
4991 .edges
4992 .iter()
4993 .filter(|edge| is_http_link(edge))
4994 .cloned()
4995 .collect::<Vec<_>>();
4996 let previous_http = previous_edges
4997 .iter()
4998 .filter(|edge| is_http_link(edge))
4999 .cloned()
5000 .collect::<Vec<_>>();
5001 let mut edges = graph
5002 .edges
5003 .iter()
5004 .filter(|edge| !is_http_link(edge))
5005 .cloned()
5006 .collect::<Vec<_>>();
5007 edges.extend(merge_affected_link_neighborhoods(
5008 &previous_http,
5009 &recomputed_http,
5010 &affected,
5011 &previous_node_keys,
5012 &graph.link_node_keys,
5013 ¤t_node_ids,
5014 ));
5015 edges.sort_by(|left, right| left.id.cmp(&right.id));
5016 edges.dedup_by(|left, right| left.id == right.id);
5017 graph.edges = edges;
5018 Ok(())
5019}
5020
5021fn run_codegraph_corroboration(
5022 context: &WorkspaceContext,
5023 focused: &FocusedBatchState,
5024 plan: &IncrementalPlan,
5025 selected_repository: Option<&str>,
5026 codegraph_binary: Option<PathBuf>,
5027) -> CorroborationSummary {
5028 let (jobs, setup_degradations) =
5029 prepare_codegraph_jobs(context, focused, plan, selected_repository);
5030 let worker = std::thread::spawn(move || -> Result<Vec<RepositoryCorroboration>, String> {
5031 let runtime = tokio::runtime::Builder::new_current_thread()
5032 .enable_all()
5033 .build()
5034 .map_err(|error| format!("cannot start CodeGraph corroboration runtime: {error}"))?;
5035 runtime.block_on(async move {
5036 let mut config = CodeGraphConfig::default();
5037 if let Some(binary) = codegraph_binary {
5038 config.binary = binary.into_os_string();
5039 }
5040 let provider = CodeGraphProvider::new(config).map_err(|error| error.to_string())?;
5041 let mut reports = Vec::new();
5042 for (repo_id, project_path, anchors, changed_files) in jobs {
5043 let report = corroborate_repository(
5044 &provider,
5045 repo_id.clone(),
5046 project_path,
5047 &anchors,
5048 &changed_files,
5049 ProviderBudget {
5050 timeout: Duration::from_secs(2),
5051 max_output_bytes: 256 * 1024,
5052 max_items: 20,
5053 },
5054 CancellationToken::new(),
5055 )
5056 .await;
5057 reports.push(RepositoryCorroboration { repo_id, report });
5058 }
5059 provider
5060 .shutdown()
5061 .await
5062 .map_err(|error| error.to_string())?;
5063 Ok(reports)
5064 })
5065 });
5066 let mut summary = CorroborationSummary {
5067 degradations: setup_degradations,
5068 ..CorroborationSummary::default()
5069 };
5070 match worker.join() {
5071 Ok(Ok(reports)) => {
5072 summary.confirmed_symbols = reports
5073 .iter()
5074 .flat_map(|item| &item.report.symbols)
5075 .filter(|symbol| matches!(symbol, SymbolCorroboration::Confirmed { .. }))
5076 .count();
5077 summary.affected_tests = reports
5078 .iter()
5079 .map(|item| item.report.affected_tests.len())
5080 .sum();
5081 summary.degradations.extend(
5082 reports
5083 .iter()
5084 .flat_map(|item| item.report.degradations.iter().cloned()),
5085 );
5086 summary.reports = reports;
5087 }
5088 Ok(Err(error)) => summary.degradations.push(error),
5089 Err(_) => summary
5090 .degradations
5091 .push("CodeGraph corroboration worker terminated unexpectedly".to_owned()),
5092 }
5093 summary.degradations.sort();
5094 summary.degradations.dedup();
5095 summary
5096}
5097
5098type CodeGraphJob = (RepoId, PathBuf, Vec<SymbolAnchor>, Vec<String>);
5099
5100fn prepare_codegraph_jobs(
5101 context: &WorkspaceContext,
5102 focused: &FocusedBatchState,
5103 plan: &IncrementalPlan,
5104 selected_repository: Option<&str>,
5105) -> (Vec<CodeGraphJob>, Vec<String>) {
5106 let mut anchors_by_repository = BTreeMap::<RepoId, Vec<SymbolAnchor>>::new();
5107 for batch in &focused.source_batches {
5108 let source_path = portable_path(&batch.source.path.display);
5109 let anchors = anchors_by_repository
5110 .entry(batch.source.repo_id.clone())
5111 .or_default();
5112 anchors.extend(batch.outputs.iter().filter_map(|observation| {
5113 if observation.role != SourceRole::Provider {
5114 return None;
5115 }
5116 Some(SymbolAnchor {
5117 symbol: observation.symbol_name.clone()?,
5118 source_path: source_path.clone(),
5119 start_line: usize::try_from(observation.lines.start).ok()?,
5120 })
5121 }));
5122 }
5123 let mut changed_files_by_repository = BTreeMap::<RepoId, Vec<String>>::new();
5124 for change in plan
5125 .changes
5126 .iter()
5127 .filter(|change| change.kind != code_system_graph_model::ArtifactChangeKind::Unchanged)
5128 {
5129 changed_files_by_repository
5130 .entry(change.repo_id.clone())
5131 .or_default()
5132 .push(portable_path(&change.path.display));
5133 }
5134
5135 let mut jobs = Vec::new();
5136 let mut setup_degradations = Vec::new();
5137 for repository in &context.registry.record.repositories {
5138 if selected_repository.is_some_and(|selected| selected != repository.alias) {
5139 continue;
5140 }
5141 let Some(project_path) = context.registry.checkout_path(&repository.alias) else {
5142 setup_degradations.push(format!(
5143 "CodeGraph skipped `{}` because its checkout is unavailable",
5144 repository.alias
5145 ));
5146 continue;
5147 };
5148 let mut anchors = anchors_by_repository
5149 .remove(&repository.id)
5150 .unwrap_or_default();
5151 anchors.sort_by(|left, right| {
5152 (&left.source_path, left.start_line, &left.symbol).cmp(&(
5153 &right.source_path,
5154 right.start_line,
5155 &right.symbol,
5156 ))
5157 });
5158 anchors.dedup();
5159 if anchors.len() > 50 {
5160 anchors.truncate(50);
5161 setup_degradations.push(format!(
5162 "CodeGraph symbol corroboration for `{}` was limited to 50 anchors",
5163 repository.alias
5164 ));
5165 }
5166 let mut changed_files = changed_files_by_repository
5167 .remove(&repository.id)
5168 .unwrap_or_default();
5169 changed_files.sort();
5170 changed_files.dedup();
5171 if changed_files.len() > 1_024 {
5172 changed_files.truncate(1_024);
5173 setup_degradations.push(format!(
5174 "CodeGraph affected-test corroboration for `{}` was limited to 1,024 changed files",
5175 repository.alias
5176 ));
5177 }
5178 if !anchors.is_empty() || !changed_files.is_empty() {
5179 jobs.push((
5180 repository.id.clone(),
5181 project_path.to_path_buf(),
5182 anchors,
5183 changed_files,
5184 ));
5185 }
5186 }
5187 (jobs, setup_degradations)
5188}
5189
5190fn apply_codegraph_corroboration(graph: &mut GraphAssembly, reports: &[RepositoryCorroboration]) {
5191 for item in reports {
5192 for outcome in &item.report.symbols {
5193 let SymbolCorroboration::Confirmed {
5194 symbol,
5195 source_path,
5196 local_id,
5197 } = outcome
5198 else {
5199 continue;
5200 };
5201 let implementation_ids = ["javascript", "typescript", "rust", "python", "go", "java"]
5202 .map(|language| {
5203 NodeId::new(stable_id(
5204 "node",
5205 &format!(
5206 "symbol:{}:{language}:{source_path}:{symbol}",
5207 item.repo_id.as_str()
5208 ),
5209 ))
5210 })
5211 .into_iter()
5212 .filter(|candidate| {
5213 graph
5214 .nodes
5215 .iter()
5216 .any(|node| node.id == *candidate && node.kind == NodeKind::SymbolRef)
5217 })
5218 .collect::<BTreeSet<_>>();
5219 if implementation_ids.is_empty() {
5220 continue;
5221 }
5222 let source_evidence = graph.evidence.iter().find(|evidence| {
5223 evidence.repo_id.as_ref() == Some(&item.repo_id)
5224 && evidence.file_path.as_deref() == Some(source_path)
5225 && evidence.extractor.starts_with("code-system-graph.source.")
5226 });
5227 let (start_line, end_line, content_hash) =
5228 source_evidence.map_or((None, None, None), |evidence| {
5229 (
5230 evidence.start_line,
5231 evidence.end_line,
5232 evidence.content_hash.clone(),
5233 )
5234 });
5235 let evidence_key = format!(
5236 "codegraph:{}:{source_path}:{symbol}:{}",
5237 item.repo_id.as_str(),
5238 local_id.as_deref().unwrap_or("anonymous")
5239 );
5240 let evidence = Evidence {
5241 id: EvidenceId::new(stable_id("evidence", &evidence_key)),
5242 repo_id: Some(item.repo_id.clone()),
5243 file_path: Some(source_path.clone()),
5244 start_line,
5245 end_line,
5246 extractor: "code-system-graph.codegraph.corroboration".to_owned(),
5247 extractor_version: "1.0.0".to_owned(),
5248 provenance: Provenance::CodeGraph,
5249 confidence: 1.0,
5250 observed_at_commit: None,
5251 content_hash,
5252 note: Some("exact provider symbol name, path, and line match".to_owned()),
5253 };
5254 for edge in graph.edges.iter_mut().filter(|edge| {
5255 edge.kind == code_system_graph_model::EdgeKind::ImplementedBy
5256 && implementation_ids.contains(&edge.target)
5257 }) {
5258 if !edge.evidence.contains(&evidence.id) {
5259 edge.evidence.push(evidence.id.clone());
5260 edge.evidence.sort();
5261 }
5262 }
5263 graph.evidence.push(evidence);
5264 }
5265 }
5266 graph.evidence.sort_by(|left, right| left.id.cmp(&right.id));
5267 graph.evidence.dedup_by(|left, right| left.id == right.id);
5268}
5269
5270fn persisted_manual_link_records(
5271 snapshot_id: &str,
5272 decisions: &[LinkDecision],
5273) -> Result<Vec<ManualLinkRecord>, ApplicationError> {
5274 decisions
5275 .iter()
5276 .map(|decision| {
5277 let disposition = match decision.status {
5278 LinkStatus::Confirmed => ManualLinkDisposition::Active,
5279 LinkStatus::Suppressed => ManualLinkDisposition::Suppression,
5280 LinkStatus::Ambiguous | LinkStatus::Rejected => {
5281 return Err(ApplicationError::Initialization(
5282 "unapplied manual-link decision reached snapshot publication".to_owned(),
5283 ));
5284 }
5285 };
5286 let kind = serde_json::to_value(decision.relation)
5287 .ok()
5288 .and_then(|value| value.as_str().map(str::to_owned))
5289 .ok_or_else(|| {
5290 ApplicationError::Initialization(
5291 "manual-link relation could not be encoded".to_owned(),
5292 )
5293 })?;
5294 let reason = decision.reasons.first().cloned().ok_or_else(|| {
5295 ApplicationError::Initialization(
5296 "manual-link decision omitted its required rationale".to_owned(),
5297 )
5298 })?;
5299 let identity = format!(
5300 "{}:{kind}:{}:{}:{reason}",
5301 decision.source.as_str(),
5302 decision.target.as_str(),
5303 match disposition {
5304 ManualLinkDisposition::Active => "active",
5305 ManualLinkDisposition::Suppression => "suppression",
5306 }
5307 );
5308 Ok(ManualLinkRecord {
5309 id: stable_id("manual-link", &identity),
5310 snapshot_id: snapshot_id.to_owned(),
5311 source_node_id: decision.source.clone(),
5312 target_node_id: decision.target.clone(),
5313 kind,
5314 disposition,
5315 reason,
5316 decision: decision.clone(),
5317 config_version: 1,
5318 })
5319 })
5320 .collect()
5321}
5322
5323fn provider_capability_record(
5324 workspace: &str,
5325 repo_id: &RepoId,
5326 capability: &ProviderCapability,
5327) -> ProviderCapabilityRecord {
5328 let mut capabilities = Vec::new();
5329 if let Some(status) = serialized_enum_name(capability.status) {
5330 capabilities.push(format!("status:{status}"));
5331 }
5332 capabilities.extend(
5333 capability
5334 .tools
5335 .iter()
5336 .filter(|tool| tool.len() <= 240)
5337 .map(|tool| format!("tool:{tool}")),
5338 );
5339 for operation in &capability.operations {
5340 if let (Some(operation_name), Some(transport)) = (
5341 serialized_enum_name(operation.operation),
5342 serialized_enum_name(operation.transport),
5343 ) {
5344 capabilities.push(format!("operation:{operation_name}:{transport}"));
5345 }
5346 }
5347 capabilities.sort();
5348 capabilities.dedup();
5349 ProviderCapabilityRecord {
5350 workspace_name: workspace.to_owned(),
5351 repo_id: repo_id.clone(),
5352 provider: capability.provider.clone(),
5353 provider_version: capability
5354 .version
5355 .clone()
5356 .unwrap_or_else(|| "unknown".to_owned()),
5357 capabilities,
5358 observed_at_unix_ms: current_unix_millis(),
5359 }
5360}
5361
5362fn community_topology_unchanged(
5363 previous_nodes: &[Node],
5364 previous_edges: &[Edge],
5365 current_nodes: &[Node],
5366 current_edges: &[Edge],
5367) -> bool {
5368 previous_nodes == current_nodes
5369 && previous_edges.len() == current_edges.len()
5370 && previous_edges
5371 .iter()
5372 .zip(current_edges)
5373 .all(|(previous, current)| {
5374 previous.id == current.id
5375 && previous.source == current.source
5376 && previous.target == current.target
5377 && previous.kind == current.kind
5378 && previous.confidence.to_bits() == current.confidence.to_bits()
5379 && previous.status == current.status
5380 })
5381}
5382
5383fn serialized_enum_name<T: Serialize>(value: T) -> Option<String> {
5384 serde_json::to_value(value)
5385 .ok()
5386 .and_then(|value| value.as_str().map(str::to_owned))
5387}
5388
5389fn extract_boundaries(context: &WorkspaceContext) -> Result<Vec<HttpBoundary>, ApplicationError> {
5390 let repository_records = context
5391 .registry
5392 .record
5393 .repositories
5394 .iter()
5395 .map(|repository| (repository.alias.as_str(), repository))
5396 .collect::<BTreeMap<_, _>>();
5397 let mut boundaries = Vec::new();
5398 for alias in context.manifest.repos.keys() {
5399 let registered = repository_records
5400 .get(alias.as_str())
5401 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5402 let effective = context
5403 .repository_configs
5404 .get(alias)
5405 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5406 boundaries.extend(
5407 effective
5408 .http_consumers
5409 .iter()
5410 .map(|consumer| HttpBoundary::consumer(registered.id.clone(), consumer)),
5411 );
5412 let repository_path = context
5413 .registry
5414 .checkout_path(alias)
5415 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5416 for openapi in &effective.openapi {
5417 let openapi_path = repository_path.join(openapi);
5418 let mut tracker = ExtractionTracker::new(
5419 openapi,
5420 "code-system-graph.http.openapi",
5421 &context.extraction_budgets,
5422 );
5423 let (openapi_source, _) = read_source_file(&openapi_path, &mut tracker)?;
5424 boundaries.extend(extract_openapi_with_tracker(
5425 ®istered.id,
5426 openapi,
5427 &openapi_source,
5428 &mut tracker,
5429 )?);
5430 }
5431 }
5432 Ok(boundaries)
5433}
5434
5435fn extract_declared_tests(
5436 context: &WorkspaceContext,
5437 fingerprints: &[ArtifactFingerprint],
5438) -> Result<Vec<DeclaredTestCase>, ApplicationError> {
5439 let repository_records = context
5440 .registry
5441 .record
5442 .repositories
5443 .iter()
5444 .map(|repository| (repository.alias.as_str(), repository))
5445 .collect::<BTreeMap<_, _>>();
5446 let mut tests = Vec::new();
5447 for alias in context.manifest.repos.keys() {
5448 let repository = repository_records
5449 .get(alias.as_str())
5450 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5451 let effective = context
5452 .repository_configs
5453 .get(alias)
5454 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5455 tests.extend(effective.integration_tests.iter().map(|test| {
5456 let content_hash = artifact_content_hash(
5457 fingerprints,
5458 &repository.id,
5459 "code-system-graph.tests.declared",
5460 &test.path,
5461 );
5462 declared_test_case(repository.id.clone(), test, content_hash)
5463 }));
5464 }
5465 tests.sort_by(|left, right| left.node.id.cmp(&right.node.id));
5466 Ok(tests)
5467}
5468
5469fn extract_declared_implementations(
5470 context: &WorkspaceContext,
5471 fingerprints: &[ArtifactFingerprint],
5472) -> Result<Vec<DeclaredImplementation>, ApplicationError> {
5473 let repository_records = context
5474 .registry
5475 .record
5476 .repositories
5477 .iter()
5478 .map(|repository| (repository.alias.as_str(), repository))
5479 .collect::<BTreeMap<_, _>>();
5480 let mut implementations = Vec::new();
5481 for alias in context.manifest.repos.keys() {
5482 let repository = repository_records
5483 .get(alias.as_str())
5484 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5485 let effective = context
5486 .repository_configs
5487 .get(alias)
5488 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5489 implementations.extend(effective.implementations.iter().map(|implementation| {
5490 let content_hash = artifact_content_hash(
5491 fingerprints,
5492 &repository.id,
5493 "code-system-graph.implementations.declared",
5494 &implementation.path,
5495 );
5496 declared_implementation(repository.id.clone(), implementation, content_hash)
5497 }));
5498 }
5499 implementations.sort_by(|left, right| left.node.id.cmp(&right.node.id));
5500 Ok(implementations)
5501}
5502
5503fn artifact_content_hash(
5504 fingerprints: &[ArtifactFingerprint],
5505 repo_id: &code_system_graph_model::RepoId,
5506 extractor: &str,
5507 path: &str,
5508) -> Option<String> {
5509 fingerprints
5510 .iter()
5511 .find(|fingerprint| {
5512 &fingerprint.repo_id == repo_id
5513 && fingerprint.extractor == extractor
5514 && portable_path(&fingerprint.path.display) == portable_path(path)
5515 })
5516 .map(|fingerprint| fingerprint.content_hash.clone())
5517}
5518
5519fn portable_path(path: &str) -> String {
5520 path.replace('\\', "/")
5521}
5522
5523fn cargo_crate_root(checkout: &Path, source_path: &Path) -> String {
5524 source_path
5525 .parent()
5526 .into_iter()
5527 .flat_map(Path::ancestors)
5528 .take_while(|candidate| candidate.starts_with(checkout))
5529 .find(|candidate| candidate.join("Cargo.toml").is_file())
5530 .and_then(|candidate| candidate.strip_prefix(checkout).ok())
5531 .map(|relative| portable_path(&relative.to_string_lossy()))
5532 .unwrap_or_default()
5533}
5534
5535fn discover_artifact_fingerprints(
5536 context: &WorkspaceContext,
5537) -> Result<Vec<ArtifactFingerprint>, ApplicationError> {
5538 let repository_records = context
5539 .registry
5540 .record
5541 .repositories
5542 .iter()
5543 .map(|repository| (repository.alias.as_str(), repository))
5544 .collect::<BTreeMap<_, _>>();
5545 let mut fingerprints = BTreeMap::new();
5546 for alias in context.manifest.repos.keys() {
5547 let repository = repository_records
5548 .get(alias.as_str())
5549 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5550 let checkout_path = context
5551 .registry
5552 .checkout_path(alias)
5553 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5554 let effective = context
5555 .repository_configs
5556 .get(alias)
5557 .ok_or_else(|| ApplicationError::RegistryAliasMissing(alias.clone()))?;
5558 for openapi in &effective.openapi {
5559 let fingerprint = fingerprint_artifact(
5560 repository,
5561 checkout_path,
5562 Path::new(openapi),
5563 "code-system-graph.http.openapi",
5564 &context.extraction_budgets,
5565 )?;
5566 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5567 }
5568 for consumer in &effective.http_consumers {
5569 let fingerprint = fingerprint_artifact(
5570 repository,
5571 checkout_path,
5572 Path::new(&consumer.source),
5573 "code-system-graph.http.declared",
5574 &context.extraction_budgets,
5575 )?;
5576 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5577 }
5578 for test in &effective.integration_tests {
5579 let fingerprint = fingerprint_artifact(
5580 repository,
5581 checkout_path,
5582 Path::new(&test.path),
5583 "code-system-graph.tests.declared",
5584 &context.extraction_budgets,
5585 )?;
5586 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5587 }
5588 for implementation in &effective.implementations {
5589 let fingerprint = fingerprint_artifact(
5590 repository,
5591 checkout_path,
5592 Path::new(&implementation.path),
5593 "code-system-graph.implementations.declared",
5594 &context.extraction_budgets,
5595 )?;
5596 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5597 }
5598 for (relative_path, extractor) in
5599 discover_focused_artifacts(checkout_path, &effective.ignore_policy)?
5600 {
5601 let fingerprint = fingerprint_artifact(
5602 repository,
5603 checkout_path,
5604 &relative_path,
5605 extractor,
5606 &context.extraction_budgets,
5607 )?;
5608 fingerprints.insert(artifact_key(&fingerprint), fingerprint);
5609 }
5610 }
5611 Ok(fingerprints.into_values().collect())
5612}
5613
5614fn discover_focused_artifacts(
5615 checkout_path: &Path,
5616 ignore_policy: &IgnorePolicy,
5617) -> Result<Vec<(PathBuf, &'static str)>, ApplicationError> {
5618 let mut pending = vec![checkout_path.to_path_buf()];
5619 let canonical_checkout =
5620 fs::canonicalize(checkout_path).map_err(|source| ApplicationError::ReadFile {
5621 path: checkout_path.to_path_buf(),
5622 source,
5623 })?;
5624 let mut visited = BTreeSet::new();
5625 let mut discovered = Vec::new();
5626 while let Some(directory) = pending.pop() {
5627 let canonical_directory =
5628 fs::canonicalize(&directory).map_err(|source| ApplicationError::ReadFile {
5629 path: directory.clone(),
5630 source,
5631 })?;
5632 if !canonical_directory.starts_with(&canonical_checkout) {
5633 return Err(ApplicationError::ArtifactOutsideCheckout {
5634 path: canonical_directory,
5635 checkout: canonical_checkout,
5636 });
5637 }
5638 if !visited.insert(canonical_directory) {
5639 continue;
5640 }
5641 worker::report_progress(code_system_graph_core::JobPhase::Discovery, 1);
5642 let entries = fs::read_dir(&directory).map_err(|source| ApplicationError::ReadFile {
5643 path: directory.clone(),
5644 source,
5645 })?;
5646 for entry in entries {
5647 let entry = entry.map_err(|source| ApplicationError::ReadFile {
5648 path: directory.clone(),
5649 source,
5650 })?;
5651 let file_type = entry
5652 .file_type()
5653 .map_err(|source| ApplicationError::ReadFile {
5654 path: entry.path(),
5655 source,
5656 })?;
5657 if file_type.is_symlink() {
5658 continue;
5659 }
5660 let path = entry.path();
5661 let relative = path.strip_prefix(checkout_path).map_err(|_| {
5662 ApplicationError::ArtifactOutsideCheckout {
5663 path: path.clone(),
5664 checkout: checkout_path.to_path_buf(),
5665 }
5666 })?;
5667 if file_type.is_dir() {
5668 if !ignore_policy.excludes(relative, true) {
5669 pending.push(path);
5670 }
5671 continue;
5672 }
5673 if !file_type.is_file() || ignore_policy.excludes(relative, false) {
5674 continue;
5675 }
5676 for extractor in focused_extractors_for_path(&path) {
5677 discovered.push((relative.to_path_buf(), extractor));
5678 }
5679 }
5680 }
5681 discovered.sort_by(|left, right| left.0.cmp(&right.0).then(left.1.cmp(right.1)));
5682 Ok(discovered)
5683}
5684
5685fn focused_extractors_for_path(path: &Path) -> Vec<&'static str> {
5686 let Some(name) = path.file_name().and_then(std::ffi::OsStr::to_str) else {
5687 return Vec::new();
5688 };
5689 let mut extractors = Vec::new();
5690 let is_generated_client_metadata = name == "openapitools.json"
5691 || (matches!(name, "FILES" | "VERSION")
5692 && path
5693 .parent()
5694 .and_then(Path::file_name)
5695 .is_some_and(|parent| parent == ".openapi-generator"));
5696 if is_generated_client_metadata {
5697 extractors.push("code-system-graph.http.generated-client");
5698 }
5699 let extension = path
5700 .extension()
5701 .and_then(std::ffi::OsStr::to_str)
5702 .map(str::to_ascii_lowercase);
5703 let source_extractor = match extension.as_deref() {
5704 Some("js" | "jsx") => Some("code-system-graph.source.javascript"),
5705 Some("ts" | "tsx") => Some("code-system-graph.source.typescript"),
5706 Some("rs") => Some("code-system-graph.source.rust"),
5707 Some("py") => Some("code-system-graph.source.python"),
5708 Some("go") => Some("code-system-graph.source.go"),
5709 Some("java") => Some("code-system-graph.source.java"),
5710 _ => None,
5711 };
5712 if let Some(extractor) = source_extractor {
5713 extractors.extend([
5714 extractor,
5715 "code-system-graph.events.source",
5716 "code-system-graph.graphql.source",
5717 "code-system-graph.protobuf.generated",
5718 "code-system-graph.data.source",
5719 ]);
5720 }
5721 if matches!(extension.as_deref(), Some("graphql" | "gql")) {
5722 extractors.push("code-system-graph.graphql.document");
5723 }
5724 if extension.as_deref() == Some("proto") {
5725 extractors.push("code-system-graph.protobuf");
5726 }
5727 let lower_name = name.to_ascii_lowercase();
5728 if matches!(
5729 lower_name.as_str(),
5730 "asyncapi.yaml" | "asyncapi.yml" | "asyncapi.json"
5731 ) {
5732 extractors.push("code-system-graph.events.asyncapi");
5733 }
5734 if matches!(
5735 lower_name.as_str(),
5736 "persisted-queries.json"
5737 | "persisted_queries.json"
5738 | "apollo-manifest.json"
5739 | "operation-manifest.json"
5740 ) {
5741 extractors.push("code-system-graph.graphql.persisted");
5742 }
5743 extractors.extend(document_extractors_for_path(
5744 path,
5745 name,
5746 extension.as_deref(),
5747 ));
5748 let is_package_artifact = matches!(
5749 name,
5750 "package.json"
5751 | "package-lock.json"
5752 | "pnpm-lock.yaml"
5753 | "yarn.lock"
5754 | "pyproject.toml"
5755 | "poetry.lock"
5756 | "Cargo.toml"
5757 | "Cargo.lock"
5758 | "go.mod"
5759 | "go.work"
5760 | "pom.xml"
5761 | "build.gradle"
5762 | "build.gradle.kts"
5763 | "packages.config"
5764 ) || extension.as_deref() == Some("csproj")
5765 || (name.starts_with("requirements")
5766 && Path::new(name)
5767 .extension()
5768 .is_some_and(|extension| extension.eq_ignore_ascii_case("txt")));
5769 if is_package_artifact {
5770 extractors.push("code-system-graph.packages");
5771 }
5772 extractors.sort_unstable();
5773 extractors.dedup();
5774 extractors
5775}
5776
5777fn document_extractors_for_path(
5778 path: &Path,
5779 name: &str,
5780 extension: Option<&str>,
5781) -> Vec<&'static str> {
5782 let mut extractors = Vec::new();
5783 let lower_name = name.to_ascii_lowercase();
5784 let lower_components = path
5785 .components()
5786 .filter_map(|component| component.as_os_str().to_str())
5787 .map(str::to_ascii_lowercase)
5788 .collect::<Vec<_>>();
5789 let in_database_migrations = lower_components
5790 .iter()
5791 .any(|component| matches!(component.as_str(), "alembic" | "migrations"));
5792 if matches!(extension, Some("sql" | "prisma"))
5793 || lower_name == "sqlx.toml"
5794 || lower_name == "schema.rs"
5795 || (extension == Some("py")
5796 && (in_database_migrations || matches!(lower_name.as_str(), "model.py" | "models.py")))
5797 {
5798 extractors.push("code-system-graph.data.artifact");
5799 }
5800
5801 let is_compose = matches!(
5802 lower_name.as_str(),
5803 "compose.yaml"
5804 | "compose.yml"
5805 | "compose.json"
5806 | "docker-compose.yaml"
5807 | "docker-compose.yml"
5808 | "docker-compose.json"
5809 );
5810 if is_compose {
5811 extractors.push("code-system-graph.infrastructure.compose");
5812 }
5813 if extension == Some("tf") {
5814 extractors.push("code-system-graph.infrastructure.terraform");
5815 }
5816 let in_helm_templates = lower_components
5817 .iter()
5818 .any(|component| component == "templates");
5819 let is_helm = in_helm_templates
5820 || matches!(
5821 lower_name.as_str(),
5822 "chart.yaml" | "chart.yml" | "values.yaml" | "values.yml"
5823 );
5824 if is_helm {
5825 extractors.push("code-system-graph.infrastructure.helm");
5826 } else if matches!(extension, Some("yaml" | "yml" | "json"))
5827 && !is_compose
5828 && (lower_components
5829 .iter()
5830 .any(|component| matches!(component.as_str(), "k8s" | "kubernetes" | "manifests"))
5831 || [
5832 "deployment",
5833 "service",
5834 "ingress",
5835 "statefulset",
5836 "daemonset",
5837 "cronjob",
5838 "job",
5839 "pod",
5840 ]
5841 .iter()
5842 .any(|kind| lower_name.starts_with(kind)))
5843 {
5844 extractors.push("code-system-graph.infrastructure.kubernetes");
5845 }
5846
5847 if extension == Some("md") || lower_name == "markdown" {
5848 extractors.push("code-system-graph.documents.markdown");
5849 }
5850 if lower_name == "codeowners" {
5851 extractors.push("code-system-graph.documents.codeowners");
5852 }
5853 if matches!(
5854 lower_name.as_str(),
5855 "catalog.yaml"
5856 | "catalog.yml"
5857 | "catalog.json"
5858 | "catalog-info.yaml"
5859 | "catalog-info.yml"
5860 | "service-catalog.yaml"
5861 | "service-catalog.yml"
5862 | "service-catalog.json"
5863 ) {
5864 extractors.push("code-system-graph.documents.catalog");
5865 }
5866
5867 let is_named_config = [
5868 "config.",
5869 "settings.",
5870 "application.",
5871 "values.",
5872 "secrets.",
5873 ]
5874 .iter()
5875 .any(|prefix| lower_name.starts_with(prefix));
5876 if lower_name == ".env"
5877 || lower_name.starts_with(".env.")
5878 || (is_named_config && matches!(extension, Some("yaml" | "yml" | "json" | "toml")))
5879 {
5880 extractors.push("code-system-graph.config.safe");
5881 }
5882 extractors
5883}
5884
5885fn fingerprint_artifact(
5886 repository: &RepositoryRecord,
5887 checkout_path: &Path,
5888 relative_path: &Path,
5889 extractor: &str,
5890 budgets: &ExtractionBudgets,
5891) -> Result<ArtifactFingerprint, ApplicationError> {
5892 let configured_path = checkout_path.join(relative_path);
5893 let canonical_path =
5894 std::fs::canonicalize(&configured_path).map_err(|source| ApplicationError::ReadFile {
5895 path: configured_path.clone(),
5896 source,
5897 })?;
5898 if !canonical_path.starts_with(checkout_path) {
5899 return Err(ApplicationError::ArtifactOutsideCheckout {
5900 path: configured_path,
5901 checkout: checkout_path.to_path_buf(),
5902 });
5903 }
5904 let metadata =
5905 std::fs::metadata(&canonical_path).map_err(|source| ApplicationError::ReadFile {
5906 path: canonical_path.clone(),
5907 source,
5908 })?;
5909 let relative = canonical_path.strip_prefix(checkout_path).map_err(|_| {
5910 ApplicationError::ArtifactOutsideCheckout {
5911 path: canonical_path.clone(),
5912 checkout: checkout_path.to_path_buf(),
5913 }
5914 })?;
5915 let path = encode_native_path(relative);
5916 code_system_graph_model::validate_safe_path_display(&path.display)
5917 .map_err(|_| ApplicationError::UnsafeArtifactPath)?;
5918 let mut tracker = ExtractionTracker::new(&path.display, extractor, budgets);
5919 let content = read_bounded_bytes(&canonical_path, &mut tracker)?;
5920 let fingerprint = ArtifactFingerprint {
5921 repo_id: repository.id.clone(),
5922 checkout_id: repository.checkout_id.clone(),
5923 path,
5924 extractor: extractor.to_owned(),
5925 content_hash: stable_id_bytes("artifact-content", &content),
5926 size_bytes: metadata.len(),
5927 };
5928 worker::report_progress(code_system_graph_core::JobPhase::Fingerprinting, 1);
5929 Ok(fingerprint)
5930}
5931
5932fn artifact_key(
5933 fingerprint: &ArtifactFingerprint,
5934) -> (CheckoutId, code_system_graph_model::NativePath, String) {
5935 (
5936 fingerprint.checkout_id.clone(),
5937 fingerprint.path.clone(),
5938 fingerprint.extractor.clone(),
5939 )
5940}
5941
5942fn extractor_runs(
5943 snapshot_id: &str,
5944 fingerprints: &[ArtifactFingerprint],
5945 plan: &IncrementalPlan,
5946) -> Vec<ExtractorRun> {
5947 let actions = plan
5948 .changes
5949 .iter()
5950 .map(|change| {
5951 (
5952 ArtifactKey {
5953 repo_id: change.repo_id.clone(),
5954 checkout_id: change.checkout_id.clone(),
5955 path: change.path.clone(),
5956 extractor: change.extractor.clone(),
5957 },
5958 change.kind,
5959 )
5960 })
5961 .collect::<BTreeMap<_, _>>();
5962 let mut groups = BTreeMap::<(String, String, String), Vec<&ArtifactFingerprint>>::new();
5963 for fingerprint in fingerprints {
5964 groups
5965 .entry((
5966 fingerprint.repo_id.as_str().to_owned(),
5967 fingerprint.checkout_id.as_str().to_owned(),
5968 fingerprint.extractor.clone(),
5969 ))
5970 .or_default()
5971 .push(fingerprint);
5972 }
5973 groups
5974 .into_iter()
5975 .map(|((repo_id, checkout_id, extractor), inputs)| {
5976 let changed = inputs
5977 .iter()
5978 .filter(|fingerprint| {
5979 actions.get(&ArtifactKey::from(**fingerprint))
5980 != Some(&code_system_graph_model::ArtifactChangeKind::Unchanged)
5981 })
5982 .count();
5983 let discovered_files = u64::try_from(inputs.len()).unwrap_or(u64::MAX);
5984 let parsed_files = u64::try_from(changed).unwrap_or(u64::MAX);
5985 let skipped_files = discovered_files.saturating_sub(parsed_files);
5986 let key = format!("{snapshot_id}:{repo_id}:{checkout_id}:{extractor}");
5987 ExtractorRun {
5988 id: stable_id("extractor-run", &key),
5989 snapshot_id: snapshot_id.to_owned(),
5990 repo_id: RepoId::new(repo_id),
5991 checkout_id: CheckoutId::new(checkout_id),
5992 extractor_version: if focused_extractor(&extractor) {
5993 EXTRACTION_CONTRACT_VERSION.to_owned()
5994 } else {
5995 env!("CARGO_PKG_VERSION").to_owned()
5996 },
5997 extractor,
5998 status: if changed == 0 {
5999 ExtractorRunStatus::SkippedUnchanged
6000 } else {
6001 ExtractorRunStatus::Success
6002 },
6003 discovered_files,
6004 parsed_files,
6005 skipped_files,
6006 elapsed_ms: 0,
6007 }
6008 })
6009 .collect()
6010}
6011
6012fn read_file(path: &Path) -> Result<String, ApplicationError> {
6013 std::fs::read_to_string(path).map_err(|source| ApplicationError::ReadFile {
6014 path: path.to_path_buf(),
6015 source,
6016 })
6017}
6018
6019fn read_source_file(
6020 path: &Path,
6021 tracker: &mut ExtractionTracker,
6022) -> Result<(String, bool), ApplicationError> {
6023 let bytes = read_bounded_bytes(path, tracker)?;
6024 match String::from_utf8(bytes) {
6025 Ok(source) => Ok((source, false)),
6026 Err(error) => Ok((String::from_utf8_lossy(error.as_bytes()).into_owned(), true)),
6027 }
6028}
6029
6030fn read_bounded_bytes(
6031 path: &Path,
6032 tracker: &mut ExtractionTracker,
6033) -> Result<Vec<u8>, ApplicationError> {
6034 let metadata = std::fs::metadata(path).map_err(|source| ApplicationError::ReadFile {
6035 path: path.to_path_buf(),
6036 source,
6037 })?;
6038 tracker.check_input_bytes(metadata.len())?;
6039 let file = std::fs::File::open(path).map_err(|source| ApplicationError::ReadFile {
6040 path: path.to_path_buf(),
6041 source,
6042 })?;
6043 let maximum = tracker.budgets().max_input_bytes_per_artifact;
6044 let mut bytes = Vec::new();
6045 file.take(maximum.saturating_add(1))
6046 .read_to_end(&mut bytes)
6047 .map_err(|source| ApplicationError::ReadFile {
6048 path: path.to_path_buf(),
6049 source,
6050 })?;
6051 tracker.check_input_bytes(u64::try_from(bytes.len()).unwrap_or(u64::MAX))?;
6052 Ok(bytes)
6053}
6054
6055#[cfg(test)]
6056mod budget_regression_tests {
6057 use super::*;
6058
6059 #[test]
6060 fn direct_scan_should_preserve_focused_source_value_limit() {
6061 let temporary = tempfile::tempdir().expect("temporary directory");
6062 let repository = temporary.path().join("api");
6063 fs::create_dir_all(repository.join("src")).expect("source directory");
6064 fs::write(
6065 repository.join("src/routes.rs"),
6066 "fn focused_source_boundary() {}",
6067 )
6068 .expect("source fixture");
6069 let config = temporary.path().join("code-system-graph.yaml");
6070 let database = temporary.path().join("code-system-graph.db");
6071 fs::write(
6072 &config,
6073 "version: 1\nname: source-budget-unit\nextractionBudgets:\n maxIdentifierBytesPerValue: 3\nrepos:\n api:\n path: api\n",
6074 )
6075 .expect("manifest fixture");
6076 let expected_artifact = encode_native_path(&Path::new("src").join("routes.rs")).display;
6077
6078 let result = scan_workspace_direct(&config, &database, &ScanOverrides::default());
6079 assert!(
6080 matches!(
6081 &result,
6082 Err(ApplicationError::ExtractionLimit(error))
6083 if error.resource == code_system_graph_core::ExtractionResource::IdentifierBytesPerValue
6084 && error.artifact == expected_artifact
6085 && error.extractor == "code-system-graph.source.rust"
6086 ),
6087 "unexpected direct source budget result: {result:?}"
6088 );
6089 }
6090
6091 #[test]
6092 fn direct_scan_should_charge_observations_independently_of_syntax_candidates() {
6093 let temporary = tempfile::tempdir().expect("temporary directory");
6094 let repository = temporary.path().join("tests");
6095 fs::create_dir_all(repository.join("tests")).expect("source directory");
6096 fs::write(
6097 repository.join("tests/test_api.py"),
6098 "import requests\ndef test_create_order():\n requests.post(\"https://api.test/v1/orders\")\n",
6099 )
6100 .expect("source fixture");
6101 let config = temporary.path().join("code-system-graph.yaml");
6102 let database = temporary.path().join("code-system-graph.db");
6103 let manifest = |maximum| {
6104 format!(
6105 "version: 1\nname: source-observation-unit\nextractionBudgets:\n maxObservationsPerArtifact: {maximum}\nrepos:\n tests:\n path: tests\n"
6106 )
6107 };
6108 fs::write(&config, manifest(1)).expect("limited manifest");
6109
6110 let rejected = scan_workspace_direct(&config, &database, &ScanOverrides::default());
6111 assert!(matches!(
6112 rejected,
6113 Err(ApplicationError::ExtractionLimit(error))
6114 if error.resource == code_system_graph_core::ExtractionResource::Observations
6115 && error.observed == 2
6116 && error.maximum == 1
6117 ));
6118
6119 fs::write(&config, manifest(2)).expect("exact manifest");
6120 let accepted = scan_workspace_direct(&config, &database, &ScanOverrides::default());
6121 assert!(
6122 accepted.is_ok(),
6123 "exact observation budget failed: {accepted:?}"
6124 );
6125 }
6126}
6127
6128#[cfg(test)]
6129mod codex_review_regression_tests {
6130 use code_system_graph_model::validate_safe_path_display;
6131
6132 use super::ApplicationError;
6133
6134 #[test]
6135 fn unsafe_artifact_path_error_must_not_echo_rejected_display() {
6136 const BIDI_PAYLOAD: &str = "invoice\u{202e}pay.pdf";
6137 assert!(validate_safe_path_display(BIDI_PAYLOAD).is_err());
6138 let rendered = ApplicationError::UnsafeArtifactPath.to_string();
6139 assert!(!rendered.contains(BIDI_PAYLOAD));
6140 assert!(!rendered.contains('\u{202e}'));
6141 assert_eq!(
6142 rendered,
6143 "artifact path contains unsafe control or bidirectional characters"
6144 );
6145 }
6146}