use std::collections::HashMap;
use std::sync::Arc;
use crate::graph::pdg::{Edge, EdgeMetadata, EdgeType, Node, NodeType, ProgramDependenceGraph};
use crate::storage::generation::migrate::{
MigrationError, StringInterner, edge_type_code_v2, edge_type_name_v2, node_type_code_v2,
node_type_name_v2,
};
pub fn encode_empty_tfidf() -> Vec<u8> {
crate::storage::generation::migrate::encode_empty_tfidf()
}
pub fn encode_empty_neural() -> Vec<u8> {
crate::storage::generation::migrate::encode_empty_neural()
}
pub fn vacuum_catalog_bytes(db_path: &std::path::Path) -> Result<Vec<u8>, MigrationError> {
crate::storage::generation::migrate::vacuum_bytes(db_path)
}
use crate::storage::generation::reader::{
PDG_EDGE_LEN, PDG_EDGE_META_LEN, PDG_HEADER_LEN, PDG_MAGIC, PDG_NODE_V2_LEN,
PDG_STRING_OFFSET_LEN, PDG_V2_NONE, PdgReader, ReaderError,
};
use crate::storage::nodes::NodeType as StorageNodeType;
#[derive(Debug, thiserror::Error)]
pub enum GraphCodecError {
#[error("PDG layer payload error: {0}")]
Payload(#[from] MigrationError),
#[error("PDG layer reader error: {0}")]
Reader(#[from] ReaderError),
#[error("invalid PDG layer: {0}")]
Invalid(String),
}
#[derive(Debug, Clone)]
pub struct LayerNodeAssignment {
node_ids: Vec<String>,
index_by_id: HashMap<String, u32>,
}
impl LayerNodeAssignment {
pub fn get(&self, node_id: &str) -> Option<u32> {
self.index_by_id.get(node_id).copied()
}
pub fn as_map(&self) -> HashMap<String, u32> {
self.index_by_id.clone()
}
pub fn len(&self) -> usize {
self.node_ids.len()
}
pub fn is_empty(&self) -> bool {
self.node_ids.is_empty()
}
pub fn node_ids(&self) -> &[String] {
&self.node_ids
}
}
pub fn encode_pdg_v2_from_graph(
pdg: &ProgramDependenceGraph,
) -> Result<(Vec<u8>, LayerNodeAssignment), GraphCodecError> {
let mut nodes_by_id: HashMap<&str, &Node> = HashMap::with_capacity(pdg.node_count());
for index in pdg.node_indices() {
let node = pdg
.get_node(index)
.ok_or_else(|| GraphCodecError::Invalid("node index has no node".into()))?;
nodes_by_id.insert(node.id.as_str(), node);
}
let node_ids: Vec<String> = {
let mut ids: Vec<&str> = nodes_by_id.keys().copied().collect();
ids.sort_unstable();
ids.into_iter().map(str::to_owned).collect()
};
let mut interner = StringInterner::new();
let mut node_records = Vec::with_capacity(node_ids.len() * PDG_NODE_V2_LEN);
for id in &node_ids {
let node = nodes_by_id[id.as_str()];
append_pdg_v2_node_record(&mut node_records, &mut interner, pdg, node)?;
}
let edges_by_key = build_pdg_v2_edge_map(pdg, &nodes_by_id, &mut interner)?;
let mut edge_keys: Vec<(u32, u32, u32)> = edges_by_key.keys().copied().collect();
edge_keys.sort_unstable();
let mut edges = Vec::with_capacity(edge_keys.len() * PDG_EDGE_LEN);
let mut edge_metadata = Vec::with_capacity(edge_keys.len() * PDG_EDGE_META_LEN);
for key @ (source_id, target_id, type_code) in edge_keys {
edges.extend_from_slice(&source_id.to_le_bytes());
edges.extend_from_slice(&target_id.to_le_bytes());
edges.extend_from_slice(&type_code.to_le_bytes());
edge_metadata.extend_from_slice(&edges_by_key[&key]);
}
let (string_table, string_bytes) = interner.into_bytes();
let num_strings = string_table.len() / PDG_STRING_OFFSET_LEN;
let mut data = Vec::with_capacity(
node_records.len()
+ edges.len()
+ edge_metadata.len()
+ string_table.len()
+ string_bytes.len(),
);
data.extend_from_slice(&node_records);
data.extend_from_slice(&edges);
data.extend_from_slice(&edge_metadata);
data.extend_from_slice(&string_table);
data.extend_from_slice(&string_bytes);
let content_hash = crate::storage::cas::blob::blob_hash(&data);
let mut payload = Vec::with_capacity(PDG_HEADER_LEN + data.len());
payload.extend_from_slice(PDG_MAGIC);
payload.push(2);
payload.extend_from_slice(&[0, 0, 0]);
payload.extend_from_slice(&(node_ids.len() as u32).to_le_bytes());
payload.extend_from_slice(&((edges.len() / PDG_EDGE_LEN) as u32).to_le_bytes());
payload.extend_from_slice(&(num_strings as u32).to_le_bytes());
payload.extend_from_slice(&(string_bytes.len() as u32).to_le_bytes());
payload.extend_from_slice(&content_hash);
payload.extend_from_slice(&data);
let index_by_id = node_ids
.iter()
.enumerate()
.map(|(index, id)| (id.clone(), index as u32))
.collect();
Ok((
payload,
LayerNodeAssignment {
node_ids,
index_by_id,
},
))
}
fn append_pdg_v2_node_record(
records: &mut Vec<u8>,
interner: &mut StringInterner,
pdg: &ProgramDependenceGraph,
node: &Node,
) -> Result<(), GraphCodecError> {
let node_type = storage_node_type(&node.node_type);
records.extend_from_slice(&interner.intern(&node.id).to_le_bytes());
records.extend_from_slice(&interner.intern(&node.name).to_le_bytes());
records.extend_from_slice(&interner.intern(&node.file_path).to_le_bytes());
records.extend_from_slice(&interner.intern(&node.language).to_le_bytes());
records.extend_from_slice(&node_type_code_v2(node_type.as_str())?.to_le_bytes());
records.extend_from_slice(&node.complexity.to_le_bytes());
records.extend_from_slice(
&u32::try_from(node.byte_range.0)
.map_err(|_| GraphCodecError::Invalid("node byte start exceeds u32".into()))?
.to_le_bytes(),
);
records.extend_from_slice(
&u32::try_from(node.byte_range.1)
.map_err(|_| GraphCodecError::Invalid("node byte end exceeds u32".into()))?
.to_le_bytes(),
);
records.extend_from_slice(&u32::from(pdg.is_precision_symbol(&node.id)).to_le_bytes());
Ok(())
}
fn build_pdg_v2_edge_map(
pdg: &ProgramDependenceGraph,
nodes_by_id: &HashMap<&str, &Node>,
interner: &mut StringInterner,
) -> Result<HashMap<(u32, u32, u32), Vec<u8>>, GraphCodecError> {
let mut edges_by_key: HashMap<(u32, u32, u32), Vec<u8>> =
HashMap::with_capacity(pdg.edge_count());
for edge_id in pdg.edge_indices() {
let edge = pdg
.get_edge(edge_id)
.ok_or_else(|| GraphCodecError::Invalid("edge index has no edge".into()))?;
let (source, target) = edge_endpoint_nodes(pdg, edge_id)?;
let source_id = intern_endpoint(nodes_by_id, interner, source, "source")?;
let target_id = intern_endpoint(nodes_by_id, interner, target, "target")?;
let edge_type = storage_edge_type(&edge.edge_type);
let type_code = edge_type_code_v2(edge_type.as_str())?;
let metadata = encode_pdg_v2_edge_metadata(edge, interner)?;
edges_by_key.insert((source_id, target_id, type_code), metadata);
}
Ok(edges_by_key)
}
fn edge_endpoint_nodes(
pdg: &ProgramDependenceGraph,
edge_id: crate::graph::pdg::EdgeId,
) -> Result<(&Node, &Node), GraphCodecError> {
let (source, target) = pdg
.edge_endpoints(edge_id)
.ok_or_else(|| GraphCodecError::Invalid("edge has no endpoints".into()))?;
let source = pdg
.get_node(source)
.ok_or_else(|| GraphCodecError::Invalid("edge source node is missing".into()))?;
let target = pdg
.get_node(target)
.ok_or_else(|| GraphCodecError::Invalid("edge target node is missing".into()))?;
Ok((source, target))
}
fn intern_endpoint(
nodes_by_id: &HashMap<&str, &Node>,
interner: &mut StringInterner,
node: &Node,
role: &str,
) -> Result<u32, GraphCodecError> {
nodes_by_id
.get(node.id.as_str())
.map(|node| interner.intern(&node.id))
.ok_or_else(|| GraphCodecError::Invalid(format!("edge {role} stable ID is missing")))
}
fn encode_pdg_v2_edge_metadata(
edge: &crate::graph::pdg::Edge,
interner: &mut StringInterner,
) -> Result<Vec<u8>, GraphCodecError> {
let mut metadata = Vec::with_capacity(PDG_EDGE_META_LEN);
encode_option_u32(&mut metadata, edge.metadata.call_count, "edge call_count")?;
let variable_name = edge
.metadata
.variable_name
.as_deref()
.map(|value| interner.intern(value));
metadata.extend_from_slice(&variable_name.unwrap_or(PDG_V2_NONE).to_le_bytes());
metadata.extend_from_slice(
&edge
.metadata
.confidence
.unwrap_or(f32::NAN)
.to_bits()
.to_le_bytes(),
);
let channel = edge
.metadata
.channel
.as_deref()
.map(|value| interner.intern(value));
metadata.extend_from_slice(&channel.unwrap_or(PDG_V2_NONE).to_le_bytes());
encode_option_u32(&mut metadata, edge.metadata.position, "edge position")?;
Ok(metadata)
}
pub fn encode_symbols_layer_from_graph(
pdg: &ProgramDependenceGraph,
) -> Result<Vec<u8>, GraphCodecError> {
use crate::storage::generation::reader::{SYMBOL_ENTRY_LEN, SYMBOLS_HEADER_LEN, SYMBOLS_MAGIC};
let mut interner = StringInterner::new();
let mut symbols: Vec<u8> = Vec::new();
let mut nodes_by_id: HashMap<&str, &Node> = HashMap::with_capacity(pdg.node_count());
for index in pdg.node_indices() {
let node = pdg
.get_node(index)
.ok_or_else(|| GraphCodecError::Invalid("node index has no node".into()))?;
nodes_by_id.insert(node.id.as_str(), node);
}
let mut ids: Vec<&str> = nodes_by_id.keys().copied().collect();
ids.sort_unstable();
for id in ids {
let node = nodes_by_id[id];
let node_type = storage_node_type(&node.node_type);
symbols.extend_from_slice(&interner.intern(&node.name).to_le_bytes());
let type_code = node_type_code_v2(node_type.as_str())?;
symbols.extend_from_slice(&type_code.to_le_bytes());
symbols.extend_from_slice(&interner.intern(&node.file_path).to_le_bytes());
symbols.extend_from_slice(&0u32.to_le_bytes()); symbols.extend_from_slice(&0u32.to_le_bytes()); symbols.extend_from_slice(&(node.complexity as f32).to_le_bytes());
}
let (string_table, string_bytes) = interner.into_bytes();
let num_symbols = symbols.len() / SYMBOL_ENTRY_LEN;
let num_strings = string_table.len() / PDG_STRING_OFFSET_LEN;
let mut data = Vec::with_capacity(symbols.len() + string_table.len() + string_bytes.len());
data.extend_from_slice(&symbols);
data.extend_from_slice(&string_table);
data.extend_from_slice(&string_bytes);
let content_hash = crate::storage::cas::blob::blob_hash(&data);
let mut payload = Vec::with_capacity(SYMBOLS_HEADER_LEN + data.len());
payload.extend_from_slice(SYMBOLS_MAGIC);
payload.push(1);
payload.extend_from_slice(&[0, 0, 0]);
payload.extend_from_slice(&(num_symbols as u32).to_le_bytes());
payload.extend_from_slice(&(num_strings as u32).to_le_bytes());
payload.extend_from_slice(&(string_bytes.len() as u32).to_le_bytes());
payload.extend_from_slice(&content_hash);
payload.extend_from_slice(&data);
Ok(payload)
}
fn encode_option_u32(
output: &mut Vec<u8>,
value: Option<usize>,
field: &str,
) -> Result<(), GraphCodecError> {
let value = value
.map(u32::try_from)
.transpose()
.map_err(|_| GraphCodecError::Invalid(format!("{field} exceeds u32")))?
.unwrap_or(PDG_V2_NONE);
output.extend_from_slice(&value.to_le_bytes());
Ok(())
}
impl PdgReader {
pub fn to_program_dependence_graph(&self) -> Result<ProgramDependenceGraph, GraphCodecError> {
if self.version() != 2 {
return Err(GraphCodecError::Invalid(format!(
"cannot hydrate graph from PDG layer version {}",
self.version()
)));
}
let mut graph = ProgramDependenceGraph::new();
graph.reserve_nodes(self.num_nodes());
let node_id_to_graph = self.decode_pdg_v2_nodes(&mut graph)?;
self.decode_pdg_v2_edges(&mut graph, &node_id_to_graph)?;
Ok(graph)
}
fn decode_pdg_v2_nodes(
&self,
graph: &mut ProgramDependenceGraph,
) -> Result<HashMap<String, crate::graph::pdg::NodeId>, GraphCodecError> {
let mut node_id_to_graph: HashMap<String, crate::graph::pdg::NodeId> = HashMap::new();
for index in 0..self.num_nodes() {
let record = self.node_full(index)?;
let id = required_string(self, record.node_id, "node id")?;
let name = required_string(self, record.symbol_name, "symbol name")?;
let file_path = required_string(self, record.file_path, "file path")?;
let language = required_string(self, record.language, "language")?;
let node_type = graph_node_type(record.node_type)?;
let graph_node = graph.add_node_without_trigrams(Node {
id: id.clone(),
node_type,
name,
file_path: Arc::from(file_path),
byte_range: (record.byte_start as usize, record.byte_end as usize),
complexity: record.complexity,
language,
});
if record.precision {
graph.mark_precision_symbol(id.clone());
}
node_id_to_graph.insert(id, graph_node);
}
Ok(node_id_to_graph)
}
fn decode_pdg_v2_edges(
&self,
graph: &mut ProgramDependenceGraph,
node_id_to_graph: &HashMap<String, crate::graph::pdg::NodeId>,
) -> Result<(), GraphCodecError> {
for index in 0..self.num_edges() {
let edge = self.edge(index)?;
let source_id = resolve_interned_node(self, edge.src, index, "source")?;
let target_id = resolve_interned_node(self, edge.dst, index, "target")?;
let (source, target) = match (
node_id_to_graph.get(&source_id),
node_id_to_graph.get(&target_id),
) {
(Some(source), Some(target)) => (*source, *target),
_ => {
return Err(GraphCodecError::Invalid(format!(
"edge {index} endpoint node id not present in the node table: {source_id} -> {target_id}"
)));
}
};
let edge_type = graph_edge_type(edge.edge_type)?;
let metadata = self.edge_meta(index)?;
graph.add_edge(
source,
target,
Edge {
edge_type,
metadata: EdgeMetadata {
call_count: metadata.call_count.map(|value| value as usize),
variable_name: metadata
.variable_name
.map(|id| required_string(self, id, "variable name"))
.transpose()?,
confidence: metadata.confidence,
channel: metadata
.channel
.map(|id| required_string(self, id, "channel"))
.transpose()?,
position: metadata.position.map(|value| value as usize),
},
},
);
}
graph.rebuild_trigram_index();
Ok(())
}
}
fn resolve_interned_node(
reader: &PdgReader,
interned: u32,
edge_index: usize,
role: &str,
) -> Result<String, GraphCodecError> {
reader
.resolve_string(interned)
.map(str::to_owned)
.ok_or_else(|| {
GraphCodecError::Invalid(format!(
"edge {edge_index} {role} string id {interned} does not resolve"
))
})
}
fn required_string(reader: &PdgReader, id: u32, field: &str) -> Result<String, GraphCodecError> {
reader
.resolve_string(id)
.map(str::to_owned)
.ok_or_else(|| GraphCodecError::Invalid(format!("invalid interned {field} string id {id}")))
}
fn graph_node_type(code: u32) -> Result<NodeType, GraphCodecError> {
let name = node_type_name_v2(code)?;
StorageNodeType::from_str_name(name)
.map(storage_node_type_to_graph)
.ok_or_else(|| {
GraphCodecError::Invalid(format!(
"unknown node type {name:?}; refusing to decode a lossless layer"
))
})
}
fn graph_edge_type(code: u32) -> Result<EdgeType, GraphCodecError> {
let name = edge_type_name_v2(code)?;
crate::storage::edges::EdgeType::from_str_name(name)
.map(storage_edge_type_to_graph)
.ok_or_else(|| {
GraphCodecError::Invalid(format!(
"unknown edge type {name:?}; refusing to decode a lossless layer"
))
})
}
pub fn collapse_for_layer(pdg: &ProgramDependenceGraph) -> ProgramDependenceGraph {
let mut collapsed = ProgramDependenceGraph::new();
let mut node_index_by_id: HashMap<String, crate::graph::pdg::NodeId> = HashMap::new();
for index in pdg.node_indices() {
let Some(node) = pdg.get_node(index) else {
continue;
};
match node_index_by_id.entry(node.id.clone()) {
std::collections::hash_map::Entry::Occupied(entry) => {
let node_index = *entry.get();
*collapsed.get_node_mut(node_index).expect("occupied node") = node.clone();
}
std::collections::hash_map::Entry::Vacant(entry) => {
let new_index = collapsed.add_node_without_trigrams(node.clone());
entry.insert(new_index);
}
}
}
for id in pdg.precision_symbols() {
collapsed.mark_precision_symbol(id.clone());
}
let mut edges_by_key: HashMap<(String, String, EdgeType), Edge> = HashMap::new();
for edge_id in pdg.edge_indices() {
let Some(edge) = pdg.get_edge(edge_id) else {
continue;
};
let Some((source, target)) = pdg.edge_endpoints(edge_id) else {
continue;
};
let (Some(source), Some(target)) = (pdg.get_node(source), pdg.get_node(target)) else {
continue;
};
if !node_index_by_id.contains_key(&source.id) || !node_index_by_id.contains_key(&target.id)
{
continue;
}
let key = (source.id.clone(), target.id.clone(), edge.edge_type.clone());
let mut edge = edge.clone();
if edge.metadata.confidence.is_some_and(f32::is_nan) {
edge.metadata.confidence = None;
}
edges_by_key.insert(key, edge);
}
for ((source_id, target_id, _), edge) in &edges_by_key {
let source = node_index_by_id[source_id];
let target = node_index_by_id[target_id];
collapsed.add_edge(source, target, edge.clone());
}
collapsed.rebuild_trigram_index();
collapsed
}
fn storage_node_type_to_graph(node_type: StorageNodeType) -> NodeType {
match node_type {
StorageNodeType::Function => NodeType::Function,
StorageNodeType::Class => NodeType::Class,
StorageNodeType::Method => NodeType::Method,
StorageNodeType::Variable => NodeType::Variable,
StorageNodeType::Module => NodeType::Module,
StorageNodeType::External => NodeType::External,
StorageNodeType::DocSection => NodeType::DocSection,
StorageNodeType::FileSummary => NodeType::FileSummary,
}
}
fn storage_edge_type_to_graph(edge_type: crate::storage::edges::EdgeType) -> EdgeType {
use crate::storage::edges::EdgeType as StorageEdgeType;
match edge_type {
StorageEdgeType::Call => EdgeType::Call,
StorageEdgeType::DataDependency => EdgeType::DataDependency,
StorageEdgeType::Inheritance => EdgeType::Inheritance,
StorageEdgeType::Import => EdgeType::Import,
StorageEdgeType::Containment => EdgeType::Containment,
StorageEdgeType::TypeOf => EdgeType::TypeOf,
StorageEdgeType::StateTransition => EdgeType::StateTransition,
StorageEdgeType::CommandArgument => EdgeType::CommandArgument,
StorageEdgeType::Environment => EdgeType::Environment,
StorageEdgeType::Stdin => EdgeType::Stdin,
}
}
pub(crate) fn graph_node_type_str(node_type: &NodeType) -> &'static str {
storage_node_type(node_type).as_str()
}
fn storage_node_type(node_type: &NodeType) -> StorageNodeType {
match node_type {
NodeType::Function => StorageNodeType::Function,
NodeType::Class => StorageNodeType::Class,
NodeType::Method => StorageNodeType::Method,
NodeType::Variable => StorageNodeType::Variable,
NodeType::Module => StorageNodeType::Module,
NodeType::External => StorageNodeType::External,
NodeType::DocSection => StorageNodeType::DocSection,
NodeType::FileSummary => StorageNodeType::FileSummary,
}
}
pub(crate) fn graph_edge_type_str(edge_type: &EdgeType) -> &'static str {
storage_edge_type(edge_type).as_str()
}
fn storage_edge_type(edge_type: &EdgeType) -> crate::storage::edges::EdgeType {
use crate::storage::edges::EdgeType as StorageEdgeType;
match edge_type {
EdgeType::Call => StorageEdgeType::Call,
EdgeType::DataDependency => StorageEdgeType::DataDependency,
EdgeType::Inheritance => StorageEdgeType::Inheritance,
EdgeType::Import => StorageEdgeType::Import,
EdgeType::Containment => StorageEdgeType::Containment,
EdgeType::TypeOf => StorageEdgeType::TypeOf,
EdgeType::StateTransition => StorageEdgeType::StateTransition,
EdgeType::CommandArgument => StorageEdgeType::CommandArgument,
EdgeType::Environment => StorageEdgeType::Environment,
EdgeType::Stdin => StorageEdgeType::Stdin,
}
}
#[cfg(test)]
#[path = "graph_codec_test.rs"]
mod tests;
#[cfg(test)]
pub(crate) use tests::canonical_pdg;