#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Additivity {
Additive,
SemiAdditive,
NonAdditive,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MetricInfo {
pub name: String,
pub owner: String,
pub declared: bool,
pub additivity: Option<Additivity>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DimensionInfo {
pub name: String,
pub owner: String,
pub is_temporal: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ModelInfo {
pub name: String,
pub has_timestamp: bool,
pub columns: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LineageKind {
Model,
Column,
Relationship,
Cube,
Metric,
Dimension,
View,
Query,
Dashboard,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LineageNode {
pub id: String,
pub kind: LineageKind,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LineageEdge {
pub from: String,
pub to: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Severity {
None,
Compatibility,
Structural,
Semantic,
}
impl Severity {
pub fn rank(self) -> u32 {
match self {
Severity::None => 0,
Severity::Compatibility => 1,
Severity::Structural => 2,
Severity::Semantic => 3,
}
}
pub fn label(self) -> &'static str {
match self {
Severity::None => "none",
Severity::Compatibility => "compatibility",
Severity::Structural => "structural",
Severity::Semantic => "semantic",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct RankedSeverity {
pub rank: u32,
pub name: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HostImpact {
pub downstream: Vec<String>,
pub severity: RankedSeverity,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct HostConsumers {
pub queries: usize,
pub dashboards: usize,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct HostAnalysis {
pub impact: std::collections::BTreeMap<String, HostImpact>,
pub consumers: Option<HostConsumers>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BlastRadius {
pub seed: String,
pub downstream: Vec<String>,
pub severity: Severity,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct LineageGraph {
pub nodes: Vec<LineageNode>,
pub edges: Vec<LineageEdge>,
}
impl LineageGraph {
pub fn is_resolvable(&self) -> bool {
self.edges
.iter()
.all(|e| self.contains(&e.from) && self.contains(&e.to))
}
pub fn contains(&self, id: &str) -> bool {
self.nodes.iter().any(|n| n.id == id)
}
pub fn node(&self, id: &str) -> Option<&LineageNode> {
self.nodes.iter().find(|n| n.id == id)
}
pub fn blast_radius(&self, seed: &str) -> BlastRadius {
let mut downstream: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
let mut stack = vec![seed.to_string()];
while let Some(current) = stack.pop() {
for edge in self.edges.iter().filter(|e| e.from == current) {
if edge.to != seed && downstream.insert(edge.to.clone()) {
stack.push(edge.to.clone());
}
}
}
let severity = downstream
.iter()
.map(|id| self.node_severity(id))
.max()
.unwrap_or(Severity::None);
BlastRadius {
seed: seed.to_string(),
downstream: downstream.into_iter().collect(),
severity,
}
}
fn node_severity(&self, id: &str) -> Severity {
match self.node(id).map(|n| n.kind) {
Some(LineageKind::Metric | LineageKind::Query | LineageKind::Dashboard) => {
Severity::Semantic
}
Some(LineageKind::Model | LineageKind::View | LineageKind::Column) => {
Severity::Structural
}
Some(LineageKind::Relationship | LineageKind::Cube | LineageKind::Dimension) => {
Severity::Compatibility
}
None => Severity::None,
}
}
}
pub trait ContextLoader {
fn is_parseable(&self) -> bool;
fn parse_error(&self) -> Option<&str> {
None
}
fn metrics(&self) -> &[MetricInfo];
fn dimensions(&self) -> &[DimensionInfo];
fn time_dimensions(&self) -> &[DimensionInfo];
fn models(&self) -> &[ModelInfo];
fn model(&self, name: &str) -> Option<&ModelInfo> {
self.models().iter().find(|m| m.name == name)
}
fn lineage(&self) -> &LineageGraph;
fn source_introspectable(&self) -> Option<bool> {
None
}
fn raw_docs_readable(&self) -> Option<bool> {
None
}
fn lineage_diagnostics(&self) -> &[String] {
&[]
}
fn metric_additivity(&self, metric: &str) -> Option<Additivity> {
self.metrics()
.iter()
.find(|m| m.name == metric)
.and_then(|m| m.additivity)
}
fn can_answer(&self, predicate: &str) -> bool {
match predicate {
"metric_additive" => self.metrics().iter().any(|m| m.declared),
"source_introspectable" => self.source_introspectable().is_some(),
"raw_docs_readable" => self.raw_docs_readable().is_some(),
"mdl_parseable"
| "wren_project_exists"
| "has_metric"
| "has_queryable_dimension"
| "has_time_dimension"
| "has_groupable_dimension"
| "model_has_timestamp"
| "lineage_resolvable" => true,
_ => false,
}
}
}
#[derive(Debug, Default)]
pub struct ExternalContext {
lineage: LineageGraph,
}
impl ExternalContext {
pub fn new() -> Self {
Self::default()
}
}
impl ContextLoader for ExternalContext {
fn is_parseable(&self) -> bool {
true
}
fn metrics(&self) -> &[MetricInfo] {
&[]
}
fn dimensions(&self) -> &[DimensionInfo] {
&[]
}
fn time_dimensions(&self) -> &[DimensionInfo] {
&[]
}
fn models(&self) -> &[ModelInfo] {
&[]
}
fn lineage(&self) -> &LineageGraph {
&self.lineage
}
fn can_answer(&self, _predicate: &str) -> bool {
false
}
}
pub const PREPARED_CONTEXT_VERSION: u32 = 2;
#[derive(Debug, thiserror::Error)]
pub enum PreparedContextError {
#[error("prepared context is not a valid document: {0}")]
Malformed(#[from] serde_json::Error),
#[error(
"prepared context declares context_version {found}, but this build reads {}",
PREPARED_CONTEXT_VERSION
)]
Version {
found: u32,
},
}
#[derive(Debug, Default)]
pub struct PreparedContext {
parseable: bool,
parse_error: Option<String>,
metrics: Vec<MetricInfo>,
dimensions: Vec<DimensionInfo>,
time_dimensions: Vec<DimensionInfo>,
models: Vec<ModelInfo>,
lineage: LineageGraph,
lineage_diagnostics: Vec<String>,
source_introspectable: Option<bool>,
raw_docs_readable: Option<bool>,
analysis: Option<HostAnalysis>,
}
impl PreparedContext {
pub fn from_json(document: &str) -> Result<Self, PreparedContextError> {
let doc: PreparedDoc = serde_json::from_str(document)?;
if doc.context_version != PREPARED_CONTEXT_VERSION {
return Err(PreparedContextError::Version {
found: doc.context_version,
});
}
let dimensions: Vec<DimensionInfo> = doc
.dimensions
.into_iter()
.map(|d| DimensionInfo {
name: d.name,
owner: d.owner,
is_temporal: d.is_temporal,
})
.collect();
let time_dimensions = dimensions
.iter()
.filter(|d| d.is_temporal)
.cloned()
.collect();
let nodes = doc
.lineage
.nodes
.into_iter()
.map(|n| LineageNode {
id: n.id,
kind: n.kind.into(),
})
.collect();
Ok(Self {
parseable: doc.parseable,
parse_error: doc.parse_error,
metrics: doc
.metrics
.into_iter()
.map(|m| MetricInfo {
name: m.name,
owner: m.owner,
declared: m.declared,
additivity: m.additivity.map(Into::into),
})
.collect(),
dimensions,
time_dimensions,
models: doc
.models
.into_iter()
.map(|m| ModelInfo {
name: m.name,
has_timestamp: m.has_timestamp,
columns: m.columns,
})
.collect(),
lineage: LineageGraph {
nodes,
edges: doc
.lineage
.edges
.into_iter()
.map(|e| LineageEdge {
from: e.from,
to: e.to,
})
.collect(),
},
lineage_diagnostics: doc.lineage_diagnostics,
source_introspectable: doc.source_introspectable,
raw_docs_readable: doc.raw_docs_readable,
analysis: doc.analysis.map(|a| HostAnalysis {
impact: a
.impact
.into_iter()
.map(|(seed, r)| {
(
seed,
HostImpact {
downstream: r.downstream,
severity: RankedSeverity {
rank: r.severity.rank,
name: r.severity.name,
},
},
)
})
.collect(),
consumers: a.consumers.map(|c| HostConsumers {
queries: c.queries,
dashboards: c.dashboards,
}),
}),
})
}
}
impl PreparedContext {
pub fn host_analysis(&self) -> Option<&HostAnalysis> {
self.analysis.as_ref()
}
}
impl ContextLoader for PreparedContext {
fn is_parseable(&self) -> bool {
self.parseable
}
fn parse_error(&self) -> Option<&str> {
self.parse_error.as_deref()
}
fn metrics(&self) -> &[MetricInfo] {
&self.metrics
}
fn dimensions(&self) -> &[DimensionInfo] {
&self.dimensions
}
fn time_dimensions(&self) -> &[DimensionInfo] {
&self.time_dimensions
}
fn models(&self) -> &[ModelInfo] {
&self.models
}
fn lineage(&self) -> &LineageGraph {
&self.lineage
}
fn lineage_diagnostics(&self) -> &[String] {
&self.lineage_diagnostics
}
fn source_introspectable(&self) -> Option<bool> {
self.source_introspectable
}
fn raw_docs_readable(&self) -> Option<bool> {
self.raw_docs_readable
}
}
fn analysis_of(lineage: &LineageGraph) -> PreparedAnalysis {
let count = |kind: LineageKind| lineage.nodes.iter().filter(|n| n.kind == kind).count();
PreparedAnalysis {
impact: lineage
.nodes
.iter()
.map(|node| {
let radius = lineage.blast_radius(&node.id);
(
node.id.clone(),
PreparedImpact {
downstream: radius.downstream,
severity: PreparedSeverity {
rank: radius.severity.rank(),
name: radius.severity.label().to_string(),
},
},
)
})
.collect(),
consumers: Some(PreparedConsumers {
queries: count(LineageKind::Query),
dashboards: count(LineageKind::Dashboard),
}),
}
}
pub fn prepared_document_from(loader: &dyn ContextLoader) -> Result<String, serde_json::Error> {
let doc = PreparedDoc {
context_version: PREPARED_CONTEXT_VERSION,
parseable: loader.is_parseable(),
parse_error: loader.parse_error().map(str::to_string),
metrics: loader
.metrics()
.iter()
.map(|m| PreparedMetric {
name: m.name.clone(),
owner: m.owner.clone(),
declared: m.declared,
additivity: m.additivity.map(Into::into),
})
.collect(),
dimensions: loader
.dimensions()
.iter()
.map(|d| PreparedDimension {
name: d.name.clone(),
owner: d.owner.clone(),
is_temporal: d.is_temporal,
})
.collect(),
models: loader
.models()
.iter()
.map(|m| PreparedModel {
name: m.name.clone(),
has_timestamp: m.has_timestamp,
columns: m.columns.clone(),
})
.collect(),
lineage: PreparedLineage {
nodes: loader
.lineage()
.nodes
.iter()
.map(|n| PreparedNode {
id: n.id.clone(),
kind: n.kind.into(),
})
.collect(),
edges: loader
.lineage()
.edges
.iter()
.map(|e| PreparedEdge {
from: e.from.clone(),
to: e.to.clone(),
})
.collect(),
},
lineage_diagnostics: loader.lineage_diagnostics().to_vec(),
source_introspectable: loader.source_introspectable(),
raw_docs_readable: loader.raw_docs_readable(),
analysis: Some(analysis_of(loader.lineage())),
};
serde_json::to_string_pretty(&doc)
}
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedAnalysis {
#[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
impact: std::collections::BTreeMap<String, PreparedImpact>,
#[serde(default, skip_serializing_if = "Option::is_none")]
consumers: Option<PreparedConsumers>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedImpact {
#[serde(default)]
downstream: Vec<String>,
severity: PreparedSeverity,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedSeverity {
rank: u32,
name: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedConsumers {
queries: usize,
dashboards: usize,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedDoc {
context_version: u32,
parseable: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
parse_error: Option<String>,
#[serde(default)]
metrics: Vec<PreparedMetric>,
#[serde(default)]
dimensions: Vec<PreparedDimension>,
#[serde(default)]
models: Vec<PreparedModel>,
#[serde(default)]
lineage: PreparedLineage,
#[serde(default)]
lineage_diagnostics: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
source_introspectable: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
raw_docs_readable: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
analysis: Option<PreparedAnalysis>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedMetric {
name: String,
owner: String,
declared: bool,
#[serde(default)]
additivity: Option<PreparedAdditivity>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedDimension {
name: String,
owner: String,
is_temporal: bool,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedModel {
name: String,
has_timestamp: bool,
#[serde(default)]
columns: Vec<String>,
}
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedLineage {
#[serde(default)]
nodes: Vec<PreparedNode>,
#[serde(default)]
edges: Vec<PreparedEdge>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedNode {
id: String,
kind: PreparedLineageKind,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct PreparedEdge {
from: String,
to: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
enum PreparedAdditivity {
Additive,
SemiAdditive,
NonAdditive,
}
impl From<PreparedAdditivity> for Additivity {
fn from(a: PreparedAdditivity) -> Self {
match a {
PreparedAdditivity::Additive => Additivity::Additive,
PreparedAdditivity::SemiAdditive => Additivity::SemiAdditive,
PreparedAdditivity::NonAdditive => Additivity::NonAdditive,
}
}
}
impl From<Additivity> for PreparedAdditivity {
fn from(a: Additivity) -> Self {
match a {
Additivity::Additive => PreparedAdditivity::Additive,
Additivity::SemiAdditive => PreparedAdditivity::SemiAdditive,
Additivity::NonAdditive => PreparedAdditivity::NonAdditive,
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
enum PreparedLineageKind {
Model,
Column,
Relationship,
Cube,
Metric,
Dimension,
View,
Query,
Dashboard,
}
impl From<PreparedLineageKind> for LineageKind {
fn from(k: PreparedLineageKind) -> Self {
match k {
PreparedLineageKind::Model => LineageKind::Model,
PreparedLineageKind::Column => LineageKind::Column,
PreparedLineageKind::Relationship => LineageKind::Relationship,
PreparedLineageKind::Cube => LineageKind::Cube,
PreparedLineageKind::Metric => LineageKind::Metric,
PreparedLineageKind::Dimension => LineageKind::Dimension,
PreparedLineageKind::View => LineageKind::View,
PreparedLineageKind::Query => LineageKind::Query,
PreparedLineageKind::Dashboard => LineageKind::Dashboard,
}
}
}
impl From<LineageKind> for PreparedLineageKind {
fn from(k: LineageKind) -> Self {
match k {
LineageKind::Model => PreparedLineageKind::Model,
LineageKind::Column => PreparedLineageKind::Column,
LineageKind::Relationship => PreparedLineageKind::Relationship,
LineageKind::Cube => PreparedLineageKind::Cube,
LineageKind::Metric => PreparedLineageKind::Metric,
LineageKind::Dimension => PreparedLineageKind::Dimension,
LineageKind::View => PreparedLineageKind::View,
LineageKind::Query => PreparedLineageKind::Query,
LineageKind::Dashboard => PreparedLineageKind::Dashboard,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
struct FakeContext {
metrics: Vec<MetricInfo>,
dimensions: Vec<DimensionInfo>,
models: Vec<ModelInfo>,
lineage: LineageGraph,
parseable: bool,
}
impl ContextLoader for FakeContext {
fn is_parseable(&self) -> bool {
self.parseable
}
fn metrics(&self) -> &[MetricInfo] {
&self.metrics
}
fn dimensions(&self) -> &[DimensionInfo] {
&self.dimensions
}
fn time_dimensions(&self) -> &[DimensionInfo] {
&self.dimensions
}
fn models(&self) -> &[ModelInfo] {
&self.models
}
fn lineage(&self) -> &LineageGraph {
&self.lineage
}
}
fn metric(name: &str, declared: bool, additivity: Option<Additivity>) -> MetricInfo {
MetricInfo {
name: name.to_string(),
owner: "m".to_string(),
declared,
additivity,
}
}
#[test]
fn metric_additivity_looks_up_declared_metric() {
let ctx = FakeContext {
metrics: vec![
metric("total_revenue", true, Some(Additivity::Additive)),
metric("avg_order", true, Some(Additivity::NonAdditive)),
metric("amount", false, None),
],
dimensions: vec![],
models: vec![],
lineage: LineageGraph::default(),
parseable: true,
};
assert_eq!(
ctx.metric_additivity("total_revenue"),
Some(Additivity::Additive)
);
assert_eq!(
ctx.metric_additivity("avg_order"),
Some(Additivity::NonAdditive)
);
assert_eq!(ctx.metric_additivity("amount"), None);
assert_eq!(ctx.metric_additivity("nope"), None);
}
#[test]
fn can_answer_metric_additive_needs_a_declared_metric() {
let cubeless = FakeContext {
metrics: vec![metric("amount", false, None)],
dimensions: vec![],
models: vec![],
lineage: LineageGraph::default(),
parseable: true,
};
assert!(!cubeless.can_answer("metric_additive"));
let with_cube = FakeContext {
metrics: vec![metric("total_revenue", true, Some(Additivity::Additive))],
dimensions: vec![],
models: vec![],
lineage: LineageGraph::default(),
parseable: true,
};
assert!(with_cube.can_answer("metric_additive"));
}
#[test]
fn can_answer_existence_predicates_always_true() {
let ctx = FakeContext {
metrics: vec![],
dimensions: vec![],
models: vec![],
lineage: LineageGraph::default(),
parseable: false,
};
for p in [
"mdl_parseable",
"wren_project_exists",
"has_metric",
"has_queryable_dimension",
"has_time_dimension",
"has_groupable_dimension",
"model_has_timestamp",
"lineage_resolvable",
] {
assert!(ctx.can_answer(p), "{p} should be answerable");
}
assert!(!ctx.can_answer("some_future_unknown_predicate"));
}
#[test]
fn mdl_only_adapter_cannot_answer_constitutive_raw_shape() {
let ctx = FakeContext {
metrics: vec![],
dimensions: vec![],
models: vec![],
lineage: LineageGraph::default(),
parseable: true,
};
assert_eq!(ctx.source_introspectable(), None);
assert_eq!(ctx.raw_docs_readable(), None);
assert!(!ctx.can_answer("source_introspectable"));
assert!(!ctx.can_answer("raw_docs_readable"));
}
#[test]
fn raw_source_adapter_answers_constitutive_raw_shape() {
struct RawFake;
impl ContextLoader for RawFake {
fn is_parseable(&self) -> bool {
true
}
fn metrics(&self) -> &[MetricInfo] {
&[]
}
fn dimensions(&self) -> &[DimensionInfo] {
&[]
}
fn time_dimensions(&self) -> &[DimensionInfo] {
&[]
}
fn models(&self) -> &[ModelInfo] {
&[]
}
fn lineage(&self) -> &LineageGraph {
static EMPTY: std::sync::OnceLock<LineageGraph> = std::sync::OnceLock::new();
EMPTY.get_or_init(LineageGraph::default)
}
fn source_introspectable(&self) -> Option<bool> {
Some(true)
}
fn raw_docs_readable(&self) -> Option<bool> {
Some(false)
}
}
let ctx = RawFake;
assert!(ctx.can_answer("source_introspectable"));
assert!(ctx.can_answer("raw_docs_readable"));
assert_eq!(ctx.source_introspectable(), Some(true));
assert_eq!(ctx.raw_docs_readable(), Some(false));
}
#[test]
fn lineage_resolvable_detects_dangling_edges() {
let good = LineageGraph {
nodes: vec![
LineageNode {
id: "model:orders".into(),
kind: LineageKind::Model,
},
LineageNode {
id: "metric:revenue".into(),
kind: LineageKind::Metric,
},
],
edges: vec![LineageEdge {
from: "model:orders".into(),
to: "metric:revenue".into(),
}],
};
assert!(good.is_resolvable());
let dangling = LineageGraph {
nodes: vec![LineageNode {
id: "model:orders".into(),
kind: LineageKind::Model,
}],
edges: vec![LineageEdge {
from: "model:orders".into(),
to: "metric:missing".into(),
}],
};
assert!(!dangling.is_resolvable());
}
fn node(id: &str, kind: LineageKind) -> LineageNode {
LineageNode {
id: id.into(),
kind,
}
}
fn edge(from: &str, to: &str) -> LineageEdge {
LineageEdge {
from: from.into(),
to: to.into(),
}
}
fn sample_graph() -> LineageGraph {
LineageGraph {
nodes: vec![
node("model:orders", LineageKind::Model),
node("cube:revenue", LineageKind::Cube),
node("metric:revenue.total", LineageKind::Metric),
node("dim:revenue.status", LineageKind::Dimension),
node("view:orders_view", LineageKind::View),
],
edges: vec![
edge("model:orders", "cube:revenue"),
edge("cube:revenue", "metric:revenue.total"),
edge("cube:revenue", "dim:revenue.status"),
edge("model:orders", "view:orders_view"),
],
}
}
#[test]
fn blast_radius_is_the_transitive_downstream_closure() {
let graph = sample_graph();
let radius = graph.blast_radius("model:orders");
assert_eq!(
radius.downstream,
vec![
"cube:revenue".to_string(),
"dim:revenue.status".to_string(),
"metric:revenue.total".to_string(),
"view:orders_view".to_string(),
],
"changing the base model reaches the cube, its members, and the view"
);
assert_eq!(radius.severity, Severity::Semantic);
}
#[test]
fn blast_radius_of_a_leaf_is_empty() {
let graph = sample_graph();
let radius = graph.blast_radius("metric:revenue.total");
assert!(radius.downstream.is_empty());
assert_eq!(radius.severity, Severity::None);
}
#[test]
fn blast_radius_severity_is_the_max_over_downstream() {
let graph = sample_graph();
assert_eq!(
graph.blast_radius("cube:revenue").severity,
Severity::Semantic
);
let structural = LineageGraph {
nodes: vec![
node("model:m", LineageKind::Model),
node("view:v", LineageKind::View),
],
edges: vec![edge("model:m", "view:v")],
};
assert_eq!(
structural.blast_radius("model:m").severity,
Severity::Structural
);
}
#[test]
fn consumer_nodes_are_semantic_severity() {
let graph = LineageGraph {
nodes: vec![
node("metric:revenue.total", LineageKind::Metric),
node("query:monthly-revenue", LineageKind::Query),
node("dashboard:exec-weekly", LineageKind::Dashboard),
],
edges: vec![
edge("metric:revenue.total", "query:monthly-revenue"),
edge("metric:revenue.total", "dashboard:exec-weekly"),
],
};
let radius = graph.blast_radius("metric:revenue.total");
assert_eq!(
radius.downstream,
vec![
"dashboard:exec-weekly".to_string(),
"query:monthly-revenue".to_string(),
],
"a metric with consumers is no longer a leaf"
);
assert_eq!(radius.severity, Severity::Semantic);
}
#[test]
fn lineage_diagnostics_default_to_empty() {
let ctx = FakeContext {
metrics: vec![],
dimensions: vec![],
models: vec![],
lineage: LineageGraph::default(),
parseable: true,
};
assert!(ctx.lineage_diagnostics().is_empty());
}
#[test]
fn blast_radius_is_cycle_safe() {
let cyclic = LineageGraph {
nodes: vec![node("a", LineageKind::Model), node("b", LineageKind::Model)],
edges: vec![edge("a", "b"), edge("b", "a")],
};
let radius = cyclic.blast_radius("a");
assert_eq!(radius.downstream, vec!["b".to_string()]);
}
#[test]
fn severity_ordering() {
assert!(Severity::Semantic > Severity::Structural);
assert!(Severity::Structural > Severity::Compatibility);
assert!(Severity::Compatibility > Severity::None);
}
fn prepared_document() -> String {
r#"{
"context_version": 2,
"parseable": true,
"metrics": [
{"name": "total_revenue", "owner": "revenue", "declared": true,
"additivity": "additive"},
{"name": "amount", "owner": "orders", "declared": false}
],
"dimensions": [
{"name": "status", "owner": "orders", "is_temporal": false},
{"name": "ordered_at", "owner": "orders", "is_temporal": true}
],
"models": [
{"name": "orders", "has_timestamp": true, "columns": ["id", "ordered_at"]}
],
"lineage": {
"nodes": [
{"id": "model:orders", "kind": "model"},
{"id": "metric:revenue.total_revenue", "kind": "metric"}
],
"edges": [{"from": "model:orders", "to": "metric:revenue.total_revenue"}]
},
"lineage_diagnostics": ["a consumer's SQL did not parse; used a whole-word scan"],
"analysis": {
"impact": {
"model:orders": {
"downstream": ["metric:revenue.total_revenue"],
"severity": {"rank": 3, "name": "semantic"}
},
"metric:revenue.total_revenue": {
"downstream": [],
"severity": {"rank": 0, "name": "none"}
}
},
"consumers": {"queries": 0, "dashboards": 0}
}
}"#
.to_string()
}
#[test]
fn prepared_context_reads_every_projection() {
let ctx = PreparedContext::from_json(&prepared_document()).expect("document parses");
assert!(ctx.is_parseable());
assert_eq!(ctx.metrics().len(), 2);
assert_eq!(ctx.metrics()[0].additivity, Some(Additivity::Additive));
assert_eq!(ctx.metrics()[1].additivity, None);
assert_eq!(ctx.models()[0].columns, vec!["id", "ordered_at"]);
assert_eq!(ctx.lineage_diagnostics().len(), 1);
}
#[test]
fn prepared_context_derives_time_dimensions_from_the_temporal_subset() {
let ctx = PreparedContext::from_json(&prepared_document()).expect("document parses");
assert_eq!(ctx.dimensions().len(), 2);
assert_eq!(ctx.time_dimensions().len(), 1);
assert_eq!(ctx.time_dimensions()[0].name, "ordered_at");
}
#[test]
fn prepared_context_supports_blast_radius() {
let ctx = PreparedContext::from_json(&prepared_document()).expect("document parses");
let radius = ctx.lineage().blast_radius("model:orders");
assert_eq!(
radius.downstream,
vec!["metric:revenue.total_revenue".to_string()]
);
assert_eq!(radius.severity, Severity::Semantic);
assert!(ctx.lineage().is_resolvable());
}
#[test]
fn prepared_context_keeps_raw_shape_probes_unanswerable_when_omitted() {
let ctx = PreparedContext::from_json(&prepared_document()).expect("document parses");
assert_eq!(ctx.source_introspectable(), None);
assert!(!ctx.can_answer("source_introspectable"));
assert!(!ctx.can_answer("raw_docs_readable"));
assert!(ctx.can_answer("metric_additive"));
}
#[test]
fn prepared_context_answers_raw_shape_probes_when_stated() {
let doc = r#"{"context_version": 2, "parseable": true,
"source_introspectable": true, "raw_docs_readable": false}"#;
let ctx = PreparedContext::from_json(doc).expect("document parses");
assert_eq!(ctx.source_introspectable(), Some(true));
assert!(ctx.can_answer("source_introspectable"));
assert_eq!(ctx.raw_docs_readable(), Some(false));
assert!(ctx.can_answer("raw_docs_readable"));
}
#[test]
fn prepared_context_carries_the_parse_error_of_an_unparseable_layer() {
let doc = r#"{"context_version": 2, "parseable": false,
"parse_error": "models/orders/metadata.yml: missing `columns`"}"#;
let ctx = PreparedContext::from_json(doc).expect("document parses");
assert!(!ctx.is_parseable());
assert!(ctx.parse_error().expect("error text").contains("columns"));
}
#[test]
fn prepared_context_rejects_an_unreadable_version() {
let doc = r#"{"context_version": 99, "parseable": true}"#;
let err = PreparedContext::from_json(doc).expect_err("version is not readable");
assert!(matches!(err, PreparedContextError::Version { found: 99 }));
}
#[test]
fn prepared_context_rejects_an_unknown_field() {
let doc = r#"{"context_version": 2, "parseable": true, "metrics_v2": []}"#;
let err = PreparedContext::from_json(doc).expect_err("unknown field is rejected");
assert!(matches!(err, PreparedContextError::Malformed(_)));
}
#[test]
fn prepared_context_reads_the_hosts_analysis() {
let ctx = PreparedContext::from_json(&prepared_document()).expect("document parses");
let analysis = ctx
.host_analysis()
.expect("the document carried an analysis");
let radius = analysis
.impact
.get("model:orders")
.expect("the host analysed the model");
assert_eq!(radius.downstream, vec!["metric:revenue.total_revenue"]);
assert_eq!(radius.severity.rank, 3);
assert_eq!(radius.severity.name, "semantic");
assert_eq!(
analysis.consumers,
Some(HostConsumers {
queries: 0,
dashboards: 0
})
);
}
#[test]
fn prepared_context_keeps_a_severity_name_it_does_not_understand() {
let doc = r#"{"context_version": 2, "parseable": true,
"analysis": {"impact": {"model:orders":
{"downstream": [], "severity": {"rank": 7, "name": "catastrophic"}}}}}"#;
let ctx = PreparedContext::from_json(doc).expect("an unknown severity name is data");
let sev = &ctx.host_analysis().unwrap().impact["model:orders"].severity;
assert_eq!(sev.rank, 7);
assert_eq!(sev.name, "catastrophic");
}
#[test]
fn prepared_context_separates_an_absent_analysis_from_an_empty_one() {
let absent = r#"{"context_version": 2, "parseable": true}"#;
assert!(PreparedContext::from_json(absent)
.expect("parses")
.host_analysis()
.is_none());
let empty = r#"{"context_version": 2, "parseable": true, "analysis": {}}"#;
let analysis = PreparedContext::from_json(empty)
.expect("parses")
.host_analysis()
.cloned()
.expect("an empty analysis is still an analysis");
assert!(analysis.impact.is_empty());
assert_eq!(analysis.consumers, None);
}
#[test]
fn prepared_analysis_rejects_an_unknown_field() {
let doc = r#"{"context_version": 2, "parseable": true,
"analysis": {"impact": {}, "consumers_v2": {}}}"#;
assert!(PreparedContext::from_json(doc).is_err());
}
#[test]
fn the_writer_emits_an_analysis_the_reader_reads_back() {
let ctx = PreparedContext::from_json(&prepared_document()).expect("document parses");
let rendered = prepared_document_from(&ctx).expect("the projection serializes");
let reread = PreparedContext::from_json(&rendered).expect("the rendering reads back");
assert_eq!(reread.host_analysis(), ctx.host_analysis());
let native = ctx.lineage().blast_radius("model:orders");
let carried = &reread.host_analysis().unwrap().impact["model:orders"];
assert_eq!(carried.downstream, native.downstream);
assert_eq!(carried.severity.rank, native.severity.rank());
}
#[test]
fn prepared_context_rejects_an_unknown_lineage_kind() {
let doc = r#"{"context_version": 2, "parseable": true,
"lineage": {"nodes": [{"id": "x", "kind": "wormhole"}], "edges": []}}"#;
let err = PreparedContext::from_json(doc).expect_err("unknown kind is rejected");
assert!(matches!(err, PreparedContextError::Malformed(_)));
}
}