1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
//! AVRO schema exporter for generating AVRO schemas from data models.
use super::{ExportError, ExportResult};
use crate::models::{DataModel, Table};
use serde_json::{Value, json};
/// Exporter for AVRO schema format.
pub struct AvroExporter;
impl AvroExporter {
/// Export tables to AVRO schema format (SDK interface).
///
/// # Arguments
///
/// * `tables` - Slice of tables to export
///
/// # Returns
///
/// An `ExportResult` containing AVRO schema(s) as JSON.
/// If a single table is provided, returns a single schema object.
/// If multiple tables are provided, returns an array of schemas.
///
/// # Example
///
/// ```rust
/// use data_modelling_core::export::avro::AvroExporter;
/// use data_modelling_core::models::{Table, Column};
///
/// let tables = vec![
/// Table::new("User".to_string(), vec![Column::new("id".to_string(), "INT64".to_string())]),
/// ];
///
/// let exporter = AvroExporter;
/// let result = exporter.export(&tables).unwrap();
/// assert_eq!(result.format, "avro");
/// ```
pub fn export(&self, tables: &[Table]) -> Result<ExportResult, ExportError> {
let schema = Self::export_model_from_tables(tables);
let content = serde_json::to_string_pretty(&schema)
.map_err(|e| ExportError::SerializationError(e.to_string()))?;
// Validate exported AVRO schema
{
use crate::validation::schema::validate_avro_internal;
validate_avro_internal(&content).map_err(|e| {
ExportError::ValidationError(format!("AVRO validation failed: {}", e))
})?;
}
Ok(ExportResult {
content,
format: "avro".to_string(),
})
}
fn export_model_from_tables(tables: &[Table]) -> serde_json::Value {
if tables.len() == 1 {
Self::export_table(&tables[0])
} else {
let schemas: Vec<serde_json::Value> = tables.iter().map(Self::export_table).collect();
serde_json::json!(schemas)
}
}
/// Export a table to AVRO schema format.
///
/// # Arguments
///
/// * `table` - The table to export
///
/// # Returns
///
/// A `serde_json::Value` representing the AVRO schema for the table.
///
/// # Example
///
/// ```rust
/// use data_modelling_core::export::avro::AvroExporter;
/// use data_modelling_core::models::{Table, Column};
///
/// let table = Table::new(
/// "User".to_string(),
/// vec![Column::new("id".to_string(), "INT64".to_string())],
/// );
///
/// let schema = AvroExporter::export_table(&table);
/// assert_eq!(schema["type"], "record");
/// assert_eq!(schema["name"], "User");
/// ```
pub fn export_table(table: &Table) -> Value {
let mut fields = Vec::new();
for column in &table.columns {
let mut field = serde_json::Map::new();
field.insert("name".to_string(), json!(column.name));
// Map data type to AVRO type
let avro_type = Self::map_data_type_to_avro(&column.data_type, column.nullable);
field.insert("type".to_string(), avro_type);
if !column.description.is_empty() {
field.insert("doc".to_string(), json!(column.description));
}
fields.push(json!(field));
}
let mut schema = serde_json::Map::new();
schema.insert("type".to_string(), json!("record"));
schema.insert("name".to_string(), json!(table.name));
// Add tags if present (AVRO doesn't have standard tags, but we can add them as metadata)
if !table.tags.is_empty() {
let tags_array: Vec<String> = table.tags.iter().map(|t| t.to_string()).collect();
schema.insert("tags".to_string(), json!(tags_array));
}
schema.insert("namespace".to_string(), json!("com.datamodel"));
schema.insert("fields".to_string(), json!(fields));
json!(schema)
}
/// Export a data model to AVRO schema format (legacy method for compatibility).
pub fn export_model(model: &DataModel, table_ids: Option<&[uuid::Uuid]>) -> Value {
let tables_to_export: Vec<&Table> = if let Some(ids) = table_ids {
model
.tables
.iter()
.filter(|t| ids.contains(&t.id))
.collect()
} else {
model.tables.iter().collect()
};
if tables_to_export.len() == 1 {
// Single table: return the schema directly
Self::export_table(tables_to_export[0])
} else {
// Multiple tables: return array of schemas
let schemas: Vec<Value> = tables_to_export
.iter()
.map(|t| Self::export_table(t))
.collect();
json!(schemas)
}
}
/// Map SQL/ODCL data types to AVRO types.
fn map_data_type_to_avro(data_type: &str, nullable: bool) -> Value {
let dt_lower = data_type.to_lowercase();
let avro_type = match dt_lower.as_str() {
"int" | "integer" | "smallint" | "tinyint" => json!("int"),
"bigint" => json!("long"),
"float" | "real" => json!("float"),
"double" | "decimal" | "numeric" => json!("double"),
"boolean" | "bool" => json!("boolean"),
"bytes" | "binary" | "varbinary" => json!("bytes"),
_ => {
// Default to string for VARCHAR, TEXT, CHAR, DATE, TIMESTAMP, etc.
json!("string")
}
};
if nullable {
json!(["null", avro_type])
} else {
avro_type
}
}
}