1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SchemaInferenceConfig {
/// Whether to treat top-level arrays as streams of objects
pub ignore_outer_array: bool,
/// Delimiter for NDJSON format (None for regular JSON)
pub delimiter: Option<u8>,
/// Schema URI to use ("AUTO" for auto-detection)
pub schema_uri: Option<String>,
/// Threshold above which non-fixed keys are treated as a map
pub map_threshold: usize,
/// Maximum number of required keys a Map can have. If None, no gating based on required keys.
/// If Some(n), objects with more than n required keys will be forced to Record type.
pub map_max_required_keys: Option<usize>,
/// Enable unification of compatible but non-homogeneous record schemas into maps
pub unify_maps: bool,
/// Force override of field treatment, e.g. {"labels": "map"}
pub force_field_types: HashMap<String, String>,
/// Whether to promote scalar values to wrapped objects when they collide with record values
/// during unification. If `true`, scalars are promoted under a synthetic property name derived from
/// the parent field and the scalar type (e.g. "foo__string"). If `false`, don't unify on conflicts.
pub wrap_scalars: bool,
/// Wrap the inferred top-level schema under a single required field with this name.
/// Example: wrap_root = Some("labels") turns `{...}` into
/// `{"type":"object","properties":{"labels":{...}},"required":["labels"]}`.
pub wrap_root: Option<String>,
/// Whether to output Avro schema rather than regular JSON Schema.
#[cfg(feature = "avro")]
pub avro: bool,
/// Enable debug output. When `true`, prints detailed information about schema inference
/// processes including field unification, map detection, and scalar wrapping decisions.
pub debug: bool,
/// Controls the verbosity level of debug output
pub verbosity: DebugVerbosity,
}
#[derive(Default, Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
pub enum DebugVerbosity {
/// Show important unification decisions and failures
#[default]
Normal,
/// Show all debug information including field introductions
Verbose,
}
impl SchemaInferenceConfig {
pub(crate) fn debug(&self, args: std::fmt::Arguments) {
if self.debug {
let message = format!("{}", args);
eprintln!("{}", self.maybe_truncate(message));
}
}
pub(crate) fn debug_verbose(&self, args: std::fmt::Arguments) {
if self.debug && matches!(self.verbosity, DebugVerbosity::Verbose) {
let message = format!("{}", args);
eprintln!("{}", self.maybe_truncate(message));
}
}
fn maybe_truncate(&self, message: String) -> String {
let lines: Vec<&str> = message.lines().collect();
if lines.len() > 20 && self.verbosity == DebugVerbosity::Normal {
let mut truncated = String::new();
// First 10 lines
for line in lines.iter().take(10) {
truncated.push_str(line);
truncated.push('\n');
}
truncated.push_str(&format!("... ({} lines truncated) ...\n", lines.len() - 15));
// Last 5 lines
for line in lines.iter().skip(lines.len() - 5) {
truncated.push_str(line);
truncated.push('\n');
}
truncated
} else {
message
}
}
}
impl Default for SchemaInferenceConfig {
fn default() -> Self {
Self {
ignore_outer_array: true,
delimiter: None,
schema_uri: Some("AUTO".to_string()),
map_threshold: 20,
map_max_required_keys: None,
unify_maps: false,
force_field_types: std::collections::HashMap::new(),
wrap_scalars: true,
wrap_root: None,
#[cfg(feature = "avro")]
avro: false,
debug: false,
verbosity: DebugVerbosity::default(),
}
}
}
#[macro_export]
macro_rules! debug {
($cfg:expr, $($arg:tt)*) => {
$cfg.debug(format_args!($($arg)*))
};
}
#[macro_export]
macro_rules! debug_verbose {
($cfg:expr, $($arg:tt)*) => {
$cfg.debug_verbose(format_args!($($arg)*))
};
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SchemaInferenceResult {
pub schema: Value,
pub processed_count: usize,
}
#[cfg(feature = "avro")]
impl SchemaInferenceResult {
pub fn to_avro_schema(
&self,
namespace: &str,
utility_namespace: Option<&str>,
base_uri: Option<&str>,
split_top_level: bool,
) -> Value {
avrotize::converter::jsons_to_avro(
&self.schema,
namespace,
utility_namespace.unwrap_or(""),
base_uri.unwrap_or("genson-core"),
split_top_level,
)
}
}