use core::panic;
use std::borrow::{Borrow, Cow};
use std::hash::Hasher;
use crate::dag::{kw, Dag, Id};
use crate::genson_rs::interned::{RowStates, StateId, Strat};
use crate::genson_rs::strategy::base::SchemaStrategy;
use crate::genson_rs::strategy::scalar::TypelessStrategy;
use crate::genson_rs::strategy::BasicSchemaStrategy;
use crate::value::object;
use ordermap::OrderMap;
use serde_json::{json, Value};
pub(crate) const PAR_MERGE_MIN: usize = 8;
#[derive(Default)]
pub(crate) struct Deferred<'a>(Vec<DeferredStrategy<'a>>);
#[derive(Default)]
pub(crate) enum DeferredStrategy<'a> {
#[default]
None,
Object(Vec<Vec<(&'a RowStates, StateId)>>),
List(Vec<(&'a RowStates, StateId)>),
}
pub(crate) fn hash_value<H: Hasher>(value: &Value, hasher: &mut H) {
struct Writer<'a, H>(&'a mut H);
impl<H: Hasher> std::io::Write for Writer<'_, H> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.write(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
serde_json::to_writer(Writer(hasher), value).expect("hashing a Value cannot fail");
}
#[derive(Clone, Debug, PartialEq)]
pub struct SchemaNode {
active_strategies: Vec<BasicSchemaStrategy>,
absorbed: Absorbed,
}
#[derive(Clone, Default)]
struct Absorbed(rustc_hash::FxHashSet<u64>);
impl PartialEq for Absorbed {
fn eq(&self, _: &Self) -> bool {
true
}
}
impl std::fmt::Debug for Absorbed {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Absorbed({})", self.0.len())
}
}
#[derive(Clone)]
pub enum DataType<'a> {
Schema(&'a Value),
Object(super::JsonValue<'a>),
SchemaNode(&'a SchemaNode),
}
impl Default for SchemaNode {
fn default() -> Self {
Self::new()
}
}
impl SchemaNode {
pub fn new() -> Self {
SchemaNode {
active_strategies: vec![],
absorbed: Absorbed::default(),
}
}
pub fn is_empty(&self) -> bool {
self.active_strategies.is_empty()
}
pub fn is_object(&self) -> bool {
matches!(
self.active_strategies.as_slice(),
[BasicSchemaStrategy::Object(_)]
)
}
pub(crate) fn unintern(&mut self) {
self.active_strategies
.iter_mut()
.for_each(BasicSchemaStrategy::unintern);
}
pub(crate) fn visit_properties_mut<F>(&mut self, f: &mut F)
where
F: FnMut(&str, &mut SchemaNode) -> bool,
{
for strategy in &mut self.active_strategies {
strategy.visit_properties_mut(f);
}
}
pub(crate) fn add_object_keyword(&mut self, key: &str, value: Value) {
for strategy in &mut self.active_strategies {
if let BasicSchemaStrategy::Object(object) = strategy {
object.add_extra_keywords(&json!({ key: value }));
return;
}
}
}
pub fn add_schema(&mut self, data: DataType) -> &mut Self {
let schema: Cow<Value> = match data {
DataType::SchemaNode(node) => Cow::Owned(node.to_schema()),
DataType::Schema(schema) => Cow::Borrowed(schema),
_ => panic!("Invalid schema type"),
};
if !SchemaNode::needs_splitting(&schema) {
let active_strategy = self.get_or_create_strategy_for_schema(&schema);
SchemaNode::add_schema_or_object_to_strategy(
active_strategy,
DataType::Schema(&schema),
);
return self;
}
for subschema in SchemaNode::get_subschemas(&schema) {
let active_strategy = self.get_or_create_strategy_for_schema(&subschema);
SchemaNode::add_schema_or_object_to_strategy(
active_strategy,
DataType::Schema(&subschema),
);
}
self
}
pub fn absorb(&mut self, other: SchemaNode) {
let mut strategies = other.active_strategies;
match strategies.len() {
0 => self.absorb_strategy(BasicSchemaStrategy::Typeless(TypelessStrategy::new())),
1 => self.absorb_strategy(strategies.pop().unwrap()),
_ => Self::schema_order(strategies)
.into_iter()
.for_each(|strategy| self.absorb_strategy(strategy)),
}
}
pub(crate) fn round_tripped(node: SchemaNode) -> SchemaNode {
let mut fresh = SchemaNode::new();
fresh.absorb(node);
fresh
}
fn schema_order<S: Borrow<BasicSchemaStrategy>>(strategies: Vec<S>) -> Vec<S> {
let (mut bare, mut ordered): (Vec<_>, Vec<_>) = strategies
.into_iter()
.partition(|strategy| strategy.borrow().bare_type().is_some());
bare.sort_by_key(|strategy| strategy.borrow().bare_type());
ordered.extend(bare);
ordered
}
pub fn absorb_all(&mut self, mut others: Vec<SchemaNode>) {
let sole_kind = |node: &SchemaNode| match node.active_strategies.as_slice() {
[strategy @ (BasicSchemaStrategy::Object(_) | BasicSchemaStrategy::List(_))] => {
Some(std::mem::discriminant(strategy))
}
_ => None,
};
let kind = others.first().and_then(sole_kind);
let parallel = others.len() >= PAR_MERGE_MIN
&& kind.is_some()
&& others.iter().all(|other| sole_kind(other) == kind)
&& (self.is_empty() || sole_kind(self) == kind);
if !parallel {
others.into_iter().for_each(|other| self.absorb(other));
return;
}
if self.is_empty() {
let created = others[0].active_strategies[0].new_of_same_kind();
self.active_strategies.reserve_exact(1);
self.active_strategies.push(created);
}
let strategies = others
.iter_mut()
.filter_map(|other| other.active_strategies.pop());
match &mut self.active_strategies[0] {
BasicSchemaStrategy::Object(own) => own.absorb_all(
strategies
.filter_map(|strategy| match strategy {
BasicSchemaStrategy::Object(object) => Some(object),
_ => None,
})
.collect(),
),
BasicSchemaStrategy::List(own) => own.absorb_all(
strategies
.filter_map(|strategy| match strategy {
BasicSchemaStrategy::List(list) => Some(list),
_ => None,
})
.collect(),
),
_ => unreachable!("only objects and lists merge in parallel"),
}
}
pub fn hash_schema<H: Hasher>(&self, hasher: &mut H) {
if let [strategy] = self.active_strategies.as_slice() {
if !matches!(strategy, BasicSchemaStrategy::Typeless(_)) {
hasher.write_usize(1);
strategy.hash_schema(hasher);
return;
}
}
let written = self.active_strategies.iter().filter(|strategy| {
!matches!(strategy, BasicSchemaStrategy::Typeless(t) if t.to_schema() == json!({}))
});
let ordered = Self::schema_order(written.collect());
hasher.write_usize(ordered.len());
for strategy in ordered {
strategy.hash_schema(hasher);
}
}
pub(crate) fn absorb_interned(&mut self, rows: &RowStates, state: StateId) {
if !self.absorbed.0.insert(rows.identity(state)) {
return;
}
let strats = rows.strats(state);
match strats.len() {
0 => self.absorb_strategy(BasicSchemaStrategy::Typeless(TypelessStrategy::new())),
1 => self.absorb_interned_strategy(rows, strats[0]),
_ => {
for strat in rows.schema_order(strats) {
self.absorb_interned_strategy(rows, strat);
}
}
}
}
pub(crate) fn absorb_interned_all(&mut self, states: &[(&RowStates, StateId)]) {
if let [(rows, state)] = states {
return self.absorb_interned(rows, *state);
}
let mut deferred = Deferred::default();
for &(rows, state) in states {
self.absorb_interned_deferred(rows, state, &mut deferred);
}
self.absorb_deferred(deferred);
}
pub(crate) fn absorb_interned_deferred<'a>(
&mut self,
rows: &'a RowStates,
state: StateId,
deferred: &mut Deferred<'a>,
) {
if !self.absorbed.0.insert(rows.identity(state)) {
return;
}
let strats = rows.strats(state);
match strats.len() {
0 => self.absorb_strategy(BasicSchemaStrategy::Typeless(TypelessStrategy::new())),
1 => self.absorb_interned_strategy_deferred(rows, strats[0], deferred),
_ => {
for strat in rows.schema_order(strats) {
self.absorb_interned_strategy_deferred(rows, strat, deferred);
}
}
}
}
fn absorb_interned_strategy_deferred<'a>(
&mut self,
rows: &'a RowStates,
strat: Strat,
deferred: &mut Deferred<'a>,
) {
let at = match self
.active_strategies
.iter()
.position(|existing| existing.same_kind_as_interned(strat))
{
Some(at) => at,
None => {
let mut created = BasicSchemaStrategy::new_of_interned_kind(strat);
if let Some(BasicSchemaStrategy::Typeless(typeless)) = self.active_strategies.last()
{
created.add_schema(&typeless.to_schema());
self.active_strategies.pop();
}
if self.active_strategies.is_empty() {
self.active_strategies.reserve_exact(1);
}
self.active_strategies.push(created);
self.active_strategies.len() - 1
}
};
deferred
.0
.resize_with(self.active_strategies.len(), Default::default);
self.active_strategies[at].absorb_interned_deferred(rows, strat, &mut deferred.0[at]);
}
pub(crate) fn absorb_deferred(&mut self, deferred: Deferred) {
for (strategy, deferred) in self.active_strategies.iter_mut().zip(deferred.0) {
strategy.absorb_deferred(deferred);
}
}
fn absorb_interned_strategy(&mut self, rows: &RowStates, strat: Strat) {
if let Some(existing) = self
.active_strategies
.iter_mut()
.find(|existing| existing.same_kind_as_interned(strat))
{
existing.absorb_interned(rows, strat);
return;
}
let mut created = BasicSchemaStrategy::new_of_interned_kind(strat);
if let Some(BasicSchemaStrategy::Typeless(typeless)) = self.active_strategies.last() {
created.add_schema(&typeless.to_schema());
self.active_strategies.pop();
}
created.absorb_interned(rows, strat);
if self.active_strategies.is_empty() {
self.active_strategies.reserve_exact(1);
}
self.active_strategies.push(created);
}
fn absorb_strategy(&mut self, strategy: BasicSchemaStrategy) {
if let Some(existing) = self
.active_strategies
.iter_mut()
.find(|existing| existing.same_kind(&strategy))
{
existing.absorb(strategy);
return;
}
if let BasicSchemaStrategy::Typeless(typeless) = strategy {
if self.active_strategies.is_empty() {
self.active_strategies
.push(BasicSchemaStrategy::Typeless(TypelessStrategy::new()));
}
self.active_strategies[0].add_schema(&typeless.to_schema());
return;
}
let mut created = strategy.new_of_same_kind();
if let Some(BasicSchemaStrategy::Typeless(typeless)) = self.active_strategies.last() {
created.add_schema(&typeless.to_schema());
self.active_strategies.pop();
}
created.absorb(strategy);
if self.active_strategies.is_empty() {
self.active_strategies.reserve_exact(1);
}
self.active_strategies.push(created);
}
fn needs_splitting(schema: &Value) -> bool {
match schema {
Value::Object(obj) => {
matches!(obj.get("anyOf"), Some(Value::Array(_)))
|| matches!(obj.get("type"), Some(Value::Array(_)))
}
_ => false,
}
}
pub fn add_schemas_par(&mut self, schemas: &[&Value]) {
if schemas.len() >= PAR_MERGE_MIN
&& (self.add_object_schemas_par(schemas) || self.add_list_schemas_par(schemas))
{
return;
}
for schema in schemas {
self.add_schema(DataType::Schema(schema));
}
}
fn add_list_schemas_par(&mut self, schemas: &[&Value]) -> bool {
let plain_lists = schemas.iter().all(|schema| {
schema["type"] == "array"
&& schema["items"].is_object()
&& !SchemaNode::needs_splitting(schema)
});
if !plain_lists {
return false;
}
match self.active_strategies.as_slice() {
[] => {
self.get_or_create_strategy_for_schema(schemas[0]);
}
[BasicSchemaStrategy::List(_)] => {}
_ => return false,
}
match self.active_strategies.first_mut() {
Some(BasicSchemaStrategy::List(strategy)) => {
let items: Vec<&Value> = schemas.iter().map(|schema| &schema["items"]).collect();
strategy.add_item_schemas_par(&items);
true
}
_ => false,
}
}
pub fn add_object_schemas_par(&mut self, schemas: &[&Value]) -> bool {
if schemas.is_empty() {
return true;
}
let plain_objects = schemas
.iter()
.all(|schema| schema["type"] == "object" && !SchemaNode::needs_splitting(schema));
if !plain_objects {
return false;
}
match self.active_strategies.as_slice() {
[] => {
self.get_or_create_strategy_for_schema(schemas[0]);
}
[BasicSchemaStrategy::Object(_)] => {}
_ => return false,
}
match self.active_strategies.first_mut() {
Some(BasicSchemaStrategy::Object(strategy)) => {
strategy.add_schemas_par(schemas);
true
}
_ => false,
}
}
pub fn add_schemas(&mut self, schemas: &[&Value]) -> &mut Self {
let mut all_subschemas: Vec<Cow<Value>> = Vec::new();
let mut schema_groups: OrderMap<usize, Vec<usize>> = OrderMap::new();
for schema in schemas {
for subschema in SchemaNode::get_subschemas(schema) {
let strategy_idx =
if let Some(idx) = self.get_strategy_for_kind(DataType::Schema(&subschema)) {
idx
} else {
if let Some(_strategy) =
self.create_strategy_for_kind(DataType::Schema(&subschema))
{
self.active_strategies.len() - 1
} else {
continue; }
};
let subschema_idx = all_subschemas.len();
all_subschemas.push(subschema);
schema_groups
.entry(strategy_idx)
.or_default()
.push(subschema_idx);
}
}
for (strategy_idx, subschema_indices) in schema_groups {
if let Some(strategy) = self.active_strategies.get_mut(strategy_idx) {
let schema_refs: Vec<&Value> = subschema_indices
.iter()
.map(|&idx| all_subschemas[idx].as_ref())
.collect();
strategy.add_schemas(&schema_refs);
}
}
self
}
fn get_subschemas(schema: &Value) -> Vec<Cow<'_, Value>> {
if let Value::Object(obj) = schema {
if let Some(Value::Array(anyof)) = obj.get("anyOf") {
return anyof.iter().flat_map(SchemaNode::get_subschemas).collect();
} else if let Some(Value::Array(types)) = obj.get("type") {
return types
.iter()
.map(|t| {
let mut new_schema = obj.clone();
new_schema["type"] = t.clone();
Cow::Owned(Value::Object(new_schema))
})
.collect();
} else {
return vec![Cow::Borrowed(schema)];
}
}
vec![Cow::Borrowed(schema)]
}
pub fn add_object(&mut self, data: DataType) -> &mut Self {
let object = match data {
DataType::Object(obj) => obj,
_ => panic!("Invalid object type"),
};
let active_strategy = self.get_or_create_strategy_for_object(object);
SchemaNode::add_schema_or_object_to_strategy(active_strategy, DataType::Object(object));
self
}
pub(crate) fn to_dag(&self, dag: &mut Dag) -> Id {
if let [strategy] = self.active_strategies.as_slice() {
let schema = strategy.to_dag(dag);
if dag.as_object(schema).is_none() {
panic!("Invalid schema type for strategy {:?}", strategy);
}
return schema;
}
let start = dag.items.len();
let mut scalar_schemas: Vec<Id> = vec![];
for strategy in &self.active_strategies {
let schema = strategy.to_dag(dag);
let Some(keywords) = dag.as_object(schema) else {
panic!("Invalid schema type for strategy {:?}", strategy);
};
if keywords.len() == 1 && keywords.first().is_some_and(|(k, _)| k == kw::TYPE) {
scalar_schemas.push(schema);
} else {
dag.items.push(schema);
}
}
if scalar_schemas.len() == 1 {
dag.items.push(scalar_schemas[0]);
} else if !scalar_schemas.is_empty() {
let mut types: Vec<&str> = scalar_schemas
.iter()
.map(|&schema| {
let js_type = dag.get(schema, kw::TYPE).and_then(|t| dag.as_sym(t));
dag.name(js_type.unwrap())
})
.collect();
types.sort();
types.dedup();
if types.len() == 1 {
dag.items.push(scalar_schemas[0]);
} else {
let types: Vec<String> = types.into_iter().map(str::to_string).collect();
let types: Vec<Id> = types.iter().map(|t| dag.str(t)).collect();
let types = dag.array(&types);
let schema = dag.object(&[(kw::TYPE, types)]);
dag.items.push(schema);
}
}
match dag.items.len() - start {
1 => dag.items.pop().unwrap(),
0 => dag.object(&[]),
_ => {
let any_of = dag.array_from(start);
dag.object(&[(kw::ANY_OF, any_of)])
}
}
}
pub fn to_schema(&self) -> Value {
let mut scalar_schemas: Vec<Value> = vec![];
let mut generated_schemas: Vec<Value> = vec![];
self.active_strategies.iter().for_each(|strategy| {
let generated_schema: Value = strategy.to_schema();
if let Value::Object(ref schema) = generated_schema {
if schema.keys().len() == 1 && schema.contains_key("type") {
scalar_schemas.push(generated_schema);
} else {
generated_schemas.push(generated_schema);
}
} else {
panic!("Invalid schema type for strategy {:?}", strategy);
}
});
if scalar_schemas.len() == 1 {
generated_schemas.push(scalar_schemas.swap_remove(0));
} else if !scalar_schemas.is_empty() {
let mut scalar_type_list: Vec<&str> = scalar_schemas
.iter()
.map(|schema| schema["type"].as_str().unwrap())
.collect();
scalar_type_list.sort();
scalar_type_list.dedup();
if scalar_type_list.len() == 1 {
generated_schemas.push(scalar_schemas.swap_remove(0));
} else {
generated_schemas.push(json!({"type": scalar_type_list}));
}
}
if generated_schemas.len() == 1 {
generated_schemas.swap_remove(0)
} else if !generated_schemas.is_empty() {
object([("anyOf", Value::Array(generated_schemas))])
} else {
json!({})
}
}
fn get_or_create_strategy_for_object(
&mut self,
object: super::JsonValue,
) -> &mut BasicSchemaStrategy {
if let Some(idx) = self.get_strategy_for_kind(DataType::Object(object)) {
return &mut self.active_strategies[idx];
}
if let Some(strategy) = self.create_strategy_for_kind(DataType::Object(object)) {
return strategy;
}
panic!("Could not find matching schema type for object: {object:?}")
}
fn get_or_create_strategy_for_schema(&mut self, schema: &Value) -> &mut BasicSchemaStrategy {
if let Some(idx) = self.get_strategy_for_kind(DataType::Schema(schema)) {
return &mut self.active_strategies[idx];
}
if let Some(strategy) = self.create_strategy_for_kind(DataType::Schema(schema)) {
return strategy;
}
panic!("Could not find matching schema type for schema: {schema}")
}
fn get_strategy_for_kind(&self, schema_or_object: DataType) -> Option<usize> {
self.active_strategies.iter().position(|strategy| {
SchemaNode::strategy_does_match_schema_or_object(strategy, &schema_or_object)
})
}
fn create_strategy_for_kind(
&mut self,
schema_or_object: DataType,
) -> Option<&mut BasicSchemaStrategy> {
if let Some(mut strategy) =
SchemaNode::create_strategy_for_schema_or_object(&schema_or_object)
{
if let Some(BasicSchemaStrategy::Typeless(typeless)) = self.active_strategies.last() {
SchemaNode::add_schema_or_object_to_strategy(
&mut strategy,
DataType::Schema(&typeless.to_schema()),
);
self.active_strategies.pop();
}
if self.active_strategies.is_empty() {
self.active_strategies.reserve_exact(1);
}
self.active_strategies.push(strategy);
return Some(self.active_strategies.last_mut().unwrap());
}
else if let DataType::Schema(schema) = schema_or_object {
if TypelessStrategy::match_schema(schema) {
if self.active_strategies.is_empty() {
self.active_strategies
.push(BasicSchemaStrategy::Typeless(TypelessStrategy::new()));
}
let first_strategy = self.active_strategies.first_mut().unwrap();
return Some(first_strategy);
}
}
None
}
fn strategy_does_match_schema_or_object(
strategy: &BasicSchemaStrategy,
schema_or_object: &DataType,
) -> bool {
match schema_or_object {
DataType::Object(obj) => strategy.match_object(*obj),
DataType::Schema(schema) => strategy.match_schema(schema),
_ => false,
}
}
fn create_strategy_for_schema_or_object(
schema_or_object: &DataType,
) -> Option<BasicSchemaStrategy> {
match schema_or_object {
DataType::Object(obj) => BasicSchemaStrategy::new_for_object(*obj),
DataType::Schema(schema) => BasicSchemaStrategy::new_for_schema(schema),
_ => None,
}
}
fn add_schema_or_object_to_strategy(
strategy: &mut BasicSchemaStrategy,
schema_or_object: DataType,
) {
match schema_or_object {
DataType::Object(obj) => strategy.add_object(obj),
DataType::Schema(schema) => strategy.add_schema(schema),
_ => (),
}
}
}
#[cfg(test)]
mod tests {
include!("../tests/node.rs");
}