use std::path::Path;
use quick_xml::events::{BytesStart, Event};
use quick_xml::Reader;
use crate::base::{intern, AttrDef, AttrKind, ClassDef, Element, FastMap, FastSet, FieldValue, TypeRegistry};
use crate::registry;
pub fn is_cim(text: &str) -> bool {
let mut end = text.len().min(8192);
while !text.is_char_boundary(end) {
end -= 1;
}
let head = &text[..end];
["http://iec.ch/TC57/", "https://cim.ucaiug.io/ns#", "https://ap.cim4.eu/", "https://energy.referencedata.eu/"]
.iter()
.any(|marker| head.contains(marker))
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct FieldForm {
pub ns: Option<&'static str>,
pub datatype: Option<&'static str>,
pub resource_ns: Option<&'static str>,
}
impl FieldForm {
fn fill_from(&mut self, other: FieldForm) {
self.ns = self.ns.or(other.ns);
self.datatype = self.datatype.or(other.datatype);
self.resource_ns = self.resource_ns.or(other.resource_ns);
}
}
pub struct CimDataset {
pub entries: FastMap<String, Element>,
pub by_type: FastMap<String, Vec<String>>,
pub field_forms: FastMap<(&'static str, &'static str), FieldForm>,
pub type_ns: FastMap<String, &'static str>,
pub prefixes: FastMap<&'static str, &'static str>,
pub foreign: FastMap<String, Element>,
}
impl Default for CimDataset {
fn default() -> Self {
Self::new()
}
}
impl CimDataset {
pub fn new() -> Self {
Self {
entries: FastMap::default(),
by_type: FastMap::default(),
field_forms: FastMap::default(),
type_ns: FastMap::default(),
prefixes: FastMap::default(),
foreign: FastMap::default(),
}
}
pub fn decode_str(content: &str) -> Result<Self, Box<dyn std::error::Error>> {
let mut ds = Self::new();
parse_rdf(content, registry::type_registry(), &mut ds)?;
Ok(ds)
}
pub fn decode_file(path: &Path) -> Result<Self, Box<dyn std::error::Error>> {
Self::decode_str(&std::fs::read_to_string(path)?)
}
pub fn decode_files(paths: &[&Path]) -> Result<Self, Box<dyn std::error::Error>> {
let mut combined = Self::new();
for path in paths {
let file_ds = Self::decode_file(path)?;
combined.merge(file_ds);
}
Ok(combined)
}
pub fn decode_files_parallel_separate(paths: &[&Path]) -> Result<Vec<Self>, Box<dyn std::error::Error>> {
if paths.len() <= 1 {
return paths.iter().map(|p| Self::decode_file(p)).collect();
}
let results: Vec<Result<Self, String>> = std::thread::scope(|s| {
paths
.iter()
.map(|p| s.spawn(|| Self::decode_file(p).map_err(|e| e.to_string())))
.collect::<Vec<_>>()
.into_iter()
.map(|h| h.join().expect("decode thread panicked"))
.collect()
});
results
.into_iter()
.collect::<Result<_, String>>()
.map_err(|e| e.into())
}
pub fn decode_files_parallel(paths: &[&Path]) -> Result<Self, Box<dyn std::error::Error>> {
let datasets = Self::decode_files_parallel_separate(paths)?;
Ok(datasets
.into_iter()
.reduce(|mut a, b| {
a.merge(b);
a
})
.unwrap_or_default())
}
pub fn merge(&mut self, other: CimDataset) {
for (key, form) in other.field_forms {
self.field_forms.entry(key).or_default().fill_from(form);
}
for (mrid, ns) in other.type_ns {
self.type_ns.entry(mrid).or_insert(ns);
}
for (ns, prefix) in other.prefixes {
self.prefixes.entry(ns).or_insert(prefix);
}
for (id, incoming) in other.foreign {
match self.foreign.get_mut(&id) {
Some(existing) => existing.merge_from(&incoming),
None => {
self.foreign.insert(id, incoming);
}
}
}
let reg = registry::type_registry();
let mut moved: FastMap<&'static str, FastSet<String>> = FastMap::default();
for (mrid, incoming) in other.entries {
if let Some(existing) = self.entries.get_mut(&mrid) {
existing.merge_from(&incoming);
let old = existing.class();
let mut changed = false;
for t in incoming.types() {
changed |= existing.add_type(t, reg);
}
if changed {
moved.entry(old.qualified).or_default().insert(mrid.clone());
self.by_type.entry(existing.type_name().to_string()).or_default().push(mrid);
}
} else {
let type_name = incoming.type_name().to_string();
self.by_type.entry(type_name).or_default().push(mrid.clone());
self.entries.insert(mrid, incoming);
}
}
for (old, mrids) in moved {
if let Some(bucket) = self.by_type.get_mut(old) {
bucket.retain(|m| !mrids.contains(m));
}
}
}
pub fn element_from_json(&self, class: &'static ClassDef, value: &serde_json::Value) -> Result<Element, String> {
let reg = registry::type_registry();
let mut element = Element::from_json(class, reg, value)?;
let keys: Vec<&'static str> = element.fields().keys().copied().collect();
for key in keys {
if let Some(reference) = self.held_as_reference(class.qualified, key) {
element.retype(key, reference);
}
}
Ok(element)
}
fn held_as_reference(&self, type_name: &str, key: &str) -> Option<bool> {
self.by_type
.get(type_name)
.into_iter()
.flatten()
.filter_map(|m| self.entries.get(m))
.find_map(|e| match e.get(key)? {
FieldValue::Resource(_) | FieldValue::ResourceList(_) => Some(true),
FieldValue::Text(_) | FieldValue::TextList(_) => Some(false),
})
.or_else(|| self.field_forms.get(&(type_name, key)).and_then(|f| f.resource_ns).map(|_| true))
}
pub fn set(&mut self, mrid: String, element: Element) {
let new_type = element.type_name().to_string();
if let Some(old) = self.entries.get(&mrid) {
let old_type = old.type_name();
if old_type != new_type
&& let Some(v) = self.by_type.get_mut(old_type) {
v.retain(|m| m != &mrid);
}
}
let already_indexed = self
.by_type
.get(&new_type)
.is_some_and(|v| v.contains(&mrid));
if !already_indexed {
self.by_type.entry(new_type).or_default().push(mrid.clone());
}
self.entries.insert(mrid, element);
}
pub fn remove(&mut self, mrid: &str) -> Option<Element> {
let removed = self.entries.remove(mrid)?;
if let Some(v) = self.by_type.get_mut(removed.type_name()) {
v.retain(|m| m != mrid);
}
Some(removed)
}
pub fn type_conflicts(&self) -> Vec<(&str, Vec<&'static str>)> {
let reg = registry::type_registry();
let related = |a: &ClassDef, b: &ClassDef| {
reg.chain(a).iter().any(|c| c.qualified == b.qualified)
|| reg.chain(b).iter().any(|c| c.qualified == a.qualified)
};
let mut out: Vec<(&str, Vec<&'static str>)> = self
.entries
.iter()
.filter_map(|(mrid, e)| {
let types: Vec<&'static ClassDef> = e.types().collect();
let conflict = types.iter().enumerate().any(|(i, a)| types[i + 1..].iter().any(|b| !related(a, b)));
conflict.then(|| (mrid.as_str(), types.iter().map(|c| c.qualified).collect()))
})
.collect();
out.sort_unstable();
out
}
}
fn index(by_type: &mut FastMap<String, Vec<String>>, type_name: &'static str, mrid: String) {
if let Some(bucket) = by_type.get_mut(type_name) {
bucket.push(mrid);
} else {
by_type.insert(type_name.to_string(), vec![mrid]);
}
}
type Table = FastMap<&'static str, &'static ClassDef>;
#[derive(Default)]
struct Scope {
prefixes: Vec<(Vec<u8>, &'static str, Option<&'static Table>)>,
default: Option<&'static Table>,
default_ns: &'static str,
}
impl Scope {
fn absorb(&mut self, e: &BytesStart, reg: &'static TypeRegistry) {
for attr in e.attributes().flatten() {
let key = attr.key.as_ref();
let Some(rest) = key.strip_prefix(b"xmlns".as_slice()) else {
continue;
};
let Ok(ns) = std::str::from_utf8(&attr.value) else {
continue;
};
let table = reg.ns_table(ns).or_else(|| {
if ns.ends_with('#') || ns.ends_with('/') {
None
} else {
reg.ns_table(&format!("{ns}#"))
}
});
let ns = intern(ns);
match rest.strip_prefix(b":".as_slice()) {
Some(prefix) => {
let prefix = prefix.to_vec();
match self.prefixes.iter_mut().find(|(k, _, _)| *k == prefix) {
Some(slot) => (slot.1, slot.2) = (ns, table),
None => self.prefixes.push((prefix, ns, table)),
}
}
None if rest.is_empty() => (self.default, self.default_ns) = (table, ns),
None => {}
}
}
}
fn table(&self, prefix: Option<&[u8]>) -> Option<&'static Table> {
match prefix {
None => self.default,
Some(p) => self
.prefixes
.iter()
.find(|(k, _, _)| k.as_slice() == p)
.and_then(|(_, _, t)| *t),
}
}
fn ns(&self, prefix: Option<&[u8]>) -> &'static str {
match prefix {
None => self.default_ns,
Some(p) => self.prefixes.iter().find(|(k, _, _)| k.as_slice() == p).map_or("", |(_, ns, _)| ns),
}
}
}
struct Subject {
mrid: String,
id: String,
described: bool,
has_xmlns: bool,
}
fn scan_attrs(e: &BytesStart) -> Result<Subject, Box<dyn std::error::Error>> {
let mut subject = Subject { mrid: String::new(), id: String::new(), described: false, has_xmlns: false };
let mut defined = false;
for attr in e.attributes().flatten() {
let key = attr.key.as_ref();
if key == b"rdf:about" || key == b"rdf:ID" {
let value = std::str::from_utf8(&attr.value)?;
subject.mrid = strip_fragment(value);
subject.id = value.strip_prefix('#').unwrap_or(value).to_string();
subject.described |= key == b"rdf:about";
defined |= key == b"rdf:ID";
} else if key == b"rdf:nodeID" {
subject.mrid = format!("_:{}", std::str::from_utf8(&attr.value)?);
subject.id = subject.mrid.clone();
} else if key.starts_with(b"xmlns") {
subject.has_xmlns = true;
}
}
subject.described &= !defined;
Ok(subject)
}
fn object(
scope: &Scope,
reg: &'static TypeRegistry,
e: &BytesStart,
subject: Subject,
) -> Result<Option<(Element, bool, Option<&'static str>)>, Box<dyn std::error::Error>> {
if subject.id.is_empty() {
return Ok(None);
}
let name = e.name();
let (prefix, local) = split_qname(name.as_ref());
let ns = scope.ns(prefix);
let (mut element, foreign) = match resolve(scope, reg, name.as_ref())? {
Some(class) => (Element::new(class, subject.mrid), false),
None => (Element::new(crate::base::foreign_class(ns, std::str::from_utf8(local)?), subject.id), true),
};
if subject.described {
element.mark_described();
}
let written = (!ns.is_empty() && !same_ns(element.type_ns(), ns)).then_some(ns);
Ok(Some((element, foreign, written)))
}
fn same_ns(a: &str, b: &str) -> bool {
a == b || a.strip_suffix('#') == Some(b) || b.strip_suffix('#') == Some(a)
}
fn store(ds: &mut CimDataset, element: Element, foreign: bool, written: Option<&'static str>) {
if let Some(ns) = written
&& !foreign
&& !element.mrid().is_empty()
{
ds.type_ns.entry(element.mrid().to_string()).or_insert(ns);
}
if foreign {
ds.foreign.insert(element.mrid().to_string(), element);
} else if !element.mrid().is_empty() {
index(&mut ds.by_type, element.type_name(), element.mrid().to_string());
ds.entries.insert(element.mrid().to_string(), element);
}
}
fn split_qname(raw: &[u8]) -> (Option<&[u8]>, &[u8]) {
match raw.iter().position(|b| *b == b':') {
Some(i) => (Some(&raw[..i]), &raw[i + 1..]),
None => (None, raw),
}
}
fn resolve(
scope: &Scope,
reg: &'static TypeRegistry,
raw: &[u8],
) -> Result<Option<&'static ClassDef>, Box<dyn std::error::Error>> {
let (prefix, local_bytes) = split_qname(raw);
let local = std::str::from_utf8(local_bytes)?;
if let Some(class) = scope.table(prefix).and_then(|t| t.get(local)) {
return Ok(Some(*class));
}
Ok(reg.bare().get(local).copied())
}
fn parse_rdf(
content: &str,
reg: &'static TypeRegistry,
ds: &mut CimDataset,
) -> Result<(), Box<dyn std::error::Error>> {
let mut reader = Reader::from_str(content);
let mut buf = Vec::new();
let mut depth: u32 = 0;
let mut scope = Scope::default();
let mut nested: Option<Scope> = None;
let mut current: Option<Element> = None;
let mut current_foreign = false;
let mut current_ns: Option<&'static str> = None;
let mut declared: Option<&'static FastMap<&'static str, &'static AttrDef>> = None;
let mut pending_key: Option<&'static str> = None;
let mut pending_lang: Option<String> = None;
let mut pending_text = String::new();
let field = |declared: Option<&'static FastMap<&'static str, &'static AttrDef>>,
class: &'static str,
e: &BytesStart,
scope: &Scope,
forms: &mut FastMap<(&'static str, &'static str), FieldForm>|
-> Result<(&'static str, FieldAttrs), Box<dyn std::error::Error>> {
let name = e.name();
let (prefix, local_bytes) = split_qname(name.as_ref());
let local = std::str::from_utf8(local_bytes)?;
let attr = declared.and_then(|d| d.get(local));
let key = attr.map_or_else(|| intern(local), |a| a.id);
let attrs = field_attrs(e.attributes())?;
let ns = scope.ns(prefix);
let undeclared_ns = (!ns.is_empty() && attr.is_none_or(|a| !same_ns(a.ns, ns))).then_some(ns);
let resource_ns = attrs.resource_ns.filter(|_| attr.is_none_or(|a| a.kind != AttrKind::Enum));
if undeclared_ns.is_some() || attrs.datatype.is_some() || resource_ns.is_some() {
forms.entry((class, key)).or_default().fill_from(FieldForm {
ns: undeclared_ns,
datatype: attrs.datatype,
resource_ns,
});
}
Ok((key, attrs))
};
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
depth += 1;
match depth {
1 => {
scope.absorb(e, reg);
for (prefix, ns, _) in &scope.prefixes {
if let Ok(prefix) = std::str::from_utf8(prefix) {
ds.prefixes.entry(ns).or_insert_with(|| intern(prefix));
}
}
}
2 => {
let subject = scan_attrs(e)?;
if subject.has_xmlns {
let mut inner = Scope {
prefixes: scope.prefixes.clone(),
default: scope.default,
default_ns: scope.default_ns,
};
inner.absorb(e, reg);
nested = Some(std::mem::replace(&mut scope, inner));
}
(current, current_foreign, current_ns) = match object(&scope, reg, e, subject)? {
Some((element, foreign, written)) => (Some(element), foreign, written),
None => (None, false, None),
};
declared = current.as_ref().and_then(|c| reg.declared(c.class()));
}
3 => {
if let Some(element) = current.as_mut() {
let (key, attrs) = field(declared, element.type_name(), e, &scope, &mut ds.field_forms)?;
if let Some(res) = attrs.resource {
element.add_field(key, FieldValue::Resource(res));
} else if !attrs.nodes {
pending_key = Some(key);
pending_lang = attrs.lang;
pending_text.clear();
}
}
}
_ => pending_key = None,
}
}
Ok(Event::Empty(ref e)) => {
match depth {
1 => {
let subject = scan_attrs(e)?;
let local_scope = if subject.has_xmlns {
let mut inner = Scope {
prefixes: scope.prefixes.clone(),
default: scope.default,
default_ns: scope.default_ns,
};
inner.absorb(e, reg);
Some(inner)
} else {
None
};
let active = local_scope.as_ref().unwrap_or(&scope);
if let Some((element, foreign, written)) = object(active, reg, e, subject)? {
store(ds, element, foreign, written);
}
}
2 => {
if let Some(element) = current.as_mut() {
let (key, attrs) = field(declared, element.type_name(), e, &scope, &mut ds.field_forms)?;
if let Some(res) = attrs.resource {
element.add_field(key, FieldValue::Resource(res));
} else if !attrs.nodes {
element.add_field(key, FieldValue::Text(String::new()));
if let Some(lang) = attrs.lang {
element.add_lang(key, &lang);
}
}
}
}
_ => pending_key = None,
}
}
Ok(Event::Text(ref e)) if depth == 3 && pending_key.is_some() => {
pending_text.push_str(&e.unescape()?);
}
Ok(Event::CData(ref e)) if depth == 3 && pending_key.is_some() => {
pending_text.push_str(std::str::from_utf8(e)?);
}
Ok(Event::End(_)) => {
if depth == 3
&& let (Some(element), Some(key)) = (current.as_mut(), pending_key.take()) {
element.add_field(key, FieldValue::Text(std::mem::take(&mut pending_text)));
if let Some(lang) = pending_lang.take() {
element.add_lang(key, &lang);
}
}
if depth == 2 {
pending_key = None;
pending_lang = None;
if let Some(element) = current.take() {
store(ds, element, current_foreign, current_ns.take());
}
if let Some(outer) = nested.take() {
scope = outer;
}
}
depth = depth.saturating_sub(1);
}
Ok(Event::Eof) => break,
Err(e) => return Err(Box::new(e)),
_ => {}
}
buf.clear();
}
Ok(())
}
fn strip_fragment(s: &str) -> String {
match s.rfind('#') {
Some(i) if i + 1 == s.len() && i > 0 => s.to_string(),
Some(i) => s[i + 1..].to_string(),
None => s.to_string(),
}
}
struct FieldAttrs {
resource: Option<String>,
resource_ns: Option<&'static str>,
lang: Option<String>,
datatype: Option<&'static str>,
nodes: bool,
}
fn field_attrs(attrs: quick_xml::events::attributes::Attributes<'_>) -> Result<FieldAttrs, Box<dyn std::error::Error>> {
let mut out = FieldAttrs { resource: None, resource_ns: None, lang: None, datatype: None, nodes: false };
for attr in attrs.flatten() {
match attr.key.as_ref() {
b"rdf:resource" => {
let iri = std::str::from_utf8(&attr.value)?;
let kept = strip_fragment(iri);
if kept.len() < iri.len() && kept.len() + 1 < iri.len() {
out.resource_ns = Some(intern(&iri[..iri.len() - kept.len()]));
}
out.resource = Some(kept);
}
b"rdf:nodeID" => out.resource = Some(format!("_:{}", std::str::from_utf8(&attr.value)?)),
b"xml:lang" => out.lang = Some(std::str::from_utf8(&attr.value)?.to_string()),
b"rdf:datatype" => out.datatype = Some(intern(std::str::from_utf8(&attr.value)?)),
b"rdf:parseType" => out.nodes = attr.value.as_ref() != b"Literal",
_ => {}
}
}
Ok(out)
}