use crate::config::Config;
use crate::config::OddityHandling;
use crate::config::RDProperty;
use crate::vocab::{basics, owl, sh};
use cli_utils::BoxResult;
use const_format::concatcp;
use enum_map::EnumMap;
use oxigraph::io::RdfFormat;
use oxigraph::io::RdfParser;
use oxigraph::model::vocab::rdf;
use oxigraph::model::{GraphName, GraphNameRef, NamedNodeRef, Quad, Term};
use oxigraph::sparql::Query;
use oxigraph::sparql::QueryResults;
use oxigraph::sparql::QuerySolution;
use oxigraph::sparql::Variable;
use oxigraph::store::Store;
use oxrdf::{BlankNode, NamedNode, VariableRef};
use regex::Regex;
use std::collections::HashSet;
use std::fs::{self, File};
use std::io::BufRead;
use std::io::BufReader;
use std::sync::LazyLock;
use strum::IntoEnumIterator;
use strum_macros::EnumIter;
const XSD_NAMESPACE: &str = "http://www.w3.org/2001/XMLSchema#";
pub static VAR_VS_TERM_STATUS: LazyLock<Variable> =
LazyLock::new(|| Variable::new("vsTermStatus").unwrap());
pub static VAR_OWL_DEPRECATED: LazyLock<Variable> =
LazyLock::new(|| Variable::new("owlDeprecated").unwrap());
pub static VAR_CC_DEPRECATED_ON: LazyLock<Variable> =
LazyLock::new(|| Variable::new("ccDeprecatedOn").unwrap());
pub static VAR_SCHEMA_SUPERSEDED_BY: LazyLock<Variable> =
LazyLock::new(|| Variable::new("schemaSupersededBy").unwrap());
pub static RE_EXACT_BASE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^\w*@base\w+<(?<base_url>.*)>\w+.").unwrap());
const QUERY_PRELUDE: &str = r"
PREFIX cc: <http://creativecommons.org/ns#>
PREFIX dcam: <http://purl.org/dc/dcam/>
PREFIX dce: <http://purl.org/dc/elements/1.1/>
PREFIX dcid: <https://datacommons.org/browser/>
PREFIX dcterms: <http://purl.org/dc/terms/>
PREFIX dfc: <http://www.virtual-assembly.org/DataFoodConsortium/BusinessOntology#>
PREFIX dtype: <http://www.linkedmodel.org/schema/dtype#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX geo: <http://www.w3.org/2003/01/geo/wgs84_pos#>
PREFIX npg: <http://ns.nature.com/terms/>
PREFIX nrl: <https://www.semanticdesktop.org/ontologies/2007/08/15/nrl/#>
PREFIX og: <http://ogp.me/ns#>
PREFIX om2: <http://www.ontology-of-units-of-measure.org/resource/om-2/>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX owl: <http://www.w3.org/2002/07/owl#>
PREFIX qudt: <http://qudt.org/schema/qudt/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX schema: <http://schema.org/>
PREFIX sh: <http://www.w3.org/ns/shacl#>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX time: <http://www.w3.org/2006/time#>
PREFIX vs: <http://www.w3.org/2003/06/sw-vocab-status/ns#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
";
macro_rules! query_parser {
($const:ident, $query_str:ident) => {
pub static $const: LazyLock<Query> = LazyLock::new(|| {
let query_str = concatcp!(QUERY_PRELUDE, '\n', $query_str);
std::fs::write(
&format!("target/{}.sparql.txt", stringify!($const)),
query_str,
)
.expect("Failed to write query to file!");
Query::parse(query_str, None).unwrap()
});
};
}
const QS_CLASSES: &str = r"
SELECT ?s ?vsTermStatus ?owlDeprecated ?ccDeprecatedOn ?schemaSupersededBy ?pSh ?oSh
WHERE {
{
{
VALUES ?t {
rdfs:Class owl:Class rdfs:Datatype
}
?s rdf:type ?t .
}
UNION
{
?s rdfs:subClassOf ?o .
}
} .
OPTIONAL {
?s vs:term_status ?vsTermStatus .
} .
OPTIONAL {
?s owl:deprecated ?owlDeprecated .
} .
OPTIONAL {
?s cc:deprecatedOn ?ccDeprecatedOn .
} .
OPTIONAL {
?s schema:supersededBy ?schemaSupersededBy .
} .
OPTIONAL {
?s ?pSh ?oSh .
filter (strstarts(str(?pSh), str(sh:)))
} .
}
GROUP BY ?s ?vsTermStatus ?owlDeprecated ?ccDeprecatedOn ?schemaSupersededBy
ORDER BY ?s
";
query_parser!(Q_CLASSES, QS_CLASSES);
const QS_PROPERTIES: &str = r"
SELECT
?s ?t ?label ?description ?cardinality ?maxCardinality ?minCardinality ?vsTermStatus ?owlDeprecated ?ccDeprecatedOn ?schemaSupersededBy ?pSh ?oSh
# HACK Due to this bug in OxiGraph, we first have to use the STR() function for the GROUP_CONCAT() argument: <https://github.com/oxigraph/oxigraph/issues/297>
( GROUP_CONCAT( DISTINCT STR(?domain); separator=',' ) as ?domainAndList )
( GROUP_CONCAT( DISTINCT STR(?range); separator=',' ) as ?rangeAndList )
( GROUP_CONCAT( DISTINCT STR(?domainIncludes); separator=',' ) as ?domainIncludesList )
( GROUP_CONCAT( DISTINCT STR(?rangeIncludes); separator=',' ) as ?rangeIncludesList )
( GROUP_CONCAT( DISTINCT STR(?domainOred); separator=',' ) as ?domainOredList )
( GROUP_CONCAT( DISTINCT STR(?rangeOred); separator=',' ) as ?rangeOredList )
WHERE {
VALUES ?t {
rdf:Property owl:ObjectProperty owl:DatatypeProperty owl:AnnotationProperty owl:FunctionalProperty owl:InverseFunctionalProperty owl:IrreflexiveProperty
} .
?s rdf:type ?t .
OPTIONAL {
?s rdfs:label | dce:title | dcterms:title | npg:title | og:title ?label .
} .
OPTIONAL {
?s rdfs:comment | dce:description | dcterms:description | og:description ?description .
} .
OPTIONAL {
?s vs:term_status ?vsTermStatus .
} .
OPTIONAL {
?s owl:deprecated ?owlDeprecated .
} .
OPTIONAL {
?s cc:deprecatedOn ?ccDeprecatedOn .
} .
OPTIONAL {
?s schema:supersededBy ?schemaSupersededBy .
} .
OPTIONAL {
?s owl:cardinality | owl:qualifiedCardinality | nrl:cardinality | qudt:cardinality ?cardinality .
} .
OPTIONAL {
?s owl:maxCardinality | owl:maxQualifiedCardinality | nrl:maxCardinality ?maxCardinality .
} .
OPTIONAL {
?s owl:minCardinality | owl:minQualifiedCardinality | nrl:minCardinality ?minCardinality .
} .
OPTIONAL {
?s rdfs:domain ?domain .
OPTIONAL {
?domain owl:unionOf ?domainOredUnion .
?domainOredUnion rdf:rest*/rdf:first ?domainOred .
}
} .
OPTIONAL {
?s rdfs:range ?range .
OPTIONAL {
?range owl:unionOf ?rangeOredUnion .
?rangeOredUnion rdf:rest*/rdf:first ?rangeOred .
}
} .
OPTIONAL {
?s schema:domainIncludes | dcam:domainIncludes | dcid:domainIncludes ?domainIncludes .
} .
OPTIONAL {
?s schema:rangeIncludes | dcam:rangeIncludes | dcid:rangeIncludes ?rangeIncludes .
} .
OPTIONAL {
?s ?pSh ?oSh .
filter (strstarts(str(?pSh), str(sh:)))
} .
}
GROUP BY ?s ?t ?label ?description ?cardinality ?maxCardinality ?minCardinality ?vsTermStatus ?owlDeprecated ?ccDeprecatedOn ?schemaSupersededBy ?pSh ?oSh
ORDER BY ?s
";
query_parser!(Q_PROPERTIES, QS_PROPERTIES);
const QS_SHAPES: &str = r"
prefix x: <urn:ex:> #-- arbitrary, used for the property path.
construct {
?s1 ?p ?o1 . #-- internal edge in the path
?s2 ?p2 ?o2 . #-- final edge in the path
}
where {
?s1 ?p ?o1 .
filter (strstarts(str(?p), str(sh:))) .
#:A (x:|!x:)* ?s1 . #-- start at :A and go any length into the path
?s1 ?p ?o1 . #-- get the triple from within the path, but make
?o1 (x:|!x:)* ?s2 . #-- sure that from ?o1 you can get to to some other
#?s2 ?p2 ?o2 . #-- ?s2 that's related to an ?o2 by property :d .
}
";
query_parser!(Q_SHAPES, QS_SHAPES);
macro_rules! ins {
($store:expr, $subj:expr, $pred:expr, $obj:expr) => {
$store.insert(&Quad::new($subj, $pred, $obj, GraphName::DefaultGraph))?;
};
}
macro_rules! ins_opt {
($store:expr, $subj:expr, $pred:expr, $solution:expr, $obj_ident:ident) => {
if let Some(obj) = $solution.get(stringify!($obj_ident)) {
ins!($store, $subj, $pred, obj.as_ref());
}
};
}
macro_rules! type2shape {
($shape_var:ident, $orig:expr) => {
let subj_iri = if let Term::NamedNode(nn) = $orig {
nn.as_str()
} else {
panic!("Only named-node properties subjects are supported!");
};
let shape_iri = format!("{subj_iri}Shape");
let $shape_var = NamedNodeRef::new(&shape_iri)?;
tracing::info!("Shape: {}", $shape_var);
};
}
fn add_list<'a>(
store: &Store,
subj: NamedNodeRef<'a>,
pred: NamedNodeRef<'a>,
items: &[BlankNode],
) -> BoxResult<()> {
assert!(!items.is_empty());
let mut cur_tuple = BlankNode::default();
ins!(store, subj, pred, cur_tuple.clone());
ins!(store, cur_tuple.clone(), rdf::FIRST, items[0].clone());
for item in &items[1..] {
let next_tuple = BlankNode::default();
ins!(store, cur_tuple, rdf::REST, next_tuple.clone());
ins!(store, next_tuple.clone(), rdf::FIRST, item.clone());
cur_tuple = next_tuple;
}
ins!(store, cur_tuple, rdf::REST, rdf::NIL);
Ok(())
}
fn extract_lit_str<'a>(sol: &'a QuerySolution, var: VariableRef<'_>) -> Option<&'a str> {
if let Some(term) = sol.get(var) {
match term {
Term::Literal(lit_val) => {
return Some(lit_val.as_ref().value());
}
_ => return None,
}
}
None
}
fn is_deprecated(sol: &QuerySolution) -> bool {
if let Some(lit_str) = extract_lit_str(sol, VAR_VS_TERM_STATUS.as_ref()) {
if lit_str == "deprecated" {
return true;
}
}
if let Some(lit_str) = extract_lit_str(sol, VAR_OWL_DEPRECATED.as_ref()) {
if lit_str == "true" {
return true;
}
}
if let Some(_lit_str) = extract_lit_str(sol, VAR_CC_DEPRECATED_ON.as_ref()) {
return true;
}
if let Some(_lit_str) = extract_lit_str(sol, VAR_SCHEMA_SUPERSEDED_BY.as_ref()) {
return true;
}
false
}
fn convert_classes(store_owl: &Store, store_shacl: &Store) -> BoxResult<()> {
tracing::info!("Converting classes ...");
if let QueryResults::Solutions(solutions) = store_owl.query(Q_CLASSES.to_owned())? {
for sol_res in solutions {
let sol = sol_res?;
let subj = sol.get("s").unwrap();
tracing::info!("Class: {subj}");
let sh_object_opt = sol.get("oSh");
if let Some(sh_object) = sh_object_opt {
let sh_pred_opt = sol.get("pSh");
if let Some(sh_pred) = sh_pred_opt {
tracing::info!("`sh:<predicate>`: {sh_pred}");
}
tracing::info!("`sh:<object>`: {sh_object}");
}
let deprecated = is_deprecated(&sol);
if deprecated {
tracing::info!("Class {subj} is deprecated.");
}
type2shape!(shape, subj);
ins!(store_shacl, shape, rdf::TYPE, sh::NODE_SHAPE);
ins!(store_shacl, shape, sh::TARGET_CLASS, subj.clone());
ins!(store_shacl, shape, sh::CLOSED, *basics::BOOL_FALSE);
}
tracing::info!("Converting classes - done.");
} else {
tracing::warn!("No classes found.");
}
Ok(())
}
#[derive(Debug, EnumIter, PartialEq, Eq, PartialOrd, Copy, Clone, Hash)]
enum ListCollectionMethod {
And,
Includes,
Ored,
}
impl ListCollectionMethod {
pub const fn to_var_postfix(self) -> &'static str {
match self {
Self::And => "AndList",
Self::Includes => "IncludesList",
Self::Ored => "OredList",
}
}
pub const fn is_and(self) -> bool {
match self {
Self::And => true,
Self::Includes | Self::Ored => false,
}
}
}
fn to_shape_iri(part: &str) -> String {
if part.starts_with(XSD_NAMESPACE) {
part.to_owned()
} else {
format!("{part}Shape")
}
}
fn convert_property_range_or_domain(
store_shacl: &Store,
shape: NamedNodeRef,
config: &Config,
sol: &QuerySolution,
prop: RDProperty,
) -> BoxResult<HashSet<ListCollectionMethod>> {
let prop_str = prop.to_str();
let mut used = HashSet::new();
let is_datatype_prop = if let Term::NamedNode(nn) = sol
.get("t")
.expect("Required SPARQL var ?t (rdfs:type) missing")
{
nn.eq(&owl::DATATYPE_PROPERTY)
} else {
panic!("Only named-node properties subjects are supported as `rdfs:type` objects!");
};
for collection_method in ListCollectionMethod::iter() {
let list_var = Variable::new(format!("{prop_str}{}", collection_method.to_var_postfix()))?;
if let Some(list) = sol.get(&list_var) {
match list {
Term::Literal(lit) => {
let lit_str = lit.as_ref().value();
if !lit_str.is_empty() {
used.insert(collection_method);
let parts_strs = lit_str.split(',').collect::<Vec<_>>();
let num_parts = parts_strs.len();
let is_list = num_parts > 1;
if is_list {
if collection_method.is_and() {
match config.settings.and_list_detected[prop] {
OddityHandling::Ignore => {
continue;
}
OddityHandling::Warn => {
tracing::warn!(
"And-list detected for property {prop_str}; this is not supported in our to-SHACL converter.");
continue;
}
OddityHandling::Error => {
return Err(format!(
"And-list detected for property {prop_str}; this is not supported in our to-SHACL converter.").into());
}
}
}
match prop {
RDProperty::Range => {
let mut types = vec![];
for part_str in parts_strs {
let part = NamedNode::new(part_str)?;
let part_shape_iri = to_shape_iri(part_str);
let part_shape = NamedNodeRef::new(&part_shape_iri)?;
let part_node = BlankNode::default();
if is_datatype_prop {
ins!(
store_shacl,
part_node.as_ref(),
sh::DATA_TYPE,
part_shape
);
} else {
ins!(store_shacl, part_node.as_ref(), sh::CLASS, part);
}
ins!(store_shacl, part_node.as_ref(), sh::NODE, part_shape);
types.push(part_node);
}
add_list(store_shacl, shape, sh::OR, &types)?;
}
RDProperty::Domain => {
for part_str in parts_strs {
let part_shape_iri = to_shape_iri(part_str);
let part_shape = NamedNodeRef::new(&part_shape_iri)?;
ins!(store_shacl, part_shape, sh::PROPERTY, shape);
}
}
}
} else {
let part_str = parts_strs[0];
let part = NamedNode::new(part_str)?;
let part_shape_iri = to_shape_iri(part_str);
let part_shape = NamedNodeRef::new(&part_shape_iri)?;
match prop {
RDProperty::Range => {
if is_datatype_prop {
ins!(store_shacl, shape, sh::DATA_TYPE, part);
} else {
ins!(store_shacl, shape, sh::CLASS, part);
}
ins!(store_shacl, shape, sh::NODE, part_shape);
}
RDProperty::Domain => {
ins!(store_shacl, part_shape, sh::PROPERTY, shape);
}
}
}
}
}
Term::NamedNode(_) | Term::BlankNode(_) | Term::Triple(_) => {
panic!("Type for SPARQL variable {list_var} should be Literal(string).")
}
}
}
}
let action = config.settings.style_mix_property[prop];
if !action.ignore() && (used.len() > 1) {
let msg = format!(
"Mixed styles of {} definitions in Property: {}",
prop_str,
used.iter()
.map(|style| format!("{style:?}"))
.collect::<Vec<_>>()
.join(", ")
);
match action {
OddityHandling::Error => return Err(msg.into()),
OddityHandling::Warn => tracing::warn!("{msg}"),
OddityHandling::Ignore => panic!("This should never happen"),
}
}
Ok(used)
}
fn convert_properties(store_owl: &Store, store_shacl: &Store, config: &Config) -> BoxResult<()> {
tracing::info!("Converting properties ...");
if let QueryResults::Solutions(solutions) = store_owl.query(Q_PROPERTIES.to_owned())? {
let mut used_prop_styles = EnumMap::from_fn(|_| HashSet::new());
for sol_res in solutions {
let sol = sol_res?;
let subj = sol.get("s").unwrap();
tracing::info!("");
tracing::info!("Property: {subj}");
let sh_object_opt = sol.get("oSh");
if let Some(sh_object) = sh_object_opt {
let sh_pred_opt = sol.get("pSh");
if let Some(sh_pred) = sh_pred_opt {
tracing::info!("`sh:<predicate>`: {sh_pred}");
}
tracing::info!("`sh:<object>`: {sh_object}");
}
let deprecated = is_deprecated(&sol);
if deprecated {
tracing::info!("Property {subj} is deprecated.");
}
type2shape!(shape, subj);
ins!(store_shacl, shape, rdf::TYPE, sh::PROPERTY_SHAPE);
ins!(store_shacl, shape, sh::PATH, subj.clone());
ins_opt!(store_shacl, shape, sh::NAME, sol, label);
ins_opt!(store_shacl, shape, sh::DESCRIPTION, sol, description);
if !deprecated {
ins_opt!(store_shacl, shape, sh::MIN_COUNT, sol, minCardinality);
ins_opt!(store_shacl, shape, sh::MAX_COUNT, sol, maxCardinality);
ins_opt!(store_shacl, shape, sh::MIN_COUNT, sol, cardinality);
ins_opt!(store_shacl, shape, sh::MAX_COUNT, sol, cardinality);
}
for (prop, used_style) in &mut used_prop_styles {
used_style.extend(convert_property_range_or_domain(
store_shacl,
shape,
config,
&sol,
prop,
)?);
}
}
for (prop, used_style) in &mut used_prop_styles {
let action = config.settings.style_mix_ontology[prop];
if !action.ignore() && (used_style.len() > 1) {
let msg = format!(
"Mixed styles of {} definitions in Ontology: {}",
prop.to_str(),
used_style
.iter()
.map(|style| format!("{style:?}"))
.collect::<Vec<_>>()
.join(", ")
);
match action {
OddityHandling::Error => return Err(msg.into()),
OddityHandling::Warn => tracing::warn!("{msg}"),
OddityHandling::Ignore => panic!("This should never happen"),
}
}
}
tracing::info!("");
tracing::info!("Converting properties done.");
} else {
tracing::warn!("No properties found.");
}
Ok(())
}
fn convert_shapes(store_owl: &Store, store_shacl: &Store, config: &Config) -> BoxResult<()> {
tracing::info!("Converting shapes ...");
if let QueryResults::Graph(triples) = store_owl.query(Q_SHAPES.to_owned())? {
for triple in triples {
tracing::info!("XXX sh: tripple: {:#?}", triple?);
}
tracing::info!("Converting shapes done.");
} else {
tracing::warn!("No shapes found.");
}
Ok(())
}
pub fn convert(config: &Config) -> BoxResult<()> {
let store_owl = Store::new()?;
tracing::debug!("Loading OWL ...");
let graph_name = GraphName::DefaultGraph;
let mut base_iri_opt = config.base_iri.clone();
if base_iri_opt.is_none() {
let buf_ttl_ont_reader = BufReader::new(File::open(&config.owl)?);
for line in buf_ttl_ont_reader.lines() {
let line_str = line?;
let base_match_opt = RE_EXACT_BASE.captures(&line_str);
if let Some(captures) = base_match_opt {
let base_url_opt = captures.name("base_url");
if let Some(base_url) = base_url_opt {
base_iri_opt = Some(base_url.as_str().to_owned());
} else {
tracing::warn!("Failed to extract base URL from OWL line: '{line_str}'");
}
break;
}
}
}
let mut load_format = RdfParser::from_format(RdfFormat::Turtle)
.without_named_graphs()
.with_default_graph(graph_name.as_ref());
if let Some(base_iri) = base_iri_opt {
load_format = load_format.with_base_iri(base_iri)?;
}
let buf_ttl_ont_reader = BufReader::new(File::open(&config.owl)?);
store_owl.load_from_reader(load_format, buf_ttl_ont_reader)?;
tracing::debug!("Loading OWL - done.");
tracing::debug!("store_owl len: {}", store_owl.len()?);
let store_shacl = Store::new()?;
tracing::debug!("Converting OWL to SHACL ...");
convert_classes(&store_owl, &store_shacl)?;
convert_properties(&store_owl, &store_shacl, config)?;
convert_shapes(&store_owl, &store_shacl, config)?;
tracing::debug!("Converting OWL to SHACL - done.");
let shacl_file = config.shacl();
tracing::debug!("Writing SHACL file ('{}') ...", shacl_file.display());
store_shacl.dump_graph_to_writer(
GraphNameRef::DefaultGraph,
RdfFormat::Turtle,
File::create(&shacl_file)?,
)?;
tracing::info!("Writing SHACL file ('{}') - done.", shacl_file.display());
let prefixes_file = config.prefixes();
tracing::debug!("Writing prefixes file ('{}') ...", prefixes_file.display());
let ttl_prefixes = QUERY_PRELUDE
.to_string()
.replace("PREFIX ", "@prefix ")
.replace('>', "> .");
fs::write(&prefixes_file, ttl_prefixes)?;
tracing::info!(
"Writing prefixes file ('{}') - done.",
prefixes_file.display()
);
Ok(())
}