Skip to main content

stamtools/
xml.rs

1use std::borrow::Cow;
2use std::collections::{BTreeMap, HashMap, BTreeSet};
3use std::fmt::Display;
4use std::fs::read_to_string;
5use std::path::Path;
6use std::hash::{Hash,DefaultHasher,Hasher};
7use std::process::{Command,  Stdio};
8use std::io::{ BufWriter, Write};
9
10use roxmltree::{Document, Node, NodeId, ParsingOptions};
11use serde::Deserialize;
12use stam::*;
13use toml;
14use upon::Engine;
15use std::fmt::Write as FmtWrite;
16use serde_json;
17
18const NS_XML: &str = "http://www.w3.org/XML/1998/namespace";
19const CONTEXT_ANNO: &str = "http://www.w3.org/ns/anno.jsonld";
20
21
22fn default_set() -> String {
23    "urn:stam-fromxml".into()
24}
25
26#[derive(Deserialize)]
27/// Holds the configuration for mapping a specific XML format to STAM
28pub struct XmlConversionConfig {
29    #[serde(default)]
30    /// Holds configurations for mapping specific XML elements to STAM, evaluated in reverse-order, so put more generic rules before specific ones
31    elements: Vec<XmlElementConfig>,
32
33    #[serde(default)]
34    /// Base elements are named templates, other elements can derive from this
35    baseelements: HashMap<String, XmlElementConfig>,
36
37    #[serde(default)]
38    /// Maps XML prefixes to namespace
39    namespaces: HashMap<String, String>,
40
41    #[serde(default = "XmlWhitespaceHandling::collapse")]
42    /// Default whitespace handling
43    whitespace: XmlWhitespaceHandling,
44
45    #[serde(default)]
46    /// Sets additional context variables that can be used in templates
47    context: HashMap<String, toml::Value>,
48
49    #[serde(default)]
50    /// Sets additional context variables that can be used in templates
51    metadata: Vec<MetadataConfig>,
52
53    #[serde(default)]
54    /// Inject a DTD (for XML entity resolution)
55    inject_dtd: Option<String>,
56
57    #[serde(default = "default_set")]
58    default_set: String,
59
60    #[serde(default)]
61    /// A prefix to assign when setting annotation IDs
62    id_prefix: Option<String>,
63
64    #[serde(default)]
65    /// A suffix to strip when setting annotation IDs
66    id_strip_suffix: Vec<String>,
67
68    #[serde(default)]
69    /// Add provenance information pointing each annotation to the appropriate node in the XML source files where it came from (translates into XPathSelector in Web Annotation output)
70    provenance: bool,
71
72    #[serde(default)]
73    external_filters: Vec<ExternalFilter>,
74
75    #[serde(skip_deserializing)]
76    debug: bool,
77
78}
79
80impl XmlConversionConfig {
81    pub fn new() -> Self {
82        Self {
83            elements: Vec::new(),
84            baseelements: HashMap::new(),
85            namespaces: HashMap::new(),
86            context: HashMap::new(),
87            metadata: Vec::new(),
88            whitespace: XmlWhitespaceHandling::Collapse,
89            default_set: default_set(),
90            inject_dtd: None,
91            id_prefix: None,
92            id_strip_suffix: Vec::new(),
93            provenance: false,
94            external_filters: Vec::new(),
95            debug: false,
96        }
97    }
98
99    pub fn resolve_baseelements(&mut self) -> Result<(), XmlConversionError> {
100        let mut replace: Vec<(usize, XmlElementConfig)> = Vec::new();
101        for (i, element) in self.elements.iter().enumerate() {
102            let mut newelement = None;
103            for basename in element.base.iter().rev() {
104                if let Some(baseelement) = self.baseelements.get(basename) {
105                    if newelement.is_none() {
106                        newelement = Some(element.clone());
107                    }
108                    newelement
109                        .as_mut()
110                        .map(|newelement| newelement.update(baseelement));
111                } else {
112                    return Err(XmlConversionError::ConfigError(format!(
113                        "No such base element: {}",
114                        basename
115                    )));
116                }
117            }
118            if let Some(newelement) = newelement {
119                replace.push((i, newelement));
120            }
121        }
122        for (i, element) in replace {
123            self.elements[i] = element;
124        }
125        Ok(())
126    }
127
128    /// Parse the configuration from a TOML string (load the data from file yourself).
129    pub fn from_toml_str(tomlstr: &str) -> Result<Self, String> {
130        let mut config: Self = toml::from_str(tomlstr).map_err(|e| format!("{}", e))?;
131        config.resolve_baseelements().map_err(|e| format!("{}", e))?;
132        Ok(config)
133    }
134
135    pub fn with_debug(mut self, value: bool) -> Self {
136        self.debug = value;
137        self
138    }
139
140    /// Add provenance information pointing each annotation to the appropriate node in the XML source files where it came from (translates into XPathSelector in Web Annotation output)
141    pub fn with_provenance(mut self, value: bool) -> Self {
142        self.provenance = value;
143        self
144    }
145
146    /// Register an XML namespace with prefix
147    pub fn with_prefix(mut self, prefix: impl Into<String>, namespace: impl Into<String>) -> Self {
148        self.namespaces.insert(prefix.into(), namespace.into());
149        self
150    }
151
152    /// A prefix to assign when setting annotation IDs, within this string you can use the special variable `{resource}` to use the resource ID.
153    pub fn with_id_prefix(mut self, prefix: impl Into<String>) -> Self {
154        self.id_prefix = Some(prefix.into());
155        self
156    }
157
158    /// A suffix to strip when assigning annotation IDs
159    pub fn with_id_strip_suffix(mut self, suffix: impl Into<String>) -> Self {
160        self.id_strip_suffix.push(suffix.into());
161        self
162    }
163
164    /// Inject a DTD (for XML entity resolution)
165    pub fn with_inject_dtd(mut self, dtd: impl Into<String>) -> Self {
166        self.inject_dtd = Some(dtd.into());
167        self
168    }
169
170    /// Set default whitespace handling
171    pub fn with_whitespace(mut self, handling: XmlWhitespaceHandling) -> Self {
172        self.whitespace = handling;
173        self
174    }
175
176    /// Set an element configuration
177    pub fn with_element<F>(mut self, expression: &str, setup: F) -> Self
178    where
179        F: Fn(XmlElementConfig) -> XmlElementConfig,
180    {
181        let expression = XPathExpression::new(expression);
182        let element = setup(XmlElementConfig::new(expression));
183        if self.debug {
184            eprintln!("[STAM fromxml] registered {:?}", element);
185        }
186        self.elements.push(element);
187        self
188    }
189
190    /// How to handle this element?
191    fn element_config(&self, node: Node, path: &NodePath) -> Option<&XmlElementConfig> {
192        for elementconfig in self.elements.iter().rev() {
193            if elementconfig.path.test(path, node, self) {
194                return Some(elementconfig);
195            }
196        }
197        None
198    }
199
200    pub fn add_context(&mut self, key: impl Into<String>, value: toml::Value) {
201        self.context.insert(key.into(), value);
202    }
203
204    pub fn debug(&self) -> bool {
205        self.debug
206    }
207}
208
209#[derive(Clone, Copy, Debug, PartialEq, Deserialize)]
210/// Determines how to handle whitespace for an XML element
211pub enum XmlWhitespaceHandling {
212    /// Not specified (used for base templates)
213    Unspecified,
214    //Inherit from parent
215    Inherit,
216    /// Whitespace is kept as is in the XML
217    Preserve,
218    /// all whitespace becomes space, consecutive whitespace is squashed
219    Collapse,
220}
221
222impl Default for XmlWhitespaceHandling {
223    fn default() -> Self {
224        XmlWhitespaceHandling::Unspecified
225    }
226}
227
228impl XmlWhitespaceHandling {
229    fn collapse() -> Self {
230        XmlWhitespaceHandling::Collapse
231    }
232}
233
234#[derive(Debug, Clone, Deserialize, PartialEq, Copy, Default)]
235pub enum XmlAnnotationHandling {
236    /// No annotation
237    #[default]
238    Unspecified,
239
240    /// No annotation
241    None,
242
243    /// Selects the text pertaining to the current element
244    TextSelector,
245
246    /// Selects the text pertaining to the current resource
247    ResourceSelector,
248
249    /// Selects the text between the current element and the next instance of the same element type
250    TextSelectorBetweenMarkers,
251}
252
253#[derive(Debug, Clone, Deserialize)]
254/// XML Element configuration, determines how to map an XML element (identified by an XPath expression) to STAM
255pub struct XmlElementConfig {
256    /// This is XPath-like expression (just a small subset of XPath) to identify an element by its path
257
258    #[serde(default)]
259    path: XPathExpression,
260
261    #[serde(default)]
262    annotation: XmlAnnotationHandling,
263
264    #[serde(default)]
265    annotationdata: Vec<XmlAnnotationDataConfig>,
266
267    /// Template or None for no text handling, prefixes are never targeted by annotations
268    #[serde(default)]
269    textprefix: Option<String>,
270
271    /// Extract text. None means unspecified and defaults to false.
272    #[serde(default)]
273    text: Option<bool>,
274
275    /// Template or None for no text handling, suffixes are never targeted by annotations
276    #[serde(default)]
277    textsuffix: Option<String>,
278
279    // Annotation data for the text prefix
280    #[serde(default)]
281    annotatetextprefix: Vec<XmlAnnotationDataConfig>,
282
283    // Annotation data for the text suffix
284    #[serde(default)]
285    annotatetextsuffix: Vec<XmlAnnotationDataConfig>,
286
287    /// Include the text prefix in the annotation's text selector. None means unspecified and defaults to false
288    #[serde(default)]
289    include_textprefix: Option<bool>,
290
291    /// Include the text suffix in the annotation's text selector. None means unspecified and defaults to false
292    #[serde(default)]
293    include_textsuffix: Option<bool>,
294
295    /// Base elements to derive from
296    #[serde(default)]
297    base: Vec<String>,
298
299    /// Template or None for no ID extraction
300    #[serde(default)]
301    id: Option<String>,
302
303    #[serde(default)]
304    /// Descend into children (false) or not? (true). None means unspecified and defaults to false
305    stop: Option<bool>,
306
307    #[serde(default)]
308    /// Whitespace handling for this element
309    whitespace: XmlWhitespaceHandling,
310
311    #[serde(default)]
312    /// Assigns a scope id to this text range, it can later be referenced to constrain marker based annotation via `marker_scope`
313    scope_id: Option<String>,
314
315    #[serde(default)]
316    /// If annotation handling is TextSelectorBetweenMarkers, this sets a scope so the last marker won't transcend (otherwise you get all text of the document)
317    /// The scope refers to the `scope_id` of another element that was used in text extraction.
318    marker_scope: Option<String>,
319
320}
321
322impl XmlElementConfig {
323    fn new(expression: XPathExpression) -> Self {
324        Self {
325            path: expression,
326            stop: None,
327            whitespace: XmlWhitespaceHandling::Unspecified,
328            annotation: XmlAnnotationHandling::Unspecified,
329            annotationdata: Vec::new(),
330            base: Vec::new(),
331            id: None,
332            textprefix: None,
333            text: None,
334            textsuffix: None,
335            annotatetextprefix: Vec::new(),
336            annotatetextsuffix: Vec::new(),
337            include_textprefix: None,
338            include_textsuffix: None,
339            scope_id: None,
340            marker_scope: None,
341        }
342    }
343
344    pub fn update(&mut self, base: &XmlElementConfig) {
345        if self.whitespace == XmlWhitespaceHandling::Unspecified
346            && base.whitespace != XmlWhitespaceHandling::Unspecified
347        {
348            self.whitespace = base.whitespace;
349        }
350        if self.annotation == XmlAnnotationHandling::Unspecified
351            && base.annotation != XmlAnnotationHandling::Unspecified
352        {
353            self.annotation = base.annotation;
354        }
355        if self.textprefix.is_none() && base.textprefix.is_some() {
356            self.textprefix = base.textprefix.clone();
357        }
358        if self.text.is_none() && base.text.is_some() {
359            self.text = base.text;
360        }
361        if self.textsuffix.is_none() && base.textsuffix.is_some() {
362            self.textsuffix = base.textsuffix.clone();
363        }
364        if self.id.is_none() && base.id.is_some() {
365            self.id = base.id.clone();
366        }
367        if self.stop.is_none() && base.stop.is_some() {
368            self.stop = base.stop;
369        }
370        for annotationdata in base.annotationdata.iter() {
371            if !self.annotationdata.contains(annotationdata) {
372                self.annotationdata.push(annotationdata.clone());
373            }
374        }
375        if self.annotatetextsuffix.is_empty() && !base.annotatetextsuffix.is_empty() {
376            self.annotatetextsuffix = base.annotatetextsuffix.clone();
377        }
378        if self.annotatetextprefix.is_empty() && !base.annotatetextprefix.is_empty() {
379            self.annotatetextprefix = base.annotatetextprefix.clone();
380        }
381        if self.include_textsuffix.is_none() {
382            self.include_textsuffix = base.include_textsuffix;
383        }
384        if self.include_textprefix.is_none() {
385            self.include_textprefix = base.include_textprefix;
386        }
387    }
388
389
390    /// This sets the mode that determines how the element is handledhttps://www.youtube.com/watch?v=G_BrbhRrP6g
391    pub fn with_stop(mut self, stop: bool) -> Self {
392        self.stop = Some(stop);
393        self
394    }
395
396    /// This sets the whitespace handling for this element
397    pub fn with_whitespace(mut self, handling: XmlWhitespaceHandling) -> Self {
398        self.whitespace = handling;
399        self
400    }
401
402    pub fn with_text(mut self, text: bool) -> Self {
403        self.text = Some(text);
404        self
405    }
406
407    pub fn with_base(mut self, iter: impl Iterator<Item = impl Into<String>>) -> Self {
408        self.base = iter.into_iter().map(|s| s.into()).collect();
409        self
410    }
411
412    pub fn without_text(mut self) -> Self {
413        self.text = None;
414        self
415    }
416
417    pub fn with_annotation(mut self, annotation: XmlAnnotationHandling) -> Self {
418        self.annotation = annotation;
419        self
420    }
421
422    /// Not a very safe hash function (just uses an address uniquely associated with this object) but works for our ends
423    fn hash(&self) -> usize {
424        self.path.0.as_ptr() as usize
425    }
426}
427
428impl PartialEq for XmlElementConfig {
429    fn eq(&self, other: &Self) -> bool {
430        self.hash() == other.hash()
431    }
432}
433
434#[derive(Debug, Clone, Deserialize, PartialEq)]
435pub struct XmlAnnotationDataConfig {
436    /// Template
437    id: Option<String>,
438    /// Template
439    set: Option<String>,
440    /// Template
441    key: Option<String>,
442    /// Any string values are interpreted as templates
443    value: Option<toml::Value>,
444
445    /// The type of the value, will be automatically detected if not set.
446    #[serde(default)]
447    valuetype: Option<String>,
448
449    /// Allow value templates that yield an empty string?
450    #[serde(default)]
451    allow_empty_value: bool,
452
453    /// Skip this data entirely if any underlying variables in the templates are undefined
454    #[serde(default)]
455    skip_if_missing: bool,
456
457
458    /// If the value is a list, convert it to multiple annotationdata instances with the same key, one for each of the values
459    #[serde(default)]
460    multiple: bool,
461}
462
463impl XmlAnnotationDataConfig {
464    pub fn with_id(mut self, id: impl Into<String>) -> Self {
465        self.id = Some(id.into());
466        self
467    }
468
469    pub fn with_set(mut self, set: impl Into<String>) -> Self {
470        self.set = Some(set.into());
471        self
472    }
473
474    pub fn with_key(mut self, key: impl Into<String>) -> Self {
475        self.key = Some(key.into());
476        self
477    }
478
479    pub fn with_value(mut self, value: impl Into<toml::Value>) -> Self {
480        self.value = Some(value.into());
481        self
482    }
483}
484
485/// Not really full XPath, just a very minor subset
486#[derive(Debug, Clone, PartialEq, Deserialize)]
487struct XPathExpression(String);
488
489impl XPathExpression {
490    pub fn new(expression: impl Into<String>) -> Self {
491        Self(expression.into())
492    }
493
494    pub fn any() -> Self {
495        Self("*".into())
496    }
497
498    pub fn iter<'a>(
499        &'a self,
500        config: &'a XmlConversionConfig,
501    ) -> impl Iterator<Item = (Option<&'a str>, &'a str, Option<&'a str>)> {
502        self.0.trim_start_matches('/').split("/").map(|segment| {
503            //eprintln!("DEBUG: segment={}", segment);
504            let (prefix, name, condition) = Self::parse_segment(segment);
505            let namespace = if let Some(prefix) = prefix {
506                if let Some(namespace) = config.namespaces.get(prefix).map(|x| x.as_str()) {
507                    Some(namespace)
508                } else {
509                    panic!(
510                        "XML namespace prefix not known in configuration: {}",
511                        prefix
512                    );
513                }
514            } else {
515                None
516            };
517            (namespace, name, condition)
518        })
519    }
520
521    /// matches a node path against an XPath-like expression
522    fn test<'a, 'b>(&self, path: &NodePath<'a, 'b>, node: Node<'a,'b>, config: &XmlConversionConfig) -> bool {
523        let refiter = self.iter(config).collect::<Vec<_>>().into_iter().rev();
524        let pathiter = path.components.iter().rev();
525        self.test_withiter(refiter, pathiter, node, config)
526    }
527
528    /// matches a node path against an XPath-like expression
529    /// The refiter is from the configuration, pathiter from the document. These paths are matched against eachother.
530    fn test_withiter<'a, 'b>(&self, mut refiter: impl Iterator<Item=(Option<&'a str>, &'a str, Option<&'a str>)> + Clone, mut pathiter: impl Iterator<Item=&'a NodePathComponent<'a, 'b>> + Clone, mut node: Node<'a,'b>, config: &XmlConversionConfig) -> bool {
531        while let Some((refns, refname, condition)) = refiter.next() {
532            if refns.is_none() && refname == "" && condition.is_none() {
533                // This is a `//` selector (empty refname/ns/condition), we bifurcate here so we match both in case SOMETHING matches as well as when NOTHING matches, the recursion covers the latter logic route
534                if self.test_withiter(refiter.clone(), pathiter.clone(), node, config) {
535                    return true;
536                }
537                // Bifurcate again for every possible component in the document path (as // is greedy and can match all) !
538                let mut pathiter2 = pathiter.clone();
539                while let Some(_) = pathiter2.next() {
540                    if self.test_withiter(refiter.clone(), pathiter2.clone(), node, config) {
541                        return true;
542                    }
543                }
544            }
545            if let Some(component) = pathiter.next() {
546                /*
547                if config.debug() {
548                    eprintln!("[STAM fromxml]          testing component {:?} against refns={:?} refname={} condition={:?}", component, refns, refname, condition);
549                }
550                */
551                if refname != "" && refname != "*" {
552                    if refns.is_none() != component.namespace.is_none() || component.namespace != refns || refname != component.tagname {
553                        return false;
554                    }
555                }
556                if let Some(condition) = condition {
557                    if !self.test_condition(condition, node, config) {
558                        return false;
559                    }
560                }
561                if let Some(parent) = node.parent() {
562                    node = parent;
563                }
564            } else {
565                if refname != "" {
566                    return false;
567                }
568            }
569        }
570        /*
571        if config.debug() {
572            eprintln!("[STAM fromxml]          match");
573        }
574        */
575        true
576    }
577
578
579    fn test_condition<'a,'b>(&self, condition: &'a str, node: Node<'a,'b>, config: &XmlConversionConfig) -> bool {
580        for condition in condition.split(" and ") { //MAYBE TODO: doesn't take literals into account yet!
581            if let Some(pos) = condition.find("!=") {
582                let var = &condition[..pos];
583                let right = condition[pos+2..].trim_matches('"');
584                if self.get_var(var, &node, config) == Some(right) {
585                    return false;
586                }
587            } else if let Some(pos) = condition.find("=") {
588                let var = &condition[..pos];
589                let right = condition[pos+1..].trim_matches('"');
590                let value = self.get_var(var, &node, config);
591                if value != Some(right) {
592                    return false;
593                }
594            } else {
595                //condition is one variable and merely needs to exist
596                let v = self.get_var(condition, &node, config);
597                if v.is_none() || v == Some("") {
598                    return false;
599                }
600            }
601        }
602        /*if config.debug() {
603            eprintln!("[STAM fromxml]          condition matches");
604        }*/
605        true
606    }
607
608    /// Resolve a variable from a conditional expression, given a variable name, node and config
609    fn get_var<'a,'b>(&self, var: &str, node: &Node<'a,'b>, config: &XmlConversionConfig) -> Option<&'a str> { 
610        if var.starts_with("@") {
611            if let Some(pos) = var.find(":") {
612                let prefix = &var[1..pos];
613                if let Some(ns) = config.namespaces.get(prefix) {
614                    let var = &var[pos+1..];
615                    node.attribute((ns.as_str(),var))
616                } else {
617                    None
618                }
619            } else {
620                node.attribute(&var[1..])
621            }
622        } else if var == "text()" {
623            node.text().map(|s|s.trim())
624        } else {
625            None
626        }
627    }
628
629    /// Parses a segment into a namespace-prefix, a name and a condition
630    fn parse_segment<'a>(s: &'a str) -> (Option<&'a str>, &'a str, Option<&'a str>) {
631        let (name, condition) = if let (Some(begin), Some(end)) = (s.find("["), s.rfind("]")) {
632            (&s[..begin], Some(&s[begin + 1..end]))
633        } else {
634            (s, None)
635        };
636        if let Some((prefix, name)) = name.split_once(":") {
637            (Some(prefix), name, condition)
638        } else {
639            (None, name, condition)
640        }
641    }
642}
643
644
645
646impl Default for XPathExpression {
647    fn default() -> Self {
648        Self::any()
649    }
650}
651
652#[derive(Clone, Debug, PartialEq)]
653struct NodePathComponent<'a,'b> {
654    namespace: Option<&'a str>,
655    tagname: &'b str,
656    /// Index sequence number, 1-indexed (as specified by XPath)
657    index: Option<usize>,
658}
659
660#[derive(Clone, Debug, PartialEq, Default)]
661struct NodePath<'a, 'b> {
662    components: Vec<NodePathComponent<'a,'b>>,
663}
664
665impl<'a, 'b> Display for NodePath<'a, 'b> {
666    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
667        for component in self.components.iter() {
668            write!(f, "/")?;
669            if let Some(ns) = component.namespace {
670                if let Some(index) = component.index {
671                    write!(f, "{{{}}}{}[{}]", ns, component.tagname, index)?;
672                } else {
673                    write!(f, "{{{}}}{}", ns, component.tagname)?;
674                }
675            } else {
676                if let Some(index) = component.index {
677                    write!(f, "{}[{}]", component.tagname, index)?;
678                } else {
679                    write!(f, "{}", component.tagname)?;
680                }
681            }
682        }
683        Ok(())
684    }
685}
686
687impl<'a,'b> NodePath<'a,'b> {
688    fn add(&mut self, node: &Node<'a,'b>, index: Option<usize>) {
689        if node.tag_name().name() != "" {
690            self.components.push(
691                NodePathComponent {
692                    namespace: node.tag_name().namespace(),
693                    tagname: node.tag_name().name(),
694                    index,
695                }
696            )
697        }
698    }
699
700    fn format_as_xpath(&self, prefixes: &HashMap<String, String>) -> String {
701        let mut out = String::new();
702        for component in self.components.iter() {
703            out.push('/');
704            if let Some(ns) = component.namespace {
705                if let Some(prefix) = prefixes.get(ns) {
706                    if let Some(index) = component.index {
707                        out += &format!("{}:{}[{}]", prefix, component.tagname, index);
708                    } else {
709                        out += &format!("{}:{}", prefix, component.tagname);
710                    }
711                } else {
712                    eprintln!("STAM fromxml WARNING: format_as_xpath: namespace {} not defined, no prefix found!", ns);
713                    if let Some(index) = component.index {
714                        out += &format!("{}[{}]", component.tagname, index);
715                    } else {
716                        out += &format!("{}", component.tagname);
717                    }
718                }
719            } else {
720                if let Some(index) = component.index {
721                    out += &format!("{}[{}]", component.tagname, index);
722                } else {
723                    out += &format!("{}", component.tagname);
724                }
725            }
726        }
727        out
728    }
729}
730
731
732/// Counts elder siblings, used to determine index values
733#[derive(Default,Debug)]
734struct SiblingCounter {
735    map: HashMap<String,usize>,
736}
737
738impl SiblingCounter {
739    fn count<'a,'b>(&mut self, node: &Node<'a,'b>) -> usize {
740        let s = format!("{:?}", node.tag_name());
741        *self.map.entry(s).and_modify(|c| {*c += 1;}).or_insert(1)
742    }
743}
744
745
746#[derive(Debug, Clone, Deserialize)]
747/// XML Element configuration, determines how to map an XML element (identified by an XPath expression) to STAM
748pub struct MetadataConfig {
749    /// This is XPath-like expression (just a small subset of XPath) to identify an element by its path
750    #[serde(default)]
751    annotation: XmlAnnotationHandling,
752
753    #[serde(default)]
754    annotationdata: Vec<XmlAnnotationDataConfig>,
755
756    /// Template or None for no ID extraction
757    #[serde(default)]
758    id: Option<String>,
759}
760
761/// Translate an XML file to STAM, given a particular configuration
762pub fn from_xml<'a>(
763    filename: &Path,
764    config: &XmlConversionConfig,
765    store: &'a mut AnnotationStore,
766) -> Result<(), String> {
767    if config.debug {
768        eprintln!("[STAM fromxml] parsing {}", filename.display());
769    }
770
771    // Read the raw XML data
772    let mut xmlstring = read_to_string(filename)
773        .map_err(|e| format!("Error opening XML file {}: {}", filename.display(), e))?;
774
775    //patchy: remove HTML5 doctype and inject our own
776    if xmlstring[..100].find("<!DOCTYPE html>").is_some() && config.inject_dtd.is_some() {
777        xmlstring = xmlstring.replacen("<!DOCTYPE html>", "", 1);
778    }
779
780    // we can only inject a DTD if there is no doctype
781    if xmlstring[..100].find("<!DOCTYPE").is_none() {
782        if let Some(dtd) = config.inject_dtd.as_ref() {
783            xmlstring = dtd.to_string() + &xmlstring
784        };
785    } else if config.inject_dtd.is_some() {
786        eprintln!("[STAM fromxml] WARNING: Can not inject DTD because file already has a DOCTYPE");
787    }
788
789    // parse the raw XML data into a DOM
790    let doc = Document::parse_with_options(
791        &xmlstring,
792        ParsingOptions {
793            allow_dtd: true,
794            ..ParsingOptions::default()
795        },
796    )
797    .map_err(|e| format!("Error parsing XML file {}: {}", filename.display(), e))?;
798
799    let mut converter = XmlToStamConverter::new(config);
800    converter
801        .compile()
802        .map_err(|e| format!("Error compiling templates: {}", e))?;
803
804    let textoutfilename = format!(
805        "{}.txt",
806        filename
807            .file_stem()
808            .expect("invalid filename")
809            .to_str()
810            .expect("invalid utf-8 in filename")
811    );
812
813    // extract text (first pass)
814    let mut path = NodePath::default();
815    path.add(&doc.root_element(), None);
816    converter
817        .extract_element_text(doc.root_element(), &path, converter.config.whitespace, Some(textoutfilename.as_str()), Some(&filename.to_string_lossy()), 0)
818        .map_err(|e| {
819            format!(
820                "Error extracting element text from {}: {}",
821                filename.display(),
822                e
823            )
824        })?;
825    if config.debug {
826        eprintln!("[STAM fromxml] extracted full text: {}", &converter.text);
827    }
828    let resource = TextResourceBuilder::new()
829        .with_id(filename_to_id(textoutfilename.as_str(), config).to_string())
830        .with_text(converter.text.clone())
831        .with_filename(&textoutfilename);
832
833    converter.resource_handle = Some(
834        store
835            .add_resource(resource)
836            .map_err(|e| format!("Failed to add resource {}: {}", &textoutfilename, e))?,
837    );
838
839    converter.add_metadata(store).map_err(|e| format!("Failed to add metadata {}: {}", &textoutfilename, e))?;
840
841    // extract annotations (second pass)
842    converter
843        .extract_element_annotation(doc.root_element(), &path,  Some(&filename.to_string_lossy()),0,  store)
844        .map_err(|e| {
845            format!(
846                "Error extracting element annotation from {}: {}",
847                filename.display(),
848                e
849            )
850        })?;
851
852    Ok(())
853}
854
855/// Translate an XML file to STAM, given a particular configuration. This translates multiple XML files to a single output file.
856pub fn from_multi_xml<'a>(
857    filenames: &Vec<&Path>,
858    outputfile: Option<&Path>,
859    config: &XmlConversionConfig,
860    store: &'a mut AnnotationStore,
861) -> Result<(), String> {
862
863    let textoutfilename = if let Some(outputfile) = outputfile {
864        format!("{}",outputfile.to_str().expect("invalid utf-8 in filename"))
865    } else {
866        format!(
867            "{}.txt",
868                filenames.iter().next().expect("1 or more filename need to be provided")
869                .file_stem()
870                .expect("invalid filename")
871                .to_str()
872                .expect("invalid utf-8 in filename")
873        )
874    };
875
876    // Read the raw XML data
877    let mut xmlstrings: Vec<String> = Vec::new();
878    let mut docs: Vec<Document> = Vec::new();
879    for filename in filenames.iter() {
880        if config.debug {
881            eprintln!("[STAM fromxml] parsing {} (one of multiple)", filename.display());
882        }
883        //patchy: remove HTML5 doctype and inject our own
884        let mut xmlstring = read_to_string(filename).map_err(|e| format!("Error opening XML file {}: {}", filename.display(), e))?;
885        if xmlstring[..100].find("<!DOCTYPE html>").is_some() && config.inject_dtd.is_some() {
886            xmlstring = xmlstring.replacen("<!DOCTYPE html>", "", 1);
887        }
888        // we can only inject a DTD if there is no doctype
889        if xmlstring[..100].find("<!DOCTYPE").is_none() {
890            if let Some(dtd) = config.inject_dtd.as_ref() {
891                xmlstring = dtd.to_string() + &xmlstring
892            };
893        } else if config.inject_dtd.is_some() {
894            eprintln!("[STAM fromxml] WARNING: Can not inject DTD because file already has a DOCTYPE");
895        }
896        xmlstrings.push(xmlstring);
897    }
898
899    for (filename, xmlstring) in filenames.iter().zip(xmlstrings.iter()) {
900        // parse the raw XML data into a DOM
901        let doc = Document::parse_with_options(
902            xmlstring,
903            ParsingOptions {
904                allow_dtd: true,
905                ..ParsingOptions::default()
906            },
907        )
908        .map_err(|e| format!("Error parsing XML file {}: {}", filename.display(), e))?;
909        docs.push(doc);
910    }
911
912    let mut converter = XmlToStamConverter::new(config);
913    converter
914        .compile()
915        .map_err(|e| format!("Error compiling templates: {}", e))?;
916
917    for (i, (doc, filename)) in docs.iter().zip(filenames.iter()).enumerate() {
918        let mut path = NodePath::default();
919        path.add(&doc.root_element(), None);
920        // extract text (first pass)
921        converter
922            .extract_element_text(doc.root_element(), &path, converter.config.whitespace, Some(textoutfilename.as_str()), Some(&filename.to_string_lossy()), i)
923            .map_err(|e| {
924                format!(
925                    "Error extracting element text from {}: {}",
926                    filename.display(),
927                    e
928                )
929            })?;
930        if config.debug {
931            eprintln!("[STAM fromxml] extracted full text: {}", &converter.text);
932        }
933    }
934
935    let resource = TextResourceBuilder::new()
936        .with_id(filename_to_id(textoutfilename.as_str(), config).to_string())
937        .with_text(converter.text.clone())
938        .with_filename(&textoutfilename);
939
940    converter.resource_handle = Some(
941        store
942            .add_resource(resource)
943            .map_err(|e| format!("Failed to add resource {}: {}", &textoutfilename, e))?,
944    );
945
946    converter.add_metadata(store).map_err(|e| format!("Failed to add metadata {}: {}", &textoutfilename, e))?;
947
948    // extract annotations (second pass)
949    for (i,(doc, filename)) in docs.iter().zip(filenames.iter()).enumerate() {
950        let mut path = NodePath::default();
951        path.add(&doc.root_element(), None);
952        converter
953            .extract_element_annotation(doc.root_element(), &path, Some(&filename.to_string_lossy()),i,  store)
954            .map_err(|e| {
955                format!(
956                    "Error extracting element annotation from {}: {}",
957                    filename.display(),
958                    e
959                )
960            })?;
961    }
962
963    Ok(())
964}
965
966/// Translate an XML file to STAM, given a particular configuration. Not writing output files and keeping all in memory. Does not support DTD injection.
967pub fn from_xml_in_memory<'a>(
968    resource_id: &str, 
969    xmlstring: &str,
970    config: &XmlConversionConfig,
971    store: &'a mut AnnotationStore,
972) -> Result<(), String> {
973    if config.debug {
974        eprintln!("[STAM fromxml] parsing XML string");
975    }
976
977    // parse the raw XML data into a DOM
978    let doc = Document::parse_with_options(
979        &xmlstring,
980        ParsingOptions {
981            allow_dtd: true,
982            ..ParsingOptions::default()
983        },
984    )
985    .map_err(|e| format!("Error parsing XML string: {}",  e))?;
986
987    let mut converter = XmlToStamConverter::new(config);
988    converter
989        .compile()
990        .map_err(|e| format!("Error compiling templates: {}", e))?;
991
992    let mut path = NodePath::default();
993    path.add(&doc.root_element(), None);
994    // extract text (first pass)
995    converter
996        .extract_element_text(doc.root_element(), &path, converter.config.whitespace, Some(resource_id), Some(resource_id), 0)
997        .map_err(|e| {
998            format!(
999                "Error extracting element text from {}: {}",
1000                resource_id,
1001                e
1002            )
1003        })?;
1004    if config.debug {
1005        eprintln!("[STAM fromxml] extracted full text: {}", &converter.text);
1006    }
1007    let resource = TextResourceBuilder::new()
1008        .with_id(resource_id)
1009        .with_text(converter.text.clone());
1010
1011    converter.resource_handle = Some(
1012        store
1013            .add_resource(resource)
1014            .map_err(|e| format!("Failed to add resource {}: {}", &resource_id, e))?,
1015    );
1016
1017    converter.add_metadata(store).map_err(|e| format!("Failed to add metadata for {}: {}", &resource_id, e))?;
1018
1019    // extract annotations (second pass)
1020    converter
1021        .extract_element_annotation(doc.root_element(), &path, Some(resource_id), 0, store)
1022        .map_err(|e| {
1023            format!(
1024                "Error extracting element annotation from {}: {}",
1025                resource_id,
1026                e
1027            )
1028        })?;
1029
1030    Ok(())
1031}
1032
1033pub fn filename_to_id<'a>(filename: &'a str, config: &XmlConversionConfig) -> &'a str {
1034    for suffix in config.id_strip_suffix.iter() {
1035        if filename.ends_with(suffix) {
1036            return &filename[..filename.len() - suffix.len()];
1037        }
1038    }
1039    return filename;
1040}
1041
1042#[derive(Clone,Copy,PartialEq, Hash, Eq)]
1043enum PositionType {
1044    Body,
1045    TextPrefix,
1046    TextSuffix,
1047}
1048
1049struct XmlToStamConverter<'a> {
1050    /// The current character position the conversion process is at
1051    cursor: usize,
1052
1053    /// The extracted plain-text after/during untangling
1054    text: String,
1055
1056    /// The template engine
1057    template_engine: Engine<'a>,
1058
1059    /// Keep track of the new positions (unicode offset) where the node starts in the untangled document. The key consist of a document sequence number and a node ID.
1060    positionmap: HashMap<(usize,NodeId,PositionType), Offset>,
1061
1062    /// Keep track of the new positions (bytes offset) where the node starts in the untangled document. The key consist of a document sequence number and a node ID.
1063    bytepositionmap: HashMap<(usize,NodeId,PositionType), (usize, usize)>,
1064
1065    /// Keep track of markers (XML elements with `XmlAnnotationHandling::TextSelectorBetweenMarkers`), the key in this map is some hash of XmlElementConfig.
1066    markers: HashMap<usize, Vec<(usize,NodeId)>>,
1067
1068    /// Keep track of scopes. These are used to find marker scopes. Only the last scope is registered. The key is an Xpath expression. The value is a sequence number and node ID which can subsequently be looked up in the position maps
1069    scopes: HashMap<String, (usize,NodeId)>,
1070
1071    /// The resource
1072    resource_handle: Option<TextResourceHandle>,
1073
1074    /// Used to keep track of whether we need to insert a whitespace before actual text
1075    pending_whitespace: bool,
1076
1077    /// The configuration
1078    config: &'a XmlConversionConfig,
1079
1080    /// Namespace to prefix map
1081    prefixes: HashMap<String, String>,
1082
1083    ///  Global context for template
1084    global_context: BTreeMap<String, upon::Value>,
1085
1086    /// Variable names per template
1087    variables: BTreeMap<String, BTreeSet<&'a str>>,
1088
1089    debugindent: String,
1090}
1091
1092pub enum XmlConversionError {
1093    StamError(StamError),
1094    TemplateError(String, Option<upon::Error>),
1095    ConfigError(String),
1096}
1097
1098impl From<StamError> for XmlConversionError {
1099    fn from(error: StamError) -> Self {
1100        Self::StamError(error)
1101    }
1102}
1103
1104impl From<upon::Error> for XmlConversionError {
1105    fn from(error: upon::Error) -> Self {
1106        Self::TemplateError("".into(), Some(error))
1107    }
1108}
1109
1110impl Display for XmlConversionError {
1111    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
1112        match self {
1113            Self::StamError(e) => e.fmt(f),
1114            Self::TemplateError(s, e) => {
1115                f.write_str(s.as_str())?;
1116                f.write_str(": ")?;
1117                if let Some(e) = e {
1118                    e.fmt(f)?;
1119                }
1120                f.write_str("")
1121            }
1122            Self::ConfigError(e) => e.fmt(f),
1123        }
1124    }
1125}
1126
1127impl<'a> XmlToStamConverter<'a> {
1128    fn new(config: &'a XmlConversionConfig) -> Self {
1129        let mut prefixes: HashMap<String, String> = HashMap::new();
1130        for (prefix, namespace) in config.namespaces.iter() {
1131            prefixes.insert(namespace.to_string(), prefix.to_string());
1132        }
1133        let mut template_engine = Engine::new();
1134        template_engine.set_default_formatter(&value_formatter); //this one serializes Lists like in JSON
1135        template_engine.add_function("capitalize", filter_capitalize);
1136        template_engine.add_function("lower", str::to_lowercase);
1137        template_engine.add_function("upper", str::to_uppercase);
1138        template_engine.add_function("trim", |s: &upon::Value| match s {
1139            upon::Value::String(s) => s.trim().to_string(),
1140            upon::Value::Integer(i) => format!("{}",i),
1141            upon::Value::Float(f) => format!("{}",f),
1142            _ => String::new(),
1143        });
1144        template_engine.add_function("add", filter_add);
1145        template_engine.add_function("sub", filter_sub);
1146        template_engine.add_function("mul", filter_mul);
1147        template_engine.add_function("div", filter_div);
1148        template_engine.add_function("eq", |a: &upon::Value, b: &upon::Value| a == b);
1149        template_engine.add_function("ne", |a: &upon::Value, b: &upon::Value| a != b);
1150        template_engine.add_function("gt", filter_gt);
1151        template_engine.add_function("lt", filter_lt);
1152        template_engine.add_function("gte", filter_gte);
1153        template_engine.add_function("lte", filter_lte);
1154        template_engine.add_function("int", filter_int);
1155        template_engine.add_function("round", filter_round);
1156        template_engine.add_function("ceil", filter_ceil);
1157        template_engine.add_function("floor", filter_floor);
1158        template_engine.add_function("map_int", |list: &upon::Value| {
1159            if let upon::Value::List(list) = list {
1160                upon::Value::List(list.iter().map(filter_int).collect())
1161            } else {
1162                panic!("map_* filters expect a list as value") //<< --^  TODO: PANIC IS WAY TO STRICT
1163            }
1164        });
1165        template_engine.add_function("float", filter_float);
1166        template_engine.add_function("map_float", |list: &upon::Value| {
1167            if let upon::Value::List(list) = list {
1168                upon::Value::List(list.iter().map(filter_float).collect())
1169            } else {
1170                panic!("map_* filters expect a list as value") //<< --^  TODO: PANIC IS WAY TO STRICT
1171            }
1172        });
1173        template_engine.add_function("str", filter_str);
1174        template_engine.add_function("map_str", |list: upon::Value| {
1175            if let upon::Value::List(list) = list {
1176                upon::Value::List(list.into_iter().map(filter_str).collect())
1177            } else {
1178                panic!("map_* filters expect a list as value") //<< --^  TODO: PANIC IS WAY TO STRICT
1179            }
1180        });
1181        template_engine.add_function("as_range", |a: i64| upon::Value::List(std::ops::Range { start: 0, end: a }.into_iter().map(|x| upon::Value::Integer(x+1)).collect::<Vec<_>>()) );
1182        template_engine.add_function("last", |list: &[upon::Value]| list.last().map(Clone::clone));
1183        template_engine.add_function("first", |list: &[upon::Value]| {
1184            list.first().map(Clone::clone)
1185        });
1186        template_engine.add_function("tokenize", |s: &str| {
1187            upon::Value::List(
1188                s.split(|c| c == ' ' || c == '\n').filter_map(|x|
1189                    if !x.is_empty() {
1190                        Some(upon::Value::String(x.to_string()))
1191                    } else {
1192                        None
1193                    }
1194                )
1195                .collect::<Vec<upon::Value>>())
1196        });
1197        template_engine.add_function("replace", |s: &upon::Value, from: &str, to: &str| {
1198            match s {
1199                upon::Value::String(s) => upon::Value::String(s.replace(from,to)),
1200                upon::Value::Integer(i) => {
1201                    let s = format!("{}",i);
1202                    upon::Value::String(s.replace(from,to))
1203                },
1204                upon::Value::Float(i) => {
1205                    let s = format!("{}",i);
1206                    upon::Value::String(s.replace(from,to))
1207                }
1208                _ => {
1209                    //empty string in other cases, especially important in case of None (this happens if the ?. operator is used)
1210                    upon::Value::String(String::new())
1211                }
1212            }
1213        });
1214        template_engine.add_function("map_replace", |list: &upon::Value, from: &str, to: &str| {
1215            if let upon::Value::List(list) = list {
1216                upon::Value::List(list.iter().filter_map(|s: &upon::Value| if let upon::Value::String(s) = s { Some(upon::Value::String(s.replace(from,to))) } else { None } ).collect())
1217            } else {
1218                panic!("map_* filters expect a list as value") //<< --^  TODO: PANIC IS WAY TO STRICT
1219            }
1220        });
1221        template_engine.add_function("starts_with", |s: &upon::Value, prefix: &str| {
1222            if let upon::Value::String(s) = s {
1223                s.starts_with(prefix)
1224            } else {
1225                false
1226            }
1227        });
1228        template_engine.add_function("ends_with", |s: &upon::Value, suffix: &str| {
1229            if let upon::Value::String(s) = s {
1230                s.ends_with(suffix)
1231            } else {
1232                false
1233            }
1234        });
1235        template_engine.add_function("substr", filter_substr);
1236        template_engine.add_function("map_substr", |list: &upon::Value, begin: isize, end: isize| {
1237            if let upon::Value::List(list) = list {
1238                upon::Value::List(list.iter().map(|s: &upon::Value| upon::Value::String(filter_substr(s, begin,end)) ).collect())
1239            } else {
1240                panic!("map_* filters expect a list as value") //<< --^  TODO: PANIC IS WAY TO STRICT
1241            }
1242        });
1243        template_engine.add_function("basename", filter_basename);
1244        template_engine.add_function("map_basename", |list: &upon::Value| {
1245            if let upon::Value::List(list) = list {
1246                upon::Value::List(list.iter().map(filter_basename).collect())
1247            } else {
1248                panic!("map_* filters expect a list as value") //<< --^  TODO: PANIC IS WAY TO STRICT
1249            }
1250        });
1251        template_engine.add_function("noext", filter_noext );
1252        template_engine.add_function("map_noext", |list: &upon::Value| {
1253            if let upon::Value::List(list) = list {
1254                upon::Value::List(list.iter().map(|s| filter_noext(s).into()).collect())
1255            } else {
1256                panic!("map_* filters expect a list as value") //<< --^  TODO: PANIC IS WAY TO STRICT
1257            }
1258        });
1259        template_engine.add_function("join", |list: &upon::Value, delimiter: &str| match list {
1260            upon::Value::List(list) => { //too much cloning but it'll do for now
1261                let newlist: Vec<String> = list.iter().map(|v| match v {
1262                    upon::Value::String(s) => s.clone(),
1263                    upon::Value::Integer(d) => format!("{}",d),
1264                    upon::Value::Float(d) => format!("{}",d),
1265                    upon::Value::Bool(d) => format!("{}",d),
1266                    _ => String::new(),
1267                }).collect();
1268                upon::Value::String(newlist.join(delimiter))
1269            },
1270            _ => {
1271                list.clone() //was not really a list after all, just pass it on so we don't need to panic
1272            }
1273        });
1274        let mut converter = Self {
1275            cursor: 0,
1276            text: String::new(),
1277            template_engine,
1278            positionmap: HashMap::new(),
1279            bytepositionmap: HashMap::new(),
1280            scopes:  HashMap::new(),
1281            markers: HashMap::new(),
1282            resource_handle: None,
1283            pending_whitespace: false,
1284            global_context: BTreeMap::new(),
1285            debugindent: String::new(),
1286            variables: BTreeMap::new(),
1287            prefixes,
1288            config,
1289        };
1290        converter.set_global_context();
1291        converter.add_external_filters();
1292        converter
1293    }
1294
1295    fn add_external_filters(&mut self) {
1296        for filter in self.config.external_filters.clone() {
1297            self.template_engine.add_function(filter.name.clone(), move |value: &upon::Value| filter.run(value)  );
1298        }
1299    }
1300
1301    /// Compile templates
1302    fn compile(&mut self) -> Result<(), XmlConversionError> {
1303        if self.config.debug {
1304            eprintln!("[STAM fromxml] compiling templates");
1305        }
1306        for element in self.config.elements.iter() {
1307            if let Some(textprefix) = element.textprefix.as_ref() {
1308                if self.template_engine.get_template(textprefix.as_str()).is_none() {
1309                    let template = self.precompile(textprefix.as_str());
1310                    self.template_engine
1311                        .add_template(textprefix.clone(), template)
1312                        .map_err(|e| {
1313                            XmlConversionError::TemplateError(
1314                                format!("element/textprefix template {}", textprefix.clone()),
1315                                Some(e),
1316                            )
1317                        })?;
1318                }
1319            }
1320            if let Some(textsuffix) = element.textsuffix.as_ref() {
1321                if self.template_engine.get_template(textsuffix.as_str()).is_none() {
1322                    let template = self.precompile(textsuffix.as_str());
1323                    self.template_engine
1324                        .add_template(textsuffix.clone(), template)
1325                        .map_err(|e| {
1326                            XmlConversionError::TemplateError(
1327                                format!("element/textsuffix template {}", textsuffix.clone()),
1328                                Some(e),
1329                            )
1330                        })?;
1331                }
1332            }
1333            if let Some(id) = element.id.as_ref() {
1334                if self.template_engine.get_template(id.as_str()).is_none() {
1335                    let template = self.precompile(id.as_str());
1336                    self.template_engine.add_template(id.clone(), template).map_err(|e| {
1337                        XmlConversionError::TemplateError(
1338                            format!("element/id template {}", id.clone()),
1339                            Some(e),
1340                        )
1341                    })?;
1342                }
1343            }
1344            for annotationdata in element.annotationdata.iter().chain(element.annotatetextprefix.iter()).chain(element.annotatetextsuffix.iter()) {
1345                if let Some(id) = annotationdata.id.as_ref() {
1346                    if self.template_engine.get_template(id.as_str()).is_none() {
1347                        let template = self.precompile(id.as_str());
1348                        self.template_engine.add_template(id.clone(), template).map_err(|e| {
1349                            XmlConversionError::TemplateError(
1350                                format!("annotationdata/id template {}", id.clone()),
1351                                Some(e),
1352                            )
1353                        })?;
1354                    }
1355                }
1356                if let Some(set) = annotationdata.set.as_ref() {
1357                    if self.template_engine.get_template(set.as_str()).is_none() {
1358                        let template = self.precompile(set.as_str());
1359                        //eprintln!("------- DEBUG: {} -> {}", set.as_str(), template);
1360                        self.template_engine.add_template(set.clone(), template).map_err(|e| {
1361                            XmlConversionError::TemplateError(
1362                                format!("annotationdata/set template {}", set.clone()),
1363                                Some(e),
1364                            )
1365                        })?;
1366                    }
1367                }
1368                if let Some(key) = annotationdata.key.as_ref() {
1369                    if self.template_engine.get_template(key.as_str()).is_none() {
1370                        let template = self.precompile(key.as_str());
1371                        self.template_engine.add_template(key.clone(), template).map_err(|e| {
1372                            XmlConversionError::TemplateError(
1373                                format!("annotationdata/key template {}", key.clone()),
1374                                Some(e),
1375                            )
1376                        })?;
1377                    }
1378                }
1379                if let Some(value) = annotationdata.value.as_ref() {
1380                    self.compile_value(value)?;
1381                }
1382            }
1383        }
1384        for metadata in self.config.metadata.iter() {
1385            if let Some(id) = metadata.id.as_ref() {
1386                if self.template_engine.get_template(id.as_str()).is_none() {
1387                    let template = self.precompile(id.as_str());
1388                    self.template_engine.add_template(id.clone(), template).map_err(|e| {
1389                        XmlConversionError::TemplateError(
1390                            format!("metadata/id template {}", id.clone()),
1391                            Some(e),
1392                        )
1393                    })?;
1394                }
1395            }
1396            for annotationdata in metadata.annotationdata.iter() {
1397                if let Some(id) = annotationdata.id.as_ref() {
1398                    if self.template_engine.get_template(id.as_str()).is_none() {
1399                        let template = self.precompile(id.as_str());
1400                        self.template_engine.add_template(id.clone(), template).map_err(|e| {
1401                            XmlConversionError::TemplateError(
1402                                format!("annotationdata/id template {}", id.clone()),
1403                                Some(e),
1404                            )
1405                        })?;
1406                    }
1407                }
1408                if let Some(set) = annotationdata.set.as_ref() {
1409                    if self.template_engine.get_template(set.as_str()).is_none() {
1410                        let template = self.precompile(set.as_str());
1411                        //eprintln!("------- DEBUG: {} -> {}", set.as_str(), template);
1412                        self.template_engine.add_template(set.clone(), template).map_err(|e| {
1413                            XmlConversionError::TemplateError(
1414                                format!("annotationdata/set template {}", set.clone()),
1415                                Some(e),
1416                            )
1417                        })?;
1418                    }
1419                }
1420                if let Some(key) = annotationdata.key.as_ref() {
1421                    if self.template_engine.get_template(key.as_str()).is_none() {
1422                        let template = self.precompile(key.as_str());
1423                        self.template_engine.add_template(key.clone(), template).map_err(|e| {
1424                            XmlConversionError::TemplateError(
1425                                format!("annotationdata/key template {}", key.clone()),
1426                                Some(e),
1427                            )
1428                        })?;
1429                    }
1430                }
1431                if let Some(value) = annotationdata.value.as_ref() {
1432                    self.compile_value(value)?;
1433                }
1434            }
1435        }
1436        Ok(())
1437    }
1438
1439    /// Compile templates from a value, all strings are considered templates
1440    fn compile_value(&mut self, value: &'a toml::Value) -> Result<(), XmlConversionError> {
1441        match value {
1442            toml::Value::String(value) => {
1443                if self.template_engine.get_template(value.as_str()).is_none() {
1444                    let template = self.precompile(value.as_str());
1445                    self.template_engine.add_template(value.clone(), template).map_err(|e| {
1446                        XmlConversionError::TemplateError(
1447                            format!("annotationdata/value template {}", value.clone()),
1448                            Some(e),
1449                        )
1450                    })?;
1451                }
1452            }
1453            toml::Value::Table(map) => {
1454                for (_key, value) in map.iter() {
1455                    self.compile_value(value)?;
1456                }
1457            },
1458            toml::Value::Array(list) => {
1459                for value in list.iter() {
1460                    self.compile_value(value)?;
1461                }
1462            }
1463            _ => {} //no templates in other types
1464        }
1465        Ok(())
1466    }
1467
1468    /// untangle text, extract the text (and only the text)
1469    /// from an XML document, according to the
1470    /// mapping configuration and creates a STAM TextResource for it.
1471    /// Records exact offsets per element/node for later use during annotation extraction.
1472    fn extract_element_text<'b>(
1473        &mut self,
1474        node: Node<'a,'b>,
1475        path: &NodePath<'a,'b>,
1476        whitespace: XmlWhitespaceHandling,
1477        resource_id: Option<&str>,
1478        inputfile: Option<&str>,
1479        doc_num: usize,
1480    ) -> Result<(), XmlConversionError> {
1481        if self.config.debug {
1482            eprintln!("[STAM fromxml]{} extracting text for element {}", self.debugindent, path);
1483        }
1484        let mut begin = self.cursor; //current character pos marks the begin
1485        let mut bytebegin = self.text.len(); //current byte pos marks the begin
1486        let mut end_discount = 0; //the discount may be needed later if textsuffixes are outputted (which we do not want as part of the annotation)
1487        let mut end_bytediscount = 0;
1488        let mut firsttext = true; //tracks whether we have already outputted some text, needed for whitespace handling
1489
1490        let mut elder_siblings = SiblingCounter::default();
1491
1492        // obtain the configuration that applies to this element
1493        if let Some(element_config) = self.config.element_config(node, path) {
1494            if self.config.debug {
1495                eprintln!("[STAM fromxml]{} matching config: {:?}", self.debugindent, element_config);
1496            }
1497
1498            if (element_config.stop == Some(false) || element_config.stop.is_none())
1499                && element_config.annotation != XmlAnnotationHandling::TextSelectorBetweenMarkers
1500            {
1501                //do text extraction for this element
1502
1503                let whitespace = if node.has_attribute((NS_XML, "space")) {
1504                    // if there is an explicit xml:space attributes, it overrides whatever whitespace handling we have set:
1505                    match node.attribute((NS_XML, "space")).unwrap() {
1506                        "preserve" => XmlWhitespaceHandling::Preserve,
1507                        "collapse" | "replace" => XmlWhitespaceHandling::Collapse,
1508                        _ => whitespace,
1509                    }
1510                } else if element_config.whitespace == XmlWhitespaceHandling::Inherit
1511                    || element_config.whitespace == XmlWhitespaceHandling::Unspecified
1512                {
1513                    whitespace //from parent, i.e. passed to this (recursive) function by caller
1514                } else {
1515                    element_config.whitespace //default from the config
1516                };
1517
1518                // process the text prefix, a text template to include prior to the actual text
1519                self.process_textprefix(element_config, node, resource_id, inputfile, doc_num, &mut begin, &mut bytebegin)?;
1520
1521                let textbegin = self.cursor;
1522                // process all child elements
1523                for child in node.children() {
1524                    if self.config.debug {
1525                        eprintln!("[STAM fromxml]{} child {:?}", self.debugindent,child);
1526                        eprintln!("[STAM fromxml]{}  cursor={} begin={} textbegin={}", self.debugindent, self.cursor, begin, textbegin);
1527                    }
1528                    if child.is_text() && element_config.text == Some(true) {
1529                        // extract the actual element text
1530                        // this may trigger multiple times if the XML element (`node`) has mixed content
1531
1532                        let mut innertext = child.text().expect("text node must have text");
1533                        let mut pending_whitespace = false;
1534                        let mut leading_whitespace = false;
1535                        if whitespace == XmlWhitespaceHandling::Collapse && !innertext.is_empty() {
1536                            // analyse what kind of whitespace we are dealing with
1537                            let mut all_whitespace = true;
1538                            leading_whitespace = innertext.chars().next().unwrap().is_whitespace();
1539
1540                            // any pending whitespace after this element is 'buffered' in this boolean
1541                            // and only written out depending on the next text's whitespace situation
1542                            // it will later be assigned to self.pending_whitespace just before going next iteration
1543                            pending_whitespace = innertext
1544                                .chars()
1545                                .inspect(|c| {
1546                                    if !c.is_whitespace() {
1547                                        all_whitespace = false
1548                                    }
1549                                })
1550                                .last()
1551                                .unwrap()
1552                                .is_whitespace();
1553                            if all_whitespace {
1554                                self.pending_whitespace = true;
1555                                if self.config.debug {
1556                                    eprintln!(
1557                                        "[STAM fromxml]{} ^- all whitespace, flag pending whitespace and skipping...",
1558                                        self.debugindent,
1559                                    );
1560                                }
1561                                continue;
1562                            }
1563                            innertext = innertext.trim();
1564                            if self.config.debug {
1565                                eprintln!(
1566                                    "[STAM fromxml]{} ^- collapsed whitespace: {:?}",
1567                                    self.debugindent,
1568                                    innertext
1569                                );
1570                            }
1571                        }
1572                        if self.pending_whitespace || leading_whitespace {
1573                            //output any pending whitespace from the previous iteration, or leading whitespace from this one
1574                            if !self.text.is_empty()
1575                                && !self.text.chars().rev().next().unwrap().is_whitespace()
1576                            {
1577                                if self.config.debug {
1578                                    eprintln!("[STAM fromxml]{} ^- outputting pending whitespace",self.debugindent);
1579                                }
1580                                self.text.push(' ');
1581                                self.cursor += 1;
1582                                if firsttext && self.pending_whitespace {
1583                                    begin += 1;
1584                                    bytebegin += 1;
1585                                    if self.config.debug {
1586                                        eprintln!("[STAM fromxml]{}  firsttext, begin is now {}, cursor {}",self.debugindent, begin, self.cursor);
1587                                    }
1588                                }
1589                            }
1590                            self.pending_whitespace = false;
1591                        }
1592
1593                        // finally we output the actual text, and advance the cursor
1594                        if whitespace == XmlWhitespaceHandling::Collapse {
1595                            let mut prevc = ' ';
1596                            let mut innertext = innertext.replace(|c: char| c.is_whitespace(), " ");
1597                            innertext.retain(|c| {
1598                                let do_retain = c != ' ' || prevc != ' ';
1599                                prevc = c;
1600                                do_retain
1601                            });
1602                            self.text += &innertext;
1603                            self.cursor += innertext.chars().count();
1604                            if self.config.debug {
1605                                eprintln!("[STAM fromxml]{} ^- outputting text child (collapsed whitespace), cursor is now {}: {}",self.debugindent, self.cursor, innertext);
1606                            }
1607                        } else {
1608                            self.text += &innertext;
1609                            self.cursor += innertext.chars().count();
1610                            if self.config.debug {
1611                                eprintln!("[STAM fromxml]{} ^- outputting text child, cursor is now {}: {}",self.debugindent, self.cursor, innertext);
1612                            }
1613                        }
1614                        firsttext = self.cursor == textbegin;
1615
1616                        //set the pending whitespace buffer (bool) for next iteration
1617                        self.pending_whitespace = pending_whitespace;
1618                    } else if child.is_element() {
1619                        if self.config.debug {
1620                            eprintln!("[STAM fromxml]{} \\- extracting text for this child", self.debugindent);
1621                        }
1622                        self.debugindent.push_str("  ");
1623                        // recursion step, process child element, pass our whitespace handling mode since it may inherit it
1624                        let mut path = path.clone();
1625                        let count = elder_siblings.count(&child);
1626                        path.add(&child, Some(count));
1627                        self.extract_element_text(child, &path, whitespace, resource_id, inputfile, doc_num)?;
1628                        firsttext = self.cursor == textbegin;
1629                        self.debugindent.pop();
1630                        self.debugindent.pop();
1631                    } else {
1632                        if self.config.debug {
1633                            eprintln!("[STAM fromxml]{} ^- skipping this child node", self.debugindent);
1634                        }
1635                        continue;
1636                    }
1637                }
1638
1639                // process the text suffix, a preconfigured string of text to include after to the actual text
1640                self.process_textsuffix(element_config, node, resource_id, inputfile, doc_num, &mut end_discount, &mut end_bytediscount, textbegin)?;
1641
1642                // Assign a scope ID if provided (used for constraining the scope of marker annotations later on)
1643                if let Some(scope_id) = element_config.scope_id.as_ref() {
1644                    self.scopes.insert( scope_id.clone(), (doc_num, node.id()) );
1645                }
1646            } else if element_config.annotation == XmlAnnotationHandling::TextSelectorBetweenMarkers
1647            {
1648                // this is a marker, keep track of it so we can extract the span between markers in [`extract_element_annotation()`] later
1649                if self.config.debug {
1650                    eprintln!("[STAM fromxml]{} adding to markers (textprefix={:?}, textsuffix={:?})", self.debugindent, element_config.textprefix, element_config.textsuffix);
1651                }
1652
1653
1654                self.markers
1655                    .entry(element_config.hash())
1656                    .and_modify(|v| v.push((doc_num, node.id())))
1657                    .or_insert(vec![(doc_num, node.id())]);
1658
1659                // for markers it doesn't matter whether something text is defined as a prefix or suffix, it's functionally the same because a marker has no text itself
1660
1661                self.process_textprefix(element_config, node, resource_id, inputfile, doc_num, &mut begin, &mut bytebegin)?;
1662                self.process_textsuffix(element_config, node, resource_id, inputfile, doc_num, &mut end_discount, &mut end_bytediscount, self.cursor)?;
1663            }
1664        } else if self.config.debug {
1665            eprintln!(
1666                "[STAM fromxml]{} WARNING: no match, skipping text extraction for element {}",
1667                self.debugindent,
1668                path
1669            );
1670        }
1671
1672        // Last, we store the new text offsets for this element/node so
1673        // we can use it in [`extract_element_annotation()`] to associate
1674        // actual annotations with this span.
1675        if begin <= (self.cursor - end_discount) {
1676            let offset = Offset::simple(begin, self.cursor - end_discount);
1677            if self.config.debug {
1678                eprintln!(
1679                    "[STAM fromxml]{} extracted text for {} @{:?}: {:?}",
1680                    self.debugindent,
1681                    path,
1682                    &offset,
1683                    &self.text[bytebegin..(self.text.len() - end_bytediscount)]
1684                );
1685            }
1686            self.positionmap.insert((doc_num, node.id(), PositionType::Body), offset);
1687            self.bytepositionmap
1688                .insert((doc_num, node.id(), PositionType::Body), (bytebegin, self.text.len() - end_bytediscount));
1689        }
1690        Ok(())
1691    }
1692
1693    /// process the text prefix, a text template to include prior to the actual text
1694    fn process_textprefix<'b>(
1695        &mut self,
1696        element_config: &XmlElementConfig,
1697        node: Node<'a,'b>,
1698        resource_id: Option<&str>,
1699        inputfile: Option<&str>,
1700        doc_num: usize,
1701        begin: &mut usize,
1702        bytebegin: &mut usize
1703    ) -> Result<(), XmlConversionError> {
1704        if let Some(textprefix) = &element_config.textprefix {
1705            self.pending_whitespace = false;
1706            if self.config.debug {
1707                eprintln!("[STAM fromxml]{} outputting textprefix: {:?}", self.debugindent, textprefix);
1708            }
1709            let result =
1710                self.render_template(textprefix, &node, Some(self.cursor), None, resource_id, inputfile, doc_num)
1711                    .map_err(|e| match e {
1712                        XmlConversionError::TemplateError(s, e) => {
1713                            XmlConversionError::TemplateError(
1714                                format!(
1715                                "whilst rendering textprefix template '{}' for node '{}': {}",
1716                                textprefix, node.tag_name().name(), s
1717                            ),
1718                                e,
1719                            )
1720                        }
1721                        e => e,
1722                    })?;
1723            let result_charlen = result.chars().count();
1724
1725            if !element_config.annotatetextprefix.is_empty() {
1726                //record the offsets for textprefix annotation later
1727                let offset = Offset::simple(self.cursor, self.cursor + result_charlen);
1728                self.positionmap.insert((doc_num, node.id(), PositionType::TextPrefix), offset);
1729                self.bytepositionmap
1730                    .insert((doc_num, node.id(), PositionType::TextPrefix), (*bytebegin, *bytebegin + result.len()));
1731            }
1732
1733            self.cursor += result_charlen;
1734            self.text += &result;
1735
1736            if element_config.include_textprefix != Some(true) {
1737                // the textprefix will not be part of the annotation's text selection, increment the offsets:
1738                *begin += result_charlen;
1739                *bytebegin += result.len();
1740            }
1741        }
1742        Ok(())
1743    }
1744
1745    /// process the text suffix, a preconfigured string of text to include after to the actual text
1746    fn process_textsuffix<'b>(
1747        &mut self,
1748        element_config: &XmlElementConfig,
1749        node: Node<'a,'b>,
1750        resource_id: Option<&str>,
1751        inputfile: Option<&str>,
1752        doc_num: usize,
1753        end_discount: &mut usize,
1754        end_bytediscount: &mut usize,
1755        textbegin: usize,
1756    ) -> Result<(), XmlConversionError> {
1757        if let Some(textsuffix) = &element_config.textsuffix {
1758            if self.config.debug {
1759                eprintln!("[STAM fromxml]{} outputting textsuffix: {:?}", self.debugindent, textsuffix);
1760            }
1761            let result = self.render_template(
1762                textsuffix.as_str(),
1763                &node,
1764                Some(textbegin),
1765                Some(self.cursor),
1766                resource_id,
1767                inputfile,
1768                doc_num
1769            ).map_err(|e| match e {
1770                    XmlConversionError::TemplateError(s, e) => {
1771                        XmlConversionError::TemplateError(
1772                            format!(
1773                                "whilst rendering textsuffix template '{}' for node '{}': {}",
1774                                textsuffix,
1775                                node.tag_name().name(),
1776                                s
1777                            ),
1778                            e,
1779                        )
1780                    }
1781                    e => e,
1782            })?;
1783            let end_discount_tmp = result.chars().count();
1784            let end_bytediscount_tmp = result.len();
1785
1786
1787            self.text += &result;
1788
1789            if !element_config.annotatetextsuffix.is_empty() {
1790                //record the offsets for textsuffix annotation later
1791                let offset = Offset::simple(self.cursor, self.cursor + end_discount_tmp);
1792                self.positionmap.insert((doc_num, node.id(), PositionType::TextSuffix), offset);
1793                self.bytepositionmap
1794                    .insert((doc_num, node.id(), PositionType::TextSuffix), (self.text.len() - end_bytediscount_tmp, self.text.len()));
1795            }
1796
1797            self.cursor += end_discount_tmp;
1798            self.pending_whitespace = false;
1799
1800            if element_config.include_textsuffix == Some(true) {
1801                // the textsuffix will be part of the annotation's text selection, no discount for later
1802                *end_discount = 0;
1803                *end_bytediscount = 0;
1804            } else {
1805                // the textsuffix will not be part of the annotation's text selection, set discounts for later
1806                *end_discount = end_discount_tmp;
1807                *end_bytediscount = end_bytediscount_tmp;
1808            }
1809        }
1810        Ok(())
1811    }
1812
1813    /// extract annotations from the XML document
1814    /// according to the mapping configuration and creates a STAM TextResource for it.
1815    /// The text, for the full document, must have already been extracted earlier with [`extract_element_text()`].
1816    /// This relies on the exact offsets per element/node computed earlier during text extraction (`positionmap`).
1817    fn extract_element_annotation<'b>(
1818        &mut self,
1819        node: Node<'a,'b>,
1820        path: &NodePath<'a,'b>,
1821        inputfile: Option<&str>,
1822        doc_num: usize,
1823        store: &mut AnnotationStore,
1824    ) -> Result<(), XmlConversionError> {
1825        if self.config.debug {
1826            eprintln!("[STAM fromxml]{} extracting annotation from {}", self.debugindent, path);
1827        }
1828
1829        let mut elder_siblings = SiblingCounter::default();
1830
1831        // obtain the configuration that applies to this element
1832        if let Some(element_config) = self.config.element_config(node, &path) {
1833            if self.config.debug {
1834                eprintln!("[STAM fromxml]{} matching config: {:?}", self.debugindent, element_config);
1835            }
1836            if element_config.annotation != XmlAnnotationHandling::None
1837                && element_config.annotation != XmlAnnotationHandling::Unspecified
1838            {
1839                let mut builder = AnnotationBuilder::new();
1840
1841                //prepare variables to pass to the template context
1842                let offset = self.positionmap.get(&(doc_num, node.id(), PositionType::Body));
1843                if element_config.annotation == XmlAnnotationHandling::TextSelector {
1844                    if let Some((beginbyte, endbyte)) = self.bytepositionmap.get(&(doc_num, node.id(), PositionType::Body)) {
1845                        if self.config.debug {
1846                            eprintln!("[STAM fromxml]{} annotation covers text {:?} (bytes {}-{})", self.debugindent, offset, beginbyte, endbyte);
1847                        }
1848                    }  else if self.text.is_empty() {
1849                        return Err(XmlConversionError::ConfigError("Can't extract annotations on text if no text was extracted!".into()));
1850                    }
1851                }
1852                let begin = if let Some(offset) = offset {
1853                    if let Cursor::BeginAligned(begin) = offset.begin {
1854                        Some(begin)
1855                    } else {
1856                        None
1857                    }
1858                } else {
1859                    None
1860                };
1861                let end = if let Some(offset) = offset {
1862                    if let Cursor::BeginAligned(end) = offset.end {
1863                        Some(end)
1864                    } else {
1865                        None
1866                    }
1867                } else {
1868                    None
1869                };
1870
1871                let resource_id = if let Some(resource_handle) = self.resource_handle {
1872                    store.resource(resource_handle).unwrap().id()
1873                } else {
1874                    None
1875                };
1876
1877                let mut have_id = false;
1878                if let Some(template) = &element_config.id {
1879                    let context = self.context_for_node(&node, begin, end, template.as_str(), resource_id, inputfile, doc_num);
1880                    let compiled_template = self.template_engine.template(template.as_str());
1881                    let id = compiled_template.render(&context).to_string().map_err(|e| 
1882                            XmlConversionError::TemplateError(
1883                                format!(
1884                                    "whilst rendering id template '{}' for node '{}'",
1885                                    template,
1886                                    node.tag_name().name(),
1887                                ),
1888                                Some(e),
1889                            )
1890                        )?;
1891                    if !id.is_empty() {
1892                        builder = builder.with_id(id);
1893                        have_id = true;
1894                    }
1895                }
1896
1897                if !have_id {
1898                    //generate a random ID if we have none
1899                    if let Some(resource_id) = resource_id {
1900                        builder = builder.with_id(stam::generate_id(&format!("{}-",resource_id), ""));
1901                    } else {
1902                        builder = builder.with_id(stam::generate_id("", ""));
1903                    }
1904                }
1905
1906                builder = self.add_annotationdata_to_builder(element_config.annotationdata.iter(), builder, node.clone(), begin, end, resource_id, inputfile, doc_num)?;
1907
1908
1909                if self.config.provenance  && inputfile.is_some() {
1910                    let path_string = if let Some(id) = node.attribute((NS_XML,"id")) {
1911                        //node has an ID, use that
1912                        format!("//{}[@xml:id=\"{}\"]", self.get_node_name_for_xpath(&node), id)
1913                    } else {
1914                        //no ID, use full XPath expression
1915                        path.format_as_xpath(&self.prefixes)
1916                    };
1917                    let databuilder = AnnotationDataBuilder::new().with_dataset(CONTEXT_ANNO.into()).with_key("target".into()).with_value(
1918                        BTreeMap::from([
1919                            ("source".to_string(),inputfile.unwrap().into()),
1920                            ("selector".to_string(),
1921                                    BTreeMap::from([
1922                                        ("type".to_string(),"XPathSelector".into()),
1923                                        ("value".to_string(),path_string.into())
1924                                    ]).into()
1925                            )
1926                        ]).into()
1927                    );
1928                    builder = builder.with_data_builder(databuilder);
1929                }
1930
1931
1932                // Finish the builder and add the actual annotation to the store, according to its element handling
1933                match element_config.annotation {
1934                    XmlAnnotationHandling::TextSelector => {
1935                        // Annotation is on text, translates to TextSelector
1936                        if let Some(selector) = self.textselector(node, doc_num, PositionType::Body) {
1937                            builder = builder.with_target(selector);
1938                            if self.config.debug {
1939                                eprintln!("[STAM fromxml]   builder AnnotateText: {:?}", builder);
1940                            }
1941                            store.annotate(builder)?;
1942                        }
1943                        if !element_config.annotatetextprefix.is_empty() || !element_config.annotatetextsuffix.is_empty() {
1944                            self.annotate_textaffixes(node, element_config, inputfile, doc_num, store)?;
1945                        }
1946                    }
1947                    XmlAnnotationHandling::ResourceSelector => {
1948                        // Annotation is metadata, translates to ResourceSelector
1949                        builder = builder.with_target(SelectorBuilder::ResourceSelector(
1950                            self.resource_handle.into(),
1951                        ));
1952                        if self.config.debug {
1953                            eprintln!("[STAM fromxml]   builder AnnotateResource: {:?}", builder);
1954                        }
1955                        store.annotate(builder)?;
1956                    }
1957                    XmlAnnotationHandling::TextSelectorBetweenMarkers => {
1958                        // Annotation is on a text span *between* two marker elements
1959                        if let Some(selector) =
1960                            self.textselector_for_markers(node, doc_num, store, element_config)
1961                        {
1962                            builder = builder.with_target(selector);
1963                            if self.config.debug {
1964                                eprintln!(
1965                                    "[STAM fromxml]   builder TextSelectorBetweenMarkers: {:?}",
1966                                    builder
1967                                );
1968                            }
1969                            store.annotate(builder)?;
1970                            if !element_config.annotatetextprefix.is_empty() || !element_config.annotatetextsuffix.is_empty() {
1971                                self.annotate_textaffixes(node, element_config, inputfile, doc_num, store)?;
1972                            }
1973                        }
1974                    }
1975                    _ => panic!(
1976                        "Invalid annotationhandling: {:?}",
1977                        element_config.annotation
1978                    ),
1979                }
1980            }
1981
1982            // Recursion step
1983            if element_config.stop == Some(false) || element_config.stop.is_none() {
1984                for child in node.children() {
1985                    if child.is_element() {
1986                        self.debugindent.push_str("  ");
1987                        let mut path = path.clone();
1988                        let count = elder_siblings.count(&child);
1989                        path.add(&child, Some(count));
1990                        //eprintln!("DEBUG: count={}, child={:?}, parent={:?}, elder_siblings={:?}", count, child.tag_name(), node.tag_name(), elder_siblings);
1991                        self.extract_element_annotation(child, &path, inputfile, doc_num, store)?;
1992                        self.debugindent.pop();
1993                        self.debugindent.pop();
1994                    }
1995                }
1996            }
1997        } else {
1998            eprintln!(
1999                "[STAM fromxml]{} WARNING: no match, skipping annotation extraction for element {}",
2000                self.debugindent,
2001                path
2002            );
2003        }
2004        Ok(())
2005    }
2006
2007    fn add_annotationdata_to_builder<'input>(&self, iter: impl Iterator<Item = &'a XmlAnnotationDataConfig>,
2008        mut builder: AnnotationBuilder<'a>,
2009        node: Node<'a, 'input>,
2010        begin: Option<usize>,
2011        end: Option<usize>,
2012        resource_id: Option<&str>,
2013        inputfile: Option<&str>,
2014        doc_num: usize,
2015    ) -> Result<AnnotationBuilder<'a>, XmlConversionError> {
2016        for annotationdata in iter {
2017            let mut databuilder = AnnotationDataBuilder::new();
2018            if let Some(template) = &annotationdata.set {
2019                let context = self.context_for_node(&node, begin, end, template.as_str(), resource_id, inputfile, doc_num);
2020                let compiled_template = self.template_engine.template(template.as_str());
2021                let dataset = compiled_template.render(&context).to_string().map_err(|e|
2022                        XmlConversionError::TemplateError(
2023                            format!(
2024                                "whilst rendering annotationdata/dataset template '{}' for node '{}'",
2025                                template,
2026                                node.tag_name().name(),
2027                            ),
2028                            Some(e),
2029                        )
2030                    )?;
2031                if !dataset.is_empty() {
2032                    databuilder = databuilder.with_dataset(dataset.into())
2033                }
2034            } else {
2035                databuilder =
2036                    databuilder.with_dataset(self.config.default_set.as_str().into());
2037            }
2038            if let Some(template) = &annotationdata.key {
2039                let context = self.context_for_node(&node, begin, end, template.as_str(), resource_id, inputfile, doc_num);
2040                let compiled_template = self.template_engine.template(template.as_str());
2041                match compiled_template.render(&context).to_string().map_err(|e| 
2042                        XmlConversionError::TemplateError(
2043                            format!(
2044                                "whilst rendering annotationdata/key template '{}' for node '{}'",
2045                                template,
2046                                node.tag_name().name(),
2047                            ),
2048                            Some(e),
2049                        )
2050                    )  {
2051                    Ok(key) if !key.is_empty() =>
2052                        databuilder = databuilder.with_key(key.into()) ,
2053                    Ok(_) if !annotationdata.skip_if_missing => {
2054                        return Err(XmlConversionError::TemplateError(
2055                            format!(
2056                                "whilst rendering annotationdata/key template '{}' for node '{}'",
2057                                template,
2058                                node.tag_name().name(),
2059                            ),
2060                            None
2061                        ));
2062                    },
2063                    Err(e) if !annotationdata.skip_if_missing => {
2064                        return Err(e)
2065                    },
2066                    _ => {
2067                        //skip whole databuilder if missing
2068                        continue
2069                    }
2070                }
2071            }
2072            if let Some(value) = &annotationdata.value {
2073                match self.extract_value(value,  node, annotationdata.allow_empty_value, annotationdata.skip_if_missing, annotationdata.valuetype.as_ref().map(|s| s.as_str()), begin, end, resource_id, inputfile, doc_num)? {
2074                    Some(DataValue::List(values)) if annotationdata.multiple => {
2075                        for value in values {
2076                            let mut databuilder_multi = databuilder.clone();
2077                            databuilder_multi = databuilder_multi.with_value(value);
2078                            builder = builder.with_data_builder(databuilder_multi);
2079                        }
2080                    },
2081                    Some(value) => {
2082                        databuilder = databuilder.with_value(value);
2083                    },
2084                    None =>  {
2085                        //skip whole databuilder if missing
2086                        continue
2087                    }
2088                }
2089            }
2090            if !annotationdata.multiple {
2091                builder = builder.with_data_builder(databuilder);
2092            }
2093        }
2094        Ok(builder)
2095    }
2096
2097    /// Annotates textprefix and textsuffix, if applicable
2098    fn annotate_textaffixes<'b>(
2099        &mut self,
2100        node: Node<'a,'b>,
2101        element_config: &XmlElementConfig,
2102        inputfile: Option<&str>,
2103        doc_num: usize,
2104        store: &mut AnnotationStore,
2105    ) -> Result<(), XmlConversionError> {
2106
2107
2108        if !element_config.annotatetextprefix.is_empty() {
2109            let mut builder = AnnotationBuilder::new().with_id(stam::generate_id("textprefix-", ""));
2110            if let Some(offset) = self.positionmap.get(&(doc_num, node.id(), PositionType::TextPrefix)) {
2111                let begin = if let Cursor::BeginAligned(begin) = offset.begin {
2112                        Some(begin)
2113                    } else {
2114                        None
2115                    };
2116                let end = if let Cursor::BeginAligned(end) = offset.end {
2117                        Some(end)
2118                    } else {
2119                        None
2120                    };
2121                builder = self.add_annotationdata_to_builder(element_config.annotatetextprefix.iter(), builder, node.clone(), begin,end, None, inputfile, doc_num)?; //MAYBE TODO: pass resource_id
2122                if let Some(selector) = self.textselector(node, doc_num, PositionType::TextPrefix) {
2123                    builder = builder.with_target(selector);
2124                    if self.config.debug {
2125                        eprintln!("[STAM fromxml]   builder AnnotateText: {:?}", builder);
2126                    }
2127                    store.annotate(builder)?;
2128                } else {
2129                    return Err(XmlConversionError::ConfigError("Failed to create textselector to target textprefix".into()));
2130                }
2131            }
2132        }
2133
2134        if !element_config.annotatetextsuffix.is_empty() {
2135            let mut builder = AnnotationBuilder::new().with_id(stam::generate_id("textsuffix-", ""));
2136            if let Some(offset) = self.positionmap.get(&(doc_num, node.id(), PositionType::TextSuffix)) {
2137                let begin = if let Cursor::BeginAligned(begin) = offset.begin {
2138                        Some(begin)
2139                    } else {
2140                        None
2141                    };
2142                let end = if let Cursor::BeginAligned(end) = offset.end {
2143                        Some(end)
2144                    } else {
2145                        None
2146                    };
2147                builder = self.add_annotationdata_to_builder(element_config.annotatetextsuffix.iter(), builder, node.clone(), begin,end, None, inputfile, doc_num)?; //MAYBE TODO: pass resource_id
2148                if let Some(selector) = self.textselector(node, doc_num, PositionType::TextSuffix) {
2149                    builder = builder.with_target(selector);
2150                    if self.config.debug {
2151                        eprintln!("[STAM fromxml]   builder AnnotateText: {:?}", builder);
2152                    }
2153                    store.annotate(builder)?;
2154                } else {
2155                    return Err(XmlConversionError::ConfigError("Failed to create textselector to target textprefix".into()));
2156                }
2157            }
2158        }
2159        Ok(())
2160    }
2161
2162    /// Extract values, running the templating engine in case of string values
2163    fn extract_value<'b>(&self, value: &'a toml::Value, node: Node<'a,'b>, allow_empty_value: bool, skip_if_missing: bool, valuetype: Option<&str>, begin: Option<usize>, end: Option<usize>, resource_id: Option<&str>, inputfile: Option<&str>, doc_num: usize) -> Result<Option<DataValue>, XmlConversionError>{
2164        match value {
2165            toml::Value::String(template) => {
2166                let context = self.context_for_node(&node, begin, end, template.as_str(), resource_id, inputfile, doc_num);
2167                /*
2168                if self.config.debug() {
2169                    eprintln!(
2170                        "[STAM fromxml]              Context for annotationdata/map template '{}' for node '{}': {:?}",
2171                        template,
2172                        node.tag_name().name(),
2173                        context
2174                    );
2175                }
2176                */
2177                let compiled_template = self.template_engine.template(template.as_str()); //panics if doesn't exist, but that can't happen
2178                match compiled_template.render(&context).to_string().map_err(|e|
2179                        XmlConversionError::TemplateError(
2180                            format!(
2181                                "whilst rendering annotationdata/map template '{}' for node '{}'.{}",
2182                                template,
2183                                node.tag_name().name(),
2184                                if self.config.debug() {
2185                                    format!("\nContext was {:?}.\nVariables are: {:?}", context, self.variables.get(template))
2186                                } else {
2187                                    String::new()
2188                                }
2189                            ),
2190                            Some(e),
2191                        )
2192                    )  {
2193                    Ok(value) => {
2194                        if !value.is_empty() || allow_empty_value {
2195                            string_to_datavalue(value, valuetype).map(|v| Some(v))
2196                        } else {
2197                            //skip
2198                            Ok(None)
2199                        }
2200                    },
2201                    Err(e) if !skip_if_missing => {
2202                        Err(e)
2203                    },
2204                    Err(_) if allow_empty_value => {
2205                        Ok(Some("".into()))
2206                    },
2207                    Err(_) => {
2208                        //skip whole databuilder if missing
2209                        Ok(None)
2210                    }
2211                }
2212            },
2213            toml::Value::Table(map) => {
2214                let mut resultmap: BTreeMap<String,DataValue> = BTreeMap::new();
2215                for (key, value) in map.iter() {
2216                    if let Some(value) = self.extract_value(value,  node, false, true, None, begin, end, resource_id, inputfile, doc_num)? {
2217                        resultmap.insert(key.clone(), value);
2218                    }
2219                }
2220                Ok(Some(resultmap.into()))
2221            },
2222            toml::Value::Array(list) => {
2223                let mut resultlist: Vec<DataValue> = Vec::new();
2224                for value in list.iter() {
2225                    if let Some(value) = self.extract_value(value, node, false, true, None,  begin, end, resource_id, inputfile, doc_num)? {
2226                        resultlist.push(value);
2227                    }
2228                }
2229                Ok(Some(resultlist.into()))
2230            }
2231            toml::Value::Boolean(v) => Ok(Some(DataValue::Bool(*v))),
2232            toml::Value::Float(v) => Ok(Some(DataValue::Float(*v))),
2233            toml::Value::Integer(v) => Ok(Some(DataValue::Int(*v as isize))),
2234            toml::Value::Datetime(_v) => {
2235                todo!("fromxml: Datetime conversion not implemented yet");
2236            }
2237        }
2238    }
2239
2240    /// Extract values for metadata (no associated node), running the templating engine in case of string values
2241    fn extract_value_metadata<'b>(&self, value: &'a toml::Value, context: &upon::Value, allow_empty_value: bool, skip_if_missing: bool, resource_id: Option<&str>) -> Result<Option<DataValue>, XmlConversionError>{
2242        match value {
2243            toml::Value::String(template) => {
2244                let compiled_template = self.template_engine.template(template.as_str()); //panics if doesn't exist, but that can't happen
2245                match compiled_template.render(&context).to_string().map_err(|e|
2246                        XmlConversionError::TemplateError(
2247                            format!(
2248                                "whilst rendering annotationdata/metadata template '{}' for metadata",
2249                                template,
2250                            ),
2251                            Some(e),
2252                        )
2253                    )  {
2254                    Ok(value) => {
2255                        if !value.is_empty() || allow_empty_value {
2256                            Ok(Some(value.into()))
2257                        } else {
2258                            //skip
2259                            Ok(None)
2260                        }
2261                    },
2262                    Err(e) if !skip_if_missing => {
2263                        Err(e)
2264                    },
2265                    Err(_) if allow_empty_value => {
2266                        Ok(Some("".into()))
2267                    },
2268                    Err(_) => {
2269                        //skip whole databuilder if missing
2270                        Ok(None)
2271                    }
2272                }
2273            },
2274            toml::Value::Table(map) => {
2275                let mut resultmap: BTreeMap<String,DataValue> = BTreeMap::new();
2276                for (key, value) in map.iter() {
2277                    if let Some(value) = self.extract_value_metadata(value, context, false, true,  resource_id)? {
2278                        resultmap.insert(key.clone(), value);
2279                    }
2280                }
2281                Ok(Some(resultmap.into()))
2282            },
2283            toml::Value::Array(list) => {
2284                let mut resultlist: Vec<DataValue> = Vec::new();
2285                for value in list.iter() {
2286                    if let Some(value) = self.extract_value_metadata(value, context, false, true, resource_id)? {
2287                        resultlist.push(value);
2288                    }
2289                }
2290                Ok(Some(resultlist.into()))
2291            }
2292            toml::Value::Boolean(v) => Ok(Some(DataValue::Bool(*v))),
2293            toml::Value::Float(v) => Ok(Some(DataValue::Float(*v))),
2294            toml::Value::Integer(v) => Ok(Some(DataValue::Int(*v as isize))),
2295            toml::Value::Datetime(_v) => {
2296                todo!("fromxml: Datetime conversion not implemented yet");
2297            }
2298        }
2299    }
2300
2301    /// Select text corresponding to the element/node and document number
2302    fn textselector<'s>(&'s self, node: Node, doc_num: usize, positiontype: PositionType) -> Option<SelectorBuilder<'s>> {
2303        let res_handle = self.resource_handle.expect("resource must be associated");
2304        if let Some(offset) = self.positionmap.get(&(doc_num, node.id(), positiontype)) {
2305            Some(SelectorBuilder::TextSelector(
2306                BuildItem::Handle(res_handle),
2307                offset.clone(),
2308            ))
2309        } else {
2310            None
2311        }
2312    }
2313
2314    /// Select text between this element/node and the next of the same type
2315    fn textselector_for_markers<'b>(
2316        &self,
2317        node: Node,
2318        doc_num: usize,
2319        store: &AnnotationStore,
2320        element_config: &'b XmlElementConfig,
2321    ) -> Option<SelectorBuilder<'b>> {
2322        let resource = store
2323            .resource(
2324                self.resource_handle
2325                    .expect("resource must have been created"),
2326            )
2327            .expect("resource must exist");
2328        let mut end: Option<usize> = None;
2329        if let Some(markers) = self.markers.get(&element_config.hash()) {
2330            let mut grab = false;
2331            for (d_num, n_id) in markers.iter() {
2332                if grab {
2333                    //this marker is the next one, it's begin position is our desired end position
2334                    end = self.positionmap.get(&(*d_num, *n_id, PositionType::Body)).map(|offset| {
2335                        offset
2336                            .begin
2337                            .try_into()
2338                            .expect("begin cursor must be beginaligned")
2339                    });
2340                    break;
2341                }
2342                if doc_num == *d_num && *n_id == node.id() {
2343                    //current node/marker found, signal grab for the next one
2344                    grab = true;
2345                }
2346            }
2347        };
2348        if end.is_none() {
2349            //no next marker found, find the end
2350            //are we in a restricted scope?
2351            if let Some(scope) = element_config.marker_scope.as_deref() {
2352                if let Some((d_num, n_id)) = self.scopes.get(scope) {
2353                    end = self.positionmap.get(&(*d_num, *n_id, PositionType::Body)).map(|offset| {
2354                        offset
2355                            .end
2356                            .try_into()
2357                            .expect("end cursor must be beginaligned")
2358                    });
2359                } else {
2360                    eprintln!("WARNING: Undefined scope referenced in marker_scope: {}, no matching text with this `scope_id` in this document! Skipping last marker!", scope);
2361                    return None;
2362                }
2363            } else {
2364                //just use end of document instead
2365                end = Some(resource.textlen());
2366            }
2367        }
2368        if let (Some(offset), Some(end)) = (self.positionmap.get(&(doc_num, node.id(), PositionType::Body)), end) {
2369            Some(SelectorBuilder::TextSelector(
2370                BuildItem::Handle(self.resource_handle.unwrap()),
2371                Offset::simple(
2372                    offset
2373                        .begin
2374                        .try_into()
2375                        .expect("begin cursor must be beginaligned"),
2376                    end,
2377                ),
2378            ))
2379        } else {
2380            None
2381        }
2382    }
2383
2384    fn set_global_context(&mut self) {
2385        self.global_context
2386            .insert("context".into(), upon::Value::Map(self.config.context.iter().map(|(k,v)| (k.clone(), map_value(v))).collect()));
2387        self.global_context
2388            .insert("namespaces".into(), self.config.namespaces.clone().into());
2389        self.global_context
2390            .insert("default_set".into(), self.config.default_set.clone().into());
2391    }
2392
2393    fn render_template<'input, 't>(
2394        &self,
2395        template: &'t str,
2396        node: &Node<'a, 'input>,
2397        begin: Option<usize>,
2398        end: Option<usize>,
2399        resource: Option<&str>,
2400        inputfile: Option<&str>,
2401        doc_num: usize,
2402    ) -> Result<Cow<'t, str>, XmlConversionError> {
2403        if template.chars().any(|c| c == '{') {
2404            //value is a template, templating engine probably needed
2405            let compiled_template = self.template_engine.template(template);
2406            let context = self.context_for_node(&node, begin, end, template, resource, inputfile, doc_num);
2407            let result = compiled_template.render(context).to_string()?;
2408            Ok(Cow::Owned(result))
2409        } else {
2410            //value is a literal: templating engine not needed
2411            Ok(Cow::Borrowed(template))
2412        }
2413    }
2414
2415    fn context_for_node<'input>(
2416        &self,
2417        node: &Node<'a, 'input>,
2418        begin: Option<usize>,
2419        end: Option<usize>,
2420        template: &str,
2421        resource: Option<&str>,
2422        inputfile: Option<&str>,
2423        doc_num: usize,
2424    ) -> upon::Value {
2425        let mut context = self.global_context.clone();
2426        let length = if let (Some(begin), Some(end)) = (begin, end) {
2427            Some(end - begin)
2428        } else {
2429            None
2430        };
2431        context.insert("localname".into(), node.tag_name().name().into());
2432        //name with name prefix (if any)
2433        context.insert("name".into(), self.get_node_name_for_template(node).into());
2434        if let Some(namespace) = node.tag_name().namespace() {
2435            //the full namespace
2436            context.insert("namespace".into(), namespace.into());
2437        }
2438
2439        // Offset in the untangled plain text
2440        if let Some(begin) = begin {
2441            context.insert("begin".into(), upon::Value::Integer(begin as i64));
2442        }
2443        if let Some(end) = end {
2444            context.insert("end".into(), upon::Value::Integer(end as i64));
2445        }
2446        if let Some(length) = length {
2447            context.insert("length".into(), upon::Value::Integer(length as i64));
2448        }
2449        if let Some(resource) = resource {
2450            //the resource ID
2451            context.insert("resource".into(), resource.into());
2452        }
2453        if let Some(inputfile) = inputfile {
2454            //the input file
2455            context.insert("inputfile".into(), inputfile.into());
2456        }
2457        //document number (0-indexed), useful in case multiple input documents are cast to a single output text
2458        context.insert("doc_num".into(), upon::Value::Integer(doc_num as i64));
2459
2460        if let Some(vars) = self.variables.get(template) {
2461            for var in vars {
2462                let mut encodedvar = String::new();
2463                if let Some(value) = self.context_for_var(node, var, &mut encodedvar, false) {
2464                    if self.config.debug() {
2465                        eprintln!(
2466                            "[STAM fromxml]              Set context variable for template '{}' for node '{}': {}={:?}   (encodedvar={})",
2467                            template,
2468                            node.tag_name().name(),
2469                            var,
2470                            value,
2471                            encodedvar
2472                        );
2473                    }
2474                    if value != upon::Value::None {
2475                        context.insert(encodedvar, value);
2476                    }
2477                } else if self.config.debug() {
2478                    eprintln!(
2479                        "[STAM fromxml]              Missed context variable for template '{}' for node '{}': {}",
2480                        template,
2481                        node.tag_name().name(),
2482                        var
2483                    );
2484                }
2485            }
2486        }
2487        upon::Value::Map(context)
2488    }
2489
2490    /// Looks up a variable value (from the DOM XML) to be used in for template context
2491    // returns value and stores full the *encoded* variable name in path (this is safe to pass to template)
2492    // return values are temporarily aggregated in multiple if multiple elements are requested, it will be emptied automatically, the caller owns it but doesn't use it itself.
2493    fn context_for_var<'input>(
2494        &self,
2495        node: &Node<'a, 'input>,
2496        var: &str,
2497        path: &mut String,
2498        mut return_all_matches: bool,
2499    ) -> Option<upon::Value> {
2500
2501        //are we the first call by the caller or are we a recursion?
2502        let first = path.is_empty();
2503
2504        let var = if var.starts_with("?.$$") {
2505            if first {
2506                path.push_str("?.ELEMENTS_");
2507                return_all_matches = true;
2508                if self.config.debug {
2509                    eprintln!("[STAM fromxml]              will return all matches for {}", var);
2510                }
2511            };
2512            &var[4..]
2513        } else if var.starts_with("?.$") {
2514            if first {
2515                path.push_str("?.ELEMENT_");
2516            };
2517            &var[3..]
2518        } else if var.starts_with("$$") {
2519            if first {
2520                path.push_str("ELEMENTS_");
2521                return_all_matches = true;
2522                if self.config.debug {
2523                    eprintln!("[STAM fromxml]              will return all matches for {}", var);
2524                }
2525            };
2526            &var[2..]
2527        } else if var.starts_with("$") {
2528            if first {
2529                path.push_str("ELEMENT_");
2530            };
2531            &var[1..]
2532        } else if var.starts_with("?.@") {
2533            if first {
2534                path.push_str("?.");
2535            };
2536            &var[2..]
2537        } else {
2538            var
2539        };
2540
2541        if !first && !var.is_empty() && !path.ends_with("ELEMENT_") && !path.ends_with("ELEMENTS_"){
2542            path.push_str("_IN_");
2543        }
2544
2545        //get the first component of the variable
2546        let (component, remainder) = var.split_once("/").unwrap_or((var,""));
2547        //eprintln!("DEBUG: component={}, remainder={}, node={}, return_all_matches={}", component, remainder, node.tag_name().name(), return_all_matches);
2548        if component.is_empty() {
2549            if first && !remainder.is_empty() {
2550                //we're asked to start at the root node
2551                let mut n = node.clone();
2552                //find the root node
2553                while let Some(parentnode) = n.parent_element() {
2554                    n = parentnode;
2555                }
2556                //recurse from root node
2557                let (rootcomponent, remainder) = remainder.split_once("/").unwrap_or((remainder,""));
2558                let (prefix, localname)  = if let Some(pos) = rootcomponent.find(":") {
2559                    (Some(&rootcomponent[0..pos]),  &rootcomponent[pos+1..])
2560                } else {
2561                    (None, rootcomponent)
2562                };
2563                //test if root name corresponds with what we expected
2564                if localname != n.tag_name().name() && localname != "*" {
2565                    None
2566                } else {
2567                    if let Some(prefix) = prefix {
2568                        path.push_str(prefix);
2569                        path.push_str("__");
2570                    }
2571                    path.push_str(localname);
2572                    self.context_for_var(&n, remainder, path, return_all_matches)
2573                }
2574            } else {
2575                //an empty component is the stop condition , this function is called recursively, stripping one
2576                //component at a time until nothing is left, we then take the text of that final node:
2577                Some(recursive_text(node).into())
2578            }
2579        } else if component.starts_with("@"){
2580            if let Some(pos) = component.find(":") {
2581                let prefix = &component[1..pos];
2582                if let Some(ns) = self.config.namespaces.get(prefix) {
2583                    let var = &component[pos+1..];
2584                    path.push_str("ATTRIB_");
2585                    path.push_str(prefix);
2586                    path.push_str("__");
2587                    path.push_str(var);
2588                    Some(
2589                        node.attribute((ns.as_str(),var)).into()
2590                    )
2591                } else {
2592                    None
2593                }
2594            } else {
2595                let var = &component[1..];
2596                path.push_str("ATTRIB_");
2597                path.push_str(var);
2598                Some(
2599                    node.attribute(var).into()
2600                )
2601            }
2602        } else if component == ".." {
2603            if let Some(parentnode) = node.parent_element().as_ref() {
2604                //recurse with parent node
2605                path.push_str("PARENT");
2606                self.context_for_var(parentnode, remainder, path, return_all_matches)
2607            } else {
2608                None
2609            }
2610        } else if component == "." {
2611            path.push_str("THIS");
2612            if !remainder.is_empty() {
2613                //a . is meaningless if not the final component
2614                self.context_for_var(node, remainder, path, return_all_matches)
2615            } else {
2616                Some(recursive_text(node).into())
2617            }
2618        } else {
2619            let (prefix, localname)  = if let Some(pos) = component.find(":") {
2620                (Some(&component[0..pos]),  &component[pos+1..])
2621            } else {
2622                (None, component)
2623            };
2624            let localname_with_condition = localname;
2625            let (localname, condition_str, condition) = self.extract_condition(localname_with_condition); //extract X-Path like conditions [@attrib="value"]  (very limited!)
2626            //eprintln!("DEBUG: looking for {} (prefix={:?},localname={}, condition={:?}) in {:?}", localname_with_condition,  prefix, localname, condition, node.tag_name());
2627            let mut multiple_value_buffer: Vec<upon::Value> = Vec::new(); //only used when multiple == true
2628            let mut final_path: String = String::new(); //only used when multiple == true
2629            for child in node.children() {
2630                if child.is_element() {
2631                    let namedata = child.tag_name();
2632                    let mut child_matches = if let Some(namespace) = namedata.namespace() {
2633                        if let Some(foundprefix) = self.prefixes.get(namespace) {
2634                            Some(foundprefix.as_str()) == prefix && localname == namedata.name()
2635                        } else {
2636                            false
2637                        }
2638                    } else {
2639                        namedata.name() == localname
2640                    };
2641                    if child_matches {
2642                        //MAYBE TODO: move to separate funtion
2643                        if let Some((attribname, negate, attribvalue)) = condition {
2644                            //test condition: falsify child_matches
2645                            if let Some(pos) = attribname.find(":") {
2646                                let prefix = &attribname[0..pos];
2647                                if let Some(ns) = self.config.namespaces.get(prefix) {
2648                                    let attribname = &attribname[pos+1..];
2649                                    if let Some(value) = child.attribute((ns.as_str(),attribname)) {
2650                                        if !negate && attribvalue != Some(value) {
2651                                            child_matches = false;
2652                                        } else if negate && attribvalue == Some(value) {
2653                                            child_matches = false;
2654                                        }
2655                                    } else {
2656                                        child_matches = false;
2657                                    }
2658                                } else {
2659                                    child_matches = false;
2660                                }
2661                            } else {
2662                                if let Some(value) = child.attribute(attribname) {
2663                                    if !negate && attribvalue != Some(value) {
2664                                        child_matches = false;
2665                                    } else if negate && attribvalue == Some(value) {
2666                                        child_matches = false;
2667                                    }
2668                                } else {
2669                                    child_matches = false;
2670                                }
2671                            }
2672                        }
2673                        if !child_matches && self.config.debug {
2674                            eprintln!("[STAM fromxml] candidate node does not meet condition: {}", localname_with_condition);
2675                        }
2676                        //end condition test
2677                    }
2678                    if child_matches {
2679                        let prevpathlen = path.len();
2680                        //update path
2681                        if let Some(prefix) = prefix {
2682                            path.push_str(prefix);
2683                            path.push_str("__");
2684                        }
2685                        path.push_str(localname);
2686                        if condition.is_some() {
2687                            //simply encode the condition as a hash (non-decodable but that's okay)
2688                            let mut hasher = DefaultHasher::new();
2689                            condition_str.hash(&mut hasher);
2690                            let h = hasher.finish();
2691                            path.push_str(&format!("_COND{}_", h));
2692                        }
2693                        if let Some(value) = self.context_for_var(&child, remainder, path, return_all_matches) {
2694                            //success
2695                            if return_all_matches {
2696                                if let upon::Value::List(v) = value {
2697                                    multiple_value_buffer.extend(v.into_iter());
2698                                } else {
2699                                    multiple_value_buffer.push(value);
2700                                }
2701                                if final_path.is_empty() {
2702                                    final_path = path.clone();
2703                                }
2704                                //do not return yet, there may be more!
2705                            } else {
2706                                //normal behaviour, get first match
2707                                return Some(value);
2708                            }
2709                        }
2710                        //child didn't match (or we want multiple matches), truncate path again and continue search (a later child may match again!)
2711                        path.truncate(prevpathlen);
2712                    }
2713                }
2714            }
2715            if !multiple_value_buffer.is_empty() {
2716                //we found multiple values, return them
2717                if self.config.debug {
2718                    eprintln!("[STAM fromxml]              returning multiple matches of {} as list", var);
2719                }
2720                //we also return the path of the match
2721                *path = final_path;
2722                Some(multiple_value_buffer.into())
2723            } else {
2724                //no match found for this variable
2725                if self.config.debug {
2726                    eprintln!("[STAM fromxml]              returning with no match found for {} in {}", var, node.tag_name().name());
2727                }
2728                None
2729            }
2730        }
2731    }
2732
2733    fn extract_condition<'b>(&self, localname: &'b str) -> (&'b str, &'b str, Option<(&'b str, bool, Option<&'b str>)>) { //(localname, condition, Option<(attrib, negation, attribvalue)>)
2734        //simple conditional statement
2735        if localname.ends_with("]") {
2736            if let Some(pos) = localname.find("[") {
2737                let condition = &localname[pos+1..localname.len()-1];
2738                let (mut attrib, negation, attribvalue) = if let Some(pos) = condition.find("=") {
2739                     let attrib = condition[0..pos].trim();
2740                     let value = condition[pos+1..].trim();
2741                     let value = &value[1..value.len() - 1]; //strips the literal quotes (") for the value
2742                     if attrib.ends_with('!') {
2743                        //negation (!= operator)
2744                        (attrib[..attrib.len() - 1].trim(), true, Some(value))
2745                     } else {
2746                        (attrib.trim(), false, Some(value))
2747                     }
2748                } else {
2749                    (condition, false, None)
2750                };
2751                if attrib.starts_with('@') {
2752                    //this should actually be mandatory and already checked during template precompilation
2753                    attrib = &attrib[1..];
2754                }
2755                return (&localname[..pos], condition, Some((attrib,  negation,attribvalue )) );
2756            }
2757        }
2758        (localname, "", None)
2759    }
2760
2761
2762    fn get_node_name_for_template<'b>(&self, node: &'b Node) -> Cow<'b,str> {
2763        let extended_name = node.tag_name();
2764        match (extended_name.namespace(), extended_name.name()) {
2765            (Some(namespace), tagname) => {
2766                if let Some(prefix) = self.prefixes.get(namespace) {
2767                    Cow::Owned(format!("{}__{}", prefix, tagname))
2768                } else {
2769                    Cow::Borrowed(tagname)
2770                }
2771            }
2772            (None, tagname) => Cow::Borrowed(tagname),
2773        }
2774    }
2775
2776    fn get_node_name_for_xpath<'b>(&self, node: &'b Node) -> Cow<'b,str> {
2777        let extended_name = node.tag_name();
2778        match (extended_name.namespace(), extended_name.name()) {
2779            (Some(namespace), tagname) => {
2780                if let Some(prefix) = self.prefixes.get(namespace) {
2781                    Cow::Owned(format!("{}:{}", prefix, tagname))
2782                } else {
2783                    Cow::Borrowed(tagname)
2784                }
2785            }
2786            (None, tagname) => Cow::Borrowed(tagname),
2787        }
2788    }
2789
2790
2791    fn precompile(&mut self, template: &'a str) -> Cow<'a,str> {
2792        let mut replacement = String::new();
2793        let mut variables: BTreeSet<&'a str> = BTreeSet::new();
2794        let mut begin = 0;
2795        let mut end = 0;
2796        for i  in 0..template.len() {
2797            let slice = &template[i..];
2798            if slice.starts_with("{{") || slice.starts_with("{%") {
2799                begin = i;
2800            } else if slice.starts_with("}}") || slice.starts_with("%}") {
2801                if end < begin+2 {
2802                    replacement.push_str(&template[end..begin+2]);
2803                }
2804                let inner = &template[begin+2..i]; //the part without the {{  }}
2805                replacement.push_str(&self.precompile_inblock(inner, &mut variables));
2806                end = i;
2807            }
2808        }
2809        if end > 0 {
2810            replacement.push_str(&template[end..]);
2811        }
2812        self.variables.insert(template.into(), variables);
2813        //eprintln!("DEBUG: precompile({}) -> {}", template, replacement);
2814
2815        if !replacement.is_empty() {
2816            Cow::Owned(replacement)
2817        } else {
2818            Cow::Borrowed(template)
2819        }
2820    }
2821
2822    fn precompile_inblock<'s>(&self, s: &'s str, vars: &mut BTreeSet<&'s str>) -> Cow<'s,str> {
2823        let mut quoted = false;
2824        let mut var = false;
2825        let mut begin = 0;
2826        let mut end = 0;
2827        let mut replacement = String::new();
2828        let mut in_condition = false;
2829        for (i,c) in s.char_indices() {
2830            if in_condition && c != ']' {
2831                continue;
2832            }
2833            if c == '"' {
2834                quoted = !quoted;
2835            } else if !quoted {
2836                if !var && (c == '@' || c == '$') {
2837                    //token is an XML variable name, its syntax needs some changes before it can be used in the templating engine
2838                    var = true;
2839                    begin = i;
2840                } else if var && c == '[' {
2841                    in_condition = true;
2842                } else if var && in_condition && c == ']' {
2843                    //end of condition
2844                    in_condition = false;
2845                } else if var && in_condition  {
2846                    //in condition
2847                    continue;
2848                } else if var && (!c.is_alphanumeric() && c != '$' && c != '.' && c != '/' && c != '_' && c != ':' && c != '@') {
2849                    //end of variable (including condition if applicable)
2850                    if end < begin {
2851                        replacement.push_str(&s[end..begin]);
2852                    }
2853                    let varname = &s[begin..i];
2854                    vars.insert(varname);
2855                    let replacement_var = self.precompile_name(varname);
2856                    replacement += &replacement_var;
2857                    end = i;
2858                    var = false;
2859                }
2860            }
2861        }
2862        if end > 0 {
2863            replacement.push_str(&s[end..]);
2864        }
2865        if var {
2866            //don't forget last one
2867            let varname = &s[begin..];
2868            vars.insert(varname);
2869            let replacement_var = self.precompile_name(varname);
2870            replacement += &replacement_var;
2871        }
2872        if !replacement.is_empty() {
2873            //eprintln!("DEBUG: precompile_inblock({}) -> {}", s, replacement);
2874            Cow::Owned(replacement)
2875        } else {
2876            Cow::Borrowed(s)
2877        }
2878    }
2879
2880    /// upon's templating syntax doesn't support some of the characters we use in names, this function substitutes them for more verbose equivalents
2881    fn precompile_name(&self, s: &str) -> String {
2882        let mut replacement = String::new();
2883        let mut begincondition = None;
2884        let mut skip = 0;
2885        for (i,c) in s.char_indices() {
2886            if begincondition.is_some() && c != ']' {
2887                continue;
2888            } else if skip > 0 {
2889                skip -= 1;
2890                continue;
2891            }
2892            if c == '$' {
2893                let slice = &s[i..];
2894                if slice.starts_with("$$..") {
2895                    replacement.push_str("ELEMENTS_PARENT");
2896                    skip = 3;
2897                } else if slice.starts_with("$$.") {
2898                    replacement.push_str("ELEMENTS_THIS");
2899                    skip = 2;
2900                } else if slice.starts_with("$$/") {
2901                    replacement.push_str("ELEMENTS_");
2902                    skip = 2;
2903                } else if slice.starts_with("$$") {
2904                    replacement.push_str("ELEMENTS_");
2905                    skip = 1;
2906                } else if slice.starts_with("$..") {
2907                    replacement.push_str("ELEMENT_PARENT");
2908                    skip = 2;
2909                } else if slice.starts_with("$.") {
2910                    replacement.push_str("ELEMENT_THIS");
2911                    skip = 1;
2912                } else if slice.starts_with("$/") {
2913                    replacement.push_str("ELEMENT_");
2914                    skip = 1;
2915                } else {
2916                    replacement.push_str("ELEMENT_");
2917                }
2918            } else if c == '@' {
2919                replacement.push_str("ATTRIB_");
2920            } else if c == '/' {
2921                replacement.push_str("_IN_");
2922            } else if c == ':' {
2923                replacement.push_str("__");
2924            } else if c == '[' {
2925                begincondition = Some(i+1);
2926            } else if c == ']' {
2927                //conditions are just stored as hashes
2928                if let Some(begin) = begincondition {
2929                    let mut hasher = DefaultHasher::new();
2930                    let _ = &s[begin..i].hash(&mut hasher);
2931                    let h = hasher.finish();
2932                    replacement.push_str(&format!("_COND{}_", h));
2933                }
2934                begincondition = None;
2935            } else {
2936                replacement.push(c);
2937            }
2938        }
2939        //eprintln!("DEBUG: precompile_name({}) -> {}", s, replacement);
2940        replacement
2941    }
2942
2943    fn add_metadata(&self, store: &mut AnnotationStore) -> Result<(), XmlConversionError> {
2944        for metadata in self.config.metadata.iter() {
2945            let mut builder = AnnotationBuilder::new();
2946
2947            let resource_id = if let Some(resource_handle) = self.resource_handle {
2948                store.resource(resource_handle).unwrap().id()
2949            } else {
2950                None
2951            };
2952
2953            let mut context = self.global_context.clone();
2954            if let Some(resource_id) = resource_id {
2955                context.insert("resource".into(), resource_id.into());
2956            }
2957
2958            if let Some(template) = &metadata.id {
2959                let compiled_template = self.template_engine.template(template.as_str());
2960                let id = compiled_template.render(&context).to_string().map_err(|e| 
2961                        XmlConversionError::TemplateError(
2962                            format!(
2963                                "whilst rendering metadata id template '{}'",
2964                                template,
2965                            ),
2966                            Some(e),
2967                        )
2968                    )?;
2969                if !id.is_empty() {
2970                    builder = builder.with_id(id);
2971                }
2972            }
2973
2974            for annotationdata in metadata.annotationdata.iter() {
2975                let mut databuilder = AnnotationDataBuilder::new();
2976                if let Some(template) = &annotationdata.set {
2977                    let compiled_template = self.template_engine.template(template.as_str());
2978                    let dataset = compiled_template.render(&context).to_string().map_err(|e| 
2979                            XmlConversionError::TemplateError(
2980                                format!(
2981                                    "whilst rendering annotationdata/dataset template '{}' for metadata",
2982                                    template,
2983                                ),
2984                                Some(e),
2985                            )
2986                        )?;
2987                    if !dataset.is_empty() {
2988                        databuilder = databuilder.with_dataset(dataset.into())
2989                    }
2990                } else {
2991                    databuilder =
2992                        databuilder.with_dataset(self.config.default_set.as_str().into());
2993                }
2994                if let Some(template) = &annotationdata.key {
2995                    let compiled_template = self.template_engine.template(template.as_str());
2996                    match compiled_template.render(&context).to_string().map_err(|e| 
2997                            XmlConversionError::TemplateError(
2998                                format!(
2999                                    "whilst rendering annotationdata/key template '{}' for metadata",
3000                                    template,
3001                                ),
3002                                Some(e),
3003                            )
3004                        )  {
3005                        Ok(key) if !key.is_empty() =>
3006                            databuilder = databuilder.with_key(key.into()) ,
3007                        Ok(_) if !annotationdata.skip_if_missing => {
3008                            return Err(XmlConversionError::TemplateError(
3009                                format!(
3010                                    "whilst rendering annotationdata/key template '{}' metadata",
3011                                    template,
3012                                ),
3013                                None
3014                            ));
3015                        },
3016                        Err(e) if !annotationdata.skip_if_missing => {
3017                            return Err(e)
3018                        },
3019                        _ => {
3020                            //skip whole databuilder if missing
3021                            continue
3022                        }
3023                    }
3024                }
3025                if let Some(value) = &annotationdata.value {
3026                    match self.extract_value_metadata(value, &upon::Value::Map(context.clone()), annotationdata.allow_empty_value, annotationdata.skip_if_missing,  resource_id.as_deref())? {
3027                        Some(value) => {
3028                            databuilder = databuilder.with_value(value);
3029                        },
3030                        None =>  {
3031                            //skip whole databuilder if missing
3032                            continue
3033                        }
3034                    }
3035                }
3036                builder = builder.with_data_builder(databuilder);
3037            }
3038
3039
3040
3041            // Finish the builder and add the actual annotation to the store, according to its element handling
3042            match metadata.annotation {
3043                XmlAnnotationHandling::TextSelector => {
3044                    // Annotation is on text, translates to TextSelector
3045                    builder = builder.with_target(SelectorBuilder::TextSelector(BuildItem::Handle(self.resource_handle.expect("resource must have handle")), Offset::whole()));
3046                    if self.config.debug {
3047                        eprintln!("[STAM fromxml]   builder AnnotateText: {:?}", builder);
3048                    }
3049                    store.annotate(builder)?;
3050                }
3051                XmlAnnotationHandling::ResourceSelector  | XmlAnnotationHandling::None | XmlAnnotationHandling::Unspecified => {
3052                    // Annotation is metadata (default), translates to ResourceSelector
3053                    builder = builder.with_target(SelectorBuilder::ResourceSelector(
3054                        self.resource_handle.into(),
3055                    ));
3056                    if self.config.debug {
3057                        eprintln!("[STAM fromxml]   builder AnnotateResource: {:?}", builder);
3058                    }
3059                    store.annotate(builder)?;
3060                }
3061                _ => panic!(
3062                    "Invalid annotationhandling for metadata: {:?}",
3063                    metadata.annotation
3064                ),
3065            }
3066        }
3067        Ok(())
3068    }
3069}
3070
3071
3072
3073/// Get recursive text without any elements
3074fn recursive_text(node: &Node) -> String {
3075    let mut s = String::new();
3076    for child in node.children() {
3077        if child.is_text() {
3078            s += child.text().expect("should have text");
3079        } else if child.is_element() {
3080            s += &recursive_text(&child);
3081        }
3082    }
3083    s
3084}
3085
3086// Filters
3087fn filter_capitalize(s: &str) -> String {
3088    let mut out = String::with_capacity(s.len());
3089    for (i, c) in s.chars().enumerate() {
3090        if i == 0 {
3091            out.push_str(&c.to_uppercase().collect::<String>())
3092        } else {
3093            out.push(c);
3094        }
3095    }
3096    out
3097}
3098
3099fn filter_gt(a: &upon::Value, b: &upon::Value) -> bool {
3100    match (a, b) {
3101        (upon::Value::Integer(a), upon::Value::Integer(b)) => *a > *b,
3102        (upon::Value::Float(a), upon::Value::Float(b)) => *a > *b,
3103        (upon::Value::String(a), upon::Value::String(b)) => *a > *b,
3104        _ => false,
3105    }
3106}
3107
3108fn filter_lt(a: &upon::Value, b: &upon::Value) -> bool {
3109    match (a, b) {
3110        (upon::Value::Integer(a), upon::Value::Integer(b)) => *a < *b,
3111        (upon::Value::Float(a), upon::Value::Float(b)) => *a < *b,
3112        (upon::Value::String(a), upon::Value::String(b)) => *a < *b,
3113        _ => false,
3114    }
3115}
3116
3117fn filter_gte(a: &upon::Value, b: &upon::Value) -> bool {
3118    match (a, b) {
3119        (upon::Value::Integer(a), upon::Value::Integer(b)) => *a >= *b,
3120        (upon::Value::Float(a), upon::Value::Float(b)) => *a >= *b,
3121        (upon::Value::String(a), upon::Value::String(b)) => *a >= *b,
3122        _ => false,
3123    }
3124}
3125
3126fn filter_lte(a: &upon::Value, b: &upon::Value) -> bool {
3127    match (a, b) {
3128        (upon::Value::Integer(a), upon::Value::Integer(b)) => *a <= *b,
3129        (upon::Value::Float(a), upon::Value::Float(b)) => *a <= *b,
3130        (upon::Value::String(a), upon::Value::String(b)) => *a <= *b,
3131        _ => false,
3132    }
3133}
3134
3135fn filter_add(a: &upon::Value, b: &upon::Value) -> upon::Value {
3136    match (a, b) {
3137        (upon::Value::Integer(a), upon::Value::Integer(b)) => upon::Value::Integer(a + b),
3138        (upon::Value::Float(a), upon::Value::Float(b)) => upon::Value::Float(a + b),
3139        (upon::Value::String(a), upon::Value::String(b)) => upon::Value::String(a.clone() + b),
3140        _ => upon::Value::None,
3141    }
3142}
3143
3144fn filter_sub(a: &upon::Value, b: &upon::Value) -> upon::Value {
3145    match (a, b) {
3146        (upon::Value::Integer(a), upon::Value::Integer(b)) => upon::Value::Integer(a - b),
3147        (upon::Value::Float(a), upon::Value::Float(b)) => upon::Value::Float(a - b),
3148        _ => upon::Value::None,
3149    }
3150}
3151
3152fn filter_mul(a: &upon::Value, b: &upon::Value) -> upon::Value {
3153    match (a, b) {
3154        (upon::Value::Integer(a), upon::Value::Integer(b)) => upon::Value::Integer(a * b),
3155        (upon::Value::Float(a), upon::Value::Float(b)) => upon::Value::Float(a * b),
3156        _ => upon::Value::None,
3157    }
3158}
3159
3160fn filter_div(a: &upon::Value, b: &upon::Value) -> upon::Value {
3161    match (a, b) {
3162        (upon::Value::Integer(a), upon::Value::Integer(b)) => upon::Value::Integer(a / b),
3163        (upon::Value::Float(a), upon::Value::Float(b)) => upon::Value::Float(a / b),
3164        _ => upon::Value::None,
3165    }
3166}
3167
3168fn filter_int(a: &upon::Value) -> upon::Value {
3169    match a {
3170        upon::Value::Integer(x) => upon::Value::Integer(*x),
3171        upon::Value::Float(x) => upon::Value::Integer(*x as i64), 
3172        upon::Value::String(s) => upon::Value::Integer(s.parse().expect("int filter expects an integer value")),
3173        _ => panic!("int filter expects an integer value"), //<< --^  TODO: PANIC IS WAY TO STRICT
3174    }
3175}
3176
3177fn filter_float(a: &upon::Value) -> upon::Value {
3178    match a {
3179        upon::Value::Float(_) => a.clone(),
3180        upon::Value::Integer(x) => upon::Value::Float(*x as f64),
3181        upon::Value::String(s) => upon::Value::Float(s.parse().expect("float filter expects a float value")),
3182        _ => panic!("float filter expects a float value"), //<< --^  TODO: PANIC IS WAY TO STRICT
3183    }
3184}
3185
3186fn filter_round(a: &upon::Value) -> upon::Value {
3187    match a {
3188        upon::Value::Integer(x) => upon::Value::Integer(*x),
3189        upon::Value::Float(x) => upon::Value::Integer(x.round() as i64),
3190        upon::Value::String(s) => {
3191            let f: f64 = s.parse().expect("round filter expects a float value");
3192            upon::Value::Integer(f.round() as i64)
3193        }
3194        _ => panic!("round filter expects a float value"), //<< --^  TODO: PANIC IS WAY TO STRICT
3195    }
3196}
3197
3198fn filter_ceil(a: &upon::Value) -> upon::Value {
3199    match a {
3200        upon::Value::Integer(x) => upon::Value::Integer(*x),
3201        upon::Value::Float(x) => upon::Value::Integer(x.ceil() as i64),
3202        upon::Value::String(s) => {
3203            let f: f64 = s.parse().expect("round filter expects a float value");
3204            upon::Value::Integer(f.ceil() as i64)
3205        }
3206        _ => panic!("ceil filter expects a float value"), //<< --^  TODO: PANIC IS WAY TO STRICT
3207    }
3208}
3209
3210fn filter_floor(a: &upon::Value) -> upon::Value {
3211    match a {
3212        upon::Value::Integer(x) => upon::Value::Integer(*x),
3213        upon::Value::Float(x) => upon::Value::Integer(x.floor() as i64),
3214        upon::Value::String(s) => {
3215            let f: f64 = s.parse().expect("round filter expects a float value");
3216            upon::Value::Integer(f.floor() as i64)
3217        }
3218        _ => panic!("ceil filter expects a float value"), //<< --^  TODO: PANIC IS WAY TO STRICT
3219    }
3220}
3221
3222fn filter_str(a: upon::Value) -> upon::Value {
3223    match a {
3224        upon::Value::Integer(x) => upon::Value::String(format!("{}",x)),
3225        upon::Value::Float(x) => upon::Value::String(format!("{}",x)),
3226        upon::Value::Bool(x) => upon::Value::String(format!("{}",x)),
3227        upon::Value::String(_) => a,
3228        upon::Value::None => upon::Value::String(String::new()),
3229        upon::Value::List(list) => { //too much cloning but it'll do for now
3230            let newlist: Vec<String> = list.iter().map(|v| match v {
3231                upon::Value::String(s) => s.clone(),
3232                upon::Value::Integer(d) => format!("{}",d),
3233                upon::Value::Float(d) => format!("{}",d),
3234                upon::Value::Bool(d) => format!("{}",d),
3235                _ => String::new(),
3236            }).collect();
3237            upon::Value::String(newlist.join(", "))
3238        },
3239        _ => panic!("map to string not implemented"), //<< --^  TODO: PANIC IS WAY TO STRICT
3240    }
3241}
3242
3243fn filter_substr(s: &upon::Value, begin: isize, end: isize) -> String {
3244    if let upon::Value::String(s) = s {
3245        let begin = if begin < 0 {
3246            s.chars().count() as isize + begin
3247        } else {
3248            begin
3249        };
3250        let end = if end < 0 {
3251            s.chars().count() as isize + end
3252        } else {
3253            end
3254        };
3255        if end > begin {
3256            s.chars().skip(begin as usize).take((end-begin) as usize).collect()
3257        } else {
3258            s.chars().skip(begin as usize).take(usize::MAX).collect()
3259        }
3260    } else {
3261        String::new()
3262    }
3263}
3264
3265fn filter_basename(a: &upon::Value) -> upon::Value {
3266    match a {
3267        upon::Value::String(s) => upon::Value::String(s.split(|c| c == '/' || c == '\\').last().expect("splitting must work").to_string()),
3268        _ => panic!("basename filter expects a string value"), //<< --^  TODO: PANIC IS WAY TO STRICT
3269    }
3270}
3271
3272fn filter_noext(a: &upon::Value) -> String {
3273    match a {
3274        upon::Value::String(s) => if let Some(pos) = s.rfind('.') {
3275            s[..pos].to_string()
3276        } else {
3277            s.to_string()
3278        },
3279        _ => panic!("basename filter expects a string value"), //<< --^  TODO: PANIC IS WAY TO STRICT
3280    }
3281}
3282
3283/// Map value between toml and upon. This makes a clone.
3284fn map_value(value: &toml::Value) -> upon::Value {
3285    match value {
3286        toml::Value::String(s) => upon::Value::String(s.clone()),
3287        toml::Value::Integer(i) => upon::Value::Integer(*i),
3288        toml::Value::Float(i) => upon::Value::Float(*i),
3289        toml::Value::Boolean(v) => upon::Value::Bool(*v),
3290        toml::Value::Datetime(s) => upon::Value::String(s.to_string()),
3291        toml::Value::Array(v) => upon::Value::List(v.iter().map(|i| map_value(i)).collect()),
3292        toml::Value::Table(v) => upon::Value::Map(v.iter().map(|(k,i)| (k.clone(),map_value(i))).collect()),
3293    }
3294}
3295
3296/// Parse a string that is a result from the template renderer to a DataValue again
3297#[inline]
3298fn string_to_datavalue(value: String, valuetype: Option<&str>) -> Result<DataValue,XmlConversionError> {
3299    match valuetype {
3300        Some("str") | Some("string")  => Ok(DataValue::String(value)),
3301        Some("int") => {
3302            if let Ok(value) = value.parse::<isize>() {
3303                Ok(DataValue::Int(value))
3304            } else {
3305                Err(XmlConversionError::TemplateError(format!("Unable to interpret value as integer: {}", value), None))
3306            }
3307        },
3308        Some("float") => {
3309            if let Ok(value) = value.parse::<f64>() {
3310                Ok(DataValue::Float(value))
3311            } else {
3312                Err(XmlConversionError::TemplateError(format!("Unable to interpret value as integer: {}", value), None))
3313            }
3314        },
3315        Some("bool") => match value.as_str() {
3316            "yes" | "true" | "enabled" | "on" | "1" | "active"  => Ok(DataValue::Bool(true)),
3317            _ => Ok(DataValue::Bool(false))
3318        },
3319        Some(x) => {
3320                Err(XmlConversionError::TemplateError(format!("Invalid valuetype: {}", x), None))
3321        }
3322        None => {
3323            //automatically determine type
3324            if let Ok(value) =  value.parse::<isize>() {
3325                Ok(DataValue::Int(value))
3326            } else if let Ok(value) =  value.parse::<f64>() {
3327                Ok(DataValue::Float(value))
3328            } else if value.starts_with("(list) [ ") && value.ends_with(" ]") {
3329                //deserialize lists again
3330                if let Ok(serde_json::Value::Array(values)) = serde_json::from_str(&value[6..]) {
3331                    Ok(DataValue::List(values.into_iter().map(|v| {
3332                        match v {
3333                            serde_json::Value::String(s) => DataValue::String(s),
3334                            serde_json::Value::Number(n) => if let Some(n) = n.as_i64() {
3335                                DataValue::Int(n as isize)
3336                            } else if let Some(n) = n.as_f64() {
3337                                DataValue::Float(n)
3338                            } else {
3339                                unreachable!("number should always be either int or float")
3340                            },
3341                            serde_json::Value::Bool(b) => DataValue::Bool(b),
3342                            _ => DataValue::Null, //nested arrays and maps are NOT supported here!
3343                        }
3344                    }).collect()))
3345                } else {
3346                    Err(XmlConversionError::TemplateError(format!("Unable to deserialize list value: {}", value), None))
3347                }
3348            } else {
3349                Ok(value.into())
3350            }
3351        }
3352    }
3353}
3354
3355fn string_to_templatevalue(value: String) -> upon::Value {
3356    if let Ok(value) =  value.parse::<i64>() {
3357        upon::Value::Integer(value)
3358    } else if let Ok(value) =  value.parse::<f64>() {
3359        upon::Value::Float(value)
3360    } else {
3361        upon::Value::String(value)
3362    }
3363}
3364
3365/// Custom formatter for templating that can also handle lists (the default one in upon can't)
3366/// Lists will be output JSON-style prepended by the marker text "(list) ", this allows deserialisers to turn it into a list again
3367fn value_formatter(f: &mut upon::fmt::Formatter<'_>, value: &upon::Value) -> upon::fmt::Result {
3368    match value {
3369        upon::Value::List(vs) => {
3370            f.write_str("(list) [ ")?;
3371            for (i, v) in vs.iter().enumerate() {
3372                if i > 0 {
3373                    f.write_str(", ")?;
3374                }
3375                if let upon::Value::String(s) = v {
3376                    write!(f, "\"{}\"", s.replace("\"","\\\"").replace("\n"," ").split_whitespace().collect::<Vec<_>>().join(" "))?;
3377                } else {
3378                    upon::fmt::default(f, v)?;
3379                    f.write_char('"')?;
3380                }
3381            }
3382            f.write_str(" ]")?;
3383        }
3384        v => upon::fmt::default(f, v)?, // fallback to default formatter
3385    };
3386    Ok(())
3387}
3388
3389#[derive(Clone,Debug,Deserialize)]
3390struct ExternalFilter {
3391    /// The name of the filter
3392    name: String,
3393
3394    /// The command to run.
3395    command: String,
3396
3397    /// The arguments to pass to the command, you can use "{{ value }}" or `$value` to represent the input value if needed. It will also be passed to stdin. No escaping needed, it is not mediated by a shell.
3398    args: Vec<String>
3399}
3400
3401impl ExternalFilter {
3402    //TODO: panic may be too strict in here:
3403    fn run(&self, input_value: &upon::Value) -> upon::Value {
3404        let process = Command::new(self.command.as_str()).args(
3405            //args are passed directly, not mediated via shell, so no escaping necessary
3406            self.args.iter().map(|x| if x == "{{value}}" || x == "{{ value }}" || x == "$value" {
3407                match input_value {
3408                    upon::Value::String(s) => s.clone(),
3409                    upon::Value::Integer(d) => format!("{}",d),
3410                    upon::Value::Float(d) => format!("{}",d),
3411                    upon::Value::Bool(d) => format!("{}",d),
3412                    upon::Value::None => String::new(),
3413                    _ => panic!("Lists and maps are not supported to be passed as parameter to  external filters yet!"), 
3414                }
3415            } else {
3416                x.clone() //too much cloning, but Cow didn't work here because it is coerced into OsStr later
3417            })
3418        ).stdin(Stdio::piped()).stdout(Stdio::piped()).spawn();
3419
3420
3421        if let Ok(mut process) = process {
3422            {
3423                let mut outstdin = process.stdin.take().expect("unable to open stdin for external filter");
3424                let mut writer = BufWriter::new(&mut outstdin);
3425                match input_value {
3426                    upon::Value::String(s) => writer.write(s.as_bytes()),
3427                    upon::Value::Integer(d) => writer.write(format!("{}",d).as_bytes()),
3428                    upon::Value::Float(d) => writer.write(format!("{}",d).as_bytes()),
3429                    upon::Value::Bool(d) => writer.write(format!("{}",d).as_bytes()),
3430                    upon::Value::None => writer.write(&[]),
3431                    _ => panic!("Lists and maps are not supported to be passed as input to external filters yet!"),
3432                }.expect("Writing to stdin for external filter failed!");
3433                //block ensures writer and outputsdin are dropped prior to waiting for output
3434            }
3435            let output = process.wait_with_output().expect("External filter wasn't running");
3436            if !output.status.success() {
3437                panic!("External filter {} failed ({:?})", self.name, output.status.code());
3438            }
3439            if let Ok(s) = String::from_utf8(output.stdout) {
3440                return string_to_templatevalue(s);
3441            } else {
3442                panic!("External filter {} produced invalid UTF-8!", self.name);
3443            }
3444        }
3445        panic!("External filter {} failed!", self.name);
3446    }
3447}
3448
3449#[cfg(test)]
3450mod tests {
3451    use super::*;
3452    //use crate::info::info;
3453
3454    const XMLSMALLEXAMPLE: &'static str = r#"<html xmlns="http://www.w3.org/1999/xhtml">
3455<head><title>test</title></head><body><h1>TEST</h1><p xml:id="p1" n="001">This  is a <em xml:id="emphasis" style="color:green">test</em>.</p></body></html>"#;
3456
3457    const XMLEXAMPLE: &'static str = r#"<!DOCTYPE entities[<!ENTITY nbsp "&#xA0;">]>
3458<html xmlns="http://www.w3.org/1999/xhtml" xmlns:my="http://example.com">
3459<head>
3460    <title>Test</title>
3461    <meta name="author" content="proycon" />
3462</head>
3463<body>
3464    <h1>Header</h1>
3465
3466    <p xml:id="par1">
3467        <span xml:id="sen1">This is a sentence.</span>
3468        <span xml:id="sen2">This is the second&nbsp;sentence.</span>
3469    </p>
3470    <p xml:id="par2">
3471        <strong>This</strong> is    the <em>second</em> paragraph.
3472            It has a <strong>bold</strong> word and one in <em>italics</em>.<br/>
3473        Let's highlight stress in the following word: <span my:stress="secondary">re</span>pu<span my:stress="primary">ta</span>tion.
3474    </p>
3475    <p xml:space="preserve"><![CDATA[This    third
3476paragraph consists
3477of CDATA and is configured to preserve whitespace, and weird &entities; ]]></p>
3478
3479    <h2>Subsection</h2>
3480
3481    <p>
3482    Have some fruits:<br/>
3483    <ul xml:id="list1" class="fruits">
3484        <li xml:id="fruit1">apple</li>
3485        <li xml:id="fruit2">banana</li>
3486        <li xml:id="fruit3">melon</li>
3487    </ul>
3488    </p>
3489
3490    Some lingering text outside of any confines...
3491</body>
3492</html>"#;
3493
3494    const XMLEXAMPLE_TEXTOUTPUT: &'static str = "Header\n\nThis is a sentence. This is the second sentence.\n\nThis is the second paragraph. It has a bold word and one in italics.\nLet's highlight stress in the following word: reputation.\n\nThis    third\nparagraph consists\nof CDATA and is configured to preserve whitespace, and weird &entities; \nSubsection\n\nHave some fruits:\n* apple\n* banana\n* melon\n\nSome lingering text outside of any confines...";
3495
3496    //fake example (not real HTML, testing TEI-like space attribute with complex template)
3497    const XMLTEISPACE: &'static str = r#"<html xmlns="http://www.w3.org/1999/xhtml">
3498<body><space dim="vertical" unit="lines" quantity="3" /></body></html>"#;
3499
3500    const CONF: &'static str = r#"#default whitespace handling (Collapse or Preserve)
3501whitespace = "Collapse"
3502default_set = "urn:stam-fromhtml"
3503
3504[namespaces]
3505#this defines the namespace prefixes you can use in this configuration
3506xml = "http://www.w3.org/XML/1998/namespace"
3507html = "http://www.w3.org/1999/xhtml"
3508xsd =  "http://www.w3.org/2001/XMLSchema"
3509xlink = "http://www.w3.org/1999/xlink"
3510
3511# elements and attributes are matched in reverse-order, so put more generic statements before more specific ones
3512
3513#Define some base elements that we reuse later for actual elements (prevents unnecessary repetition)
3514[baseelements.common]
3515id = "{% if ?.@xml:id %}{{ @xml:id }}{% endif %}"
3516
3517    [[baseelements.common.annotationdata]]
3518    key = "type"
3519    value = "{{ localname }}"
3520
3521    [[baseelements.common.annotationdata]]
3522    key = "lang"
3523    value = "{{ @xml:lang }}"
3524    skip_if_missing = true
3525
3526    [[baseelements.common.annotationdata]]
3527    key = "n"
3528    value = "{{ @n }}"
3529    skip_if_missing = true
3530    valuetype = "int"
3531
3532    [[baseelements.common.annotationdata]]
3533    key = "nstring"
3534    value = "{{ @n }}"
3535    skip_if_missing = true
3536    valuetype = "string"
3537
3538    [[baseelements.common.annotationdata]]
3539    key = "style"
3540    value = "{{ @style }}"
3541    skip_if_missing = true
3542
3543    [[baseelements.common.annotationdata]]
3544    key = "class"
3545    value = "{{ @class }}"
3546    skip_if_missing = true
3547
3548    [[baseelements.common.annotationdata]]
3549    key = "src"
3550    value = "{{ @src }}"
3551    skip_if_missing = true
3552
3553[baseelements.text]
3554text = true
3555
3556
3557[[elements]]
3558base = [ "text", "common" ]
3559path = "*"
3560text = true
3561annotation = "TextSelector"
3562
3563# Pass through the following elements without mapping to text
3564[[elements]]
3565base = [ "common" ]
3566path = "//html:head"
3567
3568[[elements]]
3569base = [ "common" ]
3570path = "//html:head//*"
3571
3572# Map metadata like <meta name="key" content="value"> to annotations with key->value data selecting the resource (ResourceSelector)
3573[[elements]]
3574base = [ "common" ]
3575path = "//html:head//html:meta"
3576
3577[[elements.annotationdata]]
3578key = "{% if ?.@name %}{{ name }}{% endif %}"
3579value = "{% if ?.@content %}{{ @content }}{% endif %}"
3580skip_if_missing = true
3581
3582# By default, ignore any tags in the head (unless they're mentioned specifically later in the config)
3583[[elements]]
3584path = "//html:head/html:title"
3585annotation = "ResourceSelector"
3586
3587[[elements.annotationdata]]
3588key = "title"
3589value = "{{ $. | trim }}"
3590
3591
3592# Determine how various structural elements are converted to text
3593
3594[[elements]]
3595base = [ "common" ]
3596path = "//html:br"
3597textsuffix = "\n"
3598
3599[[elements]]
3600base = [ "common", "text" ]
3601path = "//html:p"
3602textprefix = "\n"
3603textsuffix = "\n"
3604annotation = "TextSelector"
3605
3606# Let's do headers and bulleted lists like markdown
3607[[elements]]
3608base = [ "common", "text" ]
3609path = "//html:h1"
3610textsuffix = "\n"
3611annotation = "TextSelector"
3612id = "h1"
3613
3614[[elements]]
3615base = [ "common", "text" ]
3616path = "//html:body//html:h2"
3617textsuffix = "\n"
3618annotation = "TextSelector"
3619id = "h2"
3620
3621#Generic, will be overriden by more specific one
3622[[elements]]
3623base = [ "common", "text" ]
3624path = "//html:li"
3625textprefix = "- "
3626textsuffix = "\n"
3627
3628[[elements]]
3629base = [ "common", "text" ]
3630path = """//html:body"""
3631annotation = "TextSelector"
3632id = "body"
3633
3634    [[elements.annotationdata]]
3635    key = "title_from_parent"
3636    value = "{{ $../html:head/html:title }}"
3637    skip_if_missing = true
3638
3639    [[elements.annotationdata]]
3640    key = "title_from_root"
3641    value = "{{ $/html:html/html:head/html:title }}"
3642    skip_if_missing = true
3643
3644    [[elements.annotationdata]]
3645    key = "firstfruit"
3646    value = """{{ $./html:p/html:ul/html:li }}"""
3647    skip_if_missing = true
3648
3649    [[elements.annotationdata]]
3650    key = "fruits"
3651    value = """{{ $$./html:p/html:ul/html:li }}"""
3652    skip_if_missing = true
3653
3654    [[elements.annotationdata]]
3655    key = "multifruits"
3656    value = """{{ $$./html:p/html:ul/html:li }}"""
3657    skip_if_missing = true
3658    multiple = true
3659
3660#More specific one takes precendence over the above generic one
3661[[elements]]
3662base = [ "common", "text" ]
3663path = """//html:ul[@class="fruits"]/html:li"""
3664textprefix = "* "
3665textsuffix = "\n"
3666
3667#Not real HTML, test-case modelled after TEI space
3668[[elements]]
3669base = [ "common" ]
3670path = """//html:space[@dim="vertical" and @unit="lines"]"""
3671text = true
3672textsuffix = """\n{% for x in @quantity | int | as_range %}\n{% endfor %}"""
3673
3674
3675[[elements]]
3676base = [ "common", "text" ]
3677path = "//html:example"
3678annotation = "TextSelector"
3679
3680[[elements.annotationdata]]
3681key = "requiredattrib"
3682value = "{{ @requiredattrib }}"
3683
3684[[elements.annotationdata]]
3685key = "optattrib"
3686value = "{{ ?.@optattrib }}"
3687
3688[[elements]]
3689base = [ "common","text" ]
3690path = "//html:marquee"
3691annotation = "TextSelector"
3692
3693#map value, some bogus data to test parsing
3694[[elements.annotationdata]]
3695key = "map"
3696
3697[elements.annotationdata.value]
3698text = "{{ $. }}"
3699number = 42
3700bogus = true
3701
3702[[metadata]]
3703id = "metadata"
3704
3705[[metadata.annotationdata]]
3706key = "author"
3707value = "proycon"
3708"#;
3709
3710    const XMLREQATTRIBEXAMPLE: &'static str = r#"<html xmlns="http://www.w3.org/1999/xhtml">
3711<body><example xml:id="ann1" requiredattrib="blah">test</example></body></html>"#;
3712
3713    const XMLREQATTRIBEXAMPLE2: &'static str = r#"<html xmlns="http://www.w3.org/1999/xhtml">
3714<body><example xml:id="ann1">test</example></body></html>"#;
3715
3716    const XMLREQATTRIBEXAMPLE3: &'static str = r#"<html xmlns="http://www.w3.org/1999/xhtml">
3717<body><example xml:id="ann1" requiredattrib="blah" optattrib="blah">test</example></body></html>"#;
3718
3719    const XMLMAPEXAMPLE: &'static str = r#"<html xmlns="http://www.w3.org/1999/xhtml">
3720<body><marquee xml:id="ann1">test</marquee></body></html>"#;
3721
3722    #[test]
3723    fn test_precompile_template_nochange() -> Result<(), String> {
3724        let config = XmlConversionConfig::new();
3725        let mut conv = XmlToStamConverter::new(&config);
3726        let template_in = "{{ foo }}";
3727        let template_out = conv.precompile(template_in);
3728        assert_eq!( template_out, template_in);
3729        //foo is not a special variable
3730        assert!(!conv.variables.get(template_in).as_ref().unwrap().contains("foo"));
3731        Ok(())
3732    }
3733
3734    #[test]
3735    fn test_precompile_template_attrib() -> Result<(), String> {
3736        let config = XmlConversionConfig::new();
3737        let mut conv = XmlToStamConverter::new(&config);
3738        let template_in = "{{ @foo }}";
3739        let template_out = conv.precompile(template_in);
3740        assert_eq!(template_out, "{{ ATTRIB_foo }}");
3741        //foo is an attribute so is returned
3742        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("@foo"));
3743        Ok(())
3744    }
3745
3746    #[test]
3747    fn test_precompile_template_attrib_ns() -> Result<(), String> {
3748        let config = XmlConversionConfig::new();
3749        let mut conv = XmlToStamConverter::new(&config);
3750        let template_in = "{{ @bar:foo }}";
3751        let template_out = conv.precompile(template_in);
3752        assert_eq!(template_out, "{{ ATTRIB_bar__foo }}");
3753        //foo is an attribute so is returned
3754        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("@bar:foo"));
3755        Ok(())
3756    }
3757
3758    #[test]
3759    fn test_precompile_template_element() -> Result<(), String> {
3760        let config = XmlConversionConfig::new();
3761        let mut conv = XmlToStamConverter::new(&config);
3762        let template_in = "{{ $foo }}";
3763        let template_out = conv.precompile(template_in);
3764        assert_eq!(template_out, "{{ ELEMENT_foo }}");
3765        //foo is an element so is returned
3766        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("$foo"));
3767        Ok(())
3768    }
3769
3770    #[test]
3771    fn test_precompile_template_element_ns() -> Result<(), String> {
3772        let config = XmlConversionConfig::new();
3773        let mut conv = XmlToStamConverter::new(&config);
3774        let template_in = "{{ $bar:foo }}";
3775        let template_out = conv.precompile(template_in);
3776        assert_eq!(template_out, "{{ ELEMENT_bar__foo }}");
3777        //foo is an element so is returned
3778        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("$bar:foo"));
3779        Ok(())
3780    }
3781
3782    #[test]
3783    fn test_precompile_template_this_text() -> Result<(), String> {
3784        let config = XmlConversionConfig::new();
3785        let mut conv = XmlToStamConverter::new(&config);
3786        let template_in = "{{ $. }}";
3787        let template_out = conv.precompile(template_in);
3788        assert_eq!(template_out, "{{ ELEMENT_THIS }}");
3789        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("$."));
3790        Ok(())
3791    }
3792
3793    #[test]
3794    fn test_precompile_template_parent_text() -> Result<(), String> {
3795        let config = XmlConversionConfig::new();
3796        let mut conv = XmlToStamConverter::new(&config);
3797        let template_in = "{{ $.. }}";
3798        let template_out = conv.precompile(template_in);
3799        assert_eq!(template_out, "{{ ELEMENT_PARENT }}");
3800        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("$.."));
3801        Ok(())
3802    }
3803
3804    #[test]
3805    fn test_precompile_template_elements() -> Result<(), String> {
3806        let config = XmlConversionConfig::new();
3807        let mut conv = XmlToStamConverter::new(&config);
3808        let template_in = "{{ $$foo }}";
3809        let template_out = conv.precompile(template_in);
3810        assert_eq!(template_out, "{{ ELEMENTS_foo }}");
3811        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("$$foo"));
3812        Ok(())
3813    }
3814
3815    #[test]
3816    fn test_precompile_template_elements_ns() -> Result<(), String> {
3817        let config = XmlConversionConfig::new();
3818        let mut conv = XmlToStamConverter::new(&config);
3819        let template_in = "{{ $$bar:foo }}";
3820        let template_out = conv.precompile(template_in);
3821        assert_eq!(template_out, "{{ ELEMENTS_bar__foo }}");
3822        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("$$bar:foo"));
3823        Ok(())
3824    }
3825
3826
3827    #[test]
3828    fn test_precompile_template_attrib2() -> Result<(), String> {
3829        let config = XmlConversionConfig::new();
3830        let mut conv = XmlToStamConverter::new(&config);
3831        let template_in = "{% for x in @foo %}";
3832        let template_out = conv.precompile(template_in);
3833        assert_eq!(template_out, "{% for x in ATTRIB_foo %}");
3834        //foo is an attribute so is returned
3835        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("@foo"));
3836        Ok(())
3837    }
3838
3839    #[test]
3840    fn test_precompile_template_attrib3() -> Result<(), String> {
3841        let config = XmlConversionConfig::new();
3842        let mut conv = XmlToStamConverter::new(&config);
3843        let template_in = "{{ ?.@foo }}";
3844        let template_out = conv.precompile(template_in);
3845        assert_eq!(template_out, "{{ ?.ATTRIB_foo }}");
3846        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("@foo"));
3847        Ok(())
3848    }
3849
3850    #[test]
3851    fn test_precompile_template_path() -> Result<(), String> {
3852        let config = XmlConversionConfig::new();
3853        let mut conv = XmlToStamConverter::new(&config);
3854        let template_in = "{{ $x/y/z/@a }}";
3855        let template_out = conv.precompile(template_in);
3856        assert_eq!(template_out, "{{ ELEMENT_x_IN_y_IN_z_IN_ATTRIB_a }}");
3857        assert!(conv.variables.get(template_in).as_ref().unwrap().contains("$x/y/z/@a"));
3858        Ok(())
3859    }
3860
3861    #[test]
3862    fn test_loadconfig() -> Result<(), String> {
3863        let config = XmlConversionConfig::from_toml_str(CONF)?;
3864        let mut conv = XmlToStamConverter::new(&config);
3865        conv.compile().map_err(|e| format!("{}",e))?;
3866        assert_eq!(conv.config.namespaces.len(),4 , "number of namespaces");
3867        assert_eq!(conv.config.elements.len(), 15, "number of elements");
3868        assert_eq!(conv.config.baseelements.len(), 2, "number of baseelements");
3869        assert_eq!(conv.config.elements.get(0).unwrap().annotationdata.len(), 7,"number of annotationdata under first element");
3870        assert_eq!(conv.config.baseelements.get("common").unwrap().annotationdata.len(), 7,"number of annotationdata under baseelement common");
3871        Ok(())
3872    }
3873
3874    #[test]
3875    fn test_small() -> Result<(), String> {
3876        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
3877        let mut store = stam::AnnotationStore::new(stam::Config::new());
3878        from_xml_in_memory("test", XMLSMALLEXAMPLE, &config, &mut store)?;
3879        let res = store.resource("test").expect("resource must have been created at this point");
3880        assert_eq!(res.text(), "TEST\n\nThis is a test.\n", "resource text");
3881        assert_eq!(store.annotations_len(), 7, "number of annotations");
3882        let annotation = store.annotation("emphasis").expect("annotation must have been created at this point");
3883        assert_eq!(annotation.text_simple(), Some("test"));
3884        //eprintln!("DEBUG: {:?}",annotation.data().collect::<Vec<_>>());
3885        let key = store.key("urn:stam-fromhtml", "style").expect("key must exist");
3886        assert_eq!(annotation.data().filter_key(&key).value_as_str(), Some("color:green"));
3887        let key = store.key("urn:stam-fromhtml", "title").expect("key must exist");
3888        let annotation = res.annotations_as_metadata().filter_key(&key).next().expect("annotation");
3889        assert_eq!(annotation.data().filter_key(&key).value_as_str(), Some("test"));
3890        let bodyannotation = store.annotation("body").expect("body annotation not found");
3891        let title1 = store.key("urn:stam-fromhtml", "title_from_parent").expect("key must exist");
3892        let title2 = store.key("urn:stam-fromhtml", "title_from_root").expect("key must exist");
3893        assert_eq!(bodyannotation.data().filter_key(&title1).value_as_str(), Some("test"));
3894        assert_eq!(bodyannotation.data().filter_key(&title2).value_as_str(), Some("test"));
3895        Ok(())
3896    }
3897
3898    #[test]
3899    fn test_full() -> Result<(), String> {
3900        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
3901        let mut store = stam::AnnotationStore::new(stam::Config::new());
3902        from_xml_in_memory("test", XMLEXAMPLE, &config, &mut store)?;
3903        let res = store.resource("test").expect("resource must have been created at this point");
3904        assert_eq!(res.text(), XMLEXAMPLE_TEXTOUTPUT, "resource text");
3905        Ok(())
3906    }
3907
3908    #[test]
3909    fn test_firstfruit() -> Result<(), String> {
3910        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
3911        let mut store = stam::AnnotationStore::new(stam::Config::new());
3912        from_xml_in_memory("test", XMLEXAMPLE, &config, &mut store)?;
3913        let bodyannotation = store.annotation("body").expect("body annotation not found");
3914        let fruit = store.key("urn:stam-fromhtml", "firstfruit").expect("key must exist");
3915        assert_eq!(bodyannotation.data().filter_key(&fruit).value_as_str(), Some("apple") );
3916        Ok(())
3917    }
3918
3919    #[test]
3920    fn test_fruits() -> Result<(), String> {
3921        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
3922        let mut store = stam::AnnotationStore::new(stam::Config::new());
3923        from_xml_in_memory("test", XMLEXAMPLE, &config, &mut store)?;
3924        let bodyannotation = store.annotation("body").expect("body annotation not found");
3925        let fruits = store.key("urn:stam-fromhtml", "fruits").expect("key must exist");
3926        assert_eq!(bodyannotation.data().filter_key(&fruits).value(), Some(&DataValue::List(vec!("apple".into(),"banana".into(),"melon".into()) )));
3927        Ok(())
3928    }
3929
3930    #[test]
3931    fn test_multifruits() -> Result<(), String> {
3932        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
3933        let mut store = stam::AnnotationStore::new(stam::Config::new());
3934        from_xml_in_memory("test", XMLEXAMPLE, &config, &mut store)?;
3935        let bodyannotation = store.annotation("body").expect("body annotation not found");
3936        let fruits = store.key("urn:stam-fromhtml", "multifruits").expect("key must exist");
3937        let results: Vec<_> = bodyannotation.data().filter_key(&fruits).collect();
3938        assert_eq!(results.len(), 3);
3939        assert_eq!(results.get(0).unwrap().value(),&DataValue::String("apple".to_string()) );
3940        assert_eq!(results.get(1).unwrap().value(),&DataValue::String("banana".to_string()) );
3941        assert_eq!(results.get(2).unwrap().value(),&DataValue::String("melon".to_string()) );
3942        Ok(())
3943    }
3944
3945    #[test]
3946    fn test_teispace() -> Result<(), String> {
3947        let config = XmlConversionConfig::from_toml_str(CONF)?;
3948        let mut store = stam::AnnotationStore::new(stam::Config::new());
3949        from_xml_in_memory("test", XMLTEISPACE, &config, &mut store)?;
3950        let res = store.resource("test").expect("resource must have been created at this point");
3951        assert_eq!(res.text(), "\n\n\n\n", "resource text");
3952        Ok(())
3953    }
3954
3955
3956    #[test]
3957    fn test_reqattrib() -> Result<(), String> {
3958        let config = XmlConversionConfig::from_toml_str(CONF)?;
3959        let mut store = stam::AnnotationStore::new(stam::Config::new());
3960        from_xml_in_memory("test", XMLREQATTRIBEXAMPLE, &config, &mut store)?;
3961        let res = store.resource("test").expect("resource must have been created at this point");
3962        assert_eq!(res.text(), "test", "resource text");
3963        let key = store.key("urn:stam-fromhtml", "requiredattrib").expect("key must exist");
3964        let annotation = store.annotation("ann1").expect("annotation");
3965        assert_eq!(annotation.data().filter_key(&key).value_as_str(), Some("blah"));
3966        assert!(store.key("urn:stam-fromhtml", "optattrib").is_none(), "optional attrib is unused");
3967        Ok(())
3968    }
3969
3970    #[test]
3971    fn test_reqattrib2() -> Result<(), String> {
3972        let mut config = XmlConversionConfig::from_toml_str(CONF)?;
3973        config = config.with_debug(true);
3974        let mut store = stam::AnnotationStore::new(stam::Config::new());
3975        assert!(from_xml_in_memory("test", XMLREQATTRIBEXAMPLE2, &config, &mut store).is_err(), "checking if error is returned");
3976        Ok(())
3977    }
3978
3979    #[test]
3980    fn test_reqattrib3() -> Result<(), String> {
3981        let config = XmlConversionConfig::from_toml_str(CONF)?;
3982        let mut store = stam::AnnotationStore::new(stam::Config::new());
3983        from_xml_in_memory("test", XMLREQATTRIBEXAMPLE3, &config, &mut store)?;
3984        let res = store.resource("test").expect("resource must have been created at this point");
3985        assert_eq!(res.text(), "test", "resource text");
3986        let reqkey = store.key("urn:stam-fromhtml", "requiredattrib").expect("key must exist");
3987        let optkey = store.key("urn:stam-fromhtml", "optattrib").expect("key optattrib must exist");
3988        let annotation = store.annotation("ann1").expect("annotation");
3989        assert_eq!(annotation.data().filter_key(&reqkey).value_as_str(), Some("blah"));
3990        assert_eq!(annotation.data().filter_key(&optkey).value_as_str(), Some("blah"));
3991        Ok(())
3992    }
3993
3994    #[test]
3995    fn test_map() -> Result<(), String> {
3996        let config = XmlConversionConfig::from_toml_str(CONF)?;
3997        let mut store = stam::AnnotationStore::new(stam::Config::new());
3998        from_xml_in_memory("test", XMLMAPEXAMPLE, &config, &mut store)?;
3999        let res = store.resource("test").expect("resource must have been created at this point");
4000        assert_eq!(res.text(), "test", "resource text");
4001        let key = store.key("urn:stam-fromhtml", "map").expect("key must exist");
4002        let annotation = store.annotation("ann1").expect("annotation");
4003        let data = annotation.data().filter_key(&key).value().expect("data must exist");
4004        if let DataValue::Map(data) = data {
4005            assert_eq!(data.get("text"), Some(&DataValue::String("test".into())));
4006            assert_eq!(data.get("number"), Some(&DataValue::Int(42)));
4007            assert_eq!(data.get("bogus"), Some(&DataValue::Bool(true)));
4008            assert_eq!(data.len(), 3);
4009        } else {
4010            assert!(false, "Data is supposed to be a map");
4011        }
4012        Ok(())
4013    }
4014
4015    #[test]
4016    fn test_metadata() -> Result<(), String> {
4017        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
4018        let mut store = stam::AnnotationStore::new(stam::Config::new());
4019        from_xml_in_memory("test", XMLEXAMPLE, &config, &mut store)?;
4020        let annotation = store.annotation("metadata").expect("annotation");
4021        let key = store.key("urn:stam-fromhtml", "author").expect("key must exist");
4022        let data = annotation.data().filter_key(&key).value().expect("data must exist");
4023        assert_eq!(data, &DataValue::String("proycon".into()));
4024        Ok(())
4025    }
4026
4027    #[test]
4028    fn test_datavalue_int() -> Result<(), String> {
4029        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
4030        let mut store = stam::AnnotationStore::new(stam::Config::new());
4031        from_xml_in_memory("test", XMLSMALLEXAMPLE, &config, &mut store)?;
4032        let annotation = store.annotation("p1").expect("annotation not found");
4033        let key = store.key("urn:stam-fromhtml", "n").expect("key must exist");
4034        assert_eq!(annotation.data().filter_key(&key).value(), Some(&DataValue::Int(1)));
4035        Ok(())
4036    }
4037
4038    #[test]
4039    fn test_datavalue_string() -> Result<(), String> {
4040        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
4041        let mut store = stam::AnnotationStore::new(stam::Config::new());
4042        from_xml_in_memory("test", XMLSMALLEXAMPLE, &config, &mut store)?;
4043        let annotation = store.annotation("p1").expect("annotation not found");
4044        let key = store.key("urn:stam-fromhtml", "nstring").expect("key must exist");
4045        assert_eq!(annotation.data().filter_key(&key).value(), Some(&DataValue::String("001".to_string())));
4046        Ok(())
4047    }
4048
4049    #[test]
4050    fn test_doubleslash_selector_root() -> Result<(), String> {
4051        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
4052        let mut store = stam::AnnotationStore::new(stam::Config::new());
4053        from_xml_in_memory("test", XMLEXAMPLE, &config, &mut store)?;
4054        assert!( store.annotation("h1").is_some());
4055        Ok(())
4056    }
4057
4058    #[test]
4059    fn test_doubleslash_selector_infix_none() -> Result<(), String> {
4060        let config = XmlConversionConfig::from_toml_str(CONF)?.with_debug(true);
4061        let mut store = stam::AnnotationStore::new(stam::Config::new());
4062        from_xml_in_memory("test", XMLEXAMPLE, &config, &mut store)?;
4063        assert!( store.annotation("h2").is_some());
4064        Ok(())
4065    }
4066
4067
4068}