rete-cli 0.4.2

CLI to build and inspect Rete graph files.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153

//! How the CLI *says* it found invalid IRIs — the build-time warning and the
//! export-time sanitize summary.
//!
//! Both read the same [`IriReport`] (`rete_core::iri`), and both go to **stderr**:
//! `rete export` writes the dump to stdout, so a report on stdout would corrupt
//! the very file it is describing.
//!
//! The wording is deliberate on two points the issue this fixes turned on:
//!
//! * **A build warning is not a build failure.** Datasets that build today —
//!   including published ones — must keep building, so the default records the
//!   damage and continues. `--strict` is the flag for the other choice.
//! * **Sanitizing is lossy, and says so.** Percent-encoding changes the IRI, so
//!   a sanitized dump no longer joins against the graph it came from. The
//!   summary states that every time it does anything, rather than reading as a
//!   clean bill of health.
//!
//! # The `totals` line is a machine interface
//!
//! `report_sanitized` prints one line that is meant to be *parsed*:
//!
//! ```text
//! --sanitize-iris: totals invalid=11 repairable=11 unrepairable=0 unclassified=0
//! ```
//!
//! `scripts/export_scholar_nquads.sh` reads it to decide whether a dump may be
//! published, because `unrepairable` is the one number that answers "will a
//! strict loader take this". Prose wording may change; **this line's shape may
//! not** without changing that script in the same commit, or the script reads a
//! missing field as a zero and opens the gate on a broken dump.
//!
//! The per-class rows below it are a second machine surface: the script matches
//! a stable fragment of each `IriDefect::reason()`. A reason it does not
//! recognise is counted as unknown and warned about — never dropped.

use rete_core::iri::IriReport;

/// One indented `count  reason` row per non-empty class, with an example.
fn class_rows(report: &IriReport, indent: &str, with_examples: bool) {
    for (defect, count, sample) in report.classes() {
        let tail = if defect.repairable() {
            ""
        } else {
            "   [NOT repairable by escaping]"
        };
        eprintln!("{indent}{count:>7}  {}{tail}", defect.reason());
        if with_examples {
            if let Some(s) = sample {
                eprintln!("{indent}         e.g. {}", elide(s));
            }
        }
    }
}

/// Keep a sample short enough to stay on one line — some real offenders are
/// 400-character harvested PDF URLs.
fn elide(token: &str) -> String {
    const MAX: usize = 96;
    if token.chars().count() <= MAX {
        return token.to_string();
    }
    let head: String = token.chars().take(MAX - 1).collect();
    format!("{head}…")
}

/// The build-time warning: what was ingested, what it costs downstream, and the
/// two flags that do something about it. Silent when the input was clean.
pub(crate) fn warn_after_build(report: &IriReport) {
    if report.is_empty() {
        return;
    }
    eprintln!(
        "warning: {} statement(s) carry an invalid IRI ({} IRI occurrence(s)).",
        report.statements(),
        report.occurrences()
    );
    eprintln!(
        "         They are stored verbatim, so `rete export --format nq` emits N-Quads that a"
    );
    eprintln!(
        "         strict parser (Oxigraph, Jena, rapper) rejects — and a bulk loader rejects the"
    );
    eprintln!("         whole chunk, not the line.");
    class_rows(report, "         ", true);
    eprintln!("         `rete export --sanitize-iris` percent-encodes the repairable ones (which");
    eprintln!("         changes the IRI); `rete build --strict` refuses such input outright.");
}

/// The export-time summary for `--sanitize-iris`. Unlike the build warning this
/// prints even when it found nothing — the flag was asked for explicitly, and
/// "nothing needed changing" is the answer to that question.
pub(crate) fn report_sanitized(report: &IriReport) {
    if report.is_empty() {
        eprintln!("--sanitize-iris: totals invalid=0 repairable=0 unrepairable=0 unclassified=0");
        eprintln!(
            "--sanitize-iris: no invalid IRIs found; the dump is byte-identical to a plain export."
        );
        return;
    }
    // Parsed downstream — see the module docs. Printed first so it is there even
    // when a later line is truncated.
    eprintln!(
        "--sanitize-iris: totals invalid={} repairable={} unrepairable={} unclassified={}",
        report.occurrences(),
        report.repairable(),
        report.unrepairable(),
        report.unclassified()
    );
    if report.repaired() > 0 {
        eprintln!(
            "--sanitize-iris: percent-encoded {} IRI occurrence(s). The dump's IRIs are NOT the",
            report.repaired()
        );
        eprintln!("                 file's IRIs: it no longer joins against the source graph, and");
        eprintln!("                 rete → store → rete is no longer the identity.");
        for (defect, count, _) in report.classes().filter(|(d, _, _)| d.repairable()) {
            eprintln!("                 {count:>7}  {}", defect.reason());
        }
    }
    if report.unrepairable() > 0 {
        eprintln!(
            "--sanitize-iris: {} occurrence(s) CANNOT be repaired by escaping and were written",
            report.unrepairable()
        );
        eprintln!(
            "                 verbatim — this dump is still not valid N-Quads. Fix them at the"
        );
        eprintln!(
            "                 source; a relative IRI needs a base IRI the file never recorded."
        );
        for (defect, count, sample) in report.classes().filter(|(d, _, _)| !d.repairable()) {
            eprintln!("                 {count:>7}  {}", defect.reason());
            if let Some(s) = sample {
                eprintln!("                          e.g. {}", elide(s));
            }
        }
        if report.unclassified() > 0 {
            eprintln!(
                "                 The unclassified ones are invalid by RFC 3987 — the parser this"
            );
            eprintln!(
                "                 tool shares with Oxigraph's N-Triples reader refused them — but"
            );
            eprintln!(
                "                 match no repair rete knows, so nothing was changed. `<https://::1>`"
            );
            eprintln!(
                "                 (an IPv6 host missing its brackets) is the shape that taught us to"
            );
            eprintln!("                 count these rather than pass them.");
        }
    }
}