content-canonical 0.1.0

Content canonicalization and text normalization library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

use canonical::{
    canonicalize, collapse_whitespace, hash_text, tokenize, CanonicalizeConfig,
    CanonicalizedDocument,
};

fn main() {
    let raw_text = "  Café\tRUST!  ";

    let collapsed = collapse_whitespace(raw_text);
    println!("collapsed -> \"{collapsed}\"");

    let cfg = CanonicalizeConfig {
        strip_punctuation: true,
        ..Default::default()
    };

    let CanonicalizedDocument {
        doc_id: _,
        canonical_text,
        tokens: _tokens,
        sha256_hex,
        ..
    } = canonicalize("demo-helper", &collapsed, &cfg).expect("canonicalization succeeds");

    println!("canonical text -> \"{canonical_text}\"");
    println!("sha256 -> {sha256_hex}");

    let helper_tokens = tokenize(&canonical_text);
    println!("tokens via helper -> {helper_tokens:?}");

    let checksum_again = hash_text(&canonical_text);
    println!("hash_text helper -> {checksum_again}");
}