duckling 0.4.0

A Rust port of Facebook's Duckling library for parsing natural language into structured data
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use crate::pattern::regex;
use crate::types::{Rule, TokenData};

use super::UrlData;

pub fn rules() -> Vec<Rule> {
    vec![
        // Main URL rule: matches URLs with domain.tld pattern
        // Ported from Haskell ruleURL
        Rule {
            name: "url".to_string(),
            pattern: vec![regex(
                r"((([a-zA-Z]+)://)?(w{2,3}[0-9]*\.)?(([a-zA-Z0-9_-]+\.)+[a-z]{2,4})(:\d+)?(/[^?\s#]*)?(\?[^\s#]+)?(#[-,*=&a-zA-Z0-9]+)?)",
            )],
            production: Box::new(|nodes| {
                let m = match &nodes[0].token_data {
                    TokenData::RegexMatch(m) => m,
                    _ => return None,
                };
                let full = m.group(1)?;
                let domain = m.group(5)?;
                Some(TokenData::Url(UrlData::new(full, &domain.to_lowercase())))
            }),
        },
        // Localhost rule
        // Ported from Haskell ruleLocalhost
        Rule {
            name: "localhost".to_string(),
            pattern: vec![regex(
                r"((([a-zA-Z]+)://)?localhost(:\d+)?(/[^?\s#]*)?(\?[^\s#]+)?)",
            )],
            production: Box::new(|nodes| {
                let m = match &nodes[0].token_data {
                    TokenData::RegexMatch(m) => m,
                    _ => return None,
                };
                let full = m.group(1)?;
                Some(TokenData::Url(UrlData::new(full, "localhost")))
            }),
        },
        // Local URL rule: protocol://hostname (no TLD required)
        // Ported from Haskell ruleLocalURL
        Rule {
            name: "local url".to_string(),
            pattern: vec![regex(
                r"(([a-zA-Z]+)://([a-zA-Z0-9_-]+)(:\d+)?(/[^?\s#]*)?(\?[^\s#]+)?)",
            )],
            production: Box::new(|nodes| {
                let m = match &nodes[0].token_data {
                    TokenData::RegexMatch(m) => m,
                    _ => return None,
                };
                let full = m.group(1)?;
                let domain = m.group(3)?;
                Some(TokenData::Url(UrlData::new(full, &domain.to_lowercase())))
            }),
        },
    ]
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::engine;
    use crate::resolve::{Context, Options};
    use crate::types::DimensionKind;

    #[test]
    fn test_urls() {
        let rules = rules();
        let options = Options { with_latent: false };
        let context = Context::default();

        for (text, expected_domain) in &[
            ("https://www.example.com", "example.com"),
            ("http://google.com/search?q=test", "google.com"),
            ("www.github.com", "github.com"),
            ("cnn.com/info", "cnn.com"),
        ] {
            let entities =
                engine::parse_and_resolve(text, &rules, &context, &options, &[DimensionKind::Url]);
            let found = entities.iter().any(|e| {
                matches!(&e.value, crate::types::DimensionValue::Url { domain, .. } if domain == *expected_domain)
            });
            assert!(
                found,
                "Expected URL with domain '{}' for '{}', got: {:?}",
                expected_domain, text, entities
            );
        }
    }
}