libpostal-sys 0.1.1

Low-level wrappers for libpostal address normalization (with locks to support thread-safe initialization)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

#include "unicode_scripts.h"

#include "unicode_scripts_data.c"

#define MAX_ASCII 128

inline script_t get_char_script(uint32_t ch) {
    if (ch > NUM_CODEPOINTS - 1) return SCRIPT_UNKNOWN;
    return char_scripts[ch];
}

inline script_languages_t get_script_languages(script_t script) {
    return script_languages[script];
}

static inline bool is_common_script(script_t script) {
    return script == SCRIPT_COMMON || script == SCRIPT_INHERITED;
}

string_script_t get_string_script(char *str, size_t len) {
    int32_t ch;
    script_t last_script = SCRIPT_UNKNOWN;
    script_t script = SCRIPT_UNKNOWN;

    uint8_t *ptr = (uint8_t *)str;

    size_t script_len = 0;
    size_t idx = 0;

    bool is_ascii = true;

    while (idx < len) {
        ssize_t char_len = utf8proc_iterate(ptr, len, &ch);

        if (ch == 0) break;

        script = get_char_script((uint32_t)ch);

        if (is_common_script(script) && last_script != SCRIPT_UNKNOWN) {
            script = last_script;
        }

        if (last_script != script && last_script != SCRIPT_UNKNOWN && !is_common_script(last_script)) {
            if (script_len < len) {
                while (true) {
                    char_len = utf8proc_iterate_reversed((const uint8_t *)str, idx, &ch);
                    if (ch == 0) break;

                    script = get_char_script((uint32_t)ch);
                    if (!is_common_script(script)) {
                        break;
                    }

                    script_len -= char_len;
                    ptr -= char_len;
                    idx -= char_len;
                }
            }

            break;
        }

        is_ascii = is_ascii && ch < MAX_ASCII;

        ptr += char_len;
        idx += char_len;
        script_len += char_len;

        if (script != SCRIPT_UNKNOWN) {
            last_script = script;
        }
    
    }

    return (string_script_t) {last_script, script_len, is_ascii};
}