libpostal-sys 0.1.1

Low-level wrappers for libpostal address normalization (with locks to support thread-safe initialization)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

#include "ngrams.h"
#include "utf8proc/utf8proc.h"

bool add_ngrams(cstring_array *grams, size_t n, char *str, size_t len, bool prefix, bool suffix) {
    if (n == 0) return false;
    
    size_t lengths[n];
    size_t num_chars = 0;

    uint8_t *ptr = (uint8_t *)str;

    int32_t ch;

    size_t idx = 0;

    size_t gram_len = 0;
    size_t gram_offset = 0;
    size_t consumed = 0;

    size_t num_grams = 0;

    bool beginning = true;

    while (idx < len) {
        ssize_t char_len = utf8proc_iterate(ptr, len, &ch);
        if (char_len <= 0 || ch == 0) break;

        // Not at min characters yet
        if (num_chars < n) {
            lengths[num_chars] = (size_t)char_len;
            num_chars++;
            gram_len += char_len;
        }

        // We have a full gram of size n
        if (num_chars == n && (num_grams > 0 || idx + char_len < len)) {
            uint32_t token_index = cstring_array_start_token(grams);

            if (beginning) {
                beginning = false;
            } else {
                if (prefix) {
                    cstring_array_append_string(grams, "_");
                }
                gram_len -= lengths[0];
                gram_offset += lengths[0];
                gram_len += char_len;

                for (size_t i = 1; i < n; i++) {
                    lengths[i - 1] = lengths[i];
                }
                lengths[n - 1] = (size_t)char_len;
            }

            cstring_array_append_string_len(grams, str + gram_offset, gram_len);

            if (idx + char_len < len && suffix) {
                cstring_array_append_string(grams, "_");
            }

            cstring_array_terminate(grams);
            num_grams++;
        }

        idx += char_len;
        ptr += char_len;
        consumed += char_len;
    }

    return num_grams > 0;
}