libpostal-sys 0.1.1

Low-level wrappers for libpostal address normalization (with locks to support thread-safe initialization)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

#include "tokens.h"
#include "strndup.h"


tokenized_string_t *tokenized_string_new(void) {
    tokenized_string_t *self = malloc(sizeof(tokenized_string_t));
    self->str = NULL;
    self->strings = cstring_array_new();
    self->tokens = token_array_new();
    return self;
}

tokenized_string_t *tokenized_string_new_size(size_t len, size_t num_tokens) {
    tokenized_string_t *self = malloc(sizeof(tokenized_string_t));
    self->str = NULL;
    self->strings = cstring_array_new_size(len);
    self->tokens = token_array_new_size(len);

    return self;
}

inline tokenized_string_t *tokenized_string_new_from_str_size(char *src, size_t len, size_t num_tokens) {
    tokenized_string_t *self = tokenized_string_new_size(len, num_tokens);
    self->str = strndup(src, len);
    if (self->str == NULL) {
        tokenized_string_destroy(self);
        return NULL;
    }
    return self;
}


void tokenized_string_add_token(tokenized_string_t *self, const char *src, size_t len, uint16_t token_type, size_t position) {
    char *ptr = (char *) (src + position);

    cstring_array_add_string_len(self->strings, ptr, len);

    token_t token = (token_t){position, len, token_type};
    token_array_push(self->tokens, token);

}

tokenized_string_t *tokenized_string_from_tokens(char *src, token_array *tokens, bool copy_tokens) {
    tokenized_string_t *self = malloc(sizeof(tokenized_string_t));
    self->str = strdup(src);
    if (self->str == NULL) {
        tokenized_string_destroy(self);
        return NULL;
    }
    self->strings = cstring_array_new_size(strlen(src) + tokens->n);
    if (copy_tokens) {
        self->tokens = token_array_new_copy(tokens, tokens->n);
    } else {
        self->tokens = tokens;
    }

    token_t token;

    for (size_t i = 0; i < tokens->n; i++) {
        token = tokens->a[i];
        cstring_array_add_string_len(self->strings, src + token.offset, token.len);
    }
    return self;
}


char *tokenized_string_get_token(tokenized_string_t *self, uint32_t index) {
    if (index < self->tokens->n) {
        return cstring_array_get_string(self->strings, index);
    } else {
        return NULL;
    }
}

void tokenized_string_destroy(tokenized_string_t *self) {
    if (self == NULL) return;

    if (self->str != NULL) {
        free(self->str);
    }

    if (self->strings != NULL) {
        cstring_array_destroy(self->strings);
    }

    if (self->tokens != NULL) {
        token_array_destroy(self->tokens);
    }

    free(self);
}