libpostal-sys 0.1.1

Low-level wrappers for libpostal address normalization (with locks to support thread-safe initialization)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

#ifndef SCANNER_H
#define SCANNER_H

 

#include <stdlib.h>
#include <stdint.h>
#include <stdbool.h>

#include "token_types.h"
#include "tokens.h"

typedef struct scanner {
    unsigned char *src, *cursor, *start, *end;
} scanner_t;

uint16_t scan_token(scanner_t *s);

scanner_t scanner_from_string(const char *input, size_t len);

void tokenize_add_tokens(token_array *tokens, const char *input, size_t len, bool keep_whitespace);
token_array *tokenize_keep_whitespace(const char *input);
token_array *tokenize(const char *input);


 

#endif