geolite 0.0.17

open source geocode solution
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

// sqlite JSONB binary encoder. spec: https://sqlite.org/jsonb.html
//
// each element = [header byte][optional extra length bytes][payload].
// header byte = SSSSTTTT where TTTT is type, SSSS is size class:
//   0..=11 : literal payload length
//   12     : 1-byte length follows
//   13     : 2-byte length follows (big-endian)
//   14     : 4-byte length follows
//   15     : 8-byte length follows
//
// types used:
//   3  INT     — text decimal payload ("42")
//   5  FLOAT   — text decimal payload ("3.14")
//   10 TEXTRAW — raw utf-8 bytes (sqlite adds JSON escapes on output)
//   11 ARRAY   — concatenated elements
//   12 OBJECT  — alternating key/value elements

const TYPE_INT: u8 = 3;
const TYPE_FLOAT: u8 = 5;
const TYPE_TEXTRAW: u8 = 10;
const TYPE_ARRAY: u8 = 11;
const TYPE_OBJECT: u8 = 12;

fn write_header(out: &mut Vec<u8>, jsonb_type: u8, payload_len: usize) {
  if payload_len <= 11 {
    out.push(((payload_len as u8) << 4) | jsonb_type);
  } else if payload_len <= 0xFF {
    out.push((12u8 << 4) | jsonb_type);
    out.push(payload_len as u8);
  } else if payload_len <= 0xFFFF {
    out.push((13u8 << 4) | jsonb_type);
    out.extend_from_slice(&(payload_len as u16).to_be_bytes());
  } else if payload_len <= 0xFFFF_FFFF {
    out.push((14u8 << 4) | jsonb_type);
    out.extend_from_slice(&(payload_len as u32).to_be_bytes());
  } else {
    out.push((15u8 << 4) | jsonb_type);
    out.extend_from_slice(&(payload_len as u64).to_be_bytes());
  }
}

pub(crate) fn write_int(out: &mut Vec<u8>, n: i64) {
  let s = n.to_string();
  write_header(out, TYPE_INT, s.len());
  out.extend_from_slice(s.as_bytes());
}

pub(crate) fn write_float(out: &mut Vec<u8>, f: f64) {
  let s = format!("{f}");
  write_header(out, TYPE_FLOAT, s.len());
  out.extend_from_slice(s.as_bytes());
}

pub(crate) fn write_text(out: &mut Vec<u8>, s: &str) {
  let bytes = s.as_bytes();
  write_header(out, TYPE_TEXTRAW, bytes.len());
  out.extend_from_slice(bytes);
}

pub struct encoder {
  // pool of reusable scratch buffers, popped on alloc and pushed back on free: the payloads
  // written today nest at most three levels (an object holding an array of objects), so three
  // scratches are active at once and no row allocates after warmup
  scratches: Vec<Vec<u8>>,
}

impl encoder {
  pub fn new() -> Self {
    Self {
      scratches: Vec::with_capacity(4),
    }
  }

  fn alloc(&mut self) -> Vec<u8> {
    self
      .scratches
      .pop()
      .unwrap_or_else(|| Vec::with_capacity(256))
  }

  fn free(&mut self, mut buf: Vec<u8>) {
    buf.clear();
    self.scratches.push(buf);
  }

  pub(crate) fn write_object<F>(&mut self, out: &mut Vec<u8>, f: F)
  where
    F: FnOnce(&mut encoder, &mut Vec<u8>),
  {
    let mut scratch = self.alloc();
    f(self, &mut scratch);
    write_header(out, TYPE_OBJECT, scratch.len());
    out.extend_from_slice(&scratch);
    self.free(scratch);
  }

  pub(crate) fn write_array<F>(&mut self, out: &mut Vec<u8>, f: F)
  where
    F: FnOnce(&mut encoder, &mut Vec<u8>),
  {
    let mut scratch = self.alloc();
    f(self, &mut scratch);
    write_header(out, TYPE_ARRAY, scratch.len());
    out.extend_from_slice(&scratch);
    self.free(scratch);
  }

}