geolite 0.0.3

open source geocode solution
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202

use std::{
  io::{Read, Write},
  net::{TcpListener, TcpStream},
  path::Path,
  sync::{Arc, Mutex},
};

use super::{geofabrik, list_local, resolve_geofabrik_url};

fn start_json_server(body: String) -> String {
  start_recording_json_server(body).0
}

// serves the json index and records the raw text of every request received, so
// scenarios can assert on request count and on request headers.
fn start_recording_json_server(body: String) -> (String, Arc<Mutex<Vec<String>>>) {
  let listener = TcpListener::bind("127.0.0.1:0").unwrap();
  let port = listener.local_addr().unwrap().port();
  let url = format!("http://127.0.0.1:{port}/index.json");
  let body = Arc::new(body);
  let requests: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new()));
  let requests_srv = requests.clone();
  std::thread::spawn(move || {
    for stream in listener.incoming().flatten() {
      let body = Arc::clone(&body);
      let requests = Arc::clone(&requests_srv);
      std::thread::spawn(move || serve_json(stream, &body, &requests));
    }
  });
  (url, requests)
}

fn serve_json(mut stream: TcpStream, body: &str, requests: &Mutex<Vec<String>>) {
  let mut buf = [0u8; 4096];
  let n = stream.read(&mut buf).unwrap_or(0);
  // records before responding, so a returned call always sees its own request.
  requests
    .lock()
    .unwrap()
    .push(String::from_utf8_lossy(&buf[..n]).to_string());
  let response = format!(
    "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
    body.len(),
    body
  );
  let _ = stream.write_all(response.as_bytes());
}

fn tmp(name: &str) -> std::path::PathBuf {
  let p = std::env::temp_dir().join(name);
  let _ = std::fs::remove_dir_all(&p);
  std::fs::create_dir_all(&p).unwrap();
  p
}

fn sqlite(name: &str) -> String {
  tmp(name).join("test.sqlite3").to_str().unwrap().to_string()
}

// 00: fetches index from http, persists rows in osm_pbf_files,
// and returns items sorted by id.
#[test]
fn _00_fetches_and_caches_when_no_cache_exists() {
  let db = sqlite("ls_t00");
  let json = r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"id":"europe/germany","name":"Germany","urls":{"pbf":"http://example.com/germany.osm.pbf"}}},{"type":"Feature","properties":{"id":"europe/france","name":"France","urls":{"pbf":"http://example.com/france.osm.pbf"}}}]}"#.to_string();
  let url = start_json_server(json);
  let items = geofabrik(&db, false, &url);
  assert_eq!(items.len(), 2);
  assert_eq!(items[0].id, "europe/france");
  assert_eq!(items[0].url, "http://example.com/france.osm.pbf");
  assert_eq!(items[1].id, "europe/germany");
  assert!(Path::new(&db).exists());
}

// 01: second call with recreate_cache=false reads from sqlite, no http.
#[test]
fn _01_returns_cached_without_refetching() {
  let db = sqlite("ls_t01");
  let json = r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"id":"asia/japan","name":"Japan","urls":{"pbf":"http://example.com/japan.osm.pbf"}}}]}"#.to_string();
  let (url, requests) = start_recording_json_server(json);
  geofabrik(&db, false, &url);
  let after_first = requests.lock().unwrap().len();
  geofabrik(&db, false, &url);
  assert_eq!(requests.lock().unwrap().len(), after_first);
}

// 02: recreate_cache=true forces a new http fetch even when cache exists.
#[test]
fn _02_recreate_cache_forces_refetch() {
  let db = sqlite("ls_t02");
  let json = r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"id":"africa/kenya","name":"Kenya","urls":{"pbf":"http://example.com/kenya.osm.pbf"}}}]}"#.to_string();
  let (url, requests) = start_recording_json_server(json);
  geofabrik(&db, false, &url);
  let after_first = requests.lock().unwrap().len();
  geofabrik(&db, true, &url);
  assert!(requests.lock().unwrap().len() > after_first);
}

// 03: feature with no pbf url appears in output with "-" as url.
#[test]
fn _03_feature_without_pbf_url_has_dash_url() {
  let db = sqlite("ls_t03");
  let json = r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"id":"oceania/australia","name":"Australia"}}]}"#.to_string();
  let url = start_json_server(json);
  let items = geofabrik(&db, false, &url);
  assert_eq!(items.len(), 1);
  assert_eq!(items[0].url, "-");
}

// 04: resolve_geofabrik_url returns the pbf url for a known id.
#[test]
fn _04_resolve_geofabrik_url_returns_url_for_known_id() {
  let db = sqlite("ls_t04");
  let json = r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"id":"south-america/brazil","name":"Brazil","urls":{"pbf":"http://example.com/brazil.osm.pbf"}}}]}"#.to_string();
  let url = start_json_server(json);
  let result = resolve_geofabrik_url(&db, "south-america/brazil", &url);
  assert_eq!(result.as_deref(), Some("http://example.com/brazil.osm.pbf"));
}

// 05: resolve_geofabrik_url returns none for an unknown id.
#[test]
fn _05_resolve_geofabrik_url_returns_none_for_unknown_id() {
  let db = sqlite("ls_t05");
  let json = r#"{"type":"FeatureCollection","features":[]}"#.to_string();
  let url = start_json_server(json);
  let result = resolve_geofabrik_url(&db, "nonexistent/region", &url);
  assert!(result.is_none());
}

// 06: resolve_geofabrik_url returns none when the entry has no pbf url.
#[test]
fn _06_resolve_geofabrik_url_returns_none_for_entry_without_url() {
  let db = sqlite("ls_t06");
  let json = r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"id":"oceania/australia","name":"Australia"}}]}"#.to_string();
  let url = start_json_server(json);
  let result = resolve_geofabrik_url(&db, "oceania/australia", &url);
  assert!(result.is_none());
}

// 07: list_local returns empty vec when pbf dir does not exist.
#[test]
fn _07_list_local_returns_empty_when_no_pbf_dir() {
  let dir = tmp("ls_t07");
  let items = list_local(dir.to_str().unwrap());
  assert!(items.is_empty());
}

// 08: list_local returns pbf files from data_path root sorted by path.
#[test]
fn _08_list_local_returns_sorted_pbf_files() {
  let dir = tmp("ls_t08");
  std::fs::write(dir.join("brazil.osm.pbf"), b"fake").unwrap();
  std::fs::write(dir.join("andorra.osm.pbf"), b"fake content longer").unwrap();
  std::fs::write(dir.join("readme.txt"), b"ignored").unwrap();
  let items = list_local(dir.to_str().unwrap());
  assert_eq!(items.len(), 2);
  assert_eq!(items[0].path.file_name().unwrap(), "andorra.osm.pbf");
  assert_eq!(items[1].path.file_name().unwrap(), "brazil.osm.pbf");
  assert_eq!(items[0].size_bytes, 19);
  assert!(Path::new(&items[0].path).exists());
}

// 09: resolve_geofabrik_url serves a cached url without refetching.
#[test]
fn _09_resolve_geofabrik_url_uses_cache_without_refetching() {
  let db = sqlite("ls_t09");
  let json = r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"id":"europe/spain","name":"Spain","urls":{"pbf":"http://example.com/spain.osm.pbf"}}}]}"#.to_string();
  let (url, requests) = start_recording_json_server(json);
  geofabrik(&db, false, &url);
  let after_geofabrik = requests.lock().unwrap().len();
  let result = resolve_geofabrik_url(&db, "europe/spain", &url);
  assert_eq!(result.as_deref(), Some("http://example.com/spain.osm.pbf"));
  assert_eq!(requests.lock().unwrap().len(), after_geofabrik);
}

// 10: list_local returns empty vec when the data_path does not exist.
#[test]
fn _10_list_local_returns_empty_for_nonexistent_path() {
  let dir = tmp("ls_t10");
  let missing = dir.join("does_not_exist");
  let items = list_local(missing.to_str().unwrap());
  assert!(items.is_empty());
}

// 11: outgoing requests carry the custom user-agent with the binary version.
#[test]
fn _11_sends_custom_user_agent_header() {
  let db = sqlite("ls_t11");
  let json = r#"{"type":"FeatureCollection","features":[{"type":"Feature","properties":{"id":"europe/spain","name":"Spain","urls":{"pbf":"http://example.com/spain.osm.pbf"}}}]}"#.to_string();
  let (url, requests) = start_recording_json_server(json);
  geofabrik(&db, false, &url);
  let requests = requests.lock().unwrap();
  let user_agent = requests[0]
    .lines()
    .find(|l| l.to_lowercase().starts_with("user-agent:"))
    .and_then(|l| l.split_once(':').map(|x| x.1))
    .map(|v| v.trim().to_string());
  assert_eq!(
    user_agent,
    Some(format!("geolite/{}", env!("CARGO_PKG_VERSION")))
  );
}