Skip to main content

radio_cli/
directory.rs

1use crate::error::{DomainError, ErrorCode};
2use crate::{USER_AGENT, config::Config, model::Station, store::Store};
3use anyhow::{Context, Result, bail};
4use hickory_resolver::Resolver;
5use rand::seq::SliceRandom;
6use reqwest::blocking::Client;
7use std::{
8    sync::{
9        Arc,
10        atomic::{AtomicBool, Ordering},
11    },
12    time::Duration,
13};
14
15pub struct Directory {
16    client: Client,
17    servers: Vec<String>,
18    explicit: Option<String>,
19    stop: Arc<AtomicBool>,
20}
21
22impl Directory {
23    pub fn new(config: &Config, stop: Arc<AtomicBool>) -> Result<Self> {
24        let client = Client::builder()
25            .user_agent(USER_AGENT)
26            .connect_timeout(Duration::from_secs(config.settings.connect_timeout_secs))
27            .timeout(Duration::from_secs(config.settings.request_timeout_secs))
28            .build()?;
29        Ok(Self {
30            client,
31            servers: vec![],
32            explicit: config.settings.api_url.clone(),
33            stop,
34        })
35    }
36
37    fn discover(&mut self) -> Result<()> {
38        if !self.servers.is_empty() {
39            return Ok(());
40        }
41        if let Some(url) = &self.explicit {
42            self.servers.push(url.trim_end_matches('/').to_owned());
43            return Ok(());
44        }
45        let discovered = (|| -> Result<_> {
46            let runtime = tokio::runtime::Builder::new_current_thread()
47                .enable_all()
48                .build()?;
49            runtime.block_on(async {
50                let resolver = Resolver::builder_tokio()?.build()?;
51                Ok(tokio::time::timeout(
52                    Duration::from_secs(3),
53                    resolver.srv_lookup("_api._tcp.radio-browser.info."),
54                )
55                .await??)
56            })
57        })();
58        if let Ok(records) = discovered {
59            for answer in records.answers() {
60                let hickory_resolver::proto::rr::RData::SRV(record) = &answer.data else {
61                    continue;
62                };
63                let name = record.target.to_utf8().trim_end_matches('.').to_owned();
64                if name.ends_with(".api.radio-browser.info") {
65                    self.servers.push(format!("https://{name}"));
66                }
67            }
68        }
69        if self.servers.is_empty() {
70            #[derive(serde::Deserialize)]
71            struct Server {
72                name: String,
73            }
74            let rows: Vec<Server> = self
75                .client
76                .get("https://all.api.radio-browser.info/json/servers")
77                .send()?
78                .error_for_status()?
79                .json()?;
80            for row in rows {
81                if row.name.ends_with(".api.radio-browser.info") {
82                    self.servers.push(format!("https://{}", row.name));
83                }
84            }
85        }
86        self.servers.shuffle(&mut rand::rng());
87        if self.servers.is_empty() {
88            bail!(DomainError::new(ErrorCode::NoDirectoryServers));
89        }
90        Ok(())
91    }
92
93    fn get(&mut self, path: &str, params: &[(&str, String)]) -> Result<Vec<Station>> {
94        self.discover()?;
95        let mut last = None;
96        for server in self.servers.iter().take(3) {
97            if self.stop.load(Ordering::Relaxed) {
98                bail!(DomainError::new(ErrorCode::Cancelled));
99            }
100            let result = self.fetch(server, path, params);
101            match result {
102                Ok(mut rows) => {
103                    for row in &mut rows {
104                        row.active = true;
105                        row.custom = false;
106                    }
107                    return Ok(rows);
108                }
109                Err(e) => last = Some(e),
110            }
111        }
112        Err(last.context(DomainError::new(ErrorCode::DirectoryRequest))?)
113    }
114
115    fn fetch(&self, server: &str, path: &str, params: &[(&str, String)]) -> Result<Vec<Station>> {
116        if self.stop.load(Ordering::Relaxed) {
117            bail!(DomainError::new(ErrorCode::Cancelled));
118        }
119        let response = self
120            .client
121            .get(format!("{server}{path}"))
122            .query(params)
123            .send()?
124            .error_for_status()?;
125        use std::io::Read;
126        let mut bytes = Vec::new();
127        response
128            .take(16 * 1024 * 1024 + 1)
129            .read_to_end(&mut bytes)?;
130        if bytes.len() > 16 * 1024 * 1024 {
131            bail!(DomainError::new(ErrorCode::DirectoryResponseTooLarge));
132        }
133        let mut rows: Vec<Station> = serde_json::from_slice(&bytes)?;
134        for row in &mut rows {
135            row.active = true;
136            row.custom = false;
137        }
138        Ok(rows)
139    }
140
141    pub fn bootstrap(&mut self, store: &mut Store) -> Result<usize> {
142        let rows = self.get(
143            "/json/stations/search",
144            &[
145                ("limit", "1000".into()),
146                ("hidebroken", "true".into()),
147                ("order", "clickcount".into()),
148                ("reverse", "true".into()),
149            ],
150        )?;
151        let rows: Vec<_> = rows.into_iter().filter(valid_identity).collect();
152        store.bootstrap(&rows, || self.stop.load(Ordering::Relaxed))
153    }
154
155    pub fn sync(&mut self, store: &mut Store, progress: impl FnMut(usize)) -> Result<usize> {
156        self.sync_with_policy(store, progress, false)
157    }
158
159    pub fn sync_with_policy(
160        &mut self,
161        store: &mut Store,
162        mut progress: impl FnMut(usize),
163        accept_shrink: bool,
164    ) -> Result<usize> {
165        self.discover()?;
166        let previous = store.directory_count()?;
167        let mut failures = Vec::new();
168        let mut attempts = Vec::new();
169        // A failed page discards the entire staging snapshot before failover.
170        for server in self.servers.iter().take(3) {
171            store.begin_sync()?;
172            progress(0);
173            let mut offset = 0usize;
174            let mut invalid = 0usize;
175            let mut valid = 0usize;
176            let result = (|| -> Result<()> {
177                loop {
178                    let rows = self.fetch(
179                        server,
180                        "/json/stations/search",
181                        &[
182                            ("limit", "1000".into()),
183                            ("offset", offset.to_string()),
184                            ("hidebroken", "true".into()),
185                            ("order", "name".into()),
186                        ],
187                    )?;
188                    if rows.len() > 1000 {
189                        bail!(DomainError::new(ErrorCode::InvalidPagination));
190                    }
191                    let received = rows.len();
192                    if self.stop.load(Ordering::Relaxed) {
193                        bail!(DomainError::new(ErrorCode::Cancelled));
194                    }
195                    let rows: Vec<_> = rows.into_iter().filter(valid_identity).collect();
196                    valid += rows.len();
197                    invalid += received - rows.len();
198                    store.stage(&rows)?;
199                    offset += received;
200                    progress(offset);
201                    if received < 1000 {
202                        break;
203                    }
204                    if offset >= 200_000 {
205                        bail!(DomainError::new(ErrorCode::CatalogTooLarge));
206                    }
207                }
208                if store.incoming_count()? == 0 {
209                    bail!(DomainError::new(ErrorCode::EmptyCatalog));
210                }
211                Ok(())
212            })();
213            if self.stop.load(Ordering::Relaxed) {
214                bail!(DomainError::new(ErrorCode::Cancelled));
215            }
216            let count = store.incoming_count()?;
217            let suspicious = result.is_ok()
218                && previous > 0
219                && count.saturating_mul(4) < previous.saturating_mul(3);
220            let error = result.as_ref().err().map(|error| format!("{error:#}"));
221            let mut report = serde_json::json!({
222                "server":server, "raw":offset, "valid":valid, "invalid":invalid,
223                "duplicates":valid.saturating_sub(count), "unique":count, "previous":previous,
224                "shrink_requires_acceptance":suspicious, "accepted":accept_shrink, "error":error,
225            });
226            attempts.push(report.clone());
227            report["attempts"] = serde_json::json!(attempts);
228            store.set_meta("last_sync_attempt", &report.to_string())?;
229            match result {
230                Ok(()) => {
231                    if suspicious && !accept_shrink {
232                        failures.push(format!("{server}: {previous} → {count} (>25%)"));
233                        bail!(
234                            DomainError::new(ErrorCode::CatalogShrink)
235                                .with("servers", failures.join("; "))
236                        );
237                    }
238                    return store.finish_sync_cancellable(|| self.stop.load(Ordering::Relaxed));
239                }
240                Err(error) => failures.push(format!("{server}: {error:#}")),
241            }
242        }
243        bail!(DomainError::new(ErrorCode::DirectorySync).with("servers", failures.join("; ")))
244    }
245
246    pub fn refresh(&mut self, uuid: &str) -> Result<Option<Station>> {
247        uuid::Uuid::parse_str(uuid)?;
248        Ok(self
249            .get(&format!("/json/stations/byuuid/{uuid}"), &[])?
250            .into_iter()
251            .find(|row| row.stationuuid == uuid && valid_identity(row)))
252    }
253
254    pub fn click(&mut self, uuid: &str) -> Result<()> {
255        uuid::Uuid::parse_str(uuid)?;
256        self.discover()?;
257        self.client
258            .post(format!("{}/json/url/{uuid}", self.servers[0]))
259            .timeout(Duration::from_secs(3))
260            .send()?
261            .error_for_status()?;
262        Ok(())
263    }
264}
265
266fn valid_identity(station: &Station) -> bool {
267    !crate::clean_text(&station.name).trim().is_empty()
268        && uuid::Uuid::parse_str(&station.stationuuid).is_ok()
269}
270
271#[cfg(test)]
272mod tests {
273    use super::*;
274    use std::{
275        io::{Read, Write},
276        net::TcpListener,
277        thread,
278    };
279
280    fn server(responses: Vec<(u16, Vec<u8>)>) -> (String, thread::JoinHandle<Vec<String>>) {
281        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
282        let url = format!("http://{}", listener.local_addr().unwrap());
283        let handle = thread::spawn(move || {
284            let mut requests = Vec::new();
285            for (status, body) in responses {
286                let (mut stream, _) = listener.accept().unwrap();
287                stream
288                    .set_read_timeout(Some(Duration::from_secs(2)))
289                    .unwrap();
290                let mut bytes = [0; 8192];
291                let n = stream.read(&mut bytes).unwrap();
292                requests.push(String::from_utf8_lossy(&bytes[..n]).into());
293                write!(stream, "HTTP/1.1 {status} OK\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n", body.len()).unwrap();
294                stream.write_all(&body).unwrap();
295            }
296            requests
297        });
298        (url, handle)
299    }
300
301    #[test]
302    fn failover_restarts_the_whole_snapshot() {
303        let first = Station {
304            stationuuid: uuid::Uuid::new_v4().to_string(),
305            name: "Only A".into(),
306            ..Station::default()
307        };
308        let second = Station {
309            stationuuid: uuid::Uuid::new_v4().to_string(),
310            name: "Only B".into(),
311            ..Station::default()
312        };
313        let (a, a_handle) = server(vec![
314            (200, serde_json::to_vec(&vec![first.clone(); 1000]).unwrap()),
315            (500, b"{}".to_vec()),
316        ]);
317        let (b, b_handle) = server(vec![(
318            200,
319            serde_json::to_vec(&vec![second.clone()]).unwrap(),
320        )]);
321        let dir = tempfile::tempdir().unwrap();
322        let config = Config::load(None, Some(dir.path()), Some(&a)).unwrap();
323        let mut store = Store::open(&dir.path().join("radio.db")).unwrap();
324        let mut directory = Directory::new(&config, Arc::new(AtomicBool::new(false))).unwrap();
325        directory.servers = vec![a, b];
326        assert_eq!(directory.sync(&mut store, |_| {}).unwrap(), 1);
327        assert!(store.station(&first.stationuuid).is_err());
328        assert!(store.station(&second.stationuuid).unwrap().active);
329        assert!(a_handle.join().unwrap()[1].contains("offset=1000"));
330        assert!(b_handle.join().unwrap()[0].contains("offset=0"));
331    }
332}
333
334#[cfg(test)]
335mod policy_tests {
336    use super::*;
337    use crate::test_support::{self, Server, response};
338    use std::sync::atomic::AtomicUsize;
339
340    #[test]
341    fn shrink_rejection_does_not_fetch_another_full_replica() {
342        let row = Station {
343            stationuuid: uuid::Uuid::new_v4().to_string(),
344            name: "smaller".into(),
345            ..Station::default()
346        };
347        let body = serde_json::to_vec(&vec![row]).unwrap();
348        let requests = Arc::new(AtomicUsize::new(0));
349        let count = requests.clone();
350        let first = Server::new(move |_, stream| {
351            count.fetch_add(1, Ordering::Relaxed);
352            response(stream, 200, "application/json", &body);
353        });
354        let backup_requests = Arc::new(AtomicUsize::new(0));
355        let count = backup_requests.clone();
356        let second = Server::new(move |_, stream| {
357            count.fetch_add(1, Ordering::Relaxed);
358            response(stream, 200, "application/json", b"[]");
359        });
360        let dir = tempfile::tempdir().unwrap();
361        let config = test_support::config(dir.path());
362        let mut store = Store::open(&dir.path().join("radio.db")).unwrap();
363        for i in 0..8 {
364            store
365                .upsert(&Station {
366                    stationuuid: uuid::Uuid::new_v4().to_string(),
367                    name: format!("old {i}"),
368                    active: true,
369                    ..Station::default()
370                })
371                .unwrap();
372        }
373        let mut directory = Directory::new(&config, Arc::new(AtomicBool::new(false))).unwrap();
374        directory.servers = vec![first.url.clone(), second.url.clone()];
375        let error = directory.sync(&mut store, |_| {}).unwrap_err().to_string();
376        assert!(error.contains("accept-shrink"));
377        assert!(error.contains(&first.url));
378        assert_eq!(requests.load(Ordering::Relaxed), 1);
379        assert_eq!(backup_requests.load(Ordering::Relaxed), 0);
380        assert_eq!(store.count().unwrap(), 8);
381        assert_eq!(
382            directory
383                .sync_with_policy(&mut store, |_| {}, true)
384                .unwrap(),
385            1
386        );
387        assert_eq!(requests.load(Ordering::Relaxed), 2);
388        assert_eq!(backup_requests.load(Ordering::Relaxed), 0);
389    }
390
391    #[test]
392    fn exhausted_failover_reports_every_attempted_server() {
393        let first = Server::new(|_, stream| response(stream, 500, "application/json", b"{}"));
394        let second = Server::new(|_, stream| response(stream, 503, "application/json", b"{}"));
395        let dir = tempfile::tempdir().unwrap();
396        let config = test_support::config(dir.path());
397        let mut store = Store::open(&dir.path().join("radio.db")).unwrap();
398        let mut directory = Directory::new(&config, Arc::new(AtomicBool::new(false))).unwrap();
399        directory.servers = vec![first.url.clone(), second.url.clone()];
400        let error = directory.sync(&mut store, |_| {}).unwrap_err().to_string();
401        assert!(error.contains(&first.url));
402        assert!(error.contains(&second.url));
403        assert!(error.contains("500"));
404        assert!(error.contains("503"));
405        let report: serde_json::Value =
406            serde_json::from_str(&store.meta("last_sync_attempt").unwrap().unwrap()).unwrap();
407        assert_eq!(report["attempts"].as_array().unwrap().len(), 2);
408        assert_eq!(report["attempts"][0]["server"], first.url);
409        assert_eq!(report["attempts"][1]["server"], second.url);
410    }
411}