1use std::collections::HashSet;
2use std::sync::atomic::{AtomicUsize, Ordering};
3
4use crate::db::connection::Database;
5use crate::db::queries::{self, TrackMeta};
6use crate::remote::client::{SubsonicAlbumFull, SubsonicClient};
7
8use rayon::prelude::*;
9use rusqlite::params;
10use thiserror::Error;
11
12#[derive(Debug, Error)]
13pub enum SyncError {
14 #[error("subsonic error: {0}")]
15 Subsonic(#[from] super::client::SubsonicError),
16 #[error("db error: {0}")]
17 Db(#[from] crate::db::connection::DbError),
18}
19
20#[derive(Debug, Default)]
21pub struct SyncResult {
22 pub artists_synced: usize,
23 pub albums_synced: usize,
24 pub tracks_synced: usize,
25 pub albums_failed: usize,
28}
29
30impl SyncResult {
31 pub fn is_complete(&self) -> bool {
33 self.albums_failed == 0
34 }
35}
36
37pub fn get_last_sync(
39 db: &Database,
40 url: &str,
41) -> Result<Option<i64>, crate::db::connection::DbError> {
42 let result = db.conn.query_row(
43 "SELECT last_sync FROM remote_servers WHERE url = ?1",
44 params![url],
45 |row| row.get::<_, Option<i64>>(0),
46 );
47 match result {
48 Ok(ts) => Ok(ts),
49 Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
50 Err(e) => Err(e.into()),
51 }
52}
53
54pub fn update_last_sync(
56 db: &Database,
57 url: &str,
58 username: &str,
59 timestamp: i64,
60) -> Result<(), crate::db::connection::DbError> {
61 db.conn.execute(
62 "INSERT INTO remote_servers (url, username, last_sync)
63 VALUES (?1, ?2, ?3)
64 ON CONFLICT(url) DO UPDATE SET last_sync = ?3",
65 params![url, username, timestamp],
66 )?;
67 Ok(())
68}
69
70fn parse_iso8601_to_unix(s: &str) -> Option<i64> {
78 use chrono::{DateTime, FixedOffset, NaiveDateTime};
79
80 if let Ok(dt) = DateTime::parse_from_rfc3339(s) {
82 return Some(dt.timestamp());
83 }
84
85 if let Ok(naive) = NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f") {
88 return Some(naive.and_utc().timestamp());
89 }
90 if let Ok(naive) = NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S") {
91 return Some(naive.and_utc().timestamp());
92 }
93
94 if let Ok(dt) = DateTime::<FixedOffset>::parse_from_str(s, "%Y-%m-%d %H:%M:%S%:z") {
96 return Some(dt.timestamp());
97 }
98 if let Ok(naive) = NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") {
99 return Some(naive.and_utc().timestamp());
100 }
101
102 None
103}
104
105pub fn sync_library(
121 db: &Database,
122 client: &SubsonicClient,
123 full: bool,
124 server_url: &str,
125 username: &str,
126) -> Result<SyncResult, SyncError> {
127 let mut result = SyncResult::default();
128
129 let artists = client.get_artists()?;
130 result.artists_synced = artists.len();
131 log::info!("syncing {} artists from remote", artists.len());
132
133 let last_sync = if full {
134 None
135 } else {
136 get_last_sync(db, server_url)?
137 };
138
139 match last_sync {
140 Some(ts) => log::info!("incremental sync (albums created after {})", ts),
141 None => log::info!("full sync"),
142 }
143
144 let sync_start = std::time::SystemTime::now()
145 .duration_since(std::time::UNIX_EPOCH)
146 .unwrap_or_default()
147 .as_secs() as i64;
148
149 let mut offset = 0u32;
150 let page_size = 500u32;
151 let mut seen_ids: HashSet<String> = HashSet::new();
154
155 loop {
156 let page = client.get_album_list("alphabeticalByName", page_size, offset)?;
157 if page.is_empty() {
158 break;
159 }
160 let page_count = page.len();
161 offset += page_count as u32;
162
163 let to_fetch: Vec<String> = page
167 .into_iter()
168 .filter(|a| seen_ids.insert(a.id.clone()))
169 .filter(|a| match last_sync {
170 None => true,
171 Some(ts) => a
172 .created
173 .as_deref()
174 .and_then(parse_iso8601_to_unix)
175 .is_none_or(|created| created >= ts),
176 })
177 .map(|a| a.id)
178 .collect();
179
180 if !to_fetch.is_empty() {
181 let failures = AtomicUsize::new(0);
182 let fetched: Vec<SubsonicAlbumFull> = to_fetch
183 .into_par_iter()
184 .filter_map(|id| match client.get_album(&id) {
185 Ok(full) => Some(full),
186 Err(e) => {
187 log::warn!("failed to fetch album {}: {}", id, e);
188 failures.fetch_add(1, Ordering::Relaxed);
189 None
190 }
191 })
192 .collect();
193 result.albums_failed += failures.into_inner();
194
195 write_albums(db, client, &fetched, &mut result)?;
196 }
197
198 log::info!(
199 "synced {} albums ({} tracks) so far...",
200 result.albums_synced,
201 result.tracks_synced
202 );
203
204 if (page_count as u32) < page_size {
205 break;
206 }
207 }
208
209 if result.is_complete() {
210 update_last_sync(db, server_url, username, sync_start)?;
211 } else {
212 log::warn!(
213 "{} album(s) failed to fetch — leaving last_sync unchanged so the next sync retries them",
214 result.albums_failed
215 );
216 }
217
218 log::info!(
219 "sync complete: {} artists, {} albums, {} tracks, {} failed",
220 result.artists_synced,
221 result.albums_synced,
222 result.tracks_synced,
223 result.albums_failed,
224 );
225
226 Ok(result)
227}
228
229fn write_albums(
231 db: &Database,
232 client: &SubsonicClient,
233 albums: &[SubsonicAlbumFull],
234 result: &mut SyncResult,
235) -> Result<(), SyncError> {
236 db.conn
237 .execute_batch("BEGIN")
238 .map_err(crate::db::connection::DbError::from)?;
239
240 for album in albums {
241 result.albums_synced += 1;
242 let artist_name = album.artist.as_deref().unwrap_or("Unknown Artist");
243
244 for song in &album.song {
245 let meta = TrackMeta {
246 title: song.title.clone(),
247 artist: song
248 .artist
249 .clone()
250 .unwrap_or_else(|| artist_name.to_string()),
251 album_artist: album.artist.clone(),
252 album: album.name.clone(),
253 date: album.year.map(|y| y.to_string()),
254 disc: song.disc_number,
255 track_number: song.track,
256 genre: song.genre.clone().or_else(|| album.genre.clone()),
257 label: None,
258 duration_ms: song.duration.map(|d| d * 1000),
259 codec: song.suffix.clone(),
260 sample_rate: None,
261 bit_depth: None,
262 channels: None,
263 bitrate: song.bit_rate,
264 size_bytes: None,
265 mtime: None,
266 path: None,
267 source: "remote".to_string(),
268 remote_id: Some(song.id.clone()),
269 remote_url: Some(client.stream_url_template(&song.id)),
270 album_remote_id: Some(album.id.clone()),
271 artist_remote_id: album.artist_id.clone(),
272 album_added_at: album.created.clone(),
273 };
274
275 match queries::upsert_track(&db.conn, &meta) {
276 Ok(_) => result.tracks_synced += 1,
277 Err(e) => log::warn!("failed to insert remote track {}: {}", song.title, e),
278 }
279 }
280 }
281
282 db.conn
283 .execute_batch("COMMIT")
284 .map_err(crate::db::connection::DbError::from)?;
285
286 Ok(())
287}
288
289#[cfg(test)]
290mod tests {
291 use super::*;
292
293 #[test]
294 fn parse_rfc3339_with_z() {
295 assert_eq!(
297 parse_iso8601_to_unix("2024-01-15T10:30:00Z"),
298 Some(1705314600)
299 );
300 }
301
302 #[test]
303 fn parse_rfc3339_with_offset() {
304 assert_eq!(
306 parse_iso8601_to_unix("2024-01-15T10:30:00+05:30"),
307 Some(1705294800)
308 );
309 }
310
311 #[test]
312 fn parse_rfc3339_negative_offset() {
313 assert_eq!(
315 parse_iso8601_to_unix("2024-01-15T10:30:00-05:00"),
316 Some(1705332600)
317 );
318 }
319
320 #[test]
321 fn parse_fractional_seconds_z() {
322 assert_eq!(
323 parse_iso8601_to_unix("2024-01-15T10:30:00.123Z"),
324 Some(1705314600)
325 );
326 }
327
328 #[test]
329 fn parse_fractional_seconds_offset() {
330 assert_eq!(
331 parse_iso8601_to_unix("2024-01-15T10:30:00.999+00:00"),
332 Some(1705314600)
333 );
334 }
335
336 #[test]
337 fn parse_no_timezone_assumes_utc() {
338 assert_eq!(
339 parse_iso8601_to_unix("2024-01-15T10:30:00"),
340 Some(1705314600)
341 );
342 }
343
344 #[test]
345 fn parse_no_timezone_fractional() {
346 assert_eq!(
347 parse_iso8601_to_unix("2024-01-15T10:30:00.500"),
348 Some(1705314600)
349 );
350 }
351
352 #[test]
353 fn parse_space_separator_with_tz() {
354 assert_eq!(
355 parse_iso8601_to_unix("2024-01-15 10:30:00+00:00"),
356 Some(1705314600)
357 );
358 }
359
360 #[test]
361 fn parse_space_separator_no_tz() {
362 assert_eq!(
363 parse_iso8601_to_unix("2024-01-15 10:30:00"),
364 Some(1705314600)
365 );
366 }
367
368 #[test]
369 fn parse_garbage_returns_none() {
370 assert_eq!(parse_iso8601_to_unix("not-a-date"), None);
371 assert_eq!(parse_iso8601_to_unix(""), None);
372 assert_eq!(parse_iso8601_to_unix("2024"), None);
373 }
374
375 #[test]
376 fn parse_epoch() {
377 assert_eq!(parse_iso8601_to_unix("1970-01-01T00:00:00Z"), Some(0));
378 }
379
380 use crate::db::connection::Database;
383 use crate::db::queries;
384 use std::sync::{Arc, Mutex};
385
386 fn test_db() -> (Database, tempfile::TempDir) {
387 let dir = tempfile::tempdir().unwrap();
388 let db = Database::open(&dir.path().join("sync_test.db")).unwrap();
389 (db, dir)
390 }
391
392 fn remote_track_meta(remote_id: &str, title: &str, artist: &str, album: &str) -> TrackMeta {
394 TrackMeta {
395 title: title.into(),
396 artist: artist.into(),
397 album_artist: Some(artist.into()),
398 album: album.into(),
399 date: Some("2024".into()),
400 disc: Some(1),
401 track_number: Some(1),
402 genre: Some("Electronic".into()),
403 label: None,
404 duration_ms: Some(240_000),
405 codec: Some("FLAC".into()),
406 sample_rate: None,
407 bit_depth: None,
408 channels: None,
409 bitrate: Some(1000),
410 size_bytes: None,
411 mtime: None,
412 path: None,
413 source: "remote".into(),
414 remote_id: Some(remote_id.into()),
415 remote_url: Some(format!("https://example.com/stream?id={}", remote_id)),
416 album_remote_id: Some(format!("album-of-{remote_id}")),
417 artist_remote_id: Some(format!("artist-of-{remote_id}")),
418 album_added_at: None,
419 }
420 }
421
422 #[test]
426 fn a_sync_records_the_album_and_artist_ids_too() {
427 let (db, _dir) = test_db();
428
429 let meta = remote_track_meta("remote-100", "Enter", "Russian Circles", "Enter");
430 queries::upsert_track(&db.conn, &meta).unwrap();
431
432 let album: Option<String> = db
433 .conn
434 .query_row(
435 "SELECT remote_id FROM albums WHERE title = 'Enter'",
436 [],
437 |r| r.get(0),
438 )
439 .unwrap();
440 assert_eq!(album.as_deref(), Some("album-of-remote-100"));
441
442 let artist: Option<String> = db
443 .conn
444 .query_row(
445 "SELECT remote_id FROM artists WHERE name = 'Russian Circles'",
446 [],
447 |r| r.get(0),
448 )
449 .unwrap();
450 assert_eq!(artist.as_deref(), Some("artist-of-remote-100"));
451 }
452
453 #[test]
454 fn sync_upserts_tracks_to_database() {
455 let (db, _dir) = test_db();
456
457 let meta = remote_track_meta("remote-001", "Vordhosbn", "Aphex Twin", "Drukqs");
458 let track_id = queries::upsert_track(&db.conn, &meta).unwrap();
459 assert!(track_id > 0, "upsert should return a valid track ID");
460
461 let row = queries::get_track_row(&db.conn, track_id)
463 .unwrap()
464 .expect("track should exist in DB");
465 assert_eq!(row.title, "Vordhosbn");
466 assert_eq!(row.artist_name, "Aphex Twin");
467 assert_eq!(row.album_title, "Drukqs");
468 assert_eq!(row.remote_id.as_deref(), Some("remote-001"));
469 assert_eq!(row.source, "remote");
470 }
471
472 struct StubServer {
477 addr: std::net::SocketAddr,
478 shutdown: Arc<std::sync::atomic::AtomicBool>,
479 }
480
481 #[derive(Default)]
482 struct StubState {
483 albums: Mutex<Vec<(String, String, String)>>,
485 failing: Mutex<HashSet<String>>,
487 insert_after_first_page: Mutex<Option<(String, String, String)>>,
490 list_pages_served: AtomicUsize,
491 list_types: Mutex<Vec<String>>,
492 album_calls: Mutex<Vec<String>>,
493 }
494
495 impl StubServer {
496 fn start(state: Arc<StubState>) -> Self {
497 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
498 listener.set_nonblocking(true).unwrap();
499 let addr = listener.local_addr().unwrap();
500 let shutdown = Arc::new(std::sync::atomic::AtomicBool::new(false));
501
502 let stop = shutdown.clone();
503 std::thread::spawn(move || {
504 while !stop.load(Ordering::Relaxed) {
505 match listener.accept() {
506 Ok((stream, _)) => {
507 let _ = stream.set_nonblocking(false);
509 let state = state.clone();
510 std::thread::spawn(move || handle(stream, state));
511 }
512 Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
513 std::thread::sleep(std::time::Duration::from_millis(2));
514 }
515 Err(_) => break,
516 }
517 }
518 });
519
520 Self { addr, shutdown }
521 }
522
523 fn url(&self) -> String {
524 format!("http://{}", self.addr)
525 }
526 }
527
528 impl Drop for StubServer {
529 fn drop(&mut self) {
530 self.shutdown.store(true, Ordering::Relaxed);
531 }
532 }
533
534 fn handle(mut stream: std::net::TcpStream, state: Arc<StubState>) {
538 use std::io::{BufRead, Write};
539
540 let Ok(peek) = stream.try_clone() else { return };
541 let mut reader = std::io::BufReader::new(peek);
542
543 loop {
544 let mut request_line = String::new();
545 if reader.read_line(&mut request_line).unwrap_or(0) == 0 {
546 return;
547 }
548 let mut line = String::new();
549 while reader.read_line(&mut line).unwrap_or(0) > 0 {
550 if line == "\r\n" || line == "\n" {
551 break;
552 }
553 line.clear();
554 }
555
556 let target = request_line.split_whitespace().nth(1).unwrap_or("/");
557 let (path, query) = target.split_once('?').unwrap_or((target, ""));
558 let params: std::collections::HashMap<&str, &str> = query
559 .split('&')
560 .filter_map(|kv| kv.split_once('='))
561 .collect();
562
563 let (status, body) = respond(&state, path, ¶ms);
564
565 let write = write!(
566 stream,
567 "HTTP/1.1 {} OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n",
568 status,
569 body.len()
570 )
571 .and_then(|()| stream.write_all(body.as_bytes()))
572 .and_then(|()| stream.flush());
573 if write.is_err() {
574 return;
575 }
576 }
577 }
578
579 fn respond(
580 state: &Arc<StubState>,
581 path: &str,
582 params: &std::collections::HashMap<&str, &str>,
583 ) -> (u16, String) {
584 match path.rsplit('/').next().unwrap_or("") {
585 "getArtists" => (
586 200,
587 r#"{"subsonic-response":{"status":"ok","artists":{"index":[{"artist":[{"id":"ar1","name":"Stub Artist"}]}]}}}"#.to_string(),
588 ),
589 "getAlbumList2" => {
590 state
591 .list_types
592 .lock()
593 .unwrap()
594 .push(params.get("type").copied().unwrap_or("").to_string());
595 let offset: usize = params.get("offset").and_then(|o| o.parse().ok()).unwrap_or(0);
596 let size: usize = params.get("size").and_then(|s| s.parse().ok()).unwrap_or(500);
597
598 let albums = state.albums.lock().unwrap();
599 let slice: Vec<String> = albums
600 .iter()
601 .skip(offset)
602 .take(size)
603 .map(|(id, name, created)| {
604 format!(
605 r#"{{"id":"{}","name":"{}","artist":"Stub Artist","created":"{}"}}"#,
606 id, name, created
607 )
608 })
609 .collect();
610 drop(albums);
611
612 if state.list_pages_served.fetch_add(1, Ordering::SeqCst) == 0
613 && let Some(new_album) = state.insert_after_first_page.lock().unwrap().take()
614 {
615 state.albums.lock().unwrap().insert(0, new_album);
616 }
617
618 (
619 200,
620 format!(
621 r#"{{"subsonic-response":{{"status":"ok","albumList2":{{"album":[{}]}}}}}}"#,
622 slice.join(",")
623 ),
624 )
625 }
626 "getAlbum" => {
627 let id = params.get("id").copied().unwrap_or("");
628 state.album_calls.lock().unwrap().push(id.to_string());
629 if state.failing.lock().unwrap().contains(id) {
630 (500, r#"{"error":"boom"}"#.to_string())
631 } else {
632 (
633 200,
634 format!(
635 r#"{{"subsonic-response":{{"status":"ok","album":{{"id":"{id}","name":"Album {id}","artist":"Stub Artist","song":[{{"id":"s{id}","title":"Song {id}","track":1,"suffix":"flac"}}]}}}}}}"#
636 ),
637 )
638 }
639 }
640 _ => (404, "{}".to_string()),
641 }
642 }
643
644 fn stub_albums(n: usize) -> Vec<(String, String, String)> {
645 (0..n)
646 .map(|i| {
647 (
648 format!("a{:04}", i),
649 format!("Album {:04}", i),
650 "2024-01-15T10:30:00Z".to_string(),
651 )
652 })
653 .collect()
654 }
655
656 #[test]
657 fn failed_album_fetch_does_not_advance_last_sync_and_next_sync_retries() {
658 let (db, _dir) = test_db();
659 let state = Arc::new(StubState {
660 albums: Mutex::new(stub_albums(4)),
661 failing: Mutex::new(["a0002".to_string()].into_iter().collect()),
662 ..Default::default()
663 });
664 let server = StubServer::start(state.clone());
665 let client = SubsonicClient::new(&server.url(), "u", "p");
666
667 let first = sync_library(&db, &client, false, &server.url(), "u").unwrap();
668 assert_eq!(first.albums_failed, 1, "the failing album must be counted");
669 assert_eq!(first.albums_synced, 3);
670 assert!(!first.is_complete());
671 assert_eq!(
672 get_last_sync(&db, &server.url()).unwrap(),
673 None,
674 "an incomplete sync must not advance last_sync"
675 );
676
677 state.failing.lock().unwrap().clear();
680 state.album_calls.lock().unwrap().clear();
681 let second = sync_library(&db, &client, false, &server.url(), "u").unwrap();
682
683 assert!(
684 state
685 .album_calls
686 .lock()
687 .unwrap()
688 .contains(&"a0002".to_string()),
689 "the previously failed album must be retried"
690 );
691 assert_eq!(second.albums_failed, 0);
692 assert!(second.is_complete());
693 assert!(
694 get_last_sync(&db, &server.url()).unwrap().is_some(),
695 "a clean sync advances last_sync"
696 );
697 }
698
699 #[test]
700 fn album_inserted_mid_pagination_is_not_fetched_twice_or_skipped() {
701 let (db, _dir) = test_db();
705 let state = Arc::new(StubState {
706 albums: Mutex::new(stub_albums(600)),
707 insert_after_first_page: Mutex::new(Some((
708 "aNEW".to_string(),
709 "AAA Brand New".to_string(),
710 "2024-06-01T00:00:00Z".to_string(),
711 ))),
712 ..Default::default()
713 });
714 let server = StubServer::start(state.clone());
715 let client = SubsonicClient::new(&server.url(), "u", "p");
716
717 let result = sync_library(&db, &client, true, &server.url(), "u").unwrap();
718 assert_eq!(result.albums_failed, 0);
719
720 let calls = state.album_calls.lock().unwrap().clone();
721 let unique: HashSet<&String> = calls.iter().collect();
722 assert_eq!(
723 calls.len(),
724 unique.len(),
725 "no album may be fetched twice after the window shifts"
726 );
727
728 for i in 0..600 {
730 let id = format!("a{:04}", i);
731 assert!(unique.contains(&id), "album {} was skipped", id);
732 }
733
734 let types = state.list_types.lock().unwrap().clone();
735 assert!(
736 types.iter().all(|t| t == "alphabeticalByName"),
737 "the paginated walk must use a stable ordering, got {:?}",
738 types
739 );
740 }
741
742 #[test]
743 fn incremental_sync_only_fetches_albums_created_after_last_sync() {
744 let (db, _dir) = test_db();
745 let mut albums = stub_albums(3);
746 albums[0].2 = "2020-01-01T00:00:00Z".into();
747 albums[1].2 = "2020-01-01T00:00:00Z".into();
748 albums[2].2 = "2030-01-01T00:00:00Z".into();
749
750 let state = Arc::new(StubState {
751 albums: Mutex::new(albums),
752 ..Default::default()
753 });
754 let server = StubServer::start(state.clone());
755 let client = SubsonicClient::new(&server.url(), "u", "p");
756
757 let watermark = parse_iso8601_to_unix("2025-01-01T00:00:00Z").unwrap();
759 update_last_sync(&db, &server.url(), "u", watermark).unwrap();
760
761 let result = sync_library(&db, &client, false, &server.url(), "u").unwrap();
762
763 assert_eq!(result.albums_synced, 1, "only the new album needs fetching");
764 assert_eq!(
765 *state.album_calls.lock().unwrap(),
766 vec!["a0002".to_string()]
767 );
768 }
769
770 #[test]
771 fn album_with_unparseable_created_is_always_fetched() {
772 let (db, _dir) = test_db();
773 let state = Arc::new(StubState {
774 albums: Mutex::new(vec![("a0000".into(), "Album".into(), "who knows".into())]),
775 ..Default::default()
776 });
777 let server = StubServer::start(state.clone());
778 let client = SubsonicClient::new(&server.url(), "u", "p");
779
780 update_last_sync(&db, &server.url(), "u", 4_000_000_000).unwrap();
781 let result = sync_library(&db, &client, false, &server.url(), "u").unwrap();
782
783 assert_eq!(
784 result.albums_synced, 1,
785 "an album with no usable timestamp must not be assumed old"
786 );
787 }
788
789 #[test]
790 fn sync_deduplicates_by_remote_id() {
791 let (db, _dir) = test_db();
792
793 let meta1 = remote_track_meta("remote-dup", "Original Title", "Artist A", "Album X");
795 let id1 = queries::upsert_track(&db.conn, &meta1).unwrap();
796
797 let meta2 = remote_track_meta("remote-dup", "Updated Title", "Artist A", "Album X");
799 let id2 = queries::upsert_track(&db.conn, &meta2).unwrap();
800
801 assert_eq!(id1, id2, "same remote_id should resolve to same track row");
803
804 let row = queries::get_track_row(&db.conn, id2)
806 .unwrap()
807 .expect("track should exist");
808 assert_eq!(row.title, "Updated Title");
809 assert_eq!(row.remote_id.as_deref(), Some("remote-dup"));
810
811 let stats = queries::library_stats(&db.conn).unwrap();
813 assert_eq!(
814 stats.total_tracks, 1,
815 "should have exactly 1 track after dedup"
816 );
817 }
818}