1use std::collections::HashSet;
2use std::sync::atomic::{AtomicUsize, Ordering};
3
4use crate::db::connection::Database;
5use crate::db::queries::{self, TrackMeta};
6use crate::remote::client::{SubsonicAlbumFull, SubsonicClient};
7
8use rayon::prelude::*;
9use rusqlite::params;
10use thiserror::Error;
11
12#[derive(Debug, Error)]
13pub enum SyncError {
14 #[error("subsonic error: {0}")]
15 Subsonic(#[from] super::client::SubsonicError),
16 #[error("db error: {0}")]
17 Db(#[from] crate::db::connection::DbError),
18}
19
20#[derive(Debug, Default)]
21pub struct SyncResult {
22 pub artists_synced: usize,
23 pub albums_synced: usize,
24 pub tracks_synced: usize,
25 pub albums_failed: usize,
28}
29
30impl SyncResult {
31 pub fn is_complete(&self) -> bool {
33 self.albums_failed == 0
34 }
35}
36
37pub fn get_last_sync(
39 db: &Database,
40 url: &str,
41) -> Result<Option<i64>, crate::db::connection::DbError> {
42 let result = db.conn.query_row(
43 "SELECT last_sync FROM remote_servers WHERE url = ?1",
44 params![url],
45 |row| row.get::<_, Option<i64>>(0),
46 );
47 match result {
48 Ok(ts) => Ok(ts),
49 Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
50 Err(e) => Err(e.into()),
51 }
52}
53
54pub fn update_last_sync(
56 db: &Database,
57 url: &str,
58 username: &str,
59 timestamp: i64,
60) -> Result<(), crate::db::connection::DbError> {
61 db.conn.execute(
62 "INSERT INTO remote_servers (url, username, last_sync)
63 VALUES (?1, ?2, ?3)
64 ON CONFLICT(url) DO UPDATE SET last_sync = ?3",
65 params![url, username, timestamp],
66 )?;
67 Ok(())
68}
69
70fn parse_iso8601_to_unix(s: &str) -> Option<i64> {
78 use chrono::{DateTime, FixedOffset, NaiveDateTime};
79
80 if let Ok(dt) = DateTime::parse_from_rfc3339(s) {
82 return Some(dt.timestamp());
83 }
84
85 if let Ok(naive) = NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f") {
88 return Some(naive.and_utc().timestamp());
89 }
90 if let Ok(naive) = NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S") {
91 return Some(naive.and_utc().timestamp());
92 }
93
94 if let Ok(dt) = DateTime::<FixedOffset>::parse_from_str(s, "%Y-%m-%d %H:%M:%S%:z") {
96 return Some(dt.timestamp());
97 }
98 if let Ok(naive) = NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") {
99 return Some(naive.and_utc().timestamp());
100 }
101
102 None
103}
104
105pub fn sync_library(
121 db: &Database,
122 client: &SubsonicClient,
123 full: bool,
124 server_url: &str,
125 username: &str,
126) -> Result<SyncResult, SyncError> {
127 let mut result = SyncResult::default();
128
129 let artists = client.get_artists()?;
130 result.artists_synced = artists.len();
131 log::info!("syncing {} artists from remote", artists.len());
132
133 let last_sync = if full {
134 None
135 } else {
136 get_last_sync(db, server_url)?
137 };
138
139 match last_sync {
140 Some(ts) => log::info!("incremental sync (albums created after {})", ts),
141 None => log::info!("full sync"),
142 }
143
144 let sync_start = std::time::SystemTime::now()
145 .duration_since(std::time::UNIX_EPOCH)
146 .unwrap_or_default()
147 .as_secs() as i64;
148
149 let mut offset = 0u32;
150 let page_size = 500u32;
151 let mut seen_ids: HashSet<String> = HashSet::new();
154
155 loop {
156 let page = client.get_album_list("alphabeticalByName", page_size, offset)?;
157 if page.is_empty() {
158 break;
159 }
160 let page_count = page.len();
161 offset += page_count as u32;
162
163 let to_fetch: Vec<String> = page
167 .into_iter()
168 .filter(|a| seen_ids.insert(a.id.clone()))
169 .filter(|a| match last_sync {
170 None => true,
171 Some(ts) => a
172 .created
173 .as_deref()
174 .and_then(parse_iso8601_to_unix)
175 .is_none_or(|created| created >= ts),
176 })
177 .map(|a| a.id)
178 .collect();
179
180 if !to_fetch.is_empty() {
181 let failures = AtomicUsize::new(0);
182 let fetched: Vec<SubsonicAlbumFull> = to_fetch
183 .into_par_iter()
184 .filter_map(|id| match client.get_album(&id) {
185 Ok(full) => Some(full),
186 Err(e) => {
187 log::warn!("failed to fetch album {}: {}", id, e);
188 failures.fetch_add(1, Ordering::Relaxed);
189 None
190 }
191 })
192 .collect();
193 result.albums_failed += failures.into_inner();
194
195 write_albums(db, client, &fetched, &mut result)?;
196 }
197
198 log::info!(
199 "synced {} albums ({} tracks) so far...",
200 result.albums_synced,
201 result.tracks_synced
202 );
203
204 if (page_count as u32) < page_size {
205 break;
206 }
207 }
208
209 if result.is_complete() {
210 update_last_sync(db, server_url, username, sync_start)?;
211 } else {
212 log::warn!(
213 "{} album(s) failed to fetch — leaving last_sync unchanged so the next sync retries them",
214 result.albums_failed
215 );
216 }
217
218 log::info!(
219 "sync complete: {} artists, {} albums, {} tracks, {} failed",
220 result.artists_synced,
221 result.albums_synced,
222 result.tracks_synced,
223 result.albums_failed,
224 );
225
226 Ok(result)
227}
228
229fn write_albums(
231 db: &Database,
232 client: &SubsonicClient,
233 albums: &[SubsonicAlbumFull],
234 result: &mut SyncResult,
235) -> Result<(), SyncError> {
236 db.conn
237 .execute_batch("BEGIN")
238 .map_err(crate::db::connection::DbError::from)?;
239
240 for album in albums {
241 result.albums_synced += 1;
242 let artist_name = album.artist.as_deref().unwrap_or("Unknown Artist");
243
244 for song in &album.song {
245 let meta = TrackMeta {
246 title: song.title.clone(),
247 artist: song
248 .artist
249 .clone()
250 .unwrap_or_else(|| artist_name.to_string()),
251 album_artist: album.artist.clone(),
252 album: album.name.clone(),
253 date: album.year.map(|y| y.to_string()),
254 disc: song.disc_number,
255 track_number: song.track,
256 genre: song.genre.clone().or_else(|| album.genre.clone()),
257 label: None,
258 duration_ms: song.duration.map(|d| d * 1000),
259 codec: song.suffix.clone(),
260 sample_rate: None,
261 bit_depth: None,
262 channels: None,
263 bitrate: song.bit_rate,
264 size_bytes: None,
265 mtime: None,
266 path: None,
267 source: "remote".to_string(),
268 remote_id: Some(song.id.clone()),
269 remote_url: Some(client.stream_url_template(&song.id)),
270 album_added_at: album.created.clone(),
271 };
272
273 match queries::upsert_track(&db.conn, &meta) {
274 Ok(_) => result.tracks_synced += 1,
275 Err(e) => log::warn!("failed to insert remote track {}: {}", song.title, e),
276 }
277 }
278 }
279
280 db.conn
281 .execute_batch("COMMIT")
282 .map_err(crate::db::connection::DbError::from)?;
283
284 Ok(())
285}
286
287#[cfg(test)]
288mod tests {
289 use super::*;
290
291 #[test]
292 fn parse_rfc3339_with_z() {
293 assert_eq!(
295 parse_iso8601_to_unix("2024-01-15T10:30:00Z"),
296 Some(1705314600)
297 );
298 }
299
300 #[test]
301 fn parse_rfc3339_with_offset() {
302 assert_eq!(
304 parse_iso8601_to_unix("2024-01-15T10:30:00+05:30"),
305 Some(1705294800)
306 );
307 }
308
309 #[test]
310 fn parse_rfc3339_negative_offset() {
311 assert_eq!(
313 parse_iso8601_to_unix("2024-01-15T10:30:00-05:00"),
314 Some(1705332600)
315 );
316 }
317
318 #[test]
319 fn parse_fractional_seconds_z() {
320 assert_eq!(
321 parse_iso8601_to_unix("2024-01-15T10:30:00.123Z"),
322 Some(1705314600)
323 );
324 }
325
326 #[test]
327 fn parse_fractional_seconds_offset() {
328 assert_eq!(
329 parse_iso8601_to_unix("2024-01-15T10:30:00.999+00:00"),
330 Some(1705314600)
331 );
332 }
333
334 #[test]
335 fn parse_no_timezone_assumes_utc() {
336 assert_eq!(
337 parse_iso8601_to_unix("2024-01-15T10:30:00"),
338 Some(1705314600)
339 );
340 }
341
342 #[test]
343 fn parse_no_timezone_fractional() {
344 assert_eq!(
345 parse_iso8601_to_unix("2024-01-15T10:30:00.500"),
346 Some(1705314600)
347 );
348 }
349
350 #[test]
351 fn parse_space_separator_with_tz() {
352 assert_eq!(
353 parse_iso8601_to_unix("2024-01-15 10:30:00+00:00"),
354 Some(1705314600)
355 );
356 }
357
358 #[test]
359 fn parse_space_separator_no_tz() {
360 assert_eq!(
361 parse_iso8601_to_unix("2024-01-15 10:30:00"),
362 Some(1705314600)
363 );
364 }
365
366 #[test]
367 fn parse_garbage_returns_none() {
368 assert_eq!(parse_iso8601_to_unix("not-a-date"), None);
369 assert_eq!(parse_iso8601_to_unix(""), None);
370 assert_eq!(parse_iso8601_to_unix("2024"), None);
371 }
372
373 #[test]
374 fn parse_epoch() {
375 assert_eq!(parse_iso8601_to_unix("1970-01-01T00:00:00Z"), Some(0));
376 }
377
378 use crate::db::connection::Database;
381 use crate::db::queries;
382 use std::sync::{Arc, Mutex};
383
384 fn test_db() -> (Database, tempfile::TempDir) {
385 let dir = tempfile::tempdir().unwrap();
386 let db = Database::open(&dir.path().join("sync_test.db")).unwrap();
387 (db, dir)
388 }
389
390 fn remote_track_meta(remote_id: &str, title: &str, artist: &str, album: &str) -> TrackMeta {
392 TrackMeta {
393 title: title.into(),
394 artist: artist.into(),
395 album_artist: Some(artist.into()),
396 album: album.into(),
397 date: Some("2024".into()),
398 disc: Some(1),
399 track_number: Some(1),
400 genre: Some("Electronic".into()),
401 label: None,
402 duration_ms: Some(240_000),
403 codec: Some("FLAC".into()),
404 sample_rate: None,
405 bit_depth: None,
406 channels: None,
407 bitrate: Some(1000),
408 size_bytes: None,
409 mtime: None,
410 path: None,
411 source: "remote".into(),
412 remote_id: Some(remote_id.into()),
413 remote_url: Some(format!("https://example.com/stream?id={}", remote_id)),
414 album_added_at: None,
415 }
416 }
417
418 #[test]
419 fn sync_upserts_tracks_to_database() {
420 let (db, _dir) = test_db();
421
422 let meta = remote_track_meta("remote-001", "Vordhosbn", "Aphex Twin", "Drukqs");
423 let track_id = queries::upsert_track(&db.conn, &meta).unwrap();
424 assert!(track_id > 0, "upsert should return a valid track ID");
425
426 let row = queries::get_track_row(&db.conn, track_id)
428 .unwrap()
429 .expect("track should exist in DB");
430 assert_eq!(row.title, "Vordhosbn");
431 assert_eq!(row.artist_name, "Aphex Twin");
432 assert_eq!(row.album_title, "Drukqs");
433 assert_eq!(row.remote_id.as_deref(), Some("remote-001"));
434 assert_eq!(row.source, "remote");
435 }
436
437 struct StubServer {
442 addr: std::net::SocketAddr,
443 shutdown: Arc<std::sync::atomic::AtomicBool>,
444 }
445
446 #[derive(Default)]
447 struct StubState {
448 albums: Mutex<Vec<(String, String, String)>>,
450 failing: Mutex<HashSet<String>>,
452 insert_after_first_page: Mutex<Option<(String, String, String)>>,
455 list_pages_served: AtomicUsize,
456 list_types: Mutex<Vec<String>>,
457 album_calls: Mutex<Vec<String>>,
458 }
459
460 impl StubServer {
461 fn start(state: Arc<StubState>) -> Self {
462 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
463 listener.set_nonblocking(true).unwrap();
464 let addr = listener.local_addr().unwrap();
465 let shutdown = Arc::new(std::sync::atomic::AtomicBool::new(false));
466
467 let stop = shutdown.clone();
468 std::thread::spawn(move || {
469 while !stop.load(Ordering::Relaxed) {
470 match listener.accept() {
471 Ok((stream, _)) => {
472 let _ = stream.set_nonblocking(false);
474 let state = state.clone();
475 std::thread::spawn(move || handle(stream, state));
476 }
477 Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
478 std::thread::sleep(std::time::Duration::from_millis(2));
479 }
480 Err(_) => break,
481 }
482 }
483 });
484
485 Self { addr, shutdown }
486 }
487
488 fn url(&self) -> String {
489 format!("http://{}", self.addr)
490 }
491 }
492
493 impl Drop for StubServer {
494 fn drop(&mut self) {
495 self.shutdown.store(true, Ordering::Relaxed);
496 }
497 }
498
499 fn handle(mut stream: std::net::TcpStream, state: Arc<StubState>) {
503 use std::io::{BufRead, Write};
504
505 let Ok(peek) = stream.try_clone() else { return };
506 let mut reader = std::io::BufReader::new(peek);
507
508 loop {
509 let mut request_line = String::new();
510 if reader.read_line(&mut request_line).unwrap_or(0) == 0 {
511 return;
512 }
513 let mut line = String::new();
514 while reader.read_line(&mut line).unwrap_or(0) > 0 {
515 if line == "\r\n" || line == "\n" {
516 break;
517 }
518 line.clear();
519 }
520
521 let target = request_line.split_whitespace().nth(1).unwrap_or("/");
522 let (path, query) = target.split_once('?').unwrap_or((target, ""));
523 let params: std::collections::HashMap<&str, &str> = query
524 .split('&')
525 .filter_map(|kv| kv.split_once('='))
526 .collect();
527
528 let (status, body) = respond(&state, path, ¶ms);
529
530 let write = write!(
531 stream,
532 "HTTP/1.1 {} OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n",
533 status,
534 body.len()
535 )
536 .and_then(|()| stream.write_all(body.as_bytes()))
537 .and_then(|()| stream.flush());
538 if write.is_err() {
539 return;
540 }
541 }
542 }
543
544 fn respond(
545 state: &Arc<StubState>,
546 path: &str,
547 params: &std::collections::HashMap<&str, &str>,
548 ) -> (u16, String) {
549 match path.rsplit('/').next().unwrap_or("") {
550 "getArtists" => (
551 200,
552 r#"{"subsonic-response":{"status":"ok","artists":{"index":[{"artist":[{"id":"ar1","name":"Stub Artist"}]}]}}}"#.to_string(),
553 ),
554 "getAlbumList2" => {
555 state
556 .list_types
557 .lock()
558 .unwrap()
559 .push(params.get("type").copied().unwrap_or("").to_string());
560 let offset: usize = params.get("offset").and_then(|o| o.parse().ok()).unwrap_or(0);
561 let size: usize = params.get("size").and_then(|s| s.parse().ok()).unwrap_or(500);
562
563 let albums = state.albums.lock().unwrap();
564 let slice: Vec<String> = albums
565 .iter()
566 .skip(offset)
567 .take(size)
568 .map(|(id, name, created)| {
569 format!(
570 r#"{{"id":"{}","name":"{}","artist":"Stub Artist","created":"{}"}}"#,
571 id, name, created
572 )
573 })
574 .collect();
575 drop(albums);
576
577 if state.list_pages_served.fetch_add(1, Ordering::SeqCst) == 0
578 && let Some(new_album) = state.insert_after_first_page.lock().unwrap().take()
579 {
580 state.albums.lock().unwrap().insert(0, new_album);
581 }
582
583 (
584 200,
585 format!(
586 r#"{{"subsonic-response":{{"status":"ok","albumList2":{{"album":[{}]}}}}}}"#,
587 slice.join(",")
588 ),
589 )
590 }
591 "getAlbum" => {
592 let id = params.get("id").copied().unwrap_or("");
593 state.album_calls.lock().unwrap().push(id.to_string());
594 if state.failing.lock().unwrap().contains(id) {
595 (500, r#"{"error":"boom"}"#.to_string())
596 } else {
597 (
598 200,
599 format!(
600 r#"{{"subsonic-response":{{"status":"ok","album":{{"id":"{id}","name":"Album {id}","artist":"Stub Artist","song":[{{"id":"s{id}","title":"Song {id}","track":1,"suffix":"flac"}}]}}}}}}"#
601 ),
602 )
603 }
604 }
605 _ => (404, "{}".to_string()),
606 }
607 }
608
609 fn stub_albums(n: usize) -> Vec<(String, String, String)> {
610 (0..n)
611 .map(|i| {
612 (
613 format!("a{:04}", i),
614 format!("Album {:04}", i),
615 "2024-01-15T10:30:00Z".to_string(),
616 )
617 })
618 .collect()
619 }
620
621 #[test]
622 fn failed_album_fetch_does_not_advance_last_sync_and_next_sync_retries() {
623 let (db, _dir) = test_db();
624 let state = Arc::new(StubState {
625 albums: Mutex::new(stub_albums(4)),
626 failing: Mutex::new(["a0002".to_string()].into_iter().collect()),
627 ..Default::default()
628 });
629 let server = StubServer::start(state.clone());
630 let client = SubsonicClient::new(&server.url(), "u", "p");
631
632 let first = sync_library(&db, &client, false, &server.url(), "u").unwrap();
633 assert_eq!(first.albums_failed, 1, "the failing album must be counted");
634 assert_eq!(first.albums_synced, 3);
635 assert!(!first.is_complete());
636 assert_eq!(
637 get_last_sync(&db, &server.url()).unwrap(),
638 None,
639 "an incomplete sync must not advance last_sync"
640 );
641
642 state.failing.lock().unwrap().clear();
645 state.album_calls.lock().unwrap().clear();
646 let second = sync_library(&db, &client, false, &server.url(), "u").unwrap();
647
648 assert!(
649 state
650 .album_calls
651 .lock()
652 .unwrap()
653 .contains(&"a0002".to_string()),
654 "the previously failed album must be retried"
655 );
656 assert_eq!(second.albums_failed, 0);
657 assert!(second.is_complete());
658 assert!(
659 get_last_sync(&db, &server.url()).unwrap().is_some(),
660 "a clean sync advances last_sync"
661 );
662 }
663
664 #[test]
665 fn album_inserted_mid_pagination_is_not_fetched_twice_or_skipped() {
666 let (db, _dir) = test_db();
670 let state = Arc::new(StubState {
671 albums: Mutex::new(stub_albums(600)),
672 insert_after_first_page: Mutex::new(Some((
673 "aNEW".to_string(),
674 "AAA Brand New".to_string(),
675 "2024-06-01T00:00:00Z".to_string(),
676 ))),
677 ..Default::default()
678 });
679 let server = StubServer::start(state.clone());
680 let client = SubsonicClient::new(&server.url(), "u", "p");
681
682 let result = sync_library(&db, &client, true, &server.url(), "u").unwrap();
683 assert_eq!(result.albums_failed, 0);
684
685 let calls = state.album_calls.lock().unwrap().clone();
686 let unique: HashSet<&String> = calls.iter().collect();
687 assert_eq!(
688 calls.len(),
689 unique.len(),
690 "no album may be fetched twice after the window shifts"
691 );
692
693 for i in 0..600 {
695 let id = format!("a{:04}", i);
696 assert!(unique.contains(&id), "album {} was skipped", id);
697 }
698
699 let types = state.list_types.lock().unwrap().clone();
700 assert!(
701 types.iter().all(|t| t == "alphabeticalByName"),
702 "the paginated walk must use a stable ordering, got {:?}",
703 types
704 );
705 }
706
707 #[test]
708 fn incremental_sync_only_fetches_albums_created_after_last_sync() {
709 let (db, _dir) = test_db();
710 let mut albums = stub_albums(3);
711 albums[0].2 = "2020-01-01T00:00:00Z".into();
712 albums[1].2 = "2020-01-01T00:00:00Z".into();
713 albums[2].2 = "2030-01-01T00:00:00Z".into();
714
715 let state = Arc::new(StubState {
716 albums: Mutex::new(albums),
717 ..Default::default()
718 });
719 let server = StubServer::start(state.clone());
720 let client = SubsonicClient::new(&server.url(), "u", "p");
721
722 let watermark = parse_iso8601_to_unix("2025-01-01T00:00:00Z").unwrap();
724 update_last_sync(&db, &server.url(), "u", watermark).unwrap();
725
726 let result = sync_library(&db, &client, false, &server.url(), "u").unwrap();
727
728 assert_eq!(result.albums_synced, 1, "only the new album needs fetching");
729 assert_eq!(
730 *state.album_calls.lock().unwrap(),
731 vec!["a0002".to_string()]
732 );
733 }
734
735 #[test]
736 fn album_with_unparseable_created_is_always_fetched() {
737 let (db, _dir) = test_db();
738 let state = Arc::new(StubState {
739 albums: Mutex::new(vec![("a0000".into(), "Album".into(), "who knows".into())]),
740 ..Default::default()
741 });
742 let server = StubServer::start(state.clone());
743 let client = SubsonicClient::new(&server.url(), "u", "p");
744
745 update_last_sync(&db, &server.url(), "u", 4_000_000_000).unwrap();
746 let result = sync_library(&db, &client, false, &server.url(), "u").unwrap();
747
748 assert_eq!(
749 result.albums_synced, 1,
750 "an album with no usable timestamp must not be assumed old"
751 );
752 }
753
754 #[test]
755 fn sync_deduplicates_by_remote_id() {
756 let (db, _dir) = test_db();
757
758 let meta1 = remote_track_meta("remote-dup", "Original Title", "Artist A", "Album X");
760 let id1 = queries::upsert_track(&db.conn, &meta1).unwrap();
761
762 let meta2 = remote_track_meta("remote-dup", "Updated Title", "Artist A", "Album X");
764 let id2 = queries::upsert_track(&db.conn, &meta2).unwrap();
765
766 assert_eq!(id1, id2, "same remote_id should resolve to same track row");
768
769 let row = queries::get_track_row(&db.conn, id2)
771 .unwrap()
772 .expect("track should exist");
773 assert_eq!(row.title, "Updated Title");
774 assert_eq!(row.remote_id.as_deref(), Some("remote-dup"));
775
776 let stats = queries::library_stats(&db.conn).unwrap();
778 assert_eq!(
779 stats.total_tracks, 1,
780 "should have exactly 1 track after dedup"
781 );
782 }
783}