sift/
discogs.rs

1//! Discogs, over its JSON web service — consulted when MusicBrainz has no
2//! strong match.
3//!
4//! Discogs releases are mapped into the same [`Release`] shape MusicBrainz
5//! releases use, so matching, tagging and filing do not need to know which
6//! service a release came from. The id is namespaced `discogs:<number>`,
7//! since MusicBrainz ids are UUIDs and never contain a colon; fields that
8//! only MusicBrainz has (recording ids, artist ids, the release group's
9//! mbid) are left empty.
10
11use std::sync::LazyLock;
12use std::time::Duration;
13
14use regex::Regex;
15use serde::Deserialize;
16use tokio::sync::Mutex;
17use tokio::time::Instant;
18
19use crate::musicbrainz::{
20    Artist, ArtistCredit, Label, LabelInfo, Medium, Recording, Release, ReleaseGroup, ReleaseTrack,
21};
22
23const BASE: &str = "https://api.discogs.com";
24const SPACING: Duration = Duration::from_millis(1050);
25const MAX_SPACING: Duration = Duration::from_secs(10);
26const ATTEMPTS: u32 = 10;
27const RELEASE_TTL: Duration = Duration::from_secs(7 * 24 * 3600);
28const SEARCH_TTL: Duration = Duration::from_secs(24 * 3600);
29
30#[derive(Debug, thiserror::Error)]
31pub enum DiscogsError {
32    #[error("Discogs request failed: {0}")]
33    Http(#[from] reqwest::Error),
34    #[error("Discogs is refusing requests (HTTP {0}) and did not recover")]
35    Unavailable(u16),
36    #[error("Discogs sent something unreadable: {0}")]
37    Decode(String),
38}
39
40impl DiscogsError {
41    /// Whether trying again later could succeed.
42    pub fn is_transient(&self) -> bool {
43        match self {
44            Self::Unavailable(_) => true,
45            Self::Http(e) => e.is_timeout() || e.is_connect() || e.status().is_some_and(retryable),
46            Self::Decode(_) => false,
47        }
48    }
49}
50
51pub struct Discogs {
52    http: reqwest::Client,
53    token: String,
54    base: String,
55    gate: Mutex<Gate>,
56    /// Responses kept on disk, same cache directory as MusicBrainz's but
57    /// with a `discogs-` prefixed filename, so the two never collide.
58    cache: Option<std::path::PathBuf>,
59}
60
61impl Discogs {
62    pub fn new(token: &str, contact: &str) -> Self {
63        Self::with_base(BASE, token, contact)
64    }
65
66    pub fn with_base(base: &str, token: &str, contact: &str) -> Self {
67        let agent = format!("sift/{} ( {contact} )", env!("CARGO_PKG_VERSION"));
68        Self {
69            http: reqwest::Client::builder()
70                .user_agent(agent)
71                .timeout(Duration::from_secs(30))
72                .build()
73                .expect("static client config"),
74            token: token.to_string(),
75            base: base.trim_end_matches('/').to_string(),
76            gate: Mutex::new(Gate::default()),
77            cache: None,
78        }
79    }
80
81    /// Keep responses under `dir`.
82    pub fn with_cache(mut self, dir: std::path::PathBuf) -> Self {
83        self.cache = Some(dir);
84        self
85    }
86
87    fn cache_path(&self, path: &str, query: &[(&str, &str)]) -> Option<std::path::PathBuf> {
88        use std::hash::{Hash, Hasher};
89        let dir = self.cache.as_ref()?;
90        let mut h = std::collections::hash_map::DefaultHasher::new();
91        (path, query).hash(&mut h);
92        Some(dir.join(format!("discogs-{:016x}.json", h.finish())))
93    }
94
95    async fn get<T: for<'de> Deserialize<'de>>(
96        &self,
97        path: &str,
98        query: &[(&str, &str)],
99    ) -> Result<T, DiscogsError> {
100        let ttl = if path.starts_with("releases/") {
101            RELEASE_TTL
102        } else {
103            SEARCH_TTL
104        };
105        let cached = self.cache_path(path, query);
106        if let Some(file) = &cached {
107            let fresh = tokio::fs::metadata(file)
108                .await
109                .ok()
110                .and_then(|m| m.modified().ok())
111                .and_then(|t| t.elapsed().ok())
112                .is_some_and(|age| age < ttl);
113            if fresh
114                && let Ok(body) = tokio::fs::read(file).await
115                && let Ok(v) = serde_json::from_slice(&body)
116            {
117                return Ok(v);
118            }
119        }
120        let body: bytes::Bytes = self.fetch(path, query).await?;
121        let value =
122            serde_json::from_slice(&body).map_err(|e| DiscogsError::Decode(e.to_string()))?;
123        if let Some(file) = cached {
124            if let Some(dir) = file.parent() {
125                let _ = tokio::fs::create_dir_all(dir).await;
126            }
127            let _ = tokio::fs::write(&file, &body).await;
128        }
129        Ok(value)
130    }
131
132    async fn fetch(
133        &self,
134        path: &str,
135        query: &[(&str, &str)],
136    ) -> Result<bytes::Bytes, DiscogsError> {
137        let mut gate = self.gate.lock().await;
138        let mut attempt = 0;
139        loop {
140            gate.wait().await;
141            let sent = self
142                .http
143                .get(format!("{}/{path}", self.base))
144                .header(
145                    reqwest::header::AUTHORIZATION,
146                    format!("Discogs token={}", self.token),
147                )
148                .query(query)
149                .send()
150                .await;
151            attempt += 1;
152            let backoff = Duration::from_secs(2u64 << attempt.min(5));
153            let (wait, failure) = match sent {
154                Ok(resp) if retryable(resp.status()) => {
155                    let wait = resp
156                        .headers()
157                        .get(reqwest::header::RETRY_AFTER)
158                        .and_then(|v| v.to_str().ok())
159                        .and_then(|v| v.trim().parse::<u64>().ok())
160                        .map_or(backoff, Duration::from_secs);
161                    (wait, DiscogsError::Unavailable(resp.status().as_u16()))
162                }
163                Ok(resp) => {
164                    gate.succeeded();
165                    return Ok(resp.error_for_status()?.bytes().await?);
166                }
167                Err(e) if e.is_timeout() || e.is_connect() => (backoff, DiscogsError::Http(e)),
168                Err(e) => return Err(e.into()),
169            };
170            if attempt >= ATTEMPTS {
171                return Err(failure);
172            }
173            gate.refused(wait.clamp(Duration::from_secs(1), Duration::from_secs(60)));
174        }
175    }
176
177    /// Release ids matching an artist and album title, best first.
178    pub async fn search(&self, artist: &str, album: &str) -> Result<Vec<u64>, DiscogsError> {
179        #[derive(Debug, Deserialize, Default)]
180        struct Results {
181            #[serde(default)]
182            results: Vec<Hit>,
183        }
184        #[derive(Debug, Deserialize)]
185        struct Hit {
186            id: u64,
187        }
188        let found: Results = self
189            .get(
190                "database/search",
191                &[
192                    ("type", "release"),
193                    ("artist", artist),
194                    ("release_title", album),
195                    ("per_page", "10"),
196                ],
197            )
198            .await?;
199        Ok(found.results.into_iter().map(|h| h.id).collect())
200    }
201
202    /// One release, mapped into the same shape a MusicBrainz release takes.
203    /// With `index_tracks`, a medley grouped under an index track has the
204    /// index's own title prefixed onto each of its parts.
205    pub async fn release(&self, id: u64, index_tracks: bool) -> Result<Release, DiscogsError> {
206        let r: DRelease = self.get(&format!("releases/{id}"), &[]).await?;
207        Ok(map_release(id, r, index_tracks))
208    }
209
210    /// An image already resolved to a URL (as `Release::cover_url` gives it),
211    /// fetched with the same token used for the API itself.
212    pub async fn image(&self, url: &str) -> Result<Vec<u8>, DiscogsError> {
213        let resp = self
214            .http
215            .get(url)
216            .header(
217                reqwest::header::AUTHORIZATION,
218                format!("Discogs token={}", self.token),
219            )
220            .send()
221            .await?;
222        Ok(resp.error_for_status()?.bytes().await?.to_vec())
223    }
224}
225
226struct Gate {
227    next: Option<Instant>,
228    spacing: Duration,
229}
230
231impl Default for Gate {
232    fn default() -> Self {
233        Self {
234            next: None,
235            spacing: SPACING,
236        }
237    }
238}
239
240impl Gate {
241    async fn wait(&self) {
242        if let Some(t) = self.next {
243            tokio::time::sleep_until(t).await;
244        }
245    }
246
247    fn succeeded(&mut self) {
248        self.spacing = self.spacing.mul_f64(0.9).max(SPACING);
249        self.next = Some(Instant::now() + self.spacing);
250    }
251
252    fn refused(&mut self, wait: Duration) {
253        self.spacing = (self.spacing * 2).min(MAX_SPACING);
254        self.next = Some(Instant::now() + wait.max(self.spacing));
255    }
256}
257
258fn retryable(status: reqwest::StatusCode) -> bool {
259    status == reqwest::StatusCode::TOO_MANY_REQUESTS || status.is_server_error()
260}
261
262#[derive(Debug, Clone, Deserialize, Default)]
263struct DArtist {
264    #[serde(default)]
265    name: String,
266    /// The word (and its spacing) to place between this artist and the
267    /// next, e.g. "&", "feat.".
268    #[serde(default)]
269    join: String,
270}
271
272#[derive(Debug, Deserialize, Default)]
273struct DLabel {
274    #[serde(default)]
275    name: String,
276    #[serde(default)]
277    catno: Option<String>,
278}
279
280#[derive(Debug, Deserialize, Default)]
281struct DImage {
282    #[serde(rename = "type", default)]
283    kind: Option<String>,
284    uri: Option<String>,
285}
286
287#[derive(Debug, Deserialize, Default)]
288struct DTrack {
289    #[serde(default)]
290    position: String,
291    #[serde(default)]
292    type_: String,
293    #[serde(default)]
294    title: String,
295    #[serde(default)]
296    duration: String,
297    #[serde(default)]
298    artists: Vec<DArtist>,
299    #[serde(default)]
300    sub_tracks: Vec<DTrack>,
301}
302
303#[derive(Debug, Deserialize, Default)]
304struct DRelease {
305    #[serde(default)]
306    title: String,
307    #[serde(default)]
308    artists: Vec<DArtist>,
309    #[serde(default)]
310    labels: Vec<DLabel>,
311    released: Option<String>,
312    country: Option<String>,
313    #[serde(default)]
314    tracklist: Vec<DTrack>,
315    #[serde(default)]
316    images: Vec<DImage>,
317}
318
319fn map_release(id: u64, r: DRelease, index_tracks: bool) -> Release {
320    let date = r.released.as_deref().and_then(clean_date);
321    let cover_url = r
322        .images
323        .iter()
324        .find(|i| i.kind.as_deref() == Some("primary"))
325        .or_else(|| r.images.first())
326        .and_then(|i| i.uri.clone());
327    Release {
328        id: format!("discogs:{id}"),
329        title: r.title,
330        date: date.clone(),
331        country: r.country.filter(|c| !c.is_empty()),
332        status: None,
333        disambiguation: None,
334        artist_credit: artist_credits(&r.artists),
335        release_group: Some(ReleaseGroup {
336            id: String::new(),
337            first_release_date: date,
338            primary_type: None,
339        }),
340        label_info: r
341            .labels
342            .iter()
343            .map(|l| LabelInfo {
344                catalog_number: l.catno.clone().filter(|c| !c.is_empty()),
345                label: Some(Label {
346                    name: l.name.clone(),
347                }),
348            })
349            .collect(),
350        media: build_media(flatten_tracks(&r.tracklist, index_tracks)),
351        cover_url,
352    }
353}
354
355/// "Artist A (2)" credited to "Artist A" and "&"-joined to "Artist B" as
356/// "Artist A & Artist B", the way MusicBrainz's `artist_credit` reads.
357fn artist_credits(artists: &[DArtist]) -> Vec<ArtistCredit> {
358    artists
359        .iter()
360        .map(|a| {
361            let name = strip_disambiguation(&a.name);
362            let join = a.join.trim();
363            ArtistCredit {
364                joinphrase: if join.is_empty() {
365                    String::new()
366                } else {
367                    format!(" {join} ")
368                },
369                artist: Artist {
370                    id: String::new(),
371                    name: name.clone(),
372                },
373                name,
374            }
375        })
376        .collect()
377}
378
379/// The disambiguation suffix Discogs appends when two artists share a name,
380/// e.g. "Artist (2)".
381fn strip_disambiguation(name: &str) -> String {
382    static SUFFIX: LazyLock<Regex> =
383        LazyLock::new(|| Regex::new(r"^(.*) \(\d+\)$").expect("static regex"));
384    match SUFFIX.captures(name) {
385        Some(c) => c[1].to_string(),
386        None => name.to_string(),
387    }
388}
389
390/// "2019-00-00" as "2019", "2019-03-00" as "2019-03": trailing all-zero
391/// date parts, which mean "unknown", dropped rather than kept as zeroes.
392fn clean_date(s: &str) -> Option<String> {
393    let mut parts: Vec<&str> = s.trim().split('-').collect();
394    while matches!(parts.last(), Some(p) if !p.is_empty() && p.chars().all(|c| c == '0')) {
395        parts.pop();
396    }
397    (!parts.is_empty() && !parts[0].is_empty()).then(|| parts.join("-"))
398}
399
400/// "3:45" or "1:02:03" as milliseconds; blank is unknown.
401fn parse_duration(s: &str) -> Option<u64> {
402    let s = s.trim();
403    if s.is_empty() {
404        return None;
405    }
406    let parts: Vec<u64> = s
407        .split(':')
408        .map(|p| p.trim().parse().ok())
409        .collect::<Option<_>>()?;
410    let secs = match parts.as_slice() {
411        [m, s] => m * 60 + s,
412        [h, m, s] => h * 3600 + m * 60 + s,
413        _ => return None,
414    };
415    Some(secs * 1000)
416}
417
418struct FlatTrack {
419    position: String,
420    title: String,
421    duration: String,
422    artists: Vec<DArtist>,
423}
424
425/// Headings carry no track; an index track's `sub_tracks` are the real
426/// tracks, the index itself is not one.
427fn flatten_tracks(tracks: &[DTrack], index_tracks: bool) -> Vec<FlatTrack> {
428    let mut out = Vec::new();
429    for t in tracks {
430        match t.type_.as_str() {
431            "heading" => {}
432            "index" => {
433                for sub in &t.sub_tracks {
434                    let title = if index_tracks {
435                        format!("{}: {}", t.title, sub.title)
436                    } else {
437                        sub.title.clone()
438                    };
439                    out.push(FlatTrack {
440                        position: sub.position.clone(),
441                        title,
442                        duration: sub.duration.clone(),
443                        artists: if sub.artists.is_empty() {
444                            t.artists.clone()
445                        } else {
446                            sub.artists.clone()
447                        },
448                    });
449                }
450            }
451            _ => out.push(FlatTrack {
452                position: t.position.clone(),
453                title: t.title.clone(),
454                duration: t.duration.clone(),
455                artists: t.artists.clone(),
456            }),
457        }
458    }
459    out
460}
461
462static DISC_DASH: LazyLock<Regex> =
463    LazyLock::new(|| Regex::new(r"(?i)^[a-z]*(\d+)-(\d+)$").expect("static regex"));
464static DISC_DOT: LazyLock<Regex> =
465    LazyLock::new(|| Regex::new(r"^(\d+)\.(\d+)$").expect("static regex"));
466static DIGITS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^(\d+)$").expect("static regex"));
467
468/// "1-3"/"CD1-3" as disc 1 track 3, "2.3" as disc 2 track 3, a bare "3" as
469/// disc 1 track 3. Vinyl sides ("A1", "B2") and blank positions carry no
470/// disc/track pair Discogs numbers consistently, so they are counted in the
471/// order they appear, disc 1 throughout.
472fn parse_position(pos: &str, seq: &mut u32) -> (u32, u32) {
473    let pos = pos.trim();
474    if let Some(c) = DISC_DASH.captures(pos) {
475        return (c[1].parse().unwrap_or(1), c[2].parse().unwrap_or(1));
476    }
477    if let Some(c) = DISC_DOT.captures(pos) {
478        return (c[1].parse().unwrap_or(1), c[2].parse().unwrap_or(1));
479    }
480    if let Some(c) = DIGITS.captures(pos) {
481        return (1, c[1].parse().unwrap_or(1));
482    }
483    *seq += 1;
484    (1, *seq)
485}
486
487fn build_media(flat: Vec<FlatTrack>) -> Vec<Medium> {
488    let mut seq = 0u32;
489    let mut by_disc: std::collections::BTreeMap<u32, Vec<ReleaseTrack>> = Default::default();
490    for f in flat {
491        let (disc, position) = parse_position(&f.position, &mut seq);
492        by_disc.entry(disc).or_default().push(ReleaseTrack {
493            id: String::new(),
494            position,
495            title: f.title,
496            length: parse_duration(&f.duration),
497            artist_credit: artist_credits(&f.artists),
498            recording: Recording {
499                id: String::new(),
500                length: None,
501            },
502        });
503    }
504    by_disc
505        .into_iter()
506        .map(|(position, tracks)| Medium {
507            position,
508            format: None,
509            tracks,
510        })
511        .collect()
512}
513
514#[cfg(test)]
515mod tests {
516    use super::*;
517
518    #[test]
519    fn strips_the_discogs_disambiguation_suffix() {
520        assert_eq!(strip_disambiguation("Boards of Canada"), "Boards of Canada");
521        assert_eq!(strip_disambiguation("Artist (2)"), "Artist");
522        assert_eq!(strip_disambiguation("Artist (12)"), "Artist");
523    }
524
525    #[test]
526    fn drops_trailing_zero_date_parts() {
527        assert_eq!(clean_date("2019-00-00"), Some("2019".into()));
528        assert_eq!(clean_date("2019-03-00"), Some("2019-03".into()));
529        assert_eq!(clean_date("2002-02-18"), Some("2002-02-18".into()));
530        assert_eq!(clean_date(""), None);
531        assert_eq!(clean_date("0000-00-00"), None);
532    }
533
534    #[test]
535    fn parses_minute_and_hour_durations() {
536        assert_eq!(parse_duration("3:45"), Some(225_000));
537        assert_eq!(parse_duration("1:02:03"), Some(3_723_000));
538        assert_eq!(parse_duration(""), None);
539        assert_eq!(parse_duration("garbage"), None);
540    }
541
542    #[test]
543    fn vinyl_sides_are_counted_in_order_on_one_disc() {
544        let flat = vec!["A1", "A2", "B1"]
545            .into_iter()
546            .map(|p| FlatTrack {
547                position: p.into(),
548                title: p.into(),
549                duration: String::new(),
550                artists: Vec::new(),
551            })
552            .collect();
553        let media = build_media(flat);
554        assert_eq!(media.len(), 1);
555        assert_eq!(media[0].position, 1);
556        let positions: Vec<u32> = media[0].tracks.iter().map(|t| t.position).collect();
557        assert_eq!(positions, [1, 2, 3]);
558    }
559
560    #[test]
561    fn disc_and_track_come_from_the_numeric_forms() {
562        let mut seq = 0;
563        assert_eq!(parse_position("1-3", &mut seq), (1, 3));
564        assert_eq!(parse_position("CD1-3", &mut seq), (1, 3));
565        assert_eq!(parse_position("2.3", &mut seq), (2, 3));
566        assert_eq!(parse_position("3", &mut seq), (1, 3));
567    }
568
569    #[test]
570    fn an_index_tracks_sub_tracks_are_flattened() {
571        let json = r#"[
572            {"position":"1","type_":"track","title":"Intro","duration":"1:00"},
573            {"type_":"index","title":"Medley","sub_tracks":[
574                {"position":"2","title":"Part One","duration":"2:00"},
575                {"position":"3","title":"Part Two","duration":"3:00"}
576            ]}
577        ]"#;
578        let tracks: Vec<DTrack> = serde_json::from_str(json).unwrap();
579        let plain = flatten_tracks(&tracks, false);
580        assert_eq!(
581            plain.iter().map(|t| t.title.as_str()).collect::<Vec<_>>(),
582            ["Intro", "Part One", "Part Two"]
583        );
584        let prefixed = flatten_tracks(&tracks, true);
585        assert_eq!(
586            prefixed
587                .iter()
588                .map(|t| t.title.as_str())
589                .collect::<Vec<_>>(),
590            ["Intro", "Medley: Part One", "Medley: Part Two"]
591        );
592    }
593
594    #[test]
595    fn headings_carry_no_track() {
596        let json = r#"[
597            {"type_":"heading","title":"Side A"},
598            {"position":"A1","type_":"track","title":"Song","duration":"3:00"}
599        ]"#;
600        let tracks: Vec<DTrack> = serde_json::from_str(json).unwrap();
601        let flat = flatten_tracks(&tracks, false);
602        assert_eq!(flat.len(), 1);
603        assert_eq!(flat[0].title, "Song");
604    }
605}
sift/discogs.rs

sift/
discogs.rs
