omni_dev/transcript/sources/youtube/
metadata.rs

1//! Per-video metadata sidecar: the serde view of the WEB `/player` response
2//! and its projection to the [`VideoMetadata`] domain struct written to
3//! `<out>/<channel-id>/<video-id>.meta.yaml`.
4//!
5//! The transcript path pins the `ANDROID_VR` client (see [`super::innertube`]),
6//! whose `/player` response carries `videoDetails` but **no `microformat`** —
7//! so it lacks publish date, like count, and category. A plain WEB-client
8//! `/player` call (un-gated, no `visitorData`) supplies both `videoDetails`
9//! and `microformat.playerMicroformatRenderer`, so metadata fetching is fully
10//! independent of the bot-gated transcript path.
11//!
12//! ## Refresh signal
13//!
14//! The WEB `microformat` shape can drift independently of the `ANDROID_VR`
15//! constants. If [`parse`] starts failing or returning empty metadata for
16//! known-healthy videos, the field paths below (`publishDate`, `likeCount`,
17//! `microformat.playerMicroformatRenderer`, …) are the place to re-check.
18
19use chrono::{DateTime, Utc};
20use serde::{Deserialize, Serialize};
21
22use crate::transcript::error::{Result, TranscriptError};
23
24/// Current sidecar schema version. Bumped only on a breaking field change;
25/// new optional fields can be added under the same version.
26pub const SCHEMA_VERSION: u32 = 1;
27
28/// Per-video metadata sidecar, serialised to `<video-id>.meta.yaml`.
29///
30/// `view_count` / `like_count` are point-in-time snapshots; [`Self::fetched_at`]
31/// is what makes them honest and is the refresh-staleness key. Fields sourced
32/// from `microformat` are [`Option`] because that block is absent for
33/// private/removed videos, in which case the sidecar is written from
34/// `videoDetails` alone.
35#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
36pub struct VideoMetadata {
37    /// Sidecar schema version (always [`SCHEMA_VERSION`] when written).
38    #[serde(default = "default_schema")]
39    pub schema: u32,
40    /// 11-character video ID.
41    pub video_id: String,
42    /// Display title.
43    pub title: String,
44    /// Channel / uploader name.
45    #[serde(default, skip_serializing_if = "Option::is_none")]
46    pub channel: Option<String>,
47    /// `UC…` channel ID.
48    #[serde(default, skip_serializing_if = "Option::is_none")]
49    pub channel_id: Option<String>,
50    /// Owner profile URL (the `@handle` URL). From `microformat`.
51    #[serde(default, skip_serializing_if = "Option::is_none")]
52    pub channel_url: Option<String>,
53    /// Video category (e.g. `Music`). From `microformat`.
54    #[serde(default, skip_serializing_if = "Option::is_none")]
55    pub category: Option<String>,
56    /// Publish timestamp, full ISO 8601 with original offset preserved
57    /// (e.g. `2009-10-24T23:57:33-07:00`). From `microformat.publishDate`.
58    #[serde(default, skip_serializing_if = "Option::is_none")]
59    pub published_at: Option<String>,
60    /// Duration in seconds.
61    #[serde(default, skip_serializing_if = "Option::is_none")]
62    pub duration_seconds: Option<u64>,
63    /// Full description (`videoDetails.shortDescription`, despite the name).
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub description: Option<String>,
66    /// Free-text keywords/tags.
67    #[serde(default, skip_serializing_if = "Vec::is_empty")]
68    pub keywords: Vec<String>,
69    /// View count at fetch time.
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub view_count: Option<u64>,
72    /// Like count at fetch time. Absent when ratings are disabled.
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    pub like_count: Option<u64>,
75    /// Whether this is (or was) a live broadcast.
76    #[serde(default)]
77    pub is_live_content: bool,
78    /// Whether the video is unlisted. From `microformat`.
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub is_unlisted: Option<bool>,
81    /// Best available thumbnail URL (maxres when present).
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub thumbnail_url: Option<String>,
84    /// When this sidecar was fetched (UTC). The refresh-staleness key.
85    pub fetched_at: DateTime<Utc>,
86}
87
88fn default_schema() -> u32 {
89    SCHEMA_VERSION
90}
91
92/// Project a raw WEB `/player` response into a [`VideoMetadata`].
93///
94/// `fetched_at` is supplied by the caller (the fetch timestamp) rather than
95/// read from a clock here, keeping the projection a pure, unit-testable
96/// function. Tolerant of an absent `microformat` block: the sidecar is then
97/// written from `videoDetails` alone (microformat-only fields stay `None`).
98/// Fails with [`TranscriptError::ParseError`] only if `videoDetails` itself is
99/// missing (a removed/blocked video the WEB client won't describe).
100pub fn parse(raw: &str, fetched_at: DateTime<Utc>) -> Result<VideoMetadata> {
101    let web: WebPlayerResponse = serde_json::from_str(raw)
102        .map_err(|e| TranscriptError::ParseError(format!("WEB playerResponse: {e}")))?;
103
104    let details = web.video_details.ok_or_else(|| {
105        TranscriptError::ParseError("WEB playerResponse: missing videoDetails".to_string())
106    })?;
107    let micro = web.microformat.and_then(|m| m.player_microformat_renderer);
108
109    // viewCount appears in both blocks; videoDetails is the live snapshot.
110    let view_count = details
111        .view_count
112        .as_ref()
113        .and_then(Count::to_u64)
114        .or_else(|| {
115            micro
116                .as_ref()
117                .and_then(|m| m.view_count.as_ref())
118                .and_then(Count::to_u64)
119        });
120
121    let thumbnail_url = micro
122        .as_ref()
123        .and_then(|m| m.thumbnail.as_ref())
124        .and_then(ThumbnailList::best)
125        .or_else(|| details.thumbnail.as_ref().and_then(ThumbnailList::best));
126
127    let description = details.short_description.filter(|s| !s.is_empty());
128
129    Ok(VideoMetadata {
130        schema: SCHEMA_VERSION,
131        video_id: details.video_id,
132        title: details.title,
133        channel: details
134            .author
135            .or_else(|| micro.as_ref().and_then(|m| m.owner_channel_name.clone())),
136        channel_id: details
137            .channel_id
138            .or_else(|| micro.as_ref().and_then(|m| m.external_channel_id.clone())),
139        channel_url: micro.as_ref().and_then(|m| m.owner_profile_url.clone()),
140        category: micro.as_ref().and_then(|m| m.category.clone()),
141        published_at: micro.as_ref().and_then(|m| m.publish_date.clone()),
142        duration_seconds: details.length_seconds.as_ref().and_then(Count::to_u64),
143        description,
144        keywords: details.keywords,
145        view_count,
146        like_count: micro
147            .as_ref()
148            .and_then(|m| m.like_count.as_ref())
149            .and_then(Count::to_u64),
150        is_live_content: details.is_live_content.unwrap_or(false),
151        is_unlisted: micro.as_ref().and_then(|m| m.is_unlisted),
152        thumbnail_url,
153        fetched_at,
154    })
155}
156
157/// Read just the `fetched_at` stamp from an existing sidecar.
158///
159/// Used for staleness decisions during a directory scan. Returns `None` if the
160/// YAML is corrupt or carries no parseable `fetched_at` — callers treat that as
161/// "sidecar missing" and re-fetch.
162pub fn read_fetched_at(yaml: &str) -> Option<DateTime<Utc>> {
163    serde_yaml::from_str::<SidecarStamp>(yaml)
164        .ok()
165        .map(|s| s.fetched_at)
166}
167
168/// Minimal view used only to recover the staleness key from an on-disk
169/// sidecar without requiring the rest of the document to be well-formed.
170#[derive(Deserialize)]
171struct SidecarStamp {
172    fetched_at: DateTime<Utc>,
173}
174
175// ── Serde view of the WEB `/player` response ──
176
177#[derive(Debug, Deserialize)]
178#[serde(rename_all = "camelCase")]
179struct WebPlayerResponse {
180    #[serde(default)]
181    video_details: Option<WebVideoDetails>,
182    #[serde(default)]
183    microformat: Option<Microformat>,
184}
185
186#[derive(Debug, Deserialize)]
187#[serde(rename_all = "camelCase")]
188struct WebVideoDetails {
189    video_id: String,
190    title: String,
191    #[serde(default)]
192    author: Option<String>,
193    #[serde(default)]
194    channel_id: Option<String>,
195    #[serde(default)]
196    length_seconds: Option<Count>,
197    #[serde(default)]
198    short_description: Option<String>,
199    #[serde(default)]
200    keywords: Vec<String>,
201    #[serde(default)]
202    view_count: Option<Count>,
203    #[serde(default)]
204    is_live_content: Option<bool>,
205    #[serde(default)]
206    thumbnail: Option<ThumbnailList>,
207}
208
209#[derive(Debug, Deserialize)]
210#[serde(rename_all = "camelCase")]
211struct Microformat {
212    #[serde(default)]
213    player_microformat_renderer: Option<PlayerMicroformatRenderer>,
214}
215
216#[derive(Debug, Deserialize)]
217#[serde(rename_all = "camelCase")]
218struct PlayerMicroformatRenderer {
219    #[serde(default)]
220    publish_date: Option<String>,
221    #[serde(default)]
222    like_count: Option<Count>,
223    #[serde(default)]
224    view_count: Option<Count>,
225    #[serde(default)]
226    category: Option<String>,
227    #[serde(default)]
228    owner_channel_name: Option<String>,
229    #[serde(default)]
230    owner_profile_url: Option<String>,
231    #[serde(default)]
232    external_channel_id: Option<String>,
233    #[serde(default)]
234    is_unlisted: Option<bool>,
235    #[serde(default)]
236    thumbnail: Option<ThumbnailList>,
237}
238
239#[derive(Debug, Deserialize)]
240#[serde(rename_all = "camelCase")]
241struct ThumbnailList {
242    #[serde(default)]
243    thumbnails: Vec<Thumbnail>,
244}
245
246impl ThumbnailList {
247    /// The highest-resolution thumbnail URL (YouTube lists ascending, so the
248    /// last entry is the largest).
249    fn best(&self) -> Option<String> {
250        self.thumbnails.last().and_then(|t| t.url.clone())
251    }
252}
253
254#[derive(Debug, Deserialize)]
255struct Thumbnail {
256    #[serde(default)]
257    url: Option<String>,
258}
259
260/// A count field that YouTube encodes inconsistently as either a JSON string
261/// (`"123"`) or a number (`123`) depending on client and block. Accepting both
262/// keeps a string/number drift from failing the whole sidecar.
263#[derive(Debug, Deserialize)]
264#[serde(untagged)]
265enum Count {
266    Str(String),
267    Num(u64),
268}
269
270impl Count {
271    fn to_u64(&self) -> Option<u64> {
272        match self {
273            Self::Str(s) => s.parse().ok(),
274            Self::Num(n) => Some(*n),
275        }
276    }
277}
278
279#[cfg(test)]
280#[allow(clippy::unwrap_used, clippy::expect_used)]
281mod tests {
282    use super::*;
283
284    const WITH_MICROFORMAT: &str = include_str!("fixtures/player_response_web_metadata.json");
285    const NO_MICROFORMAT: &str = include_str!("fixtures/player_response_web_no_microformat.json");
286
287    fn fixed_fetched_at() -> DateTime<Utc> {
288        "2026-06-11T03:12:45Z".parse().unwrap()
289    }
290
291    #[test]
292    fn parse_projects_full_metadata_with_microformat() {
293        let meta = parse(WITH_MICROFORMAT, fixed_fetched_at()).unwrap();
294        assert_eq!(meta.schema, SCHEMA_VERSION);
295        assert_eq!(meta.video_id, "dQw4w9WgXcQ");
296        assert_eq!(meta.title, "Rick Astley - Never Gonna Give You Up");
297        assert_eq!(meta.channel.as_deref(), Some("Rick Astley"));
298        assert_eq!(meta.channel_id.as_deref(), Some("UCuAXFkgsw1L7xaCfnd5JJOw"));
299        assert_eq!(
300            meta.channel_url.as_deref(),
301            Some("http://www.youtube.com/@RickAstleyYT")
302        );
303        assert_eq!(meta.category.as_deref(), Some("Music"));
304        assert_eq!(
305            meta.published_at.as_deref(),
306            Some("2009-10-24T23:57:33-07:00")
307        );
308        assert_eq!(meta.duration_seconds, Some(213));
309        assert!(meta
310            .description
311            .as_deref()
312            .unwrap()
313            .contains("official video"));
314        assert_eq!(
315            meta.keywords,
316            vec!["rick astley", "Never Gonna Give You Up"]
317        );
318        assert_eq!(meta.view_count, Some(1_781_429_760));
319        assert_eq!(meta.like_count, Some(19_148_727));
320        assert!(!meta.is_live_content);
321        assert_eq!(meta.is_unlisted, Some(false));
322        assert_eq!(
323            meta.thumbnail_url.as_deref(),
324            Some("https://i.ytimg.com/vi/dQw4w9WgXcQ/maxresdefault.jpg")
325        );
326        assert_eq!(meta.fetched_at, fixed_fetched_at());
327    }
328
329    #[test]
330    fn parse_tolerates_absent_microformat() {
331        let meta = parse(NO_MICROFORMAT, fixed_fetched_at()).unwrap();
332        assert_eq!(meta.video_id, "noMicro1234");
333        assert_eq!(meta.title, "Private-ish video");
334        assert_eq!(meta.channel.as_deref(), Some("Some Channel"));
335        // microformat-only fields fall away rather than failing the parse
336        assert_eq!(meta.published_at, None);
337        assert_eq!(meta.like_count, None);
338        assert_eq!(meta.category, None);
339        assert_eq!(meta.is_unlisted, None);
340        // videoDetails-sourced fields still populate
341        assert_eq!(meta.view_count, Some(42));
342        assert_eq!(meta.duration_seconds, Some(100));
343    }
344
345    #[test]
346    fn parse_errors_when_video_details_missing() {
347        let err = parse(
348            r#"{"playabilityStatus":{"status":"ERROR"}}"#,
349            fixed_fetched_at(),
350        )
351        .unwrap_err();
352        assert!(matches!(err, TranscriptError::ParseError(_)));
353    }
354
355    #[test]
356    fn parse_errors_on_malformed_json() {
357        let err = parse("{ not json", fixed_fetched_at()).unwrap_err();
358        assert!(matches!(err, TranscriptError::ParseError(_)));
359    }
360
361    #[test]
362    fn count_accepts_string_or_number() {
363        // viewCount as a bare number (microformat sometimes does this) still
364        // projects to u64 rather than failing the sidecar.
365        let raw = r#"{"videoDetails":{"videoId":"x","title":"t","viewCount":7}}"#;
366        let meta = parse(raw, fixed_fetched_at()).unwrap();
367        assert_eq!(meta.view_count, Some(7));
368    }
369
370    #[test]
371    fn serialised_sidecar_round_trips_and_skips_empties() {
372        let meta = parse(NO_MICROFORMAT, fixed_fetched_at()).unwrap();
373        let yaml = serde_yaml::to_string(&meta).unwrap();
374        // microformat-only fields are omitted when absent
375        assert!(!yaml.contains("like_count"));
376        assert!(!yaml.contains("category"));
377        // required fields are present
378        assert!(yaml.contains("schema: 1"));
379        assert!(yaml.contains("video_id: noMicro1234"));
380        assert!(yaml.contains("fetched_at:"));
381        let back: VideoMetadata = serde_yaml::from_str(&yaml).unwrap();
382        assert_eq!(back, meta);
383    }
384
385    #[test]
386    fn read_fetched_at_recovers_stamp() {
387        let meta = parse(WITH_MICROFORMAT, fixed_fetched_at()).unwrap();
388        let yaml = serde_yaml::to_string(&meta).unwrap();
389        assert_eq!(read_fetched_at(&yaml), Some(fixed_fetched_at()));
390    }
391
392    #[test]
393    fn read_fetched_at_returns_none_for_corrupt_or_missing() {
394        assert_eq!(read_fetched_at("{ not yaml"), None);
395        assert_eq!(read_fetched_at("schema: 1\nvideo_id: x\n"), None);
396    }
397}
omni_dev/transcript/sources/youtube/metadata.rs

omni_dev/transcript/sources/youtube/
metadata.rs
