agent-p 0.4.0

//! RSS feed fetching: latest episode title, full episode list, and channel metadata.

use anyhow::{Context, Result};
use reqwest::StatusCode;
use rss::Channel;

use super::data::{EpisodeInfo, Podcast};

/// Channel-level metadata extracted from an RSS feed.
#[derive(Debug, Clone)]
pub struct FeedMetadata {
    pub title: String,
    pub author: String,
}

async fn fetch_channel(url: &str) -> Result<Channel> {
    let response = super::http::client()
        .get(url)
        .send()
        .await
        .context("could not reach the feed")?;

    let status = response.status();
    let content_type = response
        .headers()
        .get(reqwest::header::CONTENT_TYPE)
        .and_then(|v| v.to_str().ok())
        .unwrap_or("")
        .to_ascii_lowercase();
    let host = response.url().host_str().unwrap_or("unknown").to_string();
    let body = response
        .bytes()
        .await
        .context("failed to read the feed response")?;

    parse_feed_response(url, &host, status, &content_type, &body)
}

fn parse_feed_response(
    url: &str,
    host: &str,
    status: StatusCode,
    content_type: &str,
    body: &[u8],
) -> Result<Channel> {
    let is_web_page = content_type.starts_with("text/") && !content_type.contains("xml");
    let requested_host = reqwest::Url::parse(url)
        .ok()
        .and_then(|u| u.host_str().map(str::to_ascii_lowercase));
    let redirected_away = requested_host.is_some_and(|h| h != host.to_ascii_lowercase());
    let filter_page = is_web_page && redirected_away;
    if !status.is_success() {
        if filter_page {
            anyhow::bail!(
                "the feed server returned HTTP {status} and a web page - {}",
                super::http::filter_block_message(host, url)
            );
        }
        anyhow::bail!("the feed server returned HTTP {status}");
    }
    match Channel::read_from(body) {
        Ok(channel) => Ok(channel),
        Err(_) if filter_page => {
            anyhow::bail!(super::http::filter_block_message(host, url))
        }
        Err(_) if is_web_page => {
            anyhow::bail!("the feed URL returned a web page instead of an RSS feed")
        }
        Err(error) => Err(error).context("the response is not a valid RSS feed"),
    }
}

/// Fetch channel-level metadata (title and author) from an RSS feed URL.
pub async fn fetch_feed_metadata(feed_url: &str) -> Result<FeedMetadata> {
    let channel = fetch_channel(feed_url).await?;
    let title = channel.title().to_string();
    let author = channel
        .itunes_ext()
        .and_then(|e| e.author())
        .or_else(|| channel.managing_editor())
        .unwrap_or("")
        .to_string();
    Ok(FeedMetadata { title, author })
}

/// Fetch the title and publish date of the most recent episode from the feed.
pub async fn get_last_podcast_name(podcast: &Podcast) -> Result<(String, Option<String>)> {
    let channel = fetch_channel(&podcast.feed_url).await?;
    let item = channel.items.first().context("RSS feed has no episodes")?;
    let episode_title = item
        .title
        .as_ref()
        .context("Episode has no title")?
        .to_string();
    let pub_date = item.pub_date.clone();
    Ok((episode_title, pub_date))
}

const BLOCK_TAGS: &[&str] = &[
    "address",
    "article",
    "blockquote",
    "br",
    "dd",
    "div",
    "dl",
    "dt",
    "footer",
    "h1",
    "h2",
    "h3",
    "h4",
    "h5",
    "h6",
    "header",
    "hr",
    "li",
    "ol",
    "p",
    "pre",
    "section",
    "table",
    "td",
    "th",
    "tr",
    "ul",
];

/// Whether the tag body between `<` and `>` names a block-level element,
/// which separates words, as opposed to inline markup such as `<b>`.
fn is_block_tag(tag: &str) -> bool {
    let name: String = tag
        .trim_start_matches('/')
        .chars()
        .take_while(char::is_ascii_alphanumeric)
        .collect::<String>()
        .to_ascii_lowercase();
    BLOCK_TAGS.contains(&name.as_str())
}

/// Decode the character reference at the start of `text` (which begins with
/// `&`): numeric references and the named entities feeds commonly use.
/// Returns the character and the reference's length in bytes.
fn decode_entity(text: &str) -> Option<(char, usize)> {
    let end = text
        .char_indices()
        .take(12)
        .find(|&(_, c)| c == ';')
        .map(|(i, _)| i)?;
    let name = &text[1..end];
    let decoded = if let Some(number) = name.strip_prefix('#') {
        let code = match number.strip_prefix(['x', 'X']) {
            Some(hex) => u32::from_str_radix(hex, 16).ok()?,
            None => number.parse().ok()?,
        };
        char::from_u32(code)?
    } else {
        match name {
            "amp" => '&',
            "lt" => '<',
            "gt" => '>',
            "quot" => '"',
            "apos" => '\'',
            "nbsp" => ' ',
            "ndash" => '–',
            "mdash" => '—',
            "lsquo" => '‘',
            "rsquo" => '’',
            "ldquo" => '“',
            "rdquo" => '”',
            "hellip" => '…',
            "bull" => '•',
            "middot" => '·',
            "copy" => '©',
            "reg" => '®',
            "trade" => '™',
            "deg" => '°',
            "eacute" => 'é',
            "egrave" => 'è',
            "aacute" => 'á',
            "iacute" => 'í',
            "oacute" => 'ó',
            "uacute" => 'ú',
            "ntilde" => 'ñ',
            "uuml" => 'ü',
            "ouml" => 'ö',
            "auml" => 'ä',
            "ccedil" => 'ç',
            _ => return None,
        }
    };
    Some((decoded, end + 1))
}

/// Reduce a feed's HTML description to searchable plain text: tags dropped,
/// with block-level tags separating words and inline ones not, character
/// references decoded, whitespace collapsed. A `<` that does not open a tag,
/// as in `<3`, is kept as text.
fn plain_text(html: &str) -> String {
    let mut text = String::with_capacity(html.len());
    let mut rest = html;
    while let Some(c) = rest.chars().next() {
        if c == '<'
            && rest[1..]
                .starts_with(|n: char| n.is_ascii_alphabetic() || matches!(n, '/' | '!' | '?'))
            && let Some(end) = rest.find('>')
        {
            if is_block_tag(&rest[1..end]) {
                text.push(' ');
            }
            rest = &rest[end + 1..];
            continue;
        }
        if c == '&'
            && let Some((decoded, len)) = decode_entity(rest)
        {
            text.push(decoded);
            rest = &rest[len..];
            continue;
        }
        text.push(c);
        rest = &rest[c.len_utf8()..];
    }
    text.split_whitespace().collect::<Vec<_>>().join(" ")
}

/// The first of an item's description, iTunes summary, and content that still
/// has text once its HTML is stripped.
fn episode_description(item: &rss::Item) -> String {
    [
        item.description(),
        item.itunes_ext().and_then(|ext| ext.summary()),
        item.content(),
    ]
    .into_iter()
    .flatten()
    .map(plain_text)
    .find(|text| !text.is_empty())
    .unwrap_or_default()
}

/// Fetch the full episode list from an RSS feed URL.
pub async fn fetch_episode_list(feed_url: &str) -> Result<Vec<EpisodeInfo>> {
    let channel = fetch_channel(feed_url).await?;
    let length = channel.items.len();

    let episodes: Vec<EpisodeInfo> = channel
        .items
        .iter()
        .enumerate()
        .map(|(rss_idx, item)| {
            let title = item
                .title
                .as_ref()
                .cloned()
                .unwrap_or_else(|| format!("Episode {}", length - rss_idx));
            let feed_index = length - rss_idx;
            let pub_date = item.pub_date.clone();
            let description = episode_description(item);
            EpisodeInfo {
                title,
                feed_index,
                pub_date,
                description,
            }
        })
        .collect();

    Ok(episodes)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn plain_text_drops_tags_and_decodes_entities() {
        assert_eq!(
            plain_text(
                "<p>Fish &amp; chips</p><p>with&nbsp;<b>vinegar</b></p>
"
            ),
            "Fish & chips with vinegar"
        );
        assert_eq!(plain_text(""), "");
    }

    #[test]
    fn inline_tags_keep_words_whole_and_block_tags_separate_them() {
        assert_eq!(plain_text("<b>Fish</b>ing<br/>trip"), "Fishing trip");
        assert_eq!(plain_text("<li>one</li><li>two</li>"), "one two");
    }

    #[test]
    fn a_tag_only_description_falls_back_to_the_content() {
        let item = rss::Item {
            description: Some("<p></p>".to_string()),
            content: Some("<p>The real notes</p>".to_string()),
            ..Default::default()
        };
        assert_eq!(episode_description(&item), "The real notes");
    }

    #[test]
    fn a_bare_angle_bracket_is_text() {
        assert_eq!(plain_text("I <3 this > that"), "I <3 this > that");
        assert_eq!(plain_text("a < b"), "a < b");
    }

    #[test]
    fn numeric_and_named_references_decode() {
        assert_eq!(
            plain_text("It&#8217;s &ldquo;here&rdquo; &#x2014; caf&eacute; &amp;lt;"),
            "It’s “here” — café &lt;"
        );
        assert_eq!(
            plain_text("&unknown; &#xZZ; & alone"),
            "&unknown; &#xZZ; & alone"
        );
    }

    const FEED: &str = r#"<?xml version="1.0"?><rss version="2.0"><channel><title>Show</title><link>https://example.com</link><description>d</description></channel></rss>"#;
    const PAGE: &str = "<html><body>Blocked by your administrator</body></html>";
    const URL: &str = "https://feeds.example.com/show";

    fn parse(status: u16, content_type: &str, body: &str) -> Result<Channel> {
        let status = StatusCode::from_u16(status).unwrap();
        parse_feed_response(URL, "filter.corp", status, content_type, body.as_bytes())
    }

    #[test]
    fn feed_served_as_plain_text_still_parses() {
        assert_eq!(parse(200, "text/plain", FEED).unwrap().title(), "Show");
    }

    #[test]
    fn rss_content_types_parse() {
        for content_type in ["application/rss+xml", "text/xml", ""] {
            assert_eq!(parse(200, content_type, FEED).unwrap().title(), "Show");
        }
    }

    #[test]
    fn web_page_instead_of_feed_is_reported_as_filter_block() {
        let message = parse(200, "text/html; charset=utf-8", PAGE)
            .unwrap_err()
            .to_string();
        assert!(message.contains("(internet-filter/proxy) blocked URL"));
        assert!(message.contains("filter.corp"));
        assert!(message.contains(URL));
    }

    #[test]
    fn error_status_is_shown_and_flags_a_filter_only_for_web_pages() {
        let page = parse(403, "text/html", PAGE).unwrap_err().to_string();
        assert!(page.contains("HTTP 403"));
        assert!(page.contains("internet-filter/proxy"));

        let plain = parse(404, "application/xml", "").unwrap_err().to_string();
        assert!(plain.contains("HTTP 404"));
        assert!(!plain.contains("internet-filter/proxy"));
    }

    #[test]
    fn web_page_from_the_feed_host_itself_is_not_blamed_on_a_filter() {
        let not_found = StatusCode::from_u16(404).unwrap();
        let missing = parse_feed_response(
            URL,
            "feeds.example.com",
            not_found,
            "text/html",
            PAGE.as_bytes(),
        )
        .unwrap_err()
        .to_string();
        assert!(missing.contains("HTTP 404"));
        assert!(!missing.contains("internet-filter/proxy"));

        let ok = StatusCode::from_u16(200).unwrap();
        let page = parse_feed_response(URL, "FEEDS.example.com", ok, "text/html", PAGE.as_bytes())
            .unwrap_err()
            .to_string();
        assert!(page.contains("web page instead of an RSS feed"));
        assert!(!page.contains("internet-filter/proxy"));
    }

    #[test]
    fn filter_message_drops_query_tokens() {
        let message = super::super::http::filter_block_message(
            "filter.corp",
            "https://feeds.example.com/show?token=secret#frag",
        );
        assert!(message.contains("https://feeds.example.com/show"));
        assert!(!message.contains("secret"));
    }

    #[test]
    fn invalid_non_page_body_is_not_blamed_on_a_filter() {
        let message = parse(200, "application/xml", "not a feed")
            .unwrap_err()
            .to_string();
        assert!(message.contains("not a valid RSS feed"));
        assert!(!message.contains("internet-filter/proxy"));
    }
}