news_flash/util/

greader.rs

use crate::feed_api::Portal;
use crate::models::{
    ArticleID, Category, CategoryID, CategoryMapping, Direction, Enclosure, FatArticle, Feed, FeedID, FeedMapping, Headline, Marked,
    NEWSFLASH_TOPLEVEL, Read, StreamConversionResult, Tag, TagID, Tagging, Url,
};
use crate::{error::FeedApiError, util};
use chrono::Utc;
use futures::future;
use greader_api::models::{
    Category as GCategory, Feed as GFeed, Item, ItemId, ItemRefs, Stream, StreamPrefs, Summary, Tagging as GTagging, Taggings,
};
use greader_api::{ApiError as GReaderError, GReaderApi};
use reqwest::Client;
use std::collections::HashSet;
use std::convert::TryInto;
use std::sync::Arc;
use tokio::sync::RwLock;

pub const TAG_READ_STR: &str = "user/-/state/com.google/read";
pub const TAG_STARRED_STR: &str = "user/-/state/com.google/starred";
pub const TAG_READING_LIST: &str = "user/-/state/com.google/reading-list";

pub const GOOGLE_ITEM_PREFIX: &str = "tag:google.com,2005:reader/item/";

impl From<GReaderError> for FeedApiError {
    fn from(error: GReaderError) -> FeedApiError {
        match error {
            GReaderError::ApiLimit => FeedApiError::ApiLimit,
            GReaderError::Url(e) => FeedApiError::Url(e),
            GReaderError::Json { source, json } => FeedApiError::Json { source, json },
            GReaderError::Http(e) => FeedApiError::Network(e),
            GReaderError::GReader(greader_error) => {
                let error_list = greader_error.errors.iter().fold("".into(), |prev, next| format!("{prev}\n{next}"));
                FeedApiError::Api {
                    message: format!("GReader Error:\n{error_list}"),
                }
            }
            GReaderError::BadRequest => FeedApiError::Api {
                message: GReaderError::BadRequest.to_string(),
            },
            GReaderError::Input => FeedApiError::Api {
                message: GReaderError::Input.to_string(),
            },
            GReaderError::Token => FeedApiError::Api {
                message: GReaderError::Token.to_string(),
            },
            GReaderError::TokenExpired => FeedApiError::Api {
                message: GReaderError::TokenExpired.to_string(),
            },
            GReaderError::AccessDenied => FeedApiError::Auth,
            GReaderError::Parse => FeedApiError::Api {
                message: GReaderError::Parse.to_string(),
            },
            GReaderError::NotLoggedIn => FeedApiError::Login,
            GReaderError::Other(msg) => FeedApiError::Api { message: msg },
        }
    }
}

pub struct ArticleQuery<'a> {
    pub stream_id: Option<&'a str>,
    pub read: Option<Read>,
    pub marked: Option<Marked>,
    pub tag_ids: &'a HashSet<TagID>,
    pub limit: Option<u64>,
    pub last_sync: Option<i64>,
}

pub struct GReaderUtil;

impl GReaderUtil {
    pub fn generate_tag_id(user_id: Option<&str>, name: &str) -> String {
        let user_id = user_id.unwrap_or("-");
        format!("user/{user_id}/label/{name}")
    }

    pub fn root_id(user_id: Option<&str>) -> String {
        let user_id = user_id.unwrap_or("-");
        format!("user/{user_id}/state/com.google/root")
    }

    pub fn convert_category(
        category: GCategory,
        index: i32,
        taggings: Option<&Taggings>,
        prefs: Option<&StreamPrefs>,
        user_id: Option<&str>,
    ) -> (Category, CategoryMapping) {
        let sort_index = if let Some(tagging) = Self::find_tagging(taggings, &category.id) {
            Self::convert_sortid(Some(&Self::root_id(user_id)), tagging.sortid.as_deref(), prefs)
        } else {
            None
        };

        let GCategory { id, label } = category;
        let category_id = CategoryID::new(&id);
        let category = Category {
            category_id: category_id.clone(),
            label,
        };
        let category_mapping = CategoryMapping {
            parent_id: NEWSFLASH_TOPLEVEL.clone(),
            category_id,
            sort_index: if sort_index.is_none() { Some(index) } else { sort_index },
        };

        (category, category_mapping)
    }

    pub fn find_tagging<'a>(taggings: Option<&'a Taggings>, id: &str) -> Option<&'a GTagging> {
        if let Some(taggings) = taggings {
            taggings.tags.iter().find(|t| t.id == id)
        } else {
            None
        }
    }

    pub fn convert_category_vec(
        mut categories: Vec<GFeed>,
        taggings: Option<&Taggings>,
        prefs: Option<&StreamPrefs>,
        user_id: Option<&str>,
    ) -> (Vec<Category>, Vec<CategoryMapping>) {
        // skip duplicate categories by collecting all ids
        let mut category_ids: HashSet<CategoryID> = HashSet::new();

        // normalize index by only issuing a new index when a new unique category is found
        let mut index = 0;

        categories
            .drain(..)
            .filter_map(|feed| {
                let feed_categories: Vec<(Category, CategoryMapping)> = feed
                    .categories
                    .into_iter()
                    .filter_map(|c| {
                        let (category, category_mapping) = Self::convert_category(c, index, taggings, prefs, user_id);
                        if category_ids.contains(&category.category_id) {
                            None
                        } else {
                            index += 1;
                            category_ids.insert(category.category_id.clone());
                            Some((category, category_mapping))
                        }
                    })
                    .collect();

                if feed_categories.is_empty() { None } else { Some(feed_categories) }
            })
            .flatten()
            .unzip()
    }

    pub fn convert_tag_list(taggings: Taggings, categories: &[Category]) -> Vec<Tag> {
        let Taggings { tags: tag_list } = taggings;

        tag_list
            .into_iter()
            .filter_map(|tagging| {
                let GTagging {
                    id: tag_id,
                    r#type: _,
                    sortid,
                    unread_count: _,
                    unseen_count: _,
                } = tagging;

                // tags and folder both have the structure 'user/$USERID/label/$NAME'
                if tag_id.contains("/label/") {
                    // if 'tag_id' is a category ignore it
                    if categories.iter().any(|c| c.category_id.as_str() == tag_id) {
                        None
                    } else {
                        let label = tag_id
                            .split('/')
                            .next_back()
                            .map(|s| s.to_owned())
                            .unwrap_or_else(|| "Missing Label".into());

                        let sort_index = sortid
                            .and_then(|str| hex::decode(str).ok())
                            .and_then(|buf| buf.try_into().ok())
                            .map(|buf| u32::from_le_bytes(buf) as i32);

                        Some(Tag {
                            tag_id: TagID::from_owned(tag_id),
                            label,
                            color: None,
                            sort_index,
                        })
                    }
                } else {
                    None
                }
            })
            .collect()
    }

    pub fn convert_feed(feed: GFeed) -> Feed {
        let GFeed {
            id,
            title,
            categories: _,
            url,
            html_url,
            icon_url,
            sortid: _,
        } = feed;

        Feed {
            feed_id: FeedID::new(&id),
            label: title,
            website: Url::parse(&html_url).ok(),
            feed_url: Url::parse(&url).ok(),
            icon_url: Url::parse(&icon_url).ok(),
            error_count: 0,
            error_message: None,
        }
    }

    pub fn convert_sortid(parent_id: Option<&str>, sort_id: Option<&str>, prefs: Option<&StreamPrefs>) -> Option<i32> {
        if let (Some(prefs), Some(parent_id)) = (prefs, parent_id) {
            if let Some(prefs) = prefs.streamprefs.get(parent_id) {
                let mut sort_index = None;
                for pref in prefs {
                    if pref.id != "subscription-ordering" {
                        continue;
                    }

                    let cahrs = pref.value.chars().collect::<Vec<char>>();

                    sort_index = cahrs
                        .chunks(8)
                        .map(|c| c.iter().collect::<String>())
                        .enumerate()
                        .find(|(_i, id)| sort_id == Some(id))
                        .map(|(i, _id)| i as i32);

                    break;
                }

                sort_index
            } else {
                None
            }
        } else {
            None
        }
    }

    pub fn convert_feed_vec(mut feeds: Vec<GFeed>, prefs: Option<&StreamPrefs>) -> (Vec<Feed>, Vec<FeedMapping>) {
        let mut mappings: Vec<FeedMapping> = Vec::new();
        let feeds = feeds
            .drain(..)
            .enumerate()
            .map(|(i, f)| {
                for category in &f.categories {
                    let parent_id = f.categories.first().map(|category| category.id.clone());
                    let sort_index = Self::convert_sortid(parent_id.as_deref(), f.sortid.as_deref(), prefs);

                    mappings.push(FeedMapping {
                        feed_id: FeedID::new(&f.id.to_string()),
                        category_id: CategoryID::new(&category.id.to_string()),
                        sort_index: if sort_index.is_none() { Some(i as i32) } else { sort_index },
                    });
                }

                Self::convert_feed(f)
            })
            .collect();

        (feeds, mappings)
    }

    pub async fn convert_stream(stream: Stream, tag_ids: &HashSet<TagID>, portal: Arc<Box<dyn Portal>>) -> StreamConversionResult {
        let Stream {
            direction: _,
            id: _,
            title: _,
            description: _,
            own: _,
            updated: _,
            updated_usec: _,
            items,
            author: _,
            continuation: _,
        } = stream;

        GReaderUtil::convert_item_vec(items, tag_ids, portal).await
    }

    pub async fn convert_item_vec(articles: Vec<Item>, tag_ids: &HashSet<TagID>, portal: Arc<Box<dyn Portal>>) -> StreamConversionResult {
        let enclosures: Arc<RwLock<Vec<Enclosure>>> = Arc::new(RwLock::new(Vec::new()));
        let taggings: Arc<RwLock<Vec<Tagging>>> = Arc::new(RwLock::new(Vec::new()));
        let headlines: Arc<RwLock<Vec<Headline>>> = Arc::new(RwLock::new(Vec::new()));

        let tasks = articles
            .into_iter()
            .map(|item| {
                let enclosures = enclosures.clone();
                let taggings = taggings.clone();
                let headlines = headlines.clone();
                let portal = portal.clone();
                let tag_ids = tag_ids.clone();

                tokio::spawn(async move {
                    let Item {
                        origin,
                        updated: _,
                        id,
                        categories,
                        author,
                        alternate,
                        timestamp_usec: _,
                        crawl_time_msec: _,
                        published,
                        title,
                        content,
                        enclosure,
                    } = item;

                    let article_id = ArticleID::new(&Self::convert_google_item_id(id));
                    let article_exists_locally = portal.get_article_exists(&article_id).unwrap_or(false);

                    let unread = if categories.iter().any(|c| c.ends_with("/read")) {
                        Read::Read
                    } else {
                        Read::Unread
                    };
                    let marked = if categories.iter().any(|c| c.ends_with("/starred")) {
                        Marked::Marked
                    } else {
                        Marked::Unmarked
                    };

                    let mut article_taggings = categories
                        .iter()
                        .filter_map(|c| {
                            let tag_id = TagID::new(c);
                            if tag_ids.contains(&tag_id) {
                                Some(Tagging {
                                    tag_id,
                                    article_id: article_id.clone(),
                                })
                            } else {
                                None
                            }
                        })
                        .collect();

                    taggings.write().await.append(&mut article_taggings);

                    // already in db
                    // -> only need to update read/marked status
                    // FIXME: can we check if article was recrawled and updated its content?
                    if article_exists_locally {
                        headlines.write().await.push(Headline { article_id, unread, marked });
                        return None;
                    }

                    let url = alternate.first().and_then(|alt| Url::parse(&alt.href).ok());
                    let (html, direction) = if let Some(content) = content {
                        let Summary { content: html, direction } = content;
                        let direction = direction.map(|d| if d == "rtl" { Direction::RightToLeft } else { Direction::LeftToRight });
                        (Some(html), direction)
                    } else {
                        (None, None)
                    };

                    let thumbnail_url = if let Some(enclosure) = enclosure {
                        let thumbnail_url = enclosure.iter().find_map(|e| {
                            let is_image_type = e._type.as_ref().map(|t| t.starts_with("image/")).unwrap_or(false);
                            let is_image_href = e.href.ends_with(".jpeg") || e.href.ends_with(".jpg") || e.href.ends_with(".png");

                            if is_image_type || is_image_href { Some(e.href.clone()) } else { None }
                        });

                        enclosures.write().await.append(
                            &mut enclosure
                                .into_iter()
                                .filter_map(|e| {
                                    Url::parse(&e.href).ok().map(|url| Enclosure {
                                        article_id: article_id.clone(),
                                        url,
                                        mime_type: e._type,
                                        duration: e.length.map(|length| length as i32),
                                        title: None,
                                        position: None,
                                        summary: None,
                                        thumbnail_url: None,
                                        filesize: None,
                                        width: None,
                                        height: None,
                                        framerate: None,
                                        alternative: None,
                                        is_default: false,
                                    })
                                })
                                .collect(),
                        );

                        thumbnail_url
                    } else if let Some(html) = html.as_deref() {
                        crate::util::thumbnail::extract_thumbnail(html)
                    } else {
                        None
                    };

                    let plain_text = if article_exists_locally {
                        None
                    } else {
                        html.as_deref().map(util::html2text::html2text)
                    };

                    let summary = plain_text.as_deref().map(util::html2text::text2summary);

                    Some(FatArticle {
                        article_id,
                        title: title.map(|t| match escaper::decode_html(&t) {
                            Ok(title) => title,
                            Err(_error) => {
                                // This warning freaks users out for some reason
                                // warn!("Error {:?} at character {}", error.kind, error.position);
                                t
                            }
                        }),
                        author,
                        feed_id: FeedID::new(&origin.stream_id),
                        url,
                        date: util::timestamp_to_datetime(published),
                        synced: Utc::now(),
                        updated: None,
                        html,
                        direction,
                        summary,
                        plain_text,
                        scraped_content: None,
                        unread,
                        marked,
                        thumbnail_url,
                    })
                })
            })
            .collect::<Vec<_>>();

        let articles = future::join_all(tasks).await.into_iter().filter_map(|res| res.ok().flatten()).collect();

        StreamConversionResult {
            articles,
            headlines: Arc::into_inner(headlines).map(|e| e.into_inner()).unwrap_or_default(),
            taggings: Arc::into_inner(taggings).map(|e| e.into_inner()).unwrap_or_default(),
            enclosures: Arc::into_inner(enclosures).map(|e| e.into_inner()).unwrap_or_default(),
        }
    }

    fn convert_google_item_id(long_id: String) -> String {
        if long_id.starts_with(GOOGLE_ITEM_PREFIX)
            && let Some(pos) = long_id.rfind('/')
        {
            let hex_id = &long_id[pos + 1..];
            let dec_id = i64::from_str_radix(hex_id, 16).unwrap();

            return dec_id.to_string();
        }

        long_id
    }

    pub async fn get_articles(
        api: &GReaderApi,
        client: &Client,
        portal: Arc<Box<dyn Portal>>,
        query: ArticleQuery<'_>,
    ) -> Result<StreamConversionResult, GReaderError> {
        let mut continuation: Option<String> = None;
        let mut articles = Vec::new();
        let mut headlines = Vec::new();
        let mut taggings = Vec::new();
        let mut enclosures = Vec::new();
        let exclude = query.read.and_then(|r| if r == Read::Unread { Some(TAG_READ_STR) } else { None });
        let include = query.read.and_then(|r| if r == Read::Read { Some(TAG_READ_STR) } else { None });
        let include = query.marked.and_then(|m| {
            if m == Marked::Marked {
                Some(include.unwrap_or(TAG_STARRED_STR))
            } else {
                None
            }
        });

        let amount = query.limit.map(|l| u64::max(l, 1000)).unwrap_or(1000);
        let mut missing = query.limit.map(|l| l.saturating_sub(1000)).unwrap_or(u64::MAX);

        loop {
            let stream = api
                .stream_contents(
                    query.stream_id,
                    false,
                    Some(amount),
                    continuation.as_deref(),
                    exclude,
                    include,
                    query.last_sync,
                    None,
                    client,
                )
                .await?;

            let stream_continuation = stream.continuation.clone();
            let mut result = GReaderUtil::convert_stream(stream, query.tag_ids, portal.clone()).await;
            articles.append(&mut result.articles);
            headlines.append(&mut result.headlines);
            taggings.append(&mut result.taggings);
            enclosures.append(&mut result.enclosures);

            if stream_continuation.is_none() {
                break;
            }

            if missing == 0 {
                break;
            }

            missing -= amount;
            continuation = stream_continuation;
        }

        Ok(StreamConversionResult {
            articles,
            headlines,
            taggings,
            enclosures,
        })
    }

    pub async fn get_article_ids(
        api: &GReaderApi,
        client: &Client,
        stream_id: Option<&str>,
        read: Option<Read>,
        marked: Option<Marked>,
        chunk_size: Option<u64>,
    ) -> Result<Vec<ItemId>, GReaderError> {
        let mut continuation: Option<String> = None;
        let mut article_ids = Vec::new();
        let exclude = read.and_then(|r| if r == Read::Unread { Some(TAG_READ_STR) } else { None });
        let include = read.and_then(|r| if r == Read::Read { Some(TAG_READ_STR) } else { None });
        let include = marked.and_then(|m| {
            if m == Marked::Marked {
                Some(include.unwrap_or(TAG_STARRED_STR))
            } else {
                None
            }
        });
        let chunk_size = chunk_size.unwrap_or(1000);

        loop {
            let stream = api
                .items_ids(
                    stream_id,
                    Some(chunk_size),
                    false,
                    continuation.as_deref(),
                    exclude,
                    include,
                    None,
                    None,
                    client,
                )
                .await?;

            let ItemRefs { item_refs, continuation: c } = stream;
            if let Some(mut item_refs) = item_refs {
                article_ids.append(&mut item_refs);
            }

            if c.is_none() {
                break;
            }

            continuation.clone_from(&c);
        }

        Ok(article_ids)
    }
}