use crate::feed_api::Portal;
use crate::models::{
ArticleID, Category, CategoryID, CategoryMapping, Direction, Enclosure, FatArticle, Feed, FeedID, FeedMapping, Headline, Marked,
NEWSFLASH_TOPLEVEL, Read, StreamConversionResult, Tag, TagID, Tagging, Url,
};
use crate::{error::FeedApiError, util};
use chrono::Utc;
use futures::future;
use greader_api::models::{
Category as GCategory, Feed as GFeed, Item, ItemId, ItemRefs, Stream, StreamPrefs, Summary, Tagging as GTagging, Taggings,
};
use greader_api::{ApiError as GReaderError, GReaderApi};
use reqwest::Client;
use std::collections::HashSet;
use std::convert::TryInto;
use std::sync::Arc;
use tokio::sync::RwLock;
pub const TAG_READ_STR: &str = "user/-/state/com.google/read";
pub const TAG_STARRED_STR: &str = "user/-/state/com.google/starred";
pub const TAG_READING_LIST: &str = "user/-/state/com.google/reading-list";
pub const GOOGLE_ITEM_PREFIX: &str = "tag:google.com,2005:reader/item/";
impl From<GReaderError> for FeedApiError {
fn from(error: GReaderError) -> FeedApiError {
match error {
GReaderError::ApiLimit => FeedApiError::ApiLimit,
GReaderError::Url(e) => FeedApiError::Url(e),
GReaderError::Json { source, json } => FeedApiError::Json { source, json },
GReaderError::Http(e) => FeedApiError::Network(e),
GReaderError::GReader(greader_error) => {
let error_list = greader_error.errors.iter().fold("".into(), |prev, next| format!("{prev}\n{next}"));
FeedApiError::Api {
message: format!("GReader Error:\n{error_list}"),
}
}
GReaderError::BadRequest => FeedApiError::Api {
message: GReaderError::BadRequest.to_string(),
},
GReaderError::Input => FeedApiError::Api {
message: GReaderError::Input.to_string(),
},
GReaderError::Token => FeedApiError::Api {
message: GReaderError::Token.to_string(),
},
GReaderError::TokenExpired => FeedApiError::Api {
message: GReaderError::TokenExpired.to_string(),
},
GReaderError::AccessDenied => FeedApiError::Auth,
GReaderError::Parse => FeedApiError::Api {
message: GReaderError::Parse.to_string(),
},
GReaderError::NotLoggedIn => FeedApiError::Login,
GReaderError::Other(msg) => FeedApiError::Api { message: msg },
}
}
}
pub struct ArticleQuery<'a> {
pub stream_id: Option<&'a str>,
pub read: Option<Read>,
pub marked: Option<Marked>,
pub tag_ids: &'a HashSet<TagID>,
pub limit: Option<u64>,
pub last_sync: Option<i64>,
}
pub struct GReaderUtil;
impl GReaderUtil {
pub fn generate_tag_id(user_id: Option<&str>, name: &str) -> String {
let user_id = user_id.unwrap_or("-");
format!("user/{user_id}/label/{name}")
}
pub fn root_id(user_id: Option<&str>) -> String {
let user_id = user_id.unwrap_or("-");
format!("user/{user_id}/state/com.google/root")
}
pub fn convert_category(
category: GCategory,
index: i32,
taggings: Option<&Taggings>,
prefs: Option<&StreamPrefs>,
user_id: Option<&str>,
) -> (Category, CategoryMapping) {
let sort_index = if let Some(tagging) = Self::find_tagging(taggings, &category.id) {
Self::convert_sortid(Some(&Self::root_id(user_id)), tagging.sortid.as_deref(), prefs)
} else {
None
};
let GCategory { id, label } = category;
let category_id = CategoryID::new(&id);
let category = Category {
category_id: category_id.clone(),
label,
};
let category_mapping = CategoryMapping {
parent_id: NEWSFLASH_TOPLEVEL.clone(),
category_id,
sort_index: if sort_index.is_none() { Some(index) } else { sort_index },
};
(category, category_mapping)
}
pub fn find_tagging<'a>(taggings: Option<&'a Taggings>, id: &str) -> Option<&'a GTagging> {
if let Some(taggings) = taggings {
taggings.tags.iter().find(|t| t.id == id)
} else {
None
}
}
pub fn convert_category_vec(
mut categories: Vec<GFeed>,
taggings: Option<&Taggings>,
prefs: Option<&StreamPrefs>,
user_id: Option<&str>,
) -> (Vec<Category>, Vec<CategoryMapping>) {
let mut category_ids: HashSet<CategoryID> = HashSet::new();
let mut index = 0;
categories
.drain(..)
.filter_map(|feed| {
let feed_categories: Vec<(Category, CategoryMapping)> = feed
.categories
.into_iter()
.filter_map(|c| {
let (category, category_mapping) = Self::convert_category(c, index, taggings, prefs, user_id);
if category_ids.contains(&category.category_id) {
None
} else {
index += 1;
category_ids.insert(category.category_id.clone());
Some((category, category_mapping))
}
})
.collect();
if feed_categories.is_empty() { None } else { Some(feed_categories) }
})
.flatten()
.unzip()
}
pub fn convert_tag_list(taggings: Taggings, categories: &[Category]) -> Vec<Tag> {
let Taggings { tags: tag_list } = taggings;
tag_list
.into_iter()
.filter_map(|tagging| {
let GTagging {
id: tag_id,
r#type: _,
sortid,
unread_count: _,
unseen_count: _,
} = tagging;
if tag_id.contains("/label/") {
if categories.iter().any(|c| c.category_id.as_str() == tag_id) {
None
} else {
let label = tag_id
.split('/')
.next_back()
.map(|s| s.to_owned())
.unwrap_or_else(|| "Missing Label".into());
let sort_index = sortid
.and_then(|str| hex::decode(str).ok())
.and_then(|buf| buf.try_into().ok())
.map(|buf| u32::from_le_bytes(buf) as i32);
Some(Tag {
tag_id: TagID::from_owned(tag_id),
label,
color: None,
sort_index,
})
}
} else {
None
}
})
.collect()
}
pub fn convert_feed(feed: GFeed) -> Feed {
let GFeed {
id,
title,
categories: _,
url,
html_url,
icon_url,
sortid: _,
} = feed;
Feed {
feed_id: FeedID::new(&id),
label: title,
website: Url::parse(&html_url).ok(),
feed_url: Url::parse(&url).ok(),
icon_url: Url::parse(&icon_url).ok(),
error_count: 0,
error_message: None,
}
}
pub fn convert_sortid(parent_id: Option<&str>, sort_id: Option<&str>, prefs: Option<&StreamPrefs>) -> Option<i32> {
if let (Some(prefs), Some(parent_id)) = (prefs, parent_id) {
if let Some(prefs) = prefs.streamprefs.get(parent_id) {
let mut sort_index = None;
for pref in prefs {
if pref.id != "subscription-ordering" {
continue;
}
let cahrs = pref.value.chars().collect::<Vec<char>>();
sort_index = cahrs
.chunks(8)
.map(|c| c.iter().collect::<String>())
.enumerate()
.find(|(_i, id)| sort_id == Some(id))
.map(|(i, _id)| i as i32);
break;
}
sort_index
} else {
None
}
} else {
None
}
}
pub fn convert_feed_vec(mut feeds: Vec<GFeed>, prefs: Option<&StreamPrefs>) -> (Vec<Feed>, Vec<FeedMapping>) {
let mut mappings: Vec<FeedMapping> = Vec::new();
let feeds = feeds
.drain(..)
.enumerate()
.map(|(i, f)| {
for category in &f.categories {
let parent_id = f.categories.first().map(|category| category.id.clone());
let sort_index = Self::convert_sortid(parent_id.as_deref(), f.sortid.as_deref(), prefs);
mappings.push(FeedMapping {
feed_id: FeedID::new(&f.id.to_string()),
category_id: CategoryID::new(&category.id.to_string()),
sort_index: if sort_index.is_none() { Some(i as i32) } else { sort_index },
});
}
Self::convert_feed(f)
})
.collect();
(feeds, mappings)
}
pub async fn convert_stream(stream: Stream, tag_ids: &HashSet<TagID>, portal: Arc<Box<dyn Portal>>) -> StreamConversionResult {
let Stream {
direction: _,
id: _,
title: _,
description: _,
own: _,
updated: _,
updated_usec: _,
items,
author: _,
continuation: _,
} = stream;
GReaderUtil::convert_item_vec(items, tag_ids, portal).await
}
pub async fn convert_item_vec(articles: Vec<Item>, tag_ids: &HashSet<TagID>, portal: Arc<Box<dyn Portal>>) -> StreamConversionResult {
let enclosures: Arc<RwLock<Vec<Enclosure>>> = Arc::new(RwLock::new(Vec::new()));
let taggings: Arc<RwLock<Vec<Tagging>>> = Arc::new(RwLock::new(Vec::new()));
let headlines: Arc<RwLock<Vec<Headline>>> = Arc::new(RwLock::new(Vec::new()));
let tasks = articles
.into_iter()
.map(|item| {
let enclosures = enclosures.clone();
let taggings = taggings.clone();
let headlines = headlines.clone();
let portal = portal.clone();
let tag_ids = tag_ids.clone();
tokio::spawn(async move {
let Item {
origin,
updated: _,
id,
categories,
author,
alternate,
timestamp_usec: _,
crawl_time_msec: _,
published,
title,
content,
enclosure,
} = item;
let article_id = ArticleID::new(&Self::convert_google_item_id(id));
let article_exists_locally = portal.get_article_exists(&article_id).unwrap_or(false);
let unread = if categories.iter().any(|c| c.ends_with("/read")) {
Read::Read
} else {
Read::Unread
};
let marked = if categories.iter().any(|c| c.ends_with("/starred")) {
Marked::Marked
} else {
Marked::Unmarked
};
let mut article_taggings = categories
.iter()
.filter_map(|c| {
let tag_id = TagID::new(c);
if tag_ids.contains(&tag_id) {
Some(Tagging {
tag_id,
article_id: article_id.clone(),
})
} else {
None
}
})
.collect();
taggings.write().await.append(&mut article_taggings);
if article_exists_locally {
headlines.write().await.push(Headline { article_id, unread, marked });
return None;
}
let url = alternate.first().and_then(|alt| Url::parse(&alt.href).ok());
let (html, direction) = if let Some(content) = content {
let Summary { content: html, direction } = content;
let direction = direction.map(|d| if d == "rtl" { Direction::RightToLeft } else { Direction::LeftToRight });
(Some(html), direction)
} else {
(None, None)
};
let thumbnail_url = if let Some(enclosure) = enclosure {
let thumbnail_url = enclosure.iter().find_map(|e| {
let is_image_type = e._type.as_ref().map(|t| t.starts_with("image/")).unwrap_or(false);
let is_image_href = e.href.ends_with(".jpeg") || e.href.ends_with(".jpg") || e.href.ends_with(".png");
if is_image_type || is_image_href { Some(e.href.clone()) } else { None }
});
enclosures.write().await.append(
&mut enclosure
.into_iter()
.filter_map(|e| {
Url::parse(&e.href).ok().map(|url| Enclosure {
article_id: article_id.clone(),
url,
mime_type: e._type,
duration: e.length.map(|length| length as i32),
title: None,
position: None,
summary: None,
thumbnail_url: None,
filesize: None,
width: None,
height: None,
framerate: None,
alternative: None,
is_default: false,
})
})
.collect(),
);
thumbnail_url
} else if let Some(html) = html.as_deref() {
crate::util::thumbnail::extract_thumbnail(html)
} else {
None
};
let plain_text = if article_exists_locally {
None
} else {
html.as_deref().map(util::html2text::html2text)
};
let summary = plain_text.as_deref().map(util::html2text::text2summary);
Some(FatArticle {
article_id,
title: title.map(|t| match escaper::decode_html(&t) {
Ok(title) => title,
Err(_error) => {
t
}
}),
author,
feed_id: FeedID::new(&origin.stream_id),
url,
date: util::timestamp_to_datetime(published),
synced: Utc::now(),
updated: None,
html,
direction,
summary,
plain_text,
scraped_content: None,
unread,
marked,
thumbnail_url,
})
})
})
.collect::<Vec<_>>();
let articles = future::join_all(tasks).await.into_iter().filter_map(|res| res.ok().flatten()).collect();
StreamConversionResult {
articles,
headlines: Arc::into_inner(headlines).map(|e| e.into_inner()).unwrap_or_default(),
taggings: Arc::into_inner(taggings).map(|e| e.into_inner()).unwrap_or_default(),
enclosures: Arc::into_inner(enclosures).map(|e| e.into_inner()).unwrap_or_default(),
}
}
fn convert_google_item_id(long_id: String) -> String {
if long_id.starts_with(GOOGLE_ITEM_PREFIX)
&& let Some(pos) = long_id.rfind('/')
{
let hex_id = &long_id[pos + 1..];
let dec_id = i64::from_str_radix(hex_id, 16).unwrap();
return dec_id.to_string();
}
long_id
}
pub async fn get_articles(
api: &GReaderApi,
client: &Client,
portal: Arc<Box<dyn Portal>>,
query: ArticleQuery<'_>,
) -> Result<StreamConversionResult, GReaderError> {
let mut continuation: Option<String> = None;
let mut articles = Vec::new();
let mut headlines = Vec::new();
let mut taggings = Vec::new();
let mut enclosures = Vec::new();
let exclude = query.read.and_then(|r| if r == Read::Unread { Some(TAG_READ_STR) } else { None });
let include = query.read.and_then(|r| if r == Read::Read { Some(TAG_READ_STR) } else { None });
let include = query.marked.and_then(|m| {
if m == Marked::Marked {
Some(include.unwrap_or(TAG_STARRED_STR))
} else {
None
}
});
let amount = query.limit.map(|l| u64::max(l, 1000)).unwrap_or(1000);
let mut missing = query.limit.map(|l| l.saturating_sub(1000)).unwrap_or(u64::MAX);
loop {
let stream = api
.stream_contents(
query.stream_id,
false,
Some(amount),
continuation.as_deref(),
exclude,
include,
query.last_sync,
None,
client,
)
.await?;
let stream_continuation = stream.continuation.clone();
let mut result = GReaderUtil::convert_stream(stream, query.tag_ids, portal.clone()).await;
articles.append(&mut result.articles);
headlines.append(&mut result.headlines);
taggings.append(&mut result.taggings);
enclosures.append(&mut result.enclosures);
if stream_continuation.is_none() {
break;
}
if missing == 0 {
break;
}
missing -= amount;
continuation = stream_continuation;
}
Ok(StreamConversionResult {
articles,
headlines,
taggings,
enclosures,
})
}
pub async fn get_article_ids(
api: &GReaderApi,
client: &Client,
stream_id: Option<&str>,
read: Option<Read>,
marked: Option<Marked>,
chunk_size: Option<u64>,
) -> Result<Vec<ItemId>, GReaderError> {
let mut continuation: Option<String> = None;
let mut article_ids = Vec::new();
let exclude = read.and_then(|r| if r == Read::Unread { Some(TAG_READ_STR) } else { None });
let include = read.and_then(|r| if r == Read::Read { Some(TAG_READ_STR) } else { None });
let include = marked.and_then(|m| {
if m == Marked::Marked {
Some(include.unwrap_or(TAG_STARRED_STR))
} else {
None
}
});
let chunk_size = chunk_size.unwrap_or(1000);
loop {
let stream = api
.items_ids(
stream_id,
Some(chunk_size),
false,
continuation.as_deref(),
exclude,
include,
None,
None,
client,
)
.await?;
let ItemRefs { item_refs, continuation: c } = stream;
if let Some(mut item_refs) = item_refs {
article_ids.append(&mut item_refs);
}
if c.is_none() {
break;
}
continuation.clone_from(&c);
}
Ok(article_ids)
}
}