1use crate::error::Result;
2use crate::error::TwitterError;
3use crate::models::tweets::Mention;
4use crate::models::Tweet;
5use crate::profile::LegacyUserRaw;
6use crate::timeline::tweet_utils::parse_media_groups;
7use crate::timeline::v1::{LegacyTweetRaw, TimelineResultRaw};
8use chrono::Utc;
9use lazy_static::lazy_static;
10use serde::{Deserialize, Serialize};
11lazy_static! {
12 static ref EMPTY_INSTRUCTIONS: Vec<TimelineInstruction> = Vec::new();
13}
14
15#[derive(Debug, Deserialize, Serialize)]
16pub struct Timeline {
17 pub timeline: Option<TimelineItems>,
18}
19
20#[derive(Debug, Deserialize, Serialize)]
21pub struct TimelineContent {
22 pub instructions: Option<Vec<TimelineInstruction>>,
23}
24
25#[derive(Debug, Deserialize, Serialize)]
26pub struct TimelineData {
27 pub user: Option<TimelineUser>,
28}
29
30#[derive(Debug, Deserialize, Serialize)]
31pub struct TimelineEntities {
32 pub hashtags: Option<Vec<Hashtag>>,
33 pub user_mentions: Option<Vec<UserMention>>,
34 pub urls: Option<Vec<UrlEntity>>,
35}
36
37#[derive(Debug, Deserialize, Serialize)]
38pub struct TimelineEntry {
39 #[serde(rename = "entryId")]
40 pub entry_id: Option<String>,
41 pub content: Option<EntryContent>,
42}
43
44#[derive(Debug, Deserialize, Serialize)]
45pub struct TimelineEntryItemContent {
46 pub item_type: Option<String>,
47 pub tweet_display_type: Option<String>,
48 pub tweet_result: Option<TweetResult>,
49 pub tweet_results: Option<TweetResult>,
50 pub user_display_type: Option<String>,
51 pub user_results: Option<TimelineUserResult>,
52}
53
54#[derive(Debug, Deserialize, Serialize)]
55pub struct TimelineEntryItemContentRaw {
56 #[serde(rename = "itemType")]
57 pub item_type: Option<String>,
58 #[serde(rename = "tweetDisplayType")]
59 pub tweet_display_type: Option<String>,
60 #[serde(rename = "tweetResult")]
61 pub tweet_result: Option<TweetResultRaw>,
62 pub tweet_results: Option<TweetResultRaw>,
63 #[serde(rename = "userDisplayType")]
64 pub user_display_type: Option<String>,
65 pub user_results: Option<TimelineUserResultRaw>,
66}
67
68#[derive(Debug, Deserialize, Serialize)]
69pub struct TimelineItems {
70 pub instructions: Option<Vec<TimelineInstruction>>,
71}
72
73#[derive(Debug, Deserialize, Serialize)]
74pub struct TimelineUser {
75 pub result: Option<TimelineUserResult>,
76}
77
78#[derive(Debug, Deserialize, Serialize)]
79pub struct TimelineUserResult {
80 pub rest_id: Option<String>,
81 pub legacy: Option<LegacyUserRaw>,
82 pub is_blue_verified: Option<bool>,
83 pub timeline_v2: Option<Box<TimelineV2>>,
84}
85
86#[derive(Debug, Deserialize, Serialize)]
87pub struct TimelineUserResultRaw {
88 pub result: Option<TimelineUserResult>,
89}
90
91#[derive(Debug, Deserialize, Serialize)]
92pub struct TimelineV2 {
93 pub data: Option<TimelineData>,
94 pub timeline: Option<TimelineItems>,
95}
96
97#[derive(Debug, Deserialize, Serialize)]
98pub struct ThreadedConversation {
99 pub data: Option<ThreadedConversationData>,
100}
101
102#[derive(Debug, Deserialize, Serialize)]
103pub struct ThreadedConversationData {
104 pub threaded_conversation_with_injections_v2: Option<TimelineContent>,
105}
106
107#[derive(Debug, Deserialize, Serialize)]
108pub struct TweetResult {
109 pub result: Option<TimelineResultRaw>,
110}
111
112#[derive(Debug, Deserialize, Serialize)]
113pub struct TweetResultRaw {
114 pub result: Option<TimelineResultRaw>,
115}
116
117#[derive(Debug, Deserialize, Serialize)]
118pub struct EntryContent {
119 #[serde(rename = "cursorType")]
120 pub cursor_type: Option<String>,
121 pub value: Option<String>,
122 pub items: Option<Vec<EntryItem>>,
123 #[serde(rename = "itemContent")]
124 pub item_content: Option<TimelineEntryItemContent>,
125}
126
127#[derive(Debug, Deserialize, Serialize)]
128pub struct EntryItem {
129 #[serde(rename = "entryId")]
130 pub entry_id: Option<String>,
131 pub item: Option<ItemContent>,
132}
133
134#[derive(Debug, Deserialize, Serialize)]
135pub struct ItemContent {
136 pub content: Option<TimelineEntryItemContent>,
137 #[serde(rename = "itemContent")]
138 pub item_content: Option<TimelineEntryItemContent>,
139}
140
141#[derive(Debug, Deserialize, Serialize)]
142pub struct Hashtag {
143 pub text: Option<String>,
144}
145
146#[derive(Debug, Deserialize, Serialize)]
147pub struct UrlEntity {
148 pub expanded_url: Option<String>,
149}
150
151#[derive(Debug, Deserialize, Serialize)]
152pub struct UserMention {
153 pub id_str: Option<String>,
154 pub name: Option<String>,
155 pub screen_name: Option<String>,
156}
157
158#[derive(Debug, Deserialize, Serialize)]
159pub struct TimelineInstruction {
160 pub entries: Option<Vec<TimelineEntry>>,
161 pub entry: Option<TimelineEntry>,
162 #[serde(rename = "type")]
163 pub type_: Option<String>,
164}
165
166#[derive(Debug, Deserialize, Serialize)]
167pub struct SearchEntryRaw {
168 #[serde(rename = "entryId")]
169 pub entry_id: String,
170 #[serde(rename = "sortIndex")]
171 pub sort_index: String,
172 pub content: Option<SearchEntryContentRaw>,
173}
174
175#[derive(Debug, Deserialize, Serialize)]
176pub struct SearchEntryContentRaw {
177 #[serde(rename = "cursorType")]
178 pub cursor_type: Option<String>,
179 #[serde(rename = "entryType")]
180 pub entry_type: Option<String>,
181 #[serde(rename = "__typename")]
182 pub typename: Option<String>,
183 pub value: Option<String>,
184 pub items: Option<Vec<SearchEntryItemRaw>>,
185 #[serde(rename = "itemContent")]
186 pub item_content: Option<TimelineEntryItemContentRaw>,
187}
188
189#[derive(Debug, Deserialize, Serialize)]
190pub struct SearchEntryItemRaw {
191 pub item: Option<SearchEntryItemInnerRaw>,
192}
193
194#[derive(Debug, Deserialize, Serialize)]
195pub struct SearchEntryItemInnerRaw {
196 pub content: Option<TimelineEntryItemContentRaw>,
197}
198
199pub fn parse_legacy_tweet(
200 user: Option<&LegacyUserRaw>,
201 tweet: Option<&LegacyTweetRaw>,
202) -> Result<Tweet> {
203 let tweet = tweet.ok_or(TwitterError::Api(
204 "Tweet was not found in the timeline object".into(),
205 ))?;
206 let user = user.ok_or(TwitterError::Api(
207 "User was not found in the timeline object".into(),
208 ))?;
209
210 let id_str = tweet
211 .id_str
212 .as_ref()
213 .or(tweet.conversation_id_str.as_ref())
214 .ok_or(TwitterError::Api("Tweet ID was not found in object".into()))?;
215
216 let hashtags = tweet
217 .entities
218 .as_ref()
219 .and_then(|e| e.hashtags.as_ref())
220 .map(|h| h.iter().filter_map(|h| h.text.clone()).collect())
221 .unwrap_or_default();
222
223 let mentions = tweet
224 .entities
225 .as_ref()
226 .and_then(|e| e.user_mentions.as_ref())
227 .map(|mentions| {
228 mentions
229 .iter()
230 .filter_map(|m| {
231 Some(Mention {
232 id: m.id_str.clone().unwrap_or_default(),
233 name: m.name.clone(),
234 username: m.screen_name.clone(),
235 })
236 })
237 .collect()
238 })
239 .unwrap_or_default();
240
241 let (photos, videos, _) =
242 if let Some(extended_entities) = &tweet.extended_entities {
243 if let Some(media) = &extended_entities.media {
244 parse_media_groups(media)
245 } else {
246 (Vec::new(), Vec::new(), false)
247 }
248 } else {
249 (Vec::new(), Vec::new(), false)
250 };
251
252 let mut tweet = Tweet {
253 bookmark_count: tweet.bookmark_count,
254 conversation_id: tweet.conversation_id_str.clone(),
255 id: Some(id_str.clone()),
256 hashtags,
257 likes: tweet.favorite_count,
258 mentions,
259 name: user.name.clone(),
260 permanent_url: Some(format!(
261 "https://twitter.com/{}/status/{}",
262 user.screen_name.as_ref().unwrap_or(&String::new()),
263 id_str
264 )),
265 photos,
266 replies: tweet.reply_count,
267 retweets: tweet.retweet_count,
268 text: tweet.full_text.clone(),
269 thread: Vec::new(),
270 urls: tweet
271 .entities
272 .as_ref()
273 .and_then(|e| e.urls.as_ref())
274 .map(|urls| urls.iter().filter_map(|u| u.expanded_url.clone()).collect())
275 .unwrap_or_default(),
276 user_id: tweet.user_id_str.clone(),
277 username: user.screen_name.clone(),
278 videos,
279 is_quoted: Some(false),
280 is_reply: Some(false),
281 is_retweet: Some(false),
282 is_pin: Some(false),
283 sensitive_content: Some(false),
284 quoted_status: None,
285 quoted_status_id: tweet.quoted_status_id_str.clone(),
286 in_reply_to_status_id: tweet.in_reply_to_status_id_str.clone(),
287 retweeted_status: None,
288 retweeted_status_id: None,
289 views: None,
290 html: None,
291 time_parsed: None,
292 timestamp: None,
293 place: tweet.place.clone(),
294 in_reply_to_status: None,
295 is_self_thread: None,
296 poll: None,
297 created_at: tweet.created_at.clone(),
298 ext_views: None,
299 quote_count: None,
300 reply_count: None,
301 retweet_count: None,
302 screen_name: None,
303 thread_id: None,
304 };
305
306 if let Some(created_at) = &tweet.created_at {
307 if let Ok(time) = chrono::DateTime::parse_from_str(created_at, "%a %b %d %H:%M:%S %z %Y") {
308 tweet.time_parsed = Some(time.with_timezone(&Utc));
309 tweet.timestamp = Some(time.timestamp());
310 }
311 }
312
313 if let Some(views) = &tweet.ext_views {
314 tweet.views = Some(*views);
315 }
316
317 Ok(tweet)
321}
322
323pub fn parse_timeline_entry_item_content_raw(
324 content: &TimelineEntryItemContent,
325 _entry_id: &str,
326 is_conversation: bool,
327) -> Option<Tweet> {
328 let result = content
329 .tweet_results
330 .as_ref()
331 .or(content.tweet_result.as_ref())
332 .and_then(|r| r.result.as_ref())?;
333
334 let tweet_result = parse_result(result);
335 if tweet_result.success {
336 let mut tweet = tweet_result.tweet?;
337
338 if is_conversation && content.tweet_display_type.as_deref() == Some("SelfThread") {
339 tweet.is_self_thread = Some(true);
340 }
341
342 return Some(tweet);
343 }
344
345 None
346}
347
348pub fn parse_and_push(
349 tweets: &mut Vec<Tweet>,
350 content: &TimelineEntryItemContent,
351 entry_id: String,
352 is_conversation: bool,
353) {
354 if let Some(tweet) = parse_timeline_entry_item_content_raw(content, &entry_id, is_conversation)
355 {
356 tweets.push(tweet);
357 }
358}
359
360pub fn parse_result(result: &TimelineResultRaw) -> ParseTweetResult {
361 let tweet_result = parse_legacy_tweet(
362 result
363 .core
364 .as_ref()
365 .and_then(|c| c.user_results.as_ref())
366 .and_then(|u| u.result.as_ref())
367 .and_then(|r| r.legacy.as_ref()),
368 result.legacy.as_deref(),
369 );
370
371 let mut tweet = match tweet_result {
372 Ok(tweet) => tweet,
373 Err(e) => {
374 return ParseTweetResult {
375 success: false,
376 tweet: None,
377 err: Some(e),
378 }
379 }
380 };
381
382 if tweet.views.is_none() {
383 if let Some(count) = result
384 .views
385 .as_ref()
386 .and_then(|v| v.count.as_ref())
387 .and_then(|c| c.parse().ok())
388 {
389 tweet.views = Some(count);
390 }
391 }
392
393 if let Some(quoted) = result.quoted_status_result.as_ref() {
394 if let Some(quoted_result) = quoted.result.as_ref() {
395 let quoted_tweet_result = parse_result(quoted_result);
396 if quoted_tweet_result.success {
397 tweet.quoted_status = quoted_tweet_result.tweet.map(Box::new);
398 }
399 }
400 }
401
402 ParseTweetResult {
403 success: true,
404 tweet: Some(tweet),
405 err: None,
406 }
407}
408
409pub struct ParseTweetResult {
410 pub success: bool,
411 pub tweet: Option<Tweet>,
412 pub err: Option<TwitterError>,
413}
414
415#[derive(Debug, Serialize, Deserialize)]
416pub struct QueryTweetsResponse {
417 pub tweets: Vec<Tweet>,
418 pub next: Option<String>,
419 pub previous: Option<String>,
420}
421
422pub fn parse_timeline_tweets_v2(timeline: &TimelineV2) -> QueryTweetsResponse {
423 let mut tweets = Vec::new();
424 let mut bottom_cursor = None;
425 let mut top_cursor = None;
426
427 let instructions = timeline
428 .data
429 .as_ref()
430 .and_then(|data| data.user.as_ref())
431 .and_then(|user| user.result.as_ref())
432 .and_then(|result| result.timeline_v2.as_ref())
433 .and_then(|timeline| timeline.timeline.as_ref())
434 .and_then(|timeline| timeline.instructions.as_ref())
435 .unwrap_or(&EMPTY_INSTRUCTIONS);
436
437 let expected_entry_types = ["tweet-", "profile-conversation-"];
438
439 for instruction in instructions {
440 let entries = instruction
441 .entries.as_deref()
442 .unwrap_or_else(|| {
443 instruction
444 .entry
445 .as_ref()
446 .map(std::slice::from_ref)
447 .unwrap_or_default()
448 });
449
450 for entry in entries {
451 let content = match &entry.content {
452 Some(content) => content,
453 None => continue,
454 };
455
456 if let Some(cursor_type) = &content.cursor_type {
457 match cursor_type.as_str() {
458 "Bottom" => {
459 bottom_cursor = content.value.clone();
460 continue;
461 }
462 "Top" => {
463 top_cursor = content.value.clone();
464 continue;
465 }
466 _ => {}
467 }
468 }
469
470 let entry_id = match &entry.entry_id {
471 Some(id) => id,
472 None => continue,
473 };
474 if !expected_entry_types
475 .iter()
476 .any(|prefix| entry_id.starts_with(prefix))
477 {
478 continue;
479 }
480
481 if let Some(ref item_content) = content.item_content {
482 parse_and_push(&mut tweets, item_content, entry_id.clone(), false);
483 }
484
485 if let Some(items) = &content.items {
486 for item in items {
487 if let Some(item) = &item.item {
488 if let Some(item_content) = &item.item_content {
489 parse_and_push(&mut tweets, item_content, entry_id.clone(), false);
490 }
491 }
492 }
493 }
494 }
495 }
496
497 QueryTweetsResponse {
498 tweets,
499 next: bottom_cursor,
500 previous: top_cursor,
501 }
502}
503
504pub fn parse_threaded_conversation(conversation: &ThreadedConversation) -> Option<Tweet> {
505 let mut main_tweet: Option<Tweet> = None;
506 let mut replies: Vec<Tweet> = Vec::new();
507
508 let instructions = conversation
509 .data
510 .as_ref()
511 .and_then(|data| data.threaded_conversation_with_injections_v2.as_ref())
512 .and_then(|conv| conv.instructions.as_ref())
513 .unwrap_or(&EMPTY_INSTRUCTIONS);
514
515 for instruction in instructions {
516 let entries = instruction
517 .entries.as_deref()
518 .unwrap_or_default();
519
520 for entry in entries {
521 if let Some(content) = &entry.content {
522 if let Some(item_content) = &content.item_content {
523 if let Some(tweet) = parse_timeline_entry_item_content_raw(
524 item_content,
525 entry.entry_id.as_deref().unwrap_or_default(),
526 true,
527 ) {
528 if main_tweet.is_none() {
529 main_tweet = Some(tweet);
530 } else {
531 replies.push(tweet);
532 }
533 }
534 }
535
536 if let Some(items) = &content.items {
537 for item in items {
538 if let Some(item) = &item.item {
539 if let Some(item_content) = &item.item_content {
540 if let Some(tweet) = parse_timeline_entry_item_content_raw(
541 item_content,
542 entry.entry_id.as_deref().unwrap_or_default(),
543 true,
544 ) {
545 replies.push(tweet);
546 }
547 }
548 }
549 }
550 }
551 }
552 }
553 }
554
555 if let Some(mut main_tweet) = main_tweet {
556 for reply in &replies {
557 if let Some(reply_id) = &reply.in_reply_to_status_id {
558 if let Some(main_id) = &main_tweet.id {
559 if reply_id == main_id {
560 main_tweet.replies = Some(replies.len() as i32);
561 break;
562 }
563 }
564 }
565 }
566
567 if main_tweet.is_self_thread == Some(true) {
568 let thread = replies
569 .iter()
570 .filter(|t| t.is_self_thread == Some(true))
571 .cloned()
572 .collect::<Vec<_>>();
573
574 if thread.is_empty() {
575 main_tweet.is_self_thread = Some(false);
576 } else {
577 main_tweet.thread = thread;
578 }
579 }
580
581 Some(main_tweet)
584 } else {
585 None
586 }
587}