1use crate::error::Result;
2use crate::error::TwitterError;
3use crate::primitives::profile::LegacyUserRaw;
4use crate::primitives::tweets::Mention;
5use crate::primitives::Tweet;
6use crate::timeline::tweet_utils::parse_media_groups;
7use crate::timeline::v1::{LegacyTweetRaw, TimelineResultRaw};
8use chrono::Utc;
9use lazy_static::lazy_static;
10use serde::{Deserialize, Serialize};
11lazy_static! {
12 static ref EMPTY_INSTRUCTIONS: Vec<TimelineInstruction> = Vec::new();
13}
14
15#[derive(Debug, Deserialize, Serialize)]
16pub struct Timeline {
17 pub timeline: Option<TimelineItems>,
18}
19
20#[derive(Debug, Deserialize, Serialize)]
21pub struct TimelineContent {
22 pub instructions: Option<Vec<TimelineInstruction>>,
23}
24
25#[derive(Debug, Deserialize, Serialize)]
26pub struct TimelineData {
27 pub user: Option<TimelineUser>,
28}
29
30#[derive(Debug, Deserialize, Serialize)]
31pub struct TimelineEntities {
32 pub hashtags: Option<Vec<Hashtag>>,
33 pub user_mentions: Option<Vec<UserMention>>,
34 pub urls: Option<Vec<UrlEntity>>,
35}
36
37#[derive(Debug, Deserialize, Serialize)]
38pub struct TimelineEntry {
39 #[serde(rename = "entryId")]
40 pub entry_id: Option<String>,
41 pub content: Option<EntryContent>,
42}
43
44#[derive(Debug, Deserialize, Serialize)]
45pub struct TimelineEntryItemContent {
46 pub item_type: Option<String>,
47 pub tweet_display_type: Option<String>,
48 pub tweet_result: Option<TweetResult>,
49 pub tweet_results: Option<TweetResult>,
50 pub user_display_type: Option<String>,
51 pub user_results: Option<TimelineUserResult>,
52}
53
54#[derive(Debug, Deserialize, Serialize)]
55pub struct TimelineEntryItemContentRaw {
56 #[serde(rename = "itemType")]
57 pub item_type: Option<String>,
58 #[serde(rename = "tweetDisplayType")]
59 pub tweet_display_type: Option<String>,
60 #[serde(rename = "tweetResult")]
61 pub tweet_result: Option<TweetResultRaw>,
62 pub tweet_results: Option<TweetResultRaw>,
63 #[serde(rename = "userDisplayType")]
64 pub user_display_type: Option<String>,
65 pub user_results: Option<TimelineUserResultRaw>,
66}
67
68#[derive(Debug, Deserialize, Serialize)]
69pub struct TimelineItems {
70 pub instructions: Option<Vec<TimelineInstruction>>,
71}
72
73#[derive(Debug, Deserialize, Serialize)]
74pub struct TimelineUser {
75 pub result: Option<TimelineUserResult>,
76}
77
78#[derive(Debug, Deserialize, Serialize)]
79pub struct TimelineUserResult {
80 pub rest_id: Option<String>,
81 pub legacy: Option<LegacyUserRaw>,
82 pub is_blue_verified: Option<bool>,
83 pub timeline_v2: Option<Box<TimelineV2>>,
84}
85
86#[derive(Debug, Deserialize, Serialize)]
87pub struct TimelineUserResultRaw {
88 pub result: Option<TimelineUserResult>,
89}
90
91#[derive(Debug, Deserialize, Serialize)]
92pub struct TimelineV2 {
93 pub data: Option<TimelineData>,
94 pub timeline: Option<TimelineItems>,
95}
96
97#[derive(Debug, Deserialize, Serialize)]
98pub struct ThreadedConversation {
99 pub data: Option<ThreadedConversationData>,
100}
101
102#[derive(Debug, Deserialize, Serialize)]
103pub struct ThreadedConversationData {
104 pub threaded_conversation_with_injections_v2: Option<TimelineContent>,
105}
106
107#[derive(Debug, Deserialize, Serialize)]
108pub struct TweetResult {
109 pub result: Option<TimelineResultRaw>,
110}
111
112#[derive(Debug, Deserialize, Serialize)]
113pub struct TweetResultRaw {
114 pub result: Option<TimelineResultRaw>,
115}
116
117#[derive(Debug, Deserialize, Serialize)]
118pub struct EntryContent {
119 #[serde(rename = "cursorType")]
120 pub cursor_type: Option<String>,
121 pub value: Option<String>,
122 pub items: Option<Vec<EntryItem>>,
123 #[serde(rename = "itemContent")]
124 pub item_content: Option<TimelineEntryItemContent>,
125}
126
127#[derive(Debug, Deserialize, Serialize)]
128pub struct EntryItem {
129 #[serde(rename = "entryId")]
130 pub entry_id: Option<String>,
131 pub item: Option<ItemContent>,
132}
133
134#[derive(Debug, Deserialize, Serialize)]
135pub struct ItemContent {
136 pub content: Option<TimelineEntryItemContent>,
137 #[serde(rename = "itemContent")]
138 pub item_content: Option<TimelineEntryItemContent>,
139}
140
141#[derive(Debug, Deserialize, Serialize)]
142pub struct Hashtag {
143 pub text: Option<String>,
144}
145
146#[derive(Debug, Deserialize, Serialize)]
147pub struct UrlEntity {
148 pub expanded_url: Option<String>,
149}
150
151#[derive(Debug, Deserialize, Serialize)]
152pub struct UserMention {
153 pub id_str: Option<String>,
154 pub name: Option<String>,
155 pub screen_name: Option<String>,
156}
157
158#[derive(Debug, Deserialize, Serialize)]
159pub struct TimelineInstruction {
160 pub entries: Option<Vec<TimelineEntry>>,
161 pub entry: Option<TimelineEntry>,
162 #[serde(rename = "type")]
163 pub type_: Option<String>,
164}
165
166#[derive(Debug, Deserialize, Serialize)]
167pub struct SearchEntryRaw {
168 #[serde(rename = "entryId")]
169 pub entry_id: String,
170 #[serde(rename = "sortIndex")]
171 pub sort_index: String,
172 pub content: Option<SearchEntryContentRaw>,
173}
174
175#[derive(Debug, Deserialize, Serialize)]
176pub struct SearchEntryContentRaw {
177 #[serde(rename = "cursorType")]
178 pub cursor_type: Option<String>,
179 #[serde(rename = "entryType")]
180 pub entry_type: Option<String>,
181 #[serde(rename = "__typename")]
182 pub typename: Option<String>,
183 pub value: Option<String>,
184 pub items: Option<Vec<SearchEntryItemRaw>>,
185 #[serde(rename = "itemContent")]
186 pub item_content: Option<TimelineEntryItemContentRaw>,
187}
188
189#[derive(Debug, Deserialize, Serialize)]
190pub struct SearchEntryItemRaw {
191 pub item: Option<SearchEntryItemInnerRaw>,
192}
193
194#[derive(Debug, Deserialize, Serialize)]
195pub struct SearchEntryItemInnerRaw {
196 pub content: Option<TimelineEntryItemContentRaw>,
197}
198
199pub fn parse_legacy_tweet(
200 user: Option<&LegacyUserRaw>,
201 tweet: Option<&LegacyTweetRaw>,
202) -> Result<Tweet> {
203 let tweet = tweet.ok_or(TwitterError::Api(
204 "Tweet was not found in the timeline object".into(),
205 ))?;
206 let user = user.ok_or(TwitterError::Api(
207 "User was not found in the timeline object".into(),
208 ))?;
209
210 let id_str = tweet
211 .id_str
212 .as_ref()
213 .or(tweet.conversation_id_str.as_ref())
214 .ok_or(TwitterError::Api("Tweet ID was not found in object".into()))?;
215
216 let hashtags = tweet
217 .entities
218 .as_ref()
219 .and_then(|e| e.hashtags.as_ref())
220 .map(|h| h.iter().filter_map(|h| h.text.clone()).collect())
221 .unwrap_or_default();
222
223 let mentions = tweet
224 .entities
225 .as_ref()
226 .and_then(|e| e.user_mentions.as_ref())
227 .map(|mentions| {
228 mentions
229 .iter()
230 .filter_map(|m| {
231 Some(Mention {
232 id: m.id_str.clone().unwrap_or_default(),
233 name: m.name.clone(),
234 username: m.screen_name.clone(),
235 })
236 })
237 .collect()
238 })
239 .unwrap_or_default();
240
241 let (photos, videos, _) = if let Some(extended_entities) = &tweet.extended_entities {
242 if let Some(media) = &extended_entities.media {
243 parse_media_groups(media)
244 } else {
245 (Vec::new(), Vec::new(), false)
246 }
247 } else {
248 (Vec::new(), Vec::new(), false)
249 };
250
251 let mut tweet = Tweet {
252 bookmark_count: tweet.bookmark_count,
253 conversation_id: tweet.conversation_id_str.clone(),
254 id: Some(id_str.clone()),
255 hashtags,
256 likes: tweet.favorite_count,
257 mentions,
258 name: user.name.clone(),
259 permanent_url: Some(format!(
260 "https://twitter.com/{}/status/{}",
261 user.screen_name.as_ref().unwrap_or(&String::new()),
262 id_str
263 )),
264 photos,
265 replies: tweet.reply_count,
266 retweets: tweet.retweet_count,
267 text: tweet.full_text.clone(),
268 thread: Vec::new(),
269 urls: tweet
270 .entities
271 .as_ref()
272 .and_then(|e| e.urls.as_ref())
273 .map(|urls| urls.iter().filter_map(|u| u.expanded_url.clone()).collect())
274 .unwrap_or_default(),
275 user_id: tweet.user_id_str.clone(),
276 username: user.screen_name.clone(),
277 videos,
278 is_quoted: Some(false),
279 is_reply: Some(false),
280 is_retweet: Some(false),
281 is_pin: Some(false),
282 sensitive_content: Some(false),
283 quoted_status: None,
284 quoted_status_id: tweet.quoted_status_id_str.clone(),
285 in_reply_to_status_id: tweet.in_reply_to_status_id_str.clone(),
286 retweeted_status: None,
287 retweeted_status_id: None,
288 views: None,
289 html: None,
290 time_parsed: None,
291 timestamp: None,
292 place: tweet.place.clone(),
293 in_reply_to_status: None,
294 is_self_thread: None,
295 poll: None,
296 created_at: tweet.created_at.clone(),
297 ext_views: None,
298 quote_count: None,
299 reply_count: None,
300 retweet_count: None,
301 screen_name: None,
302 thread_id: None,
303 };
304
305 if let Some(created_at) = &tweet.created_at {
306 if let Ok(time) = chrono::DateTime::parse_from_str(created_at, "%a %b %d %H:%M:%S %z %Y") {
307 tweet.time_parsed = Some(time.with_timezone(&Utc));
308 tweet.timestamp = Some(time.timestamp());
309 }
310 }
311
312 if let Some(views) = &tweet.ext_views {
313 tweet.views = Some(*views);
314 }
315
316 Ok(tweet)
320}
321
322pub fn parse_timeline_entry_item_content_raw(
323 content: &TimelineEntryItemContent,
324 _entry_id: &str,
325 is_conversation: bool,
326) -> Option<Tweet> {
327 let result = content
328 .tweet_results
329 .as_ref()
330 .or(content.tweet_result.as_ref())
331 .and_then(|r| r.result.as_ref())?;
332
333 let tweet_result = parse_result(result);
334 if tweet_result.success {
335 let mut tweet = tweet_result.tweet?;
336
337 if is_conversation && content.tweet_display_type.as_deref() == Some("SelfThread") {
338 tweet.is_self_thread = Some(true);
339 }
340
341 return Some(tweet);
342 }
343
344 None
345}
346
347pub fn parse_and_push(
348 tweets: &mut Vec<Tweet>,
349 content: &TimelineEntryItemContent,
350 entry_id: String,
351 is_conversation: bool,
352) {
353 if let Some(tweet) = parse_timeline_entry_item_content_raw(content, &entry_id, is_conversation)
354 {
355 tweets.push(tweet);
356 }
357}
358
359pub fn parse_result(result: &TimelineResultRaw) -> ParseTweetResult {
360 let tweet_result = parse_legacy_tweet(
361 result
362 .core
363 .as_ref()
364 .and_then(|c| c.user_results.as_ref())
365 .and_then(|u| u.result.as_ref())
366 .and_then(|r| r.legacy.as_ref()),
367 result.legacy.as_deref(),
368 );
369
370 let mut tweet = match tweet_result {
371 Ok(tweet) => tweet,
372 Err(e) => {
373 return ParseTweetResult {
374 success: false,
375 tweet: None,
376 err: Some(e),
377 }
378 }
379 };
380
381 if tweet.views.is_none() {
382 if let Some(count) = result
383 .views
384 .as_ref()
385 .and_then(|v| v.count.as_ref())
386 .and_then(|c| c.parse().ok())
387 {
388 tweet.views = Some(count);
389 }
390 }
391
392 if let Some(quoted) = result.quoted_status_result.as_ref() {
393 if let Some(quoted_result) = quoted.result.as_ref() {
394 let quoted_tweet_result = parse_result(quoted_result);
395 if quoted_tweet_result.success {
396 tweet.quoted_status = quoted_tweet_result.tweet.map(Box::new);
397 }
398 }
399 }
400
401 ParseTweetResult {
402 success: true,
403 tweet: Some(tweet),
404 err: None,
405 }
406}
407
408pub struct ParseTweetResult {
409 pub success: bool,
410 pub tweet: Option<Tweet>,
411 pub err: Option<TwitterError>,
412}
413
414#[derive(Debug, Serialize, Deserialize)]
415pub struct QueryTweetsResponse {
416 pub tweets: Vec<Tweet>,
417 pub next: Option<String>,
418 pub previous: Option<String>,
419}
420
421pub fn parse_timeline_tweets_v2(timeline: &TimelineV2) -> QueryTweetsResponse {
422 let mut tweets = Vec::new();
423 let mut bottom_cursor = None;
424 let mut top_cursor = None;
425
426 let instructions = timeline
427 .data
428 .as_ref()
429 .and_then(|data| data.user.as_ref())
430 .and_then(|user| user.result.as_ref())
431 .and_then(|result| result.timeline_v2.as_ref())
432 .and_then(|timeline| timeline.timeline.as_ref())
433 .and_then(|timeline| timeline.instructions.as_ref())
434 .unwrap_or(&EMPTY_INSTRUCTIONS);
435
436 let expected_entry_types = ["tweet-", "profile-conversation-"];
437
438 for instruction in instructions {
439 let entries = instruction.entries.as_deref().unwrap_or_else(|| {
440 instruction
441 .entry
442 .as_ref()
443 .map(std::slice::from_ref)
444 .unwrap_or_default()
445 });
446
447 for entry in entries {
448 let content = match &entry.content {
449 Some(content) => content,
450 None => continue,
451 };
452
453 if let Some(cursor_type) = &content.cursor_type {
454 match cursor_type.as_str() {
455 "Bottom" => {
456 bottom_cursor = content.value.clone();
457 continue;
458 }
459 "Top" => {
460 top_cursor = content.value.clone();
461 continue;
462 }
463 _ => {}
464 }
465 }
466
467 let entry_id = match &entry.entry_id {
468 Some(id) => id,
469 None => continue,
470 };
471 if !expected_entry_types
472 .iter()
473 .any(|prefix| entry_id.starts_with(prefix))
474 {
475 continue;
476 }
477
478 if let Some(ref item_content) = content.item_content {
479 parse_and_push(&mut tweets, item_content, entry_id.clone(), false);
480 }
481
482 if let Some(items) = &content.items {
483 for item in items {
484 if let Some(item) = &item.item {
485 if let Some(item_content) = &item.item_content {
486 parse_and_push(&mut tweets, item_content, entry_id.clone(), false);
487 }
488 }
489 }
490 }
491 }
492 }
493
494 QueryTweetsResponse {
495 tweets,
496 next: bottom_cursor,
497 previous: top_cursor,
498 }
499}
500
501pub fn parse_threaded_conversation(conversation: &ThreadedConversation) -> Option<Tweet> {
502 let mut main_tweet: Option<Tweet> = None;
503 let mut replies: Vec<Tweet> = Vec::new();
504
505 let instructions = conversation
506 .data
507 .as_ref()
508 .and_then(|data| data.threaded_conversation_with_injections_v2.as_ref())
509 .and_then(|conv| conv.instructions.as_ref())
510 .unwrap_or(&EMPTY_INSTRUCTIONS);
511
512 for instruction in instructions {
513 let entries = instruction.entries.as_deref().unwrap_or_default();
514
515 for entry in entries {
516 if let Some(content) = &entry.content {
517 if let Some(item_content) = &content.item_content {
518 if let Some(tweet) = parse_timeline_entry_item_content_raw(
519 item_content,
520 entry.entry_id.as_deref().unwrap_or_default(),
521 true,
522 ) {
523 if main_tweet.is_none() {
524 main_tweet = Some(tweet);
525 } else {
526 replies.push(tweet);
527 }
528 }
529 }
530
531 if let Some(items) = &content.items {
532 for item in items {
533 if let Some(item) = &item.item {
534 if let Some(item_content) = &item.item_content {
535 if let Some(tweet) = parse_timeline_entry_item_content_raw(
536 item_content,
537 entry.entry_id.as_deref().unwrap_or_default(),
538 true,
539 ) {
540 replies.push(tweet);
541 }
542 }
543 }
544 }
545 }
546 }
547 }
548 }
549
550 if let Some(mut main_tweet) = main_tweet {
551 for reply in &replies {
552 if let Some(reply_id) = &reply.in_reply_to_status_id {
553 if let Some(main_id) = &main_tweet.id {
554 if reply_id == main_id {
555 main_tweet.replies = Some(replies.len() as i32);
556 break;
557 }
558 }
559 }
560 }
561
562 if main_tweet.is_self_thread == Some(true) {
563 let thread = replies
564 .iter()
565 .filter(|t| t.is_self_thread == Some(true))
566 .cloned()
567 .collect::<Vec<_>>();
568
569 if thread.is_empty() {
570 main_tweet.is_self_thread = Some(false);
571 } else {
572 main_tweet.thread = thread;
573 }
574 }
575
576 Some(main_tweet)
579 } else {
580 None
581 }
582}