agent_twitter_client/timeline/
v1.rs

1use crate::models::tweets::Mention;
2use crate::models::tweets::PlaceRaw;
3use crate::models::{Profile, Tweet};
4use crate::profile::LegacyUserRaw;
5use crate::timeline::tweet_utils::{parse_media_groups, reconstruct_tweet_html};
6use chrono::DateTime;
7use chrono::Utc;
8use serde::{Deserialize, Serialize};
9use std::collections::HashMap;
10
11#[derive(Debug, Deserialize, Serialize)]
12pub struct Hashtag {
13    pub text: Option<String>,
14}
15
16#[derive(Debug, Deserialize, Serialize)]
17pub struct TimelineUserMentionBasicRaw {
18    pub id_str: Option<String>,
19    pub name: Option<String>,
20    pub screen_name: Option<String>,
21}
22
23#[derive(Debug, Deserialize, Serialize)]
24pub struct TimelineMediaBasicRaw {
25    pub media_url_https: Option<String>,
26    pub r#type: Option<String>,
27    pub url: Option<String>,
28}
29
30#[derive(Debug, Deserialize, Serialize)]
31pub struct TimelineUrlBasicRaw {
32    pub expanded_url: Option<String>,
33    pub url: Option<String>,
34}
35
36#[derive(Debug, Deserialize, Serialize)]
37pub struct ExtSensitiveMediaWarningRaw {
38    pub adult_content: Option<bool>,
39    pub graphic_violence: Option<bool>,
40    pub other: Option<bool>,
41}
42
43#[derive(Debug, Deserialize, Serialize)]
44pub struct VideoVariant {
45    pub bitrate: Option<i32>,
46    pub url: Option<String>,
47}
48
49#[derive(Debug, Deserialize, Serialize)]
50pub struct VideoInfo {
51    pub variants: Option<Vec<VideoVariant>>,
52}
53
54#[derive(Debug, Deserialize, Serialize)]
55pub struct TimelineMediaExtendedRaw {
56    pub id_str: Option<String>,
57    pub media_url_https: Option<String>,
58    pub ext_sensitive_media_warning: Option<ExtSensitiveMediaWarningRaw>,
59    pub r#type: Option<String>,
60    pub url: Option<String>,
61    pub video_info: Option<VideoInfo>,
62    pub ext_alt_text: Option<String>,
63}
64
65#[derive(Debug, Deserialize, Serialize)]
66pub struct SearchResultRaw {
67    pub rest_id: Option<String>,
68    pub __typename: Option<String>,
69    pub core: Option<UserResultsCore>,
70    pub views: Option<Views>,
71    pub note_tweet: Option<NoteTweet>,
72    pub quoted_status_result: Option<QuotedStatusResult>,
73    pub legacy: Option<LegacyTweetRaw>,
74}
75
76#[derive(Debug, Deserialize, Serialize)]
77pub struct UserResultsCore {
78    pub user_results: Option<UserResults>,
79}
80
81#[derive(Debug, Deserialize, Serialize)]
82pub struct UserResults {
83    pub result: Option<UserResult>,
84}
85
86#[derive(Debug, Deserialize, Serialize)]
87pub struct UserResult {
88    pub is_blue_verified: Option<bool>,
89    pub legacy: Option<LegacyUserRaw>,
90}
91
92#[derive(Debug, Deserialize, Serialize)]
93pub struct Views {
94    pub count: Option<String>,
95}
96
97#[derive(Debug, Deserialize, Serialize)]
98pub struct NoteTweet {
99    pub note_tweet_results: Option<NoteTweetResults>,
100}
101
102#[derive(Debug, Deserialize, Serialize)]
103pub struct NoteTweetResults {
104    pub result: Option<NoteTweetResult>,
105}
106
107#[derive(Debug, Deserialize, Serialize)]
108pub struct NoteTweetResult {
109    pub text: Option<String>,
110}
111
112#[derive(Debug, Deserialize, Serialize)]
113pub struct QuotedStatusResult {
114    pub result: Option<Box<SearchResultRaw>>,
115}
116
117#[derive(Debug, Deserialize, Serialize)]
118pub struct TimelineResultRaw {
119    pub result: Option<Box<TimelineResultRaw>>,
120    pub rest_id: Option<String>,
121    pub __typename: Option<String>,
122    pub core: Option<TimelineCore>,
123    pub views: Option<TimelineViews>,
124    pub note_tweet: Option<TimelineNoteTweet>,
125    pub quoted_status_result: Option<Box<TimelineQuotedStatus>>,
126    pub legacy: Option<Box<LegacyTweetRaw>>,
127    pub tweet: Option<Box<TimelineResultRaw>>,
128}
129
130#[derive(Debug, Deserialize, Serialize)]
131pub struct TimelineCore {
132    pub user_results: Option<TimelineUserResults>,
133}
134
135#[derive(Debug, Deserialize, Serialize)]
136pub struct TimelineUserResults {
137    pub result: Option<TimelineUserResult>,
138}
139
140#[derive(Debug, Deserialize, Serialize)]
141pub struct TimelineUserResult {
142    pub is_blue_verified: Option<bool>,
143    pub legacy: Option<LegacyUserRaw>,
144}
145
146#[derive(Debug, Deserialize, Serialize)]
147pub struct TimelineViews {
148    pub count: Option<String>,
149}
150
151#[derive(Debug, Deserialize, Serialize)]
152pub struct TimelineNoteTweet {
153    pub note_tweet_results: Option<TimelineNoteTweetResults>,
154}
155
156#[derive(Debug, Deserialize, Serialize)]
157pub struct TimelineNoteTweetResults {
158    pub result: Option<TimelineNoteTweetResult>,
159}
160
161#[derive(Debug, Deserialize, Serialize)]
162pub struct TimelineNoteTweetResult {
163    pub text: Option<String>,
164}
165
166#[derive(Debug, Deserialize, Serialize)]
167pub struct TimelineQuotedStatus {
168    pub result: Option<Box<TimelineResultRaw>>,
169}
170
171#[derive(Debug, Deserialize, Serialize)]
172pub struct LegacyTweetRaw {
173    pub bookmark_count: Option<i32>,
174    pub conversation_id_str: Option<String>,
175    pub created_at: Option<String>,
176    pub favorite_count: Option<i32>,
177    pub full_text: Option<String>,
178    pub entities: Option<TweetEntities>,
179    pub extended_entities: Option<TweetExtendedEntities>,
180    pub id_str: Option<String>,
181    pub in_reply_to_status_id_str: Option<String>,
182    pub place: Option<PlaceRaw>,
183    pub reply_count: Option<i32>,
184    pub retweet_count: Option<i32>,
185    pub retweeted_status_id_str: Option<String>,
186    pub retweeted_status_result: Option<TimelineRetweetedStatus>,
187    pub quoted_status_id_str: Option<String>,
188    pub time: Option<String>,
189    pub user_id_str: Option<String>,
190    pub ext_views: Option<TweetExtViews>,
191}
192
193#[derive(Debug, Deserialize, Serialize)]
194pub struct TweetEntities {
195    pub hashtags: Option<Vec<Hashtag>>,
196    pub media: Option<Vec<TimelineMediaBasicRaw>>,
197    pub urls: Option<Vec<TimelineUrlBasicRaw>>,
198    pub user_mentions: Option<Vec<TimelineUserMentionBasicRaw>>,
199}
200
201#[derive(Debug, Deserialize, Serialize)]
202pub struct TweetExtendedEntities {
203    pub media: Option<Vec<TimelineMediaExtendedRaw>>,
204}
205
206#[derive(Debug, Deserialize, Serialize)]
207pub struct TimelineRetweetedStatus {
208    pub result: Option<TimelineResultRaw>,
209}
210
211#[derive(Debug, Deserialize, Serialize)]
212pub struct TweetExtViews {
213    pub state: Option<String>,
214    pub count: Option<String>,
215}
216
217#[derive(Debug, Deserialize, Serialize)]
218pub struct TimelineGlobalObjectsRaw {
219    pub tweets: Option<HashMap<String, Option<LegacyTweetRaw>>>,
220    pub users: Option<HashMap<String, Option<LegacyUserRaw>>>,
221}
222
223#[derive(Debug, Deserialize, Serialize)]
224pub struct TimelineDataRawCursor {
225    pub value: Option<String>,
226    pub cursor_type: Option<String>,
227}
228
229#[derive(Debug, Deserialize, Serialize)]
230pub struct TimelineDataRawEntity {
231    pub id: Option<String>,
232}
233
234#[derive(Debug, Deserialize, Serialize)]
235pub struct TimelineDataRawModuleItem {
236    pub client_event_info: Option<ClientEventInfo>,
237}
238
239#[derive(Debug, Deserialize, Serialize)]
240pub struct ClientEventInfo {
241    pub details: Option<ClientEventDetails>,
242}
243
244#[derive(Debug, Deserialize, Serialize)]
245pub struct ClientEventDetails {
246    pub guide_details: Option<GuideDetails>,
247}
248
249#[derive(Debug, Deserialize, Serialize)]
250pub struct GuideDetails {
251    pub transparent_guide_details: Option<TransparentGuideDetails>,
252}
253
254#[derive(Debug, Deserialize, Serialize)]
255pub struct TransparentGuideDetails {
256    pub trend_metadata: Option<TrendMetadata>,
257}
258
259#[derive(Debug, Deserialize, Serialize)]
260pub struct TrendMetadata {
261    pub trend_name: Option<String>,
262}
263
264#[derive(Debug, Deserialize, Serialize)]
265pub struct TimelineDataRawAddEntry {
266    pub content: Option<TimelineEntryContent>,
267}
268
269#[derive(Debug, Deserialize, Serialize)]
270pub struct TimelineDataRawPinEntry {
271    pub content: Option<TimelinePinContent>,
272}
273
274#[derive(Debug, Deserialize, Serialize)]
275pub struct TimelinePinContent {
276    pub item: Option<TimelineItem>,
277}
278
279#[derive(Debug, Deserialize, Serialize)]
280pub struct TimelineDataRawReplaceEntry {
281    pub content: Option<TimelineReplaceContent>,
282}
283
284#[derive(Debug, Deserialize, Serialize)]
285pub struct TimelineReplaceContent {
286    pub operation: Option<TimelineOperation>,
287}
288
289#[derive(Debug, Deserialize, Serialize)]
290pub struct TimelineDataRawInstruction {
291    pub add_entries: Option<TimelineAddEntries>,
292    pub pin_entry: Option<TimelineDataRawPinEntry>,
293    pub replace_entry: Option<TimelineDataRawReplaceEntry>,
294}
295
296#[derive(Debug, Deserialize, Serialize)]
297pub struct TimelineAddEntries {
298    pub entries: Option<Vec<TimelineDataRawAddEntry>>,
299}
300
301#[derive(Debug, Deserialize, Serialize)]
302pub struct TimelineDataRaw {
303    pub instructions: Option<Vec<TimelineDataRawInstruction>>,
304}
305
306#[derive(Debug, Deserialize, Serialize)]
307pub struct TimelineV1 {
308    pub global_objects: Option<TimelineGlobalObjectsRaw>,
309    pub timeline: Option<TimelineDataRaw>,
310}
311
312#[derive(Debug)]
313pub enum ParseTweetResult {
314    Success { tweet: Tweet },
315    Error { err: String },
316}
317
318#[derive(Debug, Serialize, Deserialize)]
319pub struct QueryTweetsResponse {
320    pub tweets: Vec<Tweet>,
321    pub next: Option<String>,
322    pub previous: Option<String>,
323}
324
325#[derive(Debug, Deserialize, Serialize)]
326pub struct QueryProfilesResponse {
327    pub profiles: Vec<Profile>,
328    pub next: Option<String>,
329    pub previous: Option<String>,
330}
331
332#[derive(Debug, Deserialize, Serialize)]
333pub struct TimelineEntryContent {
334    pub item: Option<TimelineItem>,
335    pub operation: Option<TimelineOperation>,
336    pub timeline_module: Option<TimelineModule>,
337}
338
339#[derive(Debug, Deserialize, Serialize)]
340pub struct TimelineItem {
341    pub content: Option<TimelineContent>,
342}
343
344#[derive(Debug, Deserialize, Serialize)]
345pub struct TimelineContent {
346    pub tweet: Option<TimelineDataRawEntity>,
347    pub user: Option<TimelineDataRawEntity>,
348}
349
350#[derive(Debug, Deserialize, Serialize)]
351pub struct TimelineOperation {
352    pub cursor: Option<TimelineDataRawCursor>,
353}
354
355#[derive(Debug, Deserialize, Serialize)]
356pub struct TimelineModule {
357    pub items: Option<Vec<TimelineModuleItemWrapper>>,
358}
359
360#[derive(Debug, Deserialize, Serialize)]
361pub struct TimelineModuleItemWrapper {
362    pub item: Option<TimelineDataRawModuleItem>,
363}
364
365#[derive(Debug)]
366pub struct UserMention {
367    pub id: String,
368    pub username: String,
369    pub name: String,
370}
371
372pub fn parse_timeline_tweet(timeline: &TimelineV1, id: &str) -> ParseTweetResult {
373    let empty_tweets = HashMap::new();
374    let tweets = match &timeline.global_objects {
375        Some(go) => go.tweets.as_ref().unwrap_or(&empty_tweets),
376        None => {
377            return ParseTweetResult::Error {
378                err: "No global objects found".to_string(),
379            }
380        }
381    };
382
383    let tweet = match tweets.get(id) {
384        Some(Some(t)) => t,
385        _ => {
386            return ParseTweetResult::Error {
387                err: format!("Tweet \"{}\" was not found in the timeline object.", id),
388            }
389        }
390    };
391
392    let user_id = match &tweet.user_id_str {
393        Some(id) => id,
394        None => {
395            return ParseTweetResult::Error {
396                err: "Tweet has no user ID".to_string(),
397            }
398        }
399    };
400
401    let empty_users = HashMap::new();
402    let users = match &timeline.global_objects {
403        Some(go) => go.users.as_ref().unwrap_or(&empty_users),
404        None => {
405            return ParseTweetResult::Error {
406                err: "No users found".to_string(),
407            }
408        }
409    };
410
411    let user = match users.get(user_id) {
412        Some(Some(u)) => u,
413        _ => {
414            return ParseTweetResult::Error {
415                err: format!("User \"{}\" has no username data.", user_id),
416            }
417        }
418    };
419
420    let hashtags = tweet
421        .entities
422        .as_ref()
423        .and_then(|e| e.hashtags.as_ref())
424        .map(|h| h.iter().filter_map(|tag| tag.text.clone()).collect())
425        .unwrap_or_default();
426
427    let mentions = tweet
428        .entities
429        .as_ref()
430        .and_then(|e| e.user_mentions.as_ref())
431        .map(|m| {
432            m.iter()
433                .filter_map(|mention| {
434                    if let (Some(id), Some(screen_name), Some(name)) =
435                        (&mention.id_str, &mention.screen_name, &mention.name)
436                    {
437                        Some(Mention {
438                            id: id.clone(),
439                            username: Some(screen_name.clone()),
440                            name: Some(name.clone()),
441                        })
442                    } else {
443                        None
444                    }
445                })
446                .collect()
447        })
448        .unwrap_or_default();
449
450    let empty_media = Vec::new();
451    let media = tweet
452        .extended_entities
453        .as_ref()
454        .and_then(|e| e.media.as_ref())
455        .unwrap_or(&empty_media);
456
457    let urls = tweet
458        .entities
459        .as_ref()
460        .and_then(|e| e.urls.as_ref())
461        .map(|u| {
462            u.iter()
463                .filter_map(|url| url.expanded_url.clone())
464                .collect()
465        })
466        .unwrap_or_default();
467
468    let (photos, videos, sensitive_content) = parse_media_groups(media);
469
470    let mut tweet_obj = Tweet {
471        conversation_id: tweet.conversation_id_str.clone(),
472        id: Some(id.to_string()),
473        hashtags,
474        likes: tweet.favorite_count,
475        mentions,
476        name: user.name.clone(),
477        permanent_url: Some(format!(
478            "https://twitter.com/{}/status/{}",
479            user.screen_name.as_ref().unwrap_or(&String::new()),
480            id
481        )),
482        photos,
483        replies: tweet.reply_count,
484        retweets: tweet.retweet_count,
485        text: tweet.full_text.clone(),
486        thread: Vec::new(),
487        urls,
488        user_id: tweet.user_id_str.clone(),
489        username: user.screen_name.clone(),
490        videos,
491        time_parsed: None,
492        timestamp: None,
493        place: None,
494        is_quoted: Some(false),
495        quoted_status_id: None,
496        quoted_status: None,
497        is_reply: Some(false),
498        in_reply_to_status_id: None,
499        in_reply_to_status: None,
500        is_retweet: Some(false),
501        retweeted_status_id: None,
502        retweeted_status: None,
503        views: None,
504        is_pin: Some(false),
505        sensitive_content: Some(sensitive_content),
506        html: None,
507        bookmark_count: None,
508        is_self_thread: None,
509        poll: None,
510        created_at: None,
511        ext_views: None,
512        quote_count: None,
513        reply_count: None,
514        retweet_count: None,
515        screen_name: None,
516        thread_id: None,
517    };
518
519    if let Some(created_at) = &tweet.created_at {
520        if let Ok(parsed_time) = DateTime::parse_from_str(created_at, "%a %b %d %H:%M:%S %z %Y") {
521            tweet_obj.time_parsed = Some(parsed_time.with_timezone(&Utc));
522            tweet_obj.timestamp = Some(parsed_time.timestamp());
523        }
524    }
525
526    if let Some(place) = &tweet.place {
527        tweet_obj.place = Some(place.clone());
528    }
529
530    if let Some(quoted_id) = &tweet.quoted_status_id_str {
531        tweet_obj.is_quoted = Some(true);
532        tweet_obj.quoted_status_id = Some(quoted_id.clone());
533
534        if let ParseTweetResult::Success {
535            tweet: quoted_tweet,
536        } = parse_timeline_tweet(timeline, quoted_id)
537        {
538            tweet_obj.quoted_status = Some(Box::new(quoted_tweet));
539        }
540    }
541
542    if let Some(ext_views) = &tweet.ext_views {
543        if let Some(count) = &ext_views.count {
544            if let Ok(views) = count.parse::<i32>() {
545                tweet_obj.views = Some(views);
546            }
547        }
548    }
549
550    tweet_obj.html = reconstruct_tweet_html(tweet, &tweet_obj.photos, &tweet_obj.videos);
551
552    ParseTweetResult::Success { tweet: tweet_obj }
553}
agent_twitter_client/timeline/v1.rs

agent_twitter_client/timeline/
v1.rs
