use std::collections::HashSet;
use std::path::Path;
use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE, COOKIE};
use serde_json::{json, Value};
use tail_fin_common::TailFinError;
use crate::auth::{extract_auth_from_cookies, load_cookies_from_file, BEARER_TOKEN};
use crate::graphql::default_features;
use crate::parsing::{
parse_article_response, parse_bookmarks_response, parse_likes_response,
parse_notifications_response, parse_search_response, parse_thread_response,
parse_timeline_response, parse_user_list_response,
};
use crate::types::{
Article, Notification, TimelineResponse, TimelineType, Tweet, TwitterUser, UserProfile,
};
pub struct TwitterHttpClient {
client: reqwest::Client,
ct0: String,
cookie_header: String,
query_id_cache: tokio::sync::OnceCell<Option<Value>>,
}
impl TwitterHttpClient {
pub fn from_cookie_file(path: &Path) -> Result<Self, TailFinError> {
let cookies = load_cookies_from_file(path)?;
let (ct0, _auth_token) = extract_auth_from_cookies(&cookies)?;
let cookie_header = tail_fin_common::cookies::build_cookie_header(&cookies);
let client = reqwest::Client::new();
Ok(Self {
client,
ct0,
cookie_header,
query_id_cache: tokio::sync::OnceCell::const_new(),
})
}
pub async fn from_browser(chrome_host: &str) -> Result<Self, TailFinError> {
let tmp_path = std::env::temp_dir().join(format!(
"tail-fin-twitter-cookies-{}.txt",
std::process::id()
));
crate::auth::export_cookies(chrome_host, &tmp_path).await?;
Self::from_cookie_file(&tmp_path)
}
pub fn from_tokens(ct0: String, auth_token: String) -> Self {
let cookie_header = format!("ct0={}; auth_token={}", ct0, auth_token);
let client = reqwest::Client::new();
Self {
client,
ct0,
cookie_header,
query_id_cache: tokio::sync::OnceCell::const_new(),
}
}
fn build_headers(&self) -> Result<HeaderMap, TailFinError> {
let mut headers = HeaderMap::new();
headers.insert(
"Authorization",
HeaderValue::from_str(&format!("Bearer {}", BEARER_TOKEN))
.map_err(|e| TailFinError::Parse(format!("invalid bearer token header: {}", e)))?,
);
headers.insert(
"X-Csrf-Token",
HeaderValue::from_str(&self.ct0)
.map_err(|e| TailFinError::Parse(format!("invalid ct0 header: {}", e)))?,
);
headers.insert(
"X-Twitter-Auth-Type",
HeaderValue::from_static("OAuth2Session"),
);
headers.insert("X-Twitter-Active-User", HeaderValue::from_static("yes"));
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
headers.insert(
COOKIE,
HeaderValue::from_str(&self.cookie_header)
.map_err(|e| TailFinError::Parse(format!("invalid cookie header: {}", e)))?,
);
Ok(headers)
}
async fn cached_query_ids(&self) -> &Option<Value> {
self.query_id_cache
.get_or_init(|| async {
let url = "https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json";
match self.client.get(url).send().await {
Ok(resp) => resp.json::<Value>().await.ok(),
Err(_) => None,
}
})
.await
}
async fn resolve_query_id(&self, endpoint: &str, fallback: &str) -> String {
if let Some(data) = self.cached_query_ids().await {
if let Some(query_id) = data
.get(endpoint)
.and_then(|v| v.get("queryId"))
.and_then(|v| v.as_str())
{
return query_id.to_string();
}
}
fallback.to_string()
}
pub async fn timeline(
&self,
kind: TimelineType,
count: usize,
cursor: Option<&str>,
) -> Result<TimelineResponse, TailFinError> {
let endpoint = kind.endpoint();
let query_id = self
.resolve_query_id(endpoint, kind.fallback_query_id())
.await;
let mut variables = json!({
"count": count,
"includePromotedContent": true,
"latestControlAvailable": true,
"requestContext": "launch",
});
if let Some(c) = cursor {
variables["cursor"] = json!(c);
}
let features = default_features();
let method = kind.method();
let headers = self.build_headers()?;
let data = if method == "POST" {
let body = json!({
"variables": variables,
"features": features,
});
let url = format!("https://x.com/i/api/graphql/{}/{}", query_id, endpoint);
let resp = self
.client
.post(&url)
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
Self::handle_response(resp).await?
} else {
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/{}?variables={}&features={}",
query_id,
endpoint,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
);
let resp = self
.client
.get(&url)
.headers(headers)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
Self::handle_response(resp).await?
};
parse_timeline_response(&data)
}
pub async fn search(
&self,
query: &str,
count: usize,
cursor: Option<&str>,
) -> Result<TimelineResponse, TailFinError> {
let query_id = self
.resolve_query_id("SearchTimeline", "lZ0GCEojmtQfiUQa5oJSEw")
.await;
let mut variables = json!({
"rawQuery": query,
"count": count,
"querySource": "typed_query",
"product": "Latest"
});
if let Some(c) = cursor {
variables["cursor"] = json!(c);
}
let features = default_features();
let field_toggles = json!({
"withArticleRichContentState": true,
"withArticlePlainText": false,
"withGrokAnalyze": false,
"withDisallowedReplyControls": false
});
let body = json!({
"variables": variables,
"features": features,
"fieldToggles": field_toggles,
});
let url = format!("https://x.com/i/api/graphql/{}/SearchTimeline", query_id);
let headers = self.build_headers()?;
let resp = self
.client
.post(&url)
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
let mut resp = parse_search_response(&data);
let mut seen = HashSet::new();
resp.tweets.retain(|t| seen.insert(t.id.clone()));
resp.tweets.truncate(count);
Ok(resp)
}
pub async fn profile(&self, username: &str) -> Result<UserProfile, TailFinError> {
let query_id = self
.resolve_query_id("UserByScreenName", "qRednkZG-rn1P6b48NINmQ")
.await;
let variables = json!({
"screen_name": username,
"withSafetyModeUserFields": true,
});
let features = json!({
"hidden_profile_subscriptions_enabled": true,
"rweb_tipjar_consumption_enabled": true,
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"subscriptions_verification_info_is_identity_verified_enabled": true,
"highlights_tweets_tab_ui_enabled": true,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": true,
});
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/UserByScreenName?variables={}&features={}",
query_id,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
);
let resp = self
.client
.get(&url)
.headers(self.build_headers()?)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
let result = data
.pointer("/data/user/result")
.ok_or_else(|| TailFinError::Parse(format!("User @{} not found", username)))?;
let legacy = result.get("legacy").unwrap_or(&Value::Null);
let expanded_url = legacy
.pointer("/entities/url/urls/0/expanded_url")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
Ok(UserProfile {
screen_name: legacy
.get("screen_name")
.and_then(|v| v.as_str())
.unwrap_or(username)
.to_string(),
name: legacy
.get("name")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
bio: legacy
.get("description")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
location: legacy
.get("location")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
url: expanded_url,
followers: legacy
.get("followers_count")
.and_then(|v| v.as_u64())
.unwrap_or(0),
following: legacy
.get("friends_count")
.and_then(|v| v.as_u64())
.unwrap_or(0),
tweets: legacy
.get("statuses_count")
.and_then(|v| v.as_u64())
.unwrap_or(0),
likes: legacy
.get("favourites_count")
.and_then(|v| v.as_u64())
.unwrap_or(0),
verified: result
.get("is_blue_verified")
.and_then(|v| v.as_bool())
.unwrap_or(false),
created_at: legacy
.get("created_at")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
})
}
pub async fn bookmarks(
&self,
count: usize,
cursor: Option<&str>,
) -> Result<TimelineResponse, TailFinError> {
let query_id = self
.resolve_query_id("Bookmarks", "Fy0QMy4q_aZCpkO0PnyLYw")
.await;
let mut variables = json!({
"count": count,
"includePromotedContent": false,
});
if let Some(c) = cursor {
variables["cursor"] = json!(c);
}
let features = default_features();
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/Bookmarks?variables={}&features={}",
query_id,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
);
let resp = self
.client
.get(&url)
.headers(self.build_headers()?)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
let mut resp = parse_bookmarks_response(&data);
resp.tweets.truncate(count);
Ok(resp)
}
pub async fn likes(
&self,
username: &str,
count: usize,
cursor: Option<&str>,
) -> Result<TimelineResponse, TailFinError> {
let user_id = self.resolve_user_id(username).await?;
let query_id = self
.resolve_query_id("Likes", "RozQdCp4CilQzrcuU0NY5w")
.await;
let mut variables = json!({
"userId": user_id,
"count": count,
"includePromotedContent": false,
"withClientEventToken": false,
"withBirdwatchNotes": false,
"withVoice": true,
});
if let Some(c) = cursor {
variables["cursor"] = json!(c);
}
let features = default_features();
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/Likes?variables={}&features={}",
query_id,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
);
let resp = self
.client
.get(&url)
.headers(self.build_headers()?)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
let mut resp = parse_likes_response(&data);
resp.tweets.truncate(count);
Ok(resp)
}
pub async fn thread(&self, tweet_id: &str, count: usize) -> Result<Vec<Tweet>, TailFinError> {
let query_id = self
.resolve_query_id("TweetDetail", "nBS-WpgA6ZG0CyNHD517JQ")
.await;
let variables = json!({
"focalTweetId": tweet_id,
"referrer": "tweet",
"with_rux_injections": false,
"includePromotedContent": false,
"rankingMode": "Recency",
"withCommunity": true,
"withQuickPromoteEligibilityTweetFields": true,
"withBirdwatchNotes": true,
"withVoice": true,
});
let features = json!({
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"longform_notetweets_consumption_enabled": true,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": true,
"freedom_of_speech_not_reach_fetch_enabled": true,
});
let field_toggles = json!({
"withArticleRichContentState": true,
"withArticlePlainText": false,
});
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let toggles_str = serde_json::to_string(&field_toggles).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/TweetDetail?variables={}&features={}&fieldToggles={}",
query_id,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
urlencoding::encode(&toggles_str),
);
let resp = self
.client
.get(&url)
.headers(self.build_headers()?)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
let mut tweets = parse_thread_response(&data);
tweets.truncate(count);
Ok(tweets)
}
async fn resolve_user_id(&self, username: &str) -> Result<String, TailFinError> {
let query_id = self
.resolve_query_id("UserByScreenName", "qRednkZG-rn1P6b48NINmQ")
.await;
let variables = json!({
"screen_name": username,
"withSafetyModeUserFields": true,
});
let features = json!({
"hidden_profile_subscriptions_enabled": true,
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": true,
});
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/UserByScreenName?variables={}&features={}",
query_id,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
);
let resp = self
.client
.get(&url)
.headers(self.build_headers()?)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
data.pointer("/data/user/result/rest_id")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.ok_or_else(|| TailFinError::Parse(format!("Could not find user @{}", username)))
}
pub async fn followers(
&self,
username: &str,
count: usize,
) -> Result<Vec<TwitterUser>, TailFinError> {
let user_id = self.resolve_user_id(username).await?;
let query_id = self
.resolve_query_id("Followers", "IOh4aS6UdGWGJUYTqliQ7Q")
.await;
let variables = json!({
"userId": user_id,
"count": count,
"includePromotedContent": false,
});
let features = user_list_features();
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/Followers?variables={}&features={}",
query_id,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
);
let resp = self
.client
.get(&url)
.headers(self.build_headers()?)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
let mut users = parse_user_list_response(&data);
users.truncate(count);
Ok(users)
}
pub async fn following(
&self,
username: &str,
count: usize,
) -> Result<Vec<TwitterUser>, TailFinError> {
let user_id = self.resolve_user_id(username).await?;
let query_id = self
.resolve_query_id("Following", "zx6e-TLzRkeDO_a7p4b3JQ")
.await;
let variables = json!({
"userId": user_id,
"count": count,
"includePromotedContent": false,
});
let features = user_list_features();
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/Following?variables={}&features={}",
query_id,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
);
let resp = self
.client
.get(&url)
.headers(self.build_headers()?)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
let mut users = parse_user_list_response(&data);
users.truncate(count);
Ok(users)
}
pub async fn notifications(&self, count: usize) -> Result<Vec<Notification>, TailFinError> {
let query_id = self
.resolve_query_id("Notifications", "PsW1KQFN_nhEHwvF3GBErA")
.await;
let variables = json!({
"count": count,
"includePromotedContent": false,
});
let features = default_features();
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/Notifications?variables={}&features={}",
query_id,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
);
let resp = self
.client
.get(&url)
.headers(self.build_headers()?)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
let mut notifications = parse_notifications_response(&data);
notifications.truncate(count);
Ok(notifications)
}
pub async fn article(&self, tweet_id: &str) -> Result<Article, TailFinError> {
let query_id = self
.resolve_query_id("TweetResultByRestId", "V3vfsYzNEyD9tsf4xoFRgw")
.await;
let variables = json!({
"tweetId": tweet_id,
"withCommunity": false,
"includePromotedContent": false,
"withVoice": false,
});
let features = default_features();
let field_toggles = json!({
"withArticleRichContentState": true,
"withArticlePlainText": false,
"withGrokAnalyze": false,
"withDisallowedReplyControls": false,
});
let vars_str = serde_json::to_string(&variables).unwrap_or_default();
let feats_str = serde_json::to_string(&features).unwrap_or_default();
let toggles_str = serde_json::to_string(&field_toggles).unwrap_or_default();
let url = format!(
"https://x.com/i/api/graphql/{}/TweetResultByRestId?variables={}&features={}&fieldToggles={}",
query_id,
urlencoding::encode(&vars_str),
urlencoding::encode(&feats_str),
urlencoding::encode(&toggles_str),
);
let resp = self
.client
.get(&url)
.headers(self.build_headers()?)
.send()
.await
.map_err(|e| TailFinError::Api(format!("HTTP request failed: {}", e)))?;
let data = Self::handle_response(resp).await?;
parse_article_response(&data, tweet_id)
}
async fn handle_response(resp: reqwest::Response) -> Result<Value, TailFinError> {
let status = resp.status();
if !status.is_success() {
let body = resp.text().await.unwrap_or_default();
if status.as_u16() == 401 || status.as_u16() == 403 {
return Err(TailFinError::Api(format!(
"Twitter auth failed ({}). Cookies may be expired.\n Run `tail-fin --connect twitter export-cookies` to refresh.",
status
)));
}
return Err(TailFinError::Api(format!("HTTP {}: {}", status, body)));
}
resp.json::<Value>()
.await
.map_err(|e| TailFinError::Parse(format!("Invalid JSON response: {}", e)))
}
}
impl crate::TwitterApi for TwitterHttpClient {
async fn timeline(
&self,
kind: TimelineType,
count: usize,
cursor: Option<&str>,
) -> Result<TimelineResponse, TailFinError> {
self.timeline(kind, count, cursor).await
}
async fn search(
&self,
query: &str,
count: usize,
cursor: Option<&str>,
) -> Result<TimelineResponse, TailFinError> {
self.search(query, count, cursor).await
}
}
fn user_list_features() -> Value {
json!({
"rweb_video_screen_enabled": false,
"profile_label_improvements_pcf_label_in_post_enabled": true,
"rweb_tipjar_consumption_enabled": true,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"premium_content_api_read_enabled": false,
"communities_web_enable_tweet_community_results_fetch": true,
"c9s_tweet_anatomy_moderator_badge_enabled": true,
"responsive_web_grok_analyze_button_fetch_trends_enabled": false,
"responsive_web_grok_analyze_post_followups_enabled": true,
"responsive_web_grok_share_attachment_enabled": true,
"articles_preview_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"view_counts_everywhere_api_enabled": true,
"longform_notetweets_consumption_enabled": true,
"responsive_web_twitter_article_tweet_consumption_enabled": true,
"tweet_awards_web_tipping_enabled": false,
"responsive_web_grok_show_grok_translated_post": false,
"responsive_web_grok_analysis_button_from_backend": false,
"freedom_of_speech_not_reach_fetch_enabled": true,
"standardized_nudges_misinfo": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": true,
"responsive_web_grok_image_annotation_enabled": true,
"responsive_web_enhance_cards_enabled": false,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn cached_query_ids_returns_preset_value_twice() {
let client = TwitterHttpClient::from_tokens("fake_ct0".into(), "fake_auth".into());
let preset = json!({
"TimelineHome": { "queryId": "abc123" },
"Bookmarks": { "queryId": "def456" }
});
client
.query_id_cache
.set(Some(preset.clone()))
.expect("fresh cell should accept the seed");
let first = client.cached_query_ids().await;
let second = client.cached_query_ids().await;
assert_eq!(first.as_ref(), Some(&preset));
assert_eq!(second.as_ref(), Some(&preset));
assert!(
std::ptr::eq(first, second),
"OnceCell should return the same pointer on repeat calls"
);
}
#[tokio::test]
async fn resolve_query_id_hits_cache_then_fallback() {
let client = TwitterHttpClient::from_tokens("fake_ct0".into(), "fake_auth".into());
let preset = json!({
"TimelineHome": { "queryId": "cached-timeline-id" }
});
client.query_id_cache.set(Some(preset)).unwrap();
assert_eq!(
client.resolve_query_id("TimelineHome", "fallback-A").await,
"cached-timeline-id"
);
assert_eq!(
client.resolve_query_id("NotInCache", "fallback-B").await,
"fallback-B"
);
}
}