douyin_cli/
comments.rs

1use std::path::PathBuf;
2use std::thread;
3use std::time::Duration;
4
5use clap::{Args, ValueEnum};
6use reqwest::blocking::Client;
7use serde_json::{Map, Value, json};
8
9use crate::err;
10use crate::net::{self, sign};
11use crate::{fs_utils, settings};
12
13const BASE_URL: &str = "https://www.douyin.com";
14const COMMENT_LIST: &str = "/aweme/v1/web/comment/list/";
15const COMMENT_REPLIES: &str = "/aweme/v1/web/comment/list/reply/";
16
17#[derive(Debug, Args)]
18pub struct CommentArgs {
19    /// 作品 ID、视频 URL 或图文 URL
20    target: String,
21    /// 最多抓取一级评论数，0 表示不限制
22    #[arg(short, long, default_value_t = 100)]
23    limit: usize,
24    /// 每页请求数量（抖音网页接口上限为 20）
25    #[arg(long, default_value_t = 20, value_parser = parse_comment_count)]
26    count: usize,
27    /// 同时抓取评论楼中楼回复
28    #[arg(long)]
29    with_replies: bool,
30    /// 每条评论最多抓取回复数，0 表示不限制
31    #[arg(long, default_value_t = 20)]
32    reply_limit: usize,
33    /// 分页请求间隔秒数
34    #[arg(long = "sleep", visible_alias = "sleep-seconds", default_value_t = 0.8, value_parser = parse_non_negative_f64)]
35    sleep_seconds: f64,
36    /// 输出文件；不传则输出到 stdout
37    #[arg(short, long)]
38    output: Option<PathBuf>,
39    /// 输出格式
40    #[arg(long = "format", value_enum, default_value_t = OutputFormat::Raw)]
41    output_format: OutputFormat,
42    #[arg(long, default_value = "user")]
43    comment_role: String,
44    #[arg(long, default_value = "assistant")]
45    reply_role: String,
46    #[arg(long, default_value_t = 0)]
47    min_comment_digg: i64,
48    #[arg(long, default_value_t = 0)]
49    min_reply_digg: i64,
50    #[arg(long)]
51    include_single_comments: bool,
52    /// 本次运行使用的 Cookie；默认读取保存的 Cookie
53    #[arg(short, long, env = "DOUYIN_COOKIE")]
54    cookie: Option<String>,
55}
56
57#[derive(Clone, Debug, ValueEnum)]
58enum OutputFormat {
59    Raw,
60    ChatmlJsonl,
61    ChatmlJson,
62}
63
64pub fn run(args: CommentArgs) -> Result<(), String> {
65    let saved = settings::load().map_err(err)?;
66    let (cookie_value, user_agent) = net::credentials(&saved, args.cookie.as_deref())?;
67    let aweme_id = extract_aweme_id(&args.target)?;
68    let crawler = CommentCrawler::new(&cookie_value, user_agent)?;
69    let data = crawler.crawl(&aweme_id, &args)?;
70    let output = match args.output_format {
71        OutputFormat::Raw => serde_json::to_string_pretty(&data).map_err(err)?,
72        OutputFormat::ChatmlJson => {
73            serde_json::to_string_pretty(&format_chatml(&data, &args)).map_err(err)?
74        }
75        OutputFormat::ChatmlJsonl => format_chatml(&data, &args)
76            .iter()
77            .map(serde_json::to_string)
78            .collect::<Result<Vec<_>, _>>()
79            .map_err(err)?
80            .join("\n"),
81    };
82    fs_utils::write_output(&output, args.output.as_deref())?;
83    if let Some(path) = args.output {
84        eprintln!("评论已保存: {}", path.display());
85    }
86    Ok(())
87}
88
89struct CommentCrawler {
90    client: Client,
91    user_agent: String,
92    common_params: Vec<(&'static str, String)>,
93}
94
95impl CommentCrawler {
96    fn new(cookie: &str, user_agent: &str) -> Result<Self, String> {
97        Ok(Self {
98            client: net::web_client(cookie, user_agent, 30)?,
99            user_agent: user_agent.to_owned(),
100            common_params: net::web_query_params(cookie),
101        })
102    }
103
104    fn crawl(&self, aweme_id: &str, args: &CommentArgs) -> Result<Value, String> {
105        let comments = self.crawl_pages(
106            COMMENT_LIST,
107            vec![("aweme_id", aweme_id.to_owned())],
108            args.limit,
109            args,
110            |raw| {
111                let mut comment = normalize_comment(raw);
112                if args.with_replies {
113                    let comment_id = comment.get("id").and_then(Value::as_str).unwrap_or("");
114                    comment["replies"] =
115                        Value::Array(self.crawl_replies(aweme_id, comment_id, args)?);
116                }
117                Ok(comment)
118            },
119        )?;
120        Ok(json!({"aweme_id": aweme_id, "comments": comments}))
121    }
122
123    fn crawl_replies(
124        &self,
125        aweme_id: &str,
126        comment_id: &str,
127        args: &CommentArgs,
128    ) -> Result<Vec<Value>, String> {
129        self.crawl_pages(
130            COMMENT_REPLIES,
131            vec![
132                ("item_id", aweme_id.to_owned()),
133                ("comment_id", comment_id.to_owned()),
134            ],
135            args.reply_limit,
136            args,
137            |raw| Ok(normalize_comment(raw)),
138        )
139    }
140
141    /// Pages through a comment endpoint, applying `normalize` to every raw comment
142    /// until the endpoint reports no more data or `limit` is reached.
143    fn crawl_pages(
144        &self,
145        path: &str,
146        base_params: Vec<(&'static str, String)>,
147        limit: usize,
148        args: &CommentArgs,
149        mut normalize: impl FnMut(&Value) -> Result<Value, String>,
150    ) -> Result<Vec<Value>, String> {
151        let mut items = Vec::new();
152        let mut cursor = 0_i64;
153        let mut has_more = true;
154        while has_more && !net::limit_reached(items.len(), limit) {
155            let mut params = base_params.clone();
156            params.extend([
157                ("cursor", cursor.to_string()),
158                ("count", args.count.to_string()),
159                ("item_type", "0".to_owned()),
160                ("insert_ids", String::new()),
161                ("whale_cut_token", String::new()),
162                ("cut_version", "1".to_owned()),
163                ("rcFT", String::new()),
164            ]);
165            let page = self.fetch_page(path, params)?;
166            let values = page
167                .get("comments")
168                .and_then(Value::as_array)
169                .cloned()
170                .unwrap_or_default();
171            if values.is_empty() {
172                break;
173            }
174            for raw in &values {
175                items.push(normalize(raw)?);
176                if net::limit_reached(items.len(), limit) {
177                    break;
178                }
179            }
180            let next_cursor = page
181                .get("cursor")
182                .and_then(net::value_i64)
183                .unwrap_or(cursor);
184            has_more = net::truthy(page.get("has_more"));
185            if has_more && next_cursor == cursor {
186                return Err("评论分页游标没有推进，已停止以避免重复请求".to_owned());
187            }
188            cursor = next_cursor;
189            pause(has_more, args.sleep_seconds);
190        }
191        Ok(items)
192    }
193
194    fn fetch_page(&self, path: &str, mut params: Vec<(&str, String)>) -> Result<Value, String> {
195        params.extend(self.common_params.clone());
196        let query = net::encode_query(&params);
197        let sign_function = if path.contains("reply") {
198            "sign_reply"
199        } else {
200            "sign_datail"
201        };
202        let signature = sign(sign_function, &query, &self.user_agent)?;
203        params.push(("a_bogus", signature));
204        let response = self
205            .client
206            .get(format!("{BASE_URL}{path}"))
207            .query(&params)
208            .send()
209            .map_err(err)?;
210        let status = response.status();
211        let text = response.text().map_err(err)?;
212        if !status.is_success() {
213            return Err(format!("评论请求失败: {status} {text}"));
214        }
215        if text.is_empty() {
216            return Err("响应体为空，Cookie 可能已失效".to_owned());
217        }
218        let data: Value =
219            serde_json::from_str(&text).map_err(|error| format!("评论响应不是 JSON: {error}"))?;
220        if net::contains_verify_check(&data) {
221            return Err("触发验证码，请完成验证后再继续".to_owned());
222        }
223        if data
224            .get("status_code")
225            .and_then(net::value_i64)
226            .unwrap_or(0)
227            != 0
228        {
229            return Err(format!("评论接口返回失败状态: {text}"));
230        }
231        Ok(data)
232    }
233}
234
235fn parse_comment_count(value: &str) -> Result<usize, String> {
236    let value = value
237        .parse::<usize>()
238        .map_err(|error| format!("无效页大小: {error}"))?;
239    if !(1..=20).contains(&value) {
240        return Err("每页请求数量必须在 1..=20 范围内".to_owned());
241    }
242    Ok(value)
243}
244
245fn parse_non_negative_f64(value: &str) -> Result<f64, String> {
246    let value = value
247        .parse::<f64>()
248        .map_err(|error| format!("无效秒数: {error}"))?;
249    if !value.is_finite() || value < 0.0 {
250        return Err("秒数必须是有限的非负数".to_owned());
251    }
252    Ok(value)
253}
254
255pub fn extract_aweme_id(target: &str) -> Result<String, String> {
256    let target = target.trim();
257    if target.chars().all(|value| value.is_ascii_digit()) && !target.is_empty() {
258        return Ok(target.to_owned());
259    }
260    let mut url = reqwest::Url::parse(target).map_err(|_| format!("无法识别作品 ID: {target}"))?;
261    if url.host_str() == Some("v.douyin.com") {
262        url = Client::builder()
263            .timeout(Duration::from_secs(15))
264            .build()
265            .map_err(err)?
266            .get(url)
267            .send()
268            .map_err(err)?
269            .url()
270            .clone();
271    }
272    let parts: Vec<_> = url
273        .path_segments()
274        .into_iter()
275        .flatten()
276        .filter(|value| !value.is_empty())
277        .collect();
278    for marker in ["video", "note"] {
279        if let Some(index) = parts.iter().position(|value| *value == marker)
280            && let Some(value) = parts
281                .get(index + 1)
282                .filter(|value| value.chars().all(|c| c.is_ascii_digit()))
283        {
284            return Ok((*value).to_owned());
285        }
286    }
287    parts
288        .last()
289        .filter(|value| value.chars().all(|c| c.is_ascii_digit()))
290        .map(|value| (*value).to_owned())
291        .ok_or_else(|| format!("无法识别作品 ID: {target}"))
292}
293
294pub fn normalize_comment(comment: &Value) -> Value {
295    let user = comment.get("user").and_then(Value::as_object);
296    json!({
297        "id": first_string(comment, &["cid", "comment_id"]),
298        "text": first_string(comment, &["text"]),
299        "create_time": comment.get("create_time").cloned().unwrap_or(Value::Null),
300        "digg_count": comment.get("digg_count").cloned().unwrap_or_else(|| json!(0)),
301        "reply_comment_total": comment.get("reply_comment_total").cloned().unwrap_or_else(|| json!(0)),
302        "ip_label": first_string(comment, &["ip_label"]),
303        "user": {
304            "uid": object_string(user, "uid"), "sec_uid": object_string(user, "sec_uid"),
305            "nickname": object_string(user, "nickname"), "unique_id": object_string(user, "unique_id")
306        }
307    })
308}
309
310fn format_chatml(data: &Value, args: &CommentArgs) -> Vec<Value> {
311    let aweme_id = data.get("aweme_id").and_then(Value::as_str).unwrap_or("");
312    let mut records = Vec::new();
313    for comment in data
314        .get("comments")
315        .and_then(Value::as_array)
316        .into_iter()
317        .flatten()
318    {
319        let text = comment
320            .get("text")
321            .and_then(Value::as_str)
322            .unwrap_or("")
323            .trim();
324        if text.is_empty() || digg(comment) < args.min_comment_digg {
325            continue;
326        }
327        let replies = comment
328            .get("replies")
329            .and_then(Value::as_array)
330            .cloned()
331            .unwrap_or_default();
332        if replies.is_empty() && args.include_single_comments {
333            records.push(json!({
334                "messages":[{"role":args.comment_role,"content":text}],
335                "metadata": metadata(aweme_id, comment, None)
336            }));
337        } else {
338            for reply in replies {
339                let reply_text = reply
340                    .get("text")
341                    .and_then(Value::as_str)
342                    .unwrap_or("")
343                    .trim();
344                if reply_text.is_empty() || digg(&reply) < args.min_reply_digg {
345                    continue;
346                }
347                records.push(json!({
348                    "messages":[{"role":args.comment_role,"content":text},{"role":args.reply_role,"content":reply_text}],
349                    "metadata": metadata(aweme_id, comment, Some(&reply))
350                }));
351            }
352        }
353    }
354    records
355}
356
357fn metadata(aweme_id: &str, comment: &Value, reply: Option<&Value>) -> Value {
358    let mut result = Map::from_iter([
359        (
360            "source".to_owned(),
361            json!(if reply.is_some() {
362                "douyin_comment_reply"
363            } else {
364                "douyin_comment"
365            }),
366        ),
367        ("aweme_id".to_owned(), json!(aweme_id)),
368        (
369            "comment_id".to_owned(),
370            json!(first_string(comment, &["id"])),
371        ),
372        ("comment_digg_count".to_owned(), json!(digg(comment))),
373        (
374            "comment_create_time".to_owned(),
375            comment.get("create_time").cloned().unwrap_or(Value::Null),
376        ),
377        (
378            "comment_user".to_owned(),
379            user_metadata(comment.get("user")),
380        ),
381        (
382            "quality_score".to_owned(),
383            json!(digg(comment) + reply.map_or(0, digg)),
384        ),
385    ]);
386    if let Some(reply) = reply {
387        result.extend([
388            ("reply_id".to_owned(), json!(first_string(reply, &["id"]))),
389            ("reply_digg_count".to_owned(), json!(digg(reply))),
390            (
391                "reply_create_time".to_owned(),
392                reply.get("create_time").cloned().unwrap_or(Value::Null),
393            ),
394            ("reply_user".to_owned(), user_metadata(reply.get("user"))),
395        ]);
396    }
397    Value::Object(result)
398}
399
400fn user_metadata(user: Option<&Value>) -> Value {
401    let object = user.and_then(Value::as_object);
402    json!({"uid":object_string(object,"uid"),"sec_uid":object_string(object,"sec_uid"),"nickname":object_string(object,"nickname"),"unique_id":object_string(object,"unique_id")})
403}
404
405fn first_string(value: &Value, keys: &[&str]) -> String {
406    keys.iter()
407        .find_map(|key| {
408            value
409                .get(key)
410                .and_then(Value::as_str)
411                .filter(|value| !value.is_empty())
412        })
413        .unwrap_or("")
414        .to_owned()
415}
416
417fn object_string(object: Option<&Map<String, Value>>, key: &str) -> String {
418    object
419        .and_then(|value| value.get(key))
420        .and_then(Value::as_str)
421        .unwrap_or("")
422        .to_owned()
423}
424
425fn digg(value: &Value) -> i64 {
426    value
427        .get("digg_count")
428        .and_then(|value| value.as_i64().or_else(|| value.as_str()?.parse().ok()))
429        .unwrap_or(0)
430}
431
432fn pause(has_more: bool, seconds: f64) {
433    if has_more && seconds > 0.0 {
434        thread::sleep(Duration::from_secs_f64(seconds));
435    }
436}
437
438#[cfg(test)]
439mod tests {
440    use super::{
441        CommentArgs, OutputFormat, extract_aweme_id, format_chatml, normalize_comment,
442        parse_comment_count, parse_non_negative_f64,
443    };
444    use crate::test_support::must;
445    use serde_json::json;
446
447    #[test]
448    fn validates_page_size_and_sleep_values() {
449        assert_eq!(must(parse_comment_count("20")), 20);
450        assert!(parse_comment_count("0").is_err());
451        assert!(parse_comment_count("21").is_err());
452        assert_eq!(must(parse_non_negative_f64("0.5")), 0.5);
453        assert!(parse_non_negative_f64("NaN").is_err());
454        assert!(parse_non_negative_f64("-1").is_err());
455    }
456
457    #[test]
458    fn extracts_raw_and_url_aweme_ids() {
459        assert_eq!(
460            must(extract_aweme_id("7380000000000000000")),
461            "7380000000000000000"
462        );
463        assert_eq!(
464            must(extract_aweme_id(
465                "https://www.douyin.com/video/7380000000000000000?x=1"
466            )),
467            "7380000000000000000"
468        );
469        assert_eq!(
470            must(extract_aweme_id(
471                "https://www.douyin.com/note/7380000000000000000"
472            )),
473            "7380000000000000000"
474        );
475    }
476
477    #[test]
478    fn normalizes_comment_fields() {
479        let value = normalize_comment(&json!({
480            "cid":"1","text":"你好","create_time":1_710_000_000,"digg_count":3,"reply_comment_total":2,"ip_label":"上海",
481            "user":{"uid":"u1","sec_uid":"sec","nickname":"用户","unique_id":"unique"}
482        }));
483        assert_eq!(value["id"], "1");
484        assert_eq!(value["user"]["nickname"], "用户");
485        assert_eq!(value["digg_count"], 3);
486    }
487
488    #[test]
489    fn chatml_pairs_comments_and_replies() {
490        let args = CommentArgs {
491            target: String::new(),
492            limit: 100,
493            count: 20,
494            with_replies: true,
495            reply_limit: 20,
496            sleep_seconds: 0.0,
497            output: None,
498            output_format: OutputFormat::ChatmlJsonl,
499            comment_role: "user".to_owned(),
500            reply_role: "assistant".to_owned(),
501            min_comment_digg: 0,
502            min_reply_digg: 0,
503            include_single_comments: false,
504            cookie: None,
505        };
506        let records = format_chatml(
507            &json!({
508                "aweme_id":"7380000000000000000",
509                "comments":[{"id":"c1","text":"这车能买吗？","digg_count":8,"user":{},"replies":[
510                    {"id":"r1","text":"先查维保和事故。","digg_count":12,"user":{}}
511                ]}]
512            }),
513            &args,
514        );
515        assert_eq!(records[0]["messages"][0]["role"], "user");
516        assert_eq!(records[0]["messages"][1]["content"], "先查维保和事故。");
517        assert_eq!(records[0]["metadata"]["quality_score"], 20);
518    }
519}
douyin_cli/comments.rs

douyin_cli/
comments.rs
