trendkit_core/
hn.rs

1//! Hacker News source via the Algolia HN Search API (no auth).
2//!
3//! `search` (relevance-ranked) restricted to stories created inside the
4//! window with at least one point approximates "trending new stories";
5//! velocity comes from repeated snapshots, same as every other source.
6
7use std::time::Duration;
8
9use fetch_kit::Client;
10use serde::{Deserialize, Serialize};
11
12use crate::clock::unix_seconds;
13use crate::error::Result;
14use crate::model::{Observation, SourceKind, SubjectRecord};
15use crate::source::{CollectCtx, CollectReport, TrendSource};
16
17const HN_API: &str = "https://hn.algolia.com/api/v1";
18const USER_AGENT: &str = "trendkit/0.2";
19const HITS_PER_PAGE: u32 = 100;
20/// Politeness pacing between paged requests (API is unauthenticated).
21pub const HN_PACING: Duration = Duration::from_millis(1_100);
22
23/// Hacker News trend source.
24#[derive(Debug, Clone)]
25pub struct HnSource {
26    client: Client,
27}
28
29impl HnSource {
30    /// Build a source. No auth: the Algolia HN API is open.
31    #[must_use]
32    pub fn new() -> HnSource {
33        HnSource {
34            client: Client::builder().user_agent(USER_AGENT).build(),
35        }
36    }
37
38    async fn collect_front_page(&self, ctx: &CollectCtx, now: i64) -> Result<usize> {
39        let params = HnQuery {
40            tags: "front_page",
41            numeric_filters: String::new(),
42            hits_per_page: 30,
43            page: 0,
44        };
45        self.collect_page(&params, ctx, now).await
46    }
47
48    async fn collect_trending(&self, ctx: &CollectCtx, now: i64) -> Result<usize> {
49        let since = unix_seconds(ctx.clock.as_ref()) - i64::from(ctx.window_days) * 86_400;
50        let mut total = 0usize;
51        for page in 0..ctx.pages {
52            let params = HnQuery {
53                tags: "story",
54                numeric_filters: format!("created_at_i>{since},points>0"),
55                hits_per_page: HITS_PER_PAGE,
56                page,
57            };
58            total += self.collect_page(&params, ctx, now).await?;
59            if total < HITS_PER_PAGE as usize * (page as usize + 1) {
60                break; // short page: last one
61            }
62            tokio::time::sleep(HN_PACING).await;
63        }
64        Ok(total)
65    }
66
67    async fn collect_page(
68        &self,
69        params: &HnQuery<'_>,
70        ctx: &CollectCtx,
71        now: i64,
72    ) -> Result<usize> {
73        let url = format!("{HN_API}/search");
74        let response: HnResponse = self.client.get(url).query(params).json_response().await?;
75        let mut count = 0usize;
76        for hit in response.hits {
77            let observation = Observation {
78                ts: now,
79                score: hit.points,
80                extra1: hit.num_comments.unwrap_or(0),
81                extra2: 0,
82            };
83            let subject = SubjectRecord::from(hit);
84            let subject_id = ctx.store.upsert_subject(&subject)?;
85            ctx.store.insert_observation(subject_id, &observation)?;
86            count += 1;
87        }
88        Ok(count)
89    }
90}
91
92impl Default for HnSource {
93    fn default() -> Self {
94        HnSource::new()
95    }
96}
97
98impl TrendSource for HnSource {
99    fn name(&self) -> &'static str {
100        "hn"
101    }
102
103    fn collect(
104        &self,
105        ctx: &CollectCtx,
106    ) -> impl std::future::Future<Output = Result<CollectReport>> {
107        let ctx = ctx.clone();
108        async move {
109            let now = unix_seconds(ctx.clock.as_ref());
110            let mut items = self.collect_trending(&ctx, now).await?;
111            items += self.collect_front_page(&ctx, now).await?;
112            ctx.store
113                .record_run(now, self.name(), i64::from(ctx.pages), items as i64)?;
114            Ok(CollectReport {
115                source: self.name(),
116                items,
117                pages: ctx.pages as usize,
118            })
119        }
120    }
121}
122
123#[derive(Debug, Serialize)]
124struct HnQuery<'a> {
125    tags: &'a str,
126    #[serde(rename = "numericFilters")]
127    numeric_filters: String,
128    #[serde(rename = "hitsPerPage")]
129    hits_per_page: u32,
130    page: u32,
131}
132
133#[derive(Debug, Deserialize)]
134struct HnResponse {
135    #[serde(default)]
136    hits: Vec<HnHit>,
137}
138
139/// The subset of an Algolia HN hit trendkit consumes.
140#[derive(Debug, Clone, Deserialize)]
141pub(crate) struct HnHit {
142    #[serde(rename = "objectID", default)]
143    pub(crate) object_id: String,
144    #[serde(default)]
145    pub(crate) title: Option<String>,
146    #[serde(default)]
147    pub(crate) url: Option<String>,
148    #[serde(default)]
149    pub(crate) points: i64,
150    #[serde(rename = "num_comments", default)]
151    pub(crate) num_comments: Option<i64>,
152}
153
154impl From<HnHit> for SubjectRecord {
155    fn from(hit: HnHit) -> Self {
156        let item_url = format!("https://news.ycombinator.com/item?id={}", hit.object_id);
157        SubjectRecord {
158            kind: SourceKind::HnStory,
159            key: hit.object_id,
160            title: hit.title,
161            lang: None,
162            url: Some(hit.url.unwrap_or(item_url)),
163            topics: Vec::new(),
164            created_at: None,
165        }
166    }
167}
168
169/// Parse an Algolia HN search response body into subjects.
170///
171/// # Errors
172/// Fails when the payload is not an Algolia response.
173pub fn parse_hits_json(payload: &str) -> Result<Vec<SubjectRecord>> {
174    let response: HnResponse = serde_json::from_str(payload)
175        .map_err(|e| crate::error::TrendError::Config(format!("invalid hn payload: {e}")))?;
176    Ok(response.hits.into_iter().map(SubjectRecord::from).collect())
177}
178
179#[cfg(test)]
180mod tests {
181    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
182
183    use super::*;
184
185    const SAMPLE: &str = r#"{
186        "hits": [
187            {
188                "objectID": "45000123",
189                "title": "Show HN: A Rust crawler",
190                "url": "https://github.com/example/crawler",
191                "points": 156,
192                "num_comments": 84,
193                "author": "someone",
194                "created_at_i": 1792200000
195            },
196            {
197                "objectID": "45000999",
198                "title": "Ask HN: advice?",
199                "points": 3
200            }
201        ],
202        "page": 0
203    }"#;
204
205    #[test]
206    fn parse_extracts_stories_with_fallback_url() {
207        let subjects = parse_hits_json(SAMPLE).unwrap();
208        assert_eq!(subjects.len(), 2);
209        assert_eq!(subjects[0].kind, SourceKind::HnStory);
210        assert_eq!(subjects[0].key, "45000123");
211        assert_eq!(
212            subjects[0].url.as_deref(),
213            Some("https://github.com/example/crawler")
214        );
215        // No external url -> HN item url.
216        assert_eq!(
217            subjects[1].url.as_deref(),
218            Some("https://news.ycombinator.com/item?id=45000999")
219        );
220    }
221
222    #[test]
223    fn parse_rejects_garbage() {
224        assert!(parse_hits_json("nope").is_err());
225    }
226}
trendkit_core/hn.rs

trendkit_core/
hn.rs
