trendkit_core/
trending.rs

1//! GitHub trending-page source: the one source with no API.
2//!
3//! `github.com/trending` exposes no JSON endpoint, so the daily page is
4//! fetched and parsed as HTML (via `scraper`/html5ever — no regex
5//! scraping). Subjects use `SourceKind::GithubRepo` with `owner/repo`
6//! keys, so trending-page observations land in the *same subject space*
7//! as Search-API observations: two independent signals per repository.
8//! `extra1` carries "stars today" — a second velocity signal GitHub
9//! gives us for free.
10//!
11//! Markup drift is handled by optionality: every field parses to
12//! `Option`/0 and articles without a repo path are skipped rather than
13//! guessed at.
14
15use fetch_kit::Client;
16use scraper::{ElementRef, Html, Selector};
17
18use crate::clock::unix_seconds;
19use crate::error::{Result, TrendError};
20use crate::model::{Observation, SourceKind, SubjectRecord};
21use crate::source::{CollectCtx, CollectReport, TrendSource};
22
23const TRENDING_URL: &str = "https://github.com/trending?since=daily";
24const USER_AGENT: &str = "trendkit/0.2";
25
26/// One parsed row of the trending page.
27#[derive(Debug, Clone, PartialEq, Eq)]
28pub struct TrendingRow {
29    /// `owner/repo` path from the article link (leading slash stripped).
30    pub full_name: String,
31    /// Article description, if present.
32    pub description: Option<String>,
33    /// Primary language, if present.
34    pub language: Option<String>,
35    /// Total stars.
36    pub stars: i64,
37    /// Total forks.
38    pub forks: i64,
39    /// "N stars today" from the article footer.
40    pub stars_today: i64,
41}
42
43/// GitHub trending-page source.
44#[derive(Debug, Clone)]
45pub struct TrendingSource {
46    client: Client,
47}
48
49impl TrendingSource {
50    /// Build a source.
51    #[must_use]
52    pub fn new() -> TrendingSource {
53        TrendingSource {
54            client: Client::builder().user_agent(USER_AGENT).build(),
55        }
56    }
57
58    /// Fetch the trending page body.
59    ///
60    /// # Errors
61    /// Fails on fetch errors (status codes checked by fetch-kit).
62    pub async fn fetch_html(&self) -> Result<String> {
63        let response = self.client.get(TRENDING_URL).send().await?;
64        Ok(response.text().await.unwrap_or_default())
65    }
66}
67
68impl Default for TrendingSource {
69    fn default() -> Self {
70        TrendingSource::new()
71    }
72}
73
74impl TrendSource for TrendingSource {
75    fn name(&self) -> &'static str {
76        "github_trending"
77    }
78
79    fn collect(
80        &self,
81        ctx: &CollectCtx,
82    ) -> impl std::future::Future<Output = Result<CollectReport>> {
83        let ctx = ctx.clone();
84        async move {
85            let now = unix_seconds(ctx.clock.as_ref());
86            let html = self.fetch_html().await?;
87            let rows = parse_trending_html(&html)?;
88            let count = rows.len();
89            for row in rows {
90                let observation = Observation {
91                    ts: now,
92                    score: row.stars,
93                    extra1: row.stars_today,
94                    extra2: row.forks,
95                };
96                let subject = SubjectRecord {
97                    kind: SourceKind::GithubRepo,
98                    key: row.full_name,
99                    title: row.description,
100                    lang: row.language,
101                    url: None, // resolved by the GitHub source's subjects
102                    topics: Vec::new(),
103                    created_at: None,
104                };
105                let subject_id = ctx.store.upsert_subject(&subject)?;
106                ctx.store.insert_observation(subject_id, &observation)?;
107            }
108            ctx.store.record_run(now, self.name(), 1, count as i64)?;
109            Ok(CollectReport {
110                source: self.name(),
111                items: count,
112                pages: 1,
113            })
114        }
115    }
116}
117
118fn selector(css: &str) -> Result<Selector> {
119    Selector::parse(css).map_err(|e| TrendError::Config(format!("bad selector `{css}`: {e}")))
120}
121
122/// "1,234" / "1,234 stars today" / "star today" → i64 (0 when absent).
123fn parse_count(text: &str, strip_suffix_words: bool) -> i64 {
124    let cleaned = if strip_suffix_words {
125        text.split_whitespace().next().unwrap_or("")
126    } else {
127        text.trim()
128    };
129    cleaned.replace(',', "").parse::<i64>().unwrap_or(0)
130}
131
132/// Trimmed concatenated text of an element.
133fn text_of(el: &ElementRef<'_>) -> String {
134    el.text().collect::<String>().trim().to_owned()
135}
136
137/// Parse the GitHub trending page HTML into rows.
138///
139/// # Errors
140/// Fails only on internal selector construction; rows with missing
141/// article data are skipped, and a page with no articles yields no rows.
142pub fn parse_trending_html(html: &str) -> Result<Vec<TrendingRow>> {
143    let document = Html::parse_document(html);
144    let articles = selector("article.Box-row")?;
145    let name_link = selector("h2 a")?;
146    let description = selector("p")?;
147    let language = selector("span[itemprop=\"programmingLanguage\"]")?;
148    let counters = selector("a.Link--muted")?;
149    let stars_today = selector("span.d-inline-block.float-sm-right")?;
150
151    let mut rows = Vec::new();
152    for article in document.select(&articles) {
153        let Some(link) = article.select(&name_link).next() else {
154            continue;
155        };
156        let Some(href) = link.value().attr("href") else {
157            continue;
158        };
159        let full_name = href.trim_start_matches('/').to_owned();
160        if full_name.is_empty() || !full_name.contains('/') {
161            continue;
162        }
163
164        let stars_and_forks: Vec<i64> = article
165            .select(&counters)
166            .map(|c| parse_count(&text_of(&c), false))
167            .collect();
168        // Markup order is stars then forks; missing counters read as 0.
169        let stars = stars_and_forks.first().copied().unwrap_or(0);
170        let forks = stars_and_forks.get(1).copied().unwrap_or(0);
171
172        rows.push(TrendingRow {
173            full_name,
174            description: article.select(&description).next().map(|p| text_of(&p)),
175            language: article.select(&language).next().map(|l| text_of(&l)),
176            stars,
177            forks,
178            stars_today: article
179                .select(&stars_today)
180                .next()
181                .map_or(0, |s| parse_count(&text_of(&s), true)),
182        });
183    }
184    Ok(rows)
185}
186
187#[cfg(test)]
188mod tests {
189    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
190
191    use super::*;
192
193    const SAMPLE: &str = r#"
194    <main>
195      <article class="Box-row">
196        <h2 class="h3 lh-condensed"><a href="/owner/first-repo">first-repo</a></h2>
197        <p class="col-9 color-fg-muted my-1 pr-4">A trending thing</p>
198        <span itemprop="programmingLanguage">Rust</span>
199        <a class="Link--muted" href="/owner/first-repo/stargazers">4,149</a>
200        <a class="Link--muted" href="/owner/first-repo/forks">321</a>
201        <span class="d-inline-block float-sm-right">1,234 stars today</span>
202      </article>
203      <article class="Box-row">
204        <h2 class="h3 lh-condensed"><a href="/owner/second-repo">second-repo</a></h2>
205        <a class="Link--muted" href="/owner/second-repo/stargazers">12</a>
206        <span class="d-inline-block float-sm-right">3 stars today</span>
207      </article>
208      <article class="Box-row">
209        <h2 class="h3 lh-condensed"></h2>
210      </article>
211    </main>"#;
212
213    #[test]
214    fn parses_rows_with_defaults_and_skips_incomplete() {
215        let rows = parse_trending_html(SAMPLE).unwrap();
216        assert_eq!(rows.len(), 2); // third article has no repo link
217
218        assert_eq!(rows[0].full_name, "owner/first-repo");
219        assert_eq!(rows[0].description.as_deref(), Some("A trending thing"));
220        assert_eq!(rows[0].language.as_deref(), Some("Rust"));
221        assert_eq!(rows[0].stars, 4_149);
222        assert_eq!(rows[0].forks, 321);
223        assert_eq!(rows[0].stars_today, 1_234);
224
225        assert_eq!(rows[1].full_name, "owner/second-repo");
226        assert_eq!(rows[1].description, None);
227        assert_eq!(rows[1].language, None);
228        assert_eq!(rows[1].stars, 12);
229        assert_eq!(rows[1].forks, 0);
230        assert_eq!(rows[1].stars_today, 3);
231    }
232
233    #[test]
234    fn empty_page_yields_no_rows() {
235        assert!(parse_trending_html("<html><body>sign in</body></html>")
236            .unwrap()
237            .is_empty());
238    }
239
240    #[test]
241    fn count_parser_handles_noise() {
242        assert_eq!(parse_count("4,149", false), 4_149);
243        assert_eq!(parse_count("1,234 stars today", true), 1_234);
244        assert_eq!(parse_count("1 star today", true), 1);
245        assert_eq!(parse_count("", false), 0);
246        assert_eq!(parse_count("n/a", false), 0);
247    }
248}
trendkit_core/trending.rs

trendkit_core/
trending.rs
