trendkit_core/
trending.rs

1//! GitHub trending-page source: the one source with no API.
2//!
3//! `github.com/trending` exposes no JSON endpoint, so the daily page is
4//! fetched and parsed as HTML (via `scraper`/html5ever — no regex
5//! scraping). Subjects use `SourceKind::GithubRepo` with `owner/repo`
6//! keys, so trending-page observations land in the *same subject space*
7//! as Search-API observations: two independent signals per repository.
8//! `extra1` carries "stars today" — a second velocity signal GitHub
9//! gives us for free.
10//!
11//! Markup drift is handled by optionality: every field parses to
12//! `Option`/0 and articles without a repo path are skipped rather than
13//! guessed at.
14
15use fetch_kit::Client;
16use scraper::{ElementRef, Html, Selector};
17
18use crate::clock::unix_seconds;
19use crate::error::{Result, TrendError};
20use crate::model::{Observation, SourceKind, SubjectRecord};
21use crate::source::{CollectCtx, CollectReport, TrendSource};
22
23const TRENDING_URL: &str = "https://github.com/trending?since=daily";
24const USER_AGENT: &str = "trendkit/0.2";
25
26/// One parsed row of the trending page.
27#[derive(Debug, Clone, PartialEq, Eq)]
28pub struct TrendingRow {
29    /// `owner/repo` path from the article link (leading slash stripped).
30    pub full_name: String,
31    /// Article description, if present.
32    pub description: Option<String>,
33    /// Primary language, if present.
34    pub language: Option<String>,
35    /// Total stars.
36    pub stars: i64,
37    /// Total forks.
38    pub forks: i64,
39    /// "N stars today" from the article footer.
40    pub stars_today: i64,
41}
42
43/// GitHub trending-page source.
44#[derive(Debug, Clone)]
45pub struct TrendingSource {
46    client: Client,
47}
48
49impl TrendingSource {
50    /// Build a source.
51    #[must_use]
52    pub fn new() -> TrendingSource {
53        TrendingSource {
54            client: Client::builder().user_agent(USER_AGENT).build(),
55        }
56    }
57
58    /// Fetch the trending page body.
59    ///
60    /// # Errors
61    /// Fails on fetch errors (status codes checked by fetch-kit).
62    pub async fn fetch_html(&self) -> Result<String> {
63        let response = self.client.get(TRENDING_URL).send().await?;
64        Ok(response.text().await.unwrap_or_default())
65    }
66}
67
68impl Default for TrendingSource {
69    fn default() -> Self {
70        TrendingSource::new()
71    }
72}
73
74impl TrendSource for TrendingSource {
75    fn name(&self) -> &'static str {
76        "github_trending"
77    }
78
79    fn collect(
80        &self,
81        ctx: &CollectCtx,
82    ) -> impl std::future::Future<Output = Result<CollectReport>> {
83        let ctx = ctx.clone();
84        async move {
85            let now = unix_seconds(ctx.clock.as_ref());
86            let html = self.fetch_html().await?;
87            let rows = parse_trending_html(&html)?;
88            let count = rows.len();
89            for row in rows {
90                let observation = Observation {
91                    ts: now,
92                    score: row.stars,
93                    // Same slot semantics as the Search source (forks),
94                    // so fork-ratio integrity works across origins.
95                    // Stars-today rides extra2; watchers are dropped.
96                    extra1: row.forks,
97                    extra2: row.stars_today,
98                };
99                let subject = SubjectRecord {
100                    kind: SourceKind::GithubRepo,
101                    key: row.full_name,
102                    title: row.description,
103                    lang: row.language,
104                    url: None, // resolved by the GitHub source's subjects
105                    topics: Vec::new(),
106                    created_at: None,
107                };
108                let subject_id = ctx.store.upsert_subject(&subject)?;
109                ctx.store.insert_observation(subject_id, &observation)?;
110            }
111            ctx.store.record_run(now, self.name(), 1, count as i64)?;
112            Ok(CollectReport {
113                source: self.name(),
114                items: count,
115                pages: 1,
116            })
117        }
118    }
119}
120
121fn selector(css: &str) -> Result<Selector> {
122    Selector::parse(css).map_err(|e| TrendError::Config(format!("bad selector `{css}`: {e}")))
123}
124
125/// "1,234" / "1,234 stars today" / "star today" → i64 (0 when absent).
126fn parse_count(text: &str, strip_suffix_words: bool) -> i64 {
127    let cleaned = if strip_suffix_words {
128        text.split_whitespace().next().unwrap_or("")
129    } else {
130        text.trim()
131    };
132    cleaned.replace(',', "").parse::<i64>().unwrap_or(0)
133}
134
135/// Trimmed concatenated text of an element.
136fn text_of(el: &ElementRef<'_>) -> String {
137    el.text().collect::<String>().trim().to_owned()
138}
139
140/// Parse the GitHub trending page HTML into rows.
141///
142/// # Errors
143/// Fails only on internal selector construction; rows with missing
144/// article data are skipped, and a page with no articles yields no rows.
145pub fn parse_trending_html(html: &str) -> Result<Vec<TrendingRow>> {
146    let document = Html::parse_document(html);
147    let articles = selector("article.Box-row")?;
148    let name_link = selector("h2 a")?;
149    let description = selector("p")?;
150    let language = selector("span[itemprop=\"programmingLanguage\"]")?;
151    let counters = selector("a.Link--muted")?;
152    let stars_today = selector("span.d-inline-block.float-sm-right")?;
153
154    let mut rows = Vec::new();
155    for article in document.select(&articles) {
156        let Some(link) = article.select(&name_link).next() else {
157            continue;
158        };
159        let Some(href) = link.value().attr("href") else {
160            continue;
161        };
162        let full_name = href.trim_start_matches('/').to_owned();
163        if full_name.is_empty() || !full_name.contains('/') {
164            continue;
165        }
166
167        let stars_and_forks: Vec<i64> = article
168            .select(&counters)
169            .map(|c| parse_count(&text_of(&c), false))
170            .collect();
171        // Markup order is stars then forks; missing counters read as 0.
172        let stars = stars_and_forks.first().copied().unwrap_or(0);
173        let forks = stars_and_forks.get(1).copied().unwrap_or(0);
174
175        rows.push(TrendingRow {
176            full_name,
177            description: article.select(&description).next().map(|p| text_of(&p)),
178            language: article.select(&language).next().map(|l| text_of(&l)),
179            stars,
180            forks,
181            stars_today: article
182                .select(&stars_today)
183                .next()
184                .map_or(0, |s| parse_count(&text_of(&s), true)),
185        });
186    }
187    Ok(rows)
188}
189
190#[cfg(test)]
191mod tests {
192    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
193
194    use super::*;
195
196    const SAMPLE: &str = r#"
197    <main>
198      <article class="Box-row">
199        <h2 class="h3 lh-condensed"><a href="/owner/first-repo">first-repo</a></h2>
200        <p class="col-9 color-fg-muted my-1 pr-4">A trending thing</p>
201        <span itemprop="programmingLanguage">Rust</span>
202        <a class="Link--muted" href="/owner/first-repo/stargazers">4,149</a>
203        <a class="Link--muted" href="/owner/first-repo/forks">321</a>
204        <span class="d-inline-block float-sm-right">1,234 stars today</span>
205      </article>
206      <article class="Box-row">
207        <h2 class="h3 lh-condensed"><a href="/owner/second-repo">second-repo</a></h2>
208        <a class="Link--muted" href="/owner/second-repo/stargazers">12</a>
209        <span class="d-inline-block float-sm-right">3 stars today</span>
210      </article>
211      <article class="Box-row">
212        <h2 class="h3 lh-condensed"></h2>
213      </article>
214    </main>"#;
215
216    #[test]
217    fn parses_rows_with_defaults_and_skips_incomplete() {
218        let rows = parse_trending_html(SAMPLE).unwrap();
219        assert_eq!(rows.len(), 2); // third article has no repo link
220
221        assert_eq!(rows[0].full_name, "owner/first-repo");
222        assert_eq!(rows[0].description.as_deref(), Some("A trending thing"));
223        assert_eq!(rows[0].language.as_deref(), Some("Rust"));
224        assert_eq!(rows[0].stars, 4_149);
225        assert_eq!(rows[0].forks, 321);
226        assert_eq!(rows[0].stars_today, 1_234);
227
228        assert_eq!(rows[1].full_name, "owner/second-repo");
229        assert_eq!(rows[1].description, None);
230        assert_eq!(rows[1].language, None);
231        assert_eq!(rows[1].stars, 12);
232        assert_eq!(rows[1].forks, 0);
233        assert_eq!(rows[1].stars_today, 3);
234    }
235
236    #[test]
237    fn empty_page_yields_no_rows() {
238        assert!(parse_trending_html("<html><body>sign in</body></html>")
239            .unwrap()
240            .is_empty());
241    }
242
243    #[test]
244    fn count_parser_handles_noise() {
245        assert_eq!(parse_count("4,149", false), 4_149);
246        assert_eq!(parse_count("1,234 stars today", true), 1_234);
247        assert_eq!(parse_count("1 star today", true), 1);
248        assert_eq!(parse_count("", false), 0);
249        assert_eq!(parse_count("n/a", false), 0);
250    }
251}
trendkit_core/trending.rs

trendkit_core/
trending.rs
