git_koi/
collect.rs

1//! Account snapshots collected from GitHub through the GitHub CLI, separate from
2//! interpretation. Only public data is read, so a workflow's built-in token is enough.
3
4use std::collections::BTreeMap;
5use std::error::Error;
6use std::fmt;
7use std::io::ErrorKind;
8use std::process::Command;
9
10use chrono::{DateTime, Days, Months, NaiveDate};
11use serde_json::Value;
12
13use crate::history::{
14    AccountSnapshot, ConflictingRepository, ContributionDay, ContributionHistory,
15    ContributionHistoryError, LanguageColor, RepositoryObservation, RepositoryVisibility,
16    calculate_language_shares, summarize_contributions,
17};
18
19/// Runs one GitHub GraphQL query with string variables and returns its `data` object.
20pub trait GraphqlClient {
21    /// Run `query` with the given string variables and return the response's `data`
22    /// object. Errors carry the reason `gh` or GitHub gave.
23    fn run_query(&self, query: &str, variables: &[(&str, &str)]) -> Result<Value, CollectionError>;
24}
25
26/// Runs queries through `gh api graphql` with whatever authentication `gh` has,
27/// including `GH_TOKEN` in GitHub Actions.
28pub struct GhCli;
29
30impl GraphqlClient for GhCli {
31    fn run_query(&self, query: &str, variables: &[(&str, &str)]) -> Result<Value, CollectionError> {
32        let mut gh_command = Command::new("gh");
33        gh_command
34            .args(["api", "graphql", "-f"])
35            .arg(format!("query={query}"));
36        for (variable_name, variable_value) in variables {
37            // -f sends the value as a string; -F would turn a numeric login into a number.
38            gh_command
39                .arg("-f")
40                .arg(format!("{variable_name}={variable_value}"));
41        }
42        let gh_output = gh_command.output().map_err(|spawn_error| {
43            if spawn_error.kind() == ErrorKind::NotFound {
44                CollectionError::GhNotInstalled
45            } else {
46                CollectionError::GhFailed(spawn_error.to_string())
47            }
48        })?;
49        if !gh_output.status.success() {
50            return Err(CollectionError::GhFailed(
51                String::from_utf8_lossy(&gh_output.stderr).trim().to_owned(),
52            ));
53        }
54        let response: Value = serde_json::from_slice(&gh_output.stdout).map_err(|parse_error| {
55            CollectionError::UnexpectedResponse(format!("gh output is not JSON: {parse_error}"))
56        })?;
57        Ok(read_field(&response, "data")?.clone())
58    }
59}
60
61const ACCOUNT_QUERY: &str =
62    "query($login: String!) { user(login: $login) { databaseId createdAt } }";
63const REPOSITORY_FIELDS: &str = "nodes { databaseId visibility isFork primaryLanguage { name color } } pageInfo { hasNextPage endCursor }";
64const CALENDAR_QUERY: &str = "query($login: String!, $from: DateTime!, $to: DateTime!) { \
65    user(login: $login) { contributionsCollection(from: $from, to: $to) { \
66    contributionCalendar { weeks { contributionDays { date contributionCount } } } } } }";
67
68/// Collect one account's public snapshot: its public owned repositories, the public
69/// repositories GitHub counts it as contributing to, GitHub's color for each of their
70/// languages, and daily contribution counts from the day it was created through the
71/// latest date GitHub reports.
72///
73/// GitHub limits each calendar request to a year, so the calendar is requested a year at
74/// a time. `today` is the current UTC date and its day is still in progress, so the
75/// requests end the day before and a run just after midnight observes the day that just
76/// ended. Within that bound the observation date comes from GitHub. An account created
77/// today has no complete day yet and gets an empty history observed through yesterday.
78/// The snapshot is checked against the history rules before it is returned, so a gap,
79/// a conflict, or a calendar that starts after the account was created fails instead of
80/// producing a partial snapshot.
81pub fn collect_account_snapshot(
82    github: &impl GraphqlClient,
83    login: &str,
84    today: NaiveDate,
85) -> Result<AccountSnapshot, CollectionError> {
86    let account_data = github.run_query(ACCOUNT_QUERY, &[("login", login)])?;
87    let user = account_data
88        .get("user")
89        .filter(|user| !user.is_null())
90        .ok_or_else(|| CollectionError::AccountNotFound(login.to_owned()))?;
91    let account_id = read_u64(user, "databaseId")?;
92    let created_on = DateTime::parse_from_rfc3339(read_str(user, "createdAt")?)
93        .map_err(|parse_error| {
94            CollectionError::UnexpectedResponse(format!(
95                "createdAt is not a timestamp: {parse_error}"
96            ))
97        })?
98        .date_naive();
99    let mut language_colors = BTreeMap::new();
100    let mut repositories = collect_repositories(
101        github,
102        login,
103        &mut language_colors,
104        "repositories",
105        "repositories(first: 100, after: $cursor, ownerAffiliations: [OWNER], privacy: PUBLIC)",
106    )?;
107    repositories.extend(collect_repositories(
108        github,
109        login,
110        &mut language_colors,
111        "repositoriesContributedTo",
112        "repositoriesContributedTo(first: 100, after: $cursor, privacy: PUBLIC, \
113         includeUserRepositories: false, \
114         contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, PULL_REQUEST_REVIEW, REPOSITORY])",
115    )?);
116    let account_snapshot = AccountSnapshot {
117        account_id,
118        repositories,
119        contribution_history: Some(collect_contribution_history(
120            github, login, created_on, today,
121        )?),
122        language_colors: Some(language_colors),
123    };
124    summarize_contributions(&account_snapshot).map_err(CollectionError::InvalidHistory)?;
125    calculate_language_shares(&account_snapshot).map_err(CollectionError::ConflictingRepository)?;
126    Ok(account_snapshot)
127}
128
129/// Follow every page of one repository connection on the user, recording each primary
130/// language's color as it goes.
131fn collect_repositories(
132    github: &impl GraphqlClient,
133    login: &str,
134    language_colors: &mut BTreeMap<String, Option<LanguageColor>>,
135    connection_name: &str,
136    connection_call: &str,
137) -> Result<Vec<RepositoryObservation>, CollectionError> {
138    let query = format!(
139        "query($login: String!, $cursor: String) {{ user(login: $login) {{ {connection_call} {{ {REPOSITORY_FIELDS} }} }} }}"
140    );
141    let mut repositories = Vec::new();
142    let mut page_cursor: Option<String> = None;
143    loop {
144        let mut variables = vec![("login", login)];
145        if let Some(page_cursor) = &page_cursor {
146            variables.push(("cursor", page_cursor.as_str()));
147        }
148        let page_data = github.run_query(&query, &variables)?;
149        let connection = read_field(read_field(&page_data, "user")?, connection_name)?;
150        for repository_node in read_array(connection, "nodes")? {
151            repositories.push(read_repository(repository_node, language_colors)?);
152        }
153        let page_info = read_field(connection, "pageInfo")?;
154        if !read_bool(page_info, "hasNextPage")? {
155            return Ok(repositories);
156        }
157        let next_cursor = read_str(page_info, "endCursor")?.to_owned();
158        if page_cursor.as_deref() == Some(next_cursor.as_str()) {
159            return Err(CollectionError::UnexpectedResponse(format!(
160                "{connection_name} repeated page cursor {next_cursor}"
161            )));
162        }
163        page_cursor = Some(next_cursor);
164    }
165}
166
167fn read_repository(
168    repository_node: &Value,
169    language_colors: &mut BTreeMap<String, Option<LanguageColor>>,
170) -> Result<RepositoryObservation, CollectionError> {
171    let visibility = match read_str(repository_node, "visibility")? {
172        "PUBLIC" => RepositoryVisibility::Public,
173        "PRIVATE" => RepositoryVisibility::Private,
174        "INTERNAL" => RepositoryVisibility::Internal,
175        other => {
176            return Err(CollectionError::UnexpectedResponse(format!(
177                "unknown repository visibility {other}"
178            )));
179        }
180    };
181    let primary_language = match read_field(repository_node, "primaryLanguage")? {
182        Value::Null => None,
183        language => {
184            let language_name = read_str(language, "name")?.to_owned();
185            let language_color = match read_field(language, "color")? {
186                Value::Null => None,
187                color => Some(
188                    LanguageColor::try_from(color.as_str().ok_or_else(|| {
189                        CollectionError::UnexpectedResponse(format!(
190                            "{language_name} color is not text"
191                        ))
192                    })?)
193                    .map_err(CollectionError::UnexpectedResponse)?,
194                ),
195            };
196            if let Some(recorded_color) =
197                language_colors.insert(language_name.clone(), language_color)
198                && recorded_color != language_color
199            {
200                return Err(CollectionError::ConflictingLanguageColor(language_name));
201            }
202            Some(language_name)
203        }
204    };
205    Ok(RepositoryObservation {
206        repository_id: read_u64(repository_node, "databaseId")?,
207        visibility,
208        is_fork: read_bool(repository_node, "isFork")?,
209        primary_language,
210    })
211}
212
213/// Request the calendar a year at a time through the last complete day, keeping only days
214/// inside each requested window and on or after the account's creation date.
215fn collect_contribution_history(
216    github: &impl GraphqlClient,
217    login: &str,
218    created_on: NaiveDate,
219    today: NaiveDate,
220) -> Result<ContributionHistory, CollectionError> {
221    let last_complete_day = today
222        .pred_opt()
223        .expect("calendar windows stay within supported dates");
224    if created_on > last_complete_day {
225        return Ok(ContributionHistory {
226            observed_on: last_complete_day,
227            days: Vec::new(),
228        });
229    }
230    let mut daily_counts = BTreeMap::new();
231    let mut window_start = created_on;
232    while window_start <= last_complete_day {
233        let window_end = window_start
234            .checked_add_months(Months::new(12))
235            .and_then(|next_year| next_year.pred_opt())
236            .expect("calendar windows stay within supported dates")
237            .min(last_complete_day);
238        let from = format!("{window_start}T00:00:00Z");
239        let to = format!("{window_end}T23:59:59Z");
240        let calendar_data = github.run_query(
241            CALENDAR_QUERY,
242            &[("login", login), ("from", &from), ("to", &to)],
243        )?;
244        let calendar = read_field(
245            read_field(
246                read_field(&calendar_data, "user")?,
247                "contributionsCollection",
248            )?,
249            "contributionCalendar",
250        )?;
251        for calendar_week in read_array(calendar, "weeks")? {
252            for calendar_day in read_array(calendar_week, "contributionDays")? {
253                let contribution_date =
254                    NaiveDate::parse_from_str(read_str(calendar_day, "date")?, "%Y-%m-%d")
255                        .map_err(|parse_error| {
256                            CollectionError::UnexpectedResponse(format!(
257                                "calendar date is invalid: {parse_error}"
258                            ))
259                        })?;
260                if contribution_date < window_start || contribution_date > window_end {
261                    continue;
262                }
263                let contribution_count =
264                    u32::try_from(read_u64(calendar_day, "contributionCount")?).map_err(|_| {
265                        CollectionError::UnexpectedResponse(format!(
266                            "contribution count for {contribution_date} is out of range"
267                        ))
268                    })?;
269                if let Some(previous_count) =
270                    daily_counts.insert(contribution_date, contribution_count)
271                    && previous_count != contribution_count
272                {
273                    return Err(CollectionError::InvalidHistory(
274                        ContributionHistoryError::ConflictingDate(contribution_date),
275                    ));
276                }
277            }
278        }
279        window_start = window_end
280            .checked_add_days(Days::new(1))
281            .expect("calendar windows stay within supported dates");
282    }
283    let first_reported_on = daily_counts.keys().next().copied();
284    let observed_on = match (first_reported_on, daily_counts.keys().next_back()) {
285        (Some(first_date), Some(&last_date)) if first_date == created_on => last_date,
286        _ => {
287            return Err(CollectionError::IncompleteCalendar {
288                created_on,
289                first_reported_on,
290            });
291        }
292    };
293    Ok(ContributionHistory {
294        observed_on,
295        days: daily_counts
296            .into_iter()
297            .map(|(date, contribution_count)| ContributionDay {
298                date,
299                contribution_count,
300            })
301            .collect(),
302    })
303}
304
305fn read_field<'a>(object: &'a Value, field_name: &str) -> Result<&'a Value, CollectionError> {
306    object.get(field_name).ok_or_else(|| {
307        CollectionError::UnexpectedResponse(format!("response is missing {field_name}"))
308    })
309}
310
311fn read_array<'a>(object: &'a Value, field_name: &str) -> Result<&'a Vec<Value>, CollectionError> {
312    read_field(object, field_name)?
313        .as_array()
314        .ok_or_else(|| CollectionError::UnexpectedResponse(format!("{field_name} is not a list")))
315}
316
317fn read_str<'a>(object: &'a Value, field_name: &str) -> Result<&'a str, CollectionError> {
318    read_field(object, field_name)?
319        .as_str()
320        .ok_or_else(|| CollectionError::UnexpectedResponse(format!("{field_name} is not text")))
321}
322
323fn read_u64(object: &Value, field_name: &str) -> Result<u64, CollectionError> {
324    read_field(object, field_name)?.as_u64().ok_or_else(|| {
325        CollectionError::UnexpectedResponse(format!("{field_name} is not a whole number"))
326    })
327}
328
329fn read_bool(object: &Value, field_name: &str) -> Result<bool, CollectionError> {
330    read_field(object, field_name)?.as_bool().ok_or_else(|| {
331        CollectionError::UnexpectedResponse(format!("{field_name} is not true or false"))
332    })
333}
334
335/// Why a collection produced no snapshot. Each message names the operator repair.
336#[derive(Debug, Eq, PartialEq)]
337pub enum CollectionError {
338    /// `gh` is not on `PATH`.
339    GhNotInstalled,
340    /// `gh` ran and failed; carries its own message (authentication, rate limits, errors).
341    GhFailed(String),
342    /// GitHub has no account with this login.
343    AccountNotFound(String),
344    /// GitHub answered in a shape this version does not understand; carries the detail.
345    UnexpectedResponse(String),
346    /// GitHub's contribution calendar does not start on the account's creation date.
347    IncompleteCalendar {
348        /// The day GitHub says the account was created.
349        created_on: NaiveDate,
350        /// The first day the calendar reported, or `None` when it reported none.
351        first_reported_on: Option<NaiveDate>,
352    },
353    /// The collected days break the history rules; carries the rule.
354    InvalidHistory(ContributionHistoryError),
355    /// Two observations of one repository disagree.
356    ConflictingRepository(ConflictingRepository),
357    /// GitHub reported two colors for this language.
358    ConflictingLanguageColor(String),
359}
360
361impl fmt::Display for CollectionError {
362    fn fmt(&self, output_formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
363        match self {
364            Self::GhNotInstalled => write!(
365                output_formatter,
366                "gh is not installed or not on PATH; install the GitHub CLI from https://cli.github.com"
367            ),
368            Self::GhFailed(gh_message) => write!(
369                output_formatter,
370                "gh api failed: {gh_message}; check `gh auth status`, or set GH_TOKEN in CI"
371            ),
372            Self::AccountNotFound(login) => {
373                write!(output_formatter, "GitHub has no account named {login}")
374            }
375            Self::UnexpectedResponse(detail) => write!(
376                output_formatter,
377                "unexpected GitHub response: {detail}; try again or report the response"
378            ),
379            Self::IncompleteCalendar {
380                created_on,
381                first_reported_on: Some(first_reported_on),
382            } => write!(
383                output_formatter,
384                "GitHub's contribution calendar starts on {first_reported_on}, but the account was created on {created_on}; try again"
385            ),
386            Self::IncompleteCalendar {
387                created_on,
388                first_reported_on: None,
389            } => write!(
390                output_formatter,
391                "GitHub reported no contribution days since the account was created on {created_on}; try again"
392            ),
393            Self::InvalidHistory(history_error) => {
394                write!(
395                    output_formatter,
396                    "collected history is invalid: {history_error}"
397                )
398            }
399            Self::ConflictingRepository(repository_error) => {
400                write!(output_formatter, "{repository_error}")
401            }
402            Self::ConflictingLanguageColor(language_name) => write!(
403                output_formatter,
404                "GitHub reported two colors for {language_name}; try again"
405            ),
406        }
407    }
408}
409
410impl Error for CollectionError {
411    fn source(&self) -> Option<&(dyn Error + 'static)> {
412        match self {
413            Self::InvalidHistory(history_error) => Some(history_error),
414            Self::ConflictingRepository(repository_error) => Some(repository_error),
415            _ => None,
416        }
417    }
418}
git_koi/collect.rs

git_koi/
collect.rs
