git_koi/
collect.rs

1//! Account snapshots collected from GitHub through the GitHub CLI, separate from
2//! interpretation. Only public data is read, so a workflow's built-in token is enough.
3
4use std::collections::BTreeMap;
5use std::error::Error;
6use std::fmt;
7use std::io::ErrorKind;
8use std::process::Command;
9
10use chrono::{DateTime, Days, Months, NaiveDate};
11use serde_json::Value;
12
13use crate::history::{
14    AccountSnapshot, ConflictingRepository, ContributionDay, ContributionHistory,
15    ContributionHistoryError, LanguageColor, RepositoryObservation, RepositoryVisibility,
16    calculate_language_shares, summarize_contributions,
17};
18
19/// Runs one GitHub GraphQL query with string variables and returns its `data` object.
20pub trait GraphqlClient {
21    /// Run `query` with the given string variables and return the response's `data`
22    /// object. Errors carry the reason `gh` or GitHub gave.
23    fn run_query(&self, query: &str, variables: &[(&str, &str)]) -> Result<Value, CollectionError>;
24}
25
26/// Runs queries through `gh api graphql` with whatever authentication `gh` has,
27/// including `GH_TOKEN` in GitHub Actions.
28pub struct GhCli;
29
30impl GraphqlClient for GhCli {
31    fn run_query(&self, query: &str, variables: &[(&str, &str)]) -> Result<Value, CollectionError> {
32        let mut gh_command = Command::new("gh");
33        gh_command
34            .args(["api", "graphql", "-f"])
35            .arg(format!("query={query}"));
36        for (variable_name, variable_value) in variables {
37            // -f sends the value as a string; -F would turn a numeric login into a number.
38            gh_command
39                .arg("-f")
40                .arg(format!("{variable_name}={variable_value}"));
41        }
42        let gh_output = gh_command.output().map_err(|spawn_error| {
43            if spawn_error.kind() == ErrorKind::NotFound {
44                CollectionError::GhNotInstalled
45            } else {
46                CollectionError::GhFailed(spawn_error.to_string())
47            }
48        })?;
49        if !gh_output.status.success() {
50            return Err(CollectionError::GhFailed(
51                String::from_utf8_lossy(&gh_output.stderr).trim().to_owned(),
52            ));
53        }
54        let response: Value = serde_json::from_slice(&gh_output.stdout).map_err(|parse_error| {
55            CollectionError::UnexpectedResponse(format!("gh output is not JSON: {parse_error}"))
56        })?;
57        Ok(read_field(&response, "data")?.clone())
58    }
59}
60
61const ACCOUNT_QUERY: &str =
62    "query($login: String!) { user(login: $login) { databaseId createdAt } }";
63const REPOSITORY_FIELDS: &str = "nodes { databaseId visibility isFork primaryLanguage { name color } } pageInfo { hasNextPage endCursor }";
64const CALENDAR_QUERY: &str = "query($login: String!, $from: DateTime!, $to: DateTime!) { \
65    user(login: $login) { contributionsCollection(from: $from, to: $to) { \
66    contributionCalendar { weeks { contributionDays { date contributionCount } } } } } }";
67
68/// Collect one account's public snapshot: its public owned repositories, the public
69/// repositories GitHub counts it as contributing to, GitHub's color for each of their
70/// languages, and daily contribution counts from the day it was created through the
71/// latest date GitHub reports.
72///
73/// GitHub limits each calendar request to a year, so the calendar is requested a year at
74/// a time, and any days GitHub leaves out of an answer are requested again.
75/// `today` is the current UTC date and its day is still in progress, so the
76/// requests end the day before and a run just after midnight observes the day that just
77/// ended. Within that bound the observation date comes from GitHub. An account created
78/// today has no complete day yet and gets an empty history observed through yesterday.
79/// The snapshot is checked against the history rules before it is returned, so a gap,
80/// a conflict, or a calendar that starts after the account was created fails instead of
81/// producing a partial snapshot.
82pub fn collect_account_snapshot(
83    github: &impl GraphqlClient,
84    login: &str,
85    today: NaiveDate,
86) -> Result<AccountSnapshot, CollectionError> {
87    let account_data = github.run_query(ACCOUNT_QUERY, &[("login", login)])?;
88    let user = account_data
89        .get("user")
90        .filter(|user| !user.is_null())
91        .ok_or_else(|| CollectionError::AccountNotFound(login.to_owned()))?;
92    let account_id = read_u64(user, "databaseId")?;
93    let created_on = DateTime::parse_from_rfc3339(read_str(user, "createdAt")?)
94        .map_err(|parse_error| {
95            CollectionError::UnexpectedResponse(format!(
96                "createdAt is not a timestamp: {parse_error}"
97            ))
98        })?
99        .date_naive();
100    let mut language_colors = BTreeMap::new();
101    let mut repositories = collect_repositories(
102        github,
103        login,
104        &mut language_colors,
105        "repositories",
106        "repositories(first: 100, after: $cursor, ownerAffiliations: [OWNER], privacy: PUBLIC)",
107    )?;
108    repositories.extend(collect_repositories(
109        github,
110        login,
111        &mut language_colors,
112        "repositoriesContributedTo",
113        "repositoriesContributedTo(first: 100, after: $cursor, privacy: PUBLIC, \
114         includeUserRepositories: false, \
115         contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, PULL_REQUEST_REVIEW, REPOSITORY])",
116    )?);
117    let account_snapshot = AccountSnapshot {
118        account_id,
119        repositories,
120        contribution_history: Some(collect_contribution_history(
121            github, login, created_on, today,
122        )?),
123        language_colors: Some(language_colors),
124    };
125    summarize_contributions(&account_snapshot).map_err(CollectionError::InvalidHistory)?;
126    calculate_language_shares(&account_snapshot).map_err(CollectionError::ConflictingRepository)?;
127    Ok(account_snapshot)
128}
129
130/// Follow every page of one repository connection on the user, recording each primary
131/// language's color as it goes.
132fn collect_repositories(
133    github: &impl GraphqlClient,
134    login: &str,
135    language_colors: &mut BTreeMap<String, Option<LanguageColor>>,
136    connection_name: &str,
137    connection_call: &str,
138) -> Result<Vec<RepositoryObservation>, CollectionError> {
139    let query = format!(
140        "query($login: String!, $cursor: String) {{ user(login: $login) {{ {connection_call} {{ {REPOSITORY_FIELDS} }} }} }}"
141    );
142    let mut repositories = Vec::new();
143    let mut page_cursor: Option<String> = None;
144    loop {
145        let mut variables = vec![("login", login)];
146        if let Some(page_cursor) = &page_cursor {
147            variables.push(("cursor", page_cursor.as_str()));
148        }
149        let page_data = github.run_query(&query, &variables)?;
150        let connection = read_field(read_field(&page_data, "user")?, connection_name)?;
151        for repository_node in read_array(connection, "nodes")? {
152            repositories.push(read_repository(repository_node, language_colors)?);
153        }
154        let page_info = read_field(connection, "pageInfo")?;
155        if !read_bool(page_info, "hasNextPage")? {
156            return Ok(repositories);
157        }
158        let next_cursor = read_str(page_info, "endCursor")?.to_owned();
159        if page_cursor.as_deref() == Some(next_cursor.as_str()) {
160            return Err(CollectionError::UnexpectedResponse(format!(
161                "{connection_name} repeated page cursor {next_cursor}"
162            )));
163        }
164        page_cursor = Some(next_cursor);
165    }
166}
167
168fn read_repository(
169    repository_node: &Value,
170    language_colors: &mut BTreeMap<String, Option<LanguageColor>>,
171) -> Result<RepositoryObservation, CollectionError> {
172    let visibility = match read_str(repository_node, "visibility")? {
173        "PUBLIC" => RepositoryVisibility::Public,
174        "PRIVATE" => RepositoryVisibility::Private,
175        "INTERNAL" => RepositoryVisibility::Internal,
176        other => {
177            return Err(CollectionError::UnexpectedResponse(format!(
178                "unknown repository visibility {other}"
179            )));
180        }
181    };
182    let primary_language = match read_field(repository_node, "primaryLanguage")? {
183        Value::Null => None,
184        language => {
185            let language_name = read_str(language, "name")?.to_owned();
186            let language_color = match read_field(language, "color")? {
187                Value::Null => None,
188                color => Some(
189                    LanguageColor::try_from(color.as_str().ok_or_else(|| {
190                        CollectionError::UnexpectedResponse(format!(
191                            "{language_name} color is not text"
192                        ))
193                    })?)
194                    .map_err(CollectionError::UnexpectedResponse)?,
195                ),
196            };
197            if let Some(recorded_color) =
198                language_colors.insert(language_name.clone(), language_color)
199                && recorded_color != language_color
200            {
201                return Err(CollectionError::ConflictingLanguageColor(language_name));
202            }
203            Some(language_name)
204        }
205    };
206    Ok(RepositoryObservation {
207        repository_id: read_u64(repository_node, "databaseId")?,
208        visibility,
209        is_fork: read_bool(repository_node, "isFork")?,
210        primary_language,
211    })
212}
213
214/// Collect the calendar in year-long windows from the account's creation date through the
215/// last complete day.
216fn collect_contribution_history(
217    github: &impl GraphqlClient,
218    login: &str,
219    created_on: NaiveDate,
220    today: NaiveDate,
221) -> Result<ContributionHistory, CollectionError> {
222    let last_complete_day = today
223        .pred_opt()
224        .expect("calendar windows stay within supported dates");
225    if created_on > last_complete_day {
226        return Ok(ContributionHistory {
227            observed_on: last_complete_day,
228            days: Vec::new(),
229        });
230    }
231    let mut daily_counts = BTreeMap::new();
232    let mut window_start = created_on;
233    while window_start <= last_complete_day {
234        let window_end = window_start
235            .checked_add_months(Months::new(12))
236            .and_then(|next_year| next_year.pred_opt())
237            .expect("calendar windows stay within supported dates")
238            .min(last_complete_day);
239        daily_counts.extend(collect_calendar_window(
240            github,
241            login,
242            window_start,
243            window_end,
244        )?);
245        window_start = window_end
246            .checked_add_days(Days::new(1))
247            .expect("calendar windows stay within supported dates");
248    }
249    let first_reported_on = daily_counts.keys().next().copied();
250    let observed_on = match (first_reported_on, daily_counts.keys().next_back()) {
251        (Some(first_date), Some(&last_date)) if first_date == created_on => last_date,
252        _ => {
253            return Err(CollectionError::IncompleteCalendar {
254                created_on,
255                first_reported_on,
256            });
257        }
258    };
259    Ok(ContributionHistory {
260        observed_on,
261        days: daily_counts
262            .into_iter()
263            .map(|(date, contribution_count)| ContributionDay {
264                date,
265                contribution_count,
266            })
267            .collect(),
268    })
269}
270
271/// Collect one window of the calendar. GitHub can answer a request with fewer days than it
272/// covers, leaving out the earliest, so while the window's first day is missing the days
273/// before the earliest one reported are requested again. Each request ends before the
274/// previous one, so this stops. A request that returns no days stops it with the window
275/// incomplete, and the checks on the whole history report the missing days.
276fn collect_calendar_window(
277    github: &impl GraphqlClient,
278    login: &str,
279    window_start: NaiveDate,
280    window_end: NaiveDate,
281) -> Result<BTreeMap<NaiveDate, u32>, CollectionError> {
282    let mut window_daily_counts = BTreeMap::new();
283    let mut request_end = window_end;
284    loop {
285        let request_daily_counts = request_calendar_days(github, login, window_start, request_end)?;
286        let Some(&earliest_reported_on) = request_daily_counts.keys().next() else {
287            return Ok(window_daily_counts);
288        };
289        window_daily_counts.extend(request_daily_counts);
290        if earliest_reported_on == window_start {
291            return Ok(window_daily_counts);
292        }
293        request_end = earliest_reported_on
294            .pred_opt()
295            .expect("calendar windows stay within supported dates");
296    }
297}
298
299/// Make one calendar request and return the daily counts GitHub reported from
300/// `request_start` through `request_end`. GitHub answers in whole weeks, so days outside
301/// that range are dropped. A date reported twice with different counts fails.
302fn request_calendar_days(
303    github: &impl GraphqlClient,
304    login: &str,
305    request_start: NaiveDate,
306    request_end: NaiveDate,
307) -> Result<BTreeMap<NaiveDate, u32>, CollectionError> {
308    let from = format!("{request_start}T00:00:00Z");
309    let to = format!("{request_end}T23:59:59Z");
310    let calendar_data = github.run_query(
311        CALENDAR_QUERY,
312        &[("login", login), ("from", &from), ("to", &to)],
313    )?;
314    let calendar = read_field(
315        read_field(
316            read_field(&calendar_data, "user")?,
317            "contributionsCollection",
318        )?,
319        "contributionCalendar",
320    )?;
321    let mut daily_counts = BTreeMap::new();
322    for calendar_week in read_array(calendar, "weeks")? {
323        for calendar_day in read_array(calendar_week, "contributionDays")? {
324            let contribution_date =
325                NaiveDate::parse_from_str(read_str(calendar_day, "date")?, "%Y-%m-%d").map_err(
326                    |parse_error| {
327                        CollectionError::UnexpectedResponse(format!(
328                            "calendar date is invalid: {parse_error}"
329                        ))
330                    },
331                )?;
332            if contribution_date < request_start || contribution_date > request_end {
333                continue;
334            }
335            let contribution_count = u32::try_from(read_u64(calendar_day, "contributionCount")?)
336                .map_err(|_| {
337                    CollectionError::UnexpectedResponse(format!(
338                        "contribution count for {contribution_date} is out of range"
339                    ))
340                })?;
341            if let Some(previous_count) = daily_counts.insert(contribution_date, contribution_count)
342                && previous_count != contribution_count
343            {
344                return Err(CollectionError::InvalidHistory(
345                    ContributionHistoryError::ConflictingDate(contribution_date),
346                ));
347            }
348        }
349    }
350    Ok(daily_counts)
351}
352
353fn read_field<'a>(object: &'a Value, field_name: &str) -> Result<&'a Value, CollectionError> {
354    object.get(field_name).ok_or_else(|| {
355        CollectionError::UnexpectedResponse(format!("response is missing {field_name}"))
356    })
357}
358
359fn read_array<'a>(object: &'a Value, field_name: &str) -> Result<&'a Vec<Value>, CollectionError> {
360    read_field(object, field_name)?
361        .as_array()
362        .ok_or_else(|| CollectionError::UnexpectedResponse(format!("{field_name} is not a list")))
363}
364
365fn read_str<'a>(object: &'a Value, field_name: &str) -> Result<&'a str, CollectionError> {
366    read_field(object, field_name)?
367        .as_str()
368        .ok_or_else(|| CollectionError::UnexpectedResponse(format!("{field_name} is not text")))
369}
370
371fn read_u64(object: &Value, field_name: &str) -> Result<u64, CollectionError> {
372    read_field(object, field_name)?.as_u64().ok_or_else(|| {
373        CollectionError::UnexpectedResponse(format!("{field_name} is not a whole number"))
374    })
375}
376
377fn read_bool(object: &Value, field_name: &str) -> Result<bool, CollectionError> {
378    read_field(object, field_name)?.as_bool().ok_or_else(|| {
379        CollectionError::UnexpectedResponse(format!("{field_name} is not true or false"))
380    })
381}
382
383/// Why a collection produced no snapshot. Each message names the operator repair.
384#[derive(Debug, Eq, PartialEq)]
385pub enum CollectionError {
386    /// `gh` is not on `PATH`.
387    GhNotInstalled,
388    /// `gh` ran and failed; carries its own message (authentication, rate limits, errors).
389    GhFailed(String),
390    /// GitHub has no account with this login.
391    AccountNotFound(String),
392    /// GitHub answered in a shape this version does not understand; carries the detail.
393    UnexpectedResponse(String),
394    /// GitHub's contribution calendar does not start on the account's creation date.
395    IncompleteCalendar {
396        /// The day GitHub says the account was created.
397        created_on: NaiveDate,
398        /// The first day the calendar reported, or `None` when it reported none.
399        first_reported_on: Option<NaiveDate>,
400    },
401    /// The collected days break the history rules; carries the rule.
402    InvalidHistory(ContributionHistoryError),
403    /// Two observations of one repository disagree.
404    ConflictingRepository(ConflictingRepository),
405    /// GitHub reported two colors for this language.
406    ConflictingLanguageColor(String),
407}
408
409impl fmt::Display for CollectionError {
410    fn fmt(&self, output_formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
411        match self {
412            Self::GhNotInstalled => write!(
413                output_formatter,
414                "gh is not installed or not on PATH; install the GitHub CLI from https://cli.github.com"
415            ),
416            Self::GhFailed(gh_message) => write!(
417                output_formatter,
418                "gh api failed: {gh_message}; check `gh auth status`, or set GH_TOKEN in CI"
419            ),
420            Self::AccountNotFound(login) => {
421                write!(output_formatter, "GitHub has no account named {login}")
422            }
423            Self::UnexpectedResponse(detail) => write!(
424                output_formatter,
425                "unexpected GitHub response: {detail}; try again or report the response"
426            ),
427            Self::IncompleteCalendar {
428                created_on,
429                first_reported_on: Some(first_reported_on),
430            } => write!(
431                output_formatter,
432                "GitHub's contribution calendar starts on {first_reported_on}, but the account was created on {created_on}; try again"
433            ),
434            Self::IncompleteCalendar {
435                created_on,
436                first_reported_on: None,
437            } => write!(
438                output_formatter,
439                "GitHub reported no contribution days since the account was created on {created_on}; try again"
440            ),
441            Self::InvalidHistory(history_error) => {
442                write!(
443                    output_formatter,
444                    "collected history is invalid: {history_error}"
445                )
446            }
447            Self::ConflictingRepository(repository_error) => {
448                write!(output_formatter, "{repository_error}")
449            }
450            Self::ConflictingLanguageColor(language_name) => write!(
451                output_formatter,
452                "GitHub reported two colors for {language_name}; try again"
453            ),
454        }
455    }
456}
457
458impl Error for CollectionError {
459    fn source(&self) -> Option<&(dyn Error + 'static)> {
460        match self {
461            Self::InvalidHistory(history_error) => Some(history_error),
462            Self::ConflictingRepository(repository_error) => Some(repository_error),
463            _ => None,
464        }
465    }
466}
git_koi/collect.rs

git_koi/
collect.rs
