Skip to main content

second_brain_api/
sources.rs

1use std::borrow::Cow;
2use std::collections::{HashMap, HashSet};
3use std::path::{Path, PathBuf};
4use std::sync::Arc;
5use std::time::Duration;
6
7use anyhow::{Context, Result};
8use chrono::{DateTime, Utc};
9use serde::Deserialize;
10use sha2::{Digest, Sha256};
11use uuid::Uuid;
12
13use second_brain_adapters::git::{self, Checkout, GitHubRepo, PullRequest};
14use second_brain_adapters::{
15    IngestResult, StoreOutcome, encode_project_dir, repo_instructions, store_document_result,
16};
17use second_brain_core::embedding::DocumentEmbedder;
18use second_brain_core::redact::redact_memory;
19use second_brain_core::schema::{IngestRecord, Memory, Relation, RelationType, Tier};
20use second_brain_core::sqlite_store::SqliteStore;
21use second_brain_core::store::{Store, StoreAdmin};
22
23use crate::bootstrap::Bootstrap;
24
25pub const INTERVAL: Duration = Duration::from_secs(600);
26pub const FIRST_SIGHT_COMMITS: usize = 500;
27pub const PR_PAGES: usize = 5;
28const PER_PAGE: usize = 100;
29const FIRST_PASS_DELAY: Duration = Duration::from_secs(60);
30const SLUG_WALK_READS: usize = 10_000;
31const GITHUB_API: &str = "https://api.github.com";
32const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
33
34pub trait PullRequests: Send + Sync {
35    /// PRs of `repo` updated after `since` (all when None), newest update first.
36    fn updated_since(
37        &self,
38        repo: &GitHubRepo,
39        since: Option<DateTime<Utc>>,
40    ) -> Result<Vec<PullRequest>>;
41}
42
43pub struct GitHub {
44    token: String,
45    api: String,
46    client: reqwest::Client,
47    handle: tokio::runtime::Handle,
48}
49
50impl GitHub {
51    /// From GITHUB_TOKEN, else GH_TOKEN; None when neither holds a non-blank value.
52    pub fn from_env(handle: tokio::runtime::Handle) -> Option<GitHub> {
53        let token = token_from(|name| std::env::var(name).ok())?;
54        GitHub::new(token, GITHUB_API.to_string(), handle)
55    }
56
57    fn new(token: String, api: String, handle: tokio::runtime::Handle) -> Option<GitHub> {
58        match reqwest::Client::builder()
59            .timeout(HTTP_TIMEOUT)
60            .user_agent("second-brain")
61            .build()
62        {
63            Ok(client) => Some(GitHub {
64                token,
65                api,
66                client,
67                handle,
68            }),
69            Err(e) => {
70                tracing::warn!("sources: pull requests disabled, cannot build an HTTP client: {e}");
71                None
72            }
73        }
74    }
75
76    async fn fetch(
77        &self,
78        repo: &GitHubRepo,
79        since: Option<DateTime<Utc>>,
80    ) -> Result<Vec<PullRequest>> {
81        let mut prs = Vec::new();
82        for page in 1..=PR_PAGES {
83            let items = self.page(repo, page).await?;
84            let last_page = items.len() < PER_PAGE;
85            if take_newer(items, since, &mut prs) || last_page {
86                break;
87            }
88        }
89        Ok(prs)
90    }
91
92    async fn page(&self, repo: &GitHubRepo, page: usize) -> Result<Vec<ApiPull>> {
93        let url = format!(
94            "{}/repos/{}/{}/pulls?state=all&sort=updated&direction=desc&per_page={PER_PAGE}&page={page}",
95            self.api, repo.owner, repo.name
96        );
97        let response = self
98            .client
99            .get(&url)
100            .bearer_auth(&self.token)
101            .header(reqwest::header::ACCEPT, "application/vnd.github+json")
102            .send()
103            .await
104            .with_context(|| format!("GET {url}"))?;
105        if response.status() == reqwest::StatusCode::NOT_FOUND {
106            tracing::debug!("sources: {url} is not visible to the token");
107            return Ok(Vec::new());
108        }
109        let response = response
110            .error_for_status()
111            .with_context(|| format!("GET {url}"))?;
112        response
113            .json()
114            .await
115            .with_context(|| format!("decode {url}"))
116    }
117}
118
119impl PullRequests for GitHub {
120    fn updated_since(
121        &self,
122        repo: &GitHubRepo,
123        since: Option<DateTime<Utc>>,
124    ) -> Result<Vec<PullRequest>> {
125        self.handle.block_on(self.fetch(repo, since))
126    }
127}
128
129fn token_from(var: impl Fn(&str) -> Option<String>) -> Option<String> {
130    ["GITHUB_TOKEN", "GH_TOKEN"]
131        .into_iter()
132        .filter_map(var)
133        .map(|token| token.trim().to_string())
134        .find(|token| !token.is_empty())
135}
136
137#[derive(Deserialize)]
138struct ApiPull {
139    number: u64,
140    title: String,
141    #[serde(default)]
142    body: Option<String>,
143    html_url: String,
144    updated_at: DateTime<Utc>,
145}
146
147/// Appends the PRs of one page (newest update first) updated after `since`,
148/// skipping those without a body. True once a PR at or before `since` shows
149/// up, since every later page is older still.
150fn take_newer(
151    items: Vec<ApiPull>,
152    since: Option<DateTime<Utc>>,
153    out: &mut Vec<PullRequest>,
154) -> bool {
155    for item in items {
156        if since.is_some_and(|since| item.updated_at <= since) {
157            return true;
158        }
159        let body = item.body.unwrap_or_default();
160        if body.trim().is_empty() {
161            continue;
162        }
163        out.push(PullRequest {
164            number: item.number,
165            title: item.title,
166            body,
167            url: item.html_url,
168            updated_at: item.updated_at,
169        });
170    }
171    false
172}
173
174#[derive(Debug, Default, PartialEq, Eq)]
175pub struct Pass {
176    pub repos: usize,
177    pub stored: usize,
178    pub superseded: usize,
179    pub errors: usize,
180}
181
182impl Pass {
183    fn absorb(&mut self, outcome: &StoreOutcome) {
184        self.stored += outcome.stored;
185        self.errors += outcome.errors;
186    }
187}
188
189pub struct Sources {
190    pub store: Arc<SqliteStore>,
191    pub embedder: Option<Arc<dyn DocumentEmbedder>>,
192    pub machine_id: String,
193    pub pull_requests: Option<Arc<dyn PullRequests>>,
194}
195
196impl Sources {
197    /// Every checkout behind a project slug stored in the database, deduped
198    /// by main checkout.
199    pub fn repos(&self) -> Vec<Checkout> {
200        let slugs = match self.store.distinct_project_paths() {
201            Ok(slugs) => slugs,
202            Err(e) => {
203                tracing::warn!("sources: cannot list project slugs: {e:#}");
204                return Vec::new();
205            }
206        };
207        let mut seen = HashSet::new();
208        let mut repos = Vec::new();
209        for (slug, _) in slugs {
210            let Some(checkout) = dir_for_slug(&slug).and_then(|dir| Checkout::containing(&dir))
211            else {
212                continue;
213            };
214            if seen.insert(checkout.root().to_path_buf()) {
215                repos.push(checkout);
216            }
217        }
218        repos
219    }
220
221    pub fn run_once(&self) -> Pass {
222        self.run_checkouts(&self.repos())
223    }
224
225    /// None while a pairing bootstrap holds ingest, because the snapshot
226    /// brings the commits and PRs the paired device already stored.
227    pub fn run_unless_pending(&self, bootstrap: &Bootstrap) -> Option<Pass> {
228        (!bootstrap.is_pending()).then(|| self.run_once())
229    }
230
231    pub fn run_checkouts(&self, checkouts: &[Checkout]) -> Pass {
232        let mut pass = Pass::default();
233        for checkout in checkouts {
234            pass.repos += 1;
235            let root = checkout.root().display();
236            if let Err(e) = self.commits(checkout, &mut pass) {
237                tracing::warn!("sources: commits of {root}: {e:#}");
238                pass.errors += 1;
239            }
240            if let Err(e) = self.pull_requests(checkout, &mut pass) {
241                tracing::warn!("sources: pull requests of {root}: {e:#}");
242                pass.errors += 1;
243            }
244            for path in repo_instructions::instruction_files(checkout) {
245                if let Err(e) = self.instructions(checkout, &path, &mut pass) {
246                    tracing::warn!("sources: {}: {e:#}", path.display());
247                    pass.errors += 1;
248                }
249            }
250        }
251        pass
252    }
253
254    pub fn spawn(self, bootstrap: Arc<Bootstrap>) {
255        let spawned = std::thread::Builder::new()
256            .name("sources".into())
257            .spawn(move || {
258                std::thread::sleep(FIRST_PASS_DELAY);
259                loop {
260                    if let Some(pass) = self.run_unless_pending(&bootstrap)
261                        && (pass.stored > 0 || pass.superseded > 0 || pass.errors > 0)
262                    {
263                        tracing::info!(
264                            "sources: {} repos, {} stored, {} superseded, {} errors",
265                            pass.repos,
266                            pass.stored,
267                            pass.superseded,
268                            pass.errors
269                        );
270                    }
271                    std::thread::sleep(INTERVAL);
272                }
273            });
274        if let Err(e) = spawned {
275            tracing::error!("cannot start the sources thread: {e}");
276        }
277    }
278
279    fn commits(&self, checkout: &Checkout, pass: &mut Pass) -> Result<()> {
280        let Some(head) = checkout.head()? else {
281            return Ok(());
282        };
283        let key = format!("{}:{}", git::COMMIT_SOURCE, checkout.root().display());
284        let recorded = self.store.ingest_log_entry(&key)?.map(|e| e.file_hash);
285        if recorded.as_deref() == Some(head.as_str()) {
286            return Ok(());
287        }
288        let mut written = StoreOutcome::default();
289        for commit in checkout.commits_since(recorded.as_deref(), FIRST_SIGHT_COMMITS)? {
290            let outcome =
291                self.store_document(&git::commit_result(checkout, &commit, &self.machine_id));
292            written.stored += outcome.stored;
293            written.errors += outcome.errors;
294        }
295        pass.absorb(&written);
296        if written.errors == 0 {
297            self.record(&key, &head, written.stored, git::COMMIT_SOURCE)?;
298        }
299        Ok(())
300    }
301
302    fn pull_requests(&self, checkout: &Checkout, pass: &mut Pass) -> Result<()> {
303        let Some(api) = &self.pull_requests else {
304            return Ok(());
305        };
306        let Some(repo) = checkout.github_remote() else {
307            return Ok(());
308        };
309        let key = format!("{}:{}/{}", git::PR_SOURCE, repo.owner, repo.name);
310        let since = self
311            .store
312            .ingest_log_entry(&key)?
313            .and_then(|entry| DateTime::parse_from_rfc3339(&entry.file_hash).ok())
314            .map(|at| at.with_timezone(&Utc));
315        let prs = api.updated_since(&repo, since)?;
316        let Some(newest) = prs.iter().map(|pr| pr.updated_at).max() else {
317            return Ok(());
318        };
319        let mut versions = Versions::load(&self.store, git::PR_SOURCE)?;
320        let (mut stored, mut errors) = (0, 0);
321        for pr in &prs {
322            let version = self.store_version(&mut versions, pass, |after| {
323                Some(git::pull_request_result(
324                    checkout,
325                    &repo,
326                    pr,
327                    &self.machine_id,
328                    after,
329                ))
330            });
331            stored += version.stored;
332            errors += version.errors;
333        }
334        if errors == 0 {
335            self.record(&key, &newest.to_rfc3339(), stored, git::PR_SOURCE)?;
336        }
337        Ok(())
338    }
339
340    fn instructions(&self, checkout: &Checkout, path: &Path, pass: &mut Pass) -> Result<()> {
341        let key = path.to_string_lossy();
342        let bytes = std::fs::read(path).with_context(|| format!("read {}", path.display()))?;
343        let hash = sha256_hex(&bytes);
344        if !self.store.is_file_changed(&key, &hash)? {
345            return Ok(());
346        }
347        let mut versions = Versions::load(&self.store, repo_instructions::SOURCE)?;
348        let version = self.store_version(&mut versions, pass, |after| {
349            repo_instructions::instructions_result(checkout, path, &self.machine_id, after)
350        });
351        if version.errors > 0 {
352            return Ok(());
353        }
354        self.record(&key, &hash, version.stored, repo_instructions::SOURCE)
355    }
356
357    /// Brings one document's versions in line with its current text: the
358    /// live version holding that text stays, a text `forget` retired stays
359    /// retired, and any other text lands as a new version. Every other live
360    /// version is superseded. `build` gets the version the candidate follows,
361    /// which keys its id: the newest one in any state, or the no longer live
362    /// version already holding the id that gave, so a text restored after an
363    /// edit never collides with the id its earlier appearance holds.
364    fn store_version(
365        &self,
366        versions: &mut Versions,
367        pass: &mut Pass,
368        build: impl Fn(Option<Uuid>) -> Option<IngestResult>,
369    ) -> StoreOutcome {
370        let mut version = StoreOutcome::default();
371        let Some(first) = build(None) else {
372            return version;
373        };
374        let Some(source_id) = first.memories.first().map(|m| m.source_id.clone()) else {
375            return version;
376        };
377        let mut candidate = first;
378        let mut after = versions.newest(&source_id);
379        for _ in 0..=versions.all(&source_id).len() {
380            let Some(previous) = after else {
381                break;
382            };
383            let Some(result) = build(Some(previous)) else {
384                return version;
385            };
386            candidate = result;
387            after = candidate
388                .memories
389                .first()
390                .and_then(|m| versions.retired_or_superseded(&source_id, m.id));
391        }
392        let Some(mut text) = candidate.memories.first().cloned() else {
393            return version;
394        };
395        redact_memory(&mut text);
396        let live = versions.live(&source_id);
397        let current = match versions.holding(&source_id, &text.content) {
398            Some(existing) => existing,
399            None => {
400                version = self.store_document(&candidate);
401                pass.absorb(&version);
402                if version.stored == 0 {
403                    return version;
404                }
405                versions.add(text.clone());
406                text
407            }
408        };
409        for old in live.iter().filter(|m| m.id != current.id) {
410            match self.supersede(old.id, current.id) {
411                Ok(()) => {
412                    pass.superseded += 1;
413                    versions.superseded(old.id, current.id);
414                }
415                Err(e) => {
416                    tracing::warn!(
417                        "sources: cannot supersede {} by {}: {e:#}",
418                        old.id,
419                        current.id
420                    );
421                    version.errors += 1;
422                    pass.errors += 1;
423                }
424            }
425        }
426        version
427    }
428
429    fn store_document(&self, result: &IngestResult) -> StoreOutcome {
430        store_document_result(
431            self.store.as_ref(),
432            self.embedder.as_deref(),
433            result,
434            &self.machine_id,
435        )
436    }
437
438    fn record(&self, key: &str, hash: &str, memory_count: usize, source: &str) -> Result<()> {
439        self.store.record_ingested(&IngestRecord {
440            file_path: key,
441            content_hash: hash,
442            session_key: None,
443            legacy_ids: false,
444            memory_count,
445            source,
446        })
447    }
448
449    /// The edge goes first: a supersede that fails after it leaves the old
450    /// version live, so the next pass retries both.
451    fn supersede(&self, old: Uuid, new: Uuid) -> Result<()> {
452        self.store.store_relation(&Relation {
453            from_id: new,
454            to_id: old,
455            relation_type: RelationType::Supersedes,
456            strength: 1.0,
457            context: None,
458        })?;
459        self.store.supersede_memory(old, new)
460    }
461}
462
463/// Every memory of one source, by source_id, read once per batch so a pass
464/// over many pull requests does not reread them all for each one.
465struct Versions(HashMap<String, Vec<Memory>>);
466
467impl Versions {
468    fn load(store: &SqliteStore, source: &str) -> Result<Versions> {
469        let mut by_source_id: HashMap<String, Vec<Memory>> = HashMap::new();
470        for memory in store.memories_by_source(source)? {
471            by_source_id
472                .entry(memory.source_id.clone())
473                .or_default()
474                .push(memory);
475        }
476        Ok(Versions(by_source_id))
477    }
478
479    fn all(&self, source_id: &str) -> &[Memory] {
480        self.0.get(source_id).map(Vec::as_slice).unwrap_or_default()
481    }
482
483    fn newest(&self, source_id: &str) -> Option<Uuid> {
484        self.all(source_id)
485            .iter()
486            .max_by_key(|m| (m.created_at, m.id))
487            .map(|m| m.id)
488    }
489
490    fn live(&self, source_id: &str) -> Vec<Memory> {
491        self.all(source_id)
492            .iter()
493            .filter(|m| is_live(m))
494            .cloned()
495            .collect()
496    }
497
498    /// The newest live version holding `content`, else the newest one
499    /// `forget` retired, which keeps that text out of recall.
500    fn holding(&self, source_id: &str, content: &str) -> Option<Memory> {
501        let same = |m: &&Memory| m.content == content;
502        let newest = |a: &&Memory, b: &&Memory| (a.created_at, a.id).cmp(&(b.created_at, b.id));
503        let versions = self.all(source_id);
504        versions
505            .iter()
506            .filter(|m| is_live(m))
507            .filter(same)
508            .max_by(newest)
509            .or_else(|| {
510                versions
511                    .iter()
512                    .filter(|m| m.retired_reason.is_some())
513                    .filter(same)
514                    .max_by(newest)
515            })
516            .cloned()
517    }
518
519    fn add(&mut self, memory: Memory) {
520        self.0
521            .entry(memory.source_id.clone())
522            .or_default()
523            .push(memory);
524    }
525
526    /// `id` when a version that is no longer live holds it, which happens
527    /// when `newest` ordered a peer's row by a clock that runs ahead.
528    fn retired_or_superseded(&self, source_id: &str, id: Uuid) -> Option<Uuid> {
529        self.all(source_id)
530            .iter()
531            .any(|m| m.id == id && !is_live(m))
532            .then_some(id)
533    }
534
535    fn superseded(&mut self, old: Uuid, by: Uuid) {
536        if let Some(memory) = self.0.values_mut().flatten().find(|m| m.id == old) {
537            memory.superseded_by = Some(by);
538        }
539    }
540}
541
542fn is_live(memory: &Memory) -> bool {
543    memory.tier == Tier::Hot && memory.superseded_by.is_none()
544}
545
546fn sha256_hex(bytes: &[u8]) -> String {
547    format!("{:x}", Sha256::digest(bytes))
548}
549
550/// The existing directory whose Claude Code slug is `slug`, or None.
551pub fn dir_for_slug(slug: &str) -> Option<PathBuf> {
552    dir_for_slug_under(Path::new("/"), slug)
553}
554
555/// `dir_for_slug` for a slug of the path below `root`. A slug without a
556/// leading '-' (Cursor's form) is read as if it had one. Gives up after
557/// SLUG_WALK_READS directory reads.
558pub fn dir_for_slug_under(root: &Path, slug: &str) -> Option<PathBuf> {
559    if slug.is_empty() {
560        return None;
561    }
562    let slug = if slug.starts_with('-') {
563        Cow::Borrowed(slug)
564    } else {
565        Cow::Owned(format!("-{slug}"))
566    };
567    let mut reads = 0;
568    find_slug(root, &slug, &mut reads)
569}
570
571fn find_slug(dir: &Path, rest: &str, reads: &mut usize) -> Option<PathBuf> {
572    if *reads >= SLUG_WALK_READS {
573        return None;
574    }
575    *reads += 1;
576    let mut exact = Vec::new();
577    let mut partial = Vec::new();
578    for entry in std::fs::read_dir(dir).ok()?.flatten() {
579        let encoded = format!("-{}", encode_project_dir(Path::new(&entry.file_name())));
580        let Some(after) = rest.strip_prefix(encoded.as_str()) else {
581            continue;
582        };
583        if !after.is_empty() && !after.starts_with('-') {
584            continue;
585        }
586        let path = entry.path();
587        if !path.is_dir() {
588            continue;
589        }
590        if after.is_empty() {
591            exact.push(path);
592        } else {
593            partial.push((path, encoded.len()));
594        }
595    }
596    exact.sort();
597    if let Some(found) = exact.into_iter().next() {
598        return Some(found);
599    }
600    partial.sort();
601    partial
602        .into_iter()
603        .find_map(|(path, len)| find_slug(&path, &rest[len..], reads))
604}
605
606#[cfg(test)]
607mod tests {
608    use std::sync::Mutex;
609
610    use axum::extract::{Query, State};
611    use axum::http::HeaderMap;
612    use axum::routing::get;
613    use axum::{Json, Router};
614    use chrono::TimeZone;
615    use serde_json::{Value, json};
616
617    use super::*;
618
619    #[test]
620    fn the_token_comes_from_github_token_then_gh_token() {
621        fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
622            move |name| {
623                pairs
624                    .iter()
625                    .find(|(k, _)| *k == name)
626                    .map(|(_, v)| v.to_string())
627            }
628        }
629        assert_eq!(
630            token_from(env(&[("GITHUB_TOKEN", "first"), ("GH_TOKEN", "second")])),
631            Some("first".into())
632        );
633        assert_eq!(
634            token_from(env(&[("GH_TOKEN", " second ")])),
635            Some("second".into())
636        );
637        assert_eq!(
638            token_from(env(&[("GITHUB_TOKEN", "  "), ("GH_TOKEN", "second")])),
639            Some("second".into())
640        );
641        assert_eq!(
642            token_from(env(&[("GITHUB_TOKEN", ""), ("GH_TOKEN", " ")])),
643            None
644        );
645        assert_eq!(token_from(env(&[])), None);
646    }
647
648    fn newest() -> DateTime<Utc> {
649        Utc.with_ymd_and_hms(2026, 5, 1, 12, 0, 0).unwrap()
650    }
651
652    fn updated(i: usize) -> DateTime<Utc> {
653        newest() - chrono::Duration::minutes(i as i64)
654    }
655
656    fn pull(i: usize) -> Value {
657        let body = if i % 10 == 3 {
658            Value::Null
659        } else {
660            json!(format!("Change {i} to the widget cache."))
661        };
662        json!({
663            "number": 1000 - i,
664            "title": format!("Widget change {i}"),
665            "body": body,
666            "html_url": format!("https://github.com/example/widgets/pull/{}", 1000 - i),
667            "updated_at": updated(i).to_rfc3339(),
668            "state": "open",
669        })
670    }
671
672    #[derive(Default)]
673    struct Fake {
674        total: usize,
675        requests: Mutex<Vec<(HashMap<String, String>, HeaderMap)>>,
676    }
677
678    async fn pulls(
679        State(fake): State<Arc<Fake>>,
680        Query(query): Query<HashMap<String, String>>,
681        headers: HeaderMap,
682    ) -> Json<Value> {
683        let page: usize = query["page"].parse().unwrap();
684        fake.requests.lock().unwrap().push((query, headers));
685        let start = ((page - 1) * PER_PAGE).min(fake.total);
686        let end = (page * PER_PAGE).min(fake.total);
687        Json(Value::Array((start..end).map(pull).collect()))
688    }
689
690    struct Server {
691        runtime: tokio::runtime::Runtime,
692        fake: Arc<Fake>,
693        api: String,
694    }
695
696    fn server(total: usize) -> Server {
697        let runtime = tokio::runtime::Builder::new_multi_thread()
698            .worker_threads(1)
699            .enable_all()
700            .build()
701            .unwrap();
702        let fake = Arc::new(Fake {
703            total,
704            ..Fake::default()
705        });
706        let app = Router::new()
707            .route("/repos/example/widgets/pulls", get(pulls))
708            .with_state(fake.clone());
709        let listener = runtime
710            .block_on(tokio::net::TcpListener::bind("127.0.0.1:0"))
711            .unwrap();
712        let api = format!("http://{}", listener.local_addr().unwrap());
713        runtime.spawn(async move { axum::serve(listener, app).await.unwrap() });
714        Server { runtime, fake, api }
715    }
716
717    fn fetch(server: &Server, since: Option<DateTime<Utc>>) -> Vec<PullRequest> {
718        let token = ["test", "-token"].concat();
719        let github =
720            GitHub::new(token, server.api.clone(), server.runtime.handle().clone()).unwrap();
721        let repo = GitHubRepo {
722            owner: "example".into(),
723            name: "widgets".into(),
724        };
725        github.updated_since(&repo, since).unwrap()
726    }
727
728    fn requests(server: &Server) -> Vec<(HashMap<String, String>, HeaderMap)> {
729        std::mem::take(&mut *server.fake.requests.lock().unwrap())
730    }
731
732    #[test]
733    fn github_pages_until_a_short_page_and_skips_empty_bodies() {
734        let server = server(103);
735        let prs = fetch(&server, None);
736        let expected: Vec<usize> = (0..103).filter(|i| i % 10 != 3).collect();
737        assert_eq!(
738            prs.iter().map(|p| p.number).collect::<Vec<_>>(),
739            expected
740                .iter()
741                .map(|i| (1000 - i) as u64)
742                .collect::<Vec<_>>()
743        );
744        assert_eq!(prs[0].title, "Widget change 0");
745        assert_eq!(prs[0].body, "Change 0 to the widget cache.");
746        assert_eq!(prs[0].url, "https://github.com/example/widgets/pull/1000");
747        assert_eq!(prs[0].updated_at, newest());
748
749        let requests = requests(&server);
750        assert_eq!(requests.len(), 2);
751        for (n, (query, headers)) in requests.iter().enumerate() {
752            assert_eq!(query["state"], "all");
753            assert_eq!(query["sort"], "updated");
754            assert_eq!(query["direction"], "desc");
755            assert_eq!(query["per_page"], "100");
756            assert_eq!(query["page"], (n + 1).to_string());
757            assert_eq!(headers["authorization"], "Bearer test-token");
758            assert_eq!(headers["user-agent"], "second-brain");
759            assert_eq!(headers["accept"], "application/vnd.github+json");
760        }
761    }
762
763    #[test]
764    fn github_stops_at_the_first_pr_not_newer_than_since() {
765        let server = server(400);
766        let prs = fetch(&server, Some(updated(50)));
767        assert_eq!(prs.len(), (0..50).filter(|i| i % 10 != 3).count());
768        assert!(prs.iter().all(|p| p.updated_at > updated(50)));
769        assert_eq!(requests(&server).len(), 1);
770    }
771
772    #[test]
773    fn github_reads_at_most_pr_pages_pages() {
774        let server = server(PR_PAGES * PER_PAGE + 250);
775        let prs = fetch(&server, None);
776        assert_eq!(requests(&server).len(), PR_PAGES);
777        assert_eq!(
778            prs.last().unwrap().number,
779            (1000 - (PR_PAGES * PER_PAGE - 1)) as u64
780        );
781    }
782
783    #[test]
784    fn a_repo_the_token_cannot_see_has_no_pull_requests() {
785        let server = server(3);
786        let token = ["test", "-token"].concat();
787        let github =
788            GitHub::new(token, server.api.clone(), server.runtime.handle().clone()).unwrap();
789        let repo = GitHubRepo {
790            owner: "example".into(),
791            name: "private".into(),
792        };
793        assert!(github.updated_since(&repo, None).unwrap().is_empty());
794    }
795}