Skip to main content

second_brain_api/
sources.rs

1use std::borrow::Cow;
2use std::collections::{HashMap, HashSet};
3use std::path::{Path, PathBuf};
4use std::sync::Arc;
5use std::time::Duration;
6
7use anyhow::{Context, Result};
8use chrono::{DateTime, Utc};
9use serde::Deserialize;
10use sha2::{Digest, Sha256};
11use uuid::Uuid;
12
13use second_brain_adapters::git::{self, Checkout, GitHubRepo, PullRequest};
14use second_brain_adapters::{
15    IngestResult, StoreOutcome, encode_project_dir, repo_instructions, store_document_result,
16};
17use second_brain_core::embedding::DocumentEmbedder;
18use second_brain_core::redact::redact_memory;
19use second_brain_core::schema::{IngestRecord, Memory, Relation, RelationType, Tier};
20use second_brain_core::sqlite_store::SqliteStore;
21use second_brain_core::store::{Store, StoreAdmin};
22
23pub const INTERVAL: Duration = Duration::from_secs(600);
24pub const FIRST_SIGHT_COMMITS: usize = 500;
25pub const PR_PAGES: usize = 5;
26const PER_PAGE: usize = 100;
27const FIRST_PASS_DELAY: Duration = Duration::from_secs(60);
28const SLUG_WALK_READS: usize = 10_000;
29const GITHUB_API: &str = "https://api.github.com";
30const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
31
32pub trait PullRequests: Send + Sync {
33    /// PRs of `repo` updated after `since` (all when None), newest update first.
34    fn updated_since(
35        &self,
36        repo: &GitHubRepo,
37        since: Option<DateTime<Utc>>,
38    ) -> Result<Vec<PullRequest>>;
39}
40
41pub struct GitHub {
42    token: String,
43    api: String,
44    client: reqwest::Client,
45    handle: tokio::runtime::Handle,
46}
47
48impl GitHub {
49    /// From GITHUB_TOKEN, else GH_TOKEN; None when neither holds a non-blank value.
50    pub fn from_env(handle: tokio::runtime::Handle) -> Option<GitHub> {
51        let token = token_from(|name| std::env::var(name).ok())?;
52        GitHub::new(token, GITHUB_API.to_string(), handle)
53    }
54
55    fn new(token: String, api: String, handle: tokio::runtime::Handle) -> Option<GitHub> {
56        match reqwest::Client::builder()
57            .timeout(HTTP_TIMEOUT)
58            .user_agent("second-brain")
59            .build()
60        {
61            Ok(client) => Some(GitHub {
62                token,
63                api,
64                client,
65                handle,
66            }),
67            Err(e) => {
68                tracing::warn!("sources: pull requests disabled, cannot build an HTTP client: {e}");
69                None
70            }
71        }
72    }
73
74    async fn fetch(
75        &self,
76        repo: &GitHubRepo,
77        since: Option<DateTime<Utc>>,
78    ) -> Result<Vec<PullRequest>> {
79        let mut prs = Vec::new();
80        for page in 1..=PR_PAGES {
81            let items = self.page(repo, page).await?;
82            let last_page = items.len() < PER_PAGE;
83            if take_newer(items, since, &mut prs) || last_page {
84                break;
85            }
86        }
87        Ok(prs)
88    }
89
90    async fn page(&self, repo: &GitHubRepo, page: usize) -> Result<Vec<ApiPull>> {
91        let url = format!(
92            "{}/repos/{}/{}/pulls?state=all&sort=updated&direction=desc&per_page={PER_PAGE}&page={page}",
93            self.api, repo.owner, repo.name
94        );
95        let response = self
96            .client
97            .get(&url)
98            .bearer_auth(&self.token)
99            .header(reqwest::header::ACCEPT, "application/vnd.github+json")
100            .send()
101            .await
102            .with_context(|| format!("GET {url}"))?;
103        if response.status() == reqwest::StatusCode::NOT_FOUND {
104            tracing::debug!("sources: {url} is not visible to the token");
105            return Ok(Vec::new());
106        }
107        let response = response
108            .error_for_status()
109            .with_context(|| format!("GET {url}"))?;
110        response
111            .json()
112            .await
113            .with_context(|| format!("decode {url}"))
114    }
115}
116
117impl PullRequests for GitHub {
118    fn updated_since(
119        &self,
120        repo: &GitHubRepo,
121        since: Option<DateTime<Utc>>,
122    ) -> Result<Vec<PullRequest>> {
123        self.handle.block_on(self.fetch(repo, since))
124    }
125}
126
127fn token_from(var: impl Fn(&str) -> Option<String>) -> Option<String> {
128    ["GITHUB_TOKEN", "GH_TOKEN"]
129        .into_iter()
130        .filter_map(var)
131        .map(|token| token.trim().to_string())
132        .find(|token| !token.is_empty())
133}
134
135#[derive(Deserialize)]
136struct ApiPull {
137    number: u64,
138    title: String,
139    #[serde(default)]
140    body: Option<String>,
141    html_url: String,
142    updated_at: DateTime<Utc>,
143}
144
145/// Appends the PRs of one page (newest update first) updated after `since`,
146/// skipping those without a body. True once a PR at or before `since` shows
147/// up, since every later page is older still.
148fn take_newer(
149    items: Vec<ApiPull>,
150    since: Option<DateTime<Utc>>,
151    out: &mut Vec<PullRequest>,
152) -> bool {
153    for item in items {
154        if since.is_some_and(|since| item.updated_at <= since) {
155            return true;
156        }
157        let body = item.body.unwrap_or_default();
158        if body.trim().is_empty() {
159            continue;
160        }
161        out.push(PullRequest {
162            number: item.number,
163            title: item.title,
164            body,
165            url: item.html_url,
166            updated_at: item.updated_at,
167        });
168    }
169    false
170}
171
172#[derive(Debug, Default, PartialEq, Eq)]
173pub struct Pass {
174    pub repos: usize,
175    pub stored: usize,
176    pub superseded: usize,
177    pub errors: usize,
178}
179
180impl Pass {
181    fn absorb(&mut self, outcome: &StoreOutcome) {
182        self.stored += outcome.stored;
183        self.errors += outcome.errors;
184    }
185}
186
187pub struct Sources {
188    pub store: Arc<SqliteStore>,
189    pub embedder: Option<Arc<dyn DocumentEmbedder>>,
190    pub machine_id: String,
191    pub pull_requests: Option<Arc<dyn PullRequests>>,
192}
193
194impl Sources {
195    /// Every checkout behind a project slug stored in the database, deduped
196    /// by main checkout.
197    pub fn repos(&self) -> Vec<Checkout> {
198        let slugs = match self.store.distinct_project_paths() {
199            Ok(slugs) => slugs,
200            Err(e) => {
201                tracing::warn!("sources: cannot list project slugs: {e:#}");
202                return Vec::new();
203            }
204        };
205        let mut seen = HashSet::new();
206        let mut repos = Vec::new();
207        for (slug, _) in slugs {
208            let Some(checkout) = dir_for_slug(&slug).and_then(|dir| Checkout::containing(&dir))
209            else {
210                continue;
211            };
212            if seen.insert(checkout.root().to_path_buf()) {
213                repos.push(checkout);
214            }
215        }
216        repos
217    }
218
219    pub fn run_once(&self) -> Pass {
220        self.run_checkouts(&self.repos())
221    }
222
223    pub fn run_checkouts(&self, checkouts: &[Checkout]) -> Pass {
224        let mut pass = Pass::default();
225        for checkout in checkouts {
226            pass.repos += 1;
227            let root = checkout.root().display();
228            if let Err(e) = self.commits(checkout, &mut pass) {
229                tracing::warn!("sources: commits of {root}: {e:#}");
230                pass.errors += 1;
231            }
232            if let Err(e) = self.pull_requests(checkout, &mut pass) {
233                tracing::warn!("sources: pull requests of {root}: {e:#}");
234                pass.errors += 1;
235            }
236            for path in repo_instructions::instruction_files(checkout) {
237                if let Err(e) = self.instructions(checkout, &path, &mut pass) {
238                    tracing::warn!("sources: {}: {e:#}", path.display());
239                    pass.errors += 1;
240                }
241            }
242        }
243        pass
244    }
245
246    pub fn spawn(self) {
247        let spawned = std::thread::Builder::new()
248            .name("sources".into())
249            .spawn(move || {
250                std::thread::sleep(FIRST_PASS_DELAY);
251                loop {
252                    let pass = self.run_once();
253                    if pass.stored > 0 || pass.superseded > 0 || pass.errors > 0 {
254                        tracing::info!(
255                            "sources: {} repos, {} stored, {} superseded, {} errors",
256                            pass.repos,
257                            pass.stored,
258                            pass.superseded,
259                            pass.errors
260                        );
261                    }
262                    std::thread::sleep(INTERVAL);
263                }
264            });
265        if let Err(e) = spawned {
266            tracing::error!("cannot start the sources thread: {e}");
267        }
268    }
269
270    fn commits(&self, checkout: &Checkout, pass: &mut Pass) -> Result<()> {
271        let Some(head) = checkout.head()? else {
272            return Ok(());
273        };
274        let key = format!("{}:{}", git::COMMIT_SOURCE, checkout.root().display());
275        let recorded = self.store.ingest_log_entry(&key)?.map(|e| e.file_hash);
276        if recorded.as_deref() == Some(head.as_str()) {
277            return Ok(());
278        }
279        let mut written = StoreOutcome::default();
280        for commit in checkout.commits_since(recorded.as_deref(), FIRST_SIGHT_COMMITS)? {
281            let outcome =
282                self.store_document(&git::commit_result(checkout, &commit, &self.machine_id));
283            written.stored += outcome.stored;
284            written.errors += outcome.errors;
285        }
286        pass.absorb(&written);
287        if written.errors == 0 {
288            self.record(&key, &head, written.stored, git::COMMIT_SOURCE)?;
289        }
290        Ok(())
291    }
292
293    fn pull_requests(&self, checkout: &Checkout, pass: &mut Pass) -> Result<()> {
294        let Some(api) = &self.pull_requests else {
295            return Ok(());
296        };
297        let Some(repo) = checkout.github_remote() else {
298            return Ok(());
299        };
300        let key = format!("{}:{}/{}", git::PR_SOURCE, repo.owner, repo.name);
301        let since = self
302            .store
303            .ingest_log_entry(&key)?
304            .and_then(|entry| DateTime::parse_from_rfc3339(&entry.file_hash).ok())
305            .map(|at| at.with_timezone(&Utc));
306        let prs = api.updated_since(&repo, since)?;
307        let Some(newest) = prs.iter().map(|pr| pr.updated_at).max() else {
308            return Ok(());
309        };
310        let mut versions = Versions::load(&self.store, git::PR_SOURCE)?;
311        let (mut stored, mut errors) = (0, 0);
312        for pr in &prs {
313            let version = self.store_version(&mut versions, pass, |after| {
314                Some(git::pull_request_result(
315                    checkout,
316                    &repo,
317                    pr,
318                    &self.machine_id,
319                    after,
320                ))
321            });
322            stored += version.stored;
323            errors += version.errors;
324        }
325        if errors == 0 {
326            self.record(&key, &newest.to_rfc3339(), stored, git::PR_SOURCE)?;
327        }
328        Ok(())
329    }
330
331    fn instructions(&self, checkout: &Checkout, path: &Path, pass: &mut Pass) -> Result<()> {
332        let key = path.to_string_lossy();
333        let bytes = std::fs::read(path).with_context(|| format!("read {}", path.display()))?;
334        let hash = sha256_hex(&bytes);
335        if !self.store.is_file_changed(&key, &hash)? {
336            return Ok(());
337        }
338        let mut versions = Versions::load(&self.store, repo_instructions::SOURCE)?;
339        let version = self.store_version(&mut versions, pass, |after| {
340            repo_instructions::instructions_result(checkout, path, &self.machine_id, after)
341        });
342        if version.errors > 0 {
343            return Ok(());
344        }
345        self.record(&key, &hash, version.stored, repo_instructions::SOURCE)
346    }
347
348    /// Brings one document's versions in line with its current text: the
349    /// live version holding that text stays, a text `forget` retired stays
350    /// retired, and any other text lands as a new version. Every other live
351    /// version is superseded. `build` gets the version the candidate follows,
352    /// which keys its id: the newest one in any state, or the no longer live
353    /// version already holding the id that gave, so a text restored after an
354    /// edit never collides with the id its earlier appearance holds.
355    fn store_version(
356        &self,
357        versions: &mut Versions,
358        pass: &mut Pass,
359        build: impl Fn(Option<Uuid>) -> Option<IngestResult>,
360    ) -> StoreOutcome {
361        let mut version = StoreOutcome::default();
362        let Some(first) = build(None) else {
363            return version;
364        };
365        let Some(source_id) = first.memories.first().map(|m| m.source_id.clone()) else {
366            return version;
367        };
368        let mut candidate = first;
369        let mut after = versions.newest(&source_id);
370        for _ in 0..=versions.all(&source_id).len() {
371            let Some(previous) = after else {
372                break;
373            };
374            let Some(result) = build(Some(previous)) else {
375                return version;
376            };
377            candidate = result;
378            after = candidate
379                .memories
380                .first()
381                .and_then(|m| versions.retired_or_superseded(&source_id, m.id));
382        }
383        let Some(mut text) = candidate.memories.first().cloned() else {
384            return version;
385        };
386        redact_memory(&mut text);
387        let live = versions.live(&source_id);
388        let current = match versions.holding(&source_id, &text.content) {
389            Some(existing) => existing,
390            None => {
391                version = self.store_document(&candidate);
392                pass.absorb(&version);
393                if version.stored == 0 {
394                    return version;
395                }
396                versions.add(text.clone());
397                text
398            }
399        };
400        for old in live.iter().filter(|m| m.id != current.id) {
401            match self.supersede(old.id, current.id) {
402                Ok(()) => {
403                    pass.superseded += 1;
404                    versions.superseded(old.id, current.id);
405                }
406                Err(e) => {
407                    tracing::warn!(
408                        "sources: cannot supersede {} by {}: {e:#}",
409                        old.id,
410                        current.id
411                    );
412                    version.errors += 1;
413                    pass.errors += 1;
414                }
415            }
416        }
417        version
418    }
419
420    fn store_document(&self, result: &IngestResult) -> StoreOutcome {
421        store_document_result(
422            self.store.as_ref(),
423            self.embedder.as_deref(),
424            result,
425            &self.machine_id,
426        )
427    }
428
429    fn record(&self, key: &str, hash: &str, memory_count: usize, source: &str) -> Result<()> {
430        self.store.record_ingested(&IngestRecord {
431            file_path: key,
432            content_hash: hash,
433            session_key: None,
434            legacy_ids: false,
435            memory_count,
436            source,
437        })
438    }
439
440    /// The edge goes first: a supersede that fails after it leaves the old
441    /// version live, so the next pass retries both.
442    fn supersede(&self, old: Uuid, new: Uuid) -> Result<()> {
443        self.store.store_relation(&Relation {
444            from_id: new,
445            to_id: old,
446            relation_type: RelationType::Supersedes,
447            strength: 1.0,
448            context: None,
449        })?;
450        self.store.supersede_memory(old, new)
451    }
452}
453
454/// Every memory of one source, by source_id, read once per batch so a pass
455/// over many pull requests does not reread them all for each one.
456struct Versions(HashMap<String, Vec<Memory>>);
457
458impl Versions {
459    fn load(store: &SqliteStore, source: &str) -> Result<Versions> {
460        let mut by_source_id: HashMap<String, Vec<Memory>> = HashMap::new();
461        for memory in store.memories_by_source(source)? {
462            by_source_id
463                .entry(memory.source_id.clone())
464                .or_default()
465                .push(memory);
466        }
467        Ok(Versions(by_source_id))
468    }
469
470    fn all(&self, source_id: &str) -> &[Memory] {
471        self.0.get(source_id).map(Vec::as_slice).unwrap_or_default()
472    }
473
474    fn newest(&self, source_id: &str) -> Option<Uuid> {
475        self.all(source_id)
476            .iter()
477            .max_by_key(|m| (m.created_at, m.id))
478            .map(|m| m.id)
479    }
480
481    fn live(&self, source_id: &str) -> Vec<Memory> {
482        self.all(source_id)
483            .iter()
484            .filter(|m| is_live(m))
485            .cloned()
486            .collect()
487    }
488
489    /// The newest live version holding `content`, else the newest one
490    /// `forget` retired, which keeps that text out of recall.
491    fn holding(&self, source_id: &str, content: &str) -> Option<Memory> {
492        let same = |m: &&Memory| m.content == content;
493        let newest = |a: &&Memory, b: &&Memory| (a.created_at, a.id).cmp(&(b.created_at, b.id));
494        let versions = self.all(source_id);
495        versions
496            .iter()
497            .filter(|m| is_live(m))
498            .filter(same)
499            .max_by(newest)
500            .or_else(|| {
501                versions
502                    .iter()
503                    .filter(|m| m.retired_reason.is_some())
504                    .filter(same)
505                    .max_by(newest)
506            })
507            .cloned()
508    }
509
510    fn add(&mut self, memory: Memory) {
511        self.0
512            .entry(memory.source_id.clone())
513            .or_default()
514            .push(memory);
515    }
516
517    /// `id` when a version that is no longer live holds it, which happens
518    /// when `newest` ordered a peer's row by a clock that runs ahead.
519    fn retired_or_superseded(&self, source_id: &str, id: Uuid) -> Option<Uuid> {
520        self.all(source_id)
521            .iter()
522            .any(|m| m.id == id && !is_live(m))
523            .then_some(id)
524    }
525
526    fn superseded(&mut self, old: Uuid, by: Uuid) {
527        if let Some(memory) = self.0.values_mut().flatten().find(|m| m.id == old) {
528            memory.superseded_by = Some(by);
529        }
530    }
531}
532
533fn is_live(memory: &Memory) -> bool {
534    memory.tier == Tier::Hot && memory.superseded_by.is_none()
535}
536
537fn sha256_hex(bytes: &[u8]) -> String {
538    format!("{:x}", Sha256::digest(bytes))
539}
540
541/// The existing directory whose Claude Code slug is `slug`, or None.
542pub fn dir_for_slug(slug: &str) -> Option<PathBuf> {
543    dir_for_slug_under(Path::new("/"), slug)
544}
545
546/// `dir_for_slug` for a slug of the path below `root`. A slug without a
547/// leading '-' (Cursor's form) is read as if it had one. Gives up after
548/// SLUG_WALK_READS directory reads.
549pub fn dir_for_slug_under(root: &Path, slug: &str) -> Option<PathBuf> {
550    if slug.is_empty() {
551        return None;
552    }
553    let slug = if slug.starts_with('-') {
554        Cow::Borrowed(slug)
555    } else {
556        Cow::Owned(format!("-{slug}"))
557    };
558    let mut reads = 0;
559    find_slug(root, &slug, &mut reads)
560}
561
562fn find_slug(dir: &Path, rest: &str, reads: &mut usize) -> Option<PathBuf> {
563    if *reads >= SLUG_WALK_READS {
564        return None;
565    }
566    *reads += 1;
567    let mut exact = Vec::new();
568    let mut partial = Vec::new();
569    for entry in std::fs::read_dir(dir).ok()?.flatten() {
570        let encoded = format!("-{}", encode_project_dir(Path::new(&entry.file_name())));
571        let Some(after) = rest.strip_prefix(encoded.as_str()) else {
572            continue;
573        };
574        if !after.is_empty() && !after.starts_with('-') {
575            continue;
576        }
577        let path = entry.path();
578        if !path.is_dir() {
579            continue;
580        }
581        if after.is_empty() {
582            exact.push(path);
583        } else {
584            partial.push((path, encoded.len()));
585        }
586    }
587    exact.sort();
588    if let Some(found) = exact.into_iter().next() {
589        return Some(found);
590    }
591    partial.sort();
592    partial
593        .into_iter()
594        .find_map(|(path, len)| find_slug(&path, &rest[len..], reads))
595}
596
597#[cfg(test)]
598mod tests {
599    use std::sync::Mutex;
600
601    use axum::extract::{Query, State};
602    use axum::http::HeaderMap;
603    use axum::routing::get;
604    use axum::{Json, Router};
605    use chrono::TimeZone;
606    use serde_json::{Value, json};
607
608    use super::*;
609
610    #[test]
611    fn the_token_comes_from_github_token_then_gh_token() {
612        fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
613            move |name| {
614                pairs
615                    .iter()
616                    .find(|(k, _)| *k == name)
617                    .map(|(_, v)| v.to_string())
618            }
619        }
620        assert_eq!(
621            token_from(env(&[("GITHUB_TOKEN", "first"), ("GH_TOKEN", "second")])),
622            Some("first".into())
623        );
624        assert_eq!(
625            token_from(env(&[("GH_TOKEN", " second ")])),
626            Some("second".into())
627        );
628        assert_eq!(
629            token_from(env(&[("GITHUB_TOKEN", "  "), ("GH_TOKEN", "second")])),
630            Some("second".into())
631        );
632        assert_eq!(
633            token_from(env(&[("GITHUB_TOKEN", ""), ("GH_TOKEN", " ")])),
634            None
635        );
636        assert_eq!(token_from(env(&[])), None);
637    }
638
639    fn newest() -> DateTime<Utc> {
640        Utc.with_ymd_and_hms(2026, 5, 1, 12, 0, 0).unwrap()
641    }
642
643    fn updated(i: usize) -> DateTime<Utc> {
644        newest() - chrono::Duration::minutes(i as i64)
645    }
646
647    fn pull(i: usize) -> Value {
648        let body = if i % 10 == 3 {
649            Value::Null
650        } else {
651            json!(format!("Change {i} to the widget cache."))
652        };
653        json!({
654            "number": 1000 - i,
655            "title": format!("Widget change {i}"),
656            "body": body,
657            "html_url": format!("https://github.com/example/widgets/pull/{}", 1000 - i),
658            "updated_at": updated(i).to_rfc3339(),
659            "state": "open",
660        })
661    }
662
663    #[derive(Default)]
664    struct Fake {
665        total: usize,
666        requests: Mutex<Vec<(HashMap<String, String>, HeaderMap)>>,
667    }
668
669    async fn pulls(
670        State(fake): State<Arc<Fake>>,
671        Query(query): Query<HashMap<String, String>>,
672        headers: HeaderMap,
673    ) -> Json<Value> {
674        let page: usize = query["page"].parse().unwrap();
675        fake.requests.lock().unwrap().push((query, headers));
676        let start = ((page - 1) * PER_PAGE).min(fake.total);
677        let end = (page * PER_PAGE).min(fake.total);
678        Json(Value::Array((start..end).map(pull).collect()))
679    }
680
681    struct Server {
682        runtime: tokio::runtime::Runtime,
683        fake: Arc<Fake>,
684        api: String,
685    }
686
687    fn server(total: usize) -> Server {
688        let runtime = tokio::runtime::Builder::new_multi_thread()
689            .worker_threads(1)
690            .enable_all()
691            .build()
692            .unwrap();
693        let fake = Arc::new(Fake {
694            total,
695            ..Fake::default()
696        });
697        let app = Router::new()
698            .route("/repos/example/widgets/pulls", get(pulls))
699            .with_state(fake.clone());
700        let listener = runtime
701            .block_on(tokio::net::TcpListener::bind("127.0.0.1:0"))
702            .unwrap();
703        let api = format!("http://{}", listener.local_addr().unwrap());
704        runtime.spawn(async move { axum::serve(listener, app).await.unwrap() });
705        Server { runtime, fake, api }
706    }
707
708    fn fetch(server: &Server, since: Option<DateTime<Utc>>) -> Vec<PullRequest> {
709        let token = ["test", "-token"].concat();
710        let github =
711            GitHub::new(token, server.api.clone(), server.runtime.handle().clone()).unwrap();
712        let repo = GitHubRepo {
713            owner: "example".into(),
714            name: "widgets".into(),
715        };
716        github.updated_since(&repo, since).unwrap()
717    }
718
719    fn requests(server: &Server) -> Vec<(HashMap<String, String>, HeaderMap)> {
720        std::mem::take(&mut *server.fake.requests.lock().unwrap())
721    }
722
723    #[test]
724    fn github_pages_until_a_short_page_and_skips_empty_bodies() {
725        let server = server(103);
726        let prs = fetch(&server, None);
727        let expected: Vec<usize> = (0..103).filter(|i| i % 10 != 3).collect();
728        assert_eq!(
729            prs.iter().map(|p| p.number).collect::<Vec<_>>(),
730            expected
731                .iter()
732                .map(|i| (1000 - i) as u64)
733                .collect::<Vec<_>>()
734        );
735        assert_eq!(prs[0].title, "Widget change 0");
736        assert_eq!(prs[0].body, "Change 0 to the widget cache.");
737        assert_eq!(prs[0].url, "https://github.com/example/widgets/pull/1000");
738        assert_eq!(prs[0].updated_at, newest());
739
740        let requests = requests(&server);
741        assert_eq!(requests.len(), 2);
742        for (n, (query, headers)) in requests.iter().enumerate() {
743            assert_eq!(query["state"], "all");
744            assert_eq!(query["sort"], "updated");
745            assert_eq!(query["direction"], "desc");
746            assert_eq!(query["per_page"], "100");
747            assert_eq!(query["page"], (n + 1).to_string());
748            assert_eq!(headers["authorization"], "Bearer test-token");
749            assert_eq!(headers["user-agent"], "second-brain");
750            assert_eq!(headers["accept"], "application/vnd.github+json");
751        }
752    }
753
754    #[test]
755    fn github_stops_at_the_first_pr_not_newer_than_since() {
756        let server = server(400);
757        let prs = fetch(&server, Some(updated(50)));
758        assert_eq!(prs.len(), (0..50).filter(|i| i % 10 != 3).count());
759        assert!(prs.iter().all(|p| p.updated_at > updated(50)));
760        assert_eq!(requests(&server).len(), 1);
761    }
762
763    #[test]
764    fn github_reads_at_most_pr_pages_pages() {
765        let server = server(PR_PAGES * PER_PAGE + 250);
766        let prs = fetch(&server, None);
767        assert_eq!(requests(&server).len(), PR_PAGES);
768        assert_eq!(
769            prs.last().unwrap().number,
770            (1000 - (PR_PAGES * PER_PAGE - 1)) as u64
771        );
772    }
773
774    #[test]
775    fn a_repo_the_token_cannot_see_has_no_pull_requests() {
776        let server = server(3);
777        let token = ["test", "-token"].concat();
778        let github =
779            GitHub::new(token, server.api.clone(), server.runtime.handle().clone()).unwrap();
780        let repo = GitHubRepo {
781            owner: "example".into(),
782            name: "private".into(),
783        };
784        assert!(github.updated_since(&repo, None).unwrap().is_empty());
785    }
786}