use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use uuid::Uuid;
use second_brain_adapters::git::{self, Checkout, GitHubRepo, PullRequest};
use second_brain_adapters::{
IngestResult, StoreOutcome, encode_project_dir, repo_instructions, store_document_result,
};
use second_brain_core::embedding::DocumentEmbedder;
use second_brain_core::redact::redact_memory;
use second_brain_core::schema::{IngestRecord, Memory, Relation, RelationType, Tier};
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::{Store, StoreAdmin};
use crate::bootstrap::Bootstrap;
pub const INTERVAL: Duration = Duration::from_secs(600);
pub const FIRST_SIGHT_COMMITS: usize = 500;
pub const PR_PAGES: usize = 5;
const PER_PAGE: usize = 100;
const FIRST_PASS_DELAY: Duration = Duration::from_secs(60);
const SLUG_WALK_READS: usize = 10_000;
const GITHUB_API: &str = "https://api.github.com";
const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
pub trait PullRequests: Send + Sync {
fn updated_since(
&self,
repo: &GitHubRepo,
since: Option<DateTime<Utc>>,
) -> Result<Vec<PullRequest>>;
}
pub struct GitHub {
token: String,
api: String,
client: reqwest::Client,
handle: tokio::runtime::Handle,
}
impl GitHub {
pub fn from_env(handle: tokio::runtime::Handle) -> Option<GitHub> {
let token = token_from(|name| std::env::var(name).ok())?;
GitHub::new(token, GITHUB_API.to_string(), handle)
}
fn new(token: String, api: String, handle: tokio::runtime::Handle) -> Option<GitHub> {
match reqwest::Client::builder()
.timeout(HTTP_TIMEOUT)
.user_agent("second-brain")
.build()
{
Ok(client) => Some(GitHub {
token,
api,
client,
handle,
}),
Err(e) => {
tracing::warn!("sources: pull requests disabled, cannot build an HTTP client: {e}");
None
}
}
}
async fn fetch(
&self,
repo: &GitHubRepo,
since: Option<DateTime<Utc>>,
) -> Result<Vec<PullRequest>> {
let mut prs = Vec::new();
for page in 1..=PR_PAGES {
let items = self.page(repo, page).await?;
let last_page = items.len() < PER_PAGE;
if take_newer(items, since, &mut prs) || last_page {
break;
}
}
Ok(prs)
}
async fn page(&self, repo: &GitHubRepo, page: usize) -> Result<Vec<ApiPull>> {
let url = format!(
"{}/repos/{}/{}/pulls?state=all&sort=updated&direction=desc&per_page={PER_PAGE}&page={page}",
self.api, repo.owner, repo.name
);
let response = self
.client
.get(&url)
.bearer_auth(&self.token)
.header(reqwest::header::ACCEPT, "application/vnd.github+json")
.send()
.await
.with_context(|| format!("GET {url}"))?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
tracing::debug!("sources: {url} is not visible to the token");
return Ok(Vec::new());
}
let response = response
.error_for_status()
.with_context(|| format!("GET {url}"))?;
response
.json()
.await
.with_context(|| format!("decode {url}"))
}
}
impl PullRequests for GitHub {
fn updated_since(
&self,
repo: &GitHubRepo,
since: Option<DateTime<Utc>>,
) -> Result<Vec<PullRequest>> {
self.handle.block_on(self.fetch(repo, since))
}
}
fn token_from(var: impl Fn(&str) -> Option<String>) -> Option<String> {
["GITHUB_TOKEN", "GH_TOKEN"]
.into_iter()
.filter_map(var)
.map(|token| token.trim().to_string())
.find(|token| !token.is_empty())
}
#[derive(Deserialize)]
struct ApiPull {
number: u64,
title: String,
#[serde(default)]
body: Option<String>,
html_url: String,
updated_at: DateTime<Utc>,
}
fn take_newer(
items: Vec<ApiPull>,
since: Option<DateTime<Utc>>,
out: &mut Vec<PullRequest>,
) -> bool {
for item in items {
if since.is_some_and(|since| item.updated_at <= since) {
return true;
}
let body = item.body.unwrap_or_default();
if body.trim().is_empty() {
continue;
}
out.push(PullRequest {
number: item.number,
title: item.title,
body,
url: item.html_url,
updated_at: item.updated_at,
});
}
false
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct Pass {
pub repos: usize,
pub stored: usize,
pub superseded: usize,
pub errors: usize,
}
impl Pass {
fn absorb(&mut self, outcome: &StoreOutcome) {
self.stored += outcome.stored;
self.errors += outcome.errors;
}
}
pub struct Sources {
pub store: Arc<SqliteStore>,
pub embedder: Option<Arc<dyn DocumentEmbedder>>,
pub machine_id: String,
pub pull_requests: Option<Arc<dyn PullRequests>>,
}
impl Sources {
pub fn repos(&self) -> Vec<Checkout> {
let slugs = match self.store.distinct_project_paths() {
Ok(slugs) => slugs,
Err(e) => {
tracing::warn!("sources: cannot list project slugs: {e:#}");
return Vec::new();
}
};
let mut seen = HashSet::new();
let mut repos = Vec::new();
for (slug, _) in slugs {
let Some(checkout) = dir_for_slug(&slug).and_then(|dir| Checkout::containing(&dir))
else {
continue;
};
if seen.insert(checkout.root().to_path_buf()) {
repos.push(checkout);
}
}
repos
}
pub fn run_once(&self) -> Pass {
self.run_checkouts(&self.repos())
}
pub fn run_unless_pending(&self, bootstrap: &Bootstrap) -> Option<Pass> {
(!bootstrap.is_pending()).then(|| self.run_once())
}
pub fn run_checkouts(&self, checkouts: &[Checkout]) -> Pass {
let mut pass = Pass::default();
for checkout in checkouts {
pass.repos += 1;
let root = checkout.root().display();
if let Err(e) = self.commits(checkout, &mut pass) {
tracing::warn!("sources: commits of {root}: {e:#}");
pass.errors += 1;
}
if let Err(e) = self.pull_requests(checkout, &mut pass) {
tracing::warn!("sources: pull requests of {root}: {e:#}");
pass.errors += 1;
}
for path in repo_instructions::instruction_files(checkout) {
if let Err(e) = self.instructions(checkout, &path, &mut pass) {
tracing::warn!("sources: {}: {e:#}", path.display());
pass.errors += 1;
}
}
}
pass
}
pub fn spawn(self, bootstrap: Arc<Bootstrap>) {
let spawned = std::thread::Builder::new()
.name("sources".into())
.spawn(move || {
std::thread::sleep(FIRST_PASS_DELAY);
loop {
if let Some(pass) = self.run_unless_pending(&bootstrap)
&& (pass.stored > 0 || pass.superseded > 0 || pass.errors > 0)
{
tracing::info!(
"sources: {} repos, {} stored, {} superseded, {} errors",
pass.repos,
pass.stored,
pass.superseded,
pass.errors
);
}
std::thread::sleep(INTERVAL);
}
});
if let Err(e) = spawned {
tracing::error!("cannot start the sources thread: {e}");
}
}
fn commits(&self, checkout: &Checkout, pass: &mut Pass) -> Result<()> {
let Some(head) = checkout.head()? else {
return Ok(());
};
let key = format!("{}:{}", git::COMMIT_SOURCE, checkout.root().display());
let recorded = self.store.ingest_log_entry(&key)?.map(|e| e.file_hash);
if recorded.as_deref() == Some(head.as_str()) {
return Ok(());
}
let mut written = StoreOutcome::default();
for commit in checkout.commits_since(recorded.as_deref(), FIRST_SIGHT_COMMITS)? {
let outcome =
self.store_document(&git::commit_result(checkout, &commit, &self.machine_id));
written.stored += outcome.stored;
written.errors += outcome.errors;
}
pass.absorb(&written);
if written.errors == 0 {
self.record(&key, &head, written.stored, git::COMMIT_SOURCE)?;
}
Ok(())
}
fn pull_requests(&self, checkout: &Checkout, pass: &mut Pass) -> Result<()> {
let Some(api) = &self.pull_requests else {
return Ok(());
};
let Some(repo) = checkout.github_remote() else {
return Ok(());
};
let key = format!("{}:{}/{}", git::PR_SOURCE, repo.owner, repo.name);
let since = self
.store
.ingest_log_entry(&key)?
.and_then(|entry| DateTime::parse_from_rfc3339(&entry.file_hash).ok())
.map(|at| at.with_timezone(&Utc));
let prs = api.updated_since(&repo, since)?;
let Some(newest) = prs.iter().map(|pr| pr.updated_at).max() else {
return Ok(());
};
let mut versions = Versions::load(&self.store, git::PR_SOURCE)?;
let (mut stored, mut errors) = (0, 0);
for pr in &prs {
let version = self.store_version(&mut versions, pass, |after| {
Some(git::pull_request_result(
checkout,
&repo,
pr,
&self.machine_id,
after,
))
});
stored += version.stored;
errors += version.errors;
}
if errors == 0 {
self.record(&key, &newest.to_rfc3339(), stored, git::PR_SOURCE)?;
}
Ok(())
}
fn instructions(&self, checkout: &Checkout, path: &Path, pass: &mut Pass) -> Result<()> {
let key = path.to_string_lossy();
let bytes = std::fs::read(path).with_context(|| format!("read {}", path.display()))?;
let hash = sha256_hex(&bytes);
if !self.store.is_file_changed(&key, &hash)? {
return Ok(());
}
let mut versions = Versions::load(&self.store, repo_instructions::SOURCE)?;
let version = self.store_version(&mut versions, pass, |after| {
repo_instructions::instructions_result(checkout, path, &self.machine_id, after)
});
if version.errors > 0 {
return Ok(());
}
self.record(&key, &hash, version.stored, repo_instructions::SOURCE)
}
fn store_version(
&self,
versions: &mut Versions,
pass: &mut Pass,
build: impl Fn(Option<Uuid>) -> Option<IngestResult>,
) -> StoreOutcome {
let mut version = StoreOutcome::default();
let Some(first) = build(None) else {
return version;
};
let Some(source_id) = first.memories.first().map(|m| m.source_id.clone()) else {
return version;
};
let mut candidate = first;
let mut after = versions.newest(&source_id);
for _ in 0..=versions.all(&source_id).len() {
let Some(previous) = after else {
break;
};
let Some(result) = build(Some(previous)) else {
return version;
};
candidate = result;
after = candidate
.memories
.first()
.and_then(|m| versions.retired_or_superseded(&source_id, m.id));
}
let Some(mut text) = candidate.memories.first().cloned() else {
return version;
};
redact_memory(&mut text);
let live = versions.live(&source_id);
let current = match versions.holding(&source_id, &text.content) {
Some(existing) => existing,
None => {
version = self.store_document(&candidate);
pass.absorb(&version);
if version.stored == 0 {
return version;
}
versions.add(text.clone());
text
}
};
for old in live.iter().filter(|m| m.id != current.id) {
match self.supersede(old.id, current.id) {
Ok(()) => {
pass.superseded += 1;
versions.superseded(old.id, current.id);
}
Err(e) => {
tracing::warn!(
"sources: cannot supersede {} by {}: {e:#}",
old.id,
current.id
);
version.errors += 1;
pass.errors += 1;
}
}
}
version
}
fn store_document(&self, result: &IngestResult) -> StoreOutcome {
store_document_result(
self.store.as_ref(),
self.embedder.as_deref(),
result,
&self.machine_id,
)
}
fn record(&self, key: &str, hash: &str, memory_count: usize, source: &str) -> Result<()> {
self.store.record_ingested(&IngestRecord {
file_path: key,
content_hash: hash,
session_key: None,
legacy_ids: false,
memory_count,
source,
})
}
fn supersede(&self, old: Uuid, new: Uuid) -> Result<()> {
self.store.store_relation(&Relation {
from_id: new,
to_id: old,
relation_type: RelationType::Supersedes,
strength: 1.0,
context: None,
})?;
self.store.supersede_memory(old, new)
}
}
struct Versions(HashMap<String, Vec<Memory>>);
impl Versions {
fn load(store: &SqliteStore, source: &str) -> Result<Versions> {
let mut by_source_id: HashMap<String, Vec<Memory>> = HashMap::new();
for memory in store.memories_by_source(source)? {
by_source_id
.entry(memory.source_id.clone())
.or_default()
.push(memory);
}
Ok(Versions(by_source_id))
}
fn all(&self, source_id: &str) -> &[Memory] {
self.0.get(source_id).map(Vec::as_slice).unwrap_or_default()
}
fn newest(&self, source_id: &str) -> Option<Uuid> {
self.all(source_id)
.iter()
.max_by_key(|m| (m.created_at, m.id))
.map(|m| m.id)
}
fn live(&self, source_id: &str) -> Vec<Memory> {
self.all(source_id)
.iter()
.filter(|m| is_live(m))
.cloned()
.collect()
}
fn holding(&self, source_id: &str, content: &str) -> Option<Memory> {
let same = |m: &&Memory| m.content == content;
let newest = |a: &&Memory, b: &&Memory| (a.created_at, a.id).cmp(&(b.created_at, b.id));
let versions = self.all(source_id);
versions
.iter()
.filter(|m| is_live(m))
.filter(same)
.max_by(newest)
.or_else(|| {
versions
.iter()
.filter(|m| m.retired_reason.is_some())
.filter(same)
.max_by(newest)
})
.cloned()
}
fn add(&mut self, memory: Memory) {
self.0
.entry(memory.source_id.clone())
.or_default()
.push(memory);
}
fn retired_or_superseded(&self, source_id: &str, id: Uuid) -> Option<Uuid> {
self.all(source_id)
.iter()
.any(|m| m.id == id && !is_live(m))
.then_some(id)
}
fn superseded(&mut self, old: Uuid, by: Uuid) {
if let Some(memory) = self.0.values_mut().flatten().find(|m| m.id == old) {
memory.superseded_by = Some(by);
}
}
}
fn is_live(memory: &Memory) -> bool {
memory.tier == Tier::Hot && memory.superseded_by.is_none()
}
fn sha256_hex(bytes: &[u8]) -> String {
format!("{:x}", Sha256::digest(bytes))
}
pub fn dir_for_slug(slug: &str) -> Option<PathBuf> {
dir_for_slug_under(Path::new("/"), slug)
}
pub fn dir_for_slug_under(root: &Path, slug: &str) -> Option<PathBuf> {
if slug.is_empty() {
return None;
}
let slug = if slug.starts_with('-') {
Cow::Borrowed(slug)
} else {
Cow::Owned(format!("-{slug}"))
};
let mut reads = 0;
find_slug(root, &slug, &mut reads)
}
fn find_slug(dir: &Path, rest: &str, reads: &mut usize) -> Option<PathBuf> {
if *reads >= SLUG_WALK_READS {
return None;
}
*reads += 1;
let mut exact = Vec::new();
let mut partial = Vec::new();
for entry in std::fs::read_dir(dir).ok()?.flatten() {
let encoded = format!("-{}", encode_project_dir(Path::new(&entry.file_name())));
let Some(after) = rest.strip_prefix(encoded.as_str()) else {
continue;
};
if !after.is_empty() && !after.starts_with('-') {
continue;
}
let path = entry.path();
if !path.is_dir() {
continue;
}
if after.is_empty() {
exact.push(path);
} else {
partial.push((path, encoded.len()));
}
}
exact.sort();
if let Some(found) = exact.into_iter().next() {
return Some(found);
}
partial.sort();
partial
.into_iter()
.find_map(|(path, len)| find_slug(&path, &rest[len..], reads))
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use axum::extract::{Query, State};
use axum::http::HeaderMap;
use axum::routing::get;
use axum::{Json, Router};
use chrono::TimeZone;
use serde_json::{Value, json};
use super::*;
#[test]
fn the_token_comes_from_github_token_then_gh_token() {
fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
move |name| {
pairs
.iter()
.find(|(k, _)| *k == name)
.map(|(_, v)| v.to_string())
}
}
assert_eq!(
token_from(env(&[("GITHUB_TOKEN", "first"), ("GH_TOKEN", "second")])),
Some("first".into())
);
assert_eq!(
token_from(env(&[("GH_TOKEN", " second ")])),
Some("second".into())
);
assert_eq!(
token_from(env(&[("GITHUB_TOKEN", " "), ("GH_TOKEN", "second")])),
Some("second".into())
);
assert_eq!(
token_from(env(&[("GITHUB_TOKEN", ""), ("GH_TOKEN", " ")])),
None
);
assert_eq!(token_from(env(&[])), None);
}
fn newest() -> DateTime<Utc> {
Utc.with_ymd_and_hms(2026, 5, 1, 12, 0, 0).unwrap()
}
fn updated(i: usize) -> DateTime<Utc> {
newest() - chrono::Duration::minutes(i as i64)
}
fn pull(i: usize) -> Value {
let body = if i % 10 == 3 {
Value::Null
} else {
json!(format!("Change {i} to the widget cache."))
};
json!({
"number": 1000 - i,
"title": format!("Widget change {i}"),
"body": body,
"html_url": format!("https://github.com/example/widgets/pull/{}", 1000 - i),
"updated_at": updated(i).to_rfc3339(),
"state": "open",
})
}
#[derive(Default)]
struct Fake {
total: usize,
requests: Mutex<Vec<(HashMap<String, String>, HeaderMap)>>,
}
async fn pulls(
State(fake): State<Arc<Fake>>,
Query(query): Query<HashMap<String, String>>,
headers: HeaderMap,
) -> Json<Value> {
let page: usize = query["page"].parse().unwrap();
fake.requests.lock().unwrap().push((query, headers));
let start = ((page - 1) * PER_PAGE).min(fake.total);
let end = (page * PER_PAGE).min(fake.total);
Json(Value::Array((start..end).map(pull).collect()))
}
struct Server {
runtime: tokio::runtime::Runtime,
fake: Arc<Fake>,
api: String,
}
fn server(total: usize) -> Server {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
.unwrap();
let fake = Arc::new(Fake {
total,
..Fake::default()
});
let app = Router::new()
.route("/repos/example/widgets/pulls", get(pulls))
.with_state(fake.clone());
let listener = runtime
.block_on(tokio::net::TcpListener::bind("127.0.0.1:0"))
.unwrap();
let api = format!("http://{}", listener.local_addr().unwrap());
runtime.spawn(async move { axum::serve(listener, app).await.unwrap() });
Server { runtime, fake, api }
}
fn fetch(server: &Server, since: Option<DateTime<Utc>>) -> Vec<PullRequest> {
let token = ["test", "-token"].concat();
let github =
GitHub::new(token, server.api.clone(), server.runtime.handle().clone()).unwrap();
let repo = GitHubRepo {
owner: "example".into(),
name: "widgets".into(),
};
github.updated_since(&repo, since).unwrap()
}
fn requests(server: &Server) -> Vec<(HashMap<String, String>, HeaderMap)> {
std::mem::take(&mut *server.fake.requests.lock().unwrap())
}
#[test]
fn github_pages_until_a_short_page_and_skips_empty_bodies() {
let server = server(103);
let prs = fetch(&server, None);
let expected: Vec<usize> = (0..103).filter(|i| i % 10 != 3).collect();
assert_eq!(
prs.iter().map(|p| p.number).collect::<Vec<_>>(),
expected
.iter()
.map(|i| (1000 - i) as u64)
.collect::<Vec<_>>()
);
assert_eq!(prs[0].title, "Widget change 0");
assert_eq!(prs[0].body, "Change 0 to the widget cache.");
assert_eq!(prs[0].url, "https://github.com/example/widgets/pull/1000");
assert_eq!(prs[0].updated_at, newest());
let requests = requests(&server);
assert_eq!(requests.len(), 2);
for (n, (query, headers)) in requests.iter().enumerate() {
assert_eq!(query["state"], "all");
assert_eq!(query["sort"], "updated");
assert_eq!(query["direction"], "desc");
assert_eq!(query["per_page"], "100");
assert_eq!(query["page"], (n + 1).to_string());
assert_eq!(headers["authorization"], "Bearer test-token");
assert_eq!(headers["user-agent"], "second-brain");
assert_eq!(headers["accept"], "application/vnd.github+json");
}
}
#[test]
fn github_stops_at_the_first_pr_not_newer_than_since() {
let server = server(400);
let prs = fetch(&server, Some(updated(50)));
assert_eq!(prs.len(), (0..50).filter(|i| i % 10 != 3).count());
assert!(prs.iter().all(|p| p.updated_at > updated(50)));
assert_eq!(requests(&server).len(), 1);
}
#[test]
fn github_reads_at_most_pr_pages_pages() {
let server = server(PR_PAGES * PER_PAGE + 250);
let prs = fetch(&server, None);
assert_eq!(requests(&server).len(), PR_PAGES);
assert_eq!(
prs.last().unwrap().number,
(1000 - (PR_PAGES * PER_PAGE - 1)) as u64
);
}
#[test]
fn a_repo_the_token_cannot_see_has_no_pull_requests() {
let server = server(3);
let token = ["test", "-token"].concat();
let github =
GitHub::new(token, server.api.clone(), server.runtime.handle().clone()).unwrap();
let repo = GitHubRepo {
owner: "example".into(),
name: "private".into(),
};
assert!(github.updated_since(&repo, None).unwrap().is_empty());
}
}