#![allow(private_interfaces)]
use std::path::PathBuf;
use std::sync::Arc;
use axum::body::Bytes;
use axum::extract::State;
use axum::http::{HeaderMap, StatusCode};
use axum::response::IntoResponse;
use axum::Json;
use hmac::{Hmac, Mac};
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use crate::config::Config;
use crate::storage::VectorStore;
use super::AppState;
#[derive(Debug, Clone, Copy, PartialEq)]
enum ForgeType {
GitHub,
GitLab,
Forgejo, Bitbucket,
}
impl ForgeType {
fn file_template(&self) -> &'static str {
match self {
ForgeType::GitHub => "/blob/main/{path}#L{line}",
ForgeType::GitLab => "/-/blob/main/{path}#L{line}",
ForgeType::Forgejo => "/src/branch/main/{path}#L{line}",
ForgeType::Bitbucket => "/src/main/{path}#{path}-{line}",
}
}
}
fn detect_forge(host: &str) -> ForgeType {
let h = host.to_lowercase();
if h == "github.com" || h.ends_with(".github.com") {
return ForgeType::GitHub;
}
if h == "gitlab.com" || h.ends_with(".gitlab.com") {
return ForgeType::GitLab;
}
if h == "bitbucket.org" || h.ends_with(".bitbucket.org") {
return ForgeType::Bitbucket;
}
if h.contains("gitlab") {
return ForgeType::GitLab;
}
if h.contains("bitbucket") {
return ForgeType::Bitbucket;
}
ForgeType::Forgejo
}
fn host_from_url(url: &str) -> Option<&str> {
let rest = url
.strip_prefix("https://")
.or_else(|| url.strip_prefix("http://"))?;
Some(rest.split('/').next()?.split(':').next()?)
}
fn web_base_from_remote(remote: &str) -> Option<String> {
let trimmed = remote.trim().trim_end_matches(".git");
if let Some(rest) = trimmed.strip_prefix("git@") {
let (host, path) = rest.split_once(':')?;
let path = path.trim_start_matches('/');
if path.split('/').count() < 2 {
return None;
}
return Some(format!("https://{host}/{path}"));
}
if trimmed.starts_with("https://") || trimmed.starts_with("http://") {
return Some(trimmed.to_string());
}
None
}
fn build_source_url(
base: &str,
remote_template: &str,
repo_name: &str,
forge_overrides: &std::collections::HashMap<String, String>,
) -> String {
if !remote_template.is_empty() {
return remote_template
.replace("{remote_base}", base)
.replace("{repo}", repo_name);
}
let forge = host_from_url(base)
.map(|host| {
if let Some(override_type) = forge_overrides.get(host) {
match override_type.to_lowercase().as_str() {
"github" => ForgeType::GitHub,
"gitlab" => ForgeType::GitLab,
"forgejo" | "gitea" => ForgeType::Forgejo,
"bitbucket" => ForgeType::Bitbucket,
_ => detect_forge(host),
}
} else {
detect_forge(host)
}
})
.unwrap_or(ForgeType::Forgejo);
format!("{}{}", base.trim_end_matches('/'), forge.file_template())
}
pub(crate) fn resolve_sources(
repo_root: &std::path::Path,
config_sources: &crate::config::SourcesConfig,
) -> crate::config::SourcesConfig {
let mut sources = config_sources.clone();
let repos_dir = repo_root.join("repos");
if let Ok(entries) = std::fs::read_dir(&repos_dir) {
for entry in entries.flatten() {
let repo_name = entry.file_name().to_string_lossy().to_string();
if sources.repos.contains_key(&repo_name) {
continue;
}
let repo_path = entry.path();
if let Ok(output) = std::process::Command::new("git")
.args([
"-C",
&repo_path.to_string_lossy(),
"remote",
"get-url",
"origin",
])
.output()
{
if output.status.success() {
let remote = String::from_utf8_lossy(&output.stdout).trim().to_string();
if let Some(base) = web_base_from_remote(&remote) {
let url = build_source_url(
&base,
&sources.remote_template,
&repo_name,
&sources.forge_overrides,
);
sources.repos.insert(repo_name, url);
}
} else {
sources.repos.insert(repo_name, String::new());
}
} else {
sources.repos.insert(repo_name, String::new());
}
}
}
sources
}
#[derive(Deserialize)]
pub(super) struct ForgejoPushPayload {
#[serde(rename = "ref")]
git_ref: Option<String>,
repository: Option<RepoInfo>,
}
#[derive(Deserialize)]
pub(super) struct RepoInfo {
full_name: Option<String>,
}
#[derive(Serialize)]
pub(super) struct WebhookResponse {
status: String,
message: String,
}
const SIGNATURE_HEADER: &str = "x-gitea-signature";
fn valid_signature(secret: &str, signature: Option<&str>, body: &[u8]) -> bool {
if secret.is_empty() {
return false;
}
let Some(signature) = signature.and_then(|value| hex::decode(value).ok()) else {
return false;
};
let Ok(mut mac) = Hmac::<Sha256>::new_from_slice(secret.as_bytes()) else {
return false;
};
mac.update(body);
mac.verify_slice(&signature).is_ok()
}
#[cfg(test)]
fn hmac_sha256(key: &[u8], body: &[u8]) -> [u8; 32] {
let mut mac = Hmac::<Sha256>::new_from_slice(key).expect("HMAC accepts keys of any size");
mac.update(body);
mac.finalize().into_bytes().into()
}
pub(super) async fn webhook_push(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
body: Bytes,
) -> axum::response::Response {
let signature = headers
.get(SIGNATURE_HEADER)
.and_then(|value| value.to_str().ok());
if !valid_signature(&state.config.archive.webhook_secret, signature, &body) {
tracing::warn!("Rejected push webhook with missing or invalid signature");
return (
StatusCode::UNAUTHORIZED,
Json(WebhookResponse {
status: "unauthorized".to_string(),
message: "Missing or invalid webhook signature".to_string(),
}),
)
.into_response();
}
let payload: ForgejoPushPayload = match serde_json::from_slice(&body) {
Ok(payload) => payload,
Err(error) => {
return (
StatusCode::BAD_REQUEST,
Json(WebhookResponse {
status: "invalid".to_string(),
message: format!("Invalid webhook payload: {error}"),
}),
)
.into_response();
}
};
let repo_name = payload
.repository
.as_ref()
.and_then(|r| r.full_name.as_deref())
.unwrap_or("unknown");
let git_ref = payload.git_ref.as_deref().unwrap_or("unknown");
tracing::info!(repo = repo_name, git_ref = git_ref, "Received push webhook");
let short_repo = repo_name
.rsplit('/')
.next()
.unwrap_or(repo_name)
.to_string();
let repos_dir = state.repo_root.join("repos");
let repo_dir = repos_dir.join(&short_repo);
if !repo_dir.exists() {
tracing::warn!(repo = short_repo, "Webhook for non-indexed repo, skipping");
return Json(WebhookResponse {
status: "skipped".to_string(),
message: format!("Repo '{}' not indexed", short_repo),
})
.into_response();
}
let pull_result = std::process::Command::new("git")
.args(["-C", &repo_dir.to_string_lossy(), "pull", "--ff-only"])
.output();
match pull_result {
Ok(output) if output.status.success() => {
tracing::info!(repo = short_repo, "Git pull succeeded");
}
Ok(output) => {
let stderr = String::from_utf8_lossy(&output.stderr);
tracing::warn!(repo = short_repo, error = %stderr, "Git pull failed");
}
Err(e) => {
tracing::warn!(repo = short_repo, error = %e, "Failed to run git pull");
}
}
let repo_root = state.repo_root.clone();
let config = state.config.clone();
let source_dir = repo_dir.clone();
let repo_tag = short_repo.clone();
tokio::spawn(async move {
if let Err(e) = run_incremental_index(repo_root, config, &source_dir, &repo_tag).await {
tracing::error!(repo = short_repo, error = %e, "Background re-index failed");
} else {
tracing::info!("Background re-index completed for {}", short_repo);
}
});
Json(WebhookResponse {
status: "accepted".to_string(),
message: format!("Re-index queued for push to {}", git_ref),
})
.into_response()
}
async fn run_incremental_index(
repo_root: PathBuf,
config: crate::config::Config,
source_dir: &std::path::Path,
repo_name: &str,
) -> anyhow::Result<()> {
use crate::index::{Embedder, Parser};
use ignore::WalkBuilder;
use sha2::{Digest, Sha256};
use std::path::Path;
let lance_path = Config::lance_path(&repo_root);
let mut vector_store = VectorStore::open(&lance_path).await?;
let model_dir = Config::model_cache_dir()?;
let embedder = Embedder::load(&model_dir, &config.embedding.model)?;
let mut parser = Parser::new()?.with_chunking(
config.index.chunk_size,
config.index.chunk_overlap,
embedder.max_seq().unwrap_or(0),
);
let mut walker = WalkBuilder::new(source_dir);
walker.hidden(true).git_ignore(config.index.use_gitignore);
let include_globs: Vec<glob::Pattern> = config
.index
.include
.iter()
.filter_map(|p| glob::Pattern::new(p).ok())
.collect();
let exclude_globs: Vec<glob::Pattern> = config
.index
.exclude
.iter()
.filter_map(|p| glob::Pattern::new(p).ok())
.collect();
let mut files_to_index = Vec::new();
for entry in walker.build().flatten() {
if !entry.file_type().is_some_and(|ft| ft.is_file()) {
continue;
}
let path = entry.path();
let rel_path = path
.strip_prefix(source_dir)
.unwrap_or(path)
.to_string_lossy()
.to_string();
let matches_include =
include_globs.is_empty() || include_globs.iter().any(|g| g.matches(&rel_path));
let matches_exclude = exclude_globs.iter().any(|g| g.matches(&rel_path));
if matches_include && !matches_exclude {
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(_) => continue,
};
let hash = format!("{:x}", Sha256::digest(content.as_bytes()));
let needs = vector_store
.needs_reindex(&rel_path, &hash, Some(repo_name))
.await
.unwrap_or(true);
if needs {
files_to_index.push((rel_path, content, hash));
}
}
}
tracing::info!(
"Webhook index: {} files need re-indexing",
files_to_index.len()
);
for (rel_path, content, hash) in &files_to_index {
let chunks = match parser.parse_file(Path::new(rel_path), content) {
Ok(c) => c,
Err(e) => {
tracing::warn!("Failed to parse {}: {}", rel_path, e);
continue;
}
};
if chunks.is_empty() {
continue;
}
vector_store
.delete_by_file(&[rel_path.clone()], Some(repo_name))
.await?;
let texts: Vec<&str> = chunks.iter().map(|c| c.content.as_str()).collect();
let embeddings = embedder.embed_batch(&texts).await?;
let contexts: Vec<Option<String>> = vec![None; chunks.len()];
let now = chrono::Utc::now().to_rfc3339();
vector_store
.insert(&chunks, &embeddings, &contexts, repo_name, hash, &now)
.await?;
}
Ok(())
}
#[cfg(test)]
#[path = "webhook_tests.rs"]
mod tests;