webfind 0.5.0

Self-hosted native search engine for AI agents — pure CLI, zero cost
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

//! `webfind migrate` — migrate a legacy SurrealDB JSON export into a fresh
//! Turso database (FR-7).
//!
//! Reads a JSON export of `url_nodes` + `link_edges` (+ optional `page_content`
//! and `crawl_jobs`) and writes a Turso/libSQL database. Re-derives every URL ID
//! via the shared BLAKE3 `url_id()` utility, validates embedding dimensions, and
//! rebuilds the FTS index + PageRank so the destination is queryable immediately.

use std::path::PathBuf;

use anyhow::Context;
use webfind::storage::migrate::migrate_graph;

/// Run the `migrate` subcommand.
///
/// `from` is the path to the JSON export. `to` is the destination Turso file;
/// it is created if absent and overwritten if present.
pub async fn run(
    cfg: &webfind::config::WebfindConfig,
    from: PathBuf,
    to: PathBuf,
) -> anyhow::Result<()> {
    // Allow --to to be relative to the configured data dir when it is not an
    // absolute path — matches the ergonomics of other Turso-backed commands.
    let to = resolve_dest(cfg, to);
    let to_str = to
        .to_str()
        .with_context(|| format!("destination path is not valid UTF-8: {}", to.display()))?;

    if !from.exists() {
        anyhow::bail!(
            "export file not found: {} — run `surrealdb export` or supply a valid path",
            from.display()
        );
    }

    println!(
        "Migrating\n  from: {}\n  to:   {}",
        from.display(),
        to.display()
    );

    let report = migrate_graph(&from, to_str)
        .await
        .context("migration failed")?;

    println!("\nMigration complete:");
    println!("  url_nodes:     {}", report.url_nodes_imported);
    println!("  link_edges:    {}", report.link_edges_imported);
    println!("  page_content:  {}", report.page_content_imported);
    println!("  crawl_jobs:    {}", report.crawl_jobs_imported);
    if let Some(dim) = report.embedding_dimension {
        println!("  embedding_dim: {} (validated)", dim);
    } else {
        println!("  embedding_dim: none");
    }
    Ok(())
}

/// Resolve the destination path: absolute paths are used as-is; otherwise the
/// path is joined to the configured data directory.
fn resolve_dest(_cfg: &webfind::config::WebfindConfig, to: PathBuf) -> PathBuf {
    if to.is_absolute() {
        return to;
    }
    webfind::config::data_dir().join(to)
}