Skip to main content

uqa_engine/
migration.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Migration support for legacy UQA `SQLite` catalog directories.
8
9use std::collections::{BTreeMap, BTreeSet};
10use std::fs;
11use std::path::{Path, PathBuf};
12
13use rusqlite::types::ValueRef;
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde::Deserialize;
16use uqa_analysis::analyzer::standard_analyzer;
17use uqa_core::{DocId, Edge, TemporalValue, Value, Vertex};
18use uqa_sql::ast::{ColumnDef, ColumnType, Expr};
19use uqa_storage_sqlite::ColumnStatsInput;
20
21use crate::{Engine, HNSWIndexParams, IVFIndexParams, VectorIndexSpec};
22use uqa_sql::assignment::conversion::convert_value_to_column_type;
23
24#[derive(Debug, Clone, PartialEq, Eq)]
25pub struct PythonMigrationReport {
26    pub source_path: PathBuf,
27    pub destination_path: PathBuf,
28    pub tables: usize,
29    pub documents: usize,
30    pub fts_fields: usize,
31    pub vector_fields: usize,
32    pub indexes: usize,
33    pub analyzers: usize,
34    pub table_field_analyzers: usize,
35    pub foreign_servers: usize,
36    pub foreign_tables: usize,
37    pub graphs: usize,
38    pub graph_vertices: usize,
39    pub graph_edges: usize,
40    pub path_indexes: usize,
41    pub scoring_params: usize,
42    pub models: usize,
43    pub column_stats: usize,
44}
45
46#[derive(Debug, thiserror::Error)]
47pub enum PythonMigrationError {
48    #[error("source path does not exist: {0}")]
49    SourceMissing(PathBuf),
50    #[error("source directory contains no Python UQA SQLite catalog: {0}")]
51    SourceCatalogMissing(PathBuf),
52    #[error("source directory contains multiple Python UQA SQLite catalogs: {0}")]
53    MultipleSourceCatalogs(String),
54    #[error("source is not a Python UQA SQLite catalog: {0}")]
55    NotPythonCatalog(PathBuf),
56    #[error("source and destination must be different paths: {0}")]
57    SameSourceAndDestination(PathBuf),
58    #[error("destination database is not empty: {0}")]
59    DestinationNotEmpty(String),
60    #[error("destination exists but is not an empty uqa-engine catalog: {0}")]
61    DestinationNotEmptyCatalog(PathBuf),
62    #[error("sqlite error: {0}")]
63    SQLite(#[from] rusqlite::Error),
64    #[error("storage error: {0}")]
65    Storage(#[from] uqa_storage_sqlite::SQLiteError),
66    #[error("storage backend error: {0}")]
67    StorageBackend(#[from] uqa_storage::StorageBackendError),
68    #[error("SQL error: {0}")]
69    SQL(#[from] uqa_sql::SQLError),
70    #[error("JSON error: {0}")]
71    JSON(#[from] serde_json::Error),
72    #[error("io error: {0}")]
73    Io(#[from] std::io::Error),
74    #[error("migration error: {0}")]
75    Invalid(String),
76}
77
78#[derive(Debug, Clone)]
79struct TableSpec {
80    name: String,
81    columns: Vec<PythonColumnDef>,
82    rust_columns: Vec<ColumnDef>,
83    fts_fields: Vec<String>,
84    vector_fields: Vec<VectorSpec>,
85}
86
87#[derive(Debug, Clone)]
88struct VectorSpec {
89    field: String,
90    dimensions: u32,
91    index: VectorIndexSpec,
92}
93
94#[derive(Debug, Clone)]
95struct CatalogIndex {
96    name: String,
97    index_type: String,
98    table_name: String,
99    columns: Vec<String>,
100    parameters: BTreeMap<String, String>,
101}
102
103type MigratedDocument = (DocId, BTreeMap<String, Value>);
104
105#[derive(Debug, Clone, Deserialize)]
106#[allow(clippy::struct_excessive_bools)]
107struct PythonColumnDef {
108    name: String,
109    type_name: String,
110    #[serde(default)]
111    primary_key: bool,
112    #[serde(default)]
113    not_null: bool,
114    #[serde(default)]
115    auto_increment: bool,
116    #[serde(default)]
117    default: Option<serde_json::Value>,
118    #[serde(default)]
119    vector_dimensions: Option<u32>,
120    #[serde(default)]
121    unique: bool,
122    #[serde(default)]
123    numeric_precision: Option<u32>,
124    #[serde(default)]
125    numeric_scale: Option<u32>,
126}
127
128mod documents;
129mod graphs;
130mod metadata;
131mod run;
132mod schema;
133mod source;
134mod table_setup;
135mod values;
136mod vectors;
137
138pub use run::migrate_python_database;
139
140use documents::migrate_documents;
141use graphs::migrate_graphs;
142use metadata::{
143    migrate_column_stats, migrate_foreign_servers, migrate_foreign_tables, migrate_models,
144    migrate_path_indexes, migrate_scoring_params, persist_catalog_indexes,
145};
146use schema::{column_to_rust, load_catalog_indexes, load_table_specs, python_temporal_type};
147use source::{
148    ensure_destination_empty, is_python_catalog, open_read_only, reject_same_path,
149    resolve_source_database, table_exists,
150};
151use table_setup::{
152    create_tables, install_secondary_indexes, migrate_analyzers, migrate_table_field_analyzers,
153};
154use values::{
155    json_object_to_pairs, json_object_to_value_map, json_to_value, parameters_to_string_map,
156    quote_ident, sqlite_value_to_uqa, table_columns,
157};
158use vectors::{blob_to_f32_vec, extract_vectors, read_vector_fallbacks, vector_value};
159
160#[cfg(test)]
161use documents::coerce_migrated_document;
162#[cfg(test)]
163use schema::{infer_vector_fields, rust_column_type};
164#[cfg(test)]
165use vectors::value_to_f32_vec;
166
167#[cfg(test)]
168mod tests;