1use std::collections::{BTreeMap, BTreeSet};
10use std::fs;
11use std::path::{Path, PathBuf};
12
13use rusqlite::types::ValueRef;
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde::Deserialize;
16use uqa_analysis::analyzer::standard_analyzer;
17use uqa_core::{DocId, Edge, TemporalValue, Value, Vertex};
18use uqa_sql::ast::{ColumnDef, ColumnType, Expr};
19use uqa_storage_sqlite::ColumnStatsInput;
20
21use crate::{Engine, HNSWIndexParams, IVFIndexParams, VectorIndexSpec};
22use uqa_sql::assignment::conversion::convert_value_to_column_type;
23
24#[derive(Debug, Clone, PartialEq, Eq)]
25pub struct PythonMigrationReport {
26 pub source_path: PathBuf,
27 pub destination_path: PathBuf,
28 pub tables: usize,
29 pub documents: usize,
30 pub fts_fields: usize,
31 pub vector_fields: usize,
32 pub indexes: usize,
33 pub analyzers: usize,
34 pub table_field_analyzers: usize,
35 pub foreign_servers: usize,
36 pub foreign_tables: usize,
37 pub graphs: usize,
38 pub graph_vertices: usize,
39 pub graph_edges: usize,
40 pub path_indexes: usize,
41 pub scoring_params: usize,
42 pub models: usize,
43 pub column_stats: usize,
44}
45
46#[derive(Debug, thiserror::Error)]
47pub enum PythonMigrationError {
48 #[error("source path does not exist: {0}")]
49 SourceMissing(PathBuf),
50 #[error("source directory contains no Python UQA SQLite catalog: {0}")]
51 SourceCatalogMissing(PathBuf),
52 #[error("source directory contains multiple Python UQA SQLite catalogs: {0}")]
53 MultipleSourceCatalogs(String),
54 #[error("source is not a Python UQA SQLite catalog: {0}")]
55 NotPythonCatalog(PathBuf),
56 #[error("source and destination must be different paths: {0}")]
57 SameSourceAndDestination(PathBuf),
58 #[error("destination database is not empty: {0}")]
59 DestinationNotEmpty(String),
60 #[error("destination exists but is not an empty uqa-engine catalog: {0}")]
61 DestinationNotEmptyCatalog(PathBuf),
62 #[error("sqlite error: {0}")]
63 SQLite(#[from] rusqlite::Error),
64 #[error("storage error: {0}")]
65 Storage(#[from] uqa_storage_sqlite::SQLiteError),
66 #[error("storage backend error: {0}")]
67 StorageBackend(#[from] uqa_storage::StorageBackendError),
68 #[error("SQL error: {0}")]
69 SQL(#[from] uqa_sql::SQLError),
70 #[error("JSON error: {0}")]
71 JSON(#[from] serde_json::Error),
72 #[error("io error: {0}")]
73 Io(#[from] std::io::Error),
74 #[error("migration error: {0}")]
75 Invalid(String),
76}
77
78#[derive(Debug, Clone)]
79struct TableSpec {
80 name: String,
81 columns: Vec<PythonColumnDef>,
82 rust_columns: Vec<ColumnDef>,
83 fts_fields: Vec<String>,
84 vector_fields: Vec<VectorSpec>,
85}
86
87#[derive(Debug, Clone)]
88struct VectorSpec {
89 field: String,
90 dimensions: u32,
91 index: VectorIndexSpec,
92}
93
94#[derive(Debug, Clone)]
95struct CatalogIndex {
96 name: String,
97 index_type: String,
98 table_name: String,
99 columns: Vec<String>,
100 parameters: BTreeMap<String, String>,
101}
102
103type MigratedDocument = (DocId, BTreeMap<String, Value>);
104
105#[derive(Debug, Clone, Deserialize)]
106#[allow(clippy::struct_excessive_bools)]
107struct PythonColumnDef {
108 name: String,
109 type_name: String,
110 #[serde(default)]
111 primary_key: bool,
112 #[serde(default)]
113 not_null: bool,
114 #[serde(default)]
115 auto_increment: bool,
116 #[serde(default)]
117 default: Option<serde_json::Value>,
118 #[serde(default)]
119 vector_dimensions: Option<u32>,
120 #[serde(default)]
121 unique: bool,
122 #[serde(default)]
123 numeric_precision: Option<u32>,
124 #[serde(default)]
125 numeric_scale: Option<u32>,
126}
127
128mod documents;
129mod graphs;
130mod metadata;
131mod run;
132mod schema;
133mod source;
134mod table_setup;
135mod values;
136mod vectors;
137
138pub use run::migrate_python_database;
139
140use documents::migrate_documents;
141use graphs::migrate_graphs;
142use metadata::{
143 migrate_column_stats, migrate_foreign_servers, migrate_foreign_tables, migrate_models,
144 migrate_path_indexes, migrate_scoring_params, persist_catalog_indexes,
145};
146use schema::{column_to_rust, load_catalog_indexes, load_table_specs, python_temporal_type};
147use source::{
148 ensure_destination_empty, is_python_catalog, open_read_only, reject_same_path,
149 resolve_source_database, table_exists,
150};
151use table_setup::{
152 create_tables, install_secondary_indexes, migrate_analyzers, migrate_table_field_analyzers,
153};
154use values::{
155 json_object_to_pairs, json_object_to_value_map, json_to_value, parameters_to_string_map,
156 quote_ident, sqlite_value_to_uqa, table_columns,
157};
158use vectors::{blob_to_f32_vec, extract_vectors, read_vector_fallbacks, vector_value};
159
160#[cfg(test)]
161use documents::coerce_migrated_document;
162#[cfg(test)]
163use schema::{infer_vector_fields, rust_column_type};
164#[cfg(test)]
165use vectors::value_to_f32_vec;
166
167#[cfg(test)]
168mod tests;