#![allow(non_camel_case_types)]
use core::ffi::c_ulonglong;
use std::fs::OpenOptions;
use std::io::BufReader;
use std::path::PathBuf;
use std::ptr;
use anndists::dist::distances::*;
use crate::api::*;
use crate::hnsw::*;
use crate::hnswio::*;
#[macro_export]
macro_rules! declare_myapi_type(
($name:ident, $ty:ty) => (
pub struct $name {
#[allow(dead_code)]
pub(crate) opaque: Box<dyn AnnT<Val=$ty>>,
} impl $name {
pub fn new(arg: Box<dyn AnnT<Val=$ty>>) -> Self {
$name{ opaque:arg}
} } )
);
declare_myapi_type!(HnswApiNodata, NoData);
#[repr(C)]
pub struct Vec_api<T> {
len: i64,
ptr: *const T,
} #[repr(C)]
pub struct Neighbour_api {
pub id: usize,
pub d: f32,
}
impl From<&Neighbour> for Neighbour_api {
fn from(neighbour: &Neighbour) -> Self {
Neighbour_api {
id: neighbour.d_id,
d: neighbour.distance,
}
}
}
#[repr(C)]
pub struct Neighbourhood_api {
pub nbgh: i64,
pub neighbours: *const Neighbour_api,
}
#[repr(C)]
pub struct Neighbour_api_parsearch_answer {
pub nb_answer: usize,
pub neighbourhoods: *const Neighbourhood_api,
}
super::declare_myapi_type!(HnswApif32, f32);
super::declare_myapi_type!(HnswApif64, f64);
#[allow(unused_macros)]
macro_rules! generate_insert(
($function_name:ident, $api_name:ty, $type_val:ty) => (
#[no_mangle]
pub extern "C" fn $function_name(hnsw_api : *mut $api_name, len:usize, data : *const $type_val, id : usize) {
log::trace!("entering insert, type {:?} vec len is {:?}, id : {:?} ", stringify!($type_val), len, id);
let data_v : Vec<$type_val>;
unsafe {
let slice = std::slice::from_raw_parts(data, len);
data_v = Vec::from(slice);
log::trace!("calling insert data");
(*hnsw_api).opaque.insert_data(&data_v, id);
}
log::trace!("exiting insert for type {:?}", stringify!($type_val));
} )
);
macro_rules! generate_parallel_insert(
($function_name:ident, $api_name:ty, $type_val:ty) => (
#[no_mangle]
pub extern "C" fn $function_name(hnsw_api : *mut $api_name, nb_vec: usize, vec_len : usize,
datas : *mut *const $type_val, ids : *const usize) {
log::trace!("entering parallel_insert type {:?} , vec len is {:?}, nb_vec : {:?}", stringify!($type_val), vec_len, nb_vec);
let data_ids : Vec<usize>;
let data_ptrs : Vec<*const $type_val>;
unsafe {
let slice = std::slice::from_raw_parts(ids, nb_vec);
data_ids = Vec::from(slice);
}
unsafe {
let slice = std::slice::from_raw_parts(datas, nb_vec);
data_ptrs = Vec::from(slice);
}
let mut data_v = Vec::<Vec<$type_val>>::with_capacity(nb_vec);
for i in 0..nb_vec {
unsafe {
let slice = std::slice::from_raw_parts(data_ptrs[i], vec_len);
let v = Vec::from(slice);
data_v.push(v);
}
}
let mut request : Vec<(&Vec<$type_val>, usize)> = Vec::with_capacity(nb_vec);
for i in 0..nb_vec {
request.push((&data_v[i], data_ids[i]));
}
unsafe { (*hnsw_api).opaque.parallel_insert_data(&request); };
log::trace!("exiting parallel_insert");
} )
);
macro_rules! generate_search_neighbours(
($function_name:ident, $api_name:ty, $type_val:ty) => (
#[no_mangle]
pub extern "C" fn $function_name(hnsw_api : *const $api_name, len:usize, data : *const $type_val,
knbn : usize, ef_search : usize) -> *const Neighbourhood_api {
log::trace!("entering search_neighbours type {:?}, vec len is {:?}, id : {:?} ef_search {:?}", stringify!($type_val), len, knbn, ef_search);
let data_v : Vec<$type_val>;
let neighbours : Vec<Neighbour>;
unsafe {
let slice = std::slice::from_raw_parts(data, len);
data_v = Vec::from(slice);
log::trace!("calling search neighbours {:?}", data_v);
neighbours = (*hnsw_api).opaque.search_neighbours(&data_v, knbn, ef_search);
}
let neighbours_api : Vec<Neighbour_api> = neighbours.iter().map(|n| Neighbour_api::from(n)).collect();
log::trace!(" got nb neighbours {:?}", neighbours_api.len());
let nbgh_i = neighbours.len() as i64;
let neighbours_ptr = neighbours_api.as_ptr();
std::mem::forget(neighbours_api);
let answer = Neighbourhood_api {
nbgh : nbgh_i,
neighbours : neighbours_ptr,
};
log::trace!("search_neighbours returning nb neighbours {:?} id ptr {:?} ", nbgh_i, neighbours_ptr);
Box::into_raw(Box::new(answer))
}
)
);
macro_rules! generate_parallel_search_neighbours(
($function_name:ident, $api_name:ty, $type_val:ty) => (
#[no_mangle]
pub extern "C" fn $function_name(hnsw_api : *const $api_name, nb_vec : usize, vec_len :i64,
data : *mut *const $type_val, knbn : usize, ef_search : usize) -> *const Vec_api<Neighbourhood_api> {
log::trace!("recieving // search request for type: {:?} with {:?} vectors", stringify!($type_val), nb_vec);
let neighbours : Vec<Vec<Neighbour> >;
let mut data_v = Vec::<Vec<$type_val>>::with_capacity(nb_vec);
unsafe {
let slice = std::slice::from_raw_parts(data, nb_vec);
let ptr_list : Vec<*const $type_val> = Vec::from(slice);
for i in 0..nb_vec {
let slice_i = std::slice::from_raw_parts(ptr_list[i], vec_len as usize);
let v = Vec::from(slice_i);
data_v.push(v);
}
neighbours = (*hnsw_api).opaque.parallel_search_neighbours(&data_v, knbn, ef_search);
}
let mut neighbour_lists = Vec::<Neighbourhood_api>::with_capacity(nb_vec);
for v in neighbours {
let neighbours_api : Vec<Neighbour_api> = v.iter().map(|n| Neighbour_api::from(n)).collect();
let nbgh = neighbours_api.len();
let neighbours_api_ptr = neighbours_api.as_ptr();
std::mem::forget(neighbours_api);
let v_answer = Neighbourhood_api {
nbgh : nbgh as i64,
neighbours: neighbours_api_ptr,
};
neighbour_lists.push(v_answer);
}
log::trace!(" reconstructed output pointers to vectors");
let neighbour_lists_ptr = neighbour_lists.as_ptr();
std::mem::forget(neighbour_lists);
let answer = Vec_api::<Neighbourhood_api> {
len : nb_vec as i64,
ptr : neighbour_lists_ptr,
};
Box::into_raw(Box::new(answer))
} )
);
#[allow(unused_macros)]
macro_rules! generate_file_dump(
($function_name:ident, $api_name:ty, $type_val:ty) => (
#[no_mangle]
pub extern "C" fn $function_name(hnsw_api : *const $api_name, namelen : usize, filename :*const u8) -> i64 {
log::info!("receiving request for file dump");
let slice = unsafe { std::slice::from_raw_parts(filename, namelen) } ;
let fstring = String::from_utf8_lossy(slice).into_owned();
let res = unsafe { (*hnsw_api).opaque.file_dump(&fstring) } ;
if res.is_ok() {
return 1;
}
else { return -1; }
} )
);
static mut RELOADER_OPT: Option<HnswIo> = None;
#[allow(unused_macros)]
macro_rules! generate_loadhnsw(
($function_name:ident, $api_name:ty, $type_val:ty, $type_dist : ty) => (
#[no_mangle]
pub extern "C" fn $function_name(flen : usize, name : *const u8) -> *const $api_name {
let slice = unsafe { std::slice::from_raw_parts(name, flen)} ;
let filename = String::from_utf8_lossy(slice).into_owned();
unsafe {
if RELOADER_OPT.is_some() {
log::error!("api can have nonly one mmap active");
return ptr::null();
}
RELOADER_OPT = Some(HnswIo::new(PathBuf::from("."), filename.clone()));
let hnsw_loaded_res = RELOADER_OPT.as_mut().unwrap().load_hnsw::<$type_val, $type_dist>();
if let Ok(hnsw_loaded) = hnsw_loaded_res {
let api = <$api_name>::new(Box::new(hnsw_loaded));
return Box::into_raw(Box::new(api));
}
else {
log::warn!("an error occured, could not reload data from {:?}", filename);
return ptr::null();
}
}
} )
);
generate_loadhnsw!(
load_hnswdump_f32_DistL1,
HnswApif32,
f32,
anndists::dist::distances::DistL1
);
generate_loadhnsw!(
load_hnswdump_f32_DistL2,
HnswApif32,
f32,
anndists::dist::distances::DistL2
);
generate_loadhnsw!(
load_hnswdump_f32_DistCosine,
HnswApif32,
f32,
anndists::dist::distances::DistCosine
);
generate_loadhnsw!(
load_hnswdump_f32_DistDot,
HnswApif32,
f32,
anndists::dist::distances::DistDot
);
generate_loadhnsw!(
load_hnswdump_f32_DistJensenShannon,
HnswApif32,
f32,
anndists::dist::distances::DistJensenShannon
);
generate_loadhnsw!(
load_hnswdump_f32_DistJeffreys,
HnswApif32,
f32,
anndists::dist::distances::DistJeffreys
);
generate_loadhnsw!(
load_hnswdump_i32_DistL1,
HnswApii32,
i32,
anndists::dist::distances::DistL1
);
generate_loadhnsw!(
load_hnswdump_i32_DistL2,
HnswApii32,
i32,
anndists::dist::distances::DistL2
);
generate_loadhnsw!(
load_hnswdump_i32_DistHamming,
HnswApii32,
i32,
anndists::dist::distances::DistHamming
);
generate_loadhnsw!(
load_hnswdump_u32_DistL1,
HnswApiu32,
u32,
anndists::dist::distances::DistL1
);
generate_loadhnsw!(
load_hnswdump_u32_DistL2,
HnswApiu32,
u32,
anndists::dist::distances::DistL2
);
generate_loadhnsw!(
load_hnswdump_u32_DistHamming,
HnswApiu32,
u32,
anndists::dist::distances::DistHamming
);
generate_loadhnsw!(
load_hnswdump_u32_DistJaccard,
HnswApiu32,
u32,
anndists::dist::distances::DistJaccard
);
generate_loadhnsw!(
load_hnswdump_u16_DistL1,
HnswApiu16,
u16,
anndists::dist::distances::DistL1
);
generate_loadhnsw!(
load_hnswdump_u16_DistL2,
HnswApiu16,
u16,
anndists::dist::distances::DistL2
);
generate_loadhnsw!(
load_hnswdump_u16_DistHamming,
HnswApiu16,
u16,
anndists::dist::distances::DistHamming
);
generate_loadhnsw!(
load_hnswdump_u16_DistLevenshtein,
HnswApiu16,
u16,
anndists::dist::distances::DistLevenshtein
);
generate_loadhnsw!(
load_hnswdump_u8_DistL1,
HnswApiu8,
u8,
anndists::dist::distances::DistL1
);
generate_loadhnsw!(
load_hnswdump_u8_DistL2,
HnswApiu8,
u8,
anndists::dist::distances::DistL2
);
generate_loadhnsw!(
load_hnswdump_u8_DistHamming,
HnswApiu8,
u8,
anndists::dist::distances::DistHamming
);
generate_loadhnsw!(
load_hnswdump_u8_DistJaccard,
HnswApiu8,
u8,
anndists::dist::distances::DistJaccard
);
generate_loadhnsw!(
load_hnswdump_NoData_DistNoDist,
HnswApiNodata,
NoData,
anndists::dist::NoDist
);
#[no_mangle]
pub extern "C" fn init_hnsw_f32(
max_nb_conn: usize,
ef_const: usize,
namelen: usize,
cdistname: *const u8,
) -> *const HnswApif32 {
log::info!("entering init_hnsw_f32");
let slice = unsafe { std::slice::from_raw_parts(cdistname, namelen) };
let dname = String::from_utf8_lossy(slice).into_owned();
match dname.as_str() {
"DistL1" => {
log::info!(" received DistL1");
let h = Hnsw::<f32, DistL1>::new(max_nb_conn, 10000, 16, ef_const, DistL1 {});
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistL2" => {
let h = Hnsw::<f32, DistL2>::new(max_nb_conn, 10000, 16, ef_const, DistL2 {});
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistDot" => {
let h = Hnsw::<f32, DistDot>::new(max_nb_conn, 10000, 16, ef_const, DistDot {});
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistHellinger" => {
let h =
Hnsw::<f32, DistHellinger>::new(max_nb_conn, 10000, 16, ef_const, DistHellinger {});
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistJeffreys" => {
let h =
Hnsw::<f32, DistJeffreys>::new(max_nb_conn, 10000, 16, ef_const, DistJeffreys {});
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistJensenShannon" => {
let h = Hnsw::<f32, DistJensenShannon>::new(
max_nb_conn,
10000,
16,
ef_const,
DistJensenShannon {},
);
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
_ => {
log::warn!("init_hnsw_f32 received unknow distance {:?} ", dname);
let p = ptr::null::<HnswApif32>();
return p;
}
} } #[no_mangle]
pub extern "C" fn new_hnsw_f32(
max_nb_conn: usize,
ef_const: usize,
namelen: usize,
cdistname: *const u8,
max_elements: usize,
max_layer: usize,
) -> *const HnswApif32 {
log::debug!("entering new_hnsw_f32");
let slice = unsafe { std::slice::from_raw_parts(cdistname, namelen) };
let dname = String::from_utf8_lossy(slice);
match dname.as_ref() {
"DistL1" => {
log::info!(" received DistL1");
let h =
Hnsw::<f32, DistL1>::new(max_nb_conn, max_elements, max_layer, ef_const, DistL1 {});
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistL2" => {
let h =
Hnsw::<f32, DistL2>::new(max_nb_conn, max_elements, max_layer, ef_const, DistL2 {});
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistDot" => {
let h = Hnsw::<f32, DistDot>::new(
max_nb_conn,
max_elements,
max_layer,
ef_const,
DistDot {},
);
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistHellinger" => {
let h = Hnsw::<f32, DistHellinger>::new(
max_nb_conn,
max_elements,
max_layer,
ef_const,
DistHellinger {},
);
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistJeffreys" => {
let h = Hnsw::<f32, DistJeffreys>::new(
max_nb_conn,
max_elements,
max_layer,
ef_const,
DistJeffreys {},
);
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
"DistJensenShannon" => {
let h = Hnsw::<f32, DistJensenShannon>::new(
max_nb_conn,
max_elements,
max_layer,
ef_const,
DistJensenShannon {},
);
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
_ => {
log::warn!("init_hnsw_f32 received unknow distance {:?} ", dname);
let p = ptr::null::<HnswApif32>();
return p;
}
} } #[no_mangle]
pub unsafe extern "C" fn drop_hnsw_f32(p: *const HnswApif32) {
let _raw = Box::from_raw(p as *mut HnswApif32);
}
#[no_mangle]
pub unsafe extern "C" fn drop_hnsw_u16(p: *const HnswApiu16) {
let _raw = Box::from_raw(p as *mut HnswApiu16);
}
#[no_mangle]
pub extern "C" fn init_hnsw_ptrdist_f32(
max_nb_conn: usize,
ef_const: usize,
c_func: extern "C" fn(*const f32, *const f32, c_ulonglong) -> f32,
) -> *const HnswApif32 {
log::info!("init_ hnsw_ptrdist: ptr {:?}", c_func);
let c_dist = DistCFFI::<f32>::new(c_func);
let h = Hnsw::<f32, DistCFFI<f32>>::new(max_nb_conn, 10000, 16, ef_const, c_dist);
let api = HnswApif32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
#[no_mangle]
pub extern "C" fn insert_f32(hnsw_api: *mut HnswApif32, len: usize, data: *const f32, id: usize) {
log::trace!("entering insert_f32, vec len is {:?}, id : {:?} ", len, id);
let data_v: Vec<f32>;
unsafe {
let slice = std::slice::from_raw_parts(data, len);
data_v = Vec::from(slice);
(*hnsw_api).opaque.insert_data(&data_v, id);
}
} #[no_mangle]
pub extern "C" fn parallel_insert_f32(
hnsw_api: *mut HnswApif32,
nb_vec: usize,
vec_len: usize,
datas: *mut *const f32,
ids: *const usize,
) {
let data_ids: Vec<usize>;
let data_ptrs: Vec<*const f32>;
unsafe {
let slice = std::slice::from_raw_parts(ids, nb_vec);
data_ids = Vec::from(slice);
}
unsafe {
let slice = std::slice::from_raw_parts(datas, nb_vec);
data_ptrs = Vec::from(slice);
}
let mut data_v = Vec::<Vec<f32>>::with_capacity(nb_vec);
for i in 0..nb_vec {
unsafe {
let slice = std::slice::from_raw_parts(data_ptrs[i], vec_len);
let v = Vec::from(slice);
data_v.push(v);
}
}
let mut request: Vec<(&Vec<f32>, usize)> = Vec::with_capacity(nb_vec);
for i in 0..nb_vec {
request.push((&data_v[i], data_ids[i]));
}
unsafe {
(*hnsw_api).opaque.parallel_insert_data(&request);
};
log::trace!("exiting parallel_insert");
} #[no_mangle]
pub extern "C" fn search_neighbours_f32(
hnsw_api: *const HnswApif32,
len: usize,
data: *const f32,
knbn: usize,
ef_search: usize,
) -> *const Neighbourhood_api {
log::trace!(
"entering search_neighbours , vec len is {:?}, id : {:?} ef_search {:?}",
len,
knbn,
ef_search
);
let data_v: Vec<f32>;
let neighbours: Vec<Neighbour>;
unsafe {
let slice = std::slice::from_raw_parts(data, len);
data_v = Vec::from(slice);
log::trace!("calling search neighbours {:?}", data_v);
neighbours = (*hnsw_api)
.opaque
.search_neighbours(&data_v, knbn, ef_search);
}
let neighbours_api: Vec<Neighbour_api> =
neighbours.iter().map(|n| Neighbour_api::from(n)).collect();
log::trace!(" got nb neighbours {:?}", neighbours_api.len());
let nbgh_i = neighbours.len() as i64;
let neighbours_ptr = neighbours_api.as_ptr();
std::mem::forget(neighbours_api);
let answer = Neighbourhood_api {
nbgh: nbgh_i,
neighbours: neighbours_ptr,
};
log::trace!(
"search_neighbours returning nb neighbours {:?} id ptr {:?} ",
nbgh_i,
neighbours_ptr
);
Box::into_raw(Box::new(answer))
}
generate_parallel_search_neighbours!(parallel_search_neighbours_f32, HnswApif32, f32);
generate_file_dump!(file_dump_f32, HnswApif32, f32);
#[no_mangle]
pub extern "C" fn init_hnsw_i32(
max_nb_conn: usize,
ef_const: usize,
namelen: usize,
cdistname: *const u8,
) -> *const HnswApii32 {
log::info!("entering init_hnsw_i32");
let slice = unsafe { std::slice::from_raw_parts(cdistname, namelen) };
let dname = String::from_utf8_lossy(slice);
if dname == "DistL1" {
log::info!(" received DistL1");
let h = Hnsw::<i32, DistL1>::new(max_nb_conn, 10000, 16, ef_const, DistL1 {});
let api = HnswApii32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistL2" {
let h = Hnsw::<i32, DistL2>::new(max_nb_conn, 10000, 16, ef_const, DistL2 {});
let api = HnswApii32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistHamming" {
let h = Hnsw::<i32, DistHamming>::new(max_nb_conn, 10000, 16, ef_const, DistHamming {});
let api = HnswApii32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
let p = ptr::null::<HnswApii32>();
p
} #[no_mangle]
pub extern "C" fn init_hnsw_ptrdist_i32(
max_nb_conn: usize,
ef_const: usize,
c_func: extern "C" fn(*const i32, *const i32, c_ulonglong) -> f32,
) -> *const HnswApii32 {
log::debug!("init_ hnsw_ptrdist: ptr {:?}", c_func);
let c_dist = DistCFFI::<i32>::new(c_func);
let h = Hnsw::<i32, DistCFFI<i32>>::new(max_nb_conn, 10000, 16, ef_const, c_dist);
let api = HnswApii32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
super::declare_myapi_type!(HnswApii32, i32);
generate_insert!(insert_i32, HnswApii32, i32);
generate_parallel_insert!(parallel_insert_i32, HnswApii32, i32);
generate_search_neighbours!(search_neighbours_i32, HnswApii32, i32);
generate_parallel_search_neighbours!(parallel_search_neighbours_i32, HnswApii32, i32);
generate_file_dump!(file_dump_i32, HnswApii32, i32);
#[no_mangle]
pub extern "C" fn init_hnsw_u32(
max_nb_conn: usize,
ef_const: usize,
namelen: usize,
cdistname: *const u8,
) -> *const HnswApiu32 {
log::debug!("entering init_hnsw_u32");
let slice = unsafe { std::slice::from_raw_parts(cdistname, namelen) };
let dname = String::from_utf8_lossy(slice);
if dname == "DistL1" {
log::debug!(" received DistL1");
let h = Hnsw::<u32, DistL1>::new(max_nb_conn, 10000, 16, ef_const, DistL1 {});
let api = HnswApiu32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistL2" {
let h = Hnsw::<u32, DistL2>::new(max_nb_conn, 10000, 16, ef_const, DistL2 {});
let api = HnswApiu32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistJaccard" {
let h = Hnsw::<u32, DistJaccard>::new(max_nb_conn, 10000, 16, ef_const, DistJaccard {});
let api = HnswApiu32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistHamming" {
let h = Hnsw::<u32, DistHamming>::new(max_nb_conn, 10000, 16, ef_const, DistHamming {});
let api = HnswApiu32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
let p = ptr::null::<HnswApiu32>();
p
} #[no_mangle]
pub extern "C" fn init_hnsw_ptrdist_u32(
max_nb_conn: usize,
ef_const: usize,
c_func: extern "C" fn(*const u32, *const u32, c_ulonglong) -> f32,
) -> *const HnswApiu32 {
log::info!("init_ hnsw_ptrdist: ptr {:?}", c_func);
let c_dist = DistCFFI::<u32>::new(c_func);
let h = Hnsw::<u32, DistCFFI<u32>>::new(max_nb_conn, 10000, 16, ef_const, c_dist);
let api = HnswApiu32 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
super::declare_myapi_type!(HnswApiu32, u32);
generate_insert!(insert_u32, HnswApiu32, u32);
generate_parallel_insert!(parallel_insert_u32, HnswApiu32, u32);
generate_search_neighbours!(search_neighbours_u32, HnswApiu32, u32);
generate_parallel_search_neighbours!(parallel_search_neighbours_u32, HnswApiu32, u32);
generate_file_dump!(file_dump_u32, HnswApiu32, u32);
super::declare_myapi_type!(HnswApiu16, u16);
#[no_mangle]
pub extern "C" fn init_hnsw_u16(
max_nb_conn: usize,
ef_const: usize,
namelen: usize,
cdistname: *const u8,
) -> *const HnswApiu16 {
log::info!("entering init_hnsw_u16");
let slice = unsafe { std::slice::from_raw_parts(cdistname, namelen) };
let dname = String::from_utf8_lossy(slice);
if dname == "DistL1" {
log::info!(" received DistL1");
let h = Hnsw::<u16, DistL1>::new(max_nb_conn, 10000, 16, ef_const, DistL1 {});
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistL2" {
let h = Hnsw::<u16, DistL2>::new(max_nb_conn, 10000, 16, ef_const, DistL2 {});
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistHamming" {
let h = Hnsw::<u16, DistHamming>::new(max_nb_conn, 10000, 16, ef_const, DistHamming {});
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistJaccard" {
let h = Hnsw::<u16, DistJaccard>::new(max_nb_conn, 10000, 16, ef_const, DistJaccard {});
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistLevenshtein" {
let h =
Hnsw::<u16, DistLevenshtein>::new(max_nb_conn, 10000, 16, ef_const, DistLevenshtein {});
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
let p = ptr::null::<HnswApiu16>();
p
} #[no_mangle]
pub extern "C" fn new_hnsw_u16(
max_nb_conn: usize,
ef_const: usize,
namelen: usize,
cdistname: *const u8,
max_elements: usize,
max_layer: usize,
) -> *const HnswApiu16 {
log::info!("entering init_hnsw_u16");
let slice = unsafe { std::slice::from_raw_parts(cdistname, namelen) };
let dname = String::from_utf8_lossy(slice);
if dname == "DistL1" {
log::info!(" received DistL1");
let h = Hnsw::<u16, DistL1>::new(max_nb_conn, max_elements, max_layer, ef_const, DistL1 {});
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistL2" {
let h = Hnsw::<u16, DistL2>::new(max_nb_conn, max_elements, max_layer, ef_const, DistL2 {});
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistHamming" {
let h = Hnsw::<u16, DistHamming>::new(
max_nb_conn,
max_elements,
max_layer,
ef_const,
DistHamming {},
);
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistJaccard" {
let h = Hnsw::<u16, DistJaccard>::new(
max_nb_conn,
max_elements,
max_layer,
ef_const,
DistJaccard {},
);
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistLevenshtein" {
let h = Hnsw::<u16, DistLevenshtein>::new(
max_nb_conn,
max_elements,
max_layer,
ef_const,
DistLevenshtein {},
);
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
let p = ptr::null::<HnswApiu16>();
p
} #[no_mangle]
pub extern "C" fn init_hnsw_ptrdist_u16(
max_nb_conn: usize,
ef_const: usize,
c_func: extern "C" fn(*const u16, *const u16, c_ulonglong) -> f32,
) -> *const HnswApiu16 {
log::info!("init_ hnsw_ptrdist: ptr {:?}", c_func);
let c_dist = DistCFFI::<u16>::new(c_func);
let h = Hnsw::<u16, DistCFFI<u16>>::new(max_nb_conn, 10000, 16, ef_const, c_dist);
let api = HnswApiu16 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
generate_insert!(insert_u16, HnswApiu16, u16);
generate_parallel_insert!(parallel_insert_u16, HnswApiu16, u16);
generate_search_neighbours!(search_neighbours_u16, HnswApiu16, u16);
generate_parallel_search_neighbours!(parallel_search_neighbours_u16, HnswApiu16, u16);
generate_file_dump!(file_dump_u16, HnswApiu16, u16);
super::declare_myapi_type!(HnswApiu8, u8);
#[no_mangle]
pub extern "C" fn init_hnsw_u8(
max_nb_conn: usize,
ef_const: usize,
namelen: usize,
cdistname: *const u8,
) -> *const HnswApiu8 {
log::debug!("entering init_hnsw_u8");
let slice = unsafe { std::slice::from_raw_parts(cdistname, namelen) };
let dname = String::from_utf8_lossy(slice);
if dname == "DistL1" {
log::info!(" received DistL1");
let h = Hnsw::<u8, DistL1>::new(max_nb_conn, 10000, 16, ef_const, DistL1 {});
let api = HnswApiu8 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistL2" {
let h = Hnsw::<u8, DistL2>::new(max_nb_conn, 10000, 16, ef_const, DistL2 {});
let api = HnswApiu8 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistHamming" {
let h = Hnsw::<u8, DistHamming>::new(max_nb_conn, 10000, 16, ef_const, DistHamming {});
let api = HnswApiu8 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
} else if dname == "DistJaccard" {
let h = Hnsw::<u8, DistJaccard>::new(max_nb_conn, 10000, 16, ef_const, DistJaccard {});
let api = HnswApiu8 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
let p = ptr::null::<HnswApiu8>();
p
} #[no_mangle]
pub extern "C" fn init_hnsw_ptrdist_u8(
max_nb_conn: usize,
ef_const: usize,
c_func: extern "C" fn(*const u8, *const u8, c_ulonglong) -> f32,
) -> *const HnswApiu8 {
log::info!("init_ hnsw_ptrdist: ptr {:?}", c_func);
let c_dist = DistCFFI::<u8>::new(c_func);
let h = Hnsw::<u8, DistCFFI<u8>>::new(max_nb_conn, 10000, 16, ef_const, c_dist);
let api = HnswApiu8 {
opaque: Box::new(h),
};
return Box::into_raw(Box::new(api));
}
generate_insert!(insert_u8, HnswApiu8, u8);
generate_parallel_insert!(parallel_insert_u8, HnswApiu8, u8);
generate_search_neighbours!(search_neighbours_u8, HnswApiu8, u8);
generate_parallel_search_neighbours!(parallel_search_neighbours_u8, HnswApiu8, u8);
generate_file_dump!(file_dump_u8, HnswApiu8, u8);
#[repr(C)]
pub struct DescriptionFFI {
pub dumpmode: u8,
pub max_nb_connection: u8,
pub nb_layer: u8,
pub ef: usize,
pub nb_point: usize,
pub data_dimension: usize,
pub distname_len: usize,
pub distname: *const u8,
pub t_name_len: usize,
pub t_name: *const u8,
}
impl DescriptionFFI {
pub fn new() -> Self {
DescriptionFFI {
dumpmode: 0,
max_nb_connection: 0,
nb_layer: 0,
ef: 0,
nb_point: 0,
data_dimension: 0,
distname_len: 0,
distname: ptr::null(),
t_name_len: 0,
t_name: ptr::null(),
}
} }
#[no_mangle]
pub extern "C" fn load_hnsw_description(flen: usize, name: *const u8) -> *const DescriptionFFI {
let slice = unsafe { std::slice::from_raw_parts(name, flen) };
let filename = String::from_utf8_lossy(slice).into_owned();
let fpath = PathBuf::from(filename);
let fileres = OpenOptions::new().read(true).open(&fpath);
let mut ffi_description = DescriptionFFI::new();
match fileres {
Ok(file) => {
let mut bufr = BufReader::with_capacity(10000000, file);
let res = load_description(&mut bufr);
if let Ok(description) = res {
let distname = String::clone(&description.distname);
let distname_ptr = distname.as_ptr();
let distname_len = distname.len();
std::mem::forget(distname);
let t_name = String::clone(&description.t_name);
let t_name_ptr = t_name.as_ptr();
let t_name_len = t_name.len();
std::mem::forget(t_name);
ffi_description.dumpmode = 1; ffi_description.max_nb_connection = description.max_nb_connection;
ffi_description.nb_layer = description.nb_layer;
ffi_description.ef = description.ef;
ffi_description.data_dimension = description.dimension;
ffi_description.distname_len = distname_len;
ffi_description.distname = distname_ptr;
ffi_description.t_name_len = t_name_len;
ffi_description.t_name = t_name_ptr;
return Box::into_raw(Box::new(ffi_description));
} else {
log::error!(
"could not get descrption of hnsw from file {:?}",
fpath.as_os_str()
);
println!(
"could not get descrption of hnsw from file {:?} ",
fpath.as_os_str()
);
return ptr::null();
}
}
Err(_e) => {
log::error!(
"no such file, load_hnsw_description: could not open file {:?}",
fpath.as_os_str()
);
println!(
"no such file, load_hnsw_description: could not open file {:?}",
fpath.as_os_str()
);
return ptr::null();
}
}
} #[no_mangle]
pub extern "C" fn init_rust_log() {
let _res = env_logger::Builder::from_default_env().try_init();
}