#include "scann/scann_ops/cc/scann_npy.h"
#include <algorithm>
#include <atomic>
#include <cmath>
#include <cstdint>
#include <functional>
#include <optional>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
#include "absl/strings/str_cat.h"
#include "pybind11/gil.h"
#include "pybind11/pytypes.h"
#include "scann/base/single_machine_base.h"
#include "scann/data_format/datapoint.h"
#include "scann/data_format/dataset.h"
#include "scann/scann_ops/cc/scann.h"
#include "scann/utils/common.h"
#include "scann/utils/io_oss_wrapper.h"
#include "scann/utils/single_machine_autopilot.h"
#include "scann/utils/types.h"
namespace research_scann {
using MutationOptions = UntypedSingleMachineSearcherBase::MutationOptions;
using PrecomputedMutationArtifacts =
UntypedSingleMachineSearcherBase::PrecomputedMutationArtifacts;
void RuntimeErrorIfNotOk(const char* prefix, const Status& status) {
if (!status.ok()) {
std::string msg = prefix + std::string(status.message());
throw std::runtime_error(msg);
}
}
template <typename T>
T ValueOrRuntimeError(StatusOr<T> status_or, const char* prefix) {
RuntimeErrorIfNotOk(prefix, status_or.status());
return status_or.value();
}
constexpr uint64_t kMaxDatapoints = kInvalidDatapointIndex;
ScannNumpy::ScannNumpy(const std::string& artifacts_dir,
const std::string& scann_assets_pbtxt) {
pybind11::gil_scoped_release gil_release;
auto status_or =
ScannInterface::LoadArtifacts(artifacts_dir, scann_assets_pbtxt);
RuntimeErrorIfNotOk("Error loading artifacts: ", status_or.status());
RuntimeErrorIfNotOk("Error initializing searcher: ",
scann_.Initialize(status_or.value()));
}
ScannNumpy::ScannNumpy(const np_row_major_arr<float>& np_dataset,
absl::string_view config, int training_threads) {
if (np_dataset.ndim() != 2)
throw std::invalid_argument("Dataset input must be two-dimensional");
if (static_cast<uint64_t>(np_dataset.shape()[0]) > kMaxDatapoints)
throw std::invalid_argument(absl::StrCat(
"Dataset has ", np_dataset.shape()[0], " rows; ScaNN supports at most ",
kMaxDatapoints, " (datapoint indices are 32-bit)"));
ConstSpan<float> dataset(np_dataset.data(), np_dataset.size());
pybind11::gil_scoped_release gil_release;
RuntimeErrorIfNotOk("Error initializing searcher: ",
scann_.Initialize(dataset, np_dataset.shape()[0], config,
training_threads));
}
vector<DatapointIndex> ScannNumpy::Upsert(
vector<std::optional<DatapointIndex>> indices,
vector<np_row_major_arr<float>>& vecs, int batch_size) {
if (batch_size < 1)
throw std::invalid_argument("Upsert batch_size must be >= 1.");
pybind11::gil_scoped_release gil_release;
return UpsertRows(
indices, vecs.size(),
[&](size_t row) { return ConstSpan<float>(vecs[row].data(), vecs[row].size()); },
batch_size);
}
vector<DatapointIndex> ScannNumpy::Upsert(
vector<std::optional<DatapointIndex>> indices,
const np_row_major_arr<float>& vecs, int batch_size) {
if (batch_size < 1)
throw std::invalid_argument("Upsert batch_size must be >= 1.");
if (vecs.ndim() != 2)
throw std::invalid_argument("Upsert vectors must be a two-dimensional array");
const size_t n = vecs.shape()[0];
const size_t width = vecs.shape()[1];
const float* data = vecs.data();
pybind11::gil_scoped_release gil_release;
return UpsertRows(
indices, n,
[&](size_t row) { return ConstSpan<float>(data + row * width, width); },
batch_size);
}
vector<DatapointIndex> ScannNumpy::UpsertRows(
const vector<std::optional<DatapointIndex>>& indices, size_t n_rows,
const std::function<ConstSpan<float>(size_t)>& rows, int batch_size) {
absl::MutexLock lock(&mu_);
if (indices.size() != n_rows)
throw std::runtime_error("Upsert input size must match.");
const DatapointIndex n_points = scann_.n_points();
const uint64_t n_adds =
std::count_if(indices.begin(), indices.end(),
[](const auto& index) { return !index.has_value(); });
if (n_points + n_adds > kMaxDatapoints)
throw std::invalid_argument(absl::StrCat(
"Upsert would grow the index to ", n_points + n_adds,
" datapoints; ScaNN supports at most ", kMaxDatapoints,
" (datapoint indices are 32-bit)"));
for (size_t row : Seq(n_rows)) {
const ConstSpan<float> vec = rows(row);
if (vec.size() != scann_.dimensionality())
throw std::invalid_argument(absl::StrCat(
"Upsert vector has dimensionality ", vec.size(),
", but the dataset has ", scann_.dimensionality(), " (row ", row,
")"));
for (float v : vec)
if (!std::isfinite(v))
throw std::invalid_argument(absl::StrCat(
"Upsert vector at row ", row,
" contains NaN or infinity; ScaNN only supports finite values."));
if (indices[row].has_value() && indices[row].value() >= n_points)
throw std::invalid_argument(absl::StrCat(
"Upsert index ", indices[row].value(), " at row ", row,
" is out of range for an index with ", n_points, " points"));
}
auto mutator =
ValueOrRuntimeError(scann_.GetMutator(), "Failed to fetch mutator: ");
if (batch_size > 1)
mutator->set_mutation_threadpool(scann_.parallel_query_pool());
const DatapointIndex n = n_rows;
vector<DatapointIndex> result;
result.reserve(n);
const size_t dim = scann_.dimensionality();
const size_t stored_dim = scann_.stored_dimensionality();
vector<float> stored;
if (scann_.TransformsDatapoints()) {
vector<float> given(n_rows * dim);
for (size_t row : Seq(n_rows)) {
const ConstSpan<float> vec = rows(row);
std::copy(vec.begin(), vec.end(), given.begin() + row * dim);
}
scann_.ToStoredDatapoints(given, &stored);
}
auto row_ptr = [&](size_t row) {
return MakeDatapointPtr(stored.empty() ? rows(row).data()
: stored.data() + row * stored_dim,
stored_dim);
};
for (size_t b : Seq(DivRoundUp(n, batch_size))) {
size_t begin = batch_size * b;
size_t bs = std::min<DatapointIndex>(n - begin, batch_size);
DenseDataset<float> ds;
for (size_t i : Seq(bs))
RuntimeErrorIfNotOk("Error appending datapoint.",
ds.Append(row_ptr(begin + i)));
auto precomputed = mutator->ComputePrecomputedMutationArtifacts(
ds, scann_.parallel_query_pool());
for (size_t i : Seq(bs)) {
auto& index = indices[begin + i];
auto mo = MutationOptions{.precomputed_mutation_artifacts =
precomputed[i].get()};
if (!index.has_value()) {
result.push_back(ValueOrRuntimeError(
mutator->AddDatapoint(row_ptr(begin + i), "", mo),
"Failed to add datapoint: "));
} else {
result.push_back(ValueOrRuntimeError(
mutator->UpdateDatapoint(row_ptr(begin + i), index.value(), mo),
"Failed to update datapoint: "));
}
}
}
auto statusor = mutator->IncrementalMaintenance();
RuntimeErrorIfNotOk("Error performing incremental maintenance ",
statusor.status());
if (statusor.value().has_value()) RebalanceLocked("");
return result;
}
vector<DatapointIndex> ScannNumpy::Delete(vector<DatapointIndex> indices) {
pybind11::gil_scoped_release gil_release;
absl::MutexLock lock(&mu_);
auto mutator =
ValueOrRuntimeError(scann_.GetMutator(), "Failed to fetch mutator: ");
mutator->set_mutation_threadpool(scann_.parallel_query_pool());
vector<DatapointIndex> result;
for (const auto& index : indices) {
RuntimeErrorIfNotOk("Failed to delete datapoint: ",
mutator->RemoveDatapoint(index));
auto statusor = mutator->IncrementalMaintenance();
RuntimeErrorIfNotOk("Error performing incremental maintenance ",
statusor.status());
if (statusor.value().has_value()) {
RebalanceLocked("");
mutator =
ValueOrRuntimeError(scann_.GetMutator(), "Failed to fetch mutator: ");
mutator->set_mutation_threadpool(scann_.parallel_query_pool());
}
result.push_back(scann_.n_points());
}
return result;
}
int ScannNumpy::Rebalance(const string& config) {
pybind11::gil_scoped_release gil_release;
absl::MutexLock lock(&mu_);
return RebalanceLocked(config);
}
int ScannNumpy::RebalanceLocked(const string& config) {
auto statusor = scann_.RetrainAndReindex(config);
if (!statusor.ok()) {
RuntimeErrorIfNotOk("Failed to retrain searcher: ", statusor.status());
return -1;
}
return scann_.n_points();
}
size_t ScannNumpy::Size() const {
pybind11::gil_scoped_release gil_release;
absl::ReaderMutexLock lock(&mu_);
return scann_.n_points();
}
void ScannNumpy::Reserve(size_t num_datapoints) {
pybind11::gil_scoped_release gil_release;
absl::MutexLock lock(&mu_);
auto mutator =
ValueOrRuntimeError(scann_.GetMutator(), "Failed to fetch mutator: ");
mutator->Reserve(num_datapoints);
}
void ScannNumpy::SetNumThreads(int num_threads) {
pybind11::gil_scoped_release gil_release;
absl::MutexLock lock(&mu_);
scann_.SetNumThreads(num_threads);
}
string ScannNumpy::SuggestAutopilot(absl::string_view config_str,
DatapointIndex n, DimensionIndex dim) {
ScannConfig config;
RuntimeErrorIfNotOk("Failed to parse config: ",
ParseTextProto(&config, config_str));
auto status_or = Autopilot(config, nullptr, n, dim);
RuntimeErrorIfNotOk("Failed to suggest autopilot config: ",
status_or.status());
std::string result;
google::protobuf::TextFormat::PrintToString(status_or.value(), &result);
return result;
}
string ScannNumpy::Config() {
pybind11::gil_scoped_release gil_release;
absl::MutexLock lock(&mu_);
std::string config_str;
google::protobuf::TextFormat::PrintToString(*scann_.config(), &config_str);
return config_str;
}
std::pair<pybind11::array_t<DatapointIndex>, pybind11::array_t<float>>
ScannNumpy::Search(const np_row_major_arr<float>& query, int final_nn,
int pre_reorder_nn, int leaves) {
if (query.ndim() != 1)
throw std::invalid_argument("Query must be one-dimensional");
DatapointPtr<float> ptr(nullptr, query.data(), query.size(), query.size());
NNResultsVector res;
float multiplier;
{
pybind11::gil_scoped_release gil_release;
absl::ReaderMutexLock lock(&mu_);
auto status = scann_.Search(ptr, &res, final_nn, pre_reorder_nn, leaves);
RuntimeErrorIfNotOk("Error during search: ", status);
multiplier = scann_.result_multiplier();
}
pybind11::array_t<DatapointIndex> idx(res.size());
pybind11::array_t<float> dis(res.size());
DatapointIndex* idx_out = idx.mutable_data();
float* dis_out = dis.mutable_data();
for (const auto& [i, d] : res) {
*idx_out++ = i;
*dis_out++ = multiplier * d;
}
return {std::move(idx), std::move(dis)};
}
std::pair<pybind11::array_t<DatapointIndex>, pybind11::array_t<float>>
ScannNumpy::SearchBatched(const np_row_major_arr<float>& queries, int final_nn,
int pre_reorder_nn, int leaves, bool parallel,
int batch_size) {
if (queries.ndim() != 2)
throw std::invalid_argument("Queries must be in two-dimensional array");
if (queries.shape()[0] == 0) {
if (final_nn == 0)
throw std::invalid_argument("final_num_neighbors must be > 0");
int k;
{
pybind11::gil_scoped_release gil_release;
absl::ReaderMutexLock lock(&mu_);
if (static_cast<size_t>(queries.shape()[1]) != scann_.dimensionality())
throw std::invalid_argument(absl::StrCat(
"Queries have dimensionality ", queries.shape()[1],
", but the dataset has ", scann_.dimensionality()));
k = final_nn > 0 ? final_nn : scann_.default_num_neighbors();
}
std::vector<long> shape = {0, static_cast<long>(k)};
return {pybind11::array_t<DatapointIndex>(shape),
pybind11::array_t<float>(shape)};
}
const size_t n = queries.shape()[0];
ConstSpan<float> rows(queries.data(), queries.size());
size_t width = 0;
if (final_nn > 0) {
width = final_nn;
} else if (final_nn < 0) {
pybind11::gil_scoped_release gil_release;
absl::ReaderMutexLock lock(&mu_);
width = std::max(scann_.default_num_neighbors(), 0);
}
std::vector<long> shape = {static_cast<long>(n), static_cast<long>(width)};
pybind11::array_t<DatapointIndex> idx_arr(shape);
pybind11::array_t<float> dis_arr(shape);
DatapointIndex* idx_out = idx_arr.mutable_data();
float* dis_out = dis_arr.mutable_data();
std::vector<NNResultsVector> res(n);
std::atomic<size_t> widest{0};
std::atomic<bool> too_wide{false};
{
pybind11::gil_scoped_release gil_release;
absl::ReaderMutexLock lock(&mu_);
auto write = [&](size_t begin, ConstSpan<NNResultsVector> chunk) {
size_t chunk_widest = 0;
for (const auto& r : chunk) chunk_widest = std::max(chunk_widest, r.size());
size_t prev = widest.load(std::memory_order_relaxed);
while (prev < chunk_widest &&
!widest.compare_exchange_weak(prev, chunk_widest)) {
}
if (chunk_widest > width) {
too_wide.store(true);
return;
}
scann_.ReshapeBatchedNNResult(chunk, idx_out + begin * width,
dis_out + begin * width, width);
};
Status status =
scann_.SearchBatchedRows(rows, queries.shape()[1],
MakeMutableSpan(res), final_nn,
pre_reorder_nn, leaves, parallel,
parallel ? batch_size : 256, write);
RuntimeErrorIfNotOk("Error during search: ", status);
}
if (!too_wide.load() && (final_nn > 0 || widest.load() == width))
return {idx_arr, dis_arr};
shape[1] = std::max<long>(final_nn, static_cast<long>(widest.load()));
pybind11::array_t<DatapointIndex> idx2(shape);
pybind11::array_t<float> dis2(shape);
DatapointIndex* idx2_out = idx2.mutable_data();
float* dis2_out = dis2.mutable_data();
{
pybind11::gil_scoped_release gil_release;
absl::ReaderMutexLock lock(&mu_);
scann_.ReshapeBatchedNNResult(MakeConstSpan(res), idx2_out, dis2_out,
static_cast<int>(shape[1]));
}
return {idx2, dis2};
}
void ScannNumpy::Serialize(std::string path, bool relative_path,
std::optional<std::string> docids_pkl) {
std::vector<std::pair<std::string, std::string>> extra_files;
if (docids_pkl.has_value())
extra_files.emplace_back("scann_docids.pkl", *std::move(docids_pkl));
pybind11::gil_scoped_release gil_release;
absl::MutexLock lock(&mu_);
RuntimeErrorIfNotOk(
"Failed to serialize searcher: ",
scann_.SerializeToDirectory(path, relative_path, extra_files));
}
pybind11::dict ScannNumpy::GetHealthStats() const {
StatusOr<ScannInterface::ScannHealthStats> r;
{
pybind11::gil_scoped_release gil_release;
absl::MutexLock lock(&mu_);
r = scann_.GetHealthStats();
}
RuntimeErrorIfNotOk("Error getting health stats: ", r.status());
using namespace pybind11::literals;
return pybind11::dict("avg_quantization_error"_a = r->avg_quantization_error,
"partition_weighted_avg_relative_imbalance"_a =
r->partition_weighted_avg_relative_imbalance,
"partition_avg_relative_positive_imbalance"_a =
r->partition_avg_relative_positive_imbalance,
"sum_partition_sizes"_a = r->sum_partition_sizes);
}
void ScannNumpy::InitializeHealthStats() {
pybind11::gil_scoped_release gil_release;
absl::MutexLock lock(&mu_);
Status status = scann_.InitializeHealthStats();
RuntimeErrorIfNotOk("Error initializing health stats: ", status);
}
}