#include "scann/utils/single_machine_retraining.h"
#include <cmath>
#include <memory>
#include <utility>
#include "absl/base/thread_annotations.h"
#include "absl/synchronization/mutex.h"
#include "scann/base/internal/single_machine_factory_impl.h"
#include "scann/base/single_machine_base.h"
#include "scann/base/single_machine_factory_options.h"
#include "scann/base/single_machine_factory_scann.h"
#include "scann/data_format/dataset.h"
#include "scann/distance_measures/one_to_one/l2_distance.h"
#include "scann/oss_wrappers/scann_down_cast.h"
#include "scann/oss_wrappers/scann_status.h"
#include "scann/oss_wrappers/scann_threadpool.h"
#include "scann/proto/distance_measure.pb.h"
#include "scann/proto/hash.pb.h"
#include "scann/proto/scann.pb.h"
#include "scann/utils/common.h"
#include "scann/utils/scann_config_utils.h"
#include "scann/utils/types.h"
namespace research_scann {
template <typename T>
StatusOrSearcherUntyped RetrainAndReindexSearcherImpl(
UntypedSingleMachineSearcherBase* untyped_searcher,
absl::Mutex* searcher_pointer_mutex, ScannConfig config,
shared_ptr<ThreadPool> parallelization_pool) {
if (searcher_pointer_mutex) searcher_pointer_mutex->AssertNotHeld();
SingleMachineSearcherBase<T>* searcher =
down_cast<SingleMachineSearcherBase<T>*>(untyped_searcher);
SCANN_ASSIGN_OR_RETURN(auto dataset, searcher->ReconstructFloatDataset());
if (!dataset) {
return FailedPreconditionError(
"Retraining (rebalance) needs the datapoints' float values, and this "
"searcher keeps only quantized data (e.g. int8 or bfloat16 brute "
"force, or asymmetric hashing without reordering), so it cannot be "
"retrained. Searchers with float brute force or with reordering can.");
}
auto new_dataset = std::dynamic_pointer_cast<TypedDataset<T>>(
std::const_pointer_cast<DenseDataset<float>>(dataset));
if (!new_dataset) {
return UnimplementedError(
"RetrainAndReindexSearcher only supports float searchers.");
}
if (IsSphericalPartitioning(config) && dataset->normalization() != UNITL2NORM) {
auto normalized = std::make_shared<DenseDataset<float>>(dataset->Copy());
for (size_t i = 0; i < normalized->size(); ++i)
NormalizeForSphericalPartitioning(normalized->mutable_data(i));
normalized->set_normalization_tag(UNITL2NORM);
new_dataset = std::dynamic_pointer_cast<TypedDataset<T>>(
std::shared_ptr<DenseDataset<float>>(normalized));
dataset = std::move(normalized);
}
StripPreprocessedArtifacts(&config);
SingleMachineFactoryOptions opts;
opts.parallelization_pool = std::move(parallelization_pool);
SCANN_ASSIGN_OR_RETURN(auto result,
SingleMachineFactoryScann<T>(config, new_dataset, opts));
auto lock_mutex = [&searcher_pointer_mutex]() ABSL_NO_THREAD_SAFETY_ANALYSIS {
if (searcher_pointer_mutex) searcher_pointer_mutex->WriterLock();
};
lock_mutex();
result->docids_ = dataset->docids();
result->retraining_requires_dataset_ = false;
return result;
}
bool IsSphericalPartitioning(const ScannConfig& config) {
return config.has_partitioning() &&
config.partitioning().partitioning_type() ==
PartitioningConfig::SPHERICAL;
}
void NormalizeForSphericalPartitioning(MutableSpan<float> v) {
const double squared_norm = SquaredL2Norm(MakeDatapointPtr(v.data(), v.size()));
if (squared_norm == 0 || !std::isfinite(squared_norm) ||
std::abs(squared_norm - 1) <= 1e-5)
return;
const double multiplier = 1.0 / sqrt(squared_norm);
for (float& x : v) x *= multiplier;
}
StatusOrSearcherUntyped RetrainAndReindexSearcher(
UntypedSingleMachineSearcherBase* searcher,
absl::Mutex* searcher_pointer_mutex, const ScannConfig& config,
shared_ptr<ThreadPool> parallelization_pool)
ABSL_NO_THREAD_SAFETY_ANALYSIS {
if (searcher_pointer_mutex) searcher_pointer_mutex->AssertNotHeld();
SCANN_RET_CHECK(searcher);
return SCANN_CALL_FUNCTION_BY_TAG(
searcher->TypeTag(), RetrainAndReindexSearcherImpl, searcher,
searcher_pointer_mutex, config, std::move(parallelization_pool));
}
}