#ifndef SCANN_PARTITIONING_PARTITIONER_BASE_H_
#define SCANN_PARTITIONING_PARTITIONER_BASE_H_
#include <algorithm>
#include <cstdint>
#include <vector>
#include "scann/data_format/datapoint.h"
#include "scann/data_format/dataset.h"
#include "scann/oss_wrappers/scann_threadpool.h"
#include "scann/partitioning/partitioner.pb.h"
#include "scann/utils/common.h"
#include "scann/utils/types.h"
#include "scann/utils/util_functions.h"
namespace research_scann {
class UntypedPartitioner {
public:
virtual ~UntypedPartitioner();
virtual void CopyToProto(SerializedPartitioner* result) const = 0;
virtual int32_t n_tokens() const = 0;
virtual Normalization NormalizationRequired() const { return NONE; }
enum TokenizationMode {
DATABASE,
QUERY
};
TokenizationMode tokenization_mode() const { return tokenization_mode_; }
void set_tokenization_mode(TokenizationMode val) {
tokenization_mode_ = val;
OnSetTokenizationMode();
}
virtual int8_t TypeTag() const = 0;
virtual void set_training_parallelization_pool(shared_ptr<ThreadPool> pool);
protected:
virtual void OnSetTokenizationMode() {}
void set_tokenization_mode_no_hook(TokenizationMode tokenization_mode) {
tokenization_mode_ = tokenization_mode;
}
private:
TokenizationMode tokenization_mode_ = DATABASE;
};
template <typename T>
class Partitioner : public UntypedPartitioner {
public:
virtual unique_ptr<Partitioner<T>> Clone() const = 0;
virtual Status TokenForDatapoint(const DatapointPtr<T>& query,
int32_t* result) const = 0;
virtual Status TokensForDatapointWithSpilling(
const DatapointPtr<T>& query, std::vector<int32_t>* result) const = 0;
virtual Status TokenForDatapointBatched(const TypedDatasetView<T>& queries,
std::vector<int32_t>* results,
ThreadPool* pool = nullptr) const;
virtual Status TokensForDatapointWithSpillingBatched(
const TypedDatasetView<T>& queries,
MutableSpan<std::vector<int32_t>> results,
ThreadPool* pool = nullptr) const;
virtual StatusOr<vector<std::vector<DatapointIndex>>> TokenizeDatabase(
const TypedDatasetView<T>& database, ThreadPool* pool_or_null) const;
int8_t TypeTag() const final { return TagForType<T>(); }
};
SCANN_INSTANTIATE_TYPED_CLASS(extern, Partitioner);
}
#endif