#ifndef SCANN_UTILS_BFLOAT16_HELPERS_H_
#define SCANN_UTILS_BFLOAT16_HELPERS_H_
#include <cmath>
#include <cstdint>
#include <vector>
#include "absl/base/casts.h"
#include "scann/data_format/datapoint.h"
#include "scann/data_format/dataset.h"
#include "scann/oss_wrappers/scann_threadpool.h"
#include "scann/utils/common.h"
namespace research_scann {
SCANN_INLINE int16_t Bfloat16Quantize(float value) {
uint32_t value_bits = absl::bit_cast<uint32_t>(value);
if (std::isinf(value)) return value_bits >> 16;
if (std::isnan(value)) return (value_bits >> 16) | 1;
uint32_t rounded_bits = value_bits + 0x8000;
if (((rounded_bits >> 23) & 0xff) == 255) {
return ((value_bits >> 31) << 15) | 0b1111'1110'1111'111;
}
return rounded_bits >> 16;
}
SCANN_INLINE float Bfloat16Decompress(int16_t value) {
// scann-core: shift as unsigned. Upstream shifted the signed int16_t, which
// is undefined behavior for negative values (i.e. negative numbers) in
// C++17. Same bits, no UB.
uint32_t value32 = static_cast<uint32_t>(static_cast<uint16_t>(value)) << 16;
return absl::bit_cast<float>(value32);
}
DenseDataset<int16_t> Bfloat16QuantizeFloatDataset(
const DenseDataset<float>& dataset);
DatapointPtr<int16_t> Bfloat16QuantizeFloatDatapoint(
const DatapointPtr<float>& dptr, std::vector<int16_t>* quantized_storage);
DatapointPtr<int16_t> Bfloat16QuantizeFloatDatapoint(
const DatapointPtr<float>& dptr, MutableSpan<int16_t> quantized);
DatapointPtr<int16_t> Bfloat16QuantizeFloatDatapointWithNoiseShaping(
const DatapointPtr<float>& dptr, double noise_shaping_threshold,
vector<int16_t>* quantized_storage, int* num_changes = nullptr,
double* residual_ptr = nullptr, double* parallel_residual_ptr = nullptr);
DatapointPtr<int16_t> Bfloat16QuantizeFloatDatapointWithNoiseShaping(
const DatapointPtr<float>& dptr, double noise_shaping_threshold,
MutableSpan<int16_t> quantized, int* num_changes = nullptr,
double* residual_ptr = nullptr, double* parallel_residual_ptr = nullptr);
DatapointPtr<int16_t> Bfloat16QuantizeFloatDatapointWithNoiseShaping(
const DatapointPtr<float>& dptr, double noise_shaping_threshold,
MutableSpan<int16_t> quantized, MutableSpan<float> residuals,
MutableSpan<uint32_t> dims, int* num_changes = nullptr,
double* residual_ptr = nullptr, double* parallel_residual_ptr = nullptr);
DenseDataset<int16_t> Bfloat16QuantizeFloatDatasetWithNoiseShaping(
const DenseDataset<float>& dataset, float noise_shaping_threshold,
ThreadPool* pool = nullptr);
} // namespace research_scann
#endif