#ifndef SCANN_UTILS_IO_NPY_H_
#define SCANN_UTILS_IO_NPY_H_
#include <cstddef>
#include <fstream>
#include <istream>
#include <streambuf>
#include <string>
#include <utility>
#include <vector>
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/string_view.h"
#include "scann/data_format/dataset.h"
#include "scann/oss_wrappers/scann_status.h"
#include "scann/utils/common.h"
#include "scann/utils/io_oss_wrapper.h"
#include "scann/utils/types.h"
namespace research_scann {
template <typename T>
std::string numpy_type_name();
template <typename T>
Status SpanToNumpy(absl::string_view filename, ConstSpan<T> data,
ConstSpan<size_t> dim_sizes = {},
DimensionIndex last_dim = kInvalidDimension) {
std::string shape_str = "(";
size_t dim_prod = 1;
for (size_t dim_size : dim_sizes) {
dim_prod *= dim_size;
shape_str += std::to_string(dim_size) + ",";
}
if (last_dim != kInvalidDimension) {
shape_str += std::to_string(last_dim) + ",)";
} else {
if (dim_prod == 0 || data.size() % dim_prod != 0)
return InvalidArgumentError(
"Size of data isn't compatible with given shape");
shape_str += std::to_string(data.size() / dim_prod) + ",)";
}
if (shape_str.size() > 65000)
return InvalidArgumentError("Shape string is too large for npy format: " +
shape_str);
std::string pt1("\x93NUMPY\x01\x00 ", 10);
std::string dict =
absl::StrFormat("{'descr':%s, 'fortran_order':False, 'shape':%s}",
numpy_type_name<T>(), shape_str);
while ((pt1.size() + dict.size()) % 64 != 63) dict += ' ';
dict += '\n';
pt1[8] = dict.size() % 256;
pt1[9] = dict.size() / 256;
const std::string header = pt1 + dict;
OpenSourceableFileWriter writer(filename);
SCANN_RETURN_IF_ERROR(
writer.Write(ConstSpan<char>(header.data(), header.size())));
const char* ptr = reinterpret_cast<const char*>(data.data());
return writer.Write(ConstSpan<char>(ptr, data.size() * sizeof(T)));
}
template <typename T>
Status VectorToNumpy(absl::string_view filename, const vector<T>& data,
const vector<size_t>& dim_sizes = {}) {
return SpanToNumpy(filename, ConstSpan<T>(data.data(), data.size()),
ConstSpan<size_t>(dim_sizes.data(), dim_sizes.size()));
}
template <typename T>
Status DatasetToNumpy(absl::string_view filename, const DenseDataset<T>& data) {
return SpanToNumpy(filename, data.data(), {data.size()},
data.dimensionality());
}
namespace npy_internal {
StatusOr<std::vector<size_t>> ReadNpyHeader(absl::string_view name,
absl::string_view expected_descr,
size_t word_size,
std::istream& in);
StatusOr<std::vector<size_t>> OpenNpy(absl::string_view filename,
absl::string_view expected_descr,
size_t word_size, std::ifstream& in);
template <typename T>
StatusOr<pair<std::vector<T>, std::vector<size_t>>> ReadNpyData(
absl::string_view name, std::vector<size_t> shape, std::istream& in) {
size_t total_size = 1;
for (size_t s : shape) total_size *= s;
vector<T> buffer(total_size);
if (total_size > 0 && !in.read(reinterpret_cast<char*>(buffer.data()),
total_size * sizeof(T)))
return InternalError(absl::StrCat("I/O error reading ", name));
return std::make_pair(std::move(buffer), std::move(shape));
}
}
class ConstMemoryStreamBuf : public std::streambuf {
public:
ConstMemoryStreamBuf(const char* data, size_t size) {
char* p = const_cast<char*>(data);
setg(p, p, p + size);
}
protected:
pos_type seekoff(off_type off, std::ios_base::seekdir dir,
std::ios_base::openmode which) override {
if (!(which & std::ios_base::in)) return pos_type(off_type(-1));
off_type base = 0;
if (dir == std::ios_base::cur) base = gptr() - eback();
if (dir == std::ios_base::end) base = egptr() - eback();
const off_type pos = base + off;
if (pos < 0 || pos > egptr() - eback()) return pos_type(off_type(-1));
setg(eback(), eback() + pos, egptr());
return pos_type(pos);
}
pos_type seekpos(pos_type pos, std::ios_base::openmode which) override {
return seekoff(off_type(pos), std::ios_base::beg, which);
}
};
template <typename T>
StatusOr<pair<std::vector<T>, std::vector<size_t>>> NumpyToVectorAndShape(
absl::string_view filename) {
std::ifstream in;
SCANN_ASSIGN_OR_RETURN(
std::vector<size_t> shape,
npy_internal::OpenNpy(filename, numpy_type_name<T>(), sizeof(T), in));
return npy_internal::ReadNpyData<T>(filename, std::move(shape), in);
}
template <typename T>
StatusOr<pair<std::vector<T>, std::vector<size_t>>> NumpyBytesToVectorAndShape(
absl::string_view name, absl::string_view bytes) {
ConstMemoryStreamBuf buf(bytes.data(), bytes.size());
std::istream in(&buf);
SCANN_ASSIGN_OR_RETURN(
std::vector<size_t> shape,
npy_internal::ReadNpyHeader(name, numpy_type_name<T>(), sizeof(T), in));
return npy_internal::ReadNpyData<T>(name, std::move(shape), in);
}
}
#endif