#include "kv_options.h"
#include <glog/logging.h>
#include <algorithm>
#include <bit>
#include <cctype>
#include <charconv>
#include <limits>
#include <numeric>
#include <string>
#include <string_view>
#include <vector>
#include "inih/cpp/INIReader.h"
namespace eloqstore
{
static uint64_t ParseSizeWithUnit(std::string_view s)
{
auto is_space = [](unsigned char c) { return std::isspace(c); };
while (!s.empty() && is_space(static_cast<unsigned char>(s.front())))
{
s.remove_prefix(1);
}
while (!s.empty() && is_space(static_cast<unsigned char>(s.back())))
{
s.remove_suffix(1);
}
if (s.empty())
{
return 0;
}
uint64_t mul = 1;
if (s.size() >= 2)
{
const char c1 =
std::toupper(static_cast<unsigned char>(s[s.size() - 2]));
const char c2 =
std::toupper(static_cast<unsigned char>(s[s.size() - 1]));
if (c1 == 'K' && c2 == 'B')
{
mul = 1ULL << 10;
s.remove_suffix(2);
}
else if (c1 == 'M' && c2 == 'B')
{
mul = 1ULL << 20;
s.remove_suffix(2);
}
else if (c1 == 'G' && c2 == 'B')
{
mul = 1ULL << 30;
s.remove_suffix(2);
}
else if (c1 == 'T' && c2 == 'B')
{
mul = 1ULL << 40;
s.remove_suffix(2);
}
}
while (!s.empty() && is_space(static_cast<unsigned char>(s.back())))
{
s.remove_suffix(1);
}
if (s.empty())
{
return 0;
}
uint64_t v = 0;
auto [ptr, ec] = std::from_chars(s.data(), s.data() + s.size(), v);
if (ec != std::errc() || ptr != s.data() + s.size())
{
return 0;
}
return v * mul;
}
int KvOptions::LoadFromIni(const char *path)
{
INIReader reader(path);
if (int res = reader.ParseError(); res != 0)
{
return res;
}
constexpr char sec_run[] = "run";
if (!reader.HasSection(sec_run))
{
return -2;
}
if (reader.HasValue(sec_run, "num_threads"))
{
num_threads = reader.GetUnsigned(sec_run, "num_threads", 1);
}
if (reader.HasValue(sec_run, "data_page_restart_interval"))
{
data_page_restart_interval =
reader.GetUnsigned(sec_run, "data_page_restart_interval", 16);
}
if (reader.HasValue(sec_run, "index_page_restart_interval"))
{
index_page_restart_interval =
reader.GetUnsigned(sec_run, "index_page_restart_interval", 16);
}
if (reader.HasValue(sec_run, "init_page_count"))
{
init_page_count =
reader.GetUnsigned(sec_run, "init_page_count", 1 << 15);
}
if (reader.HasValue(sec_run, "skip_verify_checksum"))
{
skip_verify_checksum =
reader.GetBoolean(sec_run, "skip_verify_checksum", false);
}
if (reader.HasValue(sec_run, "buffer_pool_size"))
{
std::string buffer_pool_size_str =
reader.Get(sec_run, "buffer_pool_size", "");
buffer_pool_size = ParseSizeWithUnit(buffer_pool_size_str);
}
if (reader.HasValue(sec_run, "root_meta_cache_size"))
{
std::string root_meta_cache_size_str =
reader.Get(sec_run, "root_meta_cache_size", "");
root_meta_cache_size = ParseSizeWithUnit(root_meta_cache_size_str);
}
if (reader.HasValue(sec_run, "manifest_limit"))
{
manifest_limit = reader.GetUnsigned(sec_run, "manifest_limit", 8 * MB);
}
if (reader.HasValue(sec_run, "fd_limit"))
{
fd_limit = reader.GetUnsigned(sec_run, "fd_limit", 1024);
}
if (reader.HasValue(sec_run, "io_queue_size"))
{
io_queue_size = reader.GetUnsigned(sec_run, "io_queue_size", 4096);
}
if (reader.HasValue(sec_run, "max_inflight_write"))
{
max_inflight_write =
reader.GetUnsigned(sec_run, "max_inflight_write", 4096);
}
if (reader.HasValue(sec_run, "max_write_batch_pages"))
{
max_write_batch_pages =
reader.GetUnsigned(sec_run, "max_write_batch_pages", 64);
}
if (reader.HasValue(sec_run, "coroutine_stack_size"))
{
coroutine_stack_size =
reader.GetUnsigned(sec_run, "coroutine_stack_size", 16 * KB);
}
if (reader.HasValue(sec_run, "num_retained_archives"))
{
num_retained_archives =
reader.GetUnsigned(sec_run, "num_retained_archives", 0);
}
if (reader.HasValue(sec_run, "archive_interval_secs"))
{
archive_interval_secs =
reader.GetUnsigned(sec_run, "archive_interval_secs", 86400);
}
if (reader.HasValue(sec_run, "max_archive_tasks"))
{
max_archive_tasks =
reader.GetUnsigned(sec_run, "max_archive_tasks", 256);
}
if (reader.HasValue(sec_run, "max_global_request_batch"))
{
max_global_request_batch = reader.GetUnsigned(
sec_run, "max_global_request_batch", max_global_request_batch);
}
if (reader.HasValue(sec_run, "file_amplify_factor"))
{
file_amplify_factor =
reader.GetUnsigned(sec_run, "file_amplify_factor", 2);
}
if (reader.HasValue(sec_run, "local_space_limit"))
{
local_space_limit = reader.GetUnsigned(sec_run, "local_space_limit", 0);
}
if (reader.HasValue(sec_run, "reserve_space_ratio"))
{
reserve_space_ratio =
reader.GetUnsigned(sec_run, "reserve_space_ratio", 100);
}
if (reader.HasValue(sec_run, "max_cloud_concurrency"))
{
max_cloud_concurrency = reader.GetUnsigned(
sec_run, "max_cloud_concurrency", max_cloud_concurrency);
}
if (reader.HasValue(sec_run, "max_write_concurrency"))
{
max_write_concurrency = reader.GetUnsigned(
sec_run, "max_write_concurrency", max_write_concurrency);
}
if (reader.HasValue(sec_run, "cloud_request_threads"))
{
cloud_request_threads = reader.GetUnsigned(
sec_run, "cloud_request_threads", cloud_request_threads);
}
if (reader.HasValue(sec_run, "standby_max_concurrency"))
{
standby_max_concurrency = reader.GetUnsigned(
sec_run, "standby_max_concurrency", standby_max_concurrency);
}
if (reader.HasValue(sec_run, "direct_io_buffer_pool_size"))
{
direct_io_buffer_pool_size = reader.GetUnsigned(
sec_run, "direct_io_buffer_pool_size", direct_io_buffer_pool_size);
}
if (reader.HasValue(sec_run, "write_buffer_size"))
{
std::string write_buffer_size_str =
reader.Get(sec_run, "write_buffer_size", "");
write_buffer_size = ParseSizeWithUnit(write_buffer_size_str);
}
if (reader.HasValue(sec_run, "non_page_io_batch_size"))
{
std::string non_page_io_batch_size_str =
reader.Get(sec_run, "non_page_io_batch_size", "");
non_page_io_batch_size = ParseSizeWithUnit(non_page_io_batch_size_str);
}
if (reader.HasValue(sec_run, "write_buffer_ratio"))
{
write_buffer_ratio =
reader.GetReal(sec_run, "write_buffer_ratio", write_buffer_ratio);
}
if (reader.HasValue(sec_run, "allow_reuse_local_caches"))
{
allow_reuse_local_caches =
reader.GetBoolean(sec_run, "allow_reuse_local_caches", false);
}
if (reader.HasValue(sec_run, "prewarm_cloud_cache"))
{
prewarm_cloud_cache =
reader.GetBoolean(sec_run, "prewarm_cloud_cache", false);
}
if (reader.HasValue(sec_run, "prewarm_task_count"))
{
prewarm_task_count =
reader.GetUnsigned(sec_run, "prewarm_task_count", 1);
}
write_buffer_ratio = std::clamp(write_buffer_ratio, 0.0, 1.0);
constexpr char sec_permanent[] = "permanent";
if (!reader.HasSection(sec_permanent))
{
return -2;
}
if (reader.HasValue(sec_permanent, "store_path"))
{
std::string input = reader.Get(sec_permanent, "store_path", "");
std::string error_message;
if (!ParseStorePathListWithWeights(
input, store_path, store_path_weights, &error_message))
{
LOG(ERROR) << "Invalid store_path: " << error_message;
return -3;
}
}
if (reader.HasValue(sec_permanent, "cloud_store_path"))
{
cloud_store_path = reader.Get(sec_permanent, "cloud_store_path", "");
}
if (reader.HasValue(sec_permanent, "standby_master_addr"))
{
standby_master_addr =
reader.Get(sec_permanent, "standby_master_addr", "");
}
if (reader.HasValue(sec_permanent, "standby_master_store_paths"))
{
std::string input =
reader.Get(sec_permanent, "standby_master_store_paths", "");
std::string error_message;
if (!ParseStorePathListWithWeights(input,
standby_master_store_paths,
standby_master_store_path_weights,
&error_message))
{
LOG(ERROR) << "Invalid standby_master_store_paths: "
<< error_message;
return -3;
}
}
if (reader.HasValue(sec_permanent, "cloud_provider"))
{
cloud_provider = reader.Get(sec_permanent, "cloud_provider", "aws");
}
if (reader.HasValue(sec_permanent, "cloud_endpoint"))
{
cloud_endpoint = reader.Get(sec_permanent, "cloud_endpoint", "");
}
if (reader.HasValue(sec_permanent, "cloud_region"))
{
cloud_region = reader.Get(sec_permanent, "cloud_region", cloud_region);
}
if (reader.HasValue(sec_permanent, "cloud_access_key"))
{
cloud_access_key =
reader.Get(sec_permanent, "cloud_access_key", cloud_access_key);
}
if (reader.HasValue(sec_permanent, "cloud_secret_key"))
{
cloud_secret_key =
reader.Get(sec_permanent, "cloud_secret_key", cloud_secret_key);
}
if (reader.HasValue(sec_permanent, "cloud_auto_credentials"))
{
cloud_auto_credentials = reader.GetBoolean(
sec_permanent, "cloud_auto_credentials", cloud_auto_credentials);
}
if (reader.HasValue(sec_permanent, "cloud_verify_ssl"))
{
cloud_verify_ssl = reader.GetBoolean(
sec_permanent, "cloud_verify_ssl", cloud_verify_ssl);
}
if (reader.HasValue(sec_permanent, "data_page_size"))
{
std::string value_str =
reader.Get(sec_permanent, "data_page_size", "4KB");
uint64_t parsed_size = ParseSizeWithUnit(value_str);
data_page_size = (parsed_size > 0) ? parsed_size : (1 << 12);
}
if (reader.HasValue(sec_permanent, "data_file_size"))
{
std::string value_str =
reader.Get(sec_permanent, "data_file_size", "8MB");
uint64_t parsed_size = ParseSizeWithUnit(value_str);
uint32_t data_file_size = (parsed_size > 0) ? parsed_size : (8 * MB);
uint32_t pages_per_file = data_file_size / data_page_size;
if (pages_per_file == 0)
{
LOG(WARNING) << "data_file_size " << data_file_size
<< " is smaller than data_page_size " << data_page_size
<< ", falling back to one page.";
pages_per_file_shift = 0;
}
else
{
pages_per_file_shift = std::numeric_limits<uint32_t>::digits -
std::countl_zero(pages_per_file) - 1;
if ((pages_per_file & (pages_per_file - 1)) != 0)
{
uint32_t adjusted_pages = 1U << pages_per_file_shift;
uint64_t adjusted_size =
static_cast<uint64_t>(adjusted_pages) * data_page_size;
LOG(WARNING) << "data_file_size " << data_file_size
<< " is not a power-of-two multiple of page size "
<< data_page_size << ", rounded down to "
<< adjusted_size << " bytes.";
}
}
}
if (reader.HasValue(sec_permanent, "overflow_pointers"))
{
overflow_pointers =
reader.GetUnsigned(sec_permanent, "overflow_pointers", 16);
}
if (reader.HasValue(sec_permanent, "data_append_mode"))
{
data_append_mode =
reader.GetBoolean(sec_permanent, "data_append_mode", false);
}
return 0;
}
bool KvOptions::operator==(const KvOptions &other) const
{
return num_threads == other.num_threads &&
data_page_restart_interval == other.data_page_restart_interval &&
index_page_restart_interval == other.index_page_restart_interval &&
init_page_count == other.init_page_count &&
skip_verify_checksum == other.skip_verify_checksum &&
buffer_pool_size == other.buffer_pool_size &&
root_meta_cache_size == other.root_meta_cache_size &&
manifest_limit == other.manifest_limit &&
fd_limit == other.fd_limit && io_queue_size == other.io_queue_size &&
max_inflight_write == other.max_inflight_write &&
max_write_batch_pages == other.max_write_batch_pages &&
coroutine_stack_size == other.coroutine_stack_size &&
num_retained_archives == other.num_retained_archives &&
archive_interval_secs == other.archive_interval_secs &&
max_archive_tasks == other.max_archive_tasks &&
max_global_request_batch == other.max_global_request_batch &&
file_amplify_factor == other.file_amplify_factor &&
local_space_limit == other.local_space_limit &&
reserve_space_ratio == other.reserve_space_ratio &&
max_cloud_concurrency == other.max_cloud_concurrency &&
max_write_concurrency == other.max_write_concurrency &&
cloud_request_threads == other.cloud_request_threads &&
standby_max_concurrency == other.standby_max_concurrency &&
direct_io_buffer_pool_size == other.direct_io_buffer_pool_size &&
write_buffer_size == other.write_buffer_size &&
non_page_io_batch_size == other.non_page_io_batch_size &&
write_buffer_ratio == other.write_buffer_ratio &&
allow_reuse_local_caches == other.allow_reuse_local_caches &&
prewarm_cloud_cache == other.prewarm_cloud_cache &&
prewarm_task_count == other.prewarm_task_count &&
auto_reopen_retry_times == other.auto_reopen_retry_times &&
store_path == other.store_path &&
store_path_weights == other.store_path_weights &&
cloud_store_path == other.cloud_store_path &&
standby_master_addr == other.standby_master_addr &&
standby_master_store_paths == other.standby_master_store_paths &&
standby_master_store_path_weights ==
other.standby_master_store_path_weights &&
cloud_provider == other.cloud_provider &&
cloud_endpoint == other.cloud_endpoint &&
cloud_region == other.cloud_region &&
cloud_access_key == other.cloud_access_key &&
cloud_secret_key == other.cloud_secret_key &&
cloud_auto_credentials == other.cloud_auto_credentials &&
cloud_verify_ssl == other.cloud_verify_ssl &&
data_page_size == other.data_page_size &&
pages_per_file_shift == other.pages_per_file_shift &&
overflow_pointers == other.overflow_pointers &&
data_append_mode == other.data_append_mode;
}
}