#include "store/enceval.h"
#include <math.h>
#include <string.h>
static inline int cmp_one(int64_t x, uint16_t op, int64_t c) {
switch (op) {
case OP_EQ: return x == c; case OP_NE: return x != c;
case OP_LT: return x < c; case OP_LE: return x <= c;
case OP_GT: return x > c; case OP_GE: return x >= c;
default: return 0;
}
}
static inline int cmp_one_f(double x, uint16_t op, double c) {
switch (op) {
case OP_EQ: return x == c; case OP_NE: return x != c;
case OP_LT: return x < c; case OP_LE: return x <= c;
case OP_GT: return x > c; case OP_GE: return x >= c;
default: return 0;
}
}
static inline uint64_t lane(const ray_enc_lanes_t* l, int T, uint32_t i) {
switch (T) { case 8: return l->b[i]; case 16: return l->h[i]; case 32: return l->s[i]; default: return l->q[i]; }
}
static inline bool is_float(int8_t t) { return t == RAY_F32 || t == RAY_F64; }
static inline bool is_cmp_op(uint16_t op) { return op >= OP_EQ && op <= OP_GE; }
void ray_enc_mask_nulls(const ray_enc_vview_t* v, uint8_t nr, uint8_t* bits) {
if (!v->nullmap) return;
for (int i = 0; i < v->n; i++)
if ((v->nullmap[i >> 3] >> (i & 7)) & 1) bits[i] = nr;
}
int ray_enc_const_decision(const ray_enc_vview_t* v, uint16_t op, int64_t c, uint8_t nr) {
if (v->enc != RAY_ENC_CONST || is_float(v->type) || !is_cmp_op(op)) return -1;
if (v->type == RAY_SYM && op != OP_EQ && op != OP_NE) return -1;
if (v->nullmap && v->n_nulls == v->n) return nr;
int r = cmp_one(ray_enc_logical(v->cbits, v->k), op, c);
if (!v->nullmap || r == nr) return r;
return -1;
}
int ray_enc_const_decision_f64(const ray_enc_vview_t* v, uint16_t op, double c, uint8_t nr) {
if (v->enc != RAY_ENC_CONST || !is_float(v->type) || !is_cmp_op(op)) return -1;
if (isnan(c)) return -1;
if (v->nullmap && v->n_nulls == v->n) return nr;
int r = cmp_one_f(ray_enc_fval(v->cbits, v->type == RAY_F64 ? 64 : 32), op, c);
if (!v->nullmap || r == nr) return r;
return -1;
}
#define EVAL_RES(UT) do { const UT* u_ = (const UT*)res; const UT t_ = (UT)tt; \
switch (op) { \
case OP_EQ: for (int i = 0; i < v->n; i++) bits[i] = u_[i] == t_; break; \
case OP_NE: for (int i = 0; i < v->n; i++) bits[i] = u_[i] != t_; break; \
case OP_LT: for (int i = 0; i < v->n; i++) bits[i] = u_[i] < t_; break; \
case OP_LE: for (int i = 0; i < v->n; i++) bits[i] = u_[i] <= t_; break; \
case OP_GT: for (int i = 0; i < v->n; i++) bits[i] = u_[i] > t_; break; \
case OP_GE: for (int i = 0; i < v->n; i++) bits[i] = u_[i] >= t_; break; \
default: memset(bits, 0, (size_t)v->n); break; } } while (0)
static void eval_for(const ray_enc_vview_t* v, uint16_t op, int64_t c, uint8_t* bits) {
const uint64_t umax = v->width >= 64 ? ~0ull : ((1ull << v->width) - 1);
const __int128 t = (__int128)c - (__int128)ray_enc_logical(v->base, v->k);
if (t < 0 || t > (__int128)umax) {
memset(bits, cmp_one(t < 0 ? 1 : -1, op, 0), (size_t)v->n);
return;
}
const uint64_t tt = (uint64_t)t;
_Alignas(64) ray_enc_lanes_t lanes;
const void* res = &lanes;
ray_enccol_unpack_resid(v, &lanes);
switch (v->k.T) {
case 8: EVAL_RES(uint8_t); break;
case 16: EVAL_RES(uint16_t); break;
case 32: EVAL_RES(uint32_t); break;
default: EVAL_RES(uint64_t); break;
}
}
#undef EVAL_RES
static uint32_t lower_bound(const ray_enc_vview_t* v, int64_t c) {
uint32_t lo = 0, hi = v->dict_n;
while (lo < hi) { uint32_t mid = lo + (hi - lo) / 2; if (ray_enc_logical(v->dict[mid], v->k) < c) lo = mid + 1; else hi = mid; }
return lo;
}
static uint32_t upper_bound(const ray_enc_vview_t* v, int64_t c) {
uint32_t lo = 0, hi = v->dict_n;
while (lo < hi) { uint32_t mid = lo + (hi - lo) / 2; if (ray_enc_logical(v->dict[mid], v->k) <= c) lo = mid + 1; else hi = mid; }
return lo;
}
static void eval_dict(const ray_enc_vview_t* v, uint16_t op, int64_t c, uint8_t* bits) {
const uint32_t lb = lower_bound(v, c), ub = upper_bound(v, c);
uint32_t lo = 0, hi = 0; uint8_t neg = 0;
switch (op) {
case OP_EQ: lo = lb; hi = ub; break;
case OP_NE: lo = lb; hi = ub; neg = 1; break;
case OP_LT: lo = 0; hi = lb; break;
case OP_LE: lo = 0; hi = ub; break;
case OP_GT: lo = ub; hi = v->dict_n; break;
case OP_GE: lo = lb; hi = v->dict_n; break;
default: break;
}
const uint32_t span = hi - lo;
for (int i = 0; i < v->n; i++)
bits[i] = (uint8_t)((v->codes[i] - lo < span) ^ neg);
}
static bool in_set(const int64_t* set, int nset, int64_t x) {
int lo = 0, hi = nset;
while (lo < hi) { int mid = lo + (hi - lo) / 2; if (set[mid] < x) lo = mid + 1; else hi = mid; }
return lo < nset && set[lo] == x;
}
typedef struct { bool member; uint16_t op; int64_t c; const int64_t* set; int nset; } run_pred_t;
static inline int run_pred(const run_pred_t* p, int64_t x) {
return p->member ? in_set(p->set, p->nset, x) : cmp_one(x, p->op, p->c);
}
static inline int64_t logical_of(uint64_t bits, ray_enc_kind_t k) {
if (k.T == 64) return (int64_t)bits;
const uint64_t m = (1ull << k.T) - 1;
bits &= m;
return k.is_signed && (bits >> (k.T - 1)) & 1 ? (int64_t)(bits | ~m) : (int64_t)bits;
}
static inline void fill_run(uint8_t* bits, uint32_t i, uint32_t e, uint32_t n, uint8_t b) {
const uint64_t pat = 0x0101010101010101ull * b;
for (; i + 8 <= e; i += 8) memcpy(bits + i, &pat, 8);
if (i < e) {
if (i + 8 <= n) memcpy(bits + i, &pat, 8);
else for (; i < e; i++) bits[i] = b;
}
}
static void eval_runs(const ray_enc_vview_t* v, const run_pred_t* p, uint8_t* bits) {
uint32_t i = 0;
for (uint32_t r = 0; r < v->n_runs; r++) {
const uint8_t b = (uint8_t)run_pred(p, logical_of(lane(&v->run_vals, v->k.T, r), v->k));
const uint32_t e = v->run_ends.h[r];
fill_run(bits, i, e, (uint32_t)v->n, b);
i = e;
}
}
#define RUNS_HOLD(UT, F) do { const UT x_ = (UT)pat; bool hit_ = false; \
for (uint32_t r = 0; r < v->n_runs; r++) hit_ |= v->run_vals.F[r] == x_; \
return hit_; } while (0)
static bool runs_hold(const ray_enc_vview_t* v, int64_t c) {
const int T = v->k.T;
if (T < 64) {
const int64_t lo = v->k.is_signed ? -((int64_t)1 << (T - 1)) : 0;
const int64_t hi = v->k.is_signed ? ((int64_t)1 << (T - 1)) - 1 : (int64_t)((1ull << T) - 1);
if (c < lo || c > hi) return false;
} else if (!v->k.is_signed && c < 0) {
return false;
}
const uint64_t pat = T == 64 ? (uint64_t)c : (uint64_t)c & ((1ull << T) - 1);
switch (T) {
case 8: RUNS_HOLD(uint8_t, b);
case 16: RUNS_HOLD(uint16_t, h);
case 32: RUNS_HOLD(uint32_t, s);
default: RUNS_HOLD(uint64_t, q);
}
}
#undef RUNS_HOLD
#define RUNS_EQ(UT, F) do { const UT x_ = (UT)pat; \
for (uint32_t r = 0; r < v->n_runs; r++) { \
const uint32_t e_ = v->run_ends.h[r]; \
fill_run(bits, i, e_, (uint32_t)v->n, (uint8_t)((v->run_vals.F[r] == x_) ^ ne)); \
i = e_; \
} } while (0)
static void eval_runs_eq(const ray_enc_vview_t* v, int64_t c, uint8_t ne, uint8_t* bits) {
const int T = v->k.T;
const uint64_t pat = T == 64 ? (uint64_t)c : (uint64_t)c & ((1ull << T) - 1);
uint32_t i = 0;
switch (T) {
case 8: RUNS_EQ(uint8_t, b); break;
case 16: RUNS_EQ(uint16_t, h); break;
case 32: RUNS_EQ(uint32_t, s); break;
default: RUNS_EQ(uint64_t, q); break;
}
}
#undef RUNS_EQ
bool ray_enc_eval_cmp_f64(const ray_enc_vview_t* v, uint16_t op, double c, uint8_t nr, uint8_t* bits) {
if (!is_float(v->type) || !is_cmp_op(op) || isnan(c)) return false;
const int T = v->k.T;
switch (v->enc) {
case RAY_ENC_CONST:
memset(bits, cmp_one_f(ray_enc_fval(v->cbits, T), op, c), (size_t)v->n);
break;
case RAY_ENC_RLE: {
uint32_t i = 0;
for (uint32_t r = 0; r < v->n_runs; r++) {
const uint32_t e = v->run_ends.h[r];
fill_run(bits, i, e, (uint32_t)v->n, (uint8_t)cmp_one_f(ray_enc_fval(lane(&v->run_vals, T, r), T), op, c));
i = e;
}
break;
}
case RAY_ENC_DICT: {
uint32_t lo = 0, hi = v->dict_n;
while (lo < hi) { uint32_t mid = lo + (hi - lo) / 2; if (ray_enc_fval(v->dict[mid], T) < c) lo = mid + 1; else hi = mid; }
const uint32_t lb = lo;
hi = v->dict_n;
while (lo < hi) { uint32_t mid = lo + (hi - lo) / 2; if (ray_enc_fval(v->dict[mid], T) <= c) lo = mid + 1; else hi = mid; }
const uint32_t ub = lo;
uint32_t a = 0, b = 0; uint8_t neg = 0;
switch (op) {
case OP_EQ: a = lb; b = ub; break;
case OP_NE: a = lb; b = ub; neg = 1; break;
case OP_LT: a = 0; b = lb; break;
case OP_LE: a = 0; b = ub; break;
case OP_GT: a = ub; b = v->dict_n; break;
default: a = lb; b = v->dict_n; break;
}
const uint32_t span = b - a;
for (int i = 0; i < v->n; i++)
bits[i] = (uint8_t)((v->codes[i] - a < span) ^ neg);
break;
}
default:
return false;
}
ray_enc_mask_nulls(v, nr, bits);
return true;
}
bool ray_enc_eval_cmp(const ray_enc_vview_t* v, uint16_t op, int64_t c, uint8_t nr, uint8_t* bits) {
if (is_float(v->type)) return false;
if (!is_cmp_op(op)) return false;
if (v->type == RAY_SYM && op != OP_EQ && op != OP_NE) return false;
switch (v->enc) {
case RAY_ENC_CONST:
memset(bits, cmp_one(ray_enc_logical(v->cbits, v->k), op, c), (size_t)v->n);
break;
case RAY_ENC_FOR_BP: eval_for(v, op, c, bits); break;
case RAY_ENC_RLE:
if ((op == OP_EQ || op == OP_NE) && !runs_hold(v, c)) memset(bits, op == OP_NE, (size_t)v->n);
else if (op == OP_EQ || op == OP_NE) eval_runs_eq(v, c, op == OP_NE, bits);
else eval_runs(v, &(run_pred_t){ false, op, c, NULL, 0 }, bits);
break;
case RAY_ENC_DICT: eval_dict(v, op, c, bits); break;
default: return false;
}
ray_enc_mask_nulls(v, nr, bits);
return true;
}
bool ray_enc_eval_cmp_runs(const ray_enc_vview_t* v, uint16_t op, int64_t c, uint8_t* pass) {
if (v->enc != RAY_ENC_RLE || v->nullmap || is_float(v->type) || !is_cmp_op(op)) return false;
if (v->type == RAY_SYM && op != OP_EQ && op != OP_NE) return false;
if ((op == OP_EQ || op == OP_NE) && !runs_hold(v, c)) {
memset(pass, op == OP_NE, v->n_runs);
return true;
}
if (op == OP_EQ || op == OP_NE) {
const int T = v->k.T;
const uint64_t pat = T == 64 ? (uint64_t)c : (uint64_t)c & ((1ull << T) - 1);
const uint8_t ne = op == OP_NE;
#define RUNS_PASS(UT, F) do { const UT x_ = (UT)pat; \
for (uint32_t r = 0; r < v->n_runs; r++) pass[r] = (uint8_t)((v->run_vals.F[r] == x_) ^ ne); \
} while (0)
switch (T) {
case 8: RUNS_PASS(uint8_t, b); break;
case 16: RUNS_PASS(uint16_t, h); break;
case 32: RUNS_PASS(uint32_t, s); break;
default: RUNS_PASS(uint64_t, q); break;
}
#undef RUNS_PASS
return true;
}
for (uint32_t r = 0; r < v->n_runs; r++)
pass[r] = (uint8_t)cmp_one(logical_of(lane(&v->run_vals, v->k.T, r), v->k), op, c);
return true;
}
bool ray_enc_eval_cmp_runs_f64(const ray_enc_vview_t* v, uint16_t op, double c, uint8_t* pass) {
if (v->enc != RAY_ENC_RLE || v->nullmap || !is_float(v->type) || !is_cmp_op(op) || isnan(c)) return false;
const int T = v->k.T;
for (uint32_t r = 0; r < v->n_runs; r++)
pass[r] = (uint8_t)cmp_one_f(ray_enc_fval(lane(&v->run_vals, T, r), T), op, c);
return true;
}
bool ray_enc_eval_in(const ray_enc_vview_t* v, const int64_t* set, int nset, uint8_t nr, uint8_t* bits) {
if (is_float(v->type)) return false;
if (nset < 0) nset = 0;
switch (v->enc) {
case RAY_ENC_CONST:
memset(bits, in_set(set, nset, ray_enc_logical(v->cbits, v->k)), (size_t)v->n);
break;
case RAY_ENC_RLE: eval_runs(v, &(run_pred_t){ true, 0, 0, set, nset }, bits); break;
case RAY_ENC_DICT: {
uint64_t pass[(1u << 16) / 64];
memset(pass, 0, ((size_t)v->dict_n + 63) / 64 * 8);
for (int s = 0; s < nset; s++) {
const uint32_t lb = lower_bound(v, set[s]);
if (lb < v->dict_n && ray_enc_logical(v->dict[lb], v->k) == set[s]) pass[lb >> 6] |= 1ull << (lb & 63);
}
for (int i = 0; i < v->n; i++) bits[i] = (uint8_t)((pass[v->codes[i] >> 6] >> (v->codes[i] & 63)) & 1);
break;
}
case RAY_ENC_FOR_BP: {
const uint64_t lb = (uint64_t)ray_enc_logical(v->base, v->k);
_Alignas(64) ray_enc_lanes_t lanes;
ray_enccol_unpack_resid(v, &lanes);
for (int i = 0; i < v->n; i++)
bits[i] = (uint8_t)in_set(set, nset, (int64_t)(lb + lane(&lanes, v->k.T, (uint32_t)i)));
break;
}
default: return false;
}
ray_enc_mask_nulls(v, nr, bits);
return true;
}
bool ray_enc_vec_sum(const ray_enc_vview_t* v, int64_t* hi, uint64_t* lo, int64_t* nn) {
switch (v->type) {
case RAY_I16: case RAY_I32: case RAY_I64: case RAY_TIME: break;
case RAY_BOOL: case RAY_U8: if (v->n_nulls) return false; break;
default: return false;
}
__int128 s = 0;
const int64_t n_nonnull = (int64_t)v->n - v->n_nulls;
switch (v->enc) {
case RAY_ENC_CONST:
if (n_nonnull) s = (__int128)n_nonnull * ray_enc_logical(v->cbits, v->k);
break;
case RAY_ENC_FOR_BP: {
_Alignas(64) ray_enc_lanes_t u;
ray_enccol_unpack_resid(v, &u);
const uint8_t* nm = v->n_nulls ? v->nullmap : NULL;
__int128 r = 0;
if (v->k.T == 64) {
if (nm) { for (int i = 0; i < v->n; i++) if (!((nm[i >> 3] >> (i & 7)) & 1)) r += u.q[i]; }
else { for (int i = 0; i < v->n; i++) r += u.q[i]; }
} else {
uint64_t r64 = 0;
for (int i = 0; i < v->n; i++) {
uint64_t x = lane(&u, v->k.T, (uint32_t)i);
if (nm) x &= (uint64_t)((nm[i >> 3] >> (i & 7)) & 1) - 1;
r64 += x;
}
r = r64;
}
s = (__int128)n_nonnull * ray_enc_logical(v->base, v->k) + r;
break;
}
case RAY_ENC_RLE: {
if (v->n_nulls) return false;
uint32_t prev = 0;
for (uint32_t r = 0; r < v->n_runs; r++) {
s += (__int128)ray_enc_logical(lane(&v->run_vals, v->k.T, r), v->k) * (int64_t)(v->run_ends.h[r] - prev);
prev = v->run_ends.h[r];
}
break;
}
default: return false;
}
*hi = (int64_t)(s >> 64);
*lo = (uint64_t)s;
*nn = n_nonnull;
return true;
}
static int64_t count_for(const ray_enc_vview_t* v, uint16_t op, int64_t c) {
const uint64_t umax = v->width >= 64 ? ~0ull : ((1ull << v->width) - 1);
const __int128 t = (__int128)c - (__int128)ray_enc_logical(v->base, v->k);
if (t < 0 || t > (__int128)umax) return cmp_one(t < 0 ? 1 : -1, op, 0) ? v->n : 0;
return ray_enccol_count_resid(v, op, (uint64_t)t);
}
bool ray_enc_count_cmp(const ray_enc_vview_t* v, uint16_t op, int64_t c, int64_t* count) {
if (v->nullmap || is_float(v->type) || !is_cmp_op(op)) return false;
if (v->type == RAY_SYM && op != OP_EQ && op != OP_NE) return false;
int64_t n = 0;
switch (v->enc) {
case RAY_ENC_CONST:
n = cmp_one(ray_enc_logical(v->cbits, v->k), op, c) ? v->n : 0;
break;
case RAY_ENC_RLE: {
uint32_t prev = 0;
for (uint32_t r = 0; r < v->n_runs; r++) {
const uint32_t e = v->run_ends.h[r];
if (cmp_one(logical_of(lane(&v->run_vals, v->k.T, r), v->k), op, c)) n += e - prev;
prev = e;
}
break;
}
case RAY_ENC_FOR_BP: n = count_for(v, op, c); break;
case RAY_ENC_DICT: {
const uint32_t lb = lower_bound(v, c), ub = upper_bound(v, c);
uint32_t lo = 0, hi = 0;
switch (op) {
case OP_EQ: case OP_NE: lo = lb; hi = ub; break;
case OP_LT: hi = lb; break;
case OP_LE: hi = ub; break;
case OP_GT: lo = ub; hi = v->dict_n; break;
default: lo = lb; hi = v->dict_n; break;
}
const uint32_t span = hi - lo;
uint32_t a = 0;
for (int i = 0; i < v->n; i++) a += v->codes[i] - lo < span;
n = op == OP_NE ? v->n - (int64_t)a : (int64_t)a;
break;
}
default: return false;
}
*count = n;
return true;
}
bool ray_enc_runs(const ray_enc_vview_t* v, void* ctx, ray_enc_run_fn fn) {
if ((v->enc != RAY_ENC_CONST && v->enc != RAY_ENC_RLE) || is_float(v->type)) return false;
uint32_t prev = 0;
for (uint32_t r = 0; r < v->n_runs; r++) {
const uint32_t e = v->run_ends.h[r];
fn(ctx, logical_of(lane(&v->run_vals, v->k.T, r), v->k), (int32_t)prev, (int32_t)e);
prev = e;
}
return true;
}
void ray_enc_view_range(const ray_enc_vview_t* v, int64_t* mn, int64_t* mx) {
int64_t a = INT64_MAX, b = INT64_MIN;
for (uint32_t q = 0; (v->enc == RAY_ENC_CONST || v->enc == RAY_ENC_RLE) && q < v->n_runs; q++) {
const int64_t x = logical_of(lane(&v->run_vals, v->k.T, q), v->k);
if (x < a) a = x;
if (x > b) b = x;
}
*mn = a; *mx = b;
}