rayforce-sys 1.3.2

/*
 *   Copyright (c) 2025-2026 Anton Kundenko <singaraiona@gmail.com>
 *   All rights reserved.

 *   Permission is hereby granted, free of charge, to any person obtaining a copy
 *   of this software and associated documentation files (the "Software"), to deal
 *   in the Software without restriction, including without limitation the rights
 *   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 *   copies of the Software, and to permit persons to whom the Software is
 *   furnished to do so, subject to the following conditions:

 *   The above copyright notice and this permission notice shall be included in all
 *   copies or substantial portions of the Software.

 *   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 *   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 *   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 *   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 *   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 *   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 *   SOFTWARE.
 */

#ifndef RAY_EVAL_H
#define RAY_EVAL_H

#include <rayforce.h>
#include <stdio.h>
#include "lang/nfo.h"

/* ===== Function Attribute Flags (stored in attrs byte) ===== */

#define RAY_FN_NONE          0x00
#define RAY_FN_LEFT_ATOMIC   0x01  /* auto-map left arg over vectors */
#define RAY_FN_RIGHT_ATOMIC  0x02  /* auto-map right arg over vectors */
#define RAY_FN_ATOMIC        0x04  /* auto-map all args over vectors */
#define RAY_FN_AGGR          0x08  /* aggregation function */
#define RAY_FN_SPECIAL_FORM  0x10  /* receives unevaluated args */
#define RAY_FN_RESTRICTED    0x20  /* forbidden during -U restricted IPC evals */
#define RAY_FN_LAZY_AWARE    0x80  /* fn extends or starts lazy chains;
                                      dispatcher must NOT materialise its
                                      args. Without this flag, dispatcher
                                      forces concrete args. */

/* AST name flag (distinguishes symbol literal from variable reference) */
#define ATTR_QUOTED  0x20  /* -RAY_SYM atom with this flag SET = quoted/literal symbol;
                              CLEAR (default) = name reference, resolved at eval. */

/* Internal eval flag for RAY_LIST values carried as nested IPC arguments.
 * The evaluator still runs callable lists normally; this only makes a
 * non-callable list head fall back to returning the list as data. */
#define RAY_EVAL_LITERAL_FALLBACK 0x04

/* Function type signatures */
typedef ray_t* (*ray_unary_fn)(ray_t*);
typedef ray_t* (*ray_binary_fn)(ray_t*, ray_t*);
typedef ray_t* (*ray_vary_fn)(ray_t**, int64_t);

/* DAG opcode stored in aux[0..1] for binary builtins with DAG-exec paths.
 * Eliminates dispatch table lookups in atomic_map_binary — just read the opcode. */
#define RAY_FN_OPCODE(fn)        (*(uint16_t*)(fn)->aux)
#define RAY_FN_SET_OPCODE(fn,op) (*(uint16_t*)(fn)->aux = (uint16_t)(op))

/* ===== VM Bytecode Opcodes ===== */

enum {
    OP_RET = 0,       /* return top of stack */
    OP_JMP,           /* unconditional jump (2-byte signed offset) */
    OP_JMPF,          /* jump if false (2-byte signed offset) */
    OP_LOADCONST,     /* push constant pool[operand] (1-byte index) */
    OP_LOADENV,       /* push local variable (1-byte slot index) */
    OP_STOREENV,      /* pop and store into local (1-byte slot index) */
    OP_POP,           /* discard top of stack */
    OP_RESOLVE,       /* resolve global name: constant pool[operand] is sym_id */
    OP_CALL1,         /* call unary: pop fn + 1 arg, push result */
    OP_CALL2,         /* call binary: pop fn + 2 args, push result */
    OP_CALLN,         /* call variadic: operand = argc, pop fn + N args */
    OP_CALLF,         /* call compiled lambda: push frame, jump to callee */
    OP_CALLS,         /* tail call: reuse frame */
    OP_CALLD,         /* dynamic dispatch: fallback to ray_eval() */
    OP_DUP,           /* duplicate top of stack */
    OP_LOADCONST_W,   /* push constant pool[operand] (2-byte index) */
    OP_RESOLVE_W,     /* resolve global name: 2-byte constant pool index */
    OP_TRAP,          /* push trap frame, 2-byte handler offset */
    OP_TRAP_END,      /* pop trap frame (success path) */
    OP_STOREGLOBAL,   /* bind global: cpool[operand] is sym_id; value stays on stack (1-byte index) */
    OP_STOREGLOBAL_W, /* bind global: 2-byte constant pool index */
    OP_SCOPE_BEGIN,   /* pop sym-id vec; push scope frame binding the live
                       * locals (+ self) so a following OP_CALLD's tree-walk
                       * eval can resolve them */
    OP_SCOPE_END,     /* pop sym-id vec; sync frame values back into the
                       * local slots, pop the frame (leaves the CALLD result
                       * on the stack) */
    OP_TRYH,          /* try-handler dispatch: pop err_val + handler; if
                       * handler is callable, call it with err_val, else
                       * push handler as a fallback value */
    OP_FORCE,         /* materialize TOS if it is a lazy handle (so a
                       * let-bound local holds a concrete, reusable value) */
    OP_BINDCAPS,      /* bind the closure's captures to local slots:
                       * cpool[operand] is an I64 vector of (name, slot)
                       * pairs; each name is looked up in the closure dict
                       * and its value stored into the slot (a name the dict
                       * lacks leaves the slot empty).  The prologue of every
                       * compiled closure */
    OP_LOADSELF,      /* push the running lambda (bare `self` as a value) */
    OP_MKCLOSURE,     /* pop slots (I64), names (SYM), template (LAMBDA):
                       * push a closure sharing the template's bytecode,
                       * capturing the current frame's named slots */
    OP_LOADENVG,      /* push local slot (1 byte) or, when the slot is still
                       * empty (a `let` that has not run), the global named
                       * by cpool[2-byte index], as the tree walker would */
    OP_SCOPE_EVAL,    /* OP_SCOPE_BEGIN for the windows compiled code opens
                       * around text it hands to the tree walker (eval, a
                       * dynamic special form, a fn literal): the frame
                       * binds the locals for that text only, not for what
                       * it calls, and ends its scope */
    OP_CALL1R,        /* OP_CALL1 / OP_CALL2 / OP_CALLN, each with an argc   */
    OP_CALL2R,        /* operand, for a builtin a closure body resolves when */
    OP_CALLNR,        /* it runs: the fast call while the name holds a builtin
                       * of that kind, OP_CALLF's general call otherwise */
    OP__COUNT
};

/* ===== Compiled Lambda Layout =====
 *
 * A RAY_LAMBDA object with attrs & RAY_FN_COMPILED stores compiled
 * bytecode in its data area:
 *
 *   data[0] = ray_t* params_list   (same as interpreted)
 *   data[1] = ray_t* body          (parsed body, same as interpreted)
 *   data[2] = ray_t* bytecode      (RAY_U8 vector of opcodes)
 *   data[3] = ray_t* constants     (RAY_LIST of constant pool entries)
 *   data[4] = int32_t n_locals    (number of local slots needed)
 *   data[5] = ray_t* nfo          (source location info, NULL if absent)
 *   data[6] = ray_t* dbg          (debug metadata, NULL if absent)
 *   data[7] = ray_t* closure      (captured lexical locals, NULL if empty)
 *   data[8] = ray_t* caprows      (RAY_U8 row flag per capture, or NULL)
 *   data[9] = uint32_t capq       (serial of the query frame the closure was
 *                                  made under, 0 if none; never serialized)
 */
#define LAMBDA_SLOTS 10

#define RAY_FN_COMPILED  0x40   /* lambda has been compiled to bytecode */

#define LAMBDA_PARAMS(lam)    (((ray_t**)ray_data(lam))[0])
#define LAMBDA_BODY(lam)      (((ray_t**)ray_data(lam))[1])
#define LAMBDA_BC(lam)        (((ray_t**)ray_data(lam))[2])
#define LAMBDA_CONSTS(lam)    (((ray_t**)ray_data(lam))[3])
#define LAMBDA_NLOCALS(lam)   (*((int32_t*)&((ray_t**)ray_data(lam))[4]))
#define LAMBDA_NFO(lam)       (((ray_t**)ray_data(lam))[5])
#define LAMBDA_DBG(lam)       (((ray_t**)ray_data(lam))[6])
#define LAMBDA_CLOSURE(lam)   (((ray_t**)ray_data(lam))[7])
#define LAMBDA_CAPROWS(lam)   (((ray_t**)ray_data(lam))[8])
#define LAMBDA_CAPQ(lam)      (*((uint32_t*)&((ray_t**)ray_data(lam))[9]))

#define LAMBDA_IS_COMPILED(lam) ((lam)->attrs & RAY_FN_COMPILED)

/* ===== Bytecode execution state =====
 *
 * Per-invocation interpreter state, allocated by the executor for each
 * compiled-lambda call and freed on return.  This is NOT the per-thread
 * VM — that is ray_vm_t in core/runtime.h, which stays stable for the
 * thread's lifetime while any number of ray_exec_t frames come and go. */

#define VM_STACK_SIZE 1024

typedef struct {
    ray_t   *fn;     /* lambda being executed */
    int32_t fp;     /* frame pointer */
    int32_t ip;     /* instruction pointer */
} vm_ctx_t;

typedef struct {
    int32_t  rp;        /* return stack depth at trap point */
    int32_t  sp;        /* stack depth at trap point */
    int32_t  handler_ip;/* IP of handler code */
    ray_t    *fn;        /* function containing handler code */
    int32_t  fp;        /* frame pointer at trap point */
    int32_t  n_locals;  /* n_locals at trap point */
    int32_t  scope_depth; /* env scope depth at trap point — an error inside
                           * an OP_SCOPE_BEGIN..END window must unwind the
                           * materialized frame(s) before the handler runs */
} vm_trap_t;

#define VM_TRAP_SIZE 16

typedef struct {
    int32_t  sp;                    /* stack pointer */
    int32_t  fp;                    /* frame pointer */
    int32_t  rp;                    /* return stack pointer */
    ray_t    *fn;                    /* current lambda */
    int32_t  tp;                    /* trap stack pointer */
    ray_t    *ps[VM_STACK_SIZE];     /* program stack */
    uint8_t  rows[VM_STACK_SIZE];    /* ps[i] is derived from query rows */
    vm_ctx_t rs[VM_STACK_SIZE];     /* return stack */
    vm_trap_t ts[VM_TRAP_SIZE];     /* trap frames */
} ray_exec_t;

/* ===== Public API ===== */

/* Initialize the Rayfall runtime: symbols, environment, builtins. */
ray_err_t ray_lang_init(void);
void     ray_lang_destroy(void);

/* Evaluate a parsed ray_t object tree. */
ray_t* ray_eval(ray_t* obj);

/* Parse + eval convenience. */
ray_t* ray_eval_str(const char* source);

/* Compile a lambda's body to bytecode. Called lazily on first invocation. */
void ray_compile(ray_t* lambda);
/* ray_compile for a lambda with nothing captured, also handing back the
 * bindings its code depends on (see compile.c). */
bool ray_compile_definition(ray_t* lambda, ray_t** deps);
/* The most bytes a literal's text may hold for the template table to keep
 * it (eval.c), and where the compiler's weight count may stop. */
#define RAY_CLOSURE_TMPL_MAX_BYTES ((int64_t)64 * 1024)
/* The capture names a closure's body can reach (its dict keys the body
 * mentions; all of them under eval/resolve), as a SYM vector or NULL. */
ray_t* ray_closure_capture_names(ray_t* lambda);
ray_t* ray_closure_mentioned_syms(ray_t* body, bool* dynamic, int64_t* weight);
ray_t* ray_closure_names_from(ray_t* closure, ray_t* mentioned, bool dynamic);
/* The closure dict shape the compiled capture binding assumes: NULL, or SYM
 * keys with a LIST of values of the same length. */
bool   ray_closure_shape_ok(ray_t* closure);
/* Compile `lambda` with `capnames` bound as locals ahead of the body; false
 * when the body cannot be compiled. */
bool   ray_compile_with_captures(ray_t* lambda, ray_t* capnames, bool closure_body);

/* Reset compiler cached state (call from ray_lang_destroy). */
void ray_compile_reset(void);

/* Look up the source span for a bytecode IP from a lambda's debug vector.
 * Returns a span with id==0 if not found. */
ray_span_t ray_bc_dbg_get(ray_t* dbg, int32_t ip);

/* Print a ray_t value to a FILE stream. */
void ray_lang_print(FILE* fp, ray_t* val);

/* Interrupt support: allow external code (REPL signal handler) to request
 * that the evaluator abort early.  ray_eval() and the bytecode VM check
 * this flag at function-call and loop boundaries. */
void ray_eval_request_interrupt(void);
void ray_eval_clear_interrupt(void);
int  ray_eval_is_interrupted(void);

/* Return the current eval context's nfo (source location) object, or NULL. */
ray_t* ray_eval_get_nfo(void);
void   ray_eval_set_nfo(ray_t* nfo);

/* Error trace: list of [span_i64, filename, fn_name, source] frames built when
 * a VM error propagates without a trap.  Cleared at the start of ray_eval_str. */
ray_t* ray_get_error_trace(void);
void   ray_clear_error_trace(void);

/* Restricted mode: when true, builtins with RAY_FN_RESTRICTED are blocked. */
void ray_eval_set_restricted(bool on);
bool ray_eval_get_restricted(void);

/* ===== Rayfall Builtin Functions ===== */

/* Arithmetic */
ray_t* ray_add_fn(ray_t* a, ray_t* b);
ray_t* ray_sub_fn(ray_t* a, ray_t* b);
ray_t* ray_mul_fn(ray_t* a, ray_t* b);
ray_t* ray_div_fn(ray_t* a, ray_t* b);
ray_t* ray_idiv_fn(ray_t* a, ray_t* b);
ray_t* ray_mod_fn(ray_t* a, ray_t* b);

/* Comparison */
ray_t* ray_gt_fn(ray_t* a, ray_t* b);
ray_t* ray_lt_fn(ray_t* a, ray_t* b);
ray_t* ray_gte_fn(ray_t* a, ray_t* b);
ray_t* ray_lte_fn(ray_t* a, ray_t* b);
ray_t* ray_min2_fn(ray_t* a, ray_t* b);
ray_t* ray_max2_fn(ray_t* a, ray_t* b);
ray_t* ray_eq_fn(ray_t* a, ray_t* b);
ray_t* ray_neq_fn(ray_t* a, ray_t* b);

/* Logic */
ray_t* ray_and_fn(ray_t* a, ray_t* b);
ray_t* ray_or_fn(ray_t* a, ray_t* b);
ray_t* ray_and_vary_fn(ray_t** args, int64_t n);
ray_t* ray_or_vary_fn(ray_t** args, int64_t n);
ray_t* ray_not_fn(ray_t* x);
ray_t* ray_neg_fn(ray_t* x);

/* Aggregation */
ray_t* ray_sum_fn(ray_t* x);
ray_t* ray_prod_fn(ray_t* x);
ray_t* ray_all_fn(ray_t* x);
ray_t* ray_any_fn(ray_t* x);
ray_t* ray_count_fn(ray_t* x);
ray_t* ray_avg_fn(ray_t* x);
ray_t* ray_min_fn(ray_t* x);
ray_t* ray_max_fn(ray_t* x);
ray_t* ray_first_fn(ray_t* x);
ray_t* ray_last_fn(ray_t* x);
ray_t* ray_med_fn(ray_t* x);
ray_t* ray_mode_fn(ray_t* x);
ray_t* ray_quantile_fn(ray_t* x, ray_t* q);
ray_t* ray_percentile_fn(ray_t* x, ray_t* p);
ray_t* ray_dev_fn(ray_t* x);
ray_t* ray_stddev_fn(ray_t* x);
ray_t* ray_stddev_pop_fn(ray_t* x);
ray_t* ray_var_fn(ray_t* x);
ray_t* ray_var_pop_fn(ray_t* x);
ray_t* ray_cov_fn(ray_t* x, ray_t* y);
ray_t* ray_scov_fn(ray_t* x, ray_t* y);
ray_t* ray_wsum_fn(ray_t* x, ray_t* y);
ray_t* ray_wavg_fn(ray_t* x, ray_t* y);

/* Higher-order */
ray_t* ray_map_fn(ray_t** args, int64_t n);
ray_t* ray_pmap_fn(ray_t** args, int64_t n);
ray_t* ray_fold_fn(ray_t** args, int64_t n);
ray_t* ray_scan_fn(ray_t** args, int64_t n);
ray_t* ray_prior_fn(ray_t** args, int64_t n);
ray_t* ray_filter_fn(ray_t* vec, ray_t* mask);
ray_t* ray_apply_fn(ray_t** args, int64_t n);

/* Collection */
ray_t* ray_distinct_fn(ray_t* x);
ray_t* ray_in_fn(ray_t* val, ray_t* vec);
ray_t* ray_in_eval_fn(ray_t* val, ray_t* vec);
/* Query text concat; str_out: a STR atom operand is a STR column's cell.
 * rows_mask bit 0 / bit 1: a / b is a column's cell, so a null one is the
 * empty text the vector kernel sees, not the null literal that nulls the
 * result. */
ray_t* ray_concat_text_fn(ray_t* a, ray_t* b, bool str_out, uint8_t rows_mask);
ray_t* ray_not_in_fn(ray_t* val, ray_t* vec);
ray_t* ray_except_fn(ray_t* vec1, ray_t* vec2);
ray_t* ray_union_fn(ray_t* vec1, ray_t* vec2);
ray_t* ray_sect_fn(ray_t* vec1, ray_t* vec2);
ray_t* ray_take_fn(ray_t* vec, ray_t* n_obj);
ray_t* ray_drop_fn(ray_t* vec, ray_t* n_obj);
ray_t* ray_rotate_fn(ray_t* vec, ray_t* n_obj);
ray_t* ray_cut_fn(ray_t* vec, ray_t* idxs);
ray_t* ray_cross_fn(ray_t* a, ray_t* b);
ray_t* ray_at_fn(ray_t* vec, ray_t* idx);
ray_t* ray_find_fn(ray_t* vec, ray_t* val);
ray_t* ray_fill_fn(ray_t* replacement, ray_t* value);
ray_t* ray_til_fn(ray_t* x);
ray_t* ray_reverse_fn(ray_t* x);
ray_t* ray_lag_fn(ray_t* x);
ray_t* ray_lead_fn(ray_t* x);
ray_t* ray_deltas_fn(ray_t* x);
ray_t* ray_ratios_fn(ray_t* x);
ray_t* ray_fills_fn(ray_t* x);
ray_t* ray_sums_fn(ray_t* x);
ray_t* ray_avgs_fn(ray_t* x);
ray_t* ray_mins_fn(ray_t* x);
ray_t* ray_maxs_fn(ray_t* x);
ray_t* ray_prds_fn(ray_t* x);
ray_t* ray_differ_fn(ray_t* x);
ray_t* ray_msum_fn(ray_t* n, ray_t* x);
ray_t* ray_mavg_fn(ray_t* n, ray_t* x);
ray_t* ray_mmin_fn(ray_t* n, ray_t* x);
ray_t* ray_mmax_fn(ray_t* n, ray_t* x);
ray_t* ray_mcount_fn(ray_t* n, ray_t* x);
ray_t* ray_mvar_fn(ray_t* n, ray_t* x);
ray_t* ray_mdev_fn(ray_t* n, ray_t* x);

/* Table construction */
ray_t* ray_list_fn(ray_t** args, int64_t n);
ray_t* ray_table_fn(ray_t* names, ray_t* cols);
ray_t* ray_key_fn(ray_t* x);
ray_t* ray_value_fn(ray_t* x);

/* Query */
ray_t* ray_select_fn(ray_t** args, int64_t n);
ray_t* ray_window_fn(ray_t** args, int64_t n);
ray_t* ray_update_fn(ray_t** args, int64_t n);
ray_t* ray_insert_fn(ray_t** args, int64_t n);
ray_t* ray_upsert_fn(ray_t** args, int64_t n);
ray_t* ray_xbar_fn(ray_t* col, ray_t* bucket);

/* Joins */
ray_t* ray_left_join_fn(ray_t** args, int64_t n);
ray_t* ray_inner_join_fn(ray_t** args, int64_t n);
ray_t* ray_full_join_fn(ray_t** args, int64_t n);
ray_t* ray_window_join_fn(ray_t** args, int64_t n);
ray_t* ray_window_join1_fn(ray_t** args, int64_t n);

/* I/O */
ray_t* ray_println_fn(ray_t** args, int64_t n);
ray_t* ray_read_csv_fn(ray_t** args, int64_t n);
ray_t* ray_write_csv_fn(ray_t** args, int64_t n);
ray_t* ray_read_file_fn(ray_t* path_obj);
ray_t* ray_read_bytes_fn(ray_t* path_obj);
ray_t* ray_write_file_fn(ray_t* path_obj, ray_t* content);
ray_t* ray_write_bytes_fn(ray_t* path_obj, ray_t* content);

/* Vector similarity / embeddings / HNSW.
 * cos-dist and l2-dist return distance (lower = closer); inner-prod is
 * the raw mathematical dot product. */
ray_t* ray_cos_dist_fn(ray_t* a, ray_t* b);
ray_t* ray_inner_prod_fn(ray_t* a, ray_t* b);
ray_t* ray_l2_dist_fn(ray_t* a, ray_t* b);
ray_t* ray_norm_fn(ray_t* x);
ray_t* ray_knn_fn(ray_t** args, int64_t n);
ray_t* ray_hnsw_build_fn(ray_t** args, int64_t n);
ray_t* ray_ann_fn(ray_t** args, int64_t n);
ray_t* ray_hnsw_free_fn(ray_t* h);
ray_t* ray_hnsw_save_fn(ray_t* h, ray_t* path);
ray_t* ray_hnsw_load_fn(ray_t* path);
ray_t* ray_hnsw_info_fn(ray_t* h);

/* Cast and type */
ray_t* ray_cast_fn(ray_t* type_sym, ray_t* val);
ray_t* ray_type_fn(ray_t* val);

/* Special forms */
ray_t* ray_set_fn(ray_t* name_obj, ray_t* val_expr);
ray_t* ray_let_fn(ray_t* name_obj, ray_t* val_expr);
ray_t* ray_cond_fn(ray_t** args, int64_t n);
ray_t* ray_do_fn(ray_t** args, int64_t n);
ray_t* ray_while_fn(ray_t** args, int64_t n);
ray_t* ray_times_fn(ray_t** args, int64_t n);
ray_t* ray_times_norm_fn(ray_t* x);
ray_t* ray_times_dec_fn(ray_t* x);
ray_t* ray_fn(ray_t** args, int64_t n);
ray_t* ray_raise_fn(ray_t* val);
ray_t* ray_try_fn(ray_t* expr, ray_t* handler_expr);


bool ray_eval_query_helper_active(void);
bool ray_eval_query_helper_literals(void);

#endif /* RAY_EVAL_H */