o41-training-data 0.1.6

Deterministic, resumable training batch streams over Parquet shards and data manifests, with a C API
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

#ifndef O41_TRAINING_DATA_H
#define O41_TRAINING_DATA_H
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#ifndef ARROW_C_DATA_INTERFACE
#define ARROW_C_DATA_INTERFACE
struct ArrowSchema {
  const char *format, *name, *metadata;
  int64_t flags, n_children;
  struct ArrowSchema **children, *dictionary;
  void (*release)(struct ArrowSchema *);
  void *private_data;
};
struct ArrowArray {
  int64_t length, null_count, offset, n_buffers, n_children;
  const void **buffers;
  struct ArrowArray **children, *dictionary;
  void (*release)(struct ArrowArray *);
  void *private_data;
};
#endif
#ifndef ARROW_C_STREAM_INTERFACE
#define ARROW_C_STREAM_INTERFACE
struct ArrowArrayStream {
  int (*get_schema)(struct ArrowArrayStream *, struct ArrowSchema *);
  int (*get_next)(struct ArrowArrayStream *, struct ArrowArray *);
  const char *(*get_last_error)(struct ArrowArrayStream *);
  void (*release)(struct ArrowArrayStream *);
  void *private_data;
};
#endif
struct Loader;
/* Callers serialize calls per handle. Arrow callbacks own exported memory.
   Strings borrowed from a handle stay valid until the next call on it.
   last_error(NULL) reports open, resume and resolve errors.
   Status: 1 success, 0 end (next only), -1 error. */
/* Resolve manifest sources into file lists. Returns an owned JSON string to
   release with loader_free_string, or NULL on error. */
char *loader_resolve(const char *config_json);
void loader_free_string(char *);
/* Open a resolved config, or resume a checkpoint with explicit storage settings. */
struct Loader *loader_open(const char *config_json);
struct Loader *loader_resume(const char *checkpoint_json, const char *storage_json);
int loader_next(struct Loader *, struct ArrowArrayStream *, uint64_t *stage, uint64_t *batch_index);
/* Borrowed JSON checkpoint: resolved config without credentials, cursor, counters. */
int loader_checkpoint(struct Loader *, const char **checkpoint_json);
int loader_advance(struct Loader *, uint64_t stage);
const char *loader_last_error(struct Loader *);
void loader_close(struct Loader *);
#ifdef __cplusplus
}
#endif
#endif