1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
# infino default configuration.
#
# This file is compiled into the binary via include_str! and acts as
# the floor of the config stack. Override at runtime via (in order of
# increasing precedence):
#
# 1. /etc/infino/config.yaml (system-wide)
# 2. $XDG_CONFIG_HOME/infino/config.yaml (user; falls back to
# $HOME/.config/infino/...)
# 3. ./infino.yaml (per-project / per-cwd)
# 4. INFINO_<UPPERCASE_FIELD> (per-process env var)
#
# Each layer is a partial override — keys not set in a higher layer
# fall through to lower layers.
#
# Nested keys are addressed with `__` (double underscore) in env vars.
# Examples below.
# Supertable runtime knobs. Reader fan-out (skip + per-superfile
# search + top-k merge) and writer commit-time rayon-shard run on
# separate pools so a long-running commit can't spike reader p99 or
# vice versa.
#
# Thread-count values:
# - `auto` — resolves at runtime to num_cpus for reader_threads
# and max(1, num_cpus / 2) for writer_threads. Hardware-portable
# default; the same shipped config works on a 4-core dev laptop
# and a 96-core production node.
# - a positive integer — explicit override. Use when you've
# measured a workload-specific tuning.
#
# Env overrides (note the double underscore for nesting):
# INFINO_SUPERTABLE__READER_THREADS=8
# INFINO_SUPERTABLE__WRITER_THREADS=4
# INFINO_SUPERTABLE__ID_COLUMN=row_id
# INFINO_SUPERTABLE__COMMIT_THRESHOLD_SIZE_MB=2048
# INFINO_SUPERTABLE__VERIFY_CRC_ON_OPEN=false
supertable:
reader_threads: auto
writer_threads: auto
# Name of the system-managed primary-key column the supertable
# injects on every append(). The column type is fixed by the
# supertable layer; only the name is configurable. Leading
# underscore signals a system-owned field. Change this if
# `_id` collides with a business field name; otherwise leave
# as default.
id_column: _id
# Threshold above which the supertable's writer triggers an
# internal commit() to flush the in-memory buffer to disk-
# equivalent (one superfile per writer-pool thread). Specified
# in mebibytes (1 MiB = 1024 × 1024 bytes). Set 0 to disable
# auto-flush — only caller-driven commit() will produce
# superfiles.
commit_threshold_size_mb: 1024
# Verify the trailing whole-blob CRC and per-subsection CRCs
# on every SuperfileReader::open. Defaults to true. Set to
# false only when the underlying storage already validates
# checksums (content-addressed object store, ZFS, etc.) —
# skipping the scan trades that storage-layer guarantee for
# faster cold opens.
verify_crc_on_open: true
# Persistent storage wiring. Defaults to in-memory-only. Set
# `backend: s3` plus `bucket` / `prefix` to make
# SupertableOptions::apply_config attach an S3StorageProvider.
# `backend: azure` attaches an AzureStorageProvider, with `bucket`
# naming the Azure container. `backend: gcs` attaches a
# GcsStorageProvider, with `bucket` naming the GCS bucket and
# `storage_options` carrying `google_*` config keys.
#
# Env overrides:
# INFINO_STORAGE__BACKEND=s3
# INFINO_STORAGE__BUCKET=my-bucket
# INFINO_STORAGE__PREFIX=tables/my-table
# INFINO_STORAGE__DISK_CACHE_ROOT=/mnt/nvme/infino-cache
storage:
backend: none
local_root:
bucket:
prefix: ""
# Object-store credentials and tuning, keyed by object_store's config
# strings. Infino reads no credentials from the environment — set them
# here (or via ConnectOptions). Empty → ambient cloud identity (IAM
# instance role / managed identity). Examples:
# backend: s3
# storage_options:
# aws_access_key_id: ...
# aws_secret_access_key: ...
# aws_region: us-east-1
# # aws_endpoint: http://localhost:9000 # MinIO / R2 / Ceph
# backend: azure # `bucket` names the container
# storage_options:
# azure_storage_account_name: ...
# azure_storage_account_key: ...
storage_options:
# When set, storage-backed reads use DiskCacheStore. For S3,
# lazy_foreground_with_background_fill is the object-store-native
# cold path: foreground opens/searches with exact range GETs (or zero
# superfile GETs at open when manifest open-batch bytes are present)
# while background fill promotes the full superfile to mmap.
disk_cache_root:
disk_budget_bytes: 10737418240
# Byte budget for the content-addressed manifest-part cache, held in a
# manifest-parts/ subdirectory of disk_cache_root. On a hit the part
# loader reads bytes from local disk instead of object storage; parts
# are content-addressed, so cached files are never stale and survive
# restarts. Independent of disk_budget_bytes. Default 2 GiB.
manifest_disk_budget_bytes: 2147483648
cold_fetch_mode: lazy_foreground_with_background_fill
cold_fetch_streams: 8
cold_fetch_chunk_bytes: 4194304
mmap_cold_threshold_secs: 300
mmap_sweep_interval_secs: 75
# Compaction merges the small superfiles produced by individual
# commits into one target-sized superfile, cutting query fan-out.
# Sizes are in mb (1 MiB = 1024 × 1024 bytes).
#
# Env overrides (note the double underscore for nesting):
# INFINO_COMPACTION__TARGET_SUPERFILE_SIZE_MB=2048
# INFINO_COMPACTION__MIN_FILL_PERCENT=80
# INFINO_COMPACTION__MAX_MEMORY_MB=3072
compaction:
# Size a compacted output aims for.
target_superfile_size_mb: 1024
# Minimum estimated live bytes to trigger a merge,
# as a percentage of `target_superfile_size_mb`.
# E.g. at 80% with a 1 GiB target, two 200 MiB superfiles (400 MiB) do not compact.
min_fill_percent: 80
# Maximum memory budget for materializing inputs during a single merge, in MiB.
max_memory_mb: 3072
# Per-connection memory budget: a ceiling on the anonymous heap the query
# and ingest paths allocate (result batches, the vector shortlist, ingest
# buffers), so one connection can't grow memory until the process is OOM-
# killed. The memory-mapped superfiles and the disk cache are bounded
# separately; this bounds heap only.
#
# Applies to connections built from this config file. Code that opens a
# connection programmatically sets the budget on ConnectOptions
# (with_connection_memory_budget_bytes) instead.
#
# Env override:
# INFINO_MEMORY__CONNECTION_BUDGET_BYTES=8589934592
memory:
# Bytes. 0 (default) is measure-only: usage is tracked but never refused.
# A positive value enforces the ceiling (the engine reserves at 90% of it,
# leaving headroom for small untracked allocations).
connection_budget_bytes: 0