hypersteeldb 0.3.2

[package]
name = "hypersteeldb"
version = "0.3.2"
edition = "2021"
description = "A database that compiles questions instead of guessing answers: typed vocabulary discovered from your documents, queries type-checked before they run, roaring-bitmap set algebra over reified hyperedges, and Dempster-Shafer evidence with an explicit conflict guard."
license = "MIT"
readme = "README.md"
homepage = "https://huggingface.co/spaces/cp500/steeldb-ontology-sensing"
documentation = "https://docs.rs/hypersteeldb"
keywords = ["hypergraph", "bitmap", "retrieval", "dempster-shafer", "llm"]
categories = ["database-implementations", "data-structures", "text-processing"]
authors = ["SteelDB contributors"]
rust-version = "1.75"
# crates.io caps a published crate at 10 MB, and models/ alone is 388 MB — weights can never ship inside the
# package. They are fetched at runtime instead (see steeldb::models). Everything here is either large, not part
# of the library, or regenerable.
exclude = [
    "stress/",   # a post-publish harness that depends on the published crate, not the tree

    "app/",                 # the Tauri desktop app
    "duckdb/",              # reference clone, not ours
    "models/",              # 388 MB of weights — fetched at runtime, never packaged
    "spaces/",              # the demo site
    "benchmark_corpus/",     # generated, 42 MB
    "pokemon_corpus/",       # generated
    "train/",               # scratch checkpoints
    "target/",
    # Credentials must never be packaged. A `cargo package --list` showed cratesio.key would have been
    # included: *.token did not cover it, and the crate registry is the last place a key should end up.
    "*.token",
    "*.key",
    "PROBLEM_FORMULATION.md",
    "PAPER.tex",
    "*.pem",
    "assets/registeel.jpg",   # the demo image, not needed by the library
    ".env",
    ".claude/",
    "**/*.log",
    "tmp/",
    "hf.token",
]

[lib]
# The PACKAGE is `hypersteeldb` (the crates.io name `steeldb` was taken in 2023 by an unrelated project), but
# the library keeps the short name, so users add `hypersteeldb` and write `use steeldb::…`.
name = "steeldb"
path = "src/lib.rs"
# cdylib so the crate can be built as a wasm module; rlib so the native binaries still link against it
crate-type = ["cdylib", "rlib"]

# Build all features on docs.rs so the ONNX + agent APIs are documented.
[package.metadata.docs.rs]
all-features = true
rustdoc-args = ["--cfg", "docsrs"]










[dependencies]
roaring = "0.10"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
csv = "1"
tokenizers = { version = "0.20", default-features = false, features = ["onig"], optional = true }
ort = { version = "2.0.0-rc.13", optional = true }
# agent LLM providers
async-trait = "0.1"
wasm-bindgen = { version = "0.2", optional = true }
tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros"], optional = true }
aws-config = { version = "1.8.18", optional = true }
aws-sdk-bedrockruntime = { version = "1.135.0", optional = true }
aws-smithy-types = { version = "1.5.0", optional = true }
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true }
# TUI front-end (ratatui re-exports crossterm as ratatui::crossterm)
ratatui = { version = "0.29", optional = true }
# document extraction (any-doc bridge): PDF text, OOXML (docx/pptx) unzip + XML text, HTML
pdf-extract = { version = "0.7", optional = true }
zip = { version = "2", optional = true, default-features = false, features = ["deflate"] }
quick-xml = { version = "0.36", optional = true }
# OCR fallback for scanned PDFs (PP-OCRv6 det+rec ONNX); image decode/resize for the pipeline
image = { version = "0.25", optional = true, default-features = false, features = ["png"] }
# Native in-process LLM inference (pure-Rust candle): run the tuned Qwen3-1.7B + shipped LoRA offline,
# no Python serve. Metal on macOS is enabled via a target-gated feature below.
candle-core = { version = "0.9", optional = true }
candle-nn = { version = "0.9", optional = true }
candle-transformers = { version = "0.9", optional = true }

[target.'cfg(target_os = "macos")'.dependencies]
candle-core = { version = "0.9", optional = true, features = ["metal"] }

[profile.release]
opt-level = 3
lto = "thin"
codegen-units = 1

[features]
# Text embedding (model2vec) + the OT ontology discovery built on it. Pulls in `tokenizers`, which links the
# Oniguruma C library and therefore cannot target wasm32 — so the roaring core, the linter and step 0 must
# not depend on it. Implied by every feature that actually runs a model.
embed = ["dep:tokenizers"]
# Browser target: expose step 0 and the linter to JavaScript via wasm-bindgen. Requires that the core stay
# free of `tokenizers` (Oniguruma is C and will not build for wasm32), which is why `embed` exists.
wasm = ["dep:wasm-bindgen"]
# ONNX inference stages (SPO tagger + SPLADE). Off by default so the roaring core and the
# Tauri app build without pulling the ONNX Runtime binary; enable with `--features onnx`.
onnx = ["dep:ort", "embed"]
# Agent runtime (async loop) + configurable LLM-provider backends. `bedrock` = Claude on AWS Bedrock
# (Converse); `paddock` = a local OpenAI-compatible Paddock server (e.g. Qwen3.5-2B). Enable either.
agent = ["dep:tokio"]
bedrock = ["agent", "dep:aws-config", "dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"]
paddock = ["agent", "dep:reqwest"]
# Native pure-Rust LLM inference (candle): tuned Qwen3-1.7B + shipped LoRA, offline, no Python serve.
native = ["agent", "embed", "dep:candle-core", "dep:candle-nn", "dep:candle-transformers"]
# Cactus Needle provider: drive the agent with the 26M tool-use-native Needle model via its /generate
# server (needle-code torch port). Tool-call-native → more reliable bitmap-program selection than a
# general small model.
needle = ["agent", "dep:reqwest"]
# Interactive terminal UI: ingest a folder once (models + index stay hot), then ask questions against a
# local Paddock/Qwen model. Build: `cargo run --features tui,paddock,onnx --bin tui -- <dir>`.
tui = ["agent", "embed", "dep:ratatui"]
# The interactive command-line tool: the three verbs in a terminal UI. Deliberately lighter than `tui` — no
# `embed`, so no tokenizers and no Oniguruma C library, and no ONNX. `cargo install hypersteeldb --features cli`
# needs a Rust toolchain and nothing else; `learn` then talks to whatever local server you already run.
cli = ["dep:ratatui", "paddock"]
# Internal development tools: corpus generators, benchmark scorers, trajectory builders, the heavy TUIs. Gated so
# `cargo install hypersteeldb --features cli` puts ONE binary on your PATH instead of a dozen. Nothing here is
# part of the library's contract.
devtools = []
# Any-doc bridge: extract text from PDF/DOCX/PPTX/HTML (+ md/txt) → text projection. Implies onnx.
docs = ["onnx", "dep:pdf-extract", "dep:zip", "dep:quick-xml"]
# OCR fallback for scanned/image PDFs (no text layer): PP-OCRv6 det+rec over rasterized pages.
ocr = ["docs", "dep:image"]

# The only binary a user installs. Everything else is behind `devtools`.
[[bin]]
name = "steel"
path = "src/bin/steel.rs"
required-features = ["cli"]

[[bin]]
name = "agent"
path = "src/bin/agent.rs"
required-features = ["devtools", "agent"]

[[bin]]
name = "bench"
path = "src/bin/bench.rs"
required-features = ["devtools"]

[[bin]]
name = "discover"
path = "src/bin/discover.rs"
required-features = ["devtools", "paddock"]

[[bin]]
name = "eval_agent"
path = "src/bin/eval_agent.rs"
required-features = ["devtools", "paddock"]

[[bin]]
name = "export_spans"
path = "src/bin/export_spans.rs"
required-features = ["devtools", "onnx", "embed"]

[[bin]]
name = "gen_benchmark_corpus"
path = "src/bin/gen_benchmark_corpus.rs"
required-features = ["devtools"]

[[bin]]
name = "gen_dsl_trajectories"
path = "src/bin/gen_dsl_trajectories.rs"
required-features = ["devtools"]

[[bin]]
name = "gen_pokemon_corpus"
path = "src/bin/gen_pokemon_corpus.rs"
required-features = ["devtools"]

[[bin]]
name = "gen_trajectories"
path = "src/bin/gen_trajectories.rs"
required-features = ["devtools"]

[[bin]]
name = "gen_trajectories_teacher"
path = "src/bin/gen_trajectories_teacher.rs"
required-features = ["devtools", "bedrock"]

[[bin]]
name = "ontology"
path = "src/bin/ontology.rs"
required-features = ["devtools", "tui"]

[[bin]]
name = "score_benchmark"
path = "src/bin/score_benchmark.rs"
required-features = ["devtools"]

[[bin]]
name = "sense_ontology"
path = "src/bin/sense_ontology.rs"
required-features = ["devtools"]

[[bin]]
name = "steeldb"
path = "src/bin/steeldb.rs"
required-features = ["devtools", "tui", "needle", "onnx"]

[[bin]]
name = "synth_prose"
path = "src/bin/synth_prose.rs"
required-features = ["devtools"]

[[bin]]
name = "tui"
path = "src/bin/tui.rs"
required-features = ["devtools", "tui", "needle", "onnx"]