tokenix 0.46.2

Semantic search, symbol graphs, secrets scanning, output filters, and CLI hooks that save 60-90% LLM tokens
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

[package]
name = "tokenix"
version = "0.46.2"
edition = "2021"
description = "Semantic search, symbol graphs, secrets scanning, output filters, and CLI hooks that save 60-90% LLM tokens"
license = "MIT"
repository = "https://github.com/juninmd/tokenix"
homepage = "https://github.com/juninmd/tokenix"
readme = "README.md"
keywords = ["llm", "embeddings", "code-search", "semantic-search", "cli"]
categories = ["command-line-utilities", "development-tools"]
exclude = ["/tokenix-logo.png", "/assets/screenshots", "/benchmark", "/.github"]

[[bin]]
name = "tokenix"
path = "src/main.rs"

[dependencies]
clap = { version = "4", features = ["derive"] }
anyhow = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
rusqlite = { version = "0.40", features = ["bundled"] }
fastembed = { version = "4" }
# default-features off: drops esaxx-rs/onig (static-CRT C/C++) which clash with
# ort-sys's dynamic CRT at link time. nomic-embed's BERT WordPiece tokenizer
# needs none of them; from_file + encode work with the pure-Rust core.
tokenizers = { version = "0.23", default-features = false, features = ["fancy-regex"] }
ort = { version = "=2.0.0-rc.9", optional = true }
once_cell = "1"
walkdir = "2"
ignore = "0.4"
sha2 = "0.11"
hex = "0.4"
chrono = { version = "0.4", features = ["serde"] }
indicatif = "0.18"
colored = "3"
# ansi feature: measures cell display-width ignoring ANSI color codes, so colored
# cells stay aligned. default-features off keeps it to the table core (no derive).
tabled = { version = "0.21", default-features = false, features = ["ansi"] }
# Interactive filter browser (`tokenix filter` in a TTY). Default features bring
# the crossterm backend, re-exported as `ratatui::crossterm` so we pin one version.
ratatui = "0.30"
dirs = "6"
rayon = "1"
toml = "1.1"
regex = "1"
rust-embed = { version = "8", features = ["include-exclude"] }
libc = "0.2"
tree-sitter = "0.26"
tree-sitter-rust = "0.24"
tree-sitter-python = "0.25"
tree-sitter-javascript = "0.25"
tree-sitter-go = "0.25"
tree-sitter-cpp = "0.23"
reqwest = { version = "0.13.4", features = ["blocking", "json"] }

[target.'cfg(windows)'.dependencies]
windows-sys = { version = "0.61", features = ["Win32_System_Threading"] }

# Debug builds overflow the main thread's stack (clap derive over the 33-variant
# Commands enum builds a huge unoptimized frame). opt-level 1 keeps frames sane.
[profile.dev]
opt-level = 1

[features]
default = []
cuda = ["dep:ort", "ort/cuda"]
directml = ["dep:ort", "ort/directml"]
# Enables tests that download the embedding model (~130 MB) and run inference.
# Excluded from the default `cargo test` so CI stays fast and offline.
model-tests = []