tokenix 0.23.4

Local semantic index CLI for LLM token optimization
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

[package]
name = "tokenix"
version = "0.23.4"
edition = "2021"
description = "Local semantic index CLI for LLM token optimization"
license = "MIT"
repository = "https://github.com/juninmd/tokenix"
homepage = "https://github.com/juninmd/tokenix"
readme = "README.md"
keywords = ["llm", "embeddings", "code-search", "semantic-search", "cli"]
categories = ["command-line-utilities", "development-tools"]
exclude = ["/tokenix-logo.png", "/assets/screenshots", "/benchmark", "/docs", "/.github"]

[[bin]]
name = "tokenix"
path = "src/main.rs"

[dependencies]
clap = { version = "4", features = ["derive"] }
anyhow = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
rusqlite = { version = "0.31", features = ["bundled"] }
fastembed = { version = "4" }
# default-features off: drops esaxx-rs/onig (static-CRT C/C++) which clash with
# ort-sys's dynamic CRT at link time. nomic-embed's BERT WordPiece tokenizer
# needs none of them; from_file + encode work with the pure-Rust core.
tokenizers = { version = "0.23", default-features = false, features = ["fancy-regex"] }
ort = { version = "=2.0.0-rc.9", optional = true }
once_cell = "1"
walkdir = "2"
ignore = "0.4"
sha2 = "0.11"
hex = "0.4"
chrono = { version = "0.4", features = ["serde"] }
indicatif = "0.18"
colored = "3"
dirs = "6"
rayon = "1"
toml = "1.1"
regex = "1"
rust-embed = { version = "8", features = ["include-exclude"] }
libc = "0.2"
tree-sitter = "0.20"
tree-sitter-rust = "0.20"
tree-sitter-python = "0.20"
tree-sitter-javascript = "0.20"
tree-sitter-go = "0.20"
tree-sitter-cpp = "0.20"

[features]
default = []
cuda = ["dep:ort", "ort/cuda"]
directml = ["dep:ort", "ort/directml"]
# Enables tests that download the embedding model (~130 MB) and run inference.
# Excluded from the default `cargo test` so CI stays fast and offline.
model-tests = []