splintr 0.10.0

Fast Rust tokenizer (BPE + SentencePiece + WordPiece) with Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

#!/bin/bash
#
# Pre-commit hook for splintr
#
# Install by copying to .git/hooks/:
#   cp hooks/pre-commit .git/hooks/pre-commit
#   chmod +x .git/hooks/pre-commit
#

set -e

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

echo "Running pre-commit checks..."
echo ""

FAILED=0

# 1. Rust formatting
echo -n "Checking formatting... "
if ! cargo fmt --all --check > /dev/null 2>&1; then
    echo -e "${RED}FAILED${NC}"
    echo -e "${YELLOW}Fix with:${NC} cargo fmt --all"
    FAILED=1
else
    echo -e "${GREEN}OK${NC}"
fi

# 2. Clippy
echo -n "Running clippy... "
if ! cargo clippy --all-targets -- -D warnings > /dev/null 2>&1; then
    echo -e "${RED}FAILED${NC}"
    echo -e "${YELLOW}Run:${NC} cargo clippy --all-targets"
    FAILED=1
else
    echo -e "${GREEN}OK${NC}"
fi

# 3. Tests
echo -n "Running tests... "
if ! cargo test --quiet > /dev/null 2>&1; then
    echo -e "${RED}FAILED${NC}"
    echo -e "${YELLOW}Run:${NC} cargo test"
    FAILED=1
else
    echo -e "${GREEN}OK${NC}"
fi

# 4. Update uv.lock if pyproject.toml changed
if git diff --cached --name-only | grep -q "pyproject.toml"; then
    echo -n "Updating uv.lock... "
    if ! uv lock > /dev/null 2>&1; then
        echo -e "${RED}FAILED${NC}"
        echo -e "${YELLOW}Run:${NC} uv lock"
        FAILED=1
    else
        git add uv.lock
        echo -e "${GREEN}OK${NC}"
    fi
fi

echo ""

if [ $FAILED -ne 0 ]; then
    echo -e "${RED}Pre-commit checks failed.${NC}"
    echo "Bypass with: git commit --no-verify"
    exit 1
fi

echo -e "${GREEN}All checks passed!${NC}"