llman 0.0.65

A tool for managing LLM application rules(prompts) ...
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

#!/usr/bin/env python3
"""
Embedding API helper for llman sdd index rebuild.

Reads JSON from stdin: {"texts": [...], "api_url": "...", "api_key": "...", "model": "..."}
Outputs JSON to stdout: {"embeddings": [[f32; dim], ...]}
Errors to stderr.
"""
import json, sys, requests, time

def main():
    input_data = json.load(sys.stdin)
    texts = input_data["texts"]
    api_url = input_data.get("api_url", "").rstrip("/")
    api_key = input_data.get("api_key", "")
    model = input_data.get("model", "bge-m3-mlx-8bit")

    # Normalize API URL
    if "/embeddings" not in api_url:
        api_url = api_url.rstrip("/") + "/embeddings"

    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json",
    }

    all_embeddings = []
    batch_size = 8  # Safe batch size from experiment

    for i in range(0, len(texts), batch_size):
        batch = texts[i:i+batch_size]
        for attempt in range(3):
            try:
                resp = requests.post(
                    api_url,
                    headers=headers,
                    json={"model": model, "input": batch, "encoding_format": "float"},
                    timeout=60,
                )
                resp.raise_for_status()
                data = resp.json()
                # Sort by index to maintain order
                embeddings = [d["embedding"] for d in sorted(data["data"], key=lambda x: x["index"])]
                all_embeddings.extend(embeddings)
                break
            except Exception as e:
                if attempt < 2:
                    time.sleep(1)
                    continue
                print(f"Error embedding batch {i//batch_size}: {e}", file=sys.stderr)
                sys.exit(1)

    print(json.dumps({"embeddings": all_embeddings}))

if __name__ == "__main__":
    main()