oxillama-py 0.1.3

Python bindings for OxiLLaMa LLM inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

"""HuggingFace Hub utilities for oxillama."""

from __future__ import annotations

from typing import Optional

from . import Engine, EngineConfig


def load_from_hub(
    repo_id: str,
    filename: Optional[str] = None,
    revision: Optional[str] = None,
    token: Optional[str] = None,
    config: Optional[EngineConfig] = None,
) -> Engine:
    """Download a GGUF model from HuggingFace Hub and load it.

    Args:
        repo_id: HuggingFace repository ID, e.g. ``"TheBloke/Llama-2-7B-GGUF"``.
        filename: Specific GGUF file within the repo.  If *None*, the first
            ``*.gguf`` file found is used.
        revision: Git revision / branch / tag.  Defaults to ``"main"``.
        token: HuggingFace access token.  Falls back to ``$HF_TOKEN`` /
            ``$HUGGINGFACE_HUB_TOKEN`` environment variables.
        config: Optional engine configuration.  Uses defaults if *None*.

    Returns:
        A loaded :class:`Engine` ready for inference.
    """
    return Engine.from_hub(
        repo_id,
        filename=filename,
        revision=revision,
        token=token,
        config=config,
    )