arms 0.0.3

🧬 The adaptive model routing system for exploration and exploitation.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

import os

from amrs.config import ModelConfig, Config


def test_model_config():
    test_cases = [
        {
            "name": "valid with default config",
            "input": {
                "id": "gpt-4",
                "base_url": "https://api.example.com/v1",
            },
            "expected": lambda: Config(
                models=[ModelConfig(id="gpt-4", provider="amrs", base_url="https://api.example.com/v1")],
            ),
            "fail": False,
        },
        {
            "name": "valid with custom config and default base_url",
            "input": {
                "id": "gpt-4",
                "provider": "openai",
                "temperature": 0.5,
                "max_tokens": 1500,
            },
            "env": {"OPENAI_API_KEY": "test_openai_key"},
            "expected": lambda: Config(
                models=[
                    ModelConfig(
                        id="gpt-4",
                        provider="openai",
                        temperature=0.5,
                        max_tokens=1500,
                        base_url="https://api.openai.com/v1",
                    )
                ],
            ),
            "fail": False,
        },
        {
            "name": "valid in-house provider with custom base_url",
            "input": {
                "id": "gpt-4",
                "provider": "openai",
                "base_url": "https://custom.api.endpoint/v1",
            },
            "env": {"OPENAI_API_KEY": "custom_key"},
            "expected": lambda: Config(
                models=[
                    ModelConfig(
                        id="gpt-4",
                        provider="openai",
                        base_url="https://custom.api.endpoint/v1",
                    )
                ],
            ),
            "fail": False,
        },
        {
            "name": "valid custom_provider with custom base_url",
            "input": {
                "id": "custom-model",
                "provider": "custom_provider",
                "base_url": "https://custom.api.endpoint/v1",
            },
            "env": {"CUSTOM_PROVIDER_API_KEY": "custom_key"},
            "expected": lambda: Config(
                models=[
                    ModelConfig(
                        id="custom-model",
                        provider="custom_provider",
                        base_url="https://custom.api.endpoint/v1",
                    )
                ],
            ),
            "fail": False,
        },
        {
            "name": "invalid unknown_provider with missing env var",
            "input": {
                "id": "gpt-4",
                "provider": "unknown_provider",
                "base_url": "https://custom.api.endpoint/v1",
            },
            "expected": None,
            "fail": True,
        },
        {
            "name": "invalid unknown_provider without base_url",
            "input": {
                "id": "gpt-4",
                "provider": "unknown_provider",
            },
            "env": {"UNKNOWN_PROVIDER_API_KEY": "some_key"},
            "expected": None,
            "fail": True,
        }
    ]

    for case in test_cases:
        if "env" in case:
            for key, value in case["env"].items():
                os.environ[key] = value

        if case["fail"]:
            try:
                Config(models=[ModelConfig(**case["input"])])
                assert False, (
                    f"Test case '{case['name']}' should have failed but didn't."
                )
            except ValueError:
                pass  # Expected to fail
        else:
            config = Config(models=[ModelConfig(**case["input"])])
            assert config == case["expected"](), (
                f"Test case '{case['name']}' failed."
            )

        # reset
        if "env" in case:
            for key in case["env"].keys():
                del os.environ[key]