dynamo_llm/
lib.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! # Dynamo LLM
5//!
6//! The `dynamo.llm` crate is a Rust library that provides a set of traits and types for building
7//! distributed LLM inference solutions.
8
9use std::{fs::File, io::BufReader, path::Path};
10
11use anyhow::Context as _;
12
13pub mod backend;
14pub mod common;
15pub mod disagg_router;
16pub mod discovery;
17pub mod endpoint_type;
18pub mod engines;
19pub mod entrypoint;
20pub mod gguf;
21pub mod grpc;
22pub mod http;
23pub mod hub;
24// pub mod key_value_store;
25pub mod kv_router;
26pub mod local_model;
27pub mod migration;
28pub mod mocker;
29pub mod model_card;
30pub mod model_type;
31pub mod namespace;
32pub mod perf;
33pub mod preprocessor;
34pub mod protocols;
35pub mod recorder;
36pub mod request_template;
37pub mod tokenizers;
38pub mod tokens;
39pub mod types;
40pub mod utils;
41
42#[cfg(feature = "block-manager")]
43pub mod block_manager;
44
45#[cfg(feature = "cuda")]
46pub mod cuda;
47
48/// Reads a JSON file, extracts a specific field, and deserializes it into type T.
49///
50/// # Arguments
51///
52/// * `json_file_path`: Path to the JSON file.
53/// * `field_name`: The name of the field to extract from the JSON map.
54///
55/// # Returns
56///
57/// A `Result` containing the deserialized value of type `T` if successful,
58/// or an `anyhow::Error` if any step fails (file I/O, JSON parsing, field not found,
59/// or deserialization to `T` fails).
60///
61/// # Type Parameters
62///
63/// * `T`: The expected type of the field's value. `T` must implement `serde::de::DeserializeOwned`.
64pub fn file_json_field<T: serde::de::DeserializeOwned>(
65    json_file_path: &Path,
66    field_name: &str,
67) -> anyhow::Result<T> {
68    // 1. Open the file
69    let file = File::open(json_file_path)
70        .with_context(|| format!("Failed to open file: {:?}", json_file_path))?;
71    let reader = BufReader::new(file);
72
73    // 2. Parse the JSON file into a generic serde_json::Value
74    // We parse into `serde_json::Value` first because we need to look up a specific field.
75    // If we tried to deserialize directly into `T`, `T` would need to represent the whole JSON structure.
76    let json_data: serde_json::Value = serde_json::from_reader(reader)
77        .with_context(|| format!("Failed to parse JSON from file: {:?}", json_file_path))?;
78
79    // 3. Ensure the root of the JSON is an object (map)
80    let map = json_data.as_object().ok_or_else(|| {
81        anyhow::anyhow!("JSON root is not an object in file: {:?}", json_file_path)
82    })?;
83
84    // 4. Get the specific field's value
85    let field_value = map.get(field_name).ok_or_else(|| {
86        anyhow::anyhow!(
87            "Field '{}' not found in JSON file: {:?}",
88            field_name,
89            json_file_path
90        )
91    })?;
92
93    // 5. Deserialize the field's value into the target type T
94    // We need to clone `field_value` because `from_value` consumes its input.
95    serde_json::from_value(field_value.clone()).with_context(|| {
96        format!(
97            "Failed to deserialize field '{}' (value: {:?}) to the expected type from file: {:?}",
98            field_name, field_value, json_file_path
99        )
100    })
101}
102
103/// Pretty-print the part of JSON that has an error.
104pub fn log_json_err(filename: &str, json: &str, err: &serde_json::Error) {
105    const ERROR_PREFIX: &str = ">>     ";
106
107    // Only log errors that relate to the content of the JSON file
108    if !(err.is_syntax() || err.is_data()) {
109        return;
110    }
111    // These are 1 based for humans so subtract
112    let line = err.line().saturating_sub(1);
113    let column = err.column().saturating_sub(1);
114
115    let json_lines: Vec<&str> = json.lines().collect();
116    if json_lines.is_empty() {
117        tracing::error!("JSON parsing error in {filename}: File is empty.");
118        return;
119    }
120
121    // Two lines before
122    let start_index = (line - 2).max(0);
123    // The problem line and two lines after
124    let end_index = (line + 3).min(json_lines.len());
125
126    // Collect the context
127    let mut context_lines: Vec<String> = (start_index..end_index)
128        .map(|i| {
129            if i == line {
130                format!("{ERROR_PREFIX}{}", json_lines[i])
131            } else {
132                // Six places because tokenizer.json is very long
133                format!("{:06} {}", i + 1, json_lines[i])
134            }
135        })
136        .collect();
137
138    // Insert the column indicator
139    let col_indicator = "_".to_string().repeat(column + ERROR_PREFIX.len()) + "^";
140    let error_in_context_idx = line - start_index;
141    if error_in_context_idx < context_lines.len() {
142        context_lines.insert(error_in_context_idx + 1, col_indicator);
143    }
144
145    tracing::error!(
146        "JSON parsing error in {filename}: Line {}, column {}:\n{}",
147        err.line(),
148        err.column(),
149        context_lines.join("\n")
150    );
151}
152
153#[cfg(test)]
154mod file_json_field_tests {
155    use super::file_json_field;
156    use serde::Deserialize;
157    use std::fs::File;
158    use std::io::Write;
159    use std::path::{Path, PathBuf};
160    use tempfile::tempdir;
161
162    // Helper function to create a temporary JSON file
163    fn create_temp_json_file(dir: &Path, file_name: &str, content: &str) -> PathBuf {
164        let file_path = dir.join(file_name);
165        let mut file = File::create(&file_path)
166            .unwrap_or_else(|_| panic!("Failed to create test file: {:?}", file_path));
167        file.write_all(content.as_bytes())
168            .unwrap_or_else(|_| panic!("Failed to write to test file: {:?}", file_path));
169        file_path
170    }
171
172    // Define a custom struct for testing deserialization
173    #[derive(Debug, PartialEq, Deserialize)]
174    struct MyConfig {
175        version: String,
176        enabled: bool,
177        count: u32,
178    }
179
180    #[test]
181    fn test_success_basic() {
182        let tmp_dir = tempdir().unwrap();
183        let file_path = create_temp_json_file(
184            tmp_dir.path(),
185            "test_basic.json",
186            r#"{ "name": "Rust", "age": 30, "is_active": true }"#,
187        );
188
189        let name: String = file_json_field(&file_path, "name").unwrap();
190        assert_eq!(name, "Rust");
191
192        let age: i32 = file_json_field(&file_path, "age").unwrap();
193        assert_eq!(age, 30);
194
195        let is_active: bool = file_json_field(&file_path, "is_active").unwrap();
196        assert!(is_active);
197    }
198
199    #[test]
200    fn test_success_custom_struct_field() {
201        let tmp_dir = tempdir().unwrap();
202        let file_path = create_temp_json_file(
203            tmp_dir.path(),
204            "test_struct.json",
205            r#"{
206                "config": {
207                    "version": "1.0.0",
208                    "enabled": true,
209                    "count": 123
210                },
211                "other_field": "value"
212            }"#,
213        );
214
215        let config: MyConfig = file_json_field(&file_path, "config").unwrap();
216        assert_eq!(
217            config,
218            MyConfig {
219                version: "1.0.0".to_string(),
220                enabled: true,
221                count: 123,
222            }
223        );
224    }
225
226    #[test]
227    fn test_file_not_found() {
228        let tmp_dir = tempdir().unwrap();
229        let non_existent_path = tmp_dir.path().join("non_existent.json");
230
231        let result: anyhow::Result<String> = file_json_field(&non_existent_path, "field");
232        assert!(result.is_err());
233        let err = result.unwrap_err();
234        assert!(err.to_string().contains("Failed to open file"));
235    }
236
237    #[test]
238    fn test_invalid_json_syntax() {
239        let tmp_dir = tempdir().unwrap();
240        let file_path = create_temp_json_file(
241            tmp_dir.path(),
242            "invalid.json",
243            r#"{ "key": "value", "bad_syntax": }"#, // Malformed JSON
244        );
245
246        let result: anyhow::Result<String> = file_json_field(&file_path, "key");
247        assert!(result.is_err());
248        let err = result.unwrap_err();
249        assert!(err.to_string().contains("Failed to parse JSON from file"));
250    }
251
252    #[test]
253    fn test_json_root_not_object_array() {
254        let tmp_dir = tempdir().unwrap();
255        let file_path = create_temp_json_file(
256            tmp_dir.path(),
257            "root_array.json",
258            r#"[ { "item": 1 }, { "item": 2 } ]"#, // Root is an array
259        );
260
261        let result: anyhow::Result<String> = file_json_field(&file_path, "item");
262        assert!(result.is_err());
263        let err = result.unwrap_err();
264        assert!(err.to_string().contains("JSON root is not an object"));
265    }
266
267    #[test]
268    fn test_json_root_not_object_primitive() {
269        let tmp_dir = tempdir().unwrap();
270        let file_path = create_temp_json_file(
271            tmp_dir.path(),
272            "root_primitive.json",
273            r#""just_a_string""#, // Root is a string
274        );
275
276        let result: anyhow::Result<String> = file_json_field(&file_path, "field");
277        assert!(result.is_err());
278        let err = result.unwrap_err();
279        assert!(err.to_string().contains("JSON root is not an object"));
280    }
281
282    #[test]
283    fn test_field_not_found() {
284        let tmp_dir = tempdir().unwrap();
285        let file_path = create_temp_json_file(
286            tmp_dir.path(),
287            "missing_field.json",
288            r#"{ "existing_field": "hello" }"#,
289        );
290
291        let result: anyhow::Result<String> = file_json_field(&file_path, "non_existent_field");
292        assert!(result.is_err());
293        let err = result.unwrap_err();
294        assert!(
295            err.to_string()
296                .contains("Field 'non_existent_field' not found")
297        );
298    }
299
300    #[test]
301    fn test_field_type_mismatch() {
302        let tmp_dir = tempdir().unwrap();
303        let file_path = create_temp_json_file(
304            tmp_dir.path(),
305            "type_mismatch.json",
306            r#"{ "count": "not_an_integer" }"#,
307        );
308
309        let result: anyhow::Result<u32> = file_json_field(&file_path, "count");
310        assert!(result.is_err());
311        let err = result.unwrap_err();
312        assert!(
313            err.to_string()
314                .contains("Failed to deserialize field 'count'")
315        );
316    }
317
318    #[test]
319    fn test_empty_file() {
320        let tmp_dir = tempdir().unwrap();
321        let file_path = create_temp_json_file(tmp_dir.path(), "empty.json", "");
322
323        let result: anyhow::Result<String> = file_json_field(&file_path, "field");
324        assert!(result.is_err());
325        let err = result.unwrap_err();
326        assert!(err.to_string().contains("Failed to parse JSON from file"));
327    }
328}
dynamo_llm/lib.rs

dynamo_llm/
lib.rs
