modelexpress-client 0.3.0

Client library for Model Express gRPC server
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""ModelExpress worker wrapper for vLLM.

This worker ensures ModelExpress loaders are registered in spawned worker processes.
When vLLM uses multiprocess executor with spawn method, worker processes start fresh
and don't inherit registered loaders from the main process.

Usage:
    Set --worker-cls=modelexpress.vllm_worker.ModelExpressWorker
"""

from vllm.v1.worker.gpu_worker import Worker


class ModelExpressWorker(Worker):
    """Worker that registers ModelExpress loaders before model loading."""

    def __init__(self, *args, **kwargs):
        # Register loaders before parent initialization.
        # This imports modelexpress.vllm_loader which has @register_model_loader
        # decorators on the MxModelLoader class.
        from modelexpress import register_modelexpress_loaders

        register_modelexpress_loaders()
        super().__init__(*args, **kwargs)