1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
use clap::{CommandFactory, Parser, Subcommand};
use llmman::{cmd, daemon, ffi, hostgpu};
// ---------------------------------------------------------------------------
// CLI definition
// ---------------------------------------------------------------------------
#[derive(Parser, Debug)]
#[command(
name = "llmman",
about = "Run any agent on any model, models stored as OCI images",
version = env!("LLMMAN_VERSION"),
propagate_version = true
)]
struct Cli {
#[command(subcommand)]
command: Option<Commands>,
}
#[derive(Subcommand, Debug)]
enum Commands {
/// Launch an integration
Launch(cmd::launch::LaunchArgs),
/// Run a model interactively or with a one-shot prompt
Run(Box<cmd::run::RunArgs>),
/// Package model files into a local OCI image
Build(cmd::build::BuildArgs),
/// Log in to a container registry or HuggingFace
Login(cmd::login::LoginArgs),
/// Log out from a container registry or HuggingFace
Logout(cmd::logout::LogoutArgs),
/// Push a local image to a registry
Push(cmd::push::PushArgs),
/// Pull an image from a registry to the local store
Pull(cmd::pull::PullArgs),
/// Search for models on Docker Hub and Hugging Face
Search(cmd::search::SearchArgs),
/// Pull (if needed) and print a model's local path as JSON (internal,
/// used by the vllm-llmman plugin)
#[command(hide = true)]
Resolve(cmd::resolve::ResolveArgs),
/// Transfer an image directly from one location to another (e.g. HuggingFace to an OCI registry)
Transfer(cmd::transfer::TransferArgs),
/// Check a registry model's signatures against trusted public keys
Verify(cmd::verify::VerifyArgs),
/// List locally stored images
#[command(alias = "ls")]
List(cmd::list::ListArgs),
/// List models currently loaded by a running `llmman serve`
Ps(cmd::ps::PsArgs),
/// Show the prompts `llmman serve` has seen, newest first (like `git log`)
Log(cmd::log::LogArgs),
/// Show the tokens and cost of what `llmman serve` has served, per model
Usage(cmd::usage::UsageArgs),
/// List the hosted providers `--provider` can route to
Providers(cmd::providers::ProvidersArgs),
/// Read and write llmman.conf settings
Config(cmd::config::ConfigArgs),
/// Copy a local image to a new reference
Cp(cmd::cp::CpArgs),
/// Remove a local image, freeing its blobs and extracted cache no
/// longer referenced by any other model (set LLMMAN_NOPRUNE to skip)
Rm(cmd::rm::RmArgs),
/// Stop (unload) a running model
Stop(cmd::stop::StopArgs),
/// Show a local model's architecture, parameters, license, and template
Show(cmd::show::ShowArgs),
/// Start an inference server (Ollama, OpenAI, Anthropic compatible APIs)
Serve(cmd::serve::ServeArgs),
/// Run a command and print its output compacted (internal, used by
/// `launch --optimizer tk`)
///
/// The command keeps this process's stdin, cwd and environment, and
/// `llmman tk` exits with its status. Filters cover `git`, `cargo`, `go`,
/// `pytest`, `npm|pnpm|yarn test`, `grep`, `rg` and `docker ps`; other
/// commands only get a generic clean-up. Commands that ask for JSON,
/// porcelain, a format or NUL separators, `cat`, and interactive commands
/// run untouched, as does everything with LLMMAN_TK=off.
///
/// Example: llmman tk git status
#[command(hide = true)]
Tk(cmd::tk::TkArgs),
/// Probe the local host's GPU/accelerator support (internal diagnostic)
#[command(hide = true)]
GpuDiscover(cmd::gpu_discover::GpuDiscoverArgs),
}
// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------
fn main() {
// Deliberately checked before anything else in this function — not a
// documented subcommand (absent from `Commands`/`--help` on purpose)
// and not routed through clap at all: this is `hostgpu::detect`'s own
// internal re-exec target, isolating its real CUDA/HIP/Vulkan FFI
// probing (see that module's doc comment) in a disposable child
// process of exactly this same binary. See
// `hostgpu::probe_subprocess_main`'s own doc comment for why that
// isolation exists at all, and why this has to run before
// `ffi::ensure_runtime_init`/`daemon::disable_std_handle_inheritance`
// below: this child is meant to do nothing but the one raw probe and
// exit, as fast and dependency-free as possible.
if std::env::args_os().nth(1).as_deref()
== Some(std::ffi::OsStr::new(hostgpu::PROBE_SUBPROCESS_ARG))
{
hostgpu::probe_subprocess_main();
}
// Must happen before any other call into the `ffi` module, from every
// process that links the Go shim in — both this CLI's own process and
// the detached `llmman serve` daemon it spawns (a separate process,
// with its own copy of the shim's Go runtime to bootstrap) each reach
// this same `main()`. See ffi::ensure_runtime_init's own doc comment
// for why this is necessary on Windows specifically.
ffi::ensure_runtime_init();
// As early as possible, before this process (directly, or via
// daemon::ensure_server) can spawn anything else on Windows — see
// daemon::disable_std_handle_inheritance's own doc comment for the
// real E2E hang this fixes.
daemon::disable_std_handle_inheritance();
let cli = Cli::parse_from(cmd::log::expand_count_shorthand(std::env::args_os()));
// A bare `llmman` is a request for help, not a usage error: print it
// and exit 0 rather than clap's 2 (package validators flag a non-zero
// exit from a bare invocation as a broken install).
let Some(command) = &cli.command else {
Cli::command()
.print_help()
.expect("failed to write help to stdout");
return;
};
let result = match command {
Commands::Launch(a) => cmd::launch::run(a),
Commands::Run(a) => cmd::run::run(a),
Commands::Build(a) => cmd::build::run(a),
Commands::Login(a) => cmd::login::run(a),
Commands::Logout(a) => cmd::logout::run(a),
Commands::Push(a) => cmd::push::run(a),
Commands::Pull(a) => cmd::pull::run(a),
Commands::Search(a) => cmd::search::run(a),
Commands::Resolve(a) => cmd::resolve::run(a),
Commands::Transfer(a) => cmd::transfer::run(a),
Commands::Verify(a) => cmd::verify::run(a),
Commands::List(a) => cmd::list::run(a),
Commands::Ps(a) => cmd::ps::run(a),
Commands::Log(a) => cmd::log::run(a),
Commands::Usage(a) => cmd::usage::run(a),
Commands::Providers(a) => cmd::providers::run(a),
Commands::Config(a) => cmd::config::run(a),
Commands::Cp(a) => cmd::cp::run(a),
Commands::Rm(a) => cmd::rm::run(a),
Commands::Stop(a) => cmd::stop::run(a),
Commands::Show(a) => cmd::show::run(a),
Commands::Serve(a) => cmd::serve::run(a),
Commands::Tk(a) => cmd::tk::run(a),
Commands::GpuDiscover(a) => cmd::gpu_discover::run(a),
};
if let Err(e) = result {
eprintln!("Error: {:#}", e);
std::process::exit(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
/// `tk` is an internal command: absent from the list, but it has its
/// own `--help` and runs anything after its name.
#[test]
fn tk_is_hidden_from_the_main_help_but_has_its_own() {
let help = Cli::command().render_help().to_string();
assert!(
!help.lines().any(|l| l.trim_start().starts_with("tk ")),
"{help}"
);
assert!(help.lines().any(|l| l.trim_start().starts_with("launch ")));
let err = Cli::command()
.try_get_matches_from(["llmman", "tk", "--help"])
.unwrap_err();
assert_eq!(err.kind(), clap::error::ErrorKind::DisplayHelp);
assert!(err.to_string().contains("llmman tk git status"), "{err}");
let parsed = Cli::try_parse_from(["llmman", "tk", "git", "diff", "--stat"]).unwrap();
assert!(matches!(parsed.command, Some(Commands::Tk(_))));
}
}