omniference 0.3.2

A multi-protocol inference engine with provider adapters
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

use omniference::{
	server::OmniferenceServer,
	types::{ProviderConfig, ProviderKind},
};

#[tokio::main]
async fn main() -> anyhow::Result<()> {
	// Initialize logging
	tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init();

	// Create and configure server
	let mut server = OmniferenceServer::new();

	// Add Ollama provider
	server
		.add_provider(ProviderConfig {
			name: "ollama".to_string(),
			endpoint: omniference::types::ProviderEndpoint {
				kind: ProviderKind::OpenAICompat,
				base_url: "http://localhost:11434".to_string(),
				api_key: None,
				extra_headers: std::collections::BTreeMap::new(),
				timeout: Some(30000),
			},
			enabled: true,
			catalog_provider_slug: None,
		})
		.await
		.map_err(|e| anyhow::anyhow!(e))?;

	// Alternative usage with builder pattern:
	/*
	let mut server = OmniferenceServerBuilder::new()
		.with_adapter(Arc::new(OllamaAdapter))
		.build();

	server.add_provider(provider_config).await?;
	*/

	// Run the server
	let addr = std::env::var("SERVER_ADDR").unwrap_or_else(|_| "127.0.0.1:8080".to_string());
	tracing::info!("Starting Omniference server on {}", addr);
	server.run(&addr).await?;

	Ok(())
}