llama_runner/runner/
gemma4.rs

1use crate::{
2    Gemma3VisionRunner, GenericVisionLmRequest, RunnerWithRecommendedSampling, VisionLmRunner,
3    error::{CreateLlamaCppRunnerError, GenericRunnerError},
4    sample::SimpleSamplingParams,
5    template::ChatTemplate,
6};
7
8pub const GEMMA_4_E2B_GUFF_MODEL_ID: &str = "unsloth/gemma-4-E2B-it-GGUF";
9pub const GEMMA_4_E2B_GUFF_MODEL_FILENAME: &str = "gemma-4-E2B-it-Q4_0.gguf";
10pub const GEMMA_4_E2B_GUFF_MULTIMODEL_FILENAME: &str = "mmproj-F16.gguf";
11
12#[repr(transparent)]
13pub struct Gemma4VisionRunner(Gemma3VisionRunner);
14
15impl Gemma4VisionRunner {
16    pub async fn default() -> Result<RunnerWithRecommendedSampling<Self>, CreateLlamaCppRunnerError>
17    {
18        let inner = Gemma3VisionRunner::new(
19            GEMMA_4_E2B_GUFF_MODEL_ID,
20            GEMMA_4_E2B_GUFF_MODEL_FILENAME,
21            GEMMA_4_E2B_GUFF_MULTIMODEL_FILENAME,
22            128_000u32.try_into().unwrap(),
23        )
24        .await?;
25        Ok(RunnerWithRecommendedSampling {
26            inner: Gemma4VisionRunner(inner),
27            default_sampling: SimpleSamplingParams {
28                top_p: Some(0.95f32),
29                top_k: Some(64),
30                temperature: Some(1.0f32),
31                presence_penalty: Some(1.5),
32                repetition_penalty: Some(1.0),
33                seed: None,
34            },
35        })
36    }
37}
38
39pub trait Gemma4ApplicableChatTemplate: ChatTemplate {}
40
41impl<'s, 'req, Tmpl: Gemma4ApplicableChatTemplate> VisionLmRunner<'s, 'req, Tmpl>
42    for Gemma4VisionRunner
43{
44    fn stream_vlm_response(
45        &'s self,
46        request: GenericVisionLmRequest<'req, Tmpl>,
47    ) -> impl Iterator<Item = Result<String, GenericRunnerError<Tmpl::Error>>> {
48        self.0.stream_vlm_response(request)
49    }
50}
51
52#[cfg(test)]
53mod test {
54    use crate::{mcp::Gemma4ChatTemplate, *};
55
56    #[tokio::test]
57    #[cfg(feature = "mcp")]
58    async fn test_vlm() {
59        let runner = Gemma4VisionRunner::default().await.unwrap();
60        let eiffel_tower_im =
61            image::load_from_memory(include_bytes!("../../assets/eiffel-tower.jpg")).unwrap();
62        let answer = runner
63            .get_vlm_response(GenericRunnerRequest {
64                messages: vec![
65                    (
66                        MessageRole::User,
67                        ImageOrText::Text("Which city is this building in?"),
68                    ),
69                    (MessageRole::User, ImageOrText::Image(&eiffel_tower_im)),
70                ],
71                tmpl: Gemma4ChatTemplate::default(),
72                ..Default::default()
73            })
74            .unwrap();
75        assert!(answer.contains("Paris"));
76    }
77}
llama_runner/runner/gemma4.rs

llama_runner/runner/
gemma4.rs
