llama-cpp-sys-4 0.10.0

// fix problem with std::min and std::max
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
#   define NOMINMAX
#endif
#include <windows.h>
#endif

#include "mtmd.h"
#include "mtmd-helper.h"
#include "mtmd-helper-common.h"
#include "llama.h"
#include "../src/llama-ext.h"

#include "hash/hash.h"

#include <algorithm>
#include <cmath>
#include <cinttypes>
#include <vector>

//#define MTMD_AUDIO_DEBUG

#define MINIAUDIO_IMPLEMENTATION
#ifndef MTMD_AUDIO_DEBUG
#   define MA_NO_ENCODING
#endif
#define MA_NO_DEVICE_IO
#define MA_NO_RESOURCE_MANAGER
#define MA_NO_NODE_GRAPH
#define MA_NO_ENGINE
#define MA_NO_GENERATION
#define MA_API static
#include "miniaudio/miniaudio.h"

#define STB_IMAGE_IMPLEMENTATION
#include "stb/stb_image.h"

#ifdef MTMD_INTERNAL_HEADER
#error "mtmd-helper is a public library outside of mtmd. it must not include internal headers"
#endif

#ifdef MTMD_VIDEO
#include "sheredom/subprocess.h"
#include <thread>
#ifndef _WIN32
#include <csignal>
#include <fcntl.h>
#include <pthread.h>
#endif
#endif

//
// internal logging functions
//

void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data) {
    if (log_callback == nullptr) {
        log_callback = g_logger.default_callback;
    }
    g_logger.log_callback = log_callback;
    g_logger.log_callback_user_data = user_data;
    mtmd_log_set(log_callback, user_data);
}

//
// helper functions
//

size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks) {
    size_t n_tokens = 0;
    for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) {
        auto chunk = mtmd_input_chunks_get(chunks, i);
        n_tokens += mtmd_input_chunk_get_n_tokens(chunk);
    }
    return n_tokens;
}

llama_pos mtmd_helper_get_n_pos(const mtmd_input_chunks * chunks) {
    llama_pos n_pos = 0;
    for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) {
        auto chunk = mtmd_input_chunks_get(chunks, i);
        n_pos += mtmd_input_chunk_get_n_pos(chunk);
    }
    return n_pos;
}

void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * chunks, llama_pos pos_0, mtmd_decoder_pos * out_pos) {
    size_t n_tokens = mtmd_image_tokens_get_n_tokens(chunks);
    for (size_t i = 0; i < n_tokens; i++) {
        out_pos[i] = mtmd_image_tokens_get_decoder_pos(chunks, pos_0, i);
    }
}

// Helper class to set non-causal attention via RAII
class scope_non_causal {
public:
    scope_non_causal(llama_context * context, bool enabled) : context_(context), enabled_(enabled) {
        if (enabled_) {
            // TODO @ngxson : need to make sure only one image is processed at a time, and n_ubatch must be enough to hold the image
            llama_set_causal_attn(context_, false);
        }
    }
    ~scope_non_causal() {
        if (enabled_) {
            llama_set_causal_attn(context_, true);
        }
    }

    scope_non_causal(const scope_non_causal &) = delete;
    scope_non_causal & operator=(const scope_non_causal &) = delete;

private:
    llama_context * context_;
    bool enabled_;
};

// Helper function for decoding an image whose embeddings have already been calculated
int32_t mtmd_helper_decode_image_chunk(
        mtmd_context * ctx,
        struct llama_context * lctx,
        const mtmd_input_chunk * chunk,
        float * encoded_embd,
        llama_pos n_past,
        llama_seq_id seq_id,
        int32_t n_batch,
        llama_pos * new_n_past,
        mtmd_helper_post_decode_callback callback,
        void * user_data) {
    GGML_ASSERT(n_batch > 0);
    auto chunk_type = mtmd_input_chunk_get_type(chunk);
    const char * name = chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE ? "image" : "audio";
    if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
        LOG_ERR("failed to decode chunk: input chunk not of image/audio type\n");
        return -1;
    }

    const llama_model * model = llama_get_model(lctx);
    int n_mmproj_embd = llama_model_n_embd_inp(model);
    int n_pos_per_embd = mtmd_decode_use_mrope(ctx) ? 4 : 1;

    int32_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk);
    int32_t i_batch = 0;
    int32_t n_img_batches = (n_tokens + n_batch - 1) / n_batch;
    decode_embd_batch batch_embd(encoded_embd, n_tokens, n_pos_per_embd, n_mmproj_embd);

    if (mtmd_decode_use_mrope(ctx)) {
        if (chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {
            const auto image_tokens = mtmd_input_chunk_get_tokens_image(chunk);
            if (!image_tokens) {
                LOG_ERR("failed to decode chunk: image tokens are null\n");
                return -1;
            }
            const auto n_tokens = mtmd_image_tokens_get_n_tokens(image_tokens);
            std::vector<mtmd_decoder_pos> rel_pos(n_tokens);
            mtmd_helper_image_get_decoder_pos(image_tokens, n_past, rel_pos.data());
            batch_embd.set_position_mrope_2d(rel_pos, seq_id);
        } else if (chunk_type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
            batch_embd.set_position_mrope_1d(n_past, seq_id);
        } else {
            GGML_ABORT("invalid chunk type for M-RoPE");
        }
    } else {
        batch_embd.set_position_normal(n_past, seq_id);
    }

    const bool use_non_causal = mtmd_decode_use_non_causal(ctx, chunk);
    const scope_non_causal non_causal(lctx, use_non_causal);

    while (i_batch < n_img_batches) { // split into batches
        int pos_offset = i_batch*n_batch;
        int n_tokens_batch = std::min(n_batch, n_tokens - pos_offset);

        LOG_INF("decoding %s batch %d/%d, n_tokens_batch = %d\n", name, i_batch+1, n_img_batches, n_tokens_batch);

        int64_t t1 = ggml_time_ms();
        int32_t ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, pos_offset, n_tokens_batch));
        if (ret != 0) {
            LOG_ERR("failed to decode %s\n", name);
            return ret;
        }

        if (callback != nullptr) {
            const mtmd_helper_embd_batch view = batch_embd.get_view(pos_offset, n_tokens_batch);
            ret = callback(&view, user_data);
            if (ret != 0) {
                LOG_ERR("post-decode callback failed\n");
                return ret;
            }
        }

        LOG_INF("%s decoded (batch %d/%d) in %" PRId64 " ms\n", name, i_batch+1, n_img_batches, ggml_time_ms() - t1);

        i_batch++;
    }

    n_past += mtmd_input_chunk_get_n_pos(chunk);
    *new_n_past = n_past;

    return 0;
}

int32_t mtmd_helper_eval_chunk_single(mtmd_context * ctx,
        struct llama_context * lctx,
        const mtmd_input_chunk * chunk,
        llama_pos n_past,
        llama_seq_id seq_id,
        int32_t n_batch,
        bool logits_last,
        llama_pos * new_n_past) {
    GGML_ASSERT(n_batch > 0);
    int32_t ret;
    auto chunk_type = mtmd_input_chunk_get_type(chunk);

    if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
        size_t n_tokens;
        const auto tokens = mtmd_input_chunk_get_tokens_text(chunk, &n_tokens);
        // LOG_INF("decoding text chunk, n_tokens = %zu\n", n_tokens);
        llama_batch_ext_ptr text_batch(llama_batch_ext_init(lctx));
        size_t i = 0;
        while (i < n_tokens) { // split into batches
            llama_batch_ext_clear(text_batch.get());
            int32_t n_added = 0;
            int32_t idx     = -1;
            for (; i < n_tokens && n_added < n_batch; i++) {
                idx = llama_batch_ext_add_token(text_batch.get(), seq_id, tokens[i]);
                GGML_ASSERT(idx >= 0);
                llama_pos pos = n_past++;
                llama_batch_ext_set_pos(text_batch.get(), idx, &pos);
                n_added++;
            }
            bool is_last_token = (i == n_tokens);
            if (logits_last && is_last_token) {
                llama_batch_ext_set_output_logits(text_batch.get(), idx, true);
            }
            ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, text_batch.get());
            if (ret != 0) {
                LOG_ERR("failed to decode text\n");
                return ret;
            }
            *new_n_past += n_added;
        }

    } else if (chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE || chunk_type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
        const char * name = chunk_type == MTMD_INPUT_CHUNK_TYPE_IMAGE ? "image" : "audio";
        int64_t t0 = ggml_time_ms();

        LOG_INF("encoding %s slice...\n", name);

        ret = mtmd_encode_chunk(ctx, chunk);
        if (ret != 0) {
            LOG_ERR("failed to encode %s slice\n", name);
            return ret;
        }

        LOG_INF("%s slice encoded in %" PRId64 " ms\n", name, ggml_time_ms() - t0);

        float * embd = mtmd_get_output_embd(ctx);
        ret = mtmd_helper_decode_image_chunk(ctx, lctx, chunk, embd, n_past, seq_id, n_batch, new_n_past, nullptr, nullptr);
        if (ret != 0) {
            LOG_ERR("failed to decode %s\n", name);
            return ret;
        }
    } else {
        GGML_ABORT("chunk type not supported");
    }

    return 0;
}

int32_t mtmd_helper_eval_chunks(mtmd_context * ctx,
                                struct llama_context * lctx,
                                const mtmd_input_chunks * chunks,
                                llama_pos n_past,
                                llama_seq_id seq_id,
                                int32_t n_batch,
                                bool logits_last,
                                llama_pos * new_n_past) {
    size_t n_chunks = mtmd_input_chunks_size(chunks);
    if (n_chunks == 0) {
        LOG_WRN("no chunks to eval\n");
        return 0;
    }

    for (size_t i = 0; i < n_chunks; i++) {
        bool chunk_logits_last = (i == n_chunks - 1) && logits_last;
        auto chunk = mtmd_input_chunks_get(chunks, i);

        int32_t res = mtmd_helper_eval_chunk_single(ctx, lctx, chunk, n_past, seq_id, n_batch, chunk_logits_last, &n_past);
        if (res != 0) {
            LOG_ERR("failed to eval chunk %zu\n", i);
            return res;
        }
        *new_n_past = n_past;
    }

    return 0;
}

namespace audio_helpers {

static bool is_audio_file(const char * buf, size_t len) {
    if (len < 12) {
        return false;
    }

    // RIFF ref: https://en.wikipedia.org/wiki/Resource_Interchange_File_Format
    // WAV ref: https://www.mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/WAVE.html
    bool is_wav = memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WAVE", 4) == 0;
    bool is_mp3 = len >= 3 && (
        memcmp(buf, "ID3", 3) == 0 ||
        // Check for MPEG sync word (simplified check)
        ((unsigned char)buf[0] == 0xFF && ((unsigned char)buf[1] & 0xE0) == 0xE0)
    );
    bool is_flac = memcmp(buf, "fLaC", 4) == 0;

    return is_wav || is_mp3 || is_flac;
}

// returns true if the buffer is a valid audio file
static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int target_sampler_rate, std::vector<float> & pcmf32_mono) {
    ma_result result;
    const int channels = 1;
    ma_decoder_config decoder_config = ma_decoder_config_init(ma_format_f32, channels, target_sampler_rate);
    ma_decoder decoder;

    result = ma_decoder_init_memory(buf_in, len, &decoder_config, &decoder);
    if (result != MA_SUCCESS) {
        return false;
    }

    ma_uint64 frame_count;
    ma_uint64 frames_read;
    result = ma_decoder_get_length_in_pcm_frames(&decoder, &frame_count);
    if (result != MA_SUCCESS) {
        ma_decoder_uninit(&decoder);
        return false;
    }

    pcmf32_mono.resize(frame_count);
    result = ma_decoder_read_pcm_frames(&decoder, pcmf32_mono.data(), frame_count, &frames_read);
    if (result != MA_SUCCESS) {
        ma_decoder_uninit(&decoder);
        return false;
    }

#ifdef MTMD_AUDIO_DEBUG
    // save audio to wav file
    ma_encoder_config config = ma_encoder_config_init(ma_encoding_format_wav, ma_format_f32, 1, target_sampler_rate);
    ma_encoder encoder;
    ma_encoder_init_file("output.wav", &config, &encoder);
    ma_encoder_write_pcm_frames(&encoder, pcmf32_mono.data(), pcmf32_mono.size(), &frames_read);
    ma_encoder_uninit(&encoder);
#endif

    ma_decoder_uninit(&decoder);
    return true;
}

} // namespace audio_helpers

static bool is_webp_file(const unsigned char * buf, size_t len) {
    // WEBP ref: https://developers.google.com/speed/webp/docs/riff_container
    return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0;
}

#ifdef MTMD_VIDEO
static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
                                             const mtmd_helper_video_init_params & params);
static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string & id);
#endif

mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder,
                                                            mtmd_helper_init_opt opt) {
    // calculate the hash if needed
    std::string id;
    mtmd_bitmap * result = nullptr;

    GGML_UNUSED(opt); // only used by video code paths

    if (!placeholder) {
        // use sha256 to prevent cache poisoning
        id = hash_sha256_hex(buf, len);
    }

    if (audio_helpers::is_audio_file((const char *)buf, len)) {
        std::vector<float> pcmf32;
        const int sample_rate = mtmd_get_audio_sample_rate(ctx);
        if (sample_rate < 0) {
            LOG_ERR("This model does not support audio input\n");
            return {nullptr, nullptr};
        }
        if (!audio_helpers::decode_audio_from_buf(buf, len, sample_rate, pcmf32)) {
            LOG_ERR("Unable to read WAV audio file from buffer\n");
            return {nullptr, nullptr};
        }
        result = mtmd_bitmap_init_from_audio(pcmf32.size(), placeholder ? nullptr : pcmf32.data());
        mtmd_bitmap_set_id(result, id.empty() ? nullptr : id.c_str());
        return {result, nullptr};
    }

    // otherwise, we assume it's an image
    if (!result) {
        int nx, ny, nc;
        auto * data = stbi_load_from_memory(buf, len, &nx, &ny, &nc, 3);
        if (data) {
            result = mtmd_bitmap_init(nx, ny, placeholder ? nullptr : data);
            mtmd_bitmap_set_id(result, id.empty() ? nullptr : id.c_str());
            stbi_image_free(data);
            return {result, nullptr};
        }
        // otherwise, fallthrough to video decoding (if supported)
    }

#ifdef MTMD_VIDEO
    // stb_image does not support webp; decode it with ffmpeg as a single frame
    if (!result && is_webp_file(buf, len)) {
        result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder, opt.video_params);
        if (!result) {
            LOG_ERR("%s: failed to decode webp buffer\n", __func__);
            return {nullptr, nullptr};
        }
        mtmd_bitmap_set_id(result, id.empty() ? nullptr : id.c_str());
        return {result, nullptr};
    }
#endif

    // last try: load as video
#ifdef MTMD_VIDEO
    if (!result) {
        auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, opt.video_params);
        if (!video_ctx) {
            LOG_ERR("%s: failed to decode buffer as either image/audio/video\n", __func__);
            return {nullptr, nullptr};
        }
        mtmd_helper_video_set_id(video_ctx, id); // propagate the hash to the frames
        result = mtmd_bitmap_init_lazy(ctx,
            id.empty() ? nullptr : id.c_str(),
            video_ctx,
            [](size_t, void * user_data, mtmd_bitmap ** out_bitmap, char ** out_text) -> int {
                auto * vctx = static_cast<mtmd_helper_video *>(user_data);
                char * text = nullptr;
                int ret = mtmd_helper_video_read_next(vctx, out_bitmap, &text);
                *out_text = text; // heap-allocated by read_next; freed automatically by mtmd
                return ret;
            });
        mtmd_bitmap_set_is_video(result, true);
        return {result, video_ctx};
    }
#else
    if (!result) {
        LOG_ERR("%s: failed to decode buffer as either image or audio (video support not compiled in)\n", __func__);
        return {nullptr, nullptr};
    }
#endif

    // should not reach here
    return {nullptr, nullptr};
}

mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context * ctx, const char * fname, bool placeholder,
                                                             mtmd_helper_init_opt opt) {
#ifdef _WIN32
    int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
    if (!wlen) {
        LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname);
        return {nullptr, nullptr};
    }
    std::vector<wchar_t> wfname(wlen);
    wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wlen);
    if (!wlen) {
        LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname);
        return {nullptr, nullptr};
    }
    FILE * f = _wfopen(wfname.data(), L"rb");
#else
    FILE * f = fopen(fname, "rb");
#endif
    if (!f) {
        LOG_ERR("Unable to open file %s: %s\n", fname, strerror(errno));
        return {nullptr, nullptr};
    }

    std::vector<unsigned char> buf;

    fseek(f, 0, SEEK_END);
    long file_size = ftell(f);
    fseek(f, 0, SEEK_SET);
    if (file_size < 0) {
        LOG_ERR("Failed to get file size of %s\n", fname);
        fclose(f);
        return {nullptr, nullptr};
    }
    buf.resize(file_size);

    size_t n_read = fread(buf.data(), 1, file_size, f);
    fclose(f);
    if (n_read != (size_t)file_size) {
        LOG_ERR("Failed to read entire file %s", fname);
        return {nullptr, nullptr};
    }

    return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt);
}

//
// Video input helpers
//

#ifdef MTMD_VIDEO

// MSVC deprecates strdup(), use a std::string version directly
static char * mtmd_strdup(const std::string & s) {
    size_t n = s.size() + 1;
    char * p = (char *) malloc(n);
    if (p) {
        memcpy(p, s.c_str(), n);
    }
    return p;
}

struct mtmd_helper_video {
    const mtmd_context * mctx;
    std::string path;
    std::vector<uint8_t> input_buf; // non-empty when initialized from buffer
    std::string ffmpeg_bin;
    std::string ffprobe_bin;
    float fps_target = 0.0f;
    mtmd_helper_video_info info = {};
    std::string id; // hash of the input video

    // RAII wrapper for managing subprocess
    struct subprocess_handle {
        struct subprocess_s proc = {};
        bool created = false; // process exists and must be cleaned up
        bool alive   = false; // process can still give us data
        std::thread feeder;

        subprocess_handle() = default;
        subprocess_handle(const subprocess_handle &) = delete;
        subprocess_handle & operator=(const subprocess_handle &) = delete;
        ~subprocess_handle() { stop(); }

        void stop() {
            // note: alive becomes false on stdout EOF, but the process still needs cleanup
            if (!created) {
                return;
            }
            subprocess_terminate(&proc);
#ifdef _WIN32
            // no SIGPIPE on windows: a blocked feeder only gets a broken pipe once we close our read end of the child stdin
            if (proc.hStdInput) {
                CloseHandle(proc.hStdInput);
                proc.hStdInput = nullptr;
            }
#endif
            // join before destroy: feeder holds a FILE* from subprocess_stdin;
            // subprocess_destroy closes it, so the thread must finish first
            if (feeder.joinable()) {
                feeder.join();
            }
            subprocess_join(&proc, nullptr); // reap the child, or else it stays a zombie
            subprocess_destroy(&proc);
            created = false;
            alive   = false;
        }

        FILE * stdout_pipe() {
            return subprocess_stdout(&proc);
        }

        // buf is tied to lifetime of mtmd_helper_video, so it's guaranteed to outlive the feeder thread
        void start_feeder(const std::vector<uint8_t> & buf) {
            feeder = std::thread([this, &buf]() {
#ifndef _WIN32
                // ffmpeg can exit before it reads all the input, for example when ffprobe already got the metadata.
                // the write below must then fail with EPIPE, instead of killing the process with SIGPIPE
                sigset_t sigpipe_set;
                sigemptyset(&sigpipe_set);
                sigaddset(&sigpipe_set, SIGPIPE);
                pthread_sigmask(SIG_BLOCK, &sigpipe_set, nullptr); // linux sends the signal to the writing thread
#endif
                FILE * f = subprocess_stdin(&proc);
                if (!f) {
                    return;
                }
#ifdef F_SETNOSIGPIPE
                fcntl(fileno(f), F_SETNOSIGPIPE, 1); // macos/bsd send it to the process, so turn it off per fd
#endif
                fwrite(buf.data(), 1, buf.size(), f);
                fclose(f);
                proc.stdin_file = nullptr; // prevent double-close in subprocess_destroy
            });
        }
    };

    subprocess_handle sp;
    int32_t current_frame = 0;

    std::string prompt_start         = "Video:";
    int32_t     timestamp_interval_ms = 5000; // emit a timestamp text every N ms (0 = disabled)
    float       next_timestamp_ms     = 0.0f; // next elapsed-ms threshold at which to emit

    // apply mtmd_get_video_config(); false for still images decoded via ffmpeg
    bool                use_video_config = false;
    mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT;

    int32_t     target_frames = 0;   // exact frame count to emit (0 = no limit)
    int32_t     out_width     = 0;   // frame size from mtmd_get_video_config (0 = source size)
    int32_t     out_height    = 0;

    // source frames to emit, as picked by the reference processors (empty = use the fps filter)
    std::vector<int32_t> frame_indices;
    size_t      next_index = 0;
    int32_t     src_frame  = 0;      // source frames read from ffmpeg so far
    double      src_fps    = 0.0;

    std::vector<uint8_t> frame_buf;
    std::string   pending_text;          // text queued to be returned before the next frame
    mtmd_bitmap * pending_frame = nullptr; // frame queued to be returned after its group timestamp

    // soundtrack: after each group of frames comes the audio from its first frame to the next group's
    double             video_start_s = 0.0;
    int32_t            n_src_frames  = 0;
    std::vector<float> soundtrack;
    mtmd_bitmap *      pending_audio = nullptr;
    bool          start_emitted = false;

    bool is_buf_input() const {
        return !input_buf.empty();
    }

    // number of frames per timestamp group (0 = timestamp every timestamp_interval_ms)
    int32_t ts_per_n_frames() const {
        return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI ? 2 : 0;
    }

    bool probe(float fps_target_arg) {
        // cache: makes stdin seekable; without it ffprobe reports width/height 0
        // for mp4 files whose moov atom sits at the end
        const char * input_arg = is_buf_input() ? "cache:pipe:0" : path.c_str();
        std::vector<const char *> cmd = {
            ffprobe_bin.c_str(),
            "-v", "quiet",
            "-show_entries", "stream=width,height,r_frame_rate,nb_frames,duration,start_time",
            "-select_streams", "v:0",
            "-of", "default=noprint_wrappers=1",
        };
        if (is_buf_input()) {
            // remove the 64KB read-ahead limit of cache:, or else ffprobe cannot reach a moov atom at end of file
            cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"});
        }
        cmd.insert(cmd.end(), {input_arg, nullptr});

        LOG_DBG("%s: launching:", __func__);
        for (size_t i = 0; cmd[i]; i++) { LOG_DBG(" %s", cmd[i]); }
        LOG_DBG("\n");

        subprocess_handle probe_sp;
        if (subprocess_create(cmd.data(),
                subprocess_option_search_user_path | subprocess_option_inherit_environment,
                &probe_sp.proc) != 0) {
            LOG_ERR("%s: failed to launch ffprobe\n", __func__);
            return false;
        }
        probe_sp.created = true;
        probe_sp.alive   = true;

        if (is_buf_input()) {
            probe_sp.start_feeder(input_buf);
        }

        uint32_t width  = 0;
        uint32_t height = 0;
        float orig_fps = 0.0f;
        float duration = -1.0f;
        int32_t n_frames_orig = -1;
        char line[256];
        FILE * fp = probe_sp.stdout_pipe();

        while (fgets(line, sizeof(line), fp)) {
            char * eq = strchr(line, '=');
            if (!eq) continue;
            *eq = '\0';
            const char * key = line;
            const char * val = eq + 1;
            char * nl = (char *)strchr(val, '\n');
            if (nl) *nl = '\0';

            if (strcmp(key, "width") == 0) {
                width = (uint32_t)atoi(val);
            } else if (strcmp(key, "height") == 0) {
                height = (uint32_t)atoi(val);
            } else if (strcmp(key, "r_frame_rate") == 0) {
                orig_fps = parse_rational(val);
            } else if (strcmp(key, "nb_frames") == 0 && strcmp(val, "N/A") != 0) {
                n_frames_orig = atoi(val);
            } else if (strcmp(key, "duration") == 0 && strcmp(val, "N/A") != 0) {
                duration = (float)atof(val);
            } else if (strcmp(key, "start_time") == 0 && strcmp(val, "N/A") != 0) {
                video_start_s = atof(val);
            }
        }

        probe_sp.stop();

        if (width == 0 || height == 0 || orig_fps <= 0.0f) {
            return false;
        }

        if (duration < 0.0f && n_frames_orig > 0) {
            duration = (float)n_frames_orig / orig_fps;
        }

        mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0};
        if (use_video_config) {
            cfg  = mtmd_get_video_config(mctx, width, height, n_frames_orig, orig_fps);
            tmpl = cfg.tmpl;
        }

        // the model's recommended sampling rate wins over the generic default
        fps_target = cfg.fps > 0.0f ? cfg.fps
                   : fps_target_arg > 0.0f ? fps_target_arg : orig_fps;

        if (cfg.n_frames > 0 && duration > 0.0f) {
            const int32_t n = cfg.n_frames;
            target_frames = n;
            fps_target    = (float) n / duration;

            // same frames as the reference processors: round(linspace(0, n_frames - 1, n))
            const double step = n > 1 ? (double) (n_frames_orig - 1) / (n - 1) : 0.0;
            frame_indices.resize(n);
            for (int32_t i = 0; i < n; i++) {
                frame_indices[i] = (int32_t) std::nearbyint(i * step);
            }
            src_fps      = orig_fps;
            n_src_frames = n_frames_orig;

            if (cfg.frame_width > 0 && cfg.frame_height > 0 &&
                    (cfg.frame_width != (int32_t) width || cfg.frame_height != (int32_t) height)) {
                out_width  = cfg.frame_width;
                out_height = cfg.frame_height;
            }
        }

        if (out_width > 0 && out_height > 0) {
            width  = (uint32_t) out_width;
            height = (uint32_t) out_height;
        }

        info.width    = width;
        info.height   = height;
        info.fps      = fps_target;
        LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d target_frames=%d\n",
                __func__, width, height, fps_target, duration, n_frames_orig, target_frames);
        info.n_frames = target_frames > 0 ? target_frames
                      : duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1;
        frame_buf.resize((size_t)width * height * 3);
        return true;
    }

    // the soundtrack over the span of the video stream: from its first frame for n_frames / fps seconds, mono at the
    // rate of the audio encoder; empty if the file has no audio
    void load_soundtrack() {
        const int rate = mtmd_get_audio_sample_rate(mctx);
        if (tmpl != MTMD_VIDEO_TEMPLATE_LFM3_OMNI || frame_indices.empty() || rate <= 0) {
            return;
        }
        const std::string rate_str = std::to_string(rate);
        std::vector<const char *> cmd = {ffmpeg_bin.c_str(), "-nostdin"};
        cmd.reserve(20); // GCC 11 -Wstringop-overflow false positive on the inserts below otherwise
        if (is_buf_input()) {
            cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"});
        }
        cmd.insert(cmd.end(), {"-i", is_buf_input() ? "cache:pipe:0" : path.c_str(), "-map", "0:a:0", "-vn", "-ac", "1",
                               "-ar", rate_str.c_str(), "-f", "f32le", "pipe:1", "-loglevel", "error", nullptr});
        subprocess_handle audio_sp;
        if (subprocess_create(cmd.data(), subprocess_option_search_user_path | subprocess_option_inherit_environment,
                              &audio_sp.proc) != 0) {
            LOG_WRN("%s: failed to launch ffmpeg for the soundtrack\n", __func__);
            return;
        }
        audio_sp.created = true;
        audio_sp.alive   = true;
        if (is_buf_input()) {
            audio_sp.start_feeder(input_buf);
        }
        std::vector<float> pcm;
        float block[4096];
        size_t n;
        while ((n = fread(block, sizeof(float), 4096, audio_sp.stdout_pipe())) > 0) {
            pcm.insert(pcm.end(), block, block + n);
        }
        audio_sp.stop();

        const size_t start = (size_t) std::max(0.0, std::nearbyint(video_start_s * rate));
        const size_t len   = (size_t) std::nearbyint((double) n_src_frames / src_fps * rate);
        if (start >= pcm.size()) {
            LOG_WRN("%s: the video has no soundtrack\n", __func__);
            return;
        }
        soundtrack.assign(pcm.begin() + start, pcm.begin() + std::min(pcm.size(), start + len));
    }

    // the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended
    mtmd_bitmap * soundtrack_slice(int32_t idx) const {
        const int32_t ts_per_n_frames = this->ts_per_n_frames();
        const int32_t group    = idx / ts_per_n_frames;
        const int32_t n_groups = ((int32_t) frame_indices.size() + ts_per_n_frames - 1) / ts_per_n_frames;
        const int     rate     = mtmd_get_audio_sample_rate(mctx);
        auto group_start = [&](int32_t g) {
            const double t = frame_indices[g * ts_per_n_frames] / src_fps;
            return std::min(soundtrack.size(), (size_t) std::nearbyint(t * rate));
        };
        const size_t begin = group == 0 ? 0 : group_start(group);
        const size_t end   = group == n_groups - 1 ? soundtrack.size() : group_start(group + 1);
        if (begin >= end) {
            return nullptr;
        }
        mtmd_bitmap * slice = mtmd_bitmap_init_from_audio(end - begin, soundtrack.data() + begin);
        if (!id.empty()) {
            mtmd_bitmap_set_id(slice, (id + "+audio" + std::to_string(group)).c_str());
        }
        return slice;
    }

    bool start_ffmpeg(float seek_seconds) {
        char seek_buf[64];
        char fps_buf[64];

        std::vector<const char *> cmd;
        cmd.push_back(ffmpeg_bin.c_str());

        if (!is_buf_input() && seek_seconds > 0.0f) {
            // input-side seek: fast, keyframe-accurate; only valid for seekable file inputs
            snprintf(seek_buf, sizeof(seek_buf), "%.6f", seek_seconds);
            cmd.push_back("-ss");
            cmd.push_back(seek_buf);
        }

        cmd.push_back("-nostdin");
        if (is_buf_input()) {
            // remove the 64KB read-ahead limit of cache:, or else ffmpeg cannot reach a moov atom at end of file
            cmd.push_back("-read_ahead_limit");
            cmd.push_back("-1");
        }
        cmd.push_back("-i");
        // cache:pipe:0 wraps stdin with a seekable in-memory cache, letting ffmpeg seek
        // backwards for container headers (e.g. MP4 moov atom at end of file)
        cmd.push_back(is_buf_input() ? "cache:pipe:0" : path.c_str());

        if (seek_seconds > 0.0f && is_buf_input()) {
            // output-side seek: frame-accurate but decodes and discards frames up to seek point
            snprintf(seek_buf, sizeof(seek_buf), "%.6f", seek_seconds);
            cmd.push_back("-ss");
            cmd.push_back(seek_buf);
        }

        std::string vf;
        if (!frame_indices.empty()) {
            // frames are picked by index in read_next_frame(); B=0 C=0.5 bicubic on RGB is closest to Pillow
            if (out_width > 0 && out_height > 0) {
                char scale_buf[160];
                snprintf(scale_buf, sizeof(scale_buf),
                        "format=rgb24,scale=%d:%d:flags=bicubic+accurate_rnd+full_chroma_int:param0=0:param1=0.5",
                        out_width, out_height);
                vf = scale_buf;
            }
        } else if (fps_target > 0.0f) {
            if (out_width > 0 && out_height > 0) {
                snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f,scale=%d:%d", fps_target, out_width, out_height);
            } else {
                snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f", fps_target);
            }
            vf = fps_buf;
        }
        if (!vf.empty()) {
            cmd.push_back("-vf");
            cmd.push_back(vf.c_str());
        }

        char frames_buf[16];
        const int32_t n_out = !frame_indices.empty() ? frame_indices.back() + 1 : target_frames;
        if (n_out > 0) {
            snprintf(frames_buf, sizeof(frames_buf), "%d", n_out);
            cmd.push_back("-frames:v");
            cmd.push_back(frames_buf);
        }

        cmd.push_back("-f");
        cmd.push_back("rawvideo");
        cmd.push_back("-pix_fmt");
        cmd.push_back("rgb24");
        cmd.push_back("pipe:1");
        cmd.push_back("-loglevel");
        cmd.push_back("error");
        cmd.push_back(nullptr);

        LOG_DBG("%s: launching:", __func__);
        for (size_t i = 0; cmd[i]; i++) {
            LOG_DBG(" %s", cmd[i]);
        }
        LOG_DBG("\n");

        int ret = subprocess_create(
            cmd.data(),
            subprocess_option_search_user_path | subprocess_option_inherit_environment,
            &sp.proc);

        sp.created = (ret == 0);
        sp.alive   = (ret == 0);
        LOG_DBG("%s: subprocess_create ret=%d proc_alive=%d\n", __func__, ret, (int)sp.alive);

        if (sp.alive && is_buf_input()) {
            LOG_DBG("%s: starting feeder thread for %zu-byte buffer\n", __func__, input_buf.size());
            sp.start_feeder(input_buf);
        }

        return sp.alive;
    }

    void stop_ffmpeg() {
        sp.stop();
    }

    // read one raw frame from ffmpeg into frame_buf
    bool read_raw_frame() {
        FILE * fp = sp.stdout_pipe();
        const size_t frame_size = (size_t)info.width * info.height * 3;
        LOG_DBG("%s: reading frame %d, expecting %zu bytes (%ux%u)\n",
                __func__, current_frame, frame_size, info.width, info.height);

        size_t total_read = 0;
        while (total_read < frame_size) {
            size_t n = fread(frame_buf.data() + total_read, 1, frame_size - total_read, fp);
            if (n == 0) {
                // clean EOF only if no bytes read yet; partial frame is an error
                LOG_DBG("%s: fread returned 0 after %zu/%zu bytes (ferror=%d)\n",
                        __func__, total_read, frame_size, ferror(fp));
                sp.alive = false;
                return false;
            }
            total_read += n;
        }
        return true;
    }

    mtmd_bitmap * read_next_frame() {
        if (!sp.alive) return nullptr;

        if (!frame_indices.empty()) {
            if (next_index >= frame_indices.size()) {
                return nullptr;
            }
            // skip to the wanted source frame; a repeated index reuses the frame in frame_buf
            const int32_t want = frame_indices[next_index];
            while (src_frame <= want) {
                if (!read_raw_frame()) {
                    return nullptr;
                }
                src_frame++;
            }
            next_index++;
        } else if (!read_raw_frame()) {
            return nullptr;
        }

        LOG_DBG("%s: frame %d read OK\n", __func__, current_frame);
        mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data());
        mtmd_bitmap_set_mergeable(frame, true);
        mtmd_bitmap_set_is_video(frame, true);
        if (!id.empty()) {
            // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache
            std::string frame_id = id + "+" + std::to_string(current_frame);
            mtmd_bitmap_set_id(frame, frame_id.c_str());
        }
        current_frame++;
        return frame;
    }

    int32_t read_next(mtmd_bitmap ** out_bitmap, char ** out_text) {
        *out_bitmap = nullptr;
        *out_text   = nullptr;

        if (!pending_text.empty()) {
            *out_text = mtmd_strdup(pending_text);
            pending_text.clear();
            return *out_text ? 0 : -2;
        }

        if (pending_frame) {
            *out_bitmap  = pending_frame;
            pending_frame = nullptr;
            return 0;
        }

        if (pending_audio) {
            *out_bitmap   = pending_audio;
            pending_audio = nullptr;
            return 0;
        }

        LOG_DBG("%s: proc_alive=%d start_emitted=%d current_frame=%d\n",
                __func__, (int)sp.alive, (int)start_emitted, current_frame);

        if (!sp.alive) {
            return (current_frame == 0) ? -2 : -1;
        }

        if (!start_emitted) {
            start_emitted = true;
            // model templates have no prompt prefix, the video begin/end tokens are added by mtmd_tokenize
            if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT && !prompt_start.empty()) {
                *out_text = mtmd_strdup(prompt_start);
                return *out_text ? 0 : -2;
            }
        }

        mtmd_bitmap * frame = read_next_frame();
        if (!frame) {
            return -1;
        }

        const int32_t ts_per_n_frames = this->ts_per_n_frames();
        if (ts_per_n_frames > 0) {
            // emit the group timestamp before the group's first frame; the value is the
            // midpoint time of the group, matching HF processors that stamp each temporal group
            const int32_t idx = current_frame - 1; // read_next_frame() already incremented
            // the soundtrack of a group follows its last frame
            const bool group_ends = idx % ts_per_n_frames == ts_per_n_frames - 1 || idx + 1 == (int32_t) frame_indices.size();
            if (!soundtrack.empty() && !frame_indices.empty() && group_ends) {
                pending_audio = soundtrack_slice(idx);
            }
            if (idx % ts_per_n_frames == 0) {
                char ts_buf[64];
                double ts_s = ((double)idx + 0.5*(double)(ts_per_n_frames - 1)) / info.fps;
                if (!frame_indices.empty()) {
                    // stamp from the source frames; a short last group repeats its last frame
                    const size_t first = idx;
                    const size_t last  = std::min(first + ts_per_n_frames - 1, frame_indices.size() - 1);
                    ts_s = (frame_indices[first] / src_fps + frame_indices[last] / src_fps) / 2.0;
                }
                snprintf(ts_buf, sizeof(ts_buf), "<%.1f seconds>", ts_s);
                pending_frame = frame;
                *out_text = mtmd_strdup(ts_buf);
                return *out_text ? 0 : -2;
            }
            *out_bitmap = frame;
            return 0;
        }

        *out_bitmap = frame;

        if (timestamp_interval_ms > 0) {
            // current_frame was already incremented by read_next_frame(); undo for elapsed calc
            float elapsed_ms = (float)(current_frame - 1) / info.fps * 1000.0f;
            if (elapsed_ms >= next_timestamp_ms) {
                char ts_buf[32];
                float elapsed_s = elapsed_ms / 1000.0f;
                int   minutes   = (int)(elapsed_s / 60);
                float seconds   = elapsed_s - minutes * 60.0f;
                snprintf(ts_buf, sizeof(ts_buf), "[%dm%.2fs]", minutes, seconds);
                pending_text = ts_buf;
                next_timestamp_ms += (float)timestamp_interval_ms;
            }
        }

        return 0;
    }

    static float parse_rational(const char * s) {
        int num = 0, den = 1;
        if (sscanf(s, "%d/%d", &num, &den) == 2 && den > 0) {
            return (float)num / (float)den;
        }
        float val;
        if (sscanf(s, "%f", &val) == 1) {
            return val;
        }
        return 0.0f;
    }
};
#endif

mtmd_helper_video_init_params mtmd_helper_video_init_params_default() {
    return {
        /* fps_target             */ 4.0f,
        /* ffmpeg_bin_dir         */ nullptr,
        /* timestamp_interval_ms  */ 5000,
        /* no_soundtrack          */ false,
    };
}

mtmd_helper_init_opt mtmd_helper_init_opt_default() {
    return {
        /* video_params */ mtmd_helper_video_init_params_default(),
    };
}

static std::string video_resolve_bin(const char * bin_dir, const char * name) {
    if (!bin_dir || bin_dir[0] == '\0') {
        return name; // rely on PATH
    }
    std::string result = bin_dir;
    char last = result.back();
    if (last != '/' && last != '\\') {
#ifdef _WIN32
        result += '\\';
#else
        result += '/';
#endif
    }
    result += name;
#ifdef _WIN32
    result += ".exe";
#endif
    return result;
}

#ifdef MTMD_VIDEO
static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string & id) {
    vctx->id = id;
}

static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
                                             const mtmd_helper_video_init_params & params) {
    mtmd_helper_video vctx;
    vctx.mctx        = mctx;
    vctx.input_buf.assign(buf, buf + len);
    vctx.ffmpeg_bin  = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg");
    vctx.ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe");
    if (!vctx.probe(0.0f)) {
        return nullptr;
    }
    if (placeholder) {
        return mtmd_bitmap_init(vctx.info.width, vctx.info.height, nullptr);
    }
    // still image: the fps filter would output no frame, so disable it
    vctx.fps_target = 0.0f;
    if (!vctx.start_ffmpeg(0.0f)) {
        return nullptr;
    }
    mtmd_bitmap * frame = vctx.read_next_frame();
    if (frame) {
        mtmd_bitmap_set_mergeable(frame, false);
        mtmd_bitmap_set_is_video(frame, false);
    }
    return frame;
}
#endif

mtmd_helper_video * mtmd_helper_video_init(
        const mtmd_context * mctx,
        const char * path,
        mtmd_helper_video_init_params params) {
#ifdef MTMD_VIDEO
    auto * ctx = new mtmd_helper_video();

    ctx->mctx                 = mctx;
    ctx->path                 = path;
    ctx->ffmpeg_bin           = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg");
    ctx->ffprobe_bin          = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe");
    ctx->timestamp_interval_ms = params.timestamp_interval_ms;
    ctx->use_video_config      = true;

    if (!ctx->probe(params.fps_target)) {
        LOG_ERR("%s: ffprobe failed for '%s' (is ffprobe in PATH?)\n", __func__, path);
        delete ctx;
        return nullptr;
    }
    if (!params.no_soundtrack) {
        ctx->load_soundtrack();
    }

    if (!ctx->start_ffmpeg(0.0f)) {
        LOG_ERR("%s: failed to start ffmpeg for '%s' (is ffmpeg in PATH?)\n", __func__, path);
        delete ctx;
        return nullptr;
    }

    return ctx;
#else
    GGML_UNUSED(mctx);
    GGML_UNUSED(path);
    GGML_UNUSED(params);
    LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__);
    return nullptr;
#endif
}

mtmd_helper_video * mtmd_helper_video_init_from_buf(
        const mtmd_context * mctx,
        const unsigned char * buf, size_t len,
        mtmd_helper_video_init_params params) {
#ifdef MTMD_VIDEO
    auto * ctx = new mtmd_helper_video();

    ctx->mctx                  = mctx;
    ctx->input_buf.assign(buf, buf + len);
    ctx->ffmpeg_bin            = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg");
    ctx->ffprobe_bin           = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe");
    ctx->timestamp_interval_ms = params.timestamp_interval_ms;
    ctx->use_video_config      = true;

    if (!ctx->probe(params.fps_target)) {
        LOG_ERR("%s: ffprobe failed on buffer (is ffprobe in PATH?)\n", __func__);
        delete ctx;
        return nullptr;
    }
    if (!params.no_soundtrack) {
        ctx->load_soundtrack();
    }

    if (!ctx->start_ffmpeg(0.0f)) {
        LOG_ERR("%s: failed to start ffmpeg on buffer (is ffmpeg in PATH?)\n", __func__);
        delete ctx;
        return nullptr;
    }

    return ctx;
#else
    GGML_UNUSED(mctx);
    GGML_UNUSED(buf);
    GGML_UNUSED(len);
    GGML_UNUSED(params);
    LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__);
    return nullptr;
#endif
}

void mtmd_helper_video_free(mtmd_helper_video * ctx) {
#ifdef MTMD_VIDEO
    if (!ctx) return;
    ctx->stop_ffmpeg();
    mtmd_bitmap_free(ctx->pending_frame);
    mtmd_bitmap_free(ctx->pending_audio);
    delete ctx;
#else
    GGML_UNUSED(ctx);
    LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__);
#endif
}

mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx) {
#ifdef MTMD_VIDEO
    return ctx->info;
#else
    GGML_UNUSED(ctx);
    GGML_ASSERT(false && "video is not supported in this build (MTMD_VIDEO is set to OFF)");
#endif
}

int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
        mtmd_bitmap ** out_bitmap, char ** out_text) {
#ifdef MTMD_VIDEO
    if (!ctx) return -2;
    return ctx->read_next(out_bitmap, out_text);
#else
    GGML_UNUSED(ctx);
    GGML_UNUSED(out_bitmap);
    GGML_UNUSED(out_text);
    GGML_ASSERT(false && "video is not supported in this build (MTMD_VIDEO is set to OFF)");
#endif
}

mtmd_helper_cap mtmd_helper_get_capabilities(const llama_model * model, const mtmd_context * mctx) {
    mtmd_helper_cap cap = {};
    cap.chat     = true;
    cap.text_gen = true;
    if (!mctx) {
        return cap;
    }

    cap.vision = mtmd_support_vision(mctx);
    cap.audio  = mtmd_support_audio(mctx);
    cap.audio_stream = mtmd_support_audio_stream(mctx);
#ifdef MTMD_VIDEO
    cap.video  = cap.vision;
#endif
    const auto gen_audio_type = mtmd_gen_audio_get_info(mctx).type;
    cap.audio_gen = gen_audio_type != MTMD_GEN_AUDIO_TYPE_NONE;
    cap.audio_gen_spk_ref = gen_audio_type == MTMD_GEN_AUDIO_TYPE_QWEN3TTS
                         || gen_audio_type == MTMD_GEN_AUDIO_TYPE_POCKETTTS;

    // a tts model without a chat template only outputs audio, it cannot be used for chat
    // neither can a realtime transcription model (e.g. Voxtral Realtime): its text follows the audio by a token delay
    cap.text_gen = !cap.audio_gen || llama_model_chat_template(model, nullptr) != nullptr;
    cap.chat     = cap.text_gen && llama_model_n_tokens_delay(model) == 0;

    return cap;
}

bool mtmd_helper_model_can_chat(const llama_context * lctx, const mtmd_context * mctx) {
    return mtmd_helper_get_capabilities(llama_get_model(lctx), mctx).chat;
}

bool mtmd_helper_support_video(const mtmd_context * ctx) {
#ifdef MTMD_VIDEO
    return mtmd_support_vision(ctx);
#else
    GGML_UNUSED(ctx);
    return false;
#endif
}