ct2rs 0.9.18

Rust bindings for OpenNMT/CTranslate2
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

#pragma once

#include "replica_pool.h"
#include "models/language_model.h"

namespace ctranslate2 {

  // Generator is the high-level class for running generation with language models.
  // It supports parallel and asynchronous generation.
  class Generator : public ReplicaPool<models::SequenceGeneratorReplica> {
  public:
    using ReplicaPool::ReplicaPool;

    std::vector<std::future<GenerationResult>>
    generate_batch_async(const std::vector<std::vector<std::string>>& start_tokens,
                         const GenerationOptions& options = GenerationOptions(),
                         const size_t max_batch_size = 0,
                         const BatchType batch_type = BatchType::Examples);

    std::vector<std::future<ScoringResult>>
    score_batch_async(const std::vector<std::vector<std::string>>& tokens,
                      const ScoringOptions& options = ScoringOptions(),
                      const size_t max_batch_size = 0,
                      const BatchType batch_type = BatchType::Examples);

    std::future<StorageView>
    forward_batch_async(std::vector<std::vector<std::string>> tokens,
                        const bool return_log_probs);

    std::future<StorageView>
    forward_batch_async(std::vector<std::vector<size_t>> ids,
                        const bool return_log_probs);

    std::future<StorageView>
    forward_batch_async(StorageView ids,
                        StorageView lengths,
                        const bool return_log_probs);
  };

}