ct2rs 0.9.19

Rust bindings for OpenNMT/CTranslate2
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

#include "backend.h"

#include "ctranslate2/utils.h"
#include "cpu_info.h"
#include "env.h"

namespace ctranslate2 {
  namespace cpu {

    static bool mayiuse_mkl_init() {
      const std::string use_mkl_env = read_string_from_env("CT2_USE_MKL");
      if (use_mkl_env.empty()) {
#ifdef CT2_WITH_MKL
        return cpu_is_genuine_intel();
#else
        return false;
#endif
      } else {
        const bool use_mkl = string_to_bool(use_mkl_env);
#ifndef CT2_WITH_MKL
        if (use_mkl)
          throw std::invalid_argument("This CTranslate2 binary was not compiled with Intel MKL");
#endif
        return use_mkl;
      }
    }

    bool mayiuse_mkl() {
      static const bool mayiuse = mayiuse_mkl_init();
      return mayiuse;
    }

    std::string gemm_backend_to_str(GemmBackend gemm_backend) {
      switch (gemm_backend) {
      case GemmBackend::MKL:
        return "MKL";
      case GemmBackend::DNNL:
        return "DNNL";
      case GemmBackend::ACCELERATE:
        return "Accelerate";
      case GemmBackend::OPENBLAS:
        return "OpenBLAS";
      case GemmBackend::RUY:
        return "Ruy";
      default:
        return "none";
      }
    }

    GemmBackend get_gemm_backend(ComputeType compute_type) {
      const bool is_int8 = (compute_type == ComputeType::INT8
                            || compute_type == ComputeType::INT8_FLOAT32
                            || compute_type == ComputeType::INT8_FLOAT16
                            || compute_type == ComputeType::INT8_BFLOAT16);

#ifdef CT2_WITH_MKL
      if (mayiuse_mkl()
          && (compute_type == ComputeType::FLOAT32
              || compute_type == ComputeType::INT16
              || is_int8)) {
        return GemmBackend::MKL;
      }
#endif

#ifdef CT2_WITH_DNNL
      if (compute_type == ComputeType::FLOAT32 || is_int8) {
        return GemmBackend::DNNL;
      }
#endif

#ifdef CT2_WITH_ACCELERATE
      if (compute_type == ComputeType::FLOAT32) {
        return GemmBackend::ACCELERATE;
      }
#endif

#ifdef CT2_WITH_OPENBLAS
      if (compute_type == ComputeType::FLOAT32) {
        return GemmBackend::OPENBLAS;
      }
#endif

#ifdef CT2_WITH_RUY
      if (is_int8 || compute_type == ComputeType::FLOAT32) {
        return GemmBackend::RUY;
      }
#endif

      return GemmBackend::NONE;
    }

    bool has_gemm_backend(ComputeType compute_type) {
      return get_gemm_backend(compute_type) != GemmBackend::NONE;
    }

    bool prefer_u8s8s32_gemm() {
      const auto gemm_s8_backend = get_gemm_backend(ComputeType::INT8);
      return gemm_s8_backend == cpu::GemmBackend::MKL || gemm_s8_backend == cpu::GemmBackend::DNNL;
    }

    bool pack_gemm_weights(ComputeType compute_type) {
      static const bool should_pack = read_bool_from_env("CT2_USE_EXPERIMENTAL_PACKED_GEMM");
      return should_pack && get_gemm_backend(compute_type) == GemmBackend::MKL;
    }

#ifdef CT2_WITH_RUY
    ruy::Context *get_ruy_context() {
      static thread_local ruy::Context context;
      return &context;
    }
#endif
  }
}