rust_i18n_autotranslate/lib.rs
1#![deny(missing_docs)]
2#![deny(missing_debug_implementations)]
3#![cfg_attr(docsrs, feature(doc_cfg))]
4#![cfg_attr(not(test), warn(unused_crate_dependencies))]
5#![cfg_attr(test, deny(warnings))]
6
7//! # rust-i18n-autotranslate
8//!
9//! The `rust-i18n-autotranslate` crate provides a simple function to autogenerate locales at runtime or buildtime
10//!
11//! This is meant to be a helper crate to [rust-i18n](<https://docs.rs/rust-i18n/latest/rust_i18n/>)
12//!
13//!## Features
14//! - Tracks the source language file and only translates when it has changed.
15//! - Set `cache = true` to reuse already translated words.
16//! - Normalizes languages to a supported language if supported.
17//!
18//! The crate supports creating translations only for version_1 type locales
19//! eg:
20//!
21//! ```text
22//! ├── Cargo.lock
23//! ├── Cargo.toml
24//! ├── locales
25//! │ ├── zh-CN.yml
26//! │ ├── en.yml
27//! └── src
28//! │ └── main.rs
29//! ```
30//!
31//!
32//!
33//! # Current support
34//! - Google Translate (Cloud Translate - Fallback to google translate web)
35//! - DeepL (Cloud Translate - Fallback to deeplx)
36//! - DeepLX (Needs installation [Install DeepLX](<https://deeplx.owo.network/install/>))
37//! - LibreTranslate (Fallback - [Install Self Hosted](<https://docs.libretranslate.com/#self-hosted>)))
38//! - Yandex (Planned)
39//! - aws ML (Planned)
40//!
41//!
42//! # Usage
43//!
44//! The crate uses env variables to set the api keys.
45//!
46//! Create a `.env` file in the root of your project and add the following key.
47//!
48//! The crate uses env variables to set the api key:
49//!
50//!- **GOOGLE_API_KEY = "xyz"** [How to generate google api key](<https://translatepress.com/docs/automatic-translation/generate-google-api-key/>)
51//!- **DEEPL_FREE_API_KEY = "xyz"**
52//!- **DEEPL_PRO_API_KEY = "xyz"**
53//!- **LIBRE_TRANSLATE_API_KEY = "xyz"**
54//!
55//!
56//! ## Language codes need to be in [ISO-639](<https://wikipedia.org/wiki/ISO_639>) format
57//!
58//! Call the translate function directly to translate your locales
59//!
60//! ```rust,no_run
61//!use rust_i18n_autotranslate::{
62//! TranslationAPI,
63//! config::{Config, TranslationProvider},
64//!};
65//!
66//!fn main() {
67//! env_logger::init();
68//!
69//! let cfg = Config::new()
70//! .locales_directory("./locales")
71//! .source_lang("en")
72//! .add_target_lang("fr")
73//! .use_cache(true)
74//! .translation_provider(TranslationProvider::GOOGLE)
75//! .build();
76//!
77//! TranslationAPI::translate(cfg).unwrap()
78//!}
79//! ```
80//!
81//!
82
83use log::{error, info};
84use rust_i18n_support::load_locales;
85
86use std::collections::BTreeMap;
87
88use crate::{
89 api::translate_data,
90 config::Config,
91 i18n::autogen_cache::Autogen,
92 utils::{match_sha256, verify_locales, write_locale_file},
93};
94
95mod api;
96pub mod config;
97mod i18n;
98mod utils;
99
100//TODO:: Setup errors correctly
101
102/// The translation api
103#[derive(Debug, Clone, PartialEq, Eq, Default)]
104pub struct TranslationAPI {}
105
106impl TranslationAPI {
107 /// Translate the source locale into multiple locales.
108 ///
109 /// Default output is json
110 ///
111 /// Choose a translation api.
112 ///
113 /// To use the paid api set any of the environment variable and select the appropriate provider
114 ///
115 /// _Environment Variables:_
116 ///- GOOGLE_API_KEY="xxx"
117 ///- DEEPL_FREE_API_KEY="xxx"
118 ///- DEEPL_PRO_API_KEY="xxx"
119 ///- LIBRE_TRANSLATE_API_KEY="xxx"
120 ///`If both deepl api keys are set, priority is given to the free key`
121 ///
122 /// Cache: Use cache to save and reuse translations.
123 ///
124
125 /// Example:
126 /// ```rust,no_run
127 ///use rust_i18n_autotranslate::{
128 /// TranslationAPI,
129 /// config::{Config, TranslationProvider},
130 ///};
131 ///
132 ///fn main() {
133 /// env_logger::init();
134 ///
135 /// let cfg = Config::new()
136 /// .locales_directory("./locales")
137 /// .source_lang("en")
138 /// .add_target_lang("fr")
139 /// .use_cache(true)
140 /// .translation_provider(TranslationProvider::GOOGLE)
141 /// .build();
142 ///
143 /// TranslationAPI::translate(cfg).unwrap()
144 ///}
145 /// ```
146 /// ## Language codes need to be in [ISO-639](<https://wikipedia.org/wiki/ISO_639>) format
147 pub fn translate(config: Config) -> Result<(), String> {
148 //verify that the sha256 checksums are different then only proceed
149 let locale_path = config.locales_dir.clone();
150
151 let verify_locales = verify_locales(
152 locale_path.as_path(),
153 &config.source_locale,
154 &config.target_locales,
155 );
156
157 let mut autogen = Autogen::load();
158
159 if config.target_locales.is_empty() {
160 info!("Already on latest");
161 autogen.data.clear();
162 let _ = autogen.update_cache();
163 return Ok(());
164 }
165
166 let checksum_res = match_sha256(
167 locale_path.as_path(),
168 &config.source_locale,
169 &autogen.checksum.unwrap_or_default(),
170 );
171
172 if checksum_res.is_some() || verify_locales.is_err() {
173 //update the sha2
174 autogen.checksum = checksum_res;
175
176 //Preload google api key from env
177 dotenvy::dotenv().ok();
178
179 let mut locales_data =
180 load_locales(config.locales_dir.to_str().unwrap_or_default(), |_| false);
181
182 let source_locale_data = locales_data.get_mut(&config.source_locale);
183
184 //use the source locale data
185 if let Some(source_data) = source_locale_data {
186 source_data.remove("_version");
187
188 if config.use_cache {
189 //use autogen cache
190 for target_locale in config.target_locales {
191 let autogen_data = autogen
192 .data
193 .get(&target_locale)
194 .cloned()
195 .unwrap_or_default();
196
197 let mut to_translate_keys = Vec::with_capacity(source_data.len());
198 let mut to_translate_values = Vec::with_capacity(source_data.len());
199 let mut og_keys = Vec::with_capacity(source_data.len());
200
201 for (key, value) in source_data.iter() {
202 //TODO: Find a more performant solution to clones and duplications
203 //maintain a seperate copy iter later
204 og_keys.push(key.as_str());
205 //if it doesnt exist in the autogen cache then send for translate
206 if autogen_data.get(value).is_none() {
207 to_translate_keys.push(key.as_str());
208 to_translate_values.push(value.as_str());
209 }
210 }
211
212 let translated_values = translate_data(
213 &config.provider,
214 &to_translate_values,
215 &config.source_locale,
216 &target_locale,
217 )?;
218
219 //get the already present data
220 let mut autogen_locale = autogen
221 .data
222 .get(&target_locale)
223 .cloned()
224 .unwrap_or_default();
225
226 //combine the translated values
227 let mut translated_kv = BTreeMap::new();
228
229 if translated_values.len() == to_translate_keys.len() {
230 if translated_values.len() > 0 && to_translate_keys.len() > 0 {
231 //Updating the autogen values
232 for (index, value) in to_translate_values.iter().enumerate() {
233 autogen_locale.insert(
234 value.to_string(),
235 translated_values[index].clone(),
236 );
237 }
238 //update the autogen value
239 autogen
240 .data
241 .insert(target_locale.to_string(), autogen_locale.clone());
242
243 for (og_key, og_value) in source_data.iter() {
244 //if contains then it was sent for translation else use cached value
245 if let Some(pos) =
246 to_translate_keys.iter().position(|x| x == &og_key)
247 {
248 //translated value
249 // use the pos to get value from translated value
250 let translated_value = translated_values.get(pos);
251 if let Some(value) = translated_value {
252 translated_kv
253 .insert(og_key.to_string(), value.to_string());
254 } else {
255 translated_kv
256 .insert(og_key.to_string(), og_value.to_string());
257 }
258 } else {
259 //cached value
260 let res = autogen_locale.get(og_value);
261 if let Some(auto_data) = res {
262 translated_kv
263 .insert(og_key.to_string(), auto_data.to_string());
264 } else {
265 //default = not found = insert source value
266 translated_kv
267 .insert(og_key.to_string(), og_value.to_string());
268 }
269 }
270 }
271 } else {
272 //cached value
273 for (og_key, og_value) in source_data.iter() {
274 let res = autogen_locale.get(og_value);
275 if let Some(auto_data) = res {
276 translated_kv
277 .insert(og_key.to_string(), auto_data.to_string());
278 } else {
279 //default = not found = insert source value
280 translated_kv
281 .insert(og_key.to_string(), og_value.to_string());
282 }
283 }
284 }
285
286 //write the locale file
287 let write_res = write_locale_file(
288 &locale_path,
289 &translated_kv,
290 &config.source_locale,
291 &target_locale,
292 );
293
294 if let Err(e) = write_res {
295 error!("{e}");
296 }
297 } else {
298 //some translations may have failed, so discard the whole translation
299 continue;
300 }
301 }
302 } else {
303 //no use autogen
304 let mut keys = Vec::with_capacity(source_data.len());
305 let mut values = Vec::with_capacity(source_data.len());
306 for (key, value) in source_data {
307 keys.push(key.as_str());
308 values.push(value.as_str());
309 }
310
311 for target_locale in config.target_locales {
312 let translated = translate_data(
313 &config.provider,
314 &values,
315 &config.source_locale,
316 &target_locale,
317 )?;
318
319 //combine the translated
320 if translated.len() == keys.len() {
321 //combine the translated values
322 let mut translated_kv = BTreeMap::new();
323 for (index, key) in keys.iter().enumerate() {
324 translated_kv.insert(key.to_string(), translated[index].clone());
325 }
326
327 //write the locale file
328 let write_res = write_locale_file(
329 &locale_path,
330 &translated_kv,
331 &config.source_locale,
332 &target_locale,
333 );
334
335 if let Err(e) = write_res {
336 error!("{e}");
337 }
338 } else {
339 //some translations may have failed, so discard the whole translation
340 continue;
341 }
342 }
343 }
344
345 //update autogen
346 let autogen_update_res = autogen.update_cache();
347 if let Err(err) = autogen_update_res {
348 error!("{}", err);
349 }
350
351 Ok(())
352 } else {
353 Err("Could not find source locale data".to_string())
354 }
355 } else {
356 info!("Already on latest");
357 Ok(())
358 }
359 }
360}