hammer/server/
languagetool.rs

1//! LanguageTool on the user's sync server: the `languagetool` service, which forwards to the
2//! LanguageTool server its admin set up. The manifest names it under `permissions.hammerServer`:
3//!
4//! ```toml
5//! [permissions]
6//! hammerServer = ["languagetool"]
7//! ```
8//!
9//! ```ignore
10//! use hammer::server::languagetool::{self, CheckRequest};
11//!
12//! let checked = languagetool::check(&CheckRequest::new(text, "en-US"))?;
13//! for found in &checked.matches {
14//!     if let Some(range) = found.byte_range(text) {
15//!         hammer::log::info(&format!("{}: {}", &text[range], found.message));
16//!     }
17//! }
18//! ```
19
20use super::{ErrorKind, Response, ServerError};
21use serde::Deserialize;
22use serde::de::DeserializeOwned;
23use std::ops::Range;
24
25pub const SERVICE: &str = "languagetool";
26pub const CHECK_PATH: &str = "/v2/check";
27pub const LANGUAGES_PATH: &str = "/v2/languages";
28
29/// The most a check's encoded form may be on a sync server left at its defaults; an admin may allow
30/// more. Encoding can triple text that isn't ASCII, so keep a check's text to about 30,000 bytes.
31pub const DEFAULT_MAX_REQUEST_BYTES: usize = 100_000;
32
33#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
34pub enum Level {
35    #[default]
36    Default,
37    /// More checks of style and typography.
38    Picky,
39}
40
41/// A check of [CheckRequest::text]. Rules and categories are LanguageTool's ids, such as
42/// `EN_A_VS_AN` or `TYPOS`.
43#[derive(Debug, Clone, Default, PartialEq)]
44pub struct CheckRequest {
45    pub text: String,
46    /// A language code such as `en-US`, or `auto` for LanguageTool to detect it.
47    pub language: String,
48    /// The writer's first language, for rules about false friends.
49    pub mother_tongue: Option<String>,
50    /// With `auto`, the variants to prefer, such as `en-GB` and `de-AT`.
51    pub preferred_variants: Vec<String>,
52    pub enabled_rules: Vec<String>,
53    pub disabled_rules: Vec<String>,
54    pub enabled_categories: Vec<String>,
55    pub disabled_categories: Vec<String>,
56    /// Only the enabled rules and categories, none of the rest.
57    pub enabled_only: bool,
58    pub level: Level,
59}
60
61impl CheckRequest {
62    pub fn new(text: impl Into<String>, language: impl Into<String>) -> CheckRequest {
63        CheckRequest { text: text.into(), language: language.into(), ..CheckRequest::default() }
64    }
65
66    /// The form LanguageTool reads, URL-encoded.
67    pub fn form(&self) -> String {
68        let language = match self.language.trim() {
69            "" => "auto",
70            language => language,
71        };
72        let lists = [
73            ("preferredVariants", &self.preferred_variants),
74            ("enabledRules", &self.enabled_rules),
75            ("disabledRules", &self.disabled_rules),
76            ("enabledCategories", &self.enabled_categories),
77            ("disabledCategories", &self.disabled_categories),
78        ]
79        .map(|(name, values)| (name, values.join(",")));
80        let mut fields = vec![("text", self.text.as_str()), ("language", language)];
81        if let Some(mother_tongue) = &self.mother_tongue {
82            fields.push(("motherTongue", mother_tongue));
83        }
84        fields.extend(lists.iter().filter(|(_, value)| !value.is_empty()).map(|(name, value)| (*name, value.as_str())));
85        if self.enabled_only {
86            fields.push(("enabledOnly", "true"));
87        }
88        if self.level == Level::Picky {
89            fields.push(("level", "picky"));
90        }
91        crate::http::form(&fields)
92    }
93}
94
95#[derive(Debug, Clone, Default, Deserialize)]
96#[serde(rename_all = "camelCase")]
97pub struct CheckResponse {
98    #[serde(default)]
99    pub language: Option<CheckedLanguage>,
100    #[serde(default)]
101    pub matches: Vec<Match>,
102    #[serde(default)]
103    pub warnings: Warnings,
104}
105
106#[derive(Debug, Clone, Default, Deserialize)]
107#[serde(rename_all = "camelCase")]
108pub struct Warnings {
109    /// LanguageTool stopped before the end of the text, as when a check takes too long.
110    #[serde(default)]
111    pub incomplete_results: bool,
112}
113
114/// The language the text was checked in, and the one LanguageTool detected in it.
115#[derive(Debug, Clone, Default, Deserialize)]
116#[serde(rename_all = "camelCase")]
117pub struct CheckedLanguage {
118    #[serde(default)]
119    pub name: String,
120    #[serde(default)]
121    pub code: String,
122    #[serde(default)]
123    pub detected_language: Option<DetectedLanguage>,
124}
125
126#[derive(Debug, Clone, Default, Deserialize)]
127pub struct DetectedLanguage {
128    #[serde(default)]
129    pub name: String,
130    #[serde(default)]
131    pub code: String,
132    #[serde(default)]
133    pub confidence: Option<f64>,
134}
135
136/// A problem LanguageTool found. [offset](Match::offset) and [length](Match::length) count UTF-16
137/// units of the text checked, as Java does; [Match::byte_range] has them as bytes.
138#[derive(Debug, Clone, Default, Deserialize)]
139#[serde(rename_all = "camelCase")]
140pub struct Match {
141    #[serde(default)]
142    pub message: String,
143    #[serde(default)]
144    pub short_message: String,
145    pub offset: usize,
146    pub length: usize,
147    #[serde(default)]
148    pub replacements: Vec<Replacement>,
149    #[serde(default)]
150    pub context: Context,
151    #[serde(default)]
152    pub sentence: String,
153    #[serde(default)]
154    pub rule: Rule,
155}
156
157impl Match {
158    /// The match's bytes in `text`, the text that was checked; None when it runs past the end, or
159    /// starts or ends inside a character.
160    pub fn byte_range(&self, text: &str) -> Option<Range<usize>> {
161        let (start, end) = (self.offset, self.offset.checked_add(self.length)?);
162        let mut units = 0;
163        let (mut start_byte, mut end_byte) = (None, None);
164        for (at, char) in text.char_indices().chain(std::iter::once((text.len(), '\0'))) {
165            if units == start {
166                start_byte = Some(at);
167            }
168            if units == end {
169                end_byte = Some(at);
170                break;
171            }
172            if units > end {
173                break;
174            }
175            units += char.len_utf16();
176        }
177        Some(start_byte?..end_byte?)
178    }
179}
180
181#[derive(Debug, Clone, Default, Deserialize)]
182#[serde(rename_all = "camelCase")]
183pub struct Replacement {
184    pub value: String,
185    #[serde(default)]
186    pub short_description: Option<String>,
187}
188
189/// Some of the text around a match, with the match's place in it in UTF-16 units.
190#[derive(Debug, Clone, Default, Deserialize)]
191pub struct Context {
192    #[serde(default)]
193    pub text: String,
194    #[serde(default)]
195    pub offset: usize,
196    #[serde(default)]
197    pub length: usize,
198}
199
200#[derive(Debug, Clone, Default, Deserialize)]
201#[serde(rename_all = "camelCase")]
202pub struct Rule {
203    #[serde(default)]
204    pub id: String,
205    #[serde(default)]
206    pub sub_id: Option<String>,
207    #[serde(default)]
208    pub description: String,
209    /// Such as `misspelling`, `grammar`, `style`, `typographical`, or `register`.
210    #[serde(default)]
211    pub issue_type: String,
212    #[serde(default)]
213    pub category: Category,
214    #[serde(default)]
215    pub urls: Vec<RuleUrl>,
216}
217
218#[derive(Debug, Clone, Default, Deserialize)]
219pub struct Category {
220    /// Such as `TYPOS`, `GRAMMAR`, `STYLE`, or `REDUNDANCY`.
221    #[serde(default)]
222    pub id: String,
223    #[serde(default)]
224    pub name: String,
225}
226
227/// A page explaining the rule.
228#[derive(Debug, Clone, Default, Deserialize)]
229pub struct RuleUrl {
230    pub value: String,
231}
232
233/// A language the server checks.
234#[derive(Debug, Clone, Default, Deserialize)]
235#[serde(rename_all = "camelCase")]
236pub struct Language {
237    pub name: String,
238    pub code: String,
239    /// The code with its variant, such as `en-US`, which a check takes as its language.
240    pub long_code: String,
241}
242
243/// Checks `request`'s text on the sync server's LanguageTool.
244pub fn check(request: &CheckRequest) -> Result<CheckResponse, ServerError> {
245    let headers = [("Content-Type", "application/x-www-form-urlencoded"), ("Accept", "application/json")];
246    parse(super::request(SERVICE, "POST", CHECK_PATH, &headers, Some(request.form().as_bytes()))?)
247}
248
249/// The languages the sync server's LanguageTool checks.
250pub fn languages() -> Result<Vec<Language>, ServerError> {
251    parse(super::request(SERVICE, "GET", LANGUAGES_PATH, &[("Accept", "application/json")], None)?)
252}
253
254fn parse<T: DeserializeOwned>(response: Response) -> Result<T, ServerError> {
255    if let Some(error) = response.error() {
256        return Err(error);
257    }
258    serde_json::from_str(&response.body).map_err(|e| ServerError {
259        kind: ErrorKind::Invalid,
260        message: format!("LanguageTool's reply couldn't be read: {e}"),
261    })
262}
263
264#[cfg(test)]
265mod tests {
266    use super::*;
267
268    #[test]
269    fn a_plain_request_sends_only_text_and_language() {
270        assert_eq!(CheckRequest::new("It are fine.", "en-US").form(), "text=It+are+fine.&language=en-US");
271        assert_eq!(CheckRequest::new("Hi", " ").form(), "text=Hi&language=auto");
272    }
273
274    #[test]
275    fn options_join_their_lists_and_skip_the_empty_ones() {
276        let request = CheckRequest {
277            mother_tongue: Some("de".into()),
278            preferred_variants: vec!["en-GB".into(), "de-AT".into()],
279            disabled_rules: vec!["WHITESPACE_RULE".into()],
280            enabled_only: true,
281            level: Level::Picky,
282            ..CheckRequest::new("Hi", "auto")
283        };
284        assert_eq!(
285            request.form(),
286            "text=Hi&language=auto&motherTongue=de&preferredVariants=en-GB%2Cde-AT&disabledRules=WHITESPACE_RULE&enabledOnly=true&level=picky"
287        );
288    }
289
290    #[test]
291    fn a_reply_reads_into_matches() {
292        let body = r#"{
293            "software": {"name": "LanguageTool", "version": "6.8"},
294            "warnings": {"incompleteResults": false},
295            "language": {"name": "English (US)", "code": "en-US",
296                "detectedLanguage": {"name": "English (US)", "code": "en-US", "confidence": 0.98}},
297            "matches": [{
298                "message": "Did you mean \"is\"?", "shortMessage": "Agreement",
299                "replacements": [{"value": "is"}, {"value": "was", "shortDescription": "past"}],
300                "offset": 3, "length": 3,
301                "context": {"text": "It are fine.", "offset": 3, "length": 3},
302                "sentence": "It are fine.",
303                "type": {"typeName": "Other"},
304                "rule": {"id": "IT_VBZ", "subId": "1", "description": "Agreement", "issueType": "grammar",
305                    "category": {"id": "GRAMMAR", "name": "Grammar"},
306                    "urls": [{"value": "https://example.com/rule"}]},
307                "ignoreForIncompleteSentence": false, "contextForSureMatch": 0
308            }]
309        }"#;
310        let checked: CheckResponse = parse(Response { status: 200, body: body.into() }).unwrap();
311        assert_eq!(checked.language.as_ref().unwrap().detected_language.as_ref().unwrap().confidence, Some(0.98));
312        let found = &checked.matches[0];
313        assert_eq!((found.offset, found.length, found.short_message.as_str()), (3, 3, "Agreement"));
314        assert_eq!(found.replacements[1].short_description.as_deref(), Some("past"));
315        assert_eq!((found.rule.id.as_str(), found.rule.issue_type.as_str(), found.rule.category.id.as_str()), ("IT_VBZ", "grammar", "GRAMMAR"));
316        assert_eq!(found.rule.urls[0].value, "https://example.com/rule");
317        assert!(!checked.warnings.incomplete_results);
318    }
319
320    #[test]
321    fn a_refusal_or_an_unreadable_reply_is_an_error() {
322        let limited = parse::<CheckResponse>(Response { status: 429, body: String::new() }).unwrap_err();
323        assert_eq!(limited.kind, ErrorKind::RateLimited);
324        let garbled = parse::<CheckResponse>(Response { status: 200, body: "<html>".into() }).unwrap_err();
325        assert_eq!(garbled.kind, ErrorKind::Invalid);
326    }
327
328    #[test]
329    fn languages_read_their_long_codes() {
330        let body = r#"[{"name": "German (Austria)", "code": "de", "longCode": "de-AT"}]"#;
331        let languages: Vec<Language> = parse(Response { status: 200, body: body.into() }).unwrap();
332        assert_eq!(languages[0].long_code, "de-AT");
333    }
334
335    #[test]
336    fn utf16_offsets_become_byte_ranges() {
337        let text = "😀 It are.";
338        let found = Match { offset: 6, length: 3, ..Match::default() };
339        assert_eq!(&text[found.byte_range(text).unwrap()], "are");
340        let to_the_end = Match { offset: 10, length: 0, ..Match::default() };
341        assert_eq!(to_the_end.byte_range(text), Some(text.len()..text.len()));
342        assert_eq!(Match { offset: 1, length: 1, ..Match::default() }.byte_range(text), None, "inside the emoji");
343        assert_eq!(Match { offset: 9, length: 5, ..Match::default() }.byte_range(text), None, "past the end");
344    }
345}
hammer/server/languagetool.rs

hammer/server/
languagetool.rs
