Skip to main content

runner_manager_platform/
service_request.rs

1//! Asking the running service to do something this process cannot.
2//!
3//! The service shares its state directory with the account that installed
4//! it, so a request left there is a channel both sides can use with no
5//! privilege and no socket. The asking command leaves a request under an
6//! identifier it chose; the service takes the request by deleting it, which is
7//! how the command knows somebody is listening; and the service answers under
8//! the same identifier, which is how the command knows the answer is to this
9//! request.
10//!
11//! [`GITHUB_DISCOVERY`] gives every request its own pair of files under
12//! `service-requests/`, so commands asking at once do not displace each other.
13//! [`CACHE_PRUNE`] keeps the single pair of files it was introduced with, which
14//! a service from that release still reads.
15
16use std::fs;
17use std::io;
18use std::path::{Path, PathBuf};
19use std::time::{Duration, Instant};
20
21use serde::de::DeserializeOwned;
22use serde::{Deserialize, Serialize};
23
24/// How often the service looks for a request.
25pub const POLL: Duration = Duration::from_secs(2);
26
27/// The directory under the state directory per-request channels use.
28const REQUESTS_DIR: &str = "service-requests";
29
30/// One kind of request and its answers.
31#[derive(Debug, Clone, Copy, PartialEq, Eq)]
32pub struct Channel {
33    slots: Slots,
34}
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq)]
37enum Slots {
38    /// One request file and one result file, shared by every asker.
39    One {
40        request: &'static str,
41        result: &'static str,
42    },
43    /// `<prefix>.<id>.request` and `<prefix>.<id>.result` per request.
44    PerRequest(&'static str),
45}
46
47/// `host cache prune` for a cache root this account cannot write: the root of
48/// a service running as another account (LocalSystem on Windows, root
49/// elsewhere).
50pub const CACHE_PRUNE: Channel = Channel {
51    slots: Slots::One {
52        request: crate::dependency_cache::PRUNE_REQUEST_FILE,
53        result: crate::dependency_cache::PRUNE_RESULT_FILE,
54    },
55};
56
57/// What the stored GitHub credential reaches, asked of the service by a
58/// command that cannot read the credential itself.
59///
60/// The reason is a login-mode Mac reached over SSH: the credential is in the
61/// login keychain, which an SSH session cannot unlock, while the service runs
62/// in the desktop session and reads it every poll. `repo add`, `org add` and
63/// `auth status` used to fail there with the keychain's error. The answer
64/// holds what GitHub said about the installations, never the credential.
65pub const GITHUB_DISCOVERY: Channel = Channel {
66    slots: Slots::PerRequest("github-discovery"),
67};
68
69/// `host reboot`: restart this machine once every running job has finished.
70/// Only the service can, because only it knows when its runners are done; the
71/// answer says what it will do.
72pub const REBOOT: Channel = Channel {
73    slots: Slots::PerRequest("reboot"),
74};
75
76/// `host reboot --cancel`: keep this machine running after all.
77pub const REBOOT_CANCEL: Channel = Channel {
78    slots: Slots::PerRequest("reboot-cancel"),
79};
80
81/// How long a command waits on the service, and how often it looks.
82#[derive(Debug, Clone, Copy, PartialEq, Eq)]
83pub struct Wait {
84    /// For the service to take the request: several of its polls, so a
85    /// running service always answers inside it.
86    pub taken: Duration,
87    /// For the answer, once the service has taken the request.
88    pub finished: Duration,
89    pub poll: Duration,
90}
91
92impl Wait {
93    /// What `host reboot` uses: the service answers at once.
94    pub const REBOOT: Self = Self {
95        taken: Duration::from_secs(4 * POLL.as_secs()),
96        finished: Duration::from_secs(10),
97        poll: Duration::from_millis(250),
98    };
99
100    /// What a command waiting on a GitHub request uses.
101    pub const GITHUB: Self = Self {
102        taken: Duration::from_secs(4 * POLL.as_secs()),
103        finished: Duration::from_secs(90),
104        poll: Duration::from_millis(250),
105    };
106}
107
108/// Why a command got no answer.
109#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
110pub enum AskError {
111    #[error("the request could not be left for the service: {0}")]
112    NotSent(String),
113    #[error("no running service took the request within {0} seconds")]
114    NotTaken(u64),
115    #[error("the service took the request but gave no answer within {0} seconds")]
116    NoAnswer(u64),
117    #[error("the service could not answer: {0}")]
118    Refused(String),
119}
120
121#[derive(Debug, Serialize, Deserialize)]
122struct Answer<T> {
123    request: String,
124    outcome: Result<T, String>,
125}
126
127/// The longest request identifier taken, and the characters it may hold. An
128/// identifier is chosen by the asking command (a timestamp and a process id)
129/// and copied into the answer, so the service takes nothing else.
130const MAX_ID: usize = 64;
131
132fn valid_id(id: &str) -> bool {
133    !id.is_empty()
134        && id.len() <= MAX_ID
135        && id
136            .chars()
137            .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '.' | '_'))
138}
139
140/// An answer nobody collected is removed after this long.
141const UNCOLLECTED: Duration = Duration::from_secs(10 * 60);
142
143/// Reads a single-slot request without following a link and without reading
144/// more than an identifier's worth.
145///
146/// The service may run as root or LocalSystem while the state directory
147/// belongs to the account that installed it, so the request path is under that
148/// account's control. Following a link there would have the service read a
149/// file of the link's choosing, and copy it into an answer that account can
150/// read; a FIFO or `/dev/zero` would hold the service's loop.
151fn read_request(path: &Path) -> Option<String> {
152    use std::io::Read as _;
153    let mut options = fs::OpenOptions::new();
154    options.read(true);
155    #[cfg(unix)]
156    std::os::unix::fs::OpenOptionsExt::custom_flags(
157        &mut options,
158        libc::O_NOFOLLOW | libc::O_NONBLOCK,
159    );
160    #[cfg(windows)]
161    std::os::windows::fs::OpenOptionsExt::custom_flags(
162        &mut options,
163        // FILE_FLAG_OPEN_REPARSE_POINT: open a link as itself.
164        0x0020_0000,
165    );
166    let file = options.open(path).ok()?;
167    if !file.metadata().ok()?.is_file() {
168        return None;
169    }
170    let mut text = String::new();
171    file.take(MAX_ID as u64 + 2)
172        .read_to_string(&mut text)
173        .ok()?;
174    let id = text.trim();
175    valid_id(id).then(|| id.to_owned())
176}
177
178/// The answer file is the service's to write and the asking account's to read,
179/// whichever account the service runs as. It holds no secret.
180fn write_readable_by_the_asker(path: &Path, bytes: &[u8]) -> io::Result<()> {
181    use std::io::Write as _;
182    let parent = path.parent().unwrap_or_else(|| Path::new("."));
183    fs::create_dir_all(parent)?;
184    let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
185    temporary.write_all(bytes)?;
186    temporary.as_file().sync_all()?;
187    // On the file before it appears under its name: the asker removes an
188    // answer as soon as it has read it, so a mode set afterwards can find
189    // nothing there.
190    #[cfg(unix)]
191    {
192        use std::os::unix::fs::PermissionsExt as _;
193        temporary
194            .as_file()
195            .set_permissions(fs::Permissions::from_mode(0o644))?;
196    }
197    temporary
198        .persist(path)
199        .map(|_| ())
200        .map_err(|error| error.error)
201}
202
203impl Channel {
204    /// Where a single-slot channel's request lives.
205    #[must_use]
206    pub fn request_path(&self, state_dir: &Path) -> PathBuf {
207        match self.slots {
208            Slots::One { request, .. } => state_dir.join(request),
209            Slots::PerRequest(prefix) => state_dir.join(REQUESTS_DIR).join(prefix),
210        }
211    }
212
213    fn request_file(&self, state_dir: &Path, id: &str) -> PathBuf {
214        match self.slots {
215            Slots::One { request, .. } => state_dir.join(request),
216            Slots::PerRequest(prefix) => state_dir
217                .join(REQUESTS_DIR)
218                .join(format!("{prefix}.{id}.request")),
219        }
220    }
221
222    fn result_file(&self, state_dir: &Path, id: &str) -> PathBuf {
223        match self.slots {
224            Slots::One { result, .. } => state_dir.join(result),
225            Slots::PerRequest(prefix) => state_dir
226                .join(REQUESTS_DIR)
227                .join(format!("{prefix}.{id}.result")),
228        }
229    }
230
231    /// Takes the pending requests and returns their identifiers. Taking one is
232    /// deleting it, so a request is answered once.
233    #[must_use]
234    pub fn take(&self, state_dir: &Path) -> Vec<String> {
235        self.take_where(state_dir, |_| true).0
236    }
237
238    /// [`Self::take`], judging each per-request file by its metadata (not
239    /// followed through a link) before it is removed: the identifiers
240    /// `accept` took, then those it refused, which are answered as refused.
241    /// A single-slot channel's request is always taken.
242    #[must_use]
243    pub fn take_where(
244        &self,
245        state_dir: &Path,
246        accept: impl Fn(&fs::Metadata) -> bool,
247    ) -> (Vec<String>, Vec<String>) {
248        let mut refused = Vec::new();
249        let taken = match self.slots {
250            Slots::One { request, .. } => {
251                let path = state_dir.join(request);
252                if fs::symlink_metadata(&path).is_err() {
253                    return (Vec::new(), refused);
254                }
255                let id = read_request(&path);
256                // Removed whatever it held: an unreadable or malformed request
257                // is not going to become a good one.
258                if fs::remove_file(&path).is_err() {
259                    return (Vec::new(), refused);
260                }
261                id.into_iter().collect()
262            }
263            Slots::PerRequest(prefix) => {
264                let Ok(entries) = fs::read_dir(state_dir.join(REQUESTS_DIR)) else {
265                    return (Vec::new(), refused);
266                };
267                let mut taken = Vec::new();
268                for entry in entries.flatten() {
269                    let name = entry.file_name();
270                    let Some(name) = name.to_str() else { continue };
271                    let Some(rest) = name.strip_prefix(prefix).and_then(|r| r.strip_prefix('.'))
272                    else {
273                        continue;
274                    };
275                    if let Some(id) = rest.strip_suffix(".request") {
276                        // The identifier is the file's name, so its contents
277                        // are never read, and a link is removed as a link.
278                        let accepted =
279                            fs::symlink_metadata(entry.path()).is_ok_and(|meta| accept(&meta));
280                        if fs::remove_file(entry.path()).is_ok() && valid_id(id) {
281                            if accepted {
282                                taken.push(id.to_owned());
283                            } else {
284                                refused.push(id.to_owned());
285                            }
286                        }
287                    } else if rest.ends_with(".result")
288                        && entry
289                            .metadata()
290                            .and_then(|meta| meta.modified())
291                            .ok()
292                            .and_then(|at| at.elapsed().ok())
293                            .is_some_and(|age| age >= UNCOLLECTED)
294                    {
295                        let _ = fs::remove_file(entry.path());
296                    }
297                }
298                taken
299            }
300        };
301        (taken, refused)
302    }
303
304    /// The service's answer to `request`.
305    ///
306    /// # Errors
307    /// The answer could not be encoded or written.
308    pub fn answer<T: Serialize>(
309        &self,
310        state_dir: &Path,
311        request: &str,
312        outcome: Result<T, String>,
313    ) -> io::Result<()> {
314        let answer = Answer {
315            request: request.to_owned(),
316            outcome,
317        };
318        let json = serde_json::to_vec_pretty(&answer).map_err(io::Error::other)?;
319        let path = self.result_file(state_dir, request);
320        write_readable_by_the_asker(&path, &json)
321    }
322
323    /// The service's answer to `request`, once it has given one.
324    #[must_use]
325    pub fn read_answer<T: DeserializeOwned>(
326        &self,
327        state_dir: &Path,
328        request: &str,
329    ) -> Option<Result<T, String>> {
330        fs::read(self.result_file(state_dir, request))
331            .ok()
332            .and_then(|bytes| serde_json::from_slice::<Answer<T>>(&bytes).ok())
333            .filter(|answer| answer.request == request)
334            .map(|answer| answer.outcome)
335    }
336
337    /// Asks the service and waits for its answer.
338    ///
339    /// # Errors
340    /// [`AskError`], naming which half of the exchange did not happen.
341    pub fn ask<T: DeserializeOwned>(&self, state_dir: &Path, wait: Wait) -> Result<T, AskError> {
342        self.ask_with(state_dir, wait, || {})
343    }
344
345    /// [`Self::ask`], calling `sent` once the request is in place, so the
346    /// command can say it is waiting.
347    ///
348    /// # Errors
349    /// As [`Self::ask`].
350    pub fn ask_with<T: DeserializeOwned>(
351        &self,
352        state_dir: &Path,
353        wait: Wait,
354        sent: impl FnOnce(),
355    ) -> Result<T, AskError> {
356        // Unique within this process as well as across processes: two threads
357        // can read the same instant where the clock is coarse (macOS).
358        static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
359        let id = format!(
360            "{}-{}-{}",
361            chrono::Utc::now().format("%Y%m%dT%H%M%S%.9fZ"),
362            std::process::id(),
363            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
364        );
365        let request = self.request_file(state_dir, &id);
366        crate::host_fitness::write_atomically(&request, id.as_bytes())
367            .map_err(|error| AskError::NotSent(format!("{}: {error}", request.display())))?;
368        sent();
369
370        let started = Instant::now();
371        let pending = |request: &Path| match self.slots {
372            Slots::One { .. } => read_request(request).as_deref() == Some(id.as_str()),
373            Slots::PerRequest(_) => fs::symlink_metadata(request).is_ok(),
374        };
375        while pending(&request) {
376            if started.elapsed() >= wait.taken {
377                // Withdrawn, so a service that starts later does not answer a
378                // command that is no longer waiting: only this command's own
379                // request, which in a single slot another command may have
380                // replaced. Gone already means it was taken after all.
381                match fs::remove_file(&request) {
382                    Err(error) if error.kind() == io::ErrorKind::NotFound => break,
383                    _ => return Err(AskError::NotTaken(wait.taken.as_secs())),
384                }
385            }
386            std::thread::sleep(wait.poll);
387        }
388
389        let taken = Instant::now();
390        loop {
391            if let Some(outcome) = self.read_answer(state_dir, &id) {
392                if matches!(self.slots, Slots::PerRequest(_)) {
393                    let _ = fs::remove_file(self.result_file(state_dir, &id));
394                }
395                return outcome.map_err(AskError::Refused);
396            }
397            if taken.elapsed() >= wait.finished {
398                return Err(AskError::NoAnswer(wait.finished.as_secs()));
399            }
400            std::thread::sleep(wait.poll);
401        }
402    }
403}
404
405#[cfg(test)]
406mod tests {
407    use super::*;
408
409    const QUICK: Wait = Wait {
410        taken: Duration::from_millis(600),
411        finished: Duration::from_secs(5),
412        poll: Duration::from_millis(20),
413    };
414
415    /// A stand-in service: takes one request and answers it with `outcome`.
416    fn service(
417        state_dir: PathBuf,
418        outcome: Result<u32, String>,
419    ) -> std::thread::JoinHandle<String> {
420        std::thread::spawn(move || {
421            loop {
422                if let Some(request) = GITHUB_DISCOVERY.take(&state_dir).pop() {
423                    GITHUB_DISCOVERY
424                        .answer(&state_dir, &request, outcome)
425                        .unwrap();
426                    return request;
427                }
428                std::thread::sleep(Duration::from_millis(10));
429            }
430        })
431    }
432
433    #[test]
434    fn a_request_its_judge_refuses_is_removed_and_named_as_refused() {
435        let dir = tempfile::tempdir().unwrap();
436        let requests = dir.path().join(REQUESTS_DIR);
437        fs::create_dir_all(&requests).unwrap();
438        fs::write(requests.join("reboot.a1.request"), "").unwrap();
439        fs::write(requests.join("reboot-cancel.b2.request"), "").unwrap();
440        assert_eq!(
441            REBOOT.take_where(dir.path(), |_| false),
442            (Vec::new(), vec!["a1".to_owned()])
443        );
444        assert!(!requests.join("reboot.a1.request").exists());
445        assert_eq!(REBOOT_CANCEL.take(dir.path()), ["b2"]);
446    }
447
448    #[test]
449    fn a_running_service_takes_the_request_once_and_answers_it() {
450        let dir = tempfile::tempdir().unwrap();
451        let answering = service(dir.path().to_path_buf(), Ok(7));
452        assert_eq!(GITHUB_DISCOVERY.ask::<u32>(dir.path(), QUICK), Ok(7));
453        answering.join().unwrap();
454        assert!(
455            GITHUB_DISCOVERY.take(dir.path()).is_empty(),
456            "answered once"
457        );
458    }
459
460    #[test]
461    fn a_service_that_cannot_answer_says_why() {
462        let dir = tempfile::tempdir().unwrap();
463        let answering = service(dir.path().to_path_buf(), Err("no credential".into()));
464        assert_eq!(
465            GITHUB_DISCOVERY.ask::<u32>(dir.path(), QUICK),
466            Err(AskError::Refused("no credential".into()))
467        );
468        answering.join().unwrap();
469    }
470
471    #[test]
472    fn with_no_service_listening_the_request_is_withdrawn() {
473        let dir = tempfile::tempdir().unwrap();
474        assert!(matches!(
475            GITHUB_DISCOVERY.ask::<u32>(dir.path(), QUICK),
476            Err(AskError::NotTaken(_))
477        ));
478        assert!(
479            GITHUB_DISCOVERY.take(dir.path()).is_empty(),
480            "a later service must not answer a command that stopped waiting"
481        );
482    }
483
484    #[test]
485    fn an_answer_to_another_request_is_not_taken_for_this_one() {
486        let dir = tempfile::tempdir().unwrap();
487        GITHUB_DISCOVERY
488            .answer(dir.path(), "an-earlier-request", Ok(1_u32))
489            .unwrap();
490        let state_dir = dir.path().to_path_buf();
491        // Takes the request and never answers it.
492        let silent = std::thread::spawn(move || {
493            while GITHUB_DISCOVERY.take(&state_dir).is_empty() {
494                std::thread::sleep(Duration::from_millis(10));
495            }
496        });
497        let wait = Wait {
498            finished: Duration::from_millis(300),
499            ..QUICK
500        };
501        assert!(matches!(
502            GITHUB_DISCOVERY.ask::<u32>(dir.path(), wait),
503            Err(AskError::NoAnswer(_))
504        ));
505        silent.join().unwrap();
506    }
507
508    /// Two commands asking at once each get their own answer: neither
509    /// replaces the other's request.
510    #[test]
511    fn commands_asking_at_once_each_get_their_own_answer() {
512        let dir = tempfile::tempdir().unwrap();
513        let state_dir = dir.path().to_path_buf();
514        let service = std::thread::spawn(move || {
515            let mut answered = 0;
516            let deadline = Instant::now() + Duration::from_secs(10);
517            while answered < 2 && Instant::now() < deadline {
518                for request in GITHUB_DISCOVERY.take(&state_dir) {
519                    GITHUB_DISCOVERY
520                        .answer(&state_dir, &request, Ok(request.len()))
521                        .unwrap();
522                    answered += 1;
523                }
524                std::thread::sleep(Duration::from_millis(10));
525            }
526            answered
527        });
528        let asking =
529            |dir: PathBuf| std::thread::spawn(move || GITHUB_DISCOVERY.ask::<usize>(&dir, QUICK));
530        let first = asking(dir.path().to_path_buf());
531        let second = asking(dir.path().to_path_buf());
532        assert!(first.join().unwrap().is_ok());
533        assert!(second.join().unwrap().is_ok());
534        assert_eq!(service.join().unwrap(), 2);
535    }
536
537    /// The service never follows a link left at a request path, and never
538    /// takes an identifier it would not have chosen.
539    #[cfg(unix)]
540    #[test]
541    fn a_request_that_is_a_link_or_not_an_identifier_is_dropped_unread() {
542        let dir = tempfile::tempdir().unwrap();
543        let secret = dir.path().join("only-the-service-may-read");
544        fs::write(&secret, "the contents of somebody else's file").unwrap();
545        let request = CACHE_PRUNE.request_path(dir.path());
546        std::os::unix::fs::symlink(&secret, &request).unwrap();
547        assert!(CACHE_PRUNE.take(dir.path()).is_empty());
548        assert!(
549            fs::symlink_metadata(&request).is_err(),
550            "the link is removed"
551        );
552        assert!(secret.exists(), "and only the link");
553
554        fs::write(&request, "not an identifier\nat all").unwrap();
555        assert!(CACHE_PRUNE.take(dir.path()).is_empty());
556        fs::write(&request, "20261007T221210.123456789Z-4242\n").unwrap();
557        assert_eq!(
558            CACHE_PRUNE.take(dir.path()),
559            ["20261007T221210.123456789Z-4242"]
560        );
561    }
562}