Skip to main content

crabshell_services/
http.rs

1//! HTTP GET through `curl`, as the configs Quickshell's users write fetch
2//! the weather, and remote cover art fetched into a cache file.
3
4#[cfg(test)]
5mod tests;
6
7pub(crate) mod art;
8
9use std::cell::RefCell;
10use std::fs::File;
11use std::io::{ErrorKind, PipeReader, Read, Write};
12use std::os::fd::AsFd;
13use std::panic::{self, AssertUnwindSafe, Location};
14use std::process::{Child, Command, ExitStatus};
15use std::rc::{Rc, Weak};
16use std::sync::Once;
17use std::time::Duration;
18
19use calloop::{Interest, RegistrationToken};
20use crabshell_reactive::{Memo, ReadSignal, Signal, untrack};
21use crabshell_ui::{Cx, IntoProp};
22use rustix::process::{Pid, PidfdFlags, Signal as Kill, kill_process_group, pidfd_open};
23
24use crate::lines::panic_message;
25use crate::{Shared, spawn};
26
27/// The largest body [`Http`] keeps unless [`Http::max_size`] says
28/// otherwise.
29pub(crate) const MAX_BODY: u64 = 4 * 1024 * 1024;
30
31/// How long a whole request may take unless [`Http::timeout`] says
32/// otherwise, and how long a cover art download may.
33pub(crate) const TIMEOUT: Duration = Duration::from_secs(30);
34
35/// The most of curl's standard error read, for its message.
36const MAX_STDERR: usize = 4096;
37
38/// What `cx.http_get` and `cx.cover_art` need from the shell: the loop,
39/// where curl is, and where art is cached.
40pub(crate) struct Service {
41  shared: Rc<Shared>,
42  /// Every curl running, so the shell can stop them all before it exits.
43  live: RefCell<Vec<Weak<RefCell<Run>>>>,
44}
45
46impl Service {
47  pub(crate) fn new(shared: Rc<Shared>) -> Service {
48    let live = RefCell::default();
49    Service { shared, live }
50  }
51}
52
53/// Stops every curl running, as the shell does before it exits: `exit`
54/// skips the drops that would, and curl has a process group of its own.
55pub(crate) fn stop_all(cx: &Cx) {
56  let Some(service) = cx.service::<Service>() else {
57    return;
58  };
59  let live: Vec<_> = service
60    .live
61    .take()
62    .iter()
63    .filter_map(Weak::upgrade)
64    .collect();
65  for run in live {
66    run.borrow_mut().stop();
67  }
68}
69
70/// Why a request failed, from [`Http::error`].
71#[derive(Clone, Debug, PartialEq, Eq)]
72pub struct HttpError {
73  /// What kind of failure it is.
74  pub kind: HttpErrorKind,
75  /// What curl or the shell said, such as `The requested URL returned
76  /// error: 404`.
77  pub message: String,
78}
79
80/// The kinds of [`HttpError`].
81#[derive(Clone, Copy, Debug, PartialEq, Eq)]
82pub enum HttpErrorKind {
83  /// `curl` isn't installed, or couldn't start.
84  NoCurl,
85  /// The server answered with this HTTP status, 400 or over.
86  Status(u16),
87  /// The body was over [`Http::max_size`].
88  TooLarge,
89  /// The request took longer than [`Http::timeout`].
90  TimedOut,
91  /// The body isn't UTF-8.
92  NotUtf8,
93  /// Anything on the way, such as a name that didn't resolve or a refused
94  /// connection.
95  Network,
96  /// The URL, or a redirect, isn't `http:` or `https:`.
97  Refused,
98}
99
100impl HttpError {
101  fn new(kind: HttpErrorKind, message: impl Into<String>) -> HttpError {
102    let message = message.into();
103    HttpError { kind, message }
104  }
105}
106
107/// Whether `url` is one curl may fetch.
108fn allowed(url: &str) -> bool {
109  let scheme = url.split_once("://").map(|(scheme, _)| scheme);
110  scheme.is_some_and(|scheme| {
111    scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https")
112  })
113}
114
115/// Where a running request puts its body.
116pub(crate) enum Sink {
117  Memory(Vec<u8>),
118  File(File),
119}
120
121type Done = Box<dyn FnOnce(Result<Sink, HttpError>)>;
122
123/// One curl run. Dropping it stops curl if it still runs, without calling
124/// its `done`.
125pub(crate) struct Fetch(Rc<RefCell<Run>>);
126
127struct Run {
128  shared: Rc<Shared>,
129  /// `None` once it ended or was stopped.
130  child: Option<Child>,
131  out: Option<(PipeReader, RegistrationToken)>,
132  err: PipeReader,
133  exit: Option<RegistrationToken>,
134  sink: Option<Sink>,
135  got: u64,
136  cap: u64,
137  done: Option<Done>,
138}
139
140/// Runs curl for `url`, an `http:` or `https:` URL, with its body going to
141/// `sink`, up to `cap` bytes, for at most `timeout`. `done` gets the sink
142/// back once the body arrived whole, or why it didn't. The URL is an
143/// argument, never seen by a shell, and curl reads no `~/.curlrc`.
144pub(crate) fn fetch(
145  service: &Rc<Service>,
146  url: &str,
147  cap: u64,
148  timeout: Duration,
149  sink: Sink,
150  done: impl FnOnce(Result<Sink, HttpError>) + 'static,
151) -> Result<Fetch, HttpError> {
152  if !allowed(url) {
153    return Err(HttpError::new(
154      HttpErrorKind::Refused,
155      "only http: and https: URLs are fetched",
156    ));
157  }
158  let shared = &service.shared;
159  let Some(event_loop) = shared.event_loop() else {
160    return Err(HttpError::new(
161      HttpErrorKind::Network,
162      "the shell is stopping",
163    ));
164  };
165  let mut command = Command::new(&shared.config.curl);
166  // `-q` must come first to skip `~/.curlrc`. `--globoff` keeps `[]` and
167  // `{}` in a URL from naming many.
168  command.args([
169    "-q",
170    "--silent",
171    "--show-error",
172    "--globoff",
173    "--fail",
174    "--location",
175  ]);
176  command.args([
177    "--max-redirs",
178    "5",
179    "--proto",
180    "=http,https",
181    "--proto-redir",
182    "=http,https",
183  ]);
184  // curl reads 0 as no limit, and refuses a number it can't hold.
185  let seconds = timeout.as_secs_f64().clamp(0.001, f64::from(u32::MAX));
186  command.arg("--max-time").arg(format!("{seconds:.3}"));
187  // Past that, the bytes counted here are the only cap.
188  if i64::try_from(cap).is_ok() {
189    command.arg("--max-filesize").arg(cap.to_string());
190  }
191  // The status goes last on standard error, after curl's message.
192  command.args(["--write-out", "%{stderr}%{response_code}", "--url", url]);
193  let spawned = crate::process::spawn_piped(&mut command, false);
194  drop(command);
195  let (child, out, err, _) = spawned.map_err(|error| {
196    let kind = match error.kind() {
197      ErrorKind::NotFound | ErrorKind::PermissionDenied => HttpErrorKind::NoCurl,
198      _ => HttpErrorKind::Network,
199    };
200    HttpError::new(kind, format!("couldn't start curl: {error}"))
201  })?;
202  let pidfd = pidfd_open(Pid::from_child(&child), PidfdFlags::NONBLOCK);
203  let run = Rc::new(RefCell::new(Run {
204    shared: Rc::clone(shared),
205    child: Some(child),
206    out: None,
207    err,
208    exit: None,
209    sink: Some(sink),
210    got: 0,
211    cap,
212    done: Some(Box::new(done)),
213  }));
214  let fetch = Fetch(Rc::clone(&run));
215  let mut live = service.live.borrow_mut();
216  live.retain(|run| run.strong_count() > 0);
217  live.push(Rc::downgrade(&run));
218  drop(live);
219  let fd = out.as_fd().try_clone_to_owned();
220  let reader = Rc::clone(&run);
221  let token = fd
222    .ok()
223    .and_then(|fd| event_loop.insert_fd(fd, Interest::READ, Box::new(move || on_output(&reader))));
224  let ended = Rc::clone(&run);
225  let exit = pidfd
226    .ok()
227    .and_then(|fd| event_loop.insert_fd(fd, Interest::READ, Box::new(move || on_exit(&ended))));
228  let mut state = run.borrow_mut();
229  state.out = token.map(|token| (out, token));
230  state.exit = exit;
231  let followed = state.out.is_some() && state.exit.is_some();
232  drop(state);
233  if !followed {
234    // Logged by the loop; dropping `fetch` stops curl.
235    return Err(HttpError::new(
236      HttpErrorKind::Network,
237      "couldn't follow curl",
238    ));
239  }
240  Ok(fetch)
241}
242
243/// Reads what curl printed, and stops it once that is over the cap.
244fn on_output(run: &Rc<RefCell<Run>>) {
245  let mut state = run.borrow_mut();
246  let error = match state.read_out() {
247    Chunk::More | Chunk::Wait => return,
248    Chunk::End => {
249      // The end of the body; its exit is still to come.
250      if let Some((_, token)) = state.out.take() {
251        state.remove(token);
252      }
253      return;
254    }
255    Chunk::Failed(error) => error,
256  };
257  drop(state);
258  finish(run, Err(error));
259}
260
261/// curl ended: what is left of its output, then how it went.
262fn on_exit(run: &Rc<RefCell<Run>>) {
263  let mut state = run.borrow_mut();
264  let status = match state.child.as_mut().map(Child::try_wait) {
265    Some(Ok(Some(status))) => Some(status),
266    Some(Ok(None)) | None => return,
267    Some(Err(_)) => None,
268  };
269  state.child = None;
270  // What it printed before it ended is in the pipe now, up to the cap.
271  let mut failed = None;
272  while failed.is_none() {
273    match state.read_out() {
274      Chunk::More => {}
275      Chunk::Wait | Chunk::End => break,
276      Chunk::Failed(error) => failed = Some(error),
277    }
278  }
279  let mut stderr = Vec::new();
280  // Non-blocking: what curl wrote before it ended.
281  let _ = (&state.err)
282    .take(MAX_STDERR as u64)
283    .read_to_end(&mut stderr);
284  let sink = state.sink.take();
285  drop(state);
286  let result = match failed {
287    Some(error) => Err(error),
288    None => outcome(status, &String::from_utf8_lossy(&stderr)),
289  };
290  let result = result
291    .and_then(|()| sink.ok_or_else(|| HttpError::new(HttpErrorKind::Network, "the body was lost")));
292  finish(run, result);
293}
294
295/// What one read of curl's output gave.
296enum Chunk {
297  More,
298  /// Nothing to read now.
299  Wait,
300  End,
301  Failed(HttpError),
302}
303
304impl Run {
305  /// Reads one chunk of the body into the sink.
306  fn read_out(&mut self) -> Chunk {
307    let mut chunk = [0; 64 * 1024];
308    let Some((out, _)) = self.out.as_mut() else {
309      return Chunk::End;
310    };
311    let read = match out.read(&mut chunk) {
312      Ok(read) => read,
313      Err(error) if error.kind() == ErrorKind::Interrupted => return Chunk::More,
314      Err(error) if error.kind() == ErrorKind::WouldBlock => return Chunk::Wait,
315      Err(_) => 0,
316    };
317    if read == 0 {
318      return Chunk::End;
319    }
320    self.got += read as u64;
321    if self.got > self.cap {
322      let cap = self.cap;
323      let message = format!("the body is over {cap} bytes");
324      return Chunk::Failed(HttpError::new(HttpErrorKind::TooLarge, message));
325    }
326    match self.sink.as_mut() {
327      Some(Sink::Memory(body)) => body.extend_from_slice(&chunk[..read]),
328      Some(Sink::File(file)) => {
329        if let Err(error) = file.write_all(&chunk[..read]) {
330          let message = format!("couldn't write the download: {error}");
331          return Chunk::Failed(HttpError::new(HttpErrorKind::Network, message));
332        }
333      }
334      None => {}
335    }
336    Chunk::More
337  }
338
339  fn remove(&self, token: RegistrationToken) {
340    if let Some(event_loop) = self.shared.event_loop() {
341      event_loop.remove(token);
342    }
343  }
344
345  /// Stops curl if it runs and takes the run out of the loop.
346  fn stop(&mut self) {
347    let tokens = [self.exit.take(), self.out.take().map(|(_, token)| token)];
348    for token in tokens.into_iter().flatten() {
349      self.remove(token);
350    }
351    if let Some(child) = self.child.take() {
352      // Its own group, from `process_group(0)`.
353      let _ = kill_process_group(Pid::from_child(&child), Kill::TERM);
354      spawn::reap(&self.shared, child, String::from("curl"));
355    }
356  }
357}
358
359/// Ends the run and calls its `done` with nothing borrowed.
360fn finish(run: &Rc<RefCell<Run>>, result: Result<Sink, HttpError>) {
361  let done = {
362    let mut state = run.borrow_mut();
363    state.stop();
364    state.done.take()
365  };
366  if let Some(done) = done {
367    done(result);
368  }
369}
370
371impl Drop for Fetch {
372  fn drop(&mut self) {
373    let done = {
374      let mut state = self.0.borrow_mut();
375      state.stop();
376      state.done.take()
377    };
378    // Dropped outside the borrow: it may hold what holds this fetch.
379    drop(done);
380  }
381}
382
383/// How curl said it went, from its exit and its standard error, whose
384/// last line is the HTTP status.
385fn outcome(status: Option<ExitStatus>, stderr: &str) -> Result<(), HttpError> {
386  let (said, code) = stderr.rsplit_once('\n').unwrap_or(("", stderr));
387  // `curl: (22) The requested URL returned error: 404`
388  let said = said.trim();
389  let said = said
390    .strip_prefix("curl: (")
391    .and_then(|rest| rest.split_once(") "))
392    .map_or(said, |(_, message)| message);
393  let exit = status.and_then(|status| status.code());
394  let message = match exit {
395    _ if !said.is_empty() => said.to_owned(),
396    Some(exit) => format!("curl exited with {exit}"),
397    None => String::from("curl was stopped"),
398  };
399  let kind = match exit {
400    Some(0) => return Ok(()),
401    Some(22) => HttpErrorKind::Status(code.trim().parse().unwrap_or(0)),
402    Some(28) => HttpErrorKind::TimedOut,
403    Some(63) => HttpErrorKind::TooLarge,
404    // An unsupported protocol, which only a redirect can reach.
405    Some(1) => HttpErrorKind::Refused,
406    _ => HttpErrorKind::Network,
407  };
408  Err(HttpError::new(kind, message))
409}
410
411/// A GET request, from `cx.http_get(url)`. It is a `Copy` handle, like a
412/// signal, and goes with the context that asked.
413#[derive(Clone, Copy)]
414pub struct Http {
415  body: Signal<Option<String>>,
416  error: Signal<Option<HttpError>>,
417  loading: Signal<bool>,
418  /// Read by the effect that starts a request, so bumping it starts one.
419  refreshes: Signal<u64>,
420  inner: Signal<Keep>,
421}
422
423/// Stops the request when the asking context's owner drops it.
424struct Keep(Rc<Inner>);
425
426impl Drop for Keep {
427  fn drop(&mut self) {
428    let fetch = self.0.state.borrow_mut().fetch.take();
429    drop(fetch);
430  }
431}
432
433struct Inner {
434  /// `None` when the shell provided no services.
435  service: Option<Rc<Service>>,
436  body: Signal<Option<String>>,
437  error: Signal<Option<HttpError>>,
438  loading: Signal<bool>,
439  state: RefCell<State>,
440}
441
442struct State {
443  fetch: Option<Fetch>,
444  stopped: bool,
445  max_size: u64,
446  timeout: Duration,
447}
448
449/// `cx.http_get(url)`.
450#[track_caller]
451pub(crate) fn http_get(cx: &Cx, url: impl IntoProp<String>) -> Http {
452  let at = Location::caller();
453  let service = cx.service::<Service>();
454  if service.is_none() {
455    static LOGGED: Once = Once::new();
456    LOGGED.call_once(|| {
457      tracing::warn!("cx.http_get() needs the services the shell provides; nothing is fetched");
458    });
459  }
460  let inner = Rc::new(Inner {
461    service,
462    body: cx.signal(None),
463    error: cx.signal(None),
464    loading: cx.signal(false),
465    state: RefCell::new(State {
466      fetch: None,
467      stopped: false,
468      max_size: MAX_BODY,
469      timeout: TIMEOUT,
470    }),
471  });
472  let refreshes = cx.signal(0);
473  let url = chosen(cx, url, at, "cx.http_get");
474  let weak = Rc::downgrade(&inner);
475  // Built in an effect, so a signal the URL read fetches again.
476  cx.effect(move || {
477    refreshes.get();
478    let Some(inner) = weak.upgrade().filter(|inner| !inner.state.borrow().stopped) else {
479      return;
480    };
481    let url = url.get();
482    untrack(|| inner.start(url));
483  });
484  Http {
485    body: inner.body,
486    error: inner.error,
487    loading: inner.loading,
488    refreshes,
489    inner: cx.signal(Keep(inner)),
490  }
491}
492
493/// The URL a config's property gives, in a memo so a closure that reads
494/// other signals fetches again only when the URL itself changes. A panic
495/// is logged with where `name` was called and gives `None`.
496fn chosen(
497  cx: &Cx,
498  url: impl IntoProp<String>,
499  at: &'static Location<'static>,
500  name: &'static str,
501) -> Memo<Option<String>> {
502  let url = url.into_prop().shared();
503  cx.memo(move || {
504    panic::catch_unwind(AssertUnwindSafe(|| url()))
505      .inspect_err(|payload| {
506        let message = panic_message(payload.as_ref());
507        tracing::error!("{name} (called at {at}) panicked choosing its URL: {message}");
508      })
509      .ok()
510  })
511}
512
513impl Http {
514  /// The last body that arrived whole, kept through a later fetch that
515  /// fails. `None` until one has.
516  pub fn body(&self) -> ReadSignal<Option<String>> {
517    self.body.read_only()
518  }
519
520  /// Why the last fetch failed, or `None` once one worked.
521  pub fn error(&self) -> ReadSignal<Option<HttpError>> {
522    self.error.read_only()
523  }
524
525  /// `true` while a fetch runs.
526  pub fn loading(&self) -> ReadSignal<bool> {
527    self.loading.read_only()
528  }
529
530  /// Fetches again now, stopping a fetch still running. A
531  /// [`stopped`](Self::stopped) request starts.
532  pub fn refresh(&self) {
533    if let Some(inner) = self.inner() {
534      inner.state.borrow_mut().stopped = false;
535    }
536    self.refreshes.try_update(|count| *count += 1);
537  }
538
539  /// Fetches nothing until the first [`refresh`](Self::refresh), for a
540  /// request made on demand. Call it right after `cx.http_get`.
541  pub fn stopped(self) -> Http {
542    if let Some(inner) = self.inner() {
543      inner.state.borrow_mut().stopped = true;
544      inner.stop();
545    }
546    self
547  }
548
549  /// The largest body taken, 4 MiB by default; a larger one stops the
550  /// fetch and is an error. Applies from the next fetch, so call it right
551  /// after `cx.http_get`.
552  pub fn max_size(self, bytes: u64) -> Http {
553    if let Some(inner) = self.inner() {
554      inner.state.borrow_mut().max_size = bytes;
555    }
556    self
557  }
558
559  /// How long the whole request may take, 30 s by default. Applies from
560  /// the next fetch, so call it right after `cx.http_get`.
561  pub fn timeout(self, timeout: Duration) -> Http {
562    if let Some(inner) = self.inner() {
563      inner.state.borrow_mut().timeout = timeout;
564    }
565    self
566  }
567
568  fn inner(&self) -> Option<Rc<Inner>> {
569    untrack(|| self.inner.try_with(|keep| Rc::clone(&keep.0)))
570  }
571}
572
573impl Inner {
574  /// Stops the fetch running now, if one is, and starts one for `url`.
575  /// Nothing, or an empty URL, fetches nothing.
576  fn start(self: &Rc<Self>, url: Option<String>) {
577    let running = self.state.borrow_mut().fetch.take();
578    drop(running);
579    let (Some(url), Some(service)) = (url.filter(|url| !url.is_empty()), self.service.clone())
580    else {
581      set(self.loading, false);
582      return;
583    };
584    let (cap, timeout) = {
585      let state = self.state.borrow();
586      (state.max_size, state.timeout)
587    };
588    let weak = Rc::downgrade(self);
589    let done = move |result| {
590      if let Some(inner) = Weak::upgrade(&weak) {
591        inner.done(result);
592      }
593    };
594    match fetch(&service, &url, cap, timeout, Sink::Memory(Vec::new()), done) {
595      Ok(fetch) => {
596        self.state.borrow_mut().fetch = Some(fetch);
597        set(self.loading, true);
598      }
599      Err(error) => self.failed(error),
600    }
601  }
602
603  /// Stops the fetch running now, if one is.
604  fn stop(&self) {
605    let fetch = self.state.borrow_mut().fetch.take();
606    drop(fetch);
607    set(self.loading, false);
608  }
609
610  fn done(&self, result: Result<Sink, HttpError>) {
611    let fetch = self.state.borrow_mut().fetch.take();
612    drop(fetch);
613    let text = result.and_then(|sink| match sink {
614      Sink::Memory(body) => String::from_utf8(body)
615        .map_err(|_| HttpError::new(HttpErrorKind::NotUtf8, "the body isn't UTF-8")),
616      Sink::File(_) => Err(HttpError::new(HttpErrorKind::Network, "no body")),
617    });
618    match text {
619      Ok(text) => {
620        set(self.body, Some(text));
621        set(self.error, None);
622        set(self.loading, false);
623      }
624      Err(error) => self.failed(error),
625    }
626  }
627
628  /// Shows `error`, logged once while the same one stays.
629  fn failed(&self, error: HttpError) {
630    if untrack(|| self.error.try_with(|old| old.as_ref() != Some(&error))) == Some(true) {
631      tracing::info!(error = error.message, "a request failed");
632    }
633    set(self.error, Some(error));
634    set(self.loading, false);
635  }
636}
637
638/// Sets `signal` unless it holds `value` or is gone, reading it untracked.
639fn set<T: PartialEq + 'static>(signal: Signal<T>, value: T) {
640  if untrack(|| signal.try_with(|old| *old != value)) == Some(true) {
641    signal.try_set(value);
642  }
643}