1use std::collections::HashSet;
2use std::fmt::{self, Display};
3use std::sync::{Arc, Mutex, OnceLock};
4use std::time::Duration;
5
6use bytes::Bytes;
7use http::request::Parts;
8use http::{Method, StatusCode};
9use hyper::body::{Body, Incoming};
10use serde::Deserialize;
11use serde_json::{Map, Value};
12use sofie_auth::SigningKeys;
13use sofie_core::{Error, Result, unpoisoned};
14use sofie_storage::{ContentType, Put, Storage, StorageKey};
15use tokio::time::Instant;
16
17use crate::Serving;
18use crate::answer::{self, Response};
19use crate::caller::Caller;
20use crate::deadline::{Deadline, Paused};
21use crate::multipart::{self, Multipart, PartHead, Stop};
22use crate::pipeline::{Asked, RequestBody, discard_rest, read_body, too_large};
23use crate::routes::BODY_LIMIT;
24use crate::target::Target;
25use crate::urlencoded::Encoding;
26
27const TEXT_LIMIT: usize = BODY_LIMIT;
30const PART_LIMIT: usize = 100;
31const FILENAME_LIMIT: usize = 255;
32const SEAL: &str = "seal";
33const SEAL_LIFETIME: Duration = Duration::from_mins(2);
35const FORM_TIME: Duration = Duration::from_mins(10);
37
38tokio::task_local! {
39 static TAKEN: Arc<Mutex<Taken>>;
41}
42
43#[derive(Default)]
44struct Taken {
45 keys: HashSet<StorageKey>,
46 committed: bool,
47}
48
49pub(crate) fn committed() {
51 let _outside_a_request = TAKEN.try_with(|taken| unpoisoned::lock(taken).committed = true);
53}
54
55#[derive(Clone, PartialEq, Eq)]
58pub struct Upload {
59 key: StorageKey,
60 filename: String,
61 content_type: ContentType,
62 size: u64,
63}
64
65impl Upload {
66 #[must_use]
67 pub fn key(&self) -> &StorageKey {
68 &self.key
69 }
70
71 #[must_use]
74 pub fn filename(&self) -> &str {
75 &self.filename
76 }
77
78 #[must_use]
80 pub fn content_type(&self) -> &ContentType {
81 &self.content_type
82 }
83
84 #[must_use]
85 pub fn size(&self) -> u64 {
86 self.size
87 }
88
89 fn sealed(&self, expires: u64) -> String {
90 format!(
91 "{}\n{}\n{}\n{}\n{expires}",
92 self.key, self.content_type, self.size, self.filename
93 )
94 }
95
96 fn to_json(&self) -> Result<Value> {
97 self.sealed_json(seconds_now() + SEAL_LIFETIME.as_secs())
98 }
99
100 fn sealed_json(&self, expires: u64) -> Result<Value> {
101 let sealed = self.sealed(expires);
102 let seal = seal_keys()?.sign(sofie_auth::Signed::UploadSeal, sealed.as_bytes());
103 Ok(serde_json::json!({
104 "key": self.key,
105 "filename": self.filename,
106 "content_type": self.content_type,
107 "size": self.size,
108 "expires": expires,
109 SEAL: seal,
110 }))
111 }
112}
113
114impl fmt::Debug for Upload {
115 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
116 formatter
117 .debug_struct("Upload")
118 .field("key", &self.key)
119 .field("content_type", &self.content_type)
120 .field("size", &self.size)
121 .finish_non_exhaustive()
122 }
123}
124
125#[derive(Deserialize)]
126#[serde(deny_unknown_fields)]
127struct Received {
128 key: StorageKey,
129 filename: String,
130 content_type: ContentType,
131 size: u64,
132 expires: u64,
133 seal: String,
134}
135
136impl<'de> Deserialize<'de> for Upload {
137 fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
138 let not_received = || serde::de::Error::custom("is not a file sent with this form");
139 let received = Received::deserialize(deserializer).map_err(|_| not_received())?;
140 let upload = Self {
141 key: received.key,
142 filename: received.filename,
143 content_type: received.content_type,
144 size: received.size,
145 };
146 let keys = seal_keys().map_err(Error::into_deserialize_error)?;
147 let sealed = upload.sealed(received.expires);
148 if received.expires <= seconds_now()
149 || !keys.verifies(
150 sofie_auth::Signed::UploadSeal,
151 sealed.as_bytes(),
152 &received.seal,
153 )
154 {
155 return Err(not_received());
156 }
157 let _outside_a_request = TAKEN.try_with(|taken| {
159 unpoisoned::lock(taken).keys.insert(upload.key.clone());
160 });
161 Ok(upload)
162 }
163}
164
165#[cfg(feature = "engine")]
166impl sofie_values::Present for Upload {
167 const INPUT: sofie_values::Input = sofie_values::Input::new(sofie_values::Control::File);
168 const SEARCH: sofie_values::Search = sofie_values::Search::Skipped;
169
170 fn csv(&self) -> String {
171 self.filename.clone()
172 }
173}
174
175#[cfg(feature = "engine")]
176pub(crate) fn is_upload(value: &Value) -> bool {
177 value.get(SEAL).is_some()
178}
179
180fn seconds_now() -> u64 {
181 std::time::SystemTime::now()
182 .duration_since(std::time::UNIX_EPOCH)
183 .map_or(0, |since| since.as_secs())
184}
185
186fn seal_keys() -> Result<&'static SigningKeys> {
188 static KEYS: OnceLock<SigningKeys> = OnceLock::new();
189 if let Some(keys) = KEYS.get() {
190 return Ok(keys);
191 }
192 let generated = SigningKeys::generate()?;
193 Ok(KEYS.get_or_init(|| generated))
194}
195
196#[derive(Default)]
200pub(crate) struct Stored {
201 storage: Option<Storage>,
202 keys: Vec<StorageKey>,
203 taken: Arc<Mutex<Taken>>,
204}
205
206impl Stored {
207 pub(crate) async fn taking<F: Future>(&self, handler: F) -> F::Output {
208 TAKEN.scope(Arc::clone(&self.taken), handler).await
209 }
210
211 pub(crate) async fn settle(mut self) {
212 let mut keys = std::mem::take(&mut self.keys);
213 {
214 let taken = unpoisoned::lock(&self.taken);
215 if taken.committed {
216 keys.retain(|key| !taken.keys.contains(key));
217 }
218 }
219 if let Some(storage) = &self.storage {
220 discard(storage, &keys).await;
221 }
222 }
223}
224
225impl Drop for Stored {
226 fn drop(&mut self) {
227 let keys = std::mem::take(&mut self.keys);
228 if let Some(storage) = self.storage.take()
229 && !keys.is_empty()
230 {
231 tokio::spawn(async move { discard(&storage, &keys).await });
232 }
233 }
234}
235
236async fn discard(storage: &Storage, keys: &[StorageKey]) {
237 for key in keys {
238 if let Err(error) = storage.delete(key).await {
239 tracing::warn!(error.kind = "storage", error = %error, "deleting an unused upload failed");
240 }
241 }
242}
243
244pub(crate) async fn read(
249 asked: Asked<'_>,
250 target: &Target,
251 serving: &Serving,
252 mut body: Incoming,
253 limit: usize,
254) -> Result<RequestBody, Box<Response>> {
255 let (request, caller) = (asked.request, asked.caller);
256 let boundary = multipart::boundary(&request.headers).filter(|_| !is_webhook(target));
257 let Some(boundary) = boundary else {
258 let limit = match target.admission.rules.file_limit {
259 Some(_) => limit.min(TEXT_LIMIT),
260 None => limit,
261 };
262 let body = read_body(&mut body, limit, caller).await?;
263 return Ok(RequestBody::plain(body, &request.headers));
264 };
265 let refused = |error: Error| Box::new(answer::failure(error, caller));
266 let boundary = match boundary {
267 Ok(boundary) => boundary,
268 Err(stop) => return Err(refused_unread(body, malformed(&stop), caller).await),
269 };
270 let storage = serving
271 .storage
272 .as_ref()
273 .filter(|_| takes_files(request, target));
274 let rules = &target.admission.rules;
275 let file_limit = rules.file_limit.unwrap_or(limit);
276 let limit = match (rules.body_limit, rules.file_limit) {
277 (None, Some(file_limit)) => file_limit.saturating_add(TEXT_LIMIT),
278 _ => limit,
279 };
280 let mut form = Form::new(body, &boundary, limit, storage, file_limit, asked.deadline);
281 let read = form.read().await;
282 match read {
283 Ok(()) => {
284 let encoded = |fields: &Map<String, Value>| {
285 serde_json::to_vec(fields)
286 .map(Bytes::from)
287 .map_err(|error| refused(Error::internal("encoding a form's fields", error)))
288 };
289 Ok(RequestBody {
290 body: encoded(&form.fields)?,
291 #[cfg(feature = "idempotency")]
292 fingerprint: encoded(&as_sent(&form.fields))?,
293 #[cfg(feature = "idempotency")]
294 form_key: form.idempotency_key,
295 encoding: Encoding::Json,
296 stored: form.stored,
297 })
298 }
299 Err(failure) => {
300 if matches!(failure, Failure::TakesNoFiles)
301 && serving.storage.is_none()
302 && target.admission.rules.file_limit.is_some()
303 {
304 tracing::warn!(
305 error.kind = "storage",
306 "a form sent a file, but the app has no storage; see `Serving::storage`"
307 );
308 }
309 form.stored.settle().await;
310 Err(Box::new(failure.answer(form.multipart, caller).await))
311 }
312 }
313}
314
315#[cfg(feature = "idempotency")]
317fn as_sent(fields: &Map<String, Value>) -> Map<String, Value> {
318 let mut sent = fields.clone();
319 for value in sent.values_mut() {
320 if let Value::Object(upload) = value
321 && upload.contains_key(SEAL)
322 {
323 upload.retain(|name, _| matches!(name.as_str(), "filename" | "content_type" | "size"));
324 }
325 }
326 sent
327}
328
329fn is_webhook(target: &Target) -> bool {
332 #[cfg(feature = "webhooks")]
333 return target.admission.rules.verify.is_some();
334 #[cfg(not(feature = "webhooks"))]
335 {
336 let _ = target;
337 false
338 }
339}
340
341fn takes_files(request: &Parts, target: &Target) -> bool {
342 #[cfg(feature = "jobs")]
343 if target.admission.rules.queues {
344 return false;
345 }
346 target.admission.rules.file_limit.is_some()
347 && request.method != Method::GET
348 && request.method != Method::HEAD
349}
350
351async fn refused_unread(mut body: Incoming, error: Error, caller: Caller<'_>) -> Box<Response> {
352 discard_rest(&mut body).await;
353 Box::new(answer::failure(error, caller))
354}
355
356enum Failure {
357 Stopped(Stop),
358 Refused(Error),
359 TakesNoFiles,
361 TooSlow,
362}
363
364impl From<Stop> for Failure {
365 fn from(stop: Stop) -> Self {
366 Self::Stopped(stop)
367 }
368}
369
370impl From<Error> for Failure {
371 fn from(error: Error) -> Self {
372 Self::Refused(error)
373 }
374}
375
376impl Failure {
377 async fn answer(self, multipart: Multipart<Incoming>, caller: Caller<'_>) -> Response {
379 discard_rest(&mut multipart.into_body()).await;
380 match self {
381 Self::Stopped(Stop::TooLarge) => too_large(caller),
382 Self::Stopped(stop) => answer::failure(malformed(&stop), caller),
383 Self::Refused(error) => answer::failure(error, caller),
384 Self::TakesNoFiles => answer::refusal(
385 StatusCode::UNSUPPORTED_MEDIA_TYPE,
386 "This address does not take files.",
387 caller,
388 ),
389 Self::TooSlow => answer::refusal(
390 StatusCode::REQUEST_TIMEOUT,
391 "The form took too long to arrive.",
392 caller,
393 ),
394 }
395 }
396}
397
398fn malformed(stop: &Stop) -> Error {
399 let reason = match stop {
400 Stop::TooLarge => "it is too large",
401 Stop::Malformed(reason) => reason,
402 Stop::Unread(reason) => reason,
403 };
404 Error::BadRequest(format!("the form cannot be read: {reason}"))
405}
406
407struct Form<'s, B = Incoming> {
408 multipart: Multipart<B>,
409 storage: Option<&'s Storage>,
410 file_limit: usize,
411 fields: Map<String, Value>,
412 #[cfg(feature = "idempotency")]
413 idempotency_key: Option<String>,
414 stored: Stored,
415 deadline: &'s Deadline,
416 paused: Option<Paused<'s>>,
417 ends: Instant,
418}
419
420impl<'s, B: Body<Data = Bytes> + Unpin> Form<'s, B>
421where
422 B::Error: Display,
423{
424 fn new(
425 body: B,
426 boundary: &str,
427 limit: usize,
428 storage: Option<&'s Storage>,
429 file_limit: usize,
430 deadline: &'s Deadline,
431 ) -> Self {
432 Self {
433 multipart: Multipart::new(body, boundary, limit),
434 storage,
435 file_limit,
436 fields: Map::new(),
437 #[cfg(feature = "idempotency")]
438 idempotency_key: None,
439 stored: Stored {
440 storage: storage.cloned(),
441 keys: Vec::new(),
442 taken: Arc::default(),
443 },
444 deadline,
445 paused: None,
446 ends: Instant::now() + FORM_TIME,
447 }
448 }
449
450 async fn read(&mut self) -> Result<(), Failure> {
451 let read = self.parts().await;
452 self.paused = None;
453 read
454 }
455
456 async fn next_part(&mut self) -> Result<Option<PartHead>, Failure> {
457 within(self.ends, self.multipart.next_part()).await
458 }
459
460 async fn next_chunk(&mut self) -> Result<Option<Bytes>, Failure> {
461 within(self.ends, self.multipart.next_chunk()).await
462 }
463
464 async fn parts(&mut self) -> Result<(), Failure> {
465 let mut parts = 0;
466 let mut text = 0;
467 while let Some(head) = self.next_part().await? {
468 parts += 1;
469 if parts > PART_LIMIT {
470 return Err(Error::BadRequest(format!(
471 "the form has more than {PART_LIMIT} fields"
472 ))
473 .into());
474 }
475 match head.filename {
476 Some(_) => self.file(head).await?,
477 None => text += self.text(&head.name, TEXT_LIMIT - text).await?,
478 }
479 }
480 Ok(())
481 }
482
483 async fn text(&mut self, name: &str, room: usize) -> Result<usize, Failure> {
484 let mut value = Vec::new();
485 while let Some(chunk) = self.next_chunk().await? {
486 if value.len() + chunk.len() > room {
487 return Err(Stop::TooLarge.into());
488 }
489 value.extend_from_slice(&chunk);
490 }
491 let length = value.len();
492 let value = String::from_utf8(value)
493 .map_err(|_| Error::BadRequest("the form is not UTF-8".to_owned()))?;
494 if name == sofie_core::IDEMPOTENCY_KEY_FIELD {
495 #[cfg(feature = "idempotency")]
496 {
497 self.idempotency_key = Some(value);
498 }
499 } else if !value.is_empty() {
500 crate::dotted_name::insert(&mut self.fields, name, value)?;
501 }
502 Ok(length)
503 }
504
505 async fn file(&mut self, head: PartHead) -> Result<(), Failure> {
506 let name = head.name;
507 if !sofie_core::is_field_name(&name) || self.fields.contains_key(&name) {
508 return Err(Error::BadRequest(format!(
509 "the form's file field `{name}` is not a plain field name or is sent twice"
510 ))
511 .into());
512 }
513 if self.storage.is_some() && self.paused.is_none() {
514 self.paused = Some(self.deadline.paused());
515 }
516 let Some(first) = self.next_chunk().await? else {
517 return Ok(());
518 };
519 let declared = head
520 .content_type
521 .as_deref()
522 .map_or_else(|| Ok(ContentType::octet_stream()), ContentType::parse)
523 .map_err(|_| Error::invalid(&name, "does not declare a media type"))?;
524 let Some(storage) = self.storage else {
525 return Err(Failure::TakesNoFiles);
526 };
527 let mut put = storage.put().await?;
528 let written = self.write(&mut put, first, &declared, &name).await;
529 match written {
530 Ok(()) => {
531 let size = put.size();
532 let key = put.finish().await?;
533 self.stored.keys.push(key.clone());
534 let upload = Upload {
535 key,
536 filename: sanitized(head.filename.as_deref().unwrap_or_default()),
537 content_type: declared,
538 size,
539 };
540 self.fields.insert(name, upload.to_json()?);
541 Ok(())
542 }
543 Err(failure) => {
544 if let Err(error) = put.abort().await {
545 tracing::warn!(error.kind = "storage", error = %error, "discarding a refused upload failed");
546 }
547 Err(failure)
548 }
549 }
550 }
551
552 async fn write(
553 &mut self,
554 put: &mut Put,
555 first: Bytes,
556 declared: &ContentType,
557 name: &str,
558 ) -> Result<(), Failure> {
559 let mut start = Vec::with_capacity(ContentType::SNIFFED_LENGTH);
560 let mut chunk = Some(first);
561 while let Some(data) = chunk {
562 let size = put.size() + u64::try_from(data.len()).unwrap_or(u64::MAX);
563 if size > u64::try_from(self.file_limit).unwrap_or(u64::MAX) {
564 return Err(Error::invalid(
565 name,
566 format!("must be at most {} bytes", self.file_limit),
567 )
568 .into());
569 }
570 let missing = ContentType::SNIFFED_LENGTH.saturating_sub(start.len());
571 start.extend_from_slice(&data[..missing.min(data.len())]);
572 put.write(data).await?;
573 chunk = self.next_chunk().await?;
574 }
575 if !declared.matches(&start) {
576 return Err(
577 Error::invalid(name, format!("is not the {declared} it claims to be")).into(),
578 );
579 }
580 Ok(())
581 }
582}
583
584async fn within<T>(
585 ends: Instant,
586 reading: impl Future<Output = Result<T, Stop>>,
587) -> Result<T, Failure> {
588 let read = tokio::time::timeout_at(ends, reading).await;
589 read.map_err(|_| Failure::TooSlow)?.map_err(Failure::from)
590}
591
592fn sanitized(filename: &str) -> String {
595 let base = filename.rsplit(['/', '\\']).next().unwrap_or_default();
596 let kept: String = base
597 .chars()
598 .filter(|character| {
599 !sofie_core::misleads_display(*character) && !"\"<>:|?*".contains(*character)
600 })
601 .collect();
602 let trimmed =
603 kept.trim_matches(|character: char| character == '.' || character.is_whitespace());
604 let mut end = trimmed.len().min(FILENAME_LIMIT);
605 while !trimmed.is_char_boundary(end) {
606 end -= 1;
607 }
608 match &trimmed[..end] {
609 "" => "file".to_owned(),
610 kept => kept.to_owned(),
611 }
612}
613
614#[cfg(test)]
615mod tests {
616 use std::convert::Infallible;
617 use std::pin::Pin;
618 use std::task::{Context, Poll};
619
620 use hyper::body::{Body, Frame};
621 use tokio::sync::mpsc;
622 use tokio::time::sleep;
623
624 use super::*;
625
626 const BOUNDARY: &str = "trickle";
627
628 struct Trickle(mpsc::Receiver<Bytes>);
629
630 impl Body for Trickle {
631 type Data = Bytes;
632 type Error = Infallible;
633
634 fn poll_frame(
635 mut self: Pin<&mut Self>,
636 context: &mut Context<'_>,
637 ) -> Poll<Option<Result<Frame<Bytes>, Infallible>>> {
638 self.0
639 .poll_recv(context)
640 .map(|chunk| chunk.map(|bytes| Ok(Frame::data(bytes))))
641 }
642 }
643
644 fn trickled(head: &'static str, bytes: usize, pause: Duration) -> Trickle {
646 let (sender, receiver) = mpsc::channel(1);
647 tokio::spawn(async move {
648 let opening = format!("--{BOUNDARY}\r\n{head}\r\n\r\n");
649 sender.send(Bytes::from(opening)).await.unwrap();
650 for _ in 0..bytes {
651 sleep(pause).await;
652 let _closed = sender.send(Bytes::from_static(b"x")).await;
653 }
654 let closing = format!("\r\n--{BOUNDARY}--\r\n");
655 let _closed = sender.send(Bytes::from(closing)).await;
656 });
657 Trickle(receiver)
658 }
659
660 async fn read_within_deadline(body: Trickle) -> Option<Result<(), Failure>> {
661 let directory = std::env::temp_dir().join(format!(
662 "sofie-http-trickle-{}",
663 StorageKey::generate().unwrap()
664 ));
665 let keys = Arc::new(SigningKeys::generate().unwrap());
666 let storage = Storage::new(sofie_storage::Local::new(&directory, keys).unwrap());
667 let deadline = Deadline::default();
668 let mut form = Form::new(body, BOUNDARY, 1024, Some(&storage), 1024, &deadline);
669 let read = deadline.within(form.read()).await;
670 drop(form);
671 std::fs::remove_dir_all(&directory).unwrap();
672 read
673 }
674
675 #[tokio::test(start_paused = true)]
676 async fn only_a_file_part_pauses_the_deadline_and_then_for_ten_minutes_at_most() {
677 let text = r#"Content-Disposition: form-data; name="title""#;
678 let file = r#"Content-Disposition: form-data; name="photo"; filename="a.bin""#;
679 let under_the_silence_timeout = Duration::from_secs(50);
680
681 let slow_text = read_within_deadline(trickled(text, 2, under_the_silence_timeout)).await;
682 let slow_file = read_within_deadline(trickled(file, 4, under_the_silence_timeout)).await;
683 let endless_file =
684 read_within_deadline(trickled(file, 15, under_the_silence_timeout)).await;
685
686 assert!(slow_text.is_none());
687 assert!(matches!(slow_file, Some(Ok(()))));
688 assert!(matches!(endless_file, Some(Err(Failure::TooSlow))));
689 }
690
691 #[test]
692 fn a_filename_keeps_only_its_last_safe_segment() {
693 for (sent, kept) in [
694 ("photo.png", "photo.png"),
695 ("C:\\Users\\ada\\photo.png", "photo.png"),
696 ("../../etc/passwd", "passwd"),
697 ("..", "file"),
698 ("", "file"),
699 (" .hidden. ", "hidden"),
700 ("a\"b<c>|?*\u{0}\n.txt", "abc.txt"),
701 ("Jürgen's résumé.pdf", "Jürgen's résumé.pdf"),
702 ] {
703 assert_eq!(sanitized(sent), kept, "{sent:?}");
704 }
705 let long = "é".repeat(200);
706 assert_eq!(sanitized(&long).len(), 254);
707 }
708
709 fn upload() -> Upload {
710 Upload {
711 key: StorageKey::generate().unwrap(),
712 filename: "photo.png".to_owned(),
713 content_type: ContentType::parse("image/png").unwrap(),
714 size: 12,
715 }
716 }
717
718 #[test]
719 fn only_an_upload_this_process_sealed_is_read_back() {
720 let upload = upload();
721 let json = upload.to_json().unwrap();
722
723 assert_eq!(
724 serde_json::from_value::<Upload>(json.clone()).unwrap(),
725 upload
726 );
727 for (field, forged) in [
728 ("key", Value::from(StorageKey::generate().unwrap().as_str())),
729 ("size", Value::from(13)),
730 ("expires", Value::from(u64::MAX)),
731 ("content_type", Value::from("text/html")),
732 ("filename", Value::from("other.png")),
733 (SEAL, Value::from("AAAA")),
734 ] {
735 let mut forged_json = json.clone();
736 forged_json[field] = forged;
737 assert!(
738 serde_json::from_value::<Upload>(forged_json).is_err(),
739 "{field}"
740 );
741 }
742 assert!(serde_json::from_value::<Upload>(Value::from("photo.png")).is_err());
743 let expired = upload.sealed_json(seconds_now() - 1).unwrap();
744 assert!(serde_json::from_value::<Upload>(expired).is_err());
745 }
746}