1pub(crate) use helpers::{kb_not_found, lookup_kb};
6
7mod browse;
8mod events;
9mod health;
10mod helpers;
11mod kbs;
12mod llms;
13mod objects;
14mod well_known;
15
16use crate::middleware::auth_middleware;
17use crate::state::AppState;
18use axum::Router;
19use axum::extract::{DefaultBodyLimit, Request};
20use axum::handler::Handler;
21use axum::http::HeaderName;
22use axum::middleware::from_fn_with_state;
23use axum::routing::get;
24use tower::ServiceBuilder;
25use tower_http::request_id::{
26 MakeRequestId, PropagateRequestIdLayer, RequestId, SetRequestIdLayer,
27};
28use tower_http::trace::TraceLayer;
29use uuid::Uuid;
30
31use browse::{browse_path, browse_root};
32use events::subscribe_events;
33use health::{healthz, readyz};
34use kbs::{list_kbs, list_objects};
35use llms::llms_txt;
36use objects::{delete_object, get_object, head_object, patch_object, post_object, put_object};
37use well_known::protected_resource_metadata;
38
39macro_rules! api_routes {
52 ($($route:ident / $matched:ident => $suffix:literal,)+) => {
53 $(
54 pub(crate) const $route: &str = $suffix;
55 pub(crate) const $matched: &str = concat!("/api/v1", $suffix);
56 )+
57 };
58}
59
60pub const API_V1_PREFIX: &str = "/api/v1";
62
63pub const BROWSE_PREFIX: &str = "/browse";
65
66api_routes! {
67 ROUTE_KBS / MATCHED_KBS => "/knowledgebases",
68 ROUTE_KB / MATCHED_KB => "/knowledgebases/{kb_slug}",
69 ROUTE_KB_SEARCH / MATCHED_KB_SEARCH => "/knowledgebases/{kb_slug}/search",
70 ROUTE_KB_EVENTS / MATCHED_KB_EVENTS => "/knowledgebases/{kb_slug}/events",
71 ROUTE_KB_OBJECT / MATCHED_KB_OBJECT => "/knowledgebases/{kb_slug}/{*object_path}",
72}
73
74pub const MAX_BODY_BYTES: u64 = 16 * 1024 * 1024;
76
77#[derive(Clone, Copy, Default)]
79pub struct MakeRequestUuidV7;
80
81impl MakeRequestId for MakeRequestUuidV7 {
82 fn make_request_id<B>(&mut self, _req: &Request<B>) -> Option<RequestId> {
83 let id = Uuid::now_v7().to_string();
84 let hv = id.parse().ok()?;
85 Some(RequestId::new(hv))
86 }
87}
88
89pub fn build_router(state: AppState) -> Router {
91 let request_id_header = HeaderName::from_static("x-request-id");
92 let api_routes = Router::new()
93 .route(ROUTE_KBS, get(list_kbs))
94 .route(ROUTE_KB, get(list_objects))
95 .route(
96 ROUTE_KB_SEARCH,
97 axum::routing::post(crate::search_route::search_kb).layer(
98 axum::extract::DefaultBodyLimit::max(crate::search_route::SEARCH_BODY_MAX_BYTES),
99 ),
100 )
101 .route(ROUTE_KB_EVENTS, get(subscribe_events))
102 .route(
103 ROUTE_KB_OBJECT,
104 get(get_object)
105 .head(head_object)
106 .put(put_object)
107 .delete(delete_object)
108 .patch(patch_object.layer(DefaultBodyLimit::disable()))
109 .post(post_object.layer(DefaultBodyLimit::disable())),
110 )
111 .layer(
112 ServiceBuilder::new()
113 .layer(DefaultBodyLimit::max(helpers::body_limit_usize(
114 MAX_BODY_BYTES,
115 )))
116 .layer(from_fn_with_state(state.clone(), auth_middleware)),
117 )
118 .with_state(state.clone());
119
120 Router::new()
124 .route("/healthz", get(healthz))
125 .route("/readyz", get(readyz))
126 .route("/llms.txt", get(llms_txt))
127 .route(
128 "/.well-known/oauth-protected-resource",
129 get(protected_resource_metadata),
130 )
131 .route(BROWSE_PREFIX, get(browse_root))
132 .route(&format!("{BROWSE_PREFIX}/"), get(browse_root))
133 .route(&format!("{BROWSE_PREFIX}/{{*path}}"), get(browse_path))
134 .nest(API_V1_PREFIX, api_routes)
135 .layer(
136 ServiceBuilder::new()
137 .layer(SetRequestIdLayer::new(
138 request_id_header.clone(),
139 MakeRequestUuidV7,
140 ))
141 .layer(PropagateRequestIdLayer::new(request_id_header))
142 .layer(TraceLayer::new_for_http()),
143 )
144 .with_state(state)
149}
150
151#[cfg(test)]
152mod route_constants {
153 use super::{
154 API_V1_PREFIX, MATCHED_KB, MATCHED_KB_EVENTS, MATCHED_KB_OBJECT, MATCHED_KB_SEARCH,
155 MATCHED_KBS, ROUTE_KB, ROUTE_KB_EVENTS, ROUTE_KB_OBJECT, ROUTE_KB_SEARCH, ROUTE_KBS,
156 };
157
158 #[test]
162 fn matched_paths_are_the_nested_routes_under_the_mount_point() {
163 assert_eq!(API_V1_PREFIX, "/api/v1");
164 for (route, matched) in [
165 (ROUTE_KBS, MATCHED_KBS),
166 (ROUTE_KB, MATCHED_KB),
167 (ROUTE_KB_SEARCH, MATCHED_KB_SEARCH),
168 (ROUTE_KB_EVENTS, MATCHED_KB_EVENTS),
169 (ROUTE_KB_OBJECT, MATCHED_KB_OBJECT),
170 ] {
171 assert_eq!(matched, format!("{API_V1_PREFIX}{route}"));
172 }
173 }
174}
175
176#[cfg(test)]
177mod patch_route {
178 use super::*;
179 use async_trait::async_trait;
180 use axum::body::{Body, to_bytes};
181 use axum::http::StatusCode;
182 use axum::response::Response;
183 use bytes::Bytes;
184 use notedthat_core::{
185 ByteRange, ConditionalHeaders, KbManifest, KbSlug, ListResponse, ObjectMeta, ObjectPath,
186 ObjectRead, PutOutcome, Storage, StorageError,
187 };
188 use notedthat_indexer::IndexEvent;
189 use std::collections::BTreeMap;
190 use std::sync::Arc;
191 use tower::util::ServiceExt;
192
193 const KB: &str = "notes";
194 const OBJECT_PATH: &str = "patch.md";
195 const TOKEN: &str = "test-token-abc";
196
197 async fn router_with_object(
198 initial_body: &'static [u8],
199 max_patchable_size: u64,
200 ) -> (axum::Router, String) {
201 let kb = KbSlug::try_new(KB).unwrap();
202 let object_path = ObjectPath::try_from_str(OBJECT_PATH).unwrap();
203 let storage = Arc::new(crate::testing::InMemoryStorage::default());
204 let outcome = storage
205 .put_object(
206 &kb,
207 &object_path,
208 Bytes::from_static(initial_body),
209 Some("text/markdown"),
210 ConditionalHeaders::default(),
211 )
212 .await
213 .unwrap();
214
215 let (indexer_tx, _rx) = tokio::sync::mpsc::channel(16);
216 let mut kbs = BTreeMap::new();
217 kbs.insert(KB.to_string(), kb);
218 let router = build_router(AppState {
219 storage,
220 access_policies: Arc::new(notedthat_core::signed_in_policies(&kbs)),
221 declared_kbs: Arc::new(kbs),
222 authenticator: Arc::new(notedthat_core::Authenticator::new(TOKEN)),
223 max_body_size: MAX_BODY_BYTES,
224 max_patchable_size,
225 indexer_tx,
226 searcher: Arc::new(crate::testing::NoopSearcher),
227 events: None,
228 });
229
230 (router, outcome.etag.unwrap())
231 }
232
233 async fn object_with_etag(
234 storage: &crate::testing::InMemoryStorage,
235 kb: &KbSlug,
236 body: &'static [u8],
237 ) -> String {
238 storage
239 .put_object(
240 kb,
241 &ObjectPath::try_from_str(OBJECT_PATH).unwrap(),
242 Bytes::from_static(body),
243 Some("text/markdown"),
244 ConditionalHeaders::default(),
245 )
246 .await
247 .unwrap()
248 .etag
249 .unwrap()
250 }
251
252 fn router_with_storage(
253 storage: Arc<dyn Storage>,
254 kb: KbSlug,
255 max_patchable_size: u64,
256 indexer_tx: tokio::sync::mpsc::Sender<IndexEvent>,
257 ) -> axum::Router {
258 let mut kbs = BTreeMap::new();
259 kbs.insert(KB.to_string(), kb);
260 build_router(AppState {
261 storage,
262 access_policies: Arc::new(notedthat_core::signed_in_policies(&kbs)),
263 declared_kbs: Arc::new(kbs),
264 authenticator: Arc::new(notedthat_core::Authenticator::new(TOKEN)),
265 max_body_size: MAX_BODY_BYTES,
266 max_patchable_size,
267 indexer_tx,
268 searcher: Arc::new(crate::testing::NoopSearcher),
269 events: None,
270 })
271 }
272
273 async fn patch_request(
274 router: axum::Router,
275 header_name: &'static str,
276 header_value: &str,
277 if_match: Option<&str>,
278 body: Bytes,
279 ) -> Response {
280 let mut builder = Request::builder()
281 .method("PATCH")
282 .uri(format!("/api/v1/knowledgebases/{KB}/{OBJECT_PATH}"))
283 .header("authorization", format!("Bearer {TOKEN}"))
284 .header(header_name, header_value);
285 if let Some(etag) = if_match {
286 builder = builder.header(axum::http::header::IF_MATCH, etag);
287 }
288
289 router
290 .oneshot(builder.body(Body::from(body)).unwrap())
291 .await
292 .unwrap()
293 }
294
295 async fn assert_error_code(response: Response, expected_status: StatusCode, expected: &str) {
296 assert_eq!(response.status(), expected_status);
297 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
298 let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
299 assert_eq!(json["error"], expected);
300 }
301
302 #[tokio::test]
303 async fn bytes_content_range_returns_ok_with_etag_and_location() {
304 let (router, etag) = router_with_object(b"0123456789abcdefghij", MAX_BODY_BYTES).await;
305
306 let response = patch_request(
307 router,
308 "content-range",
309 "bytes 0-9/*",
310 Some(&etag),
311 Bytes::from_static(b"ABCDEFGHIJ"),
312 )
313 .await;
314
315 assert_eq!(response.status(), StatusCode::OK);
316 assert!(response.headers().get(axum::http::header::ETAG).is_some());
317 assert_eq!(
318 response
319 .headers()
320 .get(axum::http::header::LOCATION)
321 .unwrap(),
322 &format!("/api/v1/knowledgebases/{KB}/{OBJECT_PATH}")
323 );
324 assert!(
325 response
326 .headers()
327 .get(axum::http::header::CONTENT_RANGE)
328 .is_none()
329 );
330 assert!(response.headers().get("nt-patch-mode").is_none());
331 }
332
333 #[tokio::test]
334 async fn lines_content_range_returns_ok() {
335 let (router, etag) = router_with_object(b"one\ntwo\nthree\nfour\n", MAX_BODY_BYTES).await;
336
337 let response = patch_request(
338 router,
339 "content-range",
340 "lines 2-3/*",
341 Some(&etag),
342 Bytes::from_static(b"TWO\nTHREE\n"),
343 )
344 .await;
345
346 assert_eq!(response.status(), StatusCode::OK);
347 }
348
349 #[tokio::test]
350 async fn append_mode_without_if_match_returns_ok() {
351 let (router, _etag) = router_with_object(b"one\n", MAX_BODY_BYTES).await;
352
353 let response = patch_request(
354 router,
355 "nt-patch-mode",
356 "append",
357 None,
358 Bytes::from_static(b"two\n"),
359 )
360 .await;
361
362 assert_eq!(response.status(), StatusCode::OK);
363 }
364
365 #[tokio::test]
366 async fn append_mode_with_if_match_returns_ok() {
367 let (router, etag) = router_with_object(b"one\n", MAX_BODY_BYTES).await;
368
369 let response = patch_request(
370 router,
371 "nt-patch-mode",
372 "append",
373 Some(&etag),
374 Bytes::from_static(b"two\n"),
375 )
376 .await;
377
378 assert_eq!(response.status(), StatusCode::OK);
379 }
380
381 #[tokio::test]
382 async fn bytes_content_range_without_if_match_returns_invalid_request() {
383 let (router, _etag) = router_with_object(b"0123456789", MAX_BODY_BYTES).await;
384
385 let response = patch_request(
386 router,
387 "content-range",
388 "bytes 0-1/*",
389 None,
390 Bytes::from_static(b"AB"),
391 )
392 .await;
393
394 assert_error_code(response, StatusCode::BAD_REQUEST, "invalid_request").await;
395 }
396
397 #[tokio::test]
398 async fn if_match_star_returns_invalid_request() {
399 let (router, _etag) = router_with_object(b"0123456789", MAX_BODY_BYTES).await;
400
401 let response = patch_request(
402 router,
403 "content-range",
404 "bytes 0-1/*",
405 Some("*"),
406 Bytes::from_static(b"AB"),
407 )
408 .await;
409
410 assert_error_code(response, StatusCode::BAD_REQUEST, "invalid_request").await;
411 }
412
413 #[tokio::test]
414 async fn multi_value_if_match_returns_invalid_request() {
415 let (router, etag) = router_with_object(b"0123456789", MAX_BODY_BYTES).await;
416
417 let response = patch_request(
418 router,
419 "content-range",
420 "bytes 0-1/*",
421 Some(&format!("{etag}, \"other\"")),
422 Bytes::from_static(b"AB"),
423 )
424 .await;
425
426 assert_error_code(response, StatusCode::BAD_REQUEST, "invalid_request").await;
427 }
428
429 #[tokio::test]
430 async fn nonexistent_object_returns_not_found() {
431 let (router, etag) = router_with_object(b"0123456789", MAX_BODY_BYTES).await;
432
433 let response = router
434 .oneshot(
435 Request::builder()
436 .method("PATCH")
437 .uri(format!("/api/v1/knowledgebases/{KB}/missing.md"))
438 .header("authorization", format!("Bearer {TOKEN}"))
439 .header("content-range", "bytes 0-1/*")
440 .header(axum::http::header::IF_MATCH, etag)
441 .body(Body::from(Bytes::from_static(b"AB")))
442 .unwrap(),
443 )
444 .await
445 .unwrap();
446
447 assert_error_code(response, StatusCode::NOT_FOUND, "not_found").await;
448 }
449
450 #[tokio::test]
451 async fn body_larger_than_max_patchable_size_returns_payload_too_large() {
452 let (router, _etag) = router_with_object(b"one\n", 4).await;
453
454 let response = patch_request(
455 router,
456 "nt-patch-mode",
457 "append",
458 None,
459 Bytes::from_static(b"abcde"),
460 )
461 .await;
462
463 assert_error_code(response, StatusCode::PAYLOAD_TOO_LARGE, "payload_too_large").await;
464 }
465
466 mod errors {
467 use super::*;
468
469 #[derive(Clone)]
470 struct PutPreconditionFailedStorage {
471 inner: crate::testing::InMemoryStorage,
472 }
473
474 #[async_trait]
475 impl Storage for PutPreconditionFailedStorage {
476 async fn ensure_bucket(&self, kb: &KbSlug) -> Result<(), StorageError> {
477 self.inner.ensure_bucket(kb).await
478 }
479
480 async fn read_manifest(&self, kb: &KbSlug) -> Result<KbManifest, StorageError> {
481 self.inner.read_manifest(kb).await
482 }
483
484 async fn write_manifest(
485 &self,
486 kb: &KbSlug,
487 manifest: &KbManifest,
488 ) -> Result<(), StorageError> {
489 self.inner.write_manifest(kb, manifest).await
490 }
491
492 async fn head_object(
493 &self,
494 kb: &KbSlug,
495 path: &ObjectPath,
496 conditionals: ConditionalHeaders,
497 ) -> Result<ObjectMeta, StorageError> {
498 self.inner.head_object(kb, path, conditionals).await
499 }
500
501 async fn get_object(
502 &self,
503 kb: &KbSlug,
504 path: &ObjectPath,
505 range: Option<Vec<ByteRange>>,
506 conditionals: ConditionalHeaders,
507 ) -> Result<ObjectRead, StorageError> {
508 self.inner.get_object(kb, path, range, conditionals).await
509 }
510
511 async fn get_object_stream(
512 &self,
513 kb: &KbSlug,
514 path: &ObjectPath,
515 range: Option<Vec<ByteRange>>,
516 conditionals: ConditionalHeaders,
517 ) -> Result<notedthat_core::ObjectStream, StorageError> {
518 self.inner
519 .get_object_stream(kb, path, range, conditionals)
520 .await
521 }
522
523 async fn put_object(
524 &self,
525 _kb: &KbSlug,
526 _path: &ObjectPath,
527 _bytes: Bytes,
528 _content_type: Option<&str>,
529 _conditionals: ConditionalHeaders,
530 ) -> Result<PutOutcome, StorageError> {
531 Err(StorageError::PreconditionFailed)
532 }
533
534 async fn put_staged_object(
535 &self,
536 _kb: &KbSlug,
537 _path: &ObjectPath,
538 _body: notedthat_core::StagedBody,
539 _content_type: Option<&str>,
540 _conditionals: ConditionalHeaders,
541 ) -> Result<PutOutcome, StorageError> {
542 Err(StorageError::PreconditionFailed)
543 }
544
545 async fn copy_object(
546 &self,
547 kb: &KbSlug,
548 source: &ObjectPath,
549 destination: &ObjectPath,
550 options: notedthat_core::CopyObjectOptions,
551 ) -> Result<PutOutcome, StorageError> {
552 self.inner
553 .copy_object(kb, source, destination, options)
554 .await
555 }
556
557 async fn delete_object(
558 &self,
559 kb: &KbSlug,
560 path: &ObjectPath,
561 conditionals: ConditionalHeaders,
562 ) -> Result<(), StorageError> {
563 self.inner.delete_object(kb, path, conditionals).await
564 }
565
566 async fn list_objects(
567 &self,
568 kb: &KbSlug,
569 prefix: Option<&str>,
570 limit: u32,
571 cursor: Option<&str>,
572 ) -> Result<ListResponse, StorageError> {
573 self.inner.list_objects(kb, prefix, limit, cursor).await
574 }
575 }
576
577 #[tokio::test]
578 async fn missing_if_match_for_bytes_mode_returns_invalid_request() {
579 let (router, _etag) = router_with_object(b"0123456789", MAX_BODY_BYTES).await;
580
581 let response = patch_request(
582 router,
583 "content-range",
584 "bytes 0-9/*",
585 None,
586 Bytes::from_static(b"ABCDEFGHIJ"),
587 )
588 .await;
589
590 assert_error_code(response, StatusCode::BAD_REQUEST, "invalid_request").await;
591 }
592
593 #[tokio::test]
594 async fn body_larger_than_max_patchable_size_returns_payload_too_large() {
595 let (router, _etag) = router_with_object(b"one\n", 10).await;
596
597 let response = patch_request(
598 router,
599 "nt-patch-mode",
600 "append",
601 None,
602 Bytes::from_static(b"more than ten bytes"),
603 )
604 .await;
605
606 assert_error_code(response, StatusCode::PAYLOAD_TOO_LARGE, "payload_too_large").await;
607 }
608
609 #[tokio::test]
610 async fn pre_splice_object_larger_than_max_patchable_size_returns_payload_too_large() {
611 let (router, _etag) = router_with_object(b"already too large", 10).await;
612
613 let response = patch_request(
614 router,
615 "nt-patch-mode",
616 "append",
617 None,
618 Bytes::from_static(b"!"),
619 )
620 .await;
621
622 assert_error_code(response, StatusCode::PAYLOAD_TOO_LARGE, "payload_too_large").await;
623 }
624
625 #[tokio::test]
626 async fn post_splice_body_larger_than_max_patchable_size_returns_payload_too_large() {
627 let (router, _etag) = router_with_object(b"123456", 10).await;
628
629 let response = patch_request(
630 router,
631 "nt-patch-mode",
632 "append",
633 None,
634 Bytes::from_static(b"78901"),
635 )
636 .await;
637
638 assert_error_code(response, StatusCode::PAYLOAD_TOO_LARGE, "payload_too_large").await;
639 }
640
641 #[tokio::test]
642 async fn line_range_out_of_bounds_returns_dual_416_headers_and_empty_body() {
643 let (router, etag) =
644 router_with_object(b"one\ntwo\nthree\nfour\nfive\n", MAX_BODY_BYTES).await;
645
646 let response = patch_request(
647 router,
648 "content-range",
649 "lines 100-200/*",
650 Some(&etag),
651 Bytes::from_static(b"replacement\n"),
652 )
653 .await;
654
655 assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE);
656 assert_eq!(
657 response.headers().get("content-range").unwrap(),
658 "lines */5"
659 );
660 assert_eq!(
661 response.headers().get("x-content-range-bytes").unwrap(),
662 "*/24"
663 );
664 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
665 assert!(body.is_empty());
666 }
667
668 #[tokio::test]
669 async fn if_match_mismatch_after_retries_returns_precondition_failed_without_content_range()
670 {
671 let kb = KbSlug::try_new(KB).unwrap();
672 let inner = crate::testing::InMemoryStorage::default();
673 let etag = object_with_etag(&inner, &kb, b"0123456789").await;
674 let (indexer_tx, _rx) = tokio::sync::mpsc::channel(16);
675 let router = router_with_storage(
676 Arc::new(PutPreconditionFailedStorage { inner }),
677 kb,
678 MAX_BODY_BYTES,
679 indexer_tx,
680 );
681
682 let response = patch_request(
683 router,
684 "content-range",
685 "bytes 0-1/*",
686 Some(&etag),
687 Bytes::from_static(b"AB"),
688 )
689 .await;
690
691 assert_eq!(response.status(), StatusCode::PRECONDITION_FAILED);
692 assert!(
693 response
694 .headers()
695 .get(axum::http::header::CONTENT_RANGE)
696 .is_none()
697 );
698 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
699 let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
700 assert_eq!(json["error"], "precondition_failed");
701 }
702
703 #[tokio::test]
704 async fn indexer_queue_full_returns_backend_unavailable_with_retry_after() {
705 let kb = KbSlug::try_new(KB).unwrap();
706 let storage = crate::testing::InMemoryStorage::default();
707 let etag = object_with_etag(&storage, &kb, b"one\n").await;
708 let (indexer_tx, _rx) = tokio::sync::mpsc::channel(1);
709 indexer_tx
710 .try_send(IndexEvent::Upsert {
711 kb: kb.clone(),
712 object_key: ObjectPath::try_from_str("queued.md").unwrap(),
713 etag: "queued".to_string(),
714 mtime: 0,
715 })
716 .unwrap();
717 let router = router_with_storage(Arc::new(storage), kb, MAX_BODY_BYTES, indexer_tx);
718
719 let response = patch_request(
720 router,
721 "nt-patch-mode",
722 "append",
723 Some(&etag),
724 Bytes::from_static(b"two\n"),
725 )
726 .await;
727
728 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
729 assert_eq!(response.headers().get("retry-after").unwrap(), "5");
730 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
731 let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
732 assert_eq!(json["error"], "backend_unavailable");
733 }
734 }
735}
736
737#[cfg(test)]
738mod line_range_get {
739 use super::*;
740 use axum::body::{Body, to_bytes};
741 use axum::http::StatusCode;
742 use axum::response::Response;
743 use bytes::Bytes;
744 use notedthat_core::{ConditionalHeaders, KbSlug, ObjectPath, Storage};
745 use std::collections::BTreeMap;
746 use std::sync::Arc;
747 use tower::util::ServiceExt;
748
749 const KB: &str = "notes";
750 const TOKEN: &str = "test-token-abc";
751
752 fn twenty_line_markdown() -> String {
753 markdown_lines(1, 20)
754 }
755
756 fn markdown_lines(first: u32, last: u32) -> String {
757 let mut body = String::new();
758 for line in first..=last {
759 std::fmt::Write::write_fmt(&mut body, format_args!("line {line:02}\n")).unwrap();
760 }
761 body
762 }
763
764 async fn router_with_markdown_object(body: String) -> axum::Router {
765 let kb = KbSlug::try_new(KB).unwrap();
766 let object_path = ObjectPath::try_from_str("ranges.md").unwrap();
767 let storage = Arc::new(crate::testing::InMemoryStorage::default());
768 storage
769 .put_object(
770 &kb,
771 &object_path,
772 Bytes::from(body),
773 Some("text/markdown"),
774 ConditionalHeaders::default(),
775 )
776 .await
777 .unwrap();
778
779 let (indexer_tx, _rx) = tokio::sync::mpsc::channel(1);
780 let mut kbs = BTreeMap::new();
781 kbs.insert(KB.to_string(), kb);
782 build_router(AppState {
783 storage,
784 access_policies: Arc::new(notedthat_core::signed_in_policies(&kbs)),
785 declared_kbs: Arc::new(kbs),
786 authenticator: Arc::new(notedthat_core::Authenticator::new(TOKEN)),
787 max_body_size: MAX_BODY_BYTES,
788 max_patchable_size: MAX_BODY_BYTES,
789 indexer_tx,
790 searcher: Arc::new(crate::testing::NoopSearcher),
791 events: None,
792 })
793 }
794
795 async fn get_ranges_md(router: axum::Router, range: &str) -> Response {
796 router
797 .oneshot(
798 Request::builder()
799 .method("GET")
800 .uri(format!("/api/v1/knowledgebases/{KB}/ranges.md"))
801 .header("authorization", format!("Bearer {TOKEN}"))
802 .header(axum::http::header::RANGE, range)
803 .body(Body::empty())
804 .unwrap(),
805 )
806 .await
807 .unwrap()
808 }
809
810 #[tokio::test]
811 async fn returns_first_five_lines_when_closed_range_requested() {
812 let router = router_with_markdown_object(twenty_line_markdown()).await;
813
814 let response = get_ranges_md(router, "lines=1-5").await;
815
816 assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
817 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
818 assert_eq!(body, Bytes::from(markdown_lines(1, 5)));
819 }
820
821 #[tokio::test]
822 async fn returns_last_three_lines_when_suffix_range_requested() {
823 let router = router_with_markdown_object(twenty_line_markdown()).await;
824
825 let response = get_ranges_md(router, "lines=-3").await;
826
827 assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
828 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
829 assert_eq!(body, Bytes::from(markdown_lines(18, 20)));
830 }
831
832 #[tokio::test]
833 async fn returns_empty_body_when_insert_range_requested() {
834 let router = router_with_markdown_object(twenty_line_markdown()).await;
835
836 let response = get_ranges_md(router, "lines=5-4").await;
837
838 assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
839 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
840 assert!(body.is_empty());
841 }
842
843 #[tokio::test]
844 async fn returns_full_body_when_unknown_range_unit_requested() {
845 let body = twenty_line_markdown();
846 let router = router_with_markdown_object(body.clone()).await;
847
848 let response = get_ranges_md(router, "items=0-5").await;
849
850 assert_eq!(response.status(), StatusCode::OK);
851 let actual = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
852 assert_eq!(actual, Bytes::from(body));
853 }
854
855 mod headers {
856 use super::*;
857
858 fn ten_line_markdown() -> String {
859 markdown_lines(1, 10)
860 }
861
862 #[tokio::test]
863 async fn returns_line_and_byte_content_ranges_when_closed_range_requested() {
864 let router = router_with_markdown_object(ten_line_markdown()).await;
865
866 let response = get_ranges_md(router, "lines=2-4").await;
867
868 assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
869 assert_eq!(
870 response.headers().get("Content-Range").unwrap(),
871 "lines 2-4/10"
872 );
873 assert_eq!(
874 response.headers().get("X-Content-Range-Bytes").unwrap(),
875 "8-31/80"
876 );
877 }
878
879 #[tokio::test]
880 async fn returns_slice_content_length_when_closed_range_requested() {
881 let router = router_with_markdown_object(ten_line_markdown()).await;
882
883 let response = get_ranges_md(router, "lines=2-4").await;
884
885 assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
886 assert_eq!(response.headers().get("content-length").unwrap(), "24");
887 }
888
889 #[tokio::test]
890 async fn returns_zero_length_and_empty_byte_range_when_insert_range_requested() {
891 let router = router_with_markdown_object(ten_line_markdown()).await;
892
893 let response = get_ranges_md(router, "lines=5-4").await;
894
895 assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
896 assert_eq!(response.headers().get("content-length").unwrap(), "0");
897 assert_eq!(
898 response.headers().get("Content-Range").unwrap(),
899 "lines 5-4/10"
900 );
901 assert_eq!(
902 response.headers().get("X-Content-Range-Bytes").unwrap(),
903 "32-31/80"
904 );
905 }
906
907 #[tokio::test]
908 async fn omits_line_byte_range_header_when_byte_range_requested() {
909 let router = router_with_markdown_object(ten_line_markdown()).await;
910
911 let response = get_ranges_md(router, "bytes=0-9").await;
912
913 assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
914 assert!(response.headers().get("X-Content-Range-Bytes").is_none());
915 }
916 }
917}
918
919#[cfg(test)]
920mod tests {
921 use super::*;
922 use axum::body::{Body, to_bytes};
923 use axum::http::StatusCode;
924 use axum::response::Response;
925 use bytes::Bytes;
926 use notedthat_core::{ConditionalHeaders, KbSlug, ObjectPath, Storage, StorageError};
927 use notedthat_indexer::IndexEvent;
928 use std::collections::BTreeMap;
929 use std::sync::Arc;
930 use tower::util::ServiceExt;
931
932 const KB: &str = "notes";
933 const TOKEN: &str = "test-token-abc";
934
935 fn router() -> axum::Router {
936 let kb = KbSlug::try_new(KB).unwrap();
937 let mut kbs = BTreeMap::new();
938 kbs.insert(KB.to_string(), kb);
939 let (indexer_tx, mut rx) = tokio::sync::mpsc::channel(16);
940 tokio::spawn(async move { while rx.recv().await.is_some() {} });
941
942 build_router(AppState {
943 storage: Arc::new(crate::testing::InMemoryStorage::default()),
944 access_policies: Arc::new(notedthat_core::signed_in_policies(&kbs)),
945 declared_kbs: Arc::new(kbs),
946 authenticator: Arc::new(notedthat_core::Authenticator::new(TOKEN)),
947 max_body_size: MAX_BODY_BYTES,
948 max_patchable_size: MAX_BODY_BYTES,
949 indexer_tx,
950 searcher: Arc::new(crate::testing::NoopSearcher),
951 events: None,
952 })
953 }
954
955 async fn put_object(router: axum::Router, path: &str, body: &'static [u8]) -> Response {
956 router
957 .oneshot(
958 Request::builder()
959 .method("PUT")
960 .uri(format!("/api/v1/knowledgebases/{KB}/{path}"))
961 .header("authorization", format!("Bearer {TOKEN}"))
962 .header(axum::http::header::CONTENT_TYPE, "text/markdown")
963 .body(Body::from(Bytes::from_static(body)))
964 .unwrap(),
965 )
966 .await
967 .unwrap()
968 }
969
970 async fn put_object_etag(router: axum::Router, path: &str, body: &'static [u8]) -> String {
971 let response = put_object(router, path, body).await;
972 assert_eq!(response.status(), StatusCode::CREATED);
973 response
974 .headers()
975 .get(axum::http::header::ETAG)
976 .unwrap()
977 .to_str()
978 .unwrap()
979 .to_string()
980 }
981
982 async fn get_object(router: axum::Router, path: &str) -> Response {
983 router
984 .oneshot(
985 Request::builder()
986 .method("GET")
987 .uri(format!("/api/v1/knowledgebases/{KB}/{path}"))
988 .header("authorization", format!("Bearer {TOKEN}"))
989 .body(Body::empty())
990 .unwrap(),
991 )
992 .await
993 .unwrap()
994 }
995
996 async fn post_replace(
997 router: axum::Router,
998 path: &str,
999 if_match: &str,
1000 body: &'static [u8],
1001 ) -> Response {
1002 router
1003 .oneshot(
1004 Request::builder()
1005 .method("POST")
1006 .uri(format!("/api/v1/knowledgebases/{KB}/replace/{path}"))
1007 .header("authorization", format!("Bearer {TOKEN}"))
1008 .header(axum::http::header::CONTENT_TYPE, "application/json")
1009 .header(axum::http::header::IF_MATCH, if_match)
1010 .body(Body::from(Bytes::from_static(body)))
1011 .unwrap(),
1012 )
1013 .await
1014 .unwrap()
1015 }
1016
1017 #[tokio::test]
1018 async fn get_on_replace_prefixed_path_still_reads_object_via_catch_all() {
1019 let router = router();
1020 put_object_etag(router.clone(), "replace/foo.md", b"hi").await;
1021
1022 let response = get_object(router, "replace/foo.md").await;
1023
1024 assert_eq!(response.status(), StatusCode::OK);
1025 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
1026 assert_eq!(&body[..], b"hi");
1027 }
1028
1029 #[tokio::test]
1030 async fn patch_on_replace_prefixed_path_still_reaches_patch_object() {
1031 let router = router();
1032 let etag = put_object_etag(router.clone(), "replace/bar.md", b"old\n").await;
1033
1034 let response = router
1035 .oneshot(
1036 Request::builder()
1037 .method("PATCH")
1038 .uri(format!("/api/v1/knowledgebases/{KB}/replace/bar.md"))
1039 .header("authorization", format!("Bearer {TOKEN}"))
1040 .header(axum::http::header::CONTENT_RANGE, "lines 1-1/*")
1041 .header(axum::http::header::IF_MATCH, etag)
1042 .body(Body::from(Bytes::from_static(b"new\n")))
1043 .unwrap(),
1044 )
1045 .await
1046 .unwrap();
1047
1048 assert_eq!(response.status(), StatusCode::OK);
1049 }
1050
1051 #[tokio::test]
1052 async fn put_and_delete_on_replace_prefixed_path_still_work() {
1053 let router = router();
1054 let put = put_object(router.clone(), "replace/delete.md", b"gone").await;
1055
1056 assert_eq!(put.status(), StatusCode::CREATED);
1057 let delete = router
1058 .oneshot(
1059 Request::builder()
1060 .method("DELETE")
1061 .uri(format!("/api/v1/knowledgebases/{KB}/replace/delete.md"))
1062 .header("authorization", format!("Bearer {TOKEN}"))
1063 .body(Body::empty())
1064 .unwrap(),
1065 )
1066 .await
1067 .unwrap();
1068 assert_eq!(delete.status(), StatusCode::NO_CONTENT);
1069 }
1070
1071 #[tokio::test]
1072 async fn post_on_non_replace_path_returns_404_not_found() {
1073 let response = router()
1074 .oneshot(
1075 Request::builder()
1076 .method("POST")
1077 .uri(format!("/api/v1/knowledgebases/{KB}/foo.md"))
1078 .header("authorization", format!("Bearer {TOKEN}"))
1079 .body(Body::empty())
1080 .unwrap(),
1081 )
1082 .await
1083 .unwrap();
1084
1085 assert_eq!(response.status(), StatusCode::NOT_FOUND);
1086 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
1087 let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
1088 assert_eq!(json["error"], "not_found");
1089 assert!(
1090 json["message"]
1091 .as_str()
1092 .unwrap()
1093 .contains("supported actions: 'replace/<path>'")
1094 );
1095 }
1096
1097 #[tokio::test]
1098 async fn post_on_replace_prefixed_path_dispatches_to_replace_handler() {
1099 let router = router();
1100 let etag = put_object_etag(router.clone(), "target.md", b"hello world").await;
1101
1102 let response = post_replace(
1103 router,
1104 "target.md",
1105 &etag,
1106 br#"{"old_string":"world","new_string":"planet"}"#,
1107 )
1108 .await;
1109
1110 assert_eq!(response.status(), StatusCode::OK);
1111 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
1112 let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
1113 assert_eq!(json["match_count"], 1);
1114 }
1115
1116 #[tokio::test]
1117 async fn post_on_replace_replace_path_targets_the_replace_prefixed_object() {
1118 let router = router();
1119 let etag = put_object_etag(router.clone(), "replace/nested.md", b"foo bar").await;
1120
1121 let response = post_replace(
1122 router.clone(),
1123 "replace/nested.md",
1124 &etag,
1125 br#"{"old_string":"bar","new_string":"baz"}"#,
1126 )
1127 .await;
1128
1129 assert_eq!(response.status(), StatusCode::OK);
1130 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
1131 let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
1132 assert_eq!(json["match_count"], 1);
1133 let get = get_object(router, "replace/nested.md").await;
1134 let body = to_bytes(get.into_body(), 64 * 1024).await.unwrap();
1135 assert_eq!(&body[..], b"foo baz");
1136 }
1137
1138 #[tokio::test]
1139 async fn test_conditional_put_503_then_naive_retry_412_keeps_object_stored() {
1140 let kb = KbSlug::try_new(KB).unwrap();
1141 let object_path = ObjectPath::try_from_str("cond.md").unwrap();
1142 let storage = Arc::new(crate::testing::InMemoryStorage::default());
1143
1144 let (indexer_tx, _rx) = tokio::sync::mpsc::channel(1);
1145 indexer_tx
1146 .try_send(IndexEvent::Upsert {
1147 kb: kb.clone(),
1148 object_key: ObjectPath::try_from_str("queued.md").unwrap(),
1149 etag: "etag".to_string(),
1150 mtime: 0,
1151 })
1152 .unwrap();
1153
1154 let mut kbs = BTreeMap::new();
1155 kbs.insert(KB.to_string(), kb.clone());
1156 let state = AppState {
1157 storage: storage.clone(),
1158 access_policies: Arc::new(notedthat_core::signed_in_policies(&kbs)),
1159 declared_kbs: Arc::new(kbs),
1160 authenticator: Arc::new(notedthat_core::Authenticator::new(TOKEN)),
1161 max_body_size: MAX_BODY_BYTES,
1162 max_patchable_size: MAX_BODY_BYTES,
1163 indexer_tx,
1164 searcher: Arc::new(crate::testing::NoopSearcher),
1165 events: None,
1166 };
1167 let router = build_router(state);
1168
1169 let response = router
1170 .clone()
1171 .oneshot(
1172 Request::builder()
1173 .method("PUT")
1174 .uri(format!("/api/v1/knowledgebases/{KB}/cond.md"))
1175 .header("authorization", format!("Bearer {TOKEN}"))
1176 .header("if-none-match", "*")
1177 .body(Body::from(Bytes::from_static(b"first content")))
1178 .unwrap(),
1179 )
1180 .await
1181 .unwrap();
1182
1183 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1184 assert_eq!(response.headers().get("retry-after").unwrap(), "5");
1185 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
1186 let body = String::from_utf8(body.to_vec()).unwrap();
1187 assert!(body.contains("\"error\":\"backend_unavailable\""));
1188 assert!(body.contains("object stored; indexer queue full — retry to re-enqueue"));
1189
1190 let stored = storage
1191 .get_object(&kb, &object_path, None, ConditionalHeaders::default())
1192 .await
1193 .unwrap();
1194 assert_eq!(stored.bytes, Bytes::from_static(b"first content"));
1195
1196 let retry = router
1197 .oneshot(
1198 Request::builder()
1199 .method("PUT")
1200 .uri(format!("/api/v1/knowledgebases/{KB}/cond.md"))
1201 .header("authorization", format!("Bearer {TOKEN}"))
1202 .header("if-none-match", "*")
1203 .body(Body::from(Bytes::from_static(b"second content")))
1204 .unwrap(),
1205 )
1206 .await
1207 .unwrap();
1208
1209 assert_eq!(retry.status(), StatusCode::PRECONDITION_FAILED);
1210 assert!(retry.headers().get("retry-after").is_none());
1211
1212 let stored = storage
1213 .get_object(&kb, &object_path, None, ConditionalHeaders::default())
1214 .await
1215 .unwrap();
1216 assert_eq!(stored.bytes, Bytes::from_static(b"first content"));
1217 }
1218
1219 #[tokio::test]
1220 async fn test_delete_returns_delete_specific_503_body_when_indexer_backpressure() {
1221 let kb = KbSlug::try_new(KB).unwrap();
1222 let object_path = ObjectPath::try_from_str("to-delete.md").unwrap();
1223 let storage = Arc::new(crate::testing::InMemoryStorage::default());
1224 storage
1225 .put_object(
1226 &kb,
1227 &object_path,
1228 Bytes::from_static(b"content"),
1229 Some("text/plain"),
1230 ConditionalHeaders::default(),
1231 )
1232 .await
1233 .unwrap();
1234
1235 let (indexer_tx, _rx) = tokio::sync::mpsc::channel(1);
1236 indexer_tx
1237 .try_send(IndexEvent::Upsert {
1238 kb: kb.clone(),
1239 object_key: ObjectPath::try_from_str("queued.md").unwrap(),
1240 etag: "etag".to_string(),
1241 mtime: 0,
1242 })
1243 .unwrap();
1244
1245 let mut kbs = BTreeMap::new();
1246 kbs.insert(KB.to_string(), kb.clone());
1247 let state = AppState {
1248 storage: storage.clone(),
1249 access_policies: Arc::new(notedthat_core::signed_in_policies(&kbs)),
1250 declared_kbs: Arc::new(kbs),
1251 authenticator: Arc::new(notedthat_core::Authenticator::new(TOKEN)),
1252 max_body_size: MAX_BODY_BYTES,
1253 max_patchable_size: MAX_BODY_BYTES,
1254 indexer_tx,
1255 searcher: Arc::new(crate::testing::NoopSearcher),
1256 events: None,
1257 };
1258 let router = build_router(state);
1259
1260 let response = router
1261 .oneshot(
1262 Request::builder()
1263 .method("DELETE")
1264 .uri(format!("/api/v1/knowledgebases/{KB}/to-delete.md"))
1265 .header("authorization", format!("Bearer {TOKEN}"))
1266 .body(Body::empty())
1267 .unwrap(),
1268 )
1269 .await
1270 .unwrap();
1271
1272 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1273 assert_eq!(response.headers().get("retry-after").unwrap(), "5");
1274 let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
1275 let body = String::from_utf8(body.to_vec()).unwrap();
1276 assert!(body.contains("\"error\":\"backend_unavailable\""));
1277 assert!(
1278 body.contains("\"message\":\"deleted from storage; retry to clear from search index\"")
1279 );
1280 assert!(!body.contains("object stored; indexer queue full — retry to re-enqueue"));
1281
1282 let deleted = storage
1283 .get_object(&kb, &object_path, None, ConditionalHeaders::default())
1284 .await;
1285 assert!(matches!(deleted, Err(StorageError::NotFound { .. })));
1286 }
1287}