mod embedded;
use crate::backend_error::{map_namespace_scoped_runtime_error, BackendError};
use crate::payload::LocalPayload;
use crate::progress::ProgressReporter;
use crate::resolve::ResolvedTarget;
use crate::uploads::UploadJournal;
use bytes::Bytes;
use loonfs_api::{
v0::{
ChangesResponse, DisableGrepIndexResponse, EnableGrepIndexResponse, GrepGcRequest,
GrepGcResponse, GrepIndexLifecycle, GrepIndexStatusResponse, StoreProbeRequest,
StoreProbeResponse, UploadStatusResponse,
},
AbsolutePath, AuthoritativePathEntry, ChangeSeq, CheckpointId, CommitResponse, ContentRef,
CreateCheckpointRequest, CreateCheckpointResponse, DeleteNamespaceResponse, GrepRequest,
GrepResponse, InodeId, ListCheckpointsResponse, ListFileRevisionsResponse,
ListPathEntriesResponse, ListTrashResponse, MaintenanceStepRequest, MaintenanceStepResponse,
NamespaceId, NamespaceStatusResponse, NamespaceSummary, ReleaseCheckpointResponse, RevisionNo,
UploadId,
};
use loonfs_client::{
CopyOptions, CreateDirectoryOptions, DeleteOptions, DirectDownloadStream, MoveOptions,
NamespacePath, PutFileOptions, RestoreRevisionOptions, UndeleteOptions,
};
use loonfs_objectstore::timing::{MonotonicTimer, StdMonotonicTimer};
use std::sync::Arc;
pub(crate) use embedded::EmbeddedBackend;
use loonfs::{
FileContentStream, MaintenanceJobId, MaintenanceStepConclusion, RuntimeError, SharedObjectStore,
};
const REMOTE_STATUS_POLL_INTERVAL_MS: u64 = 250;
#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct StepBudget {
pub max_steps: Option<u64>,
pub deadline_ms: Option<u64>,
}
impl StepBudget {
fn spent(&self, steps: u64, elapsed_ms: u64) -> bool {
self.max_steps.is_some_and(|max_steps| steps >= max_steps)
|| self
.deadline_ms
.is_some_and(|deadline_ms| elapsed_ms >= deadline_ms)
}
}
pub(crate) enum FileDownload {
Streamed {
namespace_id: NamespaceId,
stream: Box<FileContentStream<SharedObjectStore>>,
resumed_from: u64,
},
Direct {
stream: Box<DirectDownloadStream>,
resumed_from: u64,
},
Whole(Vec<u8>),
}
impl FileDownload {
pub(crate) async fn next_chunk(&mut self) -> Result<Option<Bytes>, BackendError> {
match self {
Self::Streamed {
namespace_id,
stream,
..
} => stream.next_chunk().await.map_err(|error| {
map_namespace_scoped_runtime_error(namespace_id, RuntimeError::Core(error))
}),
Self::Direct { stream, .. } => stream.next_chunk().await.map_err(BackendError::from),
Self::Whole(bytes) if bytes.is_empty() => Ok(None),
Self::Whole(bytes) => Ok(Some(Bytes::from(std::mem::take(bytes)))),
}
}
pub(crate) fn resumed_from(&self) -> u64 {
match self {
Self::Streamed { resumed_from, .. } | Self::Direct { resumed_from, .. } => {
*resumed_from
}
Self::Whole(_) => 0,
}
}
pub(crate) fn fold_resumed_prefix(&mut self, bytes: &[u8]) {
match self {
Self::Streamed { stream, .. } => stream.fold_resumed_prefix(bytes),
Self::Direct { stream, .. } => stream.fold_resumed_prefix(bytes),
Self::Whole(_) => {}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct MaintenanceKeyProgress {
pub job: MaintenanceJobId,
pub namespace_id: NamespaceId,
pub steps: u64,
pub conclusion: Option<MaintenanceStepConclusion>,
}
impl MaintenanceKeyProgress {
pub(crate) fn settled(&self) -> bool {
match self.conclusion {
Some(
MaintenanceStepConclusion::Idle
| MaintenanceStepConclusion::Blocked
| MaintenanceStepConclusion::NotEnabled,
) => true,
Some(MaintenanceStepConclusion::Progressed | MaintenanceStepConclusion::Superseded)
| None => false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct MaintenanceDrainProgress {
pub keys: Vec<MaintenanceKeyProgress>,
pub steps: u64,
}
impl MaintenanceDrainProgress {
pub(crate) fn budget_exhausted(&self) -> bool {
!self.keys.iter().all(MaintenanceKeyProgress::settled)
}
}
fn upload_sessions_need_a_remote_profile() -> BackendError {
BackendError::new(
loonfs_api::ErrorCode::NotSupported.as_str(),
"upload sessions belong to a server; an embedded profile stages content itself",
)
}
fn maintenance_host_needs_an_embedded_profile() -> BackendError {
BackendError::new(
loonfs_api::ErrorCode::NotSupported.as_str(),
"`admin run` hosts maintenance in this process and needs an embedded profile; \
a remote profile's server runs its own maintenance",
)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct GrepWaitProgress {
pub state: GrepIndexLifecycle,
pub steps: u64,
pub reached: bool,
}
#[allow(clippy::disallowed_methods)]
async fn rest_between_status_checks() {
tokio::time::sleep(std::time::Duration::from_millis(
REMOTE_STATUS_POLL_INTERVAL_MS,
))
.await;
}
impl ResolvedTarget {
pub(crate) async fn create_namespace(
&self,
namespace_id: &NamespaceId,
) -> Result<NamespaceSummary, BackendError> {
match self {
Self::Embedded(target) => target.backend.create_namespace(namespace_id).await,
Self::Remote(target) => Ok(target.client.create_namespace(namespace_id).await?),
}
}
pub(crate) async fn delete_namespace(
&self,
namespace_id: &NamespaceId,
expected_head_seq: Option<ChangeSeq>,
) -> Result<DeleteNamespaceResponse, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.delete_namespace(namespace_id, expected_head_seq)
.await
}
Self::Remote(target) => Ok(target
.client
.delete_namespace(namespace_id, expected_head_seq)
.await?),
}
}
pub(crate) async fn fork_namespace(
&self,
source_namespace_id: &NamespaceId,
new_namespace_id: &NamespaceId,
) -> Result<NamespaceSummary, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.fork_namespace(source_namespace_id, new_namespace_id)
.await
}
Self::Remote(target) => Ok(target
.client
.fork_namespace(source_namespace_id, new_namespace_id)
.await?),
}
}
pub(crate) async fn namespace_status(
&self,
namespace_id: &NamespaceId,
) -> Result<NamespaceStatusResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.namespace_status(namespace_id).await,
Self::Remote(target) => Ok(target.client.namespace_status(namespace_id).await?),
}
}
pub(crate) async fn list_path_entries_all(
&self,
spec: &NamespacePath,
) -> Result<Vec<AuthoritativePathEntry>, BackendError> {
match self {
Self::Embedded(target) => target.backend.list_path_entries_all(spec).await,
Self::Remote(target) => Ok(target.client.list_path_entries_all(spec).await?.entries),
}
}
pub(crate) async fn list_path_entries_page(
&self,
spec: &NamespacePath,
limit: Option<u32>,
cursor: Option<&str>,
) -> Result<ListPathEntriesResponse, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.list_path_entries_page(spec, limit, cursor)
.await
}
Self::Remote(target) => Ok(target
.client
.list_path_entries_page(spec, limit, cursor)
.await?),
}
}
pub(crate) async fn stat_path(
&self,
spec: &NamespacePath,
) -> Result<AuthoritativePathEntry, BackendError> {
match self {
Self::Embedded(target) => target.backend.stat_path(spec).await,
Self::Remote(target) => Ok(target.client.stat_path(spec).await?),
}
}
pub(crate) async fn get_file_bytes(
&self,
spec: &NamespacePath,
) -> Result<Vec<u8>, BackendError> {
match self {
Self::Embedded(target) => target.backend.get_file_bytes(spec).await,
Self::Remote(target) => Ok(target.client.get_file_bytes(spec).await?),
}
}
pub(crate) async fn open_file_download(
&self,
spec: &NamespacePath,
revision_no: Option<RevisionNo>,
size_bytes: Option<u64>,
start_offset: u64,
) -> Result<FileDownload, BackendError> {
if let (Self::Remote(target), Some(size_bytes)) = (self, size_bytes) {
if target.client.offers_direct_download(size_bytes).await {
let grant = target.client.begin_download(spec, revision_no).await?;
return Ok(FileDownload::Direct {
stream: Box::new(
target
.client
.open_direct_download_at(&grant, start_offset)
.await?,
),
resumed_from: start_offset,
});
}
}
if let Some(revision_no) = revision_no {
return Ok(FileDownload::Whole(
self.get_file_revision_bytes(spec, revision_no).await?,
));
}
match self {
Self::Embedded(target) => Ok(FileDownload::Streamed {
namespace_id: spec.namespace().clone(),
stream: Box::new(target.backend.read_file_stream(spec, start_offset).await?),
resumed_from: start_offset,
}),
Self::Remote(target) => Ok(FileDownload::Whole(
target.client.get_file_bytes(spec).await?,
)),
}
}
pub(crate) async fn grep(
&self,
namespace_id: &NamespaceId,
request: &GrepRequest,
) -> Result<GrepResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.grep(namespace_id, request).await,
Self::Remote(target) => Ok(target.client.grep(namespace_id, request).await?),
}
}
pub(crate) async fn enable_grep_index(
&self,
namespace_id: &NamespaceId,
) -> Result<EnableGrepIndexResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.enable_grep_index(namespace_id).await,
Self::Remote(target) => Ok(target.client.enable_grep_index(namespace_id).await?),
}
}
pub(crate) async fn disable_grep_index(
&self,
namespace_id: &NamespaceId,
) -> Result<DisableGrepIndexResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.disable_grep_index(namespace_id).await,
Self::Remote(target) => Ok(target.client.disable_grep_index(namespace_id).await?),
}
}
pub(crate) async fn grep_index_status(
&self,
namespace_id: &NamespaceId,
) -> Result<GrepIndexStatusResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.grep_index_status(namespace_id).await,
Self::Remote(target) => Ok(target.client.grep_index_status(namespace_id).await?),
}
}
pub(crate) async fn gc_grep_index(
&self,
namespace_id: &NamespaceId,
request: &GrepGcRequest,
) -> Result<GrepGcResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.gc_grep_index(namespace_id, request).await,
Self::Remote(target) => Ok(target.client.gc_grep_index(namespace_id, request).await?),
}
}
pub(crate) async fn wait_for_grep_index(
&self,
namespace_id: &NamespaceId,
target_seq: ChangeSeq,
budget: StepBudget,
) -> Result<GrepWaitProgress, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.drive_grep_index(namespace_id, target_seq, budget)
.await
}
Self::Remote(target) => {
let timer = StdMonotonicTimer::default();
let started_ms = timer.monotonic_now_ms();
let mut steps = 0;
loop {
let state = target.client.grep_index_status(namespace_id).await?.state;
let reached = state.is_built_through(target_seq);
let elapsed_ms = timer.monotonic_now_ms().saturating_sub(started_ms);
if reached || budget.spent(steps, elapsed_ms) {
return Ok(GrepWaitProgress {
state,
steps,
reached,
});
}
rest_between_status_checks().await;
steps += 1;
}
}
}
}
pub(crate) async fn get_file_revision_bytes(
&self,
spec: &NamespacePath,
revision_no: RevisionNo,
) -> Result<Vec<u8>, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.get_file_revision_bytes(spec, revision_no)
.await
}
Self::Remote(target) => Ok(target
.client
.get_file_revision_bytes(spec, revision_no)
.await?),
}
}
pub(crate) async fn list_trash(
&self,
namespace_id: &NamespaceId,
limit: Option<u32>,
cursor: Option<&str>,
) -> Result<ListTrashResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.list_trash(namespace_id, limit, cursor).await,
Self::Remote(target) => Ok(target
.client
.list_trash_page(namespace_id, limit, cursor)
.await?),
}
}
pub(crate) async fn list_file_revisions_page(
&self,
spec: &NamespacePath,
limit: Option<u32>,
cursor: Option<&str>,
) -> Result<ListFileRevisionsResponse, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.list_file_revisions_page(spec, limit, cursor)
.await
}
Self::Remote(target) => Ok(target
.client
.list_file_revisions_page(spec, limit, cursor)
.await?),
}
}
pub(crate) async fn put_file_bytes(
&self,
spec: &NamespacePath,
bytes: &[u8],
options: &PutFileOptions,
) -> Result<CommitResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.put_file_bytes(spec, bytes, options).await,
Self::Remote(target) => Ok(target.client.put_file_bytes(spec, bytes, options).await?),
}
}
pub(crate) async fn put_file_stream(
&self,
spec: &NamespacePath,
payload: &LocalPayload,
options: &PutFileOptions,
progress: &Arc<ProgressReporter>,
journal: Option<&UploadJournal>,
) -> Result<CommitResponse, BackendError> {
match self {
Self::Embedded(target) => {
let body = payload.open_byte_stream(progress).await?;
target.backend.put_file_stream(spec, body, options).await
}
Self::Remote(target) => {
let source = payload.open_source(progress).await?;
let Some(journal) = journal else {
return Ok(target.client.put_file_stream(spec, source, options).await?);
};
let resume = journal.resume();
Ok(target
.client
.put_file_stream_resumable(spec, source, options, journal, resume.as_ref())
.await?)
}
}
}
pub(crate) async fn read_upload_status(
&self,
namespace_id: &NamespaceId,
upload_id: &UploadId,
) -> Result<UploadStatusResponse, BackendError> {
match self {
Self::Embedded(_) => Err(upload_sessions_need_a_remote_profile()),
Self::Remote(target) => Ok(target
.client
.read_upload_status(namespace_id, upload_id)
.await?),
}
}
pub(crate) async fn commit_completed_upload(
&self,
spec: &NamespacePath,
content_ref: ContentRef,
validated_content_token: Option<String>,
options: &PutFileOptions,
) -> Result<CommitResponse, BackendError> {
match self {
Self::Embedded(_) => Err(upload_sessions_need_a_remote_profile()),
Self::Remote(target) => Ok(target
.client
.commit_completed_upload(spec, content_ref, validated_content_token, options)
.await?),
}
}
pub(crate) async fn create_directory(
&self,
spec: &NamespacePath,
options: &CreateDirectoryOptions,
) -> Result<CommitResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.create_directory(spec, options).await,
Self::Remote(target) => Ok(target.client.create_directory(spec, options).await?),
}
}
pub(crate) async fn delete_path(
&self,
spec: &NamespacePath,
options: &DeleteOptions,
) -> Result<CommitResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.delete_path(spec, options).await,
Self::Remote(target) => Ok(target.client.delete_path(spec, options).await?),
}
}
pub(crate) async fn move_path(
&self,
from: &NamespacePath,
to: &NamespacePath,
options: &MoveOptions,
) -> Result<CommitResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.move_path(from, to, options).await,
Self::Remote(target) => Ok(target.client.move_path(from, to, options).await?),
}
}
pub(crate) async fn copy_path(
&self,
from: &NamespacePath,
to: &NamespacePath,
options: &CopyOptions,
) -> Result<CommitResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.copy_path(from, to, options).await,
Self::Remote(target) => Ok(target.client.copy_path(from, to, options).await?),
}
}
pub(crate) async fn restore_file_revision(
&self,
spec: &NamespacePath,
source_revision_no: RevisionNo,
options: &RestoreRevisionOptions,
) -> Result<CommitResponse, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.restore_file_revision(spec, source_revision_no, options)
.await
}
Self::Remote(target) => Ok(target
.client
.restore_file_revision(spec, source_revision_no, options)
.await?),
}
}
pub(crate) async fn undelete(
&self,
namespace: &NamespaceId,
path: Option<&AbsolutePath>,
inode_id: InodeId,
deleted_at_seq: ChangeSeq,
options: &UndeleteOptions,
) -> Result<CommitResponse, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.undelete(namespace, path, inode_id, deleted_at_seq, options)
.await
}
Self::Remote(target) => Ok(target
.client
.undelete(namespace, inode_id, deleted_at_seq, path, options)
.await?),
}
}
pub(crate) async fn create_checkpoint(
&self,
namespace_id: &NamespaceId,
request: CreateCheckpointRequest,
) -> Result<CreateCheckpointResponse, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.create_checkpoint(namespace_id, request)
.await
}
Self::Remote(target) => Ok(target
.client
.create_checkpoint(namespace_id, &request)
.await?),
}
}
pub(crate) async fn list_checkpoints(
&self,
namespace_id: &NamespaceId,
) -> Result<ListCheckpointsResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.list_checkpoints(namespace_id).await,
Self::Remote(target) => Ok(target.client.list_checkpoints(namespace_id).await?),
}
}
pub(crate) async fn release_checkpoint(
&self,
namespace_id: &NamespaceId,
checkpoint_id: &CheckpointId,
) -> Result<ReleaseCheckpointResponse, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.release_checkpoint(namespace_id, checkpoint_id)
.await
}
Self::Remote(target) => Ok(target
.client
.release_checkpoint(namespace_id, checkpoint_id)
.await?),
}
}
pub(crate) async fn maintenance_step(
&self,
namespace_id: &NamespaceId,
request: MaintenanceStepRequest,
) -> Result<MaintenanceStepResponse, BackendError> {
match self {
Self::Embedded(target) => target.backend.maintenance_step(namespace_id, request).await,
Self::Remote(target) => Ok(target
.client
.maintenance_step(namespace_id, &request)
.await?),
}
}
pub(crate) async fn probe_store(&self) -> Result<StoreProbeResponse, BackendError> {
match self {
Self::Embedded(target) => Ok(target.backend.probe_store().await),
Self::Remote(target) => Ok(target.client.probe_store(&StoreProbeRequest {}).await?),
}
}
pub(crate) async fn host_maintenance(
&self,
namespaces: &[NamespaceId],
jobs: &[MaintenanceJobId],
poll_interval_ms: Option<u64>,
shutdown: impl std::future::Future<Output = ()>,
) -> Result<(), BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.host_maintenance(namespaces, jobs, poll_interval_ms, shutdown)
.await
}
Self::Remote(_) => Err(maintenance_host_needs_an_embedded_profile()),
}
}
pub(crate) async fn drain_maintenance(
&self,
namespaces: &[NamespaceId],
jobs: &[MaintenanceJobId],
budget: StepBudget,
) -> Result<MaintenanceDrainProgress, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.drain_maintenance(namespaces, jobs, budget)
.await
}
Self::Remote(_) => Err(maintenance_host_needs_an_embedded_profile()),
}
}
pub(crate) async fn list_changes(
&self,
namespace_id: &NamespaceId,
after_seq: ChangeSeq,
limit: Option<u32>,
) -> Result<ChangesResponse, BackendError> {
match self {
Self::Embedded(target) => {
target
.backend
.list_changes(namespace_id, after_seq, limit)
.await
}
Self::Remote(target) => Ok(target
.client
.list_changes(namespace_id, after_seq, limit)
.await?),
}
}
}