Trait ProviderAdmin 

pub trait ProviderAdmin:
    Any
    + Send
    + Sync {
    // Required methods
    fn list_instances<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<String>, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn list_instances_by_status<'life0, 'life1, 'async_trait>(
        &'life0 self,
        status: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<String>, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn list_executions<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<u64>, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn read_history_with_execution_id<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance: &'life1 str,
        execution_id: u64,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<Event>, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn read_history<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<Event>, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn latest_execution_id<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<u64, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn get_instance_info<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<InstanceInfo, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn get_execution_info<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance: &'life1 str,
        execution_id: u64,
    ) -> Pin<Box<dyn Future<Output = Result<ExecutionInfo, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn get_system_metrics<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<SystemMetrics, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn get_queue_depths<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<QueueDepths, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn list_children<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance_id: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<String>, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn get_parent_id<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance_id: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Option<String>, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn delete_instances_atomic<'life0, 'life1, 'async_trait>(
        &'life0 self,
        ids: &'life1 [String],
        force: bool,
    ) -> Pin<Box<dyn Future<Output = Result<DeleteInstanceResult, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn delete_instance_bulk<'life0, 'async_trait>(
        &'life0 self,
        filter: InstanceFilter,
    ) -> Pin<Box<dyn Future<Output = Result<DeleteInstanceResult, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn prune_executions<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance_id: &'life1 str,
        options: PruneOptions,
    ) -> Pin<Box<dyn Future<Output = Result<PruneResult, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn prune_executions_bulk<'life0, 'async_trait>(
        &'life0 self,
        filter: InstanceFilter,
        options: PruneOptions,
    ) -> Pin<Box<dyn Future<Output = Result<PruneResult, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;

    // Provided methods
    fn get_instance_tree<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance_id: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<InstanceTree, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn delete_instance<'life0, 'life1, 'async_trait>(
        &'life0 self,
        instance_id: &'life1 str,
        force: bool,
    ) -> Pin<Box<dyn Future<Output = Result<DeleteInstanceResult, ProviderError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
}

Administrative capability trait for observability and management operations.

This trait provides rich management and observability features that extend the core Provider functionality. Providers can implement this trait to expose administrative capabilities to the Client.

Automatic Discovery

The Client automatically discovers this capability through the Provider::as_management_capability() method. When available, management methods become accessible through the client.

Implementation Guide for LLMs

When implementing a new provider, you can optionally implement this trait to expose management features:

impl Provider for MyProvider {
    // ... implement required Provider methods
     
    fn as_management_capability(&self) -> Option<&dyn ProviderAdmin> {
        Some(self as &dyn ProviderAdmin)
    }
}

impl ProviderAdmin for MyProvider {
    async fn list_instances(&self) -> Result<Vec<String>, String> {
        // Query your storage for all instance IDs
        Ok(vec!["instance-1".to_string(), "instance-2".to_string()])
    }
     
    async fn get_instance_info(&self, instance: &str) -> Result<InstanceInfo, String> {
        // Query instance metadata from your storage
        Ok(InstanceInfo {
            instance_id: instance.to_string(),
            orchestration_name: "ProcessOrder".to_string(),
            orchestration_version: "1.0.0".to_string(),
            current_execution_id: 1,
            status: "Running".to_string(),
            output: None,
            created_at: 1234567890,
            updated_at: 1234567890,
        })
    }
     
    // ... implement other management methods
}

Required Methods (8 total)

1. Instance Discovery (2 methods)

   • list_instances() - List all instance IDs
   • list_instances_by_status() - Filter instances by status

2. Execution Inspection (3 methods)

   • list_executions() - List execution IDs for an instance
   • read_execution() - Read history for a specific execution
   • latest_execution_id() - Get the latest execution ID

3. Metadata Access (2 methods)

   • get_instance_info() - Get comprehensive instance information
   • get_execution_info() - Get detailed execution information

4. System Metrics (2 methods)

   • get_system_metrics() - Get system-wide metrics
   • get_queue_depths() - Get current queue depths

Usage

let client = Client::new(provider);

// Check if management features are available
if client.has_management_capability() {
    let instances = client.list_all_instances().await?;
    let metrics = client.get_system_metrics().await?;
    println!("System has {} instances", metrics.total_instances);
} else {
    println!("Management features not available");
}

Required Methods

fn list_instances<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<Vec<String>, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

List all known instance IDs.

Returns

Vector of instance IDs, typically sorted by creation time (newest first).

Use Cases

• Admin dashboards showing all workflows
• Bulk operations across instances
• Testing (verify instance creation)

Implementation Example

async fn list_instances(&self) -> Result<Vec<String>, String> {
    SELECT instance_id FROM instances ORDER BY created_at DESC
}

fn list_instances_by_status<'life0, 'life1, 'async_trait>( &'life0 self, status: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Vec<String>, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

List instances matching a status filter.

Parameters

• status - Filter by execution status: “Running”, “Completed”, “Failed”, “ContinuedAsNew”

Returns

Vector of instance IDs with the specified status.

Implementation Example

async fn list_instances_by_status(&self, status: &str) -> Result<Vec<String>, String> {
    SELECT i.instance_id FROM instances i
    JOIN executions e ON i.instance_id = e.instance_id AND i.current_execution_id = e.execution_id
    WHERE e.status = ?
    ORDER BY i.created_at DESC
}

fn list_executions<'life0, 'life1, 'async_trait>( &'life0 self, instance: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Vec<u64>, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

List all execution IDs for an instance.

Returns

Vector of execution IDs in ascending order: [1], [1, 2], [1, 2, 3], etc.

Multi-Execution Context

When an orchestration uses ContinueAsNew, multiple executions exist:

• Execution 1: Original execution
• Execution 2: First ContinueAsNew
• Execution 3: Second ContinueAsNew
• etc.

Implementation Example

async fn list_executions(&self, instance: &str) -> Result<Vec<u64>, String> {
    SELECT execution_id FROM executions
    WHERE instance_id = ?
    ORDER BY execution_id
}

fn read_history_with_execution_id<'life0, 'life1, 'async_trait>( &'life0 self, instance: &'life1 str, execution_id: u64, ) -> Pin<Box<dyn Future<Output = Result<Vec<Event>, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Read the full event history for a specific execution within an instance.

Parameters

• instance - The ID of the orchestration instance.
• execution_id - The specific execution ID to read history for.

Returns

Vector of events in chronological order (oldest first).

Implementation Example

async fn read_execution(&self, instance: &str, execution_id: u64) -> Result<Vec<Event>, String> {
    SELECT event_data FROM history
    WHERE instance_id = ? AND execution_id = ?
    ORDER BY event_id
}

fn read_history<'life0, 'life1, 'async_trait>( &'life0 self, instance: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Vec<Event>, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Read the full event history for the latest execution of an instance.

Parameters

• instance - The ID of the orchestration instance.

Returns

Vector of events in chronological order (oldest first) for the latest execution.

Implementation

This method gets the latest execution ID and delegates to read_history_with_execution_id.

async fn read_history(&self, instance: &str) -> Result<Vec<Event>, ProviderError> {
    let execution_id = self.latest_execution_id(instance).await?;
    self.read_history_with_execution_id(instance, execution_id).await
}

fn latest_execution_id<'life0, 'life1, 'async_trait>( &'life0 self, instance: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<u64, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the latest (current) execution ID for an instance.

Parameters

• instance - The ID of the orchestration instance.

Implementation Pattern

async fn latest_execution_id(&self, instance: &str) -> Result<u64, String> {
    SELECT COALESCE(MAX(execution_id), 1) FROM executions WHERE instance_id = ?
}

fn get_instance_info<'life0, 'life1, 'async_trait>( &'life0 self, instance: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<InstanceInfo, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get comprehensive information about an instance.

Parameters

• instance - The ID of the orchestration instance.

Returns

Detailed instance information including status, output, and metadata.

Implementation Example

async fn get_instance_info(&self, instance: &str) -> Result<InstanceInfo, String> {
    SELECT i.*, e.status, e.output
    FROM instances i
    JOIN executions e ON i.instance_id = e.instance_id AND i.current_execution_id = e.execution_id
    WHERE i.instance_id = ?
}

fn get_execution_info<'life0, 'life1, 'async_trait>( &'life0 self, instance: &'life1 str, execution_id: u64, ) -> Pin<Box<dyn Future<Output = Result<ExecutionInfo, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get detailed information about a specific execution.

Parameters

• instance - The ID of the orchestration instance.
• execution_id - The specific execution ID.

Returns

Detailed execution information including status, output, and event count.

Implementation Example

async fn get_execution_info(&self, instance: &str, execution_id: u64) -> Result<ExecutionInfo, String> {
    SELECT e.*, COUNT(h.event_id) as event_count
    FROM executions e
    LEFT JOIN history h ON e.instance_id = h.instance_id AND e.execution_id = h.execution_id
    WHERE e.instance_id = ? AND e.execution_id = ?
    GROUP BY e.execution_id
}

fn get_system_metrics<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<SystemMetrics, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Get system-wide metrics for the orchestration engine.

Returns

System metrics including instance counts, execution counts, and status breakdown.

Implementation Example

async fn get_system_metrics(&self) -> Result<SystemMetrics, String> {
    SELECT
        COUNT(*) as total_instances,
        SUM(CASE WHEN e.status = 'Running' THEN 1 ELSE 0 END) as running_instances,
        SUM(CASE WHEN e.status = 'Completed' THEN 1 ELSE 0 END) as completed_instances,
        SUM(CASE WHEN e.status = 'Failed' THEN 1 ELSE 0 END) as failed_instances
    FROM instances i
    JOIN executions e ON i.instance_id = e.instance_id AND i.current_execution_id = e.execution_id
}

fn get_queue_depths<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<QueueDepths, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Get the current depths of the internal work queues.

Returns

Queue depths for orchestrator and worker queues.

Note: Timer queue depth is not applicable since timers are handled via delayed visibility in the orchestrator queue.

Implementation Example

async fn get_queue_depths(&self) -> Result<QueueDepths, String> {
    SELECT
        (SELECT COUNT(*) FROM orchestrator_queue WHERE lock_token IS NULL) as orchestrator_queue,
        (SELECT COUNT(*) FROM worker_queue WHERE lock_token IS NULL) as worker_queue
}

fn list_children<'life0, 'life1, 'async_trait>( &'life0 self, instance_id: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Vec<String>, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

List direct children of an instance.

Returns instance IDs that have parent_instance_id = instance_id. Returns empty vec if instance has no children or doesn’t exist.

Implementation Example

async fn list_children(&self, instance_id: &str) -> Result<Vec<String>, ProviderError> {
    SELECT instance_id FROM instances WHERE parent_instance_id = ?
}

fn get_parent_id<'life0, 'life1, 'async_trait>( &'life0 self, instance_id: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Option<String>, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the parent instance ID.

Returns Some(parent_id) for sub-orchestrations, None for root orchestrations. Returns Err if instance doesn’t exist.

Implementation Example

async fn get_parent_id(&self, instance_id: &str) -> Result<Option<String>, ProviderError> {
    SELECT parent_instance_id FROM instances WHERE instance_id = ?
}

fn delete_instances_atomic<'life0, 'life1, 'async_trait>( &'life0 self, ids: &'life1 [String], force: bool, ) -> Pin<Box<dyn Future<Output = Result<DeleteInstanceResult, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Atomically delete a batch of instances.

Parameters

• ids - Instance IDs to delete.
• force - If true, delete regardless of status. If false, all instances must be terminal.

Provider Contract (MUST implement)

1. Atomicity: All deletions MUST be atomic (all-or-nothing). If any instance fails validation, the entire batch MUST be rolled back.

2. Force semantics:

   • force=false: Return error if ANY instance has status = 'Running'
   • force=true: Delete regardless of status (for stuck/abandoned instances)

3. Orphan detection: Atomically check for children not in ids:

   SELECT instance_id FROM instances
   WHERE parent_instance_id IN (?) AND instance_id NOT IN (?)

   If any exist, return error (race condition: new child spawned after get_instance_tree).

4. Complete cleanup: Remove ALL related data in a single atomic operation:

   • event history
   • executions
   • orchestrator queue entries
   • worker queue entries
   • instance locks
   • instance metadata

5. Prevent ack recreation: When force-deleting Running instances, the removal of the instance lock prevents in-flight ack_orchestration_item from recreating state.

Race Condition Protection

This method MUST validate within the transaction that no orphans would be created:

• If any instance has parent_instance_id pointing to an instance in ids, but that child is NOT also in ids, return an error.
• This prevents orphans from children spawned between get_instance_tree() and this call.

Transaction Semantics

All instances must be deleted atomically (all-or-nothing). The orphan check must be done within the transaction to prevent TOCTOU races.

Implementation Notes

• Delete all related data: event history, executions, orchestrator queue, worker queue, instance locks, instance metadata
• Order within transaction doesn’t matter for correctness (single atomic transaction)
• Count deletions and aggregate into result

fn delete_instance_bulk<'life0, 'async_trait>( &'life0 self, filter: InstanceFilter, ) -> Pin<Box<dyn Future<Output = Result<DeleteInstanceResult, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Delete multiple orchestration instances matching the filter criteria.

Only instances in terminal states (Completed, Failed) are eligible. Running instances are silently skipped (not an error).

Parameters

• filter - Criteria for selecting instances to delete. All criteria are ANDed.

Filter Behavior

• instance_ids: Allowlist of specific IDs to consider
• completed_before: Only delete instances completed before this timestamp (ms)
• limit: Maximum number of instances to delete (applied last)

Returns

Aggregated counts of all deleted data across all deleted instances.

Safety

• Running instances are NEVER deleted (silently skipped)
• Use limit to avoid long-running transactions
• Sub-orchestrations are skipped (only roots are deleted with cascade)

fn prune_executions<'life0, 'life1, 'async_trait>( &'life0 self, instance_id: &'life1 str, options: PruneOptions, ) -> Pin<Box<dyn Future<Output = Result<PruneResult, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Prune old executions from a single instance.

Use this for ContinueAsNew workflows that accumulate many executions.

Parameters

• instance_id - The instance to prune executions from.
• options - Criteria for selecting executions to delete. All criteria are ANDed.

Provider Contract (MUST implement)

1. Current execution protection: The current execution of the instance MUST NEVER be pruned, regardless of options.

2. Running execution protection: Executions with status ‘Running’ MUST NEVER be pruned.

3. Atomicity: All deletions for a single prune call MUST be atomic (all-or-nothing).

keep_last Semantics

Since current_execution_id is always the highest execution_id:

• keep_last: None → prune all except current
• keep_last: Some(0) → same as None (current is protected)
• keep_last: Some(1) → same as above (top 1 = current)
• keep_last: Some(N) → keep current + (N-1) most recent

All three (None, Some(0), Some(1)) are equivalent because the current execution is always protected regardless of this setting.

Implementation Example

DELETE FROM executions
WHERE instance_id = ?
  AND execution_id != ?  -- current_execution_id (CRITICAL)
  AND status != 'Running'
  AND execution_id NOT IN (
    SELECT execution_id FROM executions
    WHERE instance_id = ?
    ORDER BY execution_id DESC
    LIMIT ?  -- keep_last
  )

fn prune_executions_bulk<'life0, 'async_trait>( &'life0 self, filter: InstanceFilter, options: PruneOptions, ) -> Pin<Box<dyn Future<Output = Result<PruneResult, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Prune old executions from multiple instances matching the filter.

Applies the same prune options to all matching instances.

Parameters

• filter - Criteria for selecting instances to process.
• options - Criteria for selecting executions to delete within each instance.

Returns

Aggregated counts including how many instances were processed.

Provided Methods

fn get_instance_tree<'life0, 'life1, 'async_trait>( &'life0 self, instance_id: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<InstanceTree, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the full instance tree rooted at the given instance.

Returns all instances in the tree: the root, all children, grandchildren, etc.

Default implementation uses list_children recursively.

fn delete_instance<'life0, 'life1, 'async_trait>( &'life0 self, instance_id: &'life1 str, force: bool, ) -> Pin<Box<dyn Future<Output = Result<DeleteInstanceResult, ProviderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Delete a single orchestration instance and all its associated data.

This removes the instance, all executions, all history events, and any pending queue messages (orchestrator, worker, timer).

Default Implementation

Uses primitives: get_parent_id, get_instance_tree, delete_instances_atomic. Providers can override for better performance if needed.

Parameters

• instance_id - The ID of the instance to delete.
• force - If true, delete even if the instance is in Running state. WARNING: Force delete only removes stored state; it does NOT cancel in-flight tokio tasks. Use cancel_instance first for graceful termination.

Returns

• Ok(DeleteResult) - Details of what was deleted.
• Err(ProviderError) with is_retryable() = false for:
  • Instance is running and force=false
  • Instance is a sub-orchestration (must delete root instead)
  • Instance not found

Safety

• Deleting a running instance with force=true may cause in-flight operations to fail when they try to persist state.
• Sub-orchestrations cannot be deleted directly; delete the root to cascade.
• The instance lock entry is removed to prevent zombie recreation.

Implementors