axonml-server 0.6.2

//! End-to-End User Flow — Integration Tests
//!
//! Full lifecycle integration tests that exercise multi-step workflows across
//! the AxonML server API. Includes: complete user registration/login/logout
//! flow with email verification handling, model CRUD lifecycle (create, get,
//! update, list versions, delete, verify deletion), training run lifecycle
//! (create, get, record metrics, stop, delete), inference endpoint lifecycle
//! (create, get, start, stop, delete), hub browse and cache flow, system info
//! and GPU listing, admin operations (user list, stats), dataset operations
//! with analysis, a complete ML pipeline simulation (datasets -> model ->
//! training -> inference -> cleanup), concurrent API request stress testing,
//! and error handling for invalid JSON, missing fields, and 404 responses.
//!
//! # File
//! `crates/axonml-server/tests/e2e_user_flows.rs`
//!
//! # Author
//! Andrew Jewell Sr. — AutomataNexus LLC
//! ORCID: 0009-0005-2158-7060
//!
//! # Updated
//! April 16, 2026 11:15 PM EST
//!
//! # Disclaimer
//! Use at own risk. This software is provided "as is", without warranty of any
//! kind, express or implied. The author and AutomataNexus shall not be held
//! liable for any damages arising from the use of this software.

mod common;

use common::*;
use serde_json::Value;
use std::time::Duration;

// =============================================================================
// Test Helpers
// =============================================================================

/// Skip test if server not running or DB not initialized
macro_rules! require_server {
    () => {
        if !is_server_running().await {
            eprintln!("SKIP: server not running at {}", TEST_API_URL);
            return;
        }
        let _c = test_client();
        if login_as_admin(&_c).await.is_err() {
            eprintln!("SKIP: admin login failed (run AxonML_DB_Init.sh)");
            return;
        }
    };
}

// =============================================================================
// Complete User Registration and Login Flow
// =============================================================================

#[tokio::test]
async fn test_complete_registration_login_flow() {
    require_server!();

    let client = test_client();
    let unique_id = chrono::Utc::now().timestamp_millis();
    let email = format!("e2e_user_{}@test.local", unique_id);
    let password =
        std::env::var("AXONML_TEST_PASSWORD").unwrap_or_else(|_| format!("TestPw_{}!", unique_id));

    // Step 1: Register new user
    let register_response = client
        .post(format!("{}/api/auth/register", TEST_API_URL))
        .json(&serde_json::json!({
            "name": "E2E Test User",
            "email": email,
            "password": &password
        }))
        .send()
        .await
        .expect("Registration request failed");

    let status = register_response.status().as_u16();
    assert!(
        status == 200 || status == 201,
        "Registration should succeed, got {}",
        status
    );

    // Step 2: Try login - should fail with 403 (email verification required)
    let login_result = login(&client, &email, &password).await;
    // New users need email verification, so login should fail with 403
    // This is expected behavior - use admin account for full flow testing
    if login_result.is_err() {
        eprintln!("Note: New user login requires email verification (expected behavior)");
        return; // Test passes - registration worked, verification required
    }
    let token = login_result.unwrap();

    // Step 3: Verify user info
    let me_response = auth_get(&client, "/api/auth/me", &token)
        .await
        .expect("Me request failed");

    assert!(me_response.status().is_success(), "Should get user info");

    let user_info: Value = me_response.json().await.expect("Failed to parse JSON");
    assert_eq!(user_info["email"], email);
    assert_eq!(user_info["name"], "E2E Test User");

    // Step 4: Logout
    let logout_response = auth_post(&client, "/api/auth/logout", &token, serde_json::json!({}))
        .await
        .expect("Logout request failed");

    // Logout should succeed
    let status = logout_response.status().as_u16();
    assert!(status == 200 || status == 204, "Logout should succeed");

    // Step 5: Verify token is invalidated (should fail after logout)
    // Note: Depending on implementation, token might still work for a short time
    tokio::time::sleep(Duration::from_millis(100)).await;

    let post_logout_response = auth_get(&client, "/api/auth/me", &token)
        .await
        .expect("Post-logout request failed");

    // Token should be invalid after logout (401) or might still work briefly (200)
    let status = post_logout_response.status().as_u16();
    assert!(
        status == 200 || status == 401,
        "Token should be invalid or still valid briefly, got {}",
        status
    );
}

// =============================================================================
// Model Upload and Management Flow
// =============================================================================

#[tokio::test]
async fn test_model_lifecycle_flow() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Step 1: List initial models
    let list_response = auth_get(&client, "/api/models", &token)
        .await
        .expect("List models failed");

    assert!(list_response.status().is_success(), "Should list models");
    let _initial_models: Vec<Value> = list_response.json().await.expect("Failed to parse");

    // Step 2: Create a new model
    let unique_name = format!("e2e_model_{}", chrono::Utc::now().timestamp_millis());
    let create_response = auth_post(
        &client,
        "/api/models",
        &token,
        serde_json::json!({
            "name": unique_name,
            "description": "E2E test model",
            "architecture": "MLP",
            "framework": "axonml"
        }),
    )
    .await
    .expect("Create model failed");

    if create_response.status().is_success() {
        let model: Value = create_response.json().await.expect("Failed to parse");
        let model_id = model["id"].as_str().expect("Model should have ID");

        // Step 3: Get model details
        let get_response = auth_get(&client, &format!("/api/models/{}", model_id), &token)
            .await
            .expect("Get model failed");

        assert!(
            get_response.status().is_success(),
            "Should get model details"
        );
        let model_details: Value = get_response.json().await.expect("Failed to parse");
        assert_eq!(model_details["name"], unique_name);

        // Step 4: Update model
        let update_response = auth_put(
            &client,
            &format!("/api/models/{}", model_id),
            &token,
            serde_json::json!({
                "description": "Updated E2E test model"
            }),
        )
        .await
        .expect("Update model failed");

        assert!(update_response.status().is_success(), "Should update model");

        // Step 5: List versions (should be empty)
        let versions_response = auth_get(
            &client,
            &format!("/api/models/{}/versions", model_id),
            &token,
        )
        .await
        .expect("List versions failed");

        assert!(
            versions_response.status().is_success(),
            "Should list versions"
        );

        // Step 6: Delete model
        let delete_response = auth_delete(&client, &format!("/api/models/{}", model_id), &token)
            .await
            .expect("Delete model failed");

        assert!(
            delete_response.status().is_success() || delete_response.status().as_u16() == 204,
            "Should delete model"
        );

        // Step 7: Verify model is deleted
        let verify_response = auth_get(&client, &format!("/api/models/{}", model_id), &token)
            .await
            .expect("Verify delete failed");

        assert_eq!(
            verify_response.status().as_u16(),
            404,
            "Model should be deleted"
        );
    }
}

// =============================================================================
// Training Run Flow
// =============================================================================

#[tokio::test]
async fn test_training_run_lifecycle() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Step 1: List initial runs
    let list_response = auth_get(&client, "/api/training/runs", &token)
        .await
        .expect("List runs failed");

    assert!(list_response.status().is_success(), "Should list runs");

    // Step 2: Create a new training run
    let unique_name = format!("e2e_run_{}", chrono::Utc::now().timestamp_millis());
    let create_response = auth_post(
        &client,
        "/api/training/runs",
        &token,
        serde_json::json!({
            "name": unique_name,
            "model_type": "MLP",
            "config": {
                "epochs": 1,
                "batch_size": 32,
                "learning_rate": 0.001,
                "optimizer": "adam"
            }
        }),
    )
    .await
    .expect("Create run failed");

    if create_response.status().is_success() || create_response.status().as_u16() == 201 {
        let run: Value = create_response.json().await.expect("Failed to parse");
        let run_id = run["id"].as_str().expect("Run should have ID");

        // Step 3: Get run details
        let get_response = auth_get(&client, &format!("/api/training/runs/{}", run_id), &token)
            .await
            .expect("Get run failed");

        assert!(get_response.status().is_success(), "Should get run details");
        let run_details: Value = get_response.json().await.expect("Failed to parse");
        assert_eq!(run_details["name"], unique_name);

        // Step 4: Record metrics
        let metrics_response = auth_post(
            &client,
            &format!("/api/training/runs/{}/metrics", run_id),
            &token,
            serde_json::json!({
                "epoch": 0,
                "step": 10,
                "loss": 0.5,
                "accuracy": 0.75
            }),
        )
        .await
        .expect("Record metrics failed");

        // Metrics recording should succeed (or 500 if db time series not configured)
        let status = metrics_response.status().as_u16();
        if status == 500 {
            eprintln!("Note: Metrics recording returned 500 - time series may not be configured");
        } else {
            assert!(
                status == 200 || status == 201 || status == 204,
                "Should record metrics, got {}",
                status
            );
        }

        // Step 5: Get metrics
        let get_metrics_response = auth_get(
            &client,
            &format!("/api/training/runs/{}/metrics", run_id),
            &token,
        )
        .await
        .expect("Get metrics failed");

        assert!(
            get_metrics_response.status().is_success(),
            "Should get metrics"
        );

        // Step 6: Stop run
        let stop_response = auth_post(
            &client,
            &format!("/api/training/runs/{}/stop", run_id),
            &token,
            serde_json::json!({}),
        )
        .await
        .expect("Stop run failed");

        assert!(stop_response.status().is_success(), "Should stop run");

        // Step 7: Delete run
        let delete_response =
            auth_delete(&client, &format!("/api/training/runs/{}", run_id), &token)
                .await
                .expect("Delete run failed");

        let status = delete_response.status().as_u16();
        assert!(
            status == 200 || status == 204,
            "Should delete run, got {}",
            status
        );
    }
}

// =============================================================================
// Inference Endpoint Flow
// =============================================================================

#[tokio::test]
async fn test_inference_endpoint_lifecycle() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Step 1: List initial endpoints
    let list_response = auth_get(&client, "/api/inference/endpoints", &token)
        .await
        .expect("List endpoints failed");

    assert!(list_response.status().is_success(), "Should list endpoints");

    // Step 2: Create an endpoint
    let unique_name = format!("e2e_endpoint_{}", chrono::Utc::now().timestamp_millis());
    let create_response = auth_post(
        &client,
        "/api/inference/endpoints",
        &token,
        serde_json::json!({
            "name": unique_name,
            "model_id": "test-model-id",
            "version_id": "v1"
        }),
    )
    .await
    .expect("Create endpoint failed");

    // Endpoint creation might fail if model doesn't exist, that's ok for this test
    if create_response.status().is_success() || create_response.status().as_u16() == 201 {
        let endpoint: Value = create_response.json().await.expect("Failed to parse");
        let endpoint_id = endpoint["id"].as_str().expect("Endpoint should have ID");

        // Step 3: Get endpoint details
        let get_response = auth_get(
            &client,
            &format!("/api/inference/endpoints/{}", endpoint_id),
            &token,
        )
        .await
        .expect("Get endpoint failed");

        assert!(
            get_response.status().is_success(),
            "Should get endpoint details"
        );

        // Step 4: Start endpoint
        let start_response = auth_post(
            &client,
            &format!("/api/inference/endpoints/{}/start", endpoint_id),
            &token,
            serde_json::json!({}),
        )
        .await
        .expect("Start endpoint failed");

        // Start might fail if model doesn't exist, that's ok
        let status = start_response.status().as_u16();
        assert!(
            status == 200 || status == 400 || status == 404 || status == 500,
            "Start should return expected status, got {}",
            status
        );

        // Step 5: Stop endpoint
        let stop_response = auth_post(
            &client,
            &format!("/api/inference/endpoints/{}/stop", endpoint_id),
            &token,
            serde_json::json!({}),
        )
        .await
        .expect("Stop endpoint failed");

        // Stop should succeed even if not started
        assert!(
            stop_response.status().is_success() || stop_response.status().as_u16() == 400,
            "Stop should succeed or return error if not started"
        );

        // Step 6: Delete endpoint
        let delete_response = auth_delete(
            &client,
            &format!("/api/inference/endpoints/{}", endpoint_id),
            &token,
        )
        .await
        .expect("Delete endpoint failed");

        let status = delete_response.status().as_u16();
        assert!(
            status == 200 || status == 204,
            "Should delete endpoint, got {}",
            status
        );
    }
}

// =============================================================================
// Hub Browse and Download Flow
// =============================================================================

#[tokio::test]
async fn test_hub_browse_flow() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Step 1: List available models in hub
    let list_response = auth_get(&client, "/api/hub/models", &token)
        .await
        .expect("List hub models failed");

    assert!(
        list_response.status().is_success(),
        "Should list hub models"
    );

    let models: Vec<Value> = list_response.json().await.expect("Failed to parse");

    // Step 2: Get info for first model (if any)
    if !models.is_empty() {
        let model_name = models[0]["name"].as_str().expect("Model should have name");

        let info_response = auth_get(&client, &format!("/api/hub/models/{}", model_name), &token)
            .await
            .expect("Get model info failed");

        assert!(info_response.status().is_success(), "Should get model info");

        let model_info: Value = info_response.json().await.expect("Failed to parse");
        assert_eq!(model_info["name"], model_name);
    }

    // Step 3: Get cache info
    let cache_response = auth_get(&client, "/api/hub/cache", &token)
        .await
        .expect("Get cache info failed");

    assert!(
        cache_response.status().is_success(),
        "Should get cache info"
    );
}

// =============================================================================
// System Info Flow
// =============================================================================

#[tokio::test]
async fn test_system_info_flow() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Step 1: Get system info
    let info_response = auth_get(&client, "/api/system/info", &token)
        .await
        .expect("Get system info failed");

    assert!(
        info_response.status().is_success(),
        "Should get system info"
    );

    let info: Value = info_response.json().await.expect("Failed to parse");
    assert!(
        info.get("cpu_count").is_some() || info.get("cpus").is_some(),
        "Should have CPU info"
    );

    // Step 2: List GPUs
    let gpus_response = auth_get(&client, "/api/system/gpus", &token)
        .await
        .expect("List GPUs failed");

    assert!(gpus_response.status().is_success(), "Should list GPUs");
}

// =============================================================================
// Admin Operations Flow
// =============================================================================

#[tokio::test]
async fn test_admin_operations_flow() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Step 1: List users (admin only)
    let users_response = auth_get(&client, "/api/admin/users", &token)
        .await
        .expect("List users failed");

    assert!(
        users_response.status().is_success(),
        "Admin should list users"
    );

    let users: Vec<Value> = users_response.json().await.expect("Failed to parse");
    assert!(!users.is_empty(), "Should have at least admin user");

    // Step 2: Get admin stats
    let stats_response = auth_get(&client, "/api/admin/stats", &token)
        .await
        .expect("Get stats failed");

    assert!(
        stats_response.status().is_success(),
        "Admin should get stats"
    );
}

// =============================================================================
// Dataset Operations Flow
// =============================================================================

#[tokio::test]
async fn test_dataset_operations_flow() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Step 1: List datasets
    let list_response = auth_get(&client, "/api/datasets", &token)
        .await
        .expect("List datasets failed");

    assert!(list_response.status().is_success(), "Should list datasets");

    let datasets: Vec<Value> = list_response.json().await.expect("Failed to parse");

    // If there are datasets, test operations on first one
    if !datasets.is_empty() {
        let dataset_id = datasets[0]["id"].as_str().expect("Dataset should have ID");

        // Step 2: Get dataset details
        let get_response = auth_get(&client, &format!("/api/datasets/{}", dataset_id), &token)
            .await
            .expect("Get dataset failed");

        assert!(
            get_response.status().is_success(),
            "Should get dataset details"
        );

        // Step 3: Analyze dataset
        let analyze_response = auth_post(
            &client,
            &format!("/api/data/{}/analyze", dataset_id),
            &token,
            serde_json::json!({}),
        )
        .await
        .expect("Analyze dataset failed");

        // Analysis might fail if dataset file doesn't exist, that's ok
        let status = analyze_response.status().as_u16();
        assert!(
            status == 200 || status == 400 || status == 404 || status == 500,
            "Analyze should return expected status, got {}",
            status
        );
    }
}

// =============================================================================
// Complete ML Pipeline Flow
// =============================================================================

#[tokio::test]
async fn test_complete_ml_pipeline_flow() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // This test simulates a complete ML pipeline:
    // 1. Check available built-in datasets
    // 2. Create a model
    // 3. Start a training run
    // 4. Monitor progress
    // 5. Create inference endpoint
    // 6. Clean up

    // Step 1: Check built-in datasets
    let datasets_response = match auth_get(&client, "/api/builtin-datasets", &token).await {
        Ok(resp) => resp,
        Err(e) => {
            eprintln!(
                "Note: Builtin datasets request failed (network issue): {}",
                e
            );
            return; // Skip test on network failure
        }
    };

    assert!(
        datasets_response.status().is_success(),
        "Should list builtin datasets"
    );

    // Step 2: Create model
    let model_name = format!("pipeline_model_{}", chrono::Utc::now().timestamp_millis());
    let model_response = auth_post(
        &client,
        "/api/models",
        &token,
        serde_json::json!({
            "name": model_name,
            "description": "Pipeline test model",
            "architecture": "CNN",
            "framework": "axonml"
        }),
    )
    .await
    .expect("Create model failed");

    if model_response.status().is_success() {
        let model: Value = model_response.json().await.expect("Failed to parse");
        let model_id = model["id"].as_str().unwrap_or("unknown");

        // Step 3: Create training run
        let run_response = auth_post(
            &client,
            "/api/training/runs",
            &token,
            serde_json::json!({
                "name": format!("pipeline_run_{}", chrono::Utc::now().timestamp_millis()),
                "model_type": "CNN",
                "config": {
                    "epochs": 1,
                    "batch_size": 32,
                    "learning_rate": 0.001,
                    "optimizer": "adam"
                }
            }),
        )
        .await
        .expect("Create run failed");

        if run_response.status().is_success() || run_response.status().as_u16() == 201 {
            let run: Value = run_response.json().await.expect("Failed to parse");
            let run_id = run["id"].as_str().unwrap_or("unknown");

            // Step 4: Check run status
            let status_response =
                auth_get(&client, &format!("/api/training/runs/{}", run_id), &token)
                    .await
                    .expect("Get run status failed");

            assert!(
                status_response.status().is_success(),
                "Should get run status"
            );

            // Step 5: Stop run
            let _ = auth_post(
                &client,
                &format!("/api/training/runs/{}/stop", run_id),
                &token,
                serde_json::json!({}),
            )
            .await;

            // Step 6: Clean up run
            let _ = auth_delete(&client, &format!("/api/training/runs/{}", run_id), &token).await;
        }

        // Step 7: Clean up model
        let _ = auth_delete(&client, &format!("/api/models/{}", model_id), &token).await;
    }

    // Pipeline completed successfully
}

// =============================================================================
// Concurrent Operations Test
// =============================================================================

#[tokio::test]
async fn test_concurrent_api_requests() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Make multiple concurrent requests
    let futures = vec![
        auth_get(&client, "/api/models", &token),
        auth_get(&client, "/api/training/runs", &token),
        auth_get(&client, "/api/datasets", &token),
        auth_get(&client, "/api/inference/endpoints", &token),
        auth_get(&client, "/api/hub/models", &token),
        auth_get(&client, "/api/system/info", &token),
        auth_get(&client, "/api/builtin-datasets", &token),
        auth_get(&client, "/api/kaggle/status", &token),
    ];

    let results = futures::future::join_all(futures).await;

    // Check results - allow some network failures under load
    let mut success_count = 0;
    let mut network_errors = 0;
    for (i, result) in results.into_iter().enumerate() {
        match result {
            Ok(response) => {
                if response.status().is_success() {
                    success_count += 1;
                } else {
                    eprintln!("Request {} returned {}", i, response.status());
                }
            }
            Err(e) => {
                eprintln!("Request {} had network error: {}", i, e);
                network_errors += 1;
            }
        }
    }

    // At least half the requests should succeed
    assert!(
        success_count >= 4,
        "At least 4 requests should succeed, got {} (network errors: {})",
        success_count,
        network_errors
    );
}

// =============================================================================
// Error Handling Tests
// =============================================================================

#[tokio::test]
async fn test_invalid_json_handling() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Send invalid JSON to various endpoints
    let response = client
        .post(format!("{}/api/models", TEST_API_URL))
        .header("Authorization", format!("Bearer {}", token))
        .header("Content-Type", "application/json")
        .body("{ invalid json }")
        .send()
        .await
        .expect("Request failed");

    assert!(
        response.status().is_client_error(),
        "Should reject invalid JSON"
    );
}

#[tokio::test]
async fn test_missing_required_fields() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Create model without required fields
    let response = auth_post(
        &client,
        "/api/models",
        &token,
        serde_json::json!({
            // Missing "name" field
            "description": "Test"
        }),
    )
    .await
    .expect("Request failed");

    assert!(
        response.status().is_client_error(),
        "Should reject missing required fields"
    );
}

#[tokio::test]
async fn test_404_handling() {
    require_server!();

    let client = test_client();
    let token = login_as_admin(&client).await.expect("Login failed");

    // Request nonexistent resources
    let endpoints = vec![
        "/api/models/nonexistent-id-12345",
        "/api/training/runs/nonexistent-id-12345",
        "/api/datasets/nonexistent-id-12345",
        "/api/inference/endpoints/nonexistent-id-12345",
    ];

    for endpoint in endpoints {
        let response = auth_get(&client, endpoint, &token)
            .await
            .expect("Request failed");

        assert_eq!(
            response.status().as_u16(),
            404,
            "Endpoint {} should return 404",
            endpoint
        );
    }
}