runmat_runtime/builtins/io/http/

webread.rs

//! MATLAB-compatible `webread` builtin for HTTP/HTTPS downloads.

use std::collections::VecDeque;
use std::time::Duration;

use base64::engine::general_purpose::STANDARD as BASE64_ENGINE;
use base64::Engine;
use runmat_builtins::{
    BuiltinCompletionPolicy, BuiltinDescriptor, BuiltinErrorDescriptor, BuiltinOutputMode,
    BuiltinParamArity, BuiltinParamDescriptor, BuiltinParamType, BuiltinSignatureDescriptor,
    CellArray, CharArray, StructValue, Tensor, Value,
};
use runmat_macros::runtime_builtin;
use url::Url;

use super::transport::{
    self, decode_body_as_text, header_value, HttpMethod, HttpRequest, HEADER_CONTENT_TYPE,
};
use crate::builtins::common::spec::{
    BroadcastSemantics, BuiltinFusionSpec, BuiltinGpuSpec, ConstantStrategy, GpuOpKind,
    ReductionNaN, ResidencyPolicy, ShapeRequirements,
};
use crate::builtins::io::json::jsondecode::decode_json_text;
use crate::{build_runtime_error, gather_if_needed_async, BuiltinResult, RuntimeError};

const DEFAULT_TIMEOUT_SECONDS: f64 = 60.0;
const DEFAULT_USER_AGENT: &str = "RunMat webread/0.0";
const BUILTIN_NAME: &str = "webread";

const WEBREAD_OUTPUT: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
    name: "data",
    ty: BuiltinParamType::Any,
    arity: BuiltinParamArity::Required,
    default: None,
    description: "Downloaded payload decoded as JSON, text, or binary tensor.",
}];
const WEBREAD_INPUTS_URL: [BuiltinParamDescriptor; 1] = [BuiltinParamDescriptor {
    name: "url",
    ty: BuiltinParamType::StringScalar,
    arity: BuiltinParamArity::Required,
    default: None,
    description: "HTTP/HTTPS URL to fetch.",
}];
const WEBREAD_INPUTS_URL_OPTIONS: [BuiltinParamDescriptor; 2] = [
    BuiltinParamDescriptor {
        name: "url",
        ty: BuiltinParamType::StringScalar,
        arity: BuiltinParamArity::Required,
        default: None,
        description: "HTTP/HTTPS URL to fetch.",
    },
    BuiltinParamDescriptor {
        name: "optionsStruct",
        ty: BuiltinParamType::Any,
        arity: BuiltinParamArity::Required,
        default: None,
        description: "weboptions struct or option struct literal.",
    },
];
const WEBREAD_INPUTS_URL_QUERY_CELL: [BuiltinParamDescriptor; 2] = [
    BuiltinParamDescriptor {
        name: "url",
        ty: BuiltinParamType::StringScalar,
        arity: BuiltinParamArity::Required,
        default: None,
        description: "HTTP/HTTPS URL to fetch.",
    },
    BuiltinParamDescriptor {
        name: "queryParameters",
        ty: BuiltinParamType::Any,
        arity: BuiltinParamArity::Required,
        default: None,
        description: "Two-column cell array of query parameter names and values.",
    },
];
const WEBREAD_INPUTS_URL_NAME_VALUE: [BuiltinParamDescriptor; 3] = [
    BuiltinParamDescriptor {
        name: "url",
        ty: BuiltinParamType::StringScalar,
        arity: BuiltinParamArity::Required,
        default: None,
        description: "HTTP/HTTPS URL to fetch.",
    },
    BuiltinParamDescriptor {
        name: "name",
        ty: BuiltinParamType::StringScalar,
        arity: BuiltinParamArity::Variadic,
        default: None,
        description: "Option or query parameter name.",
    },
    BuiltinParamDescriptor {
        name: "value",
        ty: BuiltinParamType::Any,
        arity: BuiltinParamArity::Variadic,
        default: None,
        description: "Option or query parameter value.",
    },
];
const WEBREAD_INPUTS_URL_OPTIONS_NAME_VALUE: [BuiltinParamDescriptor; 4] = [
    BuiltinParamDescriptor {
        name: "url",
        ty: BuiltinParamType::StringScalar,
        arity: BuiltinParamArity::Required,
        default: None,
        description: "HTTP/HTTPS URL to fetch.",
    },
    BuiltinParamDescriptor {
        name: "optionsStruct",
        ty: BuiltinParamType::Any,
        arity: BuiltinParamArity::Required,
        default: None,
        description: "weboptions struct or option struct literal.",
    },
    BuiltinParamDescriptor {
        name: "name",
        ty: BuiltinParamType::StringScalar,
        arity: BuiltinParamArity::Variadic,
        default: None,
        description: "Option or query parameter name.",
    },
    BuiltinParamDescriptor {
        name: "value",
        ty: BuiltinParamType::Any,
        arity: BuiltinParamArity::Variadic,
        default: None,
        description: "Option or query parameter value.",
    },
];

const WEBREAD_SIGNATURES: [BuiltinSignatureDescriptor; 6] = [
    BuiltinSignatureDescriptor {
        label: "data = webread(url)",
        inputs: &WEBREAD_INPUTS_URL,
        outputs: &WEBREAD_OUTPUT,
    },
    BuiltinSignatureDescriptor {
        label: "data = webread(url, optionsStruct)",
        inputs: &WEBREAD_INPUTS_URL_OPTIONS,
        outputs: &WEBREAD_OUTPUT,
    },
    BuiltinSignatureDescriptor {
        label: "data = webread(url, queryParameters)",
        inputs: &WEBREAD_INPUTS_URL_QUERY_CELL,
        outputs: &WEBREAD_OUTPUT,
    },
    BuiltinSignatureDescriptor {
        label: "data = webread(url, name, value, ...)",
        inputs: &WEBREAD_INPUTS_URL_NAME_VALUE,
        outputs: &WEBREAD_OUTPUT,
    },
    BuiltinSignatureDescriptor {
        label: "data = webread(url, optionsStruct, name, value, ...)",
        inputs: &WEBREAD_INPUTS_URL_OPTIONS_NAME_VALUE,
        outputs: &WEBREAD_OUTPUT,
    },
    BuiltinSignatureDescriptor {
        label: "data = webread(url, queryParameters, name, value, ...)",
        inputs: &WEBREAD_INPUTS_URL_OPTIONS_NAME_VALUE,
        outputs: &WEBREAD_OUTPUT,
    },
];

const WEBREAD_ERROR_INVALID_ARGUMENT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
    code: "RM.WEBREAD.INVALID_ARGUMENT",
    identifier: Some("RunMat:webread:InvalidArgument"),
    when: "Argument type/shape does not match webread call contract.",
    message: "webread: invalid argument",
};
const WEBREAD_ERROR_INVALID_URL: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
    code: "RM.WEBREAD.INVALID_URL",
    identifier: Some("RunMat:webread:InvalidUrl"),
    when: "URL input is empty or invalid.",
    message: "webread: invalid URL",
};
const WEBREAD_ERROR_MISSING_OPTION_VALUE: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
    code: "RM.WEBREAD.MISSING_OPTION_VALUE",
    identifier: Some("RunMat:webread:MissingOptionValue"),
    when: "A name-value key has no corresponding value.",
    message: "webread: missing option value",
};
const WEBREAD_ERROR_INVALID_OPTION_VALUE: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
    code: "RM.WEBREAD.INVALID_OPTION_VALUE",
    identifier: Some("RunMat:webread:InvalidOptionValue"),
    when: "An option value fails validation.",
    message: "webread: invalid option value",
};
const WEBREAD_ERROR_INVALID_CREDENTIALS: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
    code: "RM.WEBREAD.INVALID_CREDENTIALS",
    identifier: Some("RunMat:webread:InvalidCredentials"),
    when: "Password is provided without username.",
    message: "webread: invalid credentials",
};
const WEBREAD_ERROR_TRANSPORT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
    code: "RM.WEBREAD.TRANSPORT",
    identifier: Some("RunMat:webread:Transport"),
    when: "HTTP transport fails.",
    message: "webread: transport failure",
};
const WEBREAD_ERROR_RESPONSE_JSON: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
    code: "RM.WEBREAD.RESPONSE_JSON",
    identifier: Some("RunMat:webread:ResponseJson"),
    when: "Response body cannot be decoded as JSON.",
    message: "webread: failed to parse JSON response",
};
const WEBREAD_ERROR_OUTPUT: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
    code: "RM.WEBREAD.OUTPUT",
    identifier: Some("RunMat:webread:Output"),
    when: "Output payload cannot be materialized.",
    message: "webread: output materialization failure",
};
const WEBREAD_ERROR_FLOW: BuiltinErrorDescriptor = BuiltinErrorDescriptor {
    code: "RM.WEBREAD.FLOW",
    identifier: Some("RunMat:webread:Flow"),
    when: "Nested flow fails while gathering inputs.",
    message: "webread: flow failure",
};

const WEBREAD_ERRORS: [BuiltinErrorDescriptor; 9] = [
    WEBREAD_ERROR_INVALID_ARGUMENT,
    WEBREAD_ERROR_INVALID_URL,
    WEBREAD_ERROR_MISSING_OPTION_VALUE,
    WEBREAD_ERROR_INVALID_OPTION_VALUE,
    WEBREAD_ERROR_INVALID_CREDENTIALS,
    WEBREAD_ERROR_TRANSPORT,
    WEBREAD_ERROR_RESPONSE_JSON,
    WEBREAD_ERROR_OUTPUT,
    WEBREAD_ERROR_FLOW,
];

pub const WEBREAD_DESCRIPTOR: BuiltinDescriptor = BuiltinDescriptor {
    signatures: &WEBREAD_SIGNATURES,
    output_mode: BuiltinOutputMode::Fixed,
    completion_policy: BuiltinCompletionPolicy::Public,
    errors: &WEBREAD_ERRORS,
};

#[allow(clippy::too_many_lines)]
#[runmat_macros::register_gpu_spec(builtin_path = "crate::builtins::io::http::webread")]
pub const GPU_SPEC: BuiltinGpuSpec = BuiltinGpuSpec {
    name: "webread",
    op_kind: GpuOpKind::Custom("http-get"),
    supported_precisions: &[],
    broadcast: BroadcastSemantics::None,
    provider_hooks: &[],
    constant_strategy: ConstantStrategy::InlineLiteral,
    residency: ResidencyPolicy::GatherImmediately,
    nan_mode: ReductionNaN::Include,
    two_pass_threshold: None,
    workgroup_size: None,
    accepts_nan_mode: false,
    notes: "HTTP requests always execute on the CPU; gpuArray inputs are gathered eagerly.",
};

fn webread_error(message: impl Into<String>) -> RuntimeError {
    webread_error_with(&WEBREAD_ERROR_INVALID_ARGUMENT, message)
}

fn webread_error_with(
    error: &'static BuiltinErrorDescriptor,
    message: impl Into<String>,
) -> RuntimeError {
    let mut builder = build_runtime_error(message).with_builtin(BUILTIN_NAME);
    if let Some(identifier) = error.identifier {
        builder = builder.with_identifier(identifier);
    }
    builder.build()
}

fn webread_error_with_source<E>(
    error: &'static BuiltinErrorDescriptor,
    message: impl Into<String>,
    source: E,
) -> RuntimeError
where
    E: std::error::Error + Send + Sync + 'static,
{
    let mut builder = build_runtime_error(message)
        .with_builtin(BUILTIN_NAME)
        .with_source(source);
    if let Some(identifier) = error.identifier {
        builder = builder.with_identifier(identifier);
    }
    builder.build()
}

fn remap_webread_flow<F>(
    error: &'static BuiltinErrorDescriptor,
    err: RuntimeError,
    message: F,
) -> RuntimeError
where
    F: FnOnce(&RuntimeError) -> String,
{
    let mut builder = build_runtime_error(message(&err))
        .with_builtin(BUILTIN_NAME)
        .with_source(err);
    if let Some(identifier) = error.identifier {
        builder = builder.with_identifier(identifier);
    }
    builder.build()
}

fn webread_flow_with_context(err: RuntimeError) -> RuntimeError {
    remap_webread_flow(&WEBREAD_ERROR_FLOW, err, |err| {
        format!("webread: {}", err.message())
    })
}

#[runmat_macros::register_fusion_spec(builtin_path = "crate::builtins::io::http::webread")]
pub const FUSION_SPEC: BuiltinFusionSpec = BuiltinFusionSpec {
    name: "webread",
    shape: ShapeRequirements::Any,
    constant_strategy: ConstantStrategy::InlineLiteral,
    elementwise: None,
    reduction: None,
    emits_nan: false,
    notes: "webread performs network I/O and terminates fusion graphs.",
};

#[runtime_builtin(
    name = "webread",
    category = "io/http",
    summary = "Read web content over HTTP/HTTPS with optional request options.",
    keywords = "webread,http get,rest client,json,api",
    accel = "sink",
    type_resolver(crate::builtins::io::type_resolvers::webread_type),
    descriptor(crate::builtins::io::http::webread::WEBREAD_DESCRIPTOR),
    builtin_path = "crate::builtins::io::http::webread"
)]
async fn webread_builtin(url: Value, rest: Vec<Value>) -> crate::BuiltinResult<Value> {
    let gathered_url = gather_if_needed_async(&url)
        .await
        .map_err(webread_flow_with_context)?;
    let gathered_args = gather_arguments(rest).await?;
    let url_text = expect_string_scalar(
        &gathered_url,
        "webread: URL must be a character vector or string scalar",
    )?;
    if url_text.trim().is_empty() {
        return Err(webread_error_with(
            &WEBREAD_ERROR_INVALID_URL,
            "webread: URL must not be empty",
        ));
    }
    let (options, query_params) = parse_arguments(gathered_args)?;
    execute_request(&url_text, options, &query_params)
}

async fn gather_arguments(values: Vec<Value>) -> BuiltinResult<Vec<Value>> {
    let mut out = Vec::with_capacity(values.len());
    for value in values {
        out.push(
            gather_if_needed_async(&value)
                .await
                .map_err(webread_flow_with_context)?,
        );
    }
    Ok(out)
}

fn parse_arguments(args: Vec<Value>) -> BuiltinResult<(WebReadOptions, Vec<(String, String)>)> {
    let mut queue: VecDeque<Value> = args.into();
    let mut options = WebReadOptions::default();
    let mut query_params = Vec::new();

    if matches!(queue.front(), Some(Value::Struct(_))) {
        if let Some(Value::Struct(struct_value)) = queue.pop_front() {
            process_struct_fields(&struct_value, &mut options, &mut query_params)?;
        }
    } else if matches!(queue.front(), Some(Value::Cell(_))) {
        if let Some(Value::Cell(cell)) = queue.pop_front() {
            append_query_from_cell(&cell, &mut query_params)?
        }
    }

    while let Some(name_value) = queue.pop_front() {
        let name = expect_string_scalar(
            &name_value,
            "webread: parameter names must be character vectors or string scalars",
        )?;
        let value = queue.pop_front().ok_or_else(|| {
            webread_error_with(
                &WEBREAD_ERROR_MISSING_OPTION_VALUE,
                "webread: missing value for name-value argument",
            )
        })?;
        process_name_value_pair(&name, &value, &mut options, &mut query_params)?;
    }

    Ok((options, query_params))
}

fn process_struct_fields(
    struct_value: &StructValue,
    options: &mut WebReadOptions,
    query_params: &mut Vec<(String, String)>,
) -> BuiltinResult<()> {
    for (key, value) in &struct_value.fields {
        process_name_value_pair(key, value, options, query_params)?;
    }
    Ok(())
}

fn process_name_value_pair(
    name: &str,
    value: &Value,
    options: &mut WebReadOptions,
    query_params: &mut Vec<(String, String)>,
) -> BuiltinResult<()> {
    let lower = name.to_ascii_lowercase();
    match lower.as_str() {
        "contenttype" => {
            options.content_type = parse_content_type(value)?;
            Ok(())
        }
        "timeout" => {
            options.timeout = parse_timeout(value)?;
            Ok(())
        }
        "headerfields" => {
            let headers = parse_header_fields(value)?;
            options.headers.extend(headers);
            Ok(())
        }
        "useragent" => {
            options.user_agent = Some(expect_string_scalar(
                value,
                "webread: UserAgent must be a character vector or string scalar",
            )?);
            Ok(())
        }
        "username" => {
            options.username = Some(expect_string_scalar(
                value,
                "webread: Username must be a character vector or string scalar",
            )?);
            Ok(())
        }
        "password" => {
            options.password = Some(expect_string_scalar(
                value,
                "webread: Password must be a character vector or string scalar",
            )?);
            Ok(())
        }
        "requestmethod" => {
            options.method = parse_request_method(value)?;
            Ok(())
        }
        "mediatype" => {
            // weboptions exposes MediaType for webwrite; accept and ignore for webread.
            expect_string_scalar(
                value,
                "webread: MediaType must be a character vector or string scalar",
            )?;
            Ok(())
        }
        "queryparameters" => append_query_from_value(value, query_params),
        _ => {
            let param_value = value_to_query_string(value, name)?;
            query_params.push((name.to_string(), param_value));
            Ok(())
        }
    }
}

fn append_query_from_value(
    value: &Value,
    query_params: &mut Vec<(String, String)>,
) -> BuiltinResult<()> {
    match value {
        Value::Struct(struct_value) => {
            for (key, val) in &struct_value.fields {
                let text = value_to_query_string(val, key)?;
                query_params.push((key.clone(), text));
            }
            Ok(())
        }
        Value::Cell(cell) => append_query_from_cell(cell, query_params),
        _ => Err(webread_error_with(
            &WEBREAD_ERROR_INVALID_OPTION_VALUE,
            "webread: QueryParameters must be a struct or cell array",
        )),
    }
}

fn append_query_from_cell(
    cell: &CellArray,
    query_params: &mut Vec<(String, String)>,
) -> BuiltinResult<()> {
    if cell.cols != 2 {
        return Err(webread_error(
            "webread: cell array of query parameters must have two columns",
        ));
    }
    for row in 0..cell.rows {
        let name_value = cell
            .get(row, 0)
            .map_err(|err| webread_error(format!("webread: {err}")))?;
        let value_value = cell
            .get(row, 1)
            .map_err(|err| webread_error(format!("webread: {err}")))?;
        let name = expect_string_scalar(
            &name_value,
            "webread: query parameter names must be text scalars",
        )?;
        let text = value_to_query_string(&value_value, &name)?;
        query_params.push((name, text));
    }
    Ok(())
}

fn execute_request(
    url_text: &str,
    options: WebReadOptions,
    query_params: &[(String, String)],
) -> BuiltinResult<Value> {
    let username_present = options
        .username
        .as_ref()
        .map(|s| !s.is_empty())
        .unwrap_or(false);
    let password_present = options
        .password
        .as_ref()
        .map(|s| !s.is_empty())
        .unwrap_or(false);
    if password_present && !username_present {
        return Err(webread_error_with(
            &WEBREAD_ERROR_INVALID_CREDENTIALS,
            "webread: Password requires a Username option",
        ));
    }

    let mut url = Url::parse(url_text).map_err(|err| {
        webread_error_with_source(
            &WEBREAD_ERROR_INVALID_URL,
            format!("webread: invalid URL '{url_text}': {err}"),
            err,
        )
    })?;
    if !query_params.is_empty() {
        {
            let mut pairs = url.query_pairs_mut();
            for (name, value) in query_params {
                pairs.append_pair(name, value);
            }
        }
    }
    let user_agent = options
        .user_agent
        .as_deref()
        .filter(|ua| !ua.trim().is_empty())
        .unwrap_or(DEFAULT_USER_AGENT)
        .to_string();

    let mut headers = options.headers.clone();
    let has_auth_header = headers
        .iter()
        .any(|(name, _)| name.eq_ignore_ascii_case("authorization"));
    if !has_auth_header {
        if let Some(username) = options.username.as_ref().filter(|s| !s.is_empty()) {
            let password = options.password.clone().unwrap_or_default();
            let token = BASE64_ENGINE.encode(format!("{username}:{password}"));
            headers.push(("Authorization".to_string(), format!("Basic {token}")));
        }
    }

    let request = HttpRequest {
        url,
        method: HttpMethod::Get,
        headers,
        body: None,
        timeout: options.timeout,
        user_agent,
    };

    let response = transport::send_request(&request).map_err(|err| {
        webread_error_with_source(
            &WEBREAD_ERROR_TRANSPORT,
            err.message_with_prefix("webread"),
            err,
        )
    })?;

    let header_content_type =
        header_value(&response.headers, HEADER_CONTENT_TYPE).map(|value| value.to_string());
    let resolved = options.resolve_content_type(header_content_type.as_deref());

    match resolved {
        ResolvedContentType::Json => {
            let body = decode_body_as_text(&response.body, header_content_type.as_deref());
            let value = decode_json_text(&body).map_err(map_json_error)?;
            Ok(value)
        }
        ResolvedContentType::Text => {
            let text = decode_body_as_text(&response.body, header_content_type.as_deref());
            let array = CharArray::new_row(&text);
            Ok(Value::CharArray(array))
        }
        ResolvedContentType::Binary => {
            let data: Vec<f64> = response.body.iter().map(|b| f64::from(*b)).collect();
            let cols = response.body.len();
            let tensor = Tensor::new(data, vec![1, cols]).map_err(|err| {
                webread_error_with(&WEBREAD_ERROR_OUTPUT, format!("webread: {err}"))
            })?;
            Ok(Value::Tensor(tensor))
        }
    }
}

fn map_json_error(err: RuntimeError) -> RuntimeError {
    let message = if let Some(rest) = err.message().strip_prefix("jsondecode: ") {
        format!("webread: failed to parse JSON response ({rest})")
    } else {
        format!("webread: failed to parse JSON response ({})", err.message())
    };
    webread_error_with_source(&WEBREAD_ERROR_RESPONSE_JSON, message, err)
}

fn parse_header_fields(value: &Value) -> BuiltinResult<Vec<(String, String)>> {
    match value {
        Value::Struct(struct_value) => {
            let mut headers = Vec::with_capacity(struct_value.fields.len());
            for (key, val) in &struct_value.fields {
                let header_value = expect_string_scalar(
                    val,
                    "webread: header values must be character vectors or string scalars",
                )?;
                headers.push((key.clone(), header_value));
            }
            Ok(headers)
        }
        Value::Cell(cell) => {
            if cell.cols != 2 {
                return Err(webread_error(
                    "webread: HeaderFields cell array must have exactly two columns",
                ));
            }
            let mut headers = Vec::with_capacity(cell.rows);
            for row in 0..cell.rows {
                let name = cell
                    .get(row, 0)
                    .map_err(|err| webread_error(format!("webread: {err}")))?;
                let value = cell
                    .get(row, 1)
                    .map_err(|err| webread_error(format!("webread: {err}")))?;
                let header_name = expect_string_scalar(
                    &name,
                    "webread: header names must be character vectors or string scalars",
                )?;
                if header_name.trim().is_empty() {
                    return Err(webread_error("webread: header names must not be empty"));
                }
                let header_value = expect_string_scalar(
                    &value,
                    "webread: header values must be character vectors or string scalars",
                )?;
                headers.push((header_name, header_value));
            }
            Ok(headers)
        }
        _ => Err(webread_error(
            "webread: HeaderFields must be provided as a struct or cell array of name/value pairs",
        )),
    }
}

fn parse_content_type(value: &Value) -> BuiltinResult<ContentTypeHint> {
    let text = expect_string_scalar(
        value,
        "webread: ContentType must be a character vector or string scalar",
    )?;
    match text.trim().to_ascii_lowercase().as_str() {
        "auto" => Ok(ContentTypeHint::Auto),
        "json" => Ok(ContentTypeHint::Json),
        "text" | "char" | "string" => Ok(ContentTypeHint::Text),
        "binary" | "octet-stream" | "raw" => Ok(ContentTypeHint::Binary),
        other => Err(webread_error(format!(
            "webread: unsupported ContentType '{}'; use 'auto', 'json', 'text', or 'binary'",
            other
        ))),
    }
}

fn parse_timeout(value: &Value) -> BuiltinResult<Duration> {
    let seconds = numeric_scalar(value, "webread: Timeout must be a finite, positive scalar")?;
    if !seconds.is_finite() || seconds <= 0.0 {
        return Err(webread_error(
            "webread: Timeout must be a finite, positive scalar",
        ));
    }
    Ok(Duration::from_secs_f64(seconds))
}

fn parse_request_method(value: &Value) -> BuiltinResult<HttpMethod> {
    let text = expect_string_scalar(
        value,
        "webread: RequestMethod must be a character vector or string scalar",
    )?;
    let lower = text.trim().to_ascii_lowercase();
    match lower.as_str() {
        "get" | "auto" => Ok(HttpMethod::Get),
        other => Err(webread_error(format!(
            "webread: RequestMethod '{}' is not supported; expected 'auto' or 'get'",
            other
        ))),
    }
}

fn numeric_scalar(value: &Value, context: &str) -> BuiltinResult<f64> {
    match value {
        Value::Num(n) => Ok(*n),
        Value::Int(i) => Ok(i.to_f64()),
        Value::Tensor(tensor) => {
            if tensor.data.len() == 1 {
                Ok(tensor.data[0])
            } else {
                Err(webread_error(context))
            }
        }
        _ => Err(webread_error(context)),
    }
}

fn expect_string_scalar(value: &Value, context: &str) -> BuiltinResult<String> {
    match value {
        Value::String(s) => Ok(s.clone()),
        Value::CharArray(ca) if ca.rows == 1 => Ok(ca.data.iter().collect()),
        Value::StringArray(sa) if sa.data.len() == 1 => Ok(sa.data[0].clone()),
        _ => Err(webread_error(context)),
    }
}

fn value_to_query_string(value: &Value, name: &str) -> BuiltinResult<String> {
    match value {
        Value::String(s) => Ok(s.clone()),
        Value::CharArray(ca) if ca.rows == 1 => Ok(ca.data.iter().collect()),
        Value::StringArray(sa) if sa.data.len() == 1 => Ok(sa.data[0].clone()),
        Value::Num(n) => Ok(format!("{}", n)),
        Value::Int(i) => Ok(i.to_i64().to_string()),
        Value::Bool(b) => Ok(if *b { "true".into() } else { "false".into() }),
        Value::Tensor(tensor) => {
            if tensor.data.len() == 1 {
                Ok(format!("{}", tensor.data[0]))
            } else {
                Err(webread_error(format!(
                    "webread: query parameter '{}' must be scalar",
                    name
                )))
            }
        }
        Value::LogicalArray(array) => {
            if array.len() == 1 {
                Ok(if array.data[0] != 0 {
                    "true".into()
                } else {
                    "false".into()
                })
            } else {
                Err(webread_error(format!(
                    "webread: query parameter '{}' must be scalar",
                    name
                )))
            }
        }
        _ => Err(webread_error(format!(
            "webread: unsupported value type for query parameter '{}'",
            name
        ))),
    }
}

#[derive(Clone, Copy, Debug)]
enum ContentTypeHint {
    Auto,
    Text,
    Json,
    Binary,
}

#[derive(Clone, Copy, Debug)]
enum ResolvedContentType {
    Text,
    Json,
    Binary,
}

#[derive(Clone, Debug)]
struct WebReadOptions {
    content_type: ContentTypeHint,
    timeout: Duration,
    headers: Vec<(String, String)>,
    user_agent: Option<String>,
    username: Option<String>,
    password: Option<String>,
    method: HttpMethod,
}

impl Default for WebReadOptions {
    fn default() -> Self {
        Self {
            content_type: ContentTypeHint::Auto,
            timeout: Duration::from_secs_f64(DEFAULT_TIMEOUT_SECONDS),
            headers: Vec::new(),
            user_agent: None,
            username: None,
            password: None,
            method: HttpMethod::Get,
        }
    }
}

impl WebReadOptions {
    fn resolve_content_type(&self, header: Option<&str>) -> ResolvedContentType {
        match self.content_type {
            ContentTypeHint::Json => ResolvedContentType::Json,
            ContentTypeHint::Text => ResolvedContentType::Text,
            ContentTypeHint::Binary => ResolvedContentType::Binary,
            ContentTypeHint::Auto => infer_content_type(header),
        }
    }
}

fn infer_content_type(header: Option<&str>) -> ResolvedContentType {
    if let Some(raw) = header {
        let mime = raw
            .split(';')
            .next()
            .map(|part| part.trim().to_ascii_lowercase())
            .unwrap_or_default();
        if mime == "application/json" || mime == "text/json" || mime.ends_with("+json") {
            ResolvedContentType::Json
        } else if mime.starts_with("text/")
            || mime == "application/xml"
            || mime.ends_with("+xml")
            || mime == "application/xhtml+xml"
            || mime == "application/javascript"
            || mime == "application/x-www-form-urlencoded"
        {
            ResolvedContentType::Text
        } else {
            ResolvedContentType::Binary
        }
    } else {
        ResolvedContentType::Text
    }
}

#[cfg(test)]
pub(crate) mod tests {
    use super::*;
    use std::io::{Read, Write};
    use std::net::{TcpListener, TcpStream};
    use std::sync::mpsc;
    use std::thread;

    fn error_message(err: RuntimeError) -> String {
        err.message().to_string()
    }

    fn run_webread(url: Value, args: Vec<Value>) -> BuiltinResult<Value> {
        futures::executor::block_on(webread_builtin(url, args))
    }

    fn spawn_server<F>(handler: F) -> String
    where
        F: FnOnce(TcpStream) + Send + 'static,
    {
        let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server");
        let addr = listener.local_addr().unwrap();
        thread::spawn(move || {
            if let Ok((stream, _)) = listener.accept() {
                handler(stream);
            }
        });
        format!("http://{}", addr)
    }

    fn respond_with(mut stream: TcpStream, content_type: &str, body: &[u8]) {
        let response = format!(
            "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: {}\r\nConnection: close\r\n\r\n",
            body.len(),
            content_type
        );
        let _ = stream.write_all(response.as_bytes());
        let _ = stream.write_all(body);
    }

    fn read_request_headers(stream: &mut TcpStream) -> String {
        let mut buffer = Vec::new();
        let mut chunk = [0u8; 256];
        while let Ok(read) = stream.read(&mut chunk) {
            if read == 0 {
                break;
            }
            buffer.extend_from_slice(&chunk[..read]);
            if buffer.windows(4).any(|w| w == b"\r\n\r\n") {
                break;
            }
            if buffer.len() > 16 * 1024 {
                break;
            }
        }
        String::from_utf8_lossy(&buffer).to_string()
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_descriptor_signatures_cover_core_forms() {
        let labels: Vec<&str> = WEBREAD_DESCRIPTOR
            .signatures
            .iter()
            .map(|sig| sig.label)
            .collect();
        assert!(labels.contains(&"data = webread(url)"));
        assert!(labels.contains(&"data = webread(url, optionsStruct)"));
        assert!(labels.contains(&"data = webread(url, name, value, ...)"));
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_fetches_json_response() {
        let url = spawn_server(|mut stream| {
            let mut buffer = [0u8; 1024];
            let _ = stream.read(&mut buffer);
            respond_with(
                stream,
                "application/json",
                br#"{"message":"hello","value":42}"#,
            );
        });

        let result = run_webread(Value::from(url), vec![]).expect("webread JSON response");

        match result {
            Value::Struct(struct_value) => {
                let message = struct_value.fields.get("message").expect("message field");
                let value = struct_value.fields.get("value").expect("value field");
                match message {
                    Value::CharArray(ca) => {
                        let text: String = ca.data.iter().collect();
                        assert_eq!(text, "hello");
                    }
                    other => panic!("expected char array, got {other:?}"),
                }
                match value {
                    Value::Num(n) => assert_eq!(*n, 42.0),
                    other => panic!("expected numeric value, got {other:?}"),
                }
            }
            other => panic!("expected struct, got {other:?}"),
        }
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_fetches_text_response() {
        let url = spawn_server(|mut stream| {
            let mut buffer = [0u8; 512];
            let _ = stream.read(&mut buffer);
            respond_with(stream, "text/plain; charset=utf-8", b"RunMat webread test");
        });

        let result = run_webread(Value::from(url), vec![]).expect("webread text response");

        match result {
            Value::CharArray(ca) => {
                let text: String = ca.data.iter().collect();
                assert_eq!(text, "RunMat webread test");
            }
            other => panic!("expected char array, got {other:?}"),
        }
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_fetches_binary_payload() {
        let payload = [1u8, 2, 3, 254, 255];
        let url = spawn_server(move |mut stream| {
            let mut buffer = [0u8; 512];
            let _ = stream.read(&mut buffer);
            respond_with(stream, "application/octet-stream", &payload);
        });

        let args = vec![Value::from("ContentType"), Value::from("binary")];
        let result = run_webread(Value::from(url), args).expect("webread binary response");

        match result {
            Value::Tensor(tensor) => {
                assert_eq!(tensor.shape, vec![1, 5]);
                let bytes: Vec<u8> = tensor.data.iter().map(|v| *v as u8).collect();
                assert_eq!(bytes, payload);
            }
            other => panic!("expected tensor, got {other:?}"),
        }
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_appends_query_parameters() {
        let (tx, rx) = mpsc::channel();
        let url = spawn_server(move |mut stream| {
            let request = read_request_headers(&mut stream);
            let _ = tx.send(request);
            respond_with(stream, "application/json", br#"{"ok":true}"#);
        });

        let args = vec![
            Value::from("count"),
            Value::Num(42.0),
            Value::from("ContentType"),
            Value::from("json"),
        ];
        let result = run_webread(Value::from(url.clone()), args).expect("webread query");
        match result {
            Value::Struct(struct_value) => {
                assert!(struct_value.fields.contains_key("ok"));
            }
            other => panic!("expected struct result, got {other:?}"),
        }
        let request = rx.recv().expect("request log");
        assert!(
            request.starts_with("GET /"),
            "unexpected request line: {request}"
        );
        assert!(
            request.contains("count=42"),
            "query parameters missing: {request}"
        );
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_struct_argument_supports_options_and_query() {
        let (tx, rx) = mpsc::channel();
        let url = spawn_server(move |mut stream| {
            let request = read_request_headers(&mut stream);
            let _ = tx.send(request);
            respond_with(stream, "application/json", br#"{"value":123}"#);
        });

        let mut fields = StructValue::new();
        fields
            .fields
            .insert("ContentType".to_string(), Value::from("json"));
        fields.fields.insert("limit".to_string(), Value::Num(5.0));

        let result = run_webread(Value::from(url.clone()), vec![Value::Struct(fields)])
            .expect("webread struct arg");

        let request = rx.recv().expect("request log");
        assert!(
            request.contains("GET /?limit=5"),
            "expected limit query parameter: {request}"
        );

        match result {
            Value::Struct(struct_value) => match struct_value.fields.get("value") {
                Some(Value::Num(n)) => assert_eq!(*n, 123.0),
                other => panic!("unexpected JSON decode result: {other:?}"),
            },
            other => panic!("expected struct, got {other:?}"),
        }
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_headerfields_struct_applies_custom_headers() {
        let (tx, rx) = mpsc::channel();
        let url = spawn_server(move |mut stream| {
            let request = read_request_headers(&mut stream);
            let _ = tx.send(request);
            respond_with(stream, "application/json", br#"{"ok":true}"#);
        });

        let mut headers = StructValue::new();
        headers
            .fields
            .insert("X-Test".to_string(), Value::from("RunMat"));

        let args = vec![
            Value::from("HeaderFields"),
            Value::Struct(headers),
            Value::from("ContentType"),
            Value::from("json"),
        ];

        let result = run_webread(Value::from(url), args).expect("webread header fields");
        assert!(matches!(result, Value::Struct(_)));

        let request = rx.recv().expect("request log");
        assert!(
            request.to_ascii_lowercase().contains("x-test: runmat"),
            "custom header missing: {request}"
        );
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_queryparameters_option_struct() {
        let (tx, rx) = mpsc::channel();
        let url = spawn_server(move |mut stream| {
            let request = read_request_headers(&mut stream);
            let _ = tx.send(request);
            respond_with(stream, "application/json", br#"{"ok":true}"#);
        });

        let mut params = StructValue::new();
        params.fields.insert("page".to_string(), Value::Num(2.0));

        let args = vec![
            Value::from("QueryParameters"),
            Value::Struct(params),
            Value::from("ContentType"),
            Value::from("json"),
        ];

        let result = run_webread(Value::from(url.clone()), args).expect("webread query parameters");
        assert!(matches!(result, Value::Struct(_)));

        let request = rx.recv().expect("request log");
        assert!(
            request.contains("page=2"),
            "query parameter missing: {request}"
        );
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_errors_on_missing_name_value_pair() {
        let err = run_webread(
            Value::from("https://example.com"),
            vec![Value::from("Timeout")],
        )
        .expect_err("expected missing value error");
        let err = error_message(err);
        assert!(
            err.contains("missing value"),
            "unexpected error message: {err}"
        );
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_rejects_non_positive_timeout() {
        let args = vec![Value::from("Timeout"), Value::Num(0.0)];
        let err = run_webread(Value::from("https://example.com"), args).expect_err("timeout error");
        let err = error_message(err);
        assert!(
            err.contains("Timeout must be a finite, positive scalar"),
            "unexpected error message: {err}"
        );
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_rejects_password_without_username() {
        let args = vec![Value::from("Password"), Value::from("secret")];
        let err = run_webread(Value::from("https://example.com"), args).expect_err("auth error");
        let err = error_message(err);
        assert!(
            err.contains("Password requires a Username"),
            "unexpected error message: {err}"
        );
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_rejects_unsupported_content_type() {
        let args = vec![Value::from("ContentType"), Value::from("table")];
        let err = run_webread(Value::from("https://example.com"), args).expect_err("format error");
        let err = error_message(err);
        assert!(
            err.contains("unsupported ContentType"),
            "unexpected error message: {err}"
        );
    }

    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
    #[test]
    fn webread_rejects_invalid_headerfields_shape() {
        let cell = crate::make_cell(
            vec![Value::from("A"), Value::from("B"), Value::from("C")],
            1,
            3,
        )
        .expect("make cell");

        let args = vec![Value::from("HeaderFields"), cell];
        let err = run_webread(Value::from("https://example.com"), args).expect_err("header error");
        let err = error_message(err);
        assert!(
            err.contains("HeaderFields cell array must have exactly two columns"),
            "unexpected error message: {err}"
        );
    }
}