use crate::hub::get_hub;
use crate::runtime::Runtime;
use crate::script::support::into_option_string;
use crate::support::W;
use crate::types::{DEFAULT_UA_AIPACK, DEFAULT_UA_BROWSER, WebOptions, WebResponse};
use crate::{Error, Result};
use mlua::{FromLua as _, IntoLua, Lua, LuaSerdeExt, Table, Value};
use reqwest::{Client, header};
use std::collections::HashMap;
use url::Url;
pub fn init_module(lua: &Lua, _runtime_context: &Runtime) -> Result<Table> {
let table = lua.create_table()?;
let web_get_fn = lua.create_function(web_get)?;
let web_post_fn = lua.create_function(web_post)?;
let parse_url_fn = lua.create_function(web_parse_url)?;
let resolve_href_fn = lua.create_function(web_resolve_href)?;
table.set("get", web_get_fn)?;
table.set("post", web_post_fn)?;
table.set("parse_url", parse_url_fn)?;
table.set("resolve_href", resolve_href_fn)?;
table.set("UA_AIPACK", DEFAULT_UA_AIPACK)?;
table.set("UA_BROWSER", DEFAULT_UA_BROWSER)?;
Ok(table)
}
pub fn web_parse_url(lua: &Lua, url: Value) -> mlua::Result<Value> {
let Some(url) = into_option_string(url, "aip.web.parse_url argument")? else {
return Ok(Value::Nil);
};
match Url::parse(&url) {
Ok(url) => Ok(W(url).into_lua(lua)?),
Err(err) => Err(crate::Error::Custom(format!("Cannot parse url '{url}'.\nCause: {err}")).into()),
}
}
fn web_resolve_href(lua: &Lua, (href_val, base_url_str): (Value, String)) -> mlua::Result<Value> {
let href_opt_str = into_option_string(href_val, "aip.web.resolve_href 'href' argument")?;
let Some(href_str) = href_opt_str else {
return Ok(Value::Nil);
};
if let Ok(parsed_href_url) = Url::parse(&href_str)
&& !parsed_href_url.scheme().is_empty()
{
return Ok(Value::String(lua.create_string(&href_str)?));
}
let base_url = Url::parse(&base_url_str).map_err(|e| {
Error::custom(format!(
"aip.web.resolve_href: Invalid base_url '{base_url_str}'.\nCause: {e}"
))
})?;
match base_url.join(&href_str) {
Ok(resolved_url) => Ok(Value::String(lua.create_string(resolved_url.as_str())?)),
Err(e) => Err(Error::custom(format!(
"aip.web.resolve_href: Failed to join href '{href_str}' with base_url '{base_url_str}'.\nCause: {e}"
))
.into()),
}
}
impl IntoLua for W<Url> {
fn into_lua(self, lua: &Lua) -> mlua::Result<Value> {
let url = self.0;
let table = lua.create_table()?;
table.set("scheme", url.scheme())?;
table.set("host", url.host_str())?;
table.set("port", url.port())?;
table.set("path", url.path())?;
let query = url.query_pairs().into_owned().collect::<HashMap<String, String>>();
let query_table = if query.is_empty() {
Value::Nil
} else {
lua.to_value(&query)?
};
table.set("query", query_table)?;
table.set("fragment", url.fragment())?;
table.set("username", url.username())?;
table.set("password", url.password())?;
table.set("url", url.as_str())?;
let mut page_url = format!("{}://{}", url.scheme(), url.host_str().unwrap_or_default());
if let Some(port) = url.port() {
page_url.push(':');
page_url.push_str(&format!("{port}"));
}
page_url.push_str(url.path());
table.set("page_url", page_url)?;
Ok(Value::Table(table))
}
}
fn web_get(lua: &Lua, (url, opts): (String, Option<Value>)) -> mlua::Result<Value> {
let rt = tokio::runtime::Handle::try_current().map_err(Error::TokioTryCurrent)?;
let res: mlua::Result<Value> = tokio::task::block_in_place(|| {
rt.block_on(async {
let mut builder = Client::builder();
let opts_val = opts.unwrap_or(Value::Nil);
let web_opts = WebOptions::from_lua(opts_val, lua)?;
let parse_response = web_opts.parse;
builder = web_opts.apply_to_reqwest_builder(builder);
let client = builder.build().map_err(crate::Error::from)?;
let res: mlua::Result<Value> = match client.get(&url).send().await {
Ok(response) => {
let web_res = WebResponse::from_reqwest_response(response, parse_response).await?;
Ok(web_res.into_lua(lua)?)
}
Err(err) => Err(crate::Error::custom(format!(
"\
Fail to do aip.web.get for url: {url}
Cause: {err}"
))
.into()),
};
if res.is_ok() {
get_hub().publish_sync(format!("-> lua web::get OK ({url}) "));
}
res
})
});
res
}
fn web_post(lua: &Lua, (url, data, opts): (String, Value, Option<Value>)) -> mlua::Result<Value> {
let rt = tokio::runtime::Handle::try_current().map_err(Error::TokioTryCurrent)?;
let res: mlua::Result<Value> = tokio::task::block_in_place(|| {
rt.block_on(async {
let mut builder = Client::builder();
let opts_val = opts.unwrap_or(Value::Nil);
let web_opts = WebOptions::from_lua(opts_val, lua)?;
let parse_response = web_opts.parse;
builder = web_opts.apply_to_reqwest_builder(builder);
let client = builder.build().map_err(crate::Error::from)?;
let mut request_builder = client.post(&url);
match data {
Value::String(s) => {
request_builder = request_builder
.header(header::CONTENT_TYPE, "plain/text")
.body(s.to_string_lossy());
}
Value::Table(table) => {
let json: serde_json::Value = serde_json::to_value(table).map_err(|err| {
crate::Error::custom(format!(
"Cannot searlize to json the argument given to the post.\n Cause: {err}"
))
})?;
request_builder = request_builder
.header(header::CONTENT_TYPE, "application/json")
.body(json.to_string());
}
_ => {
return Err(mlua::Error::RuntimeError(
"Data must be a string or a table".to_string(),
));
}
}
let res: mlua::Result<Value> = match request_builder.send().await {
Ok(response) => {
let web_res = WebResponse::from_reqwest_response(response, parse_response).await?;
Ok(web_res.into_lua(lua)?)
}
Err(err) => Err(crate::Error::custom(format!(
"\
Fail to do aip.web.post for url: {url}
Cause: {err}"
))
.into()),
};
if res.is_ok() {
get_hub().publish_sync(format!("-> lua web::post OK ({url}) "));
}
res
})
});
res
}
#[cfg(test)]
mod tests {
type Result<T> = core::result::Result<T, Box<dyn std::error::Error>>;
use crate::_test_support::{assert_contains, eval_lua, setup_lua};
use crate::script::aip_modules::aip_web;
use serde_json::Value as JsonValue;
use value_ext::JsonValueExt;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_script_aip_web_get_simple_ok() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
local url = "https://phet-dev.colorado.edu/html/build-an-atom/0.0.0-3/simple-text-only-test-page.html"
return aip.web.get(url)
"#;
let res = eval_lua(&lua, script)?;
let content = res.x_get_str("content")?;
assert_contains(content, "This page tests that simple text can be");
assert_eq!(res.x_get_i64("status")?, 200, "status code");
assert!(res.x_get_bool("success")?, "success should be true");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_script_aip_web_post_json_ok() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
local url = "https://postman-echo.com/post"
local res = aip.web.post(url, {some = "stuff"}, {parse = true})
return res
"#;
let res = eval_lua(&lua, script)?;
let content = res.pointer("/content").ok_or("Should have content")?;
assert_eq!(content.x_get_str("/json/some")?, "stuff");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_script_aip_web_get_invalid_url() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
local url = "https://this-cannot-go/anywhere-or-can-it.aip"
return aip.web.get(url)
"#;
let err = match eval_lua(&lua, script) {
Ok(_) => return Err("Should have returned an error".into()),
Err(e) => e,
};
let err_str = err.to_string();
assert_contains(&err_str, "Fail to do aip.web.get");
assert_contains(&err_str, "https://this-cannot-go/anywhere-or-can-it.aip");
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_parse_url_ok() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.parse_url("https://user:pass@example.com:8080/path/to/resource?key1=val1&key2=val2#fragment")
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(res.x_get_str("scheme")?, "https");
assert_eq!(res.x_get_str("host")?, "example.com");
assert_eq!(res.x_get_i64("port")?, 8080);
assert_eq!(res.x_get_str("path")?, "/path/to/resource");
assert_eq!(res.x_get_str("/query/key1")?, "val1");
assert_eq!(res.x_get_str("/query/key2")?, "val2");
assert_eq!(res.x_get_str("fragment")?, "fragment");
assert_eq!(res.x_get_str("username")?, "user");
assert_eq!(res.x_get_str("password")?, "pass");
assert_eq!(
res.x_get_str("url")?,
"https://user:pass@example.com:8080/path/to/resource?key1=val1&key2=val2#fragment"
);
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_parse_url_invalid() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.parse_url("not a valid url")
"#;
let err = match eval_lua(&lua, script) {
Ok(_) => return Err("Should have returned an error".into()),
Err(e) => e,
};
let err_str = err.to_string();
assert_contains(&err_str, "Cannot parse url 'not a valid url'");
assert_contains(&err_str, "relative URL without a base");
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_parse_url_nil() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.parse_url(nil)
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(res, JsonValue::Null, "Result should be JSON null for Lua nil");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_script_aip_web_get_with_headers_capture() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
local url = "https://postman-echo.com/response-headers?Content-Type=text/plain&Set-Cookie=session1=a;%20HttpOnly&Set-Cookie=session2=b&X-Custom=val_single"
local res = aip.web.get(url)
return res
"#;
let res = eval_lua(&lua, script)?;
let date_header = res.x_get_str("/headers/content-type")?;
assert!(!date_header.is_empty(), "text/plain; charset=utf-8");
let custom_header = res.x_get_str("/headers/set-cookie")?;
assert_eq!(custom_header, "session1=a; HttpOnly");
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_ok_absolute_href() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.resolve_href("https://another.com/page.html", "https://base.com/docs/")
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(res.as_str().ok_or("should be string")?, "https://another.com/page.html");
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_ok_relative_path_href() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.resolve_href("sub/page.html", "https://base.com/docs/")
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(
res.as_str().ok_or("should be string")?,
"https://base.com/docs/sub/page.html"
);
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_ok_absolute_path_href() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.resolve_href("/other/resource", "https://base.com/docs/path/")
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(
res.as_str().ok_or("should be string")?,
"https://base.com/other/resource"
);
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_ok_scheme_relative_href_https() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.resolve_href("//cdn.example.com/script.js", "https://base.com/docs/")
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(
res.as_str().ok_or("should be string")?,
"https://cdn.example.com/script.js"
);
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_ok_scheme_relative_href_http() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.resolve_href("//cdn.example.com/script.js", "http://base.com/docs/")
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(
res.as_str().ok_or("should be string")?,
"http://cdn.example.com/script.js"
);
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_ok_nil_href() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.resolve_href(nil, "https://base.com/docs/")
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(res, JsonValue::Null);
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_err_invalid_base_url() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.resolve_href("path", "not-a-base-url")
"#;
let err = match eval_lua(&lua, script) {
Ok(_) => return Err("Should have returned an error".into()),
Err(e) => e,
};
let err_str = err.to_string();
assert_contains(&err_str, "aip.web.resolve_href: Invalid base_url 'not-a-base-url'");
assert_contains(&err_str, "relative URL without a base");
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_ok_empty_href() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.resolve_href("", "https://base.com/docs/")
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(&res, "https://base.com/docs/");
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_ok_href_is_fragment() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r##"
return aip.web.resolve_href("#section1", "https://base.com/page.html")
"##;
let res = eval_lua(&lua, script)?;
assert_eq!(&res, "https://base.com/page.html#section1");
Ok(())
}
#[tokio::test]
async fn test_script_aip_web_resolve_href_ok_href_is_query() -> Result<()> {
let lua = setup_lua(aip_web::init_module, "web").await?;
let script = r#"
return aip.web.resolve_href("?key=val", "https://base.com/page.html")
"#;
let res = eval_lua(&lua, script)?;
assert_eq!(&res, "https://base.com/page.html?key=val");
Ok(())
}
}