halldyll_parser/

forms.rs

//! Form extraction for halldyll-parser
//!
//! This module handles extraction of:
//! - Login forms (username/password)
//! - Search forms
//! - Contact forms
//! - Newsletter/subscription forms
//! - File upload forms
//! - Generic form analysis

use scraper::{Html, ElementRef, Selector};
use serde::{Deserialize, Serialize};

use crate::types::ParserResult;

// ============================================================================
// TYPES
// ============================================================================

/// Represents an HTML form
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Form {
    /// Form ID attribute
    pub id: Option<String>,
    /// Form name attribute
    pub name: Option<String>,
    /// Form action URL
    pub action: Option<String>,
    /// HTTP method (GET, POST)
    pub method: FormMethod,
    /// Encoding type
    pub enctype: Option<String>,
    /// Form fields
    pub fields: Vec<FormField>,
    /// Detected form type
    pub form_type: FormType,
    /// Whether form has CSRF token
    pub has_csrf: bool,
    /// Whether form has captcha
    pub has_captcha: bool,
    /// Submit button text
    pub submit_text: Option<String>,
}

impl Form {
    /// Create a new form with default values
    pub fn new() -> Self {
        Self {
            id: None,
            name: None,
            action: None,
            method: FormMethod::Get,
            enctype: None,
            fields: Vec::new(),
            form_type: FormType::Unknown,
            has_csrf: false,
            has_captcha: false,
            submit_text: None,
        }
    }

    /// Check if form is a login form
    pub fn is_login(&self) -> bool {
        matches!(self.form_type, FormType::Login)
    }

    /// Check if form is a search form
    pub fn is_search(&self) -> bool {
        matches!(self.form_type, FormType::Search)
    }

    /// Check if form has file upload
    pub fn has_file_upload(&self) -> bool {
        self.fields.iter().any(|f| f.field_type == FieldType::File)
    }

    /// Get required fields
    pub fn required_fields(&self) -> Vec<&FormField> {
        self.fields.iter().filter(|f| f.required).collect()
    }

    /// Get field by name
    pub fn get_field(&self, name: &str) -> Option<&FormField> {
        self.fields.iter().find(|f| f.name.as_deref() == Some(name))
    }
}

impl Default for Form {
    fn default() -> Self {
        Self::new()
    }
}

/// HTTP form method
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "lowercase")]
pub enum FormMethod {
    #[default]
    Get,
    Post,
    Dialog,
}

impl From<&str> for FormMethod {
    fn from(s: &str) -> Self {
        match s.to_lowercase().as_str() {
            "post" => FormMethod::Post,
            "dialog" => FormMethod::Dialog,
            _ => FormMethod::Get,
        }
    }
}

/// Detected form type
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
pub enum FormType {
    /// Login form (username/password)
    Login,
    /// Registration form
    Registration,
    /// Search form
    Search,
    /// Contact form
    Contact,
    /// Newsletter/subscription form
    Newsletter,
    /// Password reset form
    PasswordReset,
    /// Checkout/payment form
    Checkout,
    /// Comment form
    Comment,
    /// File upload form
    Upload,
    /// Unknown form type
    #[default]
    Unknown,
}

/// Form field
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct FormField {
    /// Field name
    pub name: Option<String>,
    /// Field ID
    pub id: Option<String>,
    /// Field type
    pub field_type: FieldType,
    /// Field label
    pub label: Option<String>,
    /// Placeholder text
    pub placeholder: Option<String>,
    /// Default value
    pub value: Option<String>,
    /// Whether field is required
    pub required: bool,
    /// Whether field is disabled
    pub disabled: bool,
    /// Whether field is readonly
    pub readonly: bool,
    /// Autocomplete attribute
    pub autocomplete: Option<String>,
    /// Pattern attribute (for validation)
    pub pattern: Option<String>,
    /// Min length
    pub min_length: Option<u32>,
    /// Max length
    pub max_length: Option<u32>,
    /// Select options (for select fields)
    pub options: Vec<SelectOption>,
}

impl FormField {
    pub fn new(field_type: FieldType) -> Self {
        Self {
            name: None,
            id: None,
            field_type,
            label: None,
            placeholder: None,
            value: None,
            required: false,
            disabled: false,
            readonly: false,
            autocomplete: None,
            pattern: None,
            min_length: None,
            max_length: None,
            options: Vec::new(),
        }
    }

    /// Check if this is a password field
    pub fn is_password(&self) -> bool {
        matches!(self.field_type, FieldType::Password)
    }

    /// Check if this looks like an email field
    pub fn is_email(&self) -> bool {
        matches!(self.field_type, FieldType::Email) ||
        self.name.as_ref().map(|n| n.to_lowercase().contains("email")).unwrap_or(false) ||
        self.autocomplete.as_ref().map(|a| a.contains("email")).unwrap_or(false)
    }

    /// Check if this looks like a username field
    pub fn is_username(&self) -> bool {
        let name_lower = self.name.as_ref().map(|n| n.to_lowercase()).unwrap_or_default();
        let id_lower = self.id.as_ref().map(|n| n.to_lowercase()).unwrap_or_default();
        
        name_lower.contains("user") || name_lower.contains("login") ||
        id_lower.contains("user") || id_lower.contains("login") ||
        self.autocomplete.as_ref().map(|a| a.contains("username")).unwrap_or(false)
    }
}

/// Form field type
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "lowercase")]
pub enum FieldType {
    #[default]
    Text,
    Password,
    Email,
    Tel,
    Url,
    Number,
    Search,
    Date,
    DateTime,
    Time,
    Month,
    Week,
    Color,
    Range,
    File,
    Hidden,
    Checkbox,
    Radio,
    Select,
    Textarea,
    Submit,
    Button,
    Reset,
    Image,
}

impl From<&str> for FieldType {
    fn from(s: &str) -> Self {
        match s.to_lowercase().as_str() {
            "password" => FieldType::Password,
            "email" => FieldType::Email,
            "tel" | "telephone" | "phone" => FieldType::Tel,
            "url" => FieldType::Url,
            "number" => FieldType::Number,
            "search" => FieldType::Search,
            "date" => FieldType::Date,
            "datetime" | "datetime-local" => FieldType::DateTime,
            "time" => FieldType::Time,
            "month" => FieldType::Month,
            "week" => FieldType::Week,
            "color" => FieldType::Color,
            "range" => FieldType::Range,
            "file" => FieldType::File,
            "hidden" => FieldType::Hidden,
            "checkbox" => FieldType::Checkbox,
            "radio" => FieldType::Radio,
            "select" | "select-one" | "select-multiple" => FieldType::Select,
            "textarea" => FieldType::Textarea,
            "submit" => FieldType::Submit,
            "button" => FieldType::Button,
            "reset" => FieldType::Reset,
            "image" => FieldType::Image,
            _ => FieldType::Text,
        }
    }
}

/// Select option
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct SelectOption {
    pub value: String,
    pub text: String,
    pub selected: bool,
    pub disabled: bool,
}

// ============================================================================
// EXTRACTION FUNCTIONS
// ============================================================================

/// Extract all forms from HTML document
pub fn extract_forms(document: &Html) -> ParserResult<Vec<Form>> {
    let form_selector = Selector::parse("form").unwrap();
    let mut forms = Vec::new();

    for form_el in document.select(&form_selector) {
        if let Some(form) = extract_form(&form_el) {
            forms.push(form);
        }
    }

    Ok(forms)
}

/// Extract a single form
fn extract_form(element: &ElementRef) -> Option<Form> {
    let mut form = Form::new();

    // Basic attributes
    form.id = element.value().attr("id").map(|s| s.to_string());
    form.name = element.value().attr("name").map(|s| s.to_string());
    form.action = element.value().attr("action").map(|s| s.to_string());
    form.method = element.value().attr("method")
        .map(FormMethod::from)
        .unwrap_or_default();
    form.enctype = element.value().attr("enctype").map(|s| s.to_string());

    // Extract fields
    form.fields = extract_form_fields(element);

    // Detect form type
    form.form_type = detect_form_type(&form);

    // Detect CSRF token
    form.has_csrf = detect_csrf_token(&form);

    // Detect captcha
    form.has_captcha = detect_captcha(element);

    // Extract submit button text
    form.submit_text = extract_submit_text(element);

    Some(form)
}

/// Extract all fields from a form
fn extract_form_fields(form: &ElementRef) -> Vec<FormField> {
    let mut fields = Vec::new();

    // Input elements
    let input_sel = Selector::parse("input").unwrap();
    for input in form.select(&input_sel) {
        if let Some(field) = extract_input_field(&input) {
            fields.push(field);
        }
    }

    // Select elements
    let select_sel = Selector::parse("select").unwrap();
    for select in form.select(&select_sel) {
        if let Some(field) = extract_select_field(&select) {
            fields.push(field);
        }
    }

    // Textarea elements
    let textarea_sel = Selector::parse("textarea").unwrap();
    for textarea in form.select(&textarea_sel) {
        if let Some(field) = extract_textarea_field(&textarea) {
            fields.push(field);
        }
    }

    // Find and associate labels
    associate_labels(form, &mut fields);

    fields
}

/// Extract input field
fn extract_input_field(element: &ElementRef) -> Option<FormField> {
    let input_type = element.value().attr("type").unwrap_or("text");
    let mut field = FormField::new(FieldType::from(input_type));

    field.name = element.value().attr("name").map(|s| s.to_string());
    field.id = element.value().attr("id").map(|s| s.to_string());
    field.placeholder = element.value().attr("placeholder").map(|s| s.to_string());
    field.value = element.value().attr("value").map(|s| s.to_string());
    field.required = element.value().attr("required").is_some();
    field.disabled = element.value().attr("disabled").is_some();
    field.readonly = element.value().attr("readonly").is_some();
    field.autocomplete = element.value().attr("autocomplete").map(|s| s.to_string());
    field.pattern = element.value().attr("pattern").map(|s| s.to_string());
    field.min_length = element.value().attr("minlength").and_then(|s| s.parse().ok());
    field.max_length = element.value().attr("maxlength").and_then(|s| s.parse().ok());

    Some(field)
}

/// Extract select field
fn extract_select_field(element: &ElementRef) -> Option<FormField> {
    let mut field = FormField::new(FieldType::Select);

    field.name = element.value().attr("name").map(|s| s.to_string());
    field.id = element.value().attr("id").map(|s| s.to_string());
    field.required = element.value().attr("required").is_some();
    field.disabled = element.value().attr("disabled").is_some();

    // Extract options
    let option_sel = Selector::parse("option").unwrap();
    for option in element.select(&option_sel) {
        let opt = SelectOption {
            value: option.value().attr("value")
                .unwrap_or("")
                .to_string(),
            text: option.text().collect::<String>().trim().to_string(),
            selected: option.value().attr("selected").is_some(),
            disabled: option.value().attr("disabled").is_some(),
        };
        field.options.push(opt);
    }

    Some(field)
}

/// Extract textarea field
fn extract_textarea_field(element: &ElementRef) -> Option<FormField> {
    let mut field = FormField::new(FieldType::Textarea);

    field.name = element.value().attr("name").map(|s| s.to_string());
    field.id = element.value().attr("id").map(|s| s.to_string());
    field.placeholder = element.value().attr("placeholder").map(|s| s.to_string());
    field.value = Some(element.text().collect::<String>());
    field.required = element.value().attr("required").is_some();
    field.disabled = element.value().attr("disabled").is_some();
    field.readonly = element.value().attr("readonly").is_some();
    field.min_length = element.value().attr("minlength").and_then(|s| s.parse().ok());
    field.max_length = element.value().attr("maxlength").and_then(|s| s.parse().ok());

    Some(field)
}

/// Associate labels with form fields
fn associate_labels(form: &ElementRef, fields: &mut [FormField]) {
    let label_sel = Selector::parse("label").unwrap();
    
    for label in form.select(&label_sel) {
        let label_text = label.text().collect::<String>().trim().to_string();
        
        // Find by "for" attribute
        if let Some(for_id) = label.value().attr("for") {
            for field in fields.iter_mut() {
                if field.id.as_deref() == Some(for_id) {
                    field.label = Some(label_text.clone());
                    break;
                }
            }
        }
    }
}

/// Detect form type based on fields and attributes
fn detect_form_type(form: &Form) -> FormType {
    let has_password = form.fields.iter().any(|f| f.field_type == FieldType::Password);
    let has_email = form.fields.iter().any(|f| f.is_email());
    let has_username = form.fields.iter().any(|f| f.is_username());
    let has_search = form.fields.iter().any(|f| f.field_type == FieldType::Search);
    let has_file = form.fields.iter().any(|f| f.field_type == FieldType::File);
    let has_textarea = form.fields.iter().any(|f| f.field_type == FieldType::Textarea);
    let password_count = form.fields.iter().filter(|f| f.field_type == FieldType::Password).count();

    // Check form attributes for hints
    let action_lower = form.action.as_ref().map(|a| a.to_lowercase()).unwrap_or_default();
    let name_lower = form.name.as_ref().map(|n| n.to_lowercase()).unwrap_or_default();
    let id_lower = form.id.as_ref().map(|i| i.to_lowercase()).unwrap_or_default();

    // Search form detection
    if has_search || 
       action_lower.contains("search") ||
       name_lower.contains("search") ||
       id_lower.contains("search") {
        return FormType::Search;
    }

    // Login form: username/email + single password
    if has_password && password_count == 1 && (has_email || has_username) {
        return FormType::Login;
    }

    // Registration: multiple passwords (password + confirm)
    if password_count >= 2 && has_email {
        return FormType::Registration;
    }

    // Password reset
    if has_password && !has_email && !has_username
        && (action_lower.contains("reset") || action_lower.contains("password") ||
           name_lower.contains("reset") || id_lower.contains("reset")) {
        return FormType::PasswordReset;
    }

    // Newsletter/subscription
    if has_email && !has_password && form.fields.len() <= 3
        && (action_lower.contains("subscribe") || action_lower.contains("newsletter") ||
           name_lower.contains("subscribe") || name_lower.contains("newsletter")) {
        return FormType::Newsletter;
    }

    // Contact form
    if has_email && has_textarea && !has_password {
        return FormType::Contact;
    }

    // Comment form
    if has_textarea && !has_password && 
       (action_lower.contains("comment") || name_lower.contains("comment") || id_lower.contains("comment")) {
        return FormType::Comment;
    }

    // Upload form
    if has_file {
        return FormType::Upload;
    }

    // Checkout form
    if action_lower.contains("checkout") || action_lower.contains("payment") ||
       action_lower.contains("order") || action_lower.contains("cart") {
        return FormType::Checkout;
    }

    FormType::Unknown
}

/// Detect CSRF token in form
fn detect_csrf_token(form: &Form) -> bool {
    let csrf_patterns = [
        "csrf", "token", "_token", "authenticity_token",
        "xsrf", "__requestverificationtoken", "anti-forgery",
    ];

    form.fields.iter().any(|f| {
        if f.field_type != FieldType::Hidden {
            return false;
        }
        
        let name_lower = f.name.as_ref().map(|n| n.to_lowercase()).unwrap_or_default();
        csrf_patterns.iter().any(|p| name_lower.contains(p))
    })
}

/// Detect captcha in form
fn detect_captcha(form: &ElementRef) -> bool {
    let html = form.html().to_lowercase();
    
    // Common captcha indicators
    html.contains("recaptcha") ||
    html.contains("hcaptcha") ||
    html.contains("captcha") ||
    html.contains("g-recaptcha") ||
    html.contains("cf-turnstile") ||
    html.contains("data-sitekey")
}

/// Extract submit button text
fn extract_submit_text(form: &ElementRef) -> Option<String> {
    // Check input[type=submit]
    let submit_sel = Selector::parse("input[type='submit'], button[type='submit'], button:not([type])").unwrap();
    
    if let Some(submit) = form.select(&submit_sel).next() {
        // For input, use value attribute
        if let Some(value) = submit.value().attr("value") {
            return Some(value.to_string());
        }
        // For button, use text content
        let text = submit.text().collect::<String>().trim().to_string();
        if !text.is_empty() {
            return Some(text);
        }
    }

    None
}

// ============================================================================
// CONVENIENCE FUNCTIONS
// ============================================================================

/// Get all login forms from document
pub fn get_login_forms(document: &Html) -> ParserResult<Vec<Form>> {
    let forms = extract_forms(document)?;
    Ok(forms.into_iter().filter(|f| f.is_login()).collect())
}

/// Get all search forms from document
pub fn get_search_forms(document: &Html) -> ParserResult<Vec<Form>> {
    let forms = extract_forms(document)?;
    Ok(forms.into_iter().filter(|f| f.is_search()).collect())
}

/// Get all contact forms from document
pub fn get_contact_forms(document: &Html) -> ParserResult<Vec<Form>> {
    let forms = extract_forms(document)?;
    Ok(forms.into_iter()
        .filter(|f| matches!(f.form_type, FormType::Contact))
        .collect())
}

/// Check if page has any forms
pub fn has_forms(document: &Html) -> bool {
    let form_selector = Selector::parse("form").unwrap();
    document.select(&form_selector).next().is_some()
}

/// Check if page has login form
pub fn has_login_form(document: &Html) -> bool {
    get_login_forms(document).map(|f| !f.is_empty()).unwrap_or(false)
}

/// Check if page has search form
pub fn has_search_form(document: &Html) -> bool {
    get_search_forms(document).map(|f| !f.is_empty()).unwrap_or(false)
}

// ============================================================================
// TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    fn parse_html(html: &str) -> Html {
        Html::parse_document(html)
    }

    #[test]
    fn test_extract_login_form() {
        let html = r#"
            <form action="/login" method="post">
                <input type="email" name="email" required>
                <input type="password" name="password" required>
                <input type="submit" value="Sign In">
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        
        assert_eq!(forms.len(), 1);
        assert_eq!(forms[0].form_type, FormType::Login);
        assert_eq!(forms[0].method, FormMethod::Post);
        assert_eq!(forms[0].submit_text, Some("Sign In".to_string()));
    }

    #[test]
    fn test_extract_search_form() {
        let html = r#"
            <form action="/search" method="get">
                <input type="search" name="q" placeholder="Search...">
                <button type="submit">Search</button>
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        
        assert_eq!(forms.len(), 1);
        assert_eq!(forms[0].form_type, FormType::Search);
        assert!(forms[0].is_search());
    }

    #[test]
    fn test_extract_contact_form() {
        let html = r#"
            <form action="/contact" method="post">
                <input type="text" name="name" required>
                <input type="email" name="email" required>
                <textarea name="message" required></textarea>
                <button type="submit">Send</button>
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        
        assert_eq!(forms.len(), 1);
        assert_eq!(forms[0].form_type, FormType::Contact);
    }

    #[test]
    fn test_extract_registration_form() {
        let html = r#"
            <form action="/register" method="post">
                <input type="email" name="email" required>
                <input type="password" name="password" required>
                <input type="password" name="password_confirm" required>
                <button type="submit">Register</button>
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        
        assert_eq!(forms.len(), 1);
        assert_eq!(forms[0].form_type, FormType::Registration);
    }

    #[test]
    fn test_detect_csrf_token() {
        let html = r#"
            <form action="/login" method="post">
                <input type="hidden" name="csrf_token" value="abc123">
                <input type="email" name="email">
                <input type="password" name="password">
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        
        assert!(forms[0].has_csrf);
    }

    #[test]
    fn test_detect_captcha() {
        let html = r#"
            <form action="/login" method="post">
                <input type="email" name="email">
                <input type="password" name="password">
                <div class="g-recaptcha" data-sitekey="xxx"></div>
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        
        assert!(forms[0].has_captcha);
    }

    #[test]
    fn test_extract_select_field() {
        let html = r#"
            <form>
                <select name="country" required>
                    <option value="">Select...</option>
                    <option value="us" selected>United States</option>
                    <option value="ca">Canada</option>
                </select>
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        let field = forms[0].get_field("country").unwrap();
        
        assert_eq!(field.field_type, FieldType::Select);
        assert_eq!(field.options.len(), 3);
        assert!(field.options[1].selected);
    }

    #[test]
    fn test_form_with_labels() {
        let html = r#"
            <form>
                <label for="email">Email Address</label>
                <input type="email" id="email" name="email">
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        let field = forms[0].get_field("email").unwrap();
        
        assert_eq!(field.label, Some("Email Address".to_string()));
    }

    #[test]
    fn test_newsletter_form() {
        let html = r#"
            <form action="/subscribe" method="post" id="newsletter">
                <input type="email" name="email" placeholder="Enter your email">
                <button type="submit">Subscribe</button>
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        
        assert_eq!(forms[0].form_type, FormType::Newsletter);
    }

    #[test]
    fn test_upload_form() {
        let html = r#"
            <form action="/upload" method="post" enctype="multipart/form-data">
                <input type="file" name="document" accept=".pdf,.doc">
                <button type="submit">Upload</button>
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        
        assert_eq!(forms[0].form_type, FormType::Upload);
        assert!(forms[0].has_file_upload());
    }

    #[test]
    fn test_has_forms() {
        let html = "<html><body><form></form></body></html>";
        let doc = parse_html(html);
        assert!(has_forms(&doc));

        let html_no_form = "<html><body><p>No forms here</p></body></html>";
        let doc_no_form = parse_html(html_no_form);
        assert!(!has_forms(&doc_no_form));
    }

    #[test]
    fn test_required_fields() {
        let html = r#"
            <form>
                <input type="email" name="email" required>
                <input type="text" name="name">
                <input type="password" name="password" required>
            </form>
        "#;
        
        let doc = parse_html(html);
        let forms = extract_forms(&doc).unwrap();
        let required = forms[0].required_fields();
        
        assert_eq!(required.len(), 2);
    }

    #[test]
    fn test_form_method_parsing() {
        assert_eq!(FormMethod::from("POST"), FormMethod::Post);
        assert_eq!(FormMethod::from("get"), FormMethod::Get);
        assert_eq!(FormMethod::from("dialog"), FormMethod::Dialog);
        assert_eq!(FormMethod::from("unknown"), FormMethod::Get);
    }

    #[test]
    fn test_field_type_parsing() {
        assert_eq!(FieldType::from("password"), FieldType::Password);
        assert_eq!(FieldType::from("EMAIL"), FieldType::Email);
        assert_eq!(FieldType::from("tel"), FieldType::Tel);
        assert_eq!(FieldType::from("unknown"), FieldType::Text);
    }
}