opensearch-api 0.1.0

#!/usr/bin/env python3
"""
OpenSearch Cluster Setup & Management
Suporta instalação progressiva: 1 node → 3 nodes → N nodes
"""

import os
import sys
import json
import subprocess
import socket
import time
import argparse
from pathlib import Path
from typing import Dict, List, Optional, Tuple
from datetime import datetime

class OpenSearchClusterManager:
    def __init__(self):
        self.cluster_name = "opensearch-production"
        self.base_path = "/opt/prod/opensearch"
        self.data_path = "/var/lib/opensearch"
        self.config_path = "/etc/opensearch"
        self.nodes = []
        self.current_node = None
        
    def detect_current_setup(self) -> Dict:
        """Detecta configuração atual do OpenSearch"""
        setup = {
            "installed": False,
            "running": False,
            "cluster_mode": "none",
            "nodes": [],
            "role": None
        }
        
        # Verifica se OpenSearch está instalado
        if os.path.exists(self.base_path):
            setup["installed"] = True
            
            # Verifica configuração atual
            config_file = f"{self.base_path}/config/opensearch.yml"
            if os.path.exists(config_file):
                with open(config_file, 'r') as f:
                    config = f.read()
                    
                    if "discovery.type: single-node" in config:
                        setup["cluster_mode"] = "single"
                        setup["role"] = "single"
                    elif "cluster.initial_master_nodes" in config:
                        setup["cluster_mode"] = "cluster"
                        
                        # Detecta role do node
                        if "node.master: true" in config:
                            setup["role"] = "master"
                        elif "node.data: true" in config:
                            setup["role"] = "data"
                        elif "node.ingest: true" in config:
                            setup["role"] = "ingest"
                        else:
                            setup["role"] = "coordinating"
        
        # Verifica se está rodando
        try:
            result = subprocess.run(
                ["systemctl", "is-active", "opensearch"],
                capture_output=True,
                text=True
            )
            setup["running"] = result.stdout.strip() == "active"
        except:
            pass
        
        return setup
    
    def install_single_node(self):
        """Instala OpenSearch em modo single-node (desenvolvimento/teste)"""
        print("\n=== Instalando OpenSearch (Single-Node) ===")
        
        config = f"""# OpenSearch Configuration - Single Node
cluster.name: {self.cluster_name}
node.name: node-1
node.master: true
node.data: true
node.ingest: true

path.data: {self.data_path}
path.logs: {self.base_path}/logs

network.host: 0.0.0.0
http.port: 9200
transport.port: 9300

# Single-node - SEM descoberta de cluster
discovery.type: single-node

# Segurança desabilitada inicialmente (habilitar em produção!)
plugins.security.disabled: true

# Configurações de memória
bootstrap.memory_lock: true

# CORS para desenvolvimento
http.cors.enabled: true
http.cors.allow-origin: "*"
"""
        
        self._write_config(config)
        self._configure_jvm(heap_size="2g")  # Metade da RAM para single-node
        print("✅ Configurado para single-node")
    
    def convert_to_cluster_master(self):
        """Converte single-node para master node de um cluster"""
        print("\n=== Convertendo para Cluster (Master Node) ===")
        
        hostname = socket.gethostname()
        ip = self._get_primary_ip()
        
        config = f"""# OpenSearch Configuration - Master Node
cluster.name: {self.cluster_name}
node.name: master-1
node.master: true
node.data: true
node.ingest: true
node.ml: false

path.data: {self.data_path}
path.logs: {self.base_path}/logs

# Rede - IMPORTANTE: ajustar para sua rede
network.host: {ip}
http.port: 9200
transport.port: 9300

# Discovery - Configuração inicial do cluster
discovery.seed_hosts:
  - {ip}:9300
  
cluster.initial_master_nodes:
  - master-1

# Configurações de cluster
cluster.max_shards_per_node: 1000
cluster.routing.allocation.disk.threshold_enabled: true
cluster.routing.allocation.disk.watermark.low: 85%
cluster.routing.allocation.disk.watermark.high: 90%

# Segurança (habilitar em produção!)
plugins.security.disabled: true
# plugins.security.ssl.transport.enabled: true
# plugins.security.ssl.http.enabled: true

# Performance
bootstrap.memory_lock: true
indices.memory.index_buffer_size: 20%
indices.fielddata.cache.size: 30%
indices.queries.cache.size: 10%

# CORS
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: "X-Requested-With,Content-Type,Content-Length,Authorization"
"""
        
        self._write_config(config)
        self._configure_jvm(heap_size="4g")  # Mais memória para master
        
        print(f"✅ Master node configurado")
        print(f"   IP: {ip}")
        print(f"   Ports: 9200 (HTTP), 9300 (Transport)")
        
        return ip
    
    def add_data_node(self, master_ip: str, node_number: int = 2):
        """Adiciona um data node ao cluster"""
        print(f"\n=== Configurando Data Node {node_number} ===")
        
        hostname = socket.gethostname()
        ip = self._get_primary_ip()
        
        config = f"""# OpenSearch Configuration - Data Node {node_number}
cluster.name: {self.cluster_name}
node.name: data-{node_number}
node.master: false
node.data: true
node.ingest: true
node.ml: false

path.data: {self.data_path}
path.logs: {self.base_path}/logs

# Rede
network.host: {ip}
http.port: 9200
transport.port: 9300

# Discovery - Conecta ao master
discovery.seed_hosts:
  - {master_ip}:9300
  - {ip}:9300

# Segurança (deve corresponder ao master)
plugins.security.disabled: true

# Performance
bootstrap.memory_lock: true
indices.memory.index_buffer_size: 20%

# CORS
http.cors.enabled: true
http.cors.allow-origin: "*"
"""
        
        self._write_config(config)
        self._configure_jvm(heap_size="4g")
        
        print(f"✅ Data node {node_number} configurado")
        print(f"   IP: {ip}")
        print(f"   Master: {master_ip}")
    
    def add_coordinating_node(self, master_ip: str, node_number: int = 1):
        """Adiciona um coordinating node (para queries)"""
        print(f"\n=== Configurando Coordinating Node {node_number} ===")
        
        ip = self._get_primary_ip()
        
        config = f"""# OpenSearch Configuration - Coordinating Node {node_number}
cluster.name: {self.cluster_name}
node.name: coord-{node_number}
node.master: false
node.data: false
node.ingest: false
node.ml: false

path.data: {self.data_path}
path.logs: {self.base_path}/logs

# Rede
network.host: {ip}
http.port: 9200
transport.port: 9300

# Discovery
discovery.seed_hosts:
  - {master_ip}:9300

# Segurança
plugins.security.disabled: true

# Performance (menos memória, só roteia queries)
bootstrap.memory_lock: true

# CORS
http.cors.enabled: true
http.cors.allow-origin: "*"
"""
        
        self._write_config(config)
        self._configure_jvm(heap_size="2g")  # Menos memória para coordinator
        
        print(f"✅ Coordinating node {node_number} configurado")
    
    def setup_3_node_cluster(self, ips: List[str]):
        """Configura cluster com 3 nodes (recomendado para produção)"""
        print("\n=== Configurando Cluster de 3 Nodes ===")
        
        if len(ips) != 3:
            print("❌ Erro: Forneça exatamente 3 IPs")
            return
        
        # Configuração para cada node
        for i, ip in enumerate(ips, 1):
            print(f"\n--- Configurando Node {i} ({ip}) ---")
            
            config = f"""# OpenSearch Configuration - Node {i} (Cluster 3 nodes)
cluster.name: {self.cluster_name}
node.name: node-{i}
node.master: true
node.data: true
node.ingest: true

path.data: {self.data_path}
path.logs: {self.base_path}/logs

# Rede
network.host: {ip}
http.port: 9200
transport.port: 9300

# Discovery - Todos os nodes
discovery.seed_hosts:
  - {ips[0]}:9300
  - {ips[1]}:9300
  - {ips[2]}:9300

# Initial master nodes (primeira vez apenas)
cluster.initial_master_nodes:
  - node-1
  - node-2
  - node-3

# Quorum (maioria para evitar split-brain)
discovery.zen.minimum_master_nodes: 2

# Configurações de cluster
cluster.max_shards_per_node: 1000
cluster.routing.allocation.disk.threshold_enabled: true
cluster.routing.allocation.disk.watermark.low: 85%
cluster.routing.allocation.disk.watermark.high: 90%

# Segurança
plugins.security.disabled: true

# Performance
bootstrap.memory_lock: true
indices.memory.index_buffer_size: 20%

# CORS
http.cors.enabled: true
http.cors.allow-origin: "*"
"""
            
            print(f"Salve esta configuração no node {ip}:")
            print(f"Arquivo: {self.base_path}/config/opensearch.yml")
            print("-" * 50)
            print(config)
            print("-" * 50)
    
    def generate_security_config(self):
        """Gera configuração de segurança para produção"""
        print("\n=== Configuração de Segurança ===")
        
        config = """
# Adicionar ao opensearch.yml para habilitar segurança:

# Segurança - Transport Layer
plugins.security.ssl.transport.pemcert_filepath: certificates/node.pem
plugins.security.ssl.transport.pemkey_filepath: certificates/node-key.pem
plugins.security.ssl.transport.pemtrustedcas_filepath: certificates/root-ca.pem
plugins.security.ssl.transport.enforce_hostname_verification: false

# Segurança - REST Layer
plugins.security.ssl.http.enabled: true
plugins.security.ssl.http.pemcert_filepath: certificates/node.pem
plugins.security.ssl.http.pemkey_filepath: certificates/node-key.pem
plugins.security.ssl.http.pemtrustedcas_filepath: certificates/root-ca.pem

# Autenticação e Autorização
plugins.security.authcz.admin_dn:
  - CN=admin,OU=IT,O=Company,L=City,ST=State,C=US

# Audit
plugins.security.audit.type: internal_opensearch
plugins.security.enable_snapshot_restore_privilege: true
plugins.security.check_snapshot_restore_write_privileges: true

# Certificados auto-assinados (desenvolvimento)
plugins.security.allow_unsafe_democertificates: false
"""
        
        print(config)
        print("\n⚠️  IMPORTANTE:")
        print("1. Gere certificados SSL válidos para produção")
        print("2. Configure senhas fortes")
        print("3. Limite acesso por IP no firewall")
        print("4. Use TLS/SSL entre nodes")
    
    def check_cluster_health(self):
        """Verifica saúde do cluster"""
        try:
            import requests
            
            # Tenta conectar em localhost primeiro
            response = requests.get("http://localhost:9200/_cluster/health", timeout=5)
            health = response.json()
            
            print("\n=== Saúde do Cluster ===")
            print(f"Cluster: {health.get('cluster_name')}")
            print(f"Status: {health.get('status')}")
            print(f"Nodes: {health.get('number_of_nodes')}")
            print(f"Data Nodes: {health.get('number_of_data_nodes')}")
            print(f"Shards Ativos: {health.get('active_shards')}")
            print(f"Shards Não Alocados: {health.get('unassigned_shards')}")
            
            # Verifica nodes
            response = requests.get("http://localhost:9200/_cat/nodes?v", timeout=5)
            print("\n=== Nodes do Cluster ===")
            print(response.text)
            
        except Exception as e:
            print(f"❌ Erro ao verificar cluster: {e}")
    
    def _write_config(self, config: str):
        """Escreve configuração no arquivo"""
        config_file = f"{self.base_path}/config/opensearch.yml"
        
        # Backup da config atual
        if os.path.exists(config_file):
            backup = f"{config_file}.backup.{datetime.now().strftime('%Y%m%d_%H%M%S')}"
            subprocess.run(["cp", config_file, backup])
            print(f"📋 Backup criado: {backup}")
        
        with open(config_file, 'w') as f:
            f.write(config)
    
    def _configure_jvm(self, heap_size: str = "2g"):
        """Configura JVM options"""
        jvm_options = f"""# JVM Configuration
-Xms{heap_size}
-Xmx{heap_size}

# GC
-XX:+UseG1GC
-XX:MaxGCPauseMillis=200
-XX:+ParallelRefProcEnabled
-XX:+AlwaysPreTouch

# Heap dump on OOM
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath={self.base_path}/logs/

# Performance
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djdk.io.permissionsUseCanonicalPath=true

# Netty
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0

# Log4j2
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
"""
        
        jvm_file = f"{self.base_path}/config/jvm.options"
        with open(jvm_file, 'w') as f:
            f.write(jvm_options)
        
        print(f"📝 JVM configurada: Heap={heap_size}")
    
    def _get_primary_ip(self) -> str:
        """Obtém IP primário da máquina"""
        try:
            # Conecta a um servidor externo para descobrir o IP local
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            s.connect(("8.8.8.8", 80))
            ip = s.getsockname()[0]
            s.close()
            return ip
        except:
            return "127.0.0.1"
    
    def show_expansion_guide(self):
        """Mostra guia de expansão do cluster"""
        print("""
=== GUIA DE EXPANSÃO DO CLUSTER ===

1️⃣  SINGLE NODE → 3 NODES (Recomendado)
   ----------------------------------------
   a) No node atual (master):
      ./cluster-setup.py --convert-to-master
      
   b) Nos 2 novos servidores:
      - Instale OpenSearch
      - Execute: ./cluster-setup.py --add-data-node --master-ip <IP_DO_MASTER>
      
   c) Reinicie todos os nodes na ordem:
      1. Master
      2. Data nodes

2️⃣  3 NODES → 5+ NODES
   ----------------------------------------
   a) Para adicionar mais data nodes:
      ./cluster-setup.py --add-data-node --master-ip <IP> --node-num 4
      
   b) Para adicionar coordinating nodes (query):
      ./cluster-setup.py --add-coord-node --master-ip <IP>

3️⃣  TOPOLOGIA RECOMENDADA
   ----------------------------------------
   Pequeno (3 nodes):
   - 3x master-eligible + data
   
   Médio (5-7 nodes):
   - 3x master-eligible
   - 2-4x data nodes
   
   Grande (10+ nodes):
   - 3x dedicated master
   - 5+x data nodes
   - 2+x coordinating nodes
   - 1-2x ingest nodes

4️⃣  CHECKLIST DE PRODUÇÃO
   ----------------------------------------
   □ Mínimo 3 master-eligible nodes
   □ discovery.zen.minimum_master_nodes = (masters / 2) + 1
   □ Certificados SSL configurados
   □ Segurança habilitada
   □ Backup configurado
   □ Monitoring ativo
   □ Firewall configurado
   □ Limites de sistema ajustados (/etc/security/limits.conf)

5️⃣  COMANDOS ÚTEIS
   ----------------------------------------
   # Verificar saúde do cluster
   curl -X GET "localhost:9200/_cluster/health?pretty"
   
   # Listar nodes
   curl -X GET "localhost:9200/_cat/nodes?v"
   
   # Verificar alocação de shards
   curl -X GET "localhost:9200/_cat/shards?v"
   
   # Estatísticas do cluster
   curl -X GET "localhost:9200/_cluster/stats?pretty"
""")

def main():
    parser = argparse.ArgumentParser(description='OpenSearch Cluster Setup')
    parser.add_argument('--install-single', action='store_true', 
                       help='Instala single-node (dev/test)')
    parser.add_argument('--convert-to-master', action='store_true',
                       help='Converte single-node para master de cluster')
    parser.add_argument('--add-data-node', action='store_true',
                       help='Adiciona data node ao cluster')
    parser.add_argument('--add-coord-node', action='store_true',
                       help='Adiciona coordinating node')
    parser.add_argument('--setup-3-nodes', nargs=3, metavar='IP',
                       help='Configura cluster de 3 nodes')
    parser.add_argument('--master-ip', help='IP do master node')
    parser.add_argument('--node-num', type=int, default=2,
                       help='Número do node')
    parser.add_argument('--check-health', action='store_true',
                       help='Verifica saúde do cluster')
    parser.add_argument('--show-guide', action='store_true',
                       help='Mostra guia de expansão')
    parser.add_argument('--security-config', action='store_true',
                       help='Gera configuração de segurança')
    
    args = parser.parse_args()
    manager = OpenSearchClusterManager()
    
    # Detecta setup atual
    current = manager.detect_current_setup()
    print(f"Setup atual: {current['cluster_mode']}")
    if current['role']:
        print(f"Role: {current['role']}")
    
    # Executa ação solicitada
    if args.install_single:
        manager.install_single_node()
    elif args.convert_to_master:
        manager.convert_to_cluster_master()
    elif args.add_data_node:
        if not args.master_ip:
            print("❌ Erro: --master-ip é obrigatório")
            sys.exit(1)
        manager.add_data_node(args.master_ip, args.node_num)
    elif args.add_coord_node:
        if not args.master_ip:
            print("❌ Erro: --master-ip é obrigatório")
            sys.exit(1)
        manager.add_coordinating_node(args.master_ip, args.node_num)
    elif args.setup_3_nodes:
        manager.setup_3_node_cluster(args.setup_3_nodes)
    elif args.check_health:
        manager.check_cluster_health()
    elif args.security_config:
        manager.generate_security_config()
    elif args.show_guide:
        manager.show_expansion_guide()
    else:
        print("\nUse --help para ver opções disponíveis")
        print("\nExemplos:")
        print("  Instalar single-node:")
        print("    python3 cluster-setup.py --install-single")
        print("\n  Converter para cluster:")
        print("    python3 cluster-setup.py --convert-to-master")
        print("\n  Adicionar data node:")
        print("    python3 cluster-setup.py --add-data-node --master-ip 192.168.1.10")

if __name__ == "__main__":
    main()