proxmox/scripts/troubleshoot-rpc-2500.sh

#!/usr/bin/env bash
set -euo pipefail

# Troubleshoot RPC-01 at VMID 2500
# Usage: ./troubleshoot-rpc-2500.sh

set -e

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true

VMID=2500
CONTAINER_NAME="besu-rpc-1"
EXPECTED_IP="${RPC_PUBLIC_1:-192.168.11.221}"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Troubleshooting RPC-01 (VMID $VMID)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Check if running on Proxmox host
if ! command -v pct &>/dev/null; then
    log_error "This script must be run on Proxmox host (pct command not found)"
    exit 1
fi

# 1. Check container status
log_info "1. Checking container status..."
CONTAINER_STATUS=$(pct status $VMID 2>&1 | grep -oP 'status: \K\w+' || echo "unknown")
log_info "   Status: $CONTAINER_STATUS"

if [ "$CONTAINER_STATUS" != "running" ]; then
    log_warn "   Container is not running. Attempting to start..."
    pct start $VMID 2>&1
    sleep 5
    CONTAINER_STATUS=$(pct status $VMID 2>&1 | grep -oP 'status: \K\w+' || echo "unknown")
    if [ "$CONTAINER_STATUS" != "running" ]; then
        log_error "   Failed to start container"
        log_info "   Check container logs: pct config $VMID"
        exit 1
    fi
    log_success "   Container started"
fi

# 2. Check network configuration
log_info ""
log_info "2. Checking network configuration..."
ACTUAL_IP=$(pct exec $VMID -- ip -4 addr show eth0 2>/dev/null | grep -oP 'inet \K[0-9.]+' | head -1 || echo "")
if [ -n "$ACTUAL_IP" ]; then
    log_info "   IP Address: $ACTUAL_IP"
    if [ "$ACTUAL_IP" = "$EXPECTED_IP" ]; then
        log_success "   IP address matches expected: $EXPECTED_IP"
    else
        log_warn "   IP address mismatch. Expected: $EXPECTED_IP, Got: $ACTUAL_IP"
    fi
else
    log_error "   Could not determine IP address"
fi

# 3. Check service status
log_info ""
log_info "3. Checking Besu RPC service status..."
SERVICE_STATUS=$(pct exec $VMID -- systemctl is-active besu-rpc.service 2>&1 || echo "unknown")
SERVICE_ENABLED=$(pct exec $VMID -- systemctl is-enabled besu-rpc.service 2>&1 || echo "unknown")

log_info "   Service Status: $SERVICE_STATUS"
log_info "   Service Enabled: $SERVICE_ENABLED"

if [ "$SERVICE_STATUS" != "active" ]; then
    log_warn "   Service is not active"
    
    # Check recent logs for errors
    log_info ""
    log_info "   Recent service logs (last 20 lines):"
    pct exec $VMID -- journalctl -u besu-rpc.service -n 20 --no-pager 2>&1 | tail -20
fi

# 4. Check configuration files
log_info ""
log_info "4. Checking configuration files..."

CONFIG_FILE="/etc/besu/config-rpc.toml"
if pct exec $VMID -- test -f "$CONFIG_FILE" 2>/dev/null; then
    log_success "   Config file exists: $CONFIG_FILE"
    
    # Check for common configuration issues
    log_info "   Checking for configuration issues..."
    
    # Check RPC is enabled
    if pct exec $VMID -- grep -q "rpc-http-enabled=true" "$CONFIG_FILE" 2>/dev/null; then
        log_success "   RPC HTTP is enabled"
    else
        log_error "   RPC HTTP is NOT enabled!"
    fi
    
    # Check RPC port
    RPC_PORT=$(pct exec $VMID -- grep -oP 'rpc-http-port=\K\d+' "$CONFIG_FILE" 2>/dev/null | head -1 || echo "")
    if [ -n "$RPC_PORT" ]; then
        log_info "   RPC HTTP Port: $RPC_PORT"
        if [ "$RPC_PORT" = "8545" ]; then
            log_success "   Port is correct (8545)"
        else
            log_warn "   Port is $RPC_PORT (expected 8545)"
        fi
    fi
    
    # Check for deprecated options
    DEPRECATED_OPTS=$(pct exec $VMID -- grep -E "log-destination|max-remote-initiated-connections|trie-logs-enabled|accounts-enabled|database-path|rpc-http-host-allowlist" "$CONFIG_FILE" 2>/dev/null | wc -l)
    if [ "$DEPRECATED_OPTS" -gt 0 ]; then
        log_error "   Found $DEPRECATED_OPTS deprecated configuration options"
        log_info "   Deprecated options found:"
        pct exec $VMID -- grep -E "log-destination|max-remote-initiated-connections|trie-logs-enabled|accounts-enabled|database-path|rpc-http-host-allowlist" "$CONFIG_FILE" 2>/dev/null
    else
        log_success "   No deprecated options found"
    fi
else
    log_error "   Config file NOT found: $CONFIG_FILE"
    log_info "   Expected location: $CONFIG_FILE"
fi

# 5. Check required files
log_info ""
log_info "5. Checking required files..."

GENESIS_FILE="/genesis/genesis.json"
STATIC_NODES="/genesis/static-nodes.json"
PERMISSIONS_NODES="/permissions/permissions-nodes.toml"

for file in "$GENESIS_FILE" "$STATIC_NODES" "$PERMISSIONS_NODES"; do
    if pct exec $VMID -- test -f "$file" 2>/dev/null; then
        log_success "   Found: $file"
    else
        log_error "   Missing: $file"
    fi
done

# 6. Check ports
log_info ""
log_info "6. Checking listening ports..."
PORTS=$(pct exec $VMID -- ss -tlnp 2>/dev/null | grep -E "8545|8546|30303|9545" || echo "")
if [ -n "$PORTS" ]; then
    log_success "   Listening ports:"
    echo "$PORTS" | while read line; do
        log_info "     $line"
    done
else
    log_error "   No Besu ports are listening (8545, 8546, 30303, 9545)"
fi

# 7. Check RPC endpoint
log_info ""
log_info "7. Testing RPC endpoint..."
RPC_RESPONSE=$(pct exec $VMID -- curl -s -X POST -H "Content-Type: application/json" \
    --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
    http://localhost:8545 2>&1 || echo "FAILED")

if echo "$RPC_RESPONSE" | grep -q "result"; then
    BLOCK_NUM=$(echo "$RPC_RESPONSE" | grep -oP '"result":"\K[^"]+' | head -1)
    log_success "   RPC endpoint is responding"
    log_info "   Current block: $BLOCK_NUM"
else
    log_error "   RPC endpoint is NOT responding"
    log_info "   Response: $RPC_RESPONSE"
fi

# 8. Check process
log_info ""
log_info "8. Checking Besu process..."
BESU_PROCESS=$(pct exec $VMID -- ps aux | grep -E "[b]esu|java.*besu" | head -1 || echo "")
if [ -n "$BESU_PROCESS" ]; then
    log_success "   Besu process is running"
    log_info "   Process: $(echo "$BESU_PROCESS" | awk '{print $2, $11, $12, $13}')"
else
    log_error "   Besu process is NOT running"
fi

# 9. Check disk space
log_info ""
log_info "9. Checking disk space..."
DISK_USAGE=$(pct exec $VMID -- df -h / 2>/dev/null | tail -1 | awk '{print $5}' || echo "unknown")
log_info "   Disk usage: $DISK_USAGE"

# 10. Check memory
log_info ""
log_info "10. Checking memory..."
MEM_INFO=$(pct exec $VMID -- free -h 2>/dev/null | grep Mem || echo "")
if [ -n "$MEM_INFO" ]; then
    log_info "   $MEM_INFO"
fi

# 11. Recent errors
log_info ""
log_info "11. Checking for recent errors in logs..."
RECENT_ERRORS=$(pct exec $VMID -- journalctl -u besu-rpc.service --since "10 minutes ago" --no-pager 2>&1 | \
    grep -iE "error|fail|exception|unable|cannot" | tail -10 || echo "")
if [ -n "$RECENT_ERRORS" ]; then
    log_error "   Recent errors found:"
    echo "$RECENT_ERRORS" | while read line; do
        log_error "     $line"
    done
else
    log_success "   No recent errors found"
fi

# Summary
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Troubleshooting Summary"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Provide recommendations
if [ "$SERVICE_STATUS" != "active" ]; then
    log_warn "RECOMMENDATIONS:"
    log_info "1. Check service logs: pct exec $VMID -- journalctl -u besu-rpc.service -f"
    log_info "2. Verify configuration: pct exec $VMID -- cat /etc/besu/config-rpc.toml"
    log_info "3. Restart service: pct exec $VMID -- systemctl restart besu-rpc.service"
    log_info "4. Check for configuration errors in logs"
fi

if [ -z "$PORTS" ]; then
    log_warn "RECOMMENDATIONS:"
    log_info "1. Service may not be running - check service status"
    log_info "2. Check firewall rules if service is running"
    log_info "3. Verify RPC is enabled in config file"
fi

echo ""
log_info "For detailed logs, run:"
log_info "  pct exec $VMID -- journalctl -u besu-rpc.service -f"
echo ""