253 lines
9.1 KiB
Bash
Executable File
253 lines
9.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Comprehensive Blockchain Health Monitoring Script
|
|
# Monitors block production, transaction inclusion, and node health
|
|
|
|
set -euo pipefail
|
|
|
|
# Load shared project environment / VMID host mapping
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
|
|
source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true
|
|
|
|
RPC_CORE_1="${RPC_CORE_1:-192.168.11.211}"
|
|
RPC_URL="${RPC_URL:-http://${RPC_CORE_1}:8545}"
|
|
DEPLOYER="${DEPLOYER:-0x4A666F96fC8764181194447A7dFdb7d471b301C8}"
|
|
PROXMOX_SSH_USER="${PROXMOX_SSH_USER:-root}"
|
|
PROXMOX_R630="${PROXMOX_R630:-${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}}"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
|
|
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
|
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[✗]${NC} $1"; }
|
|
log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; }
|
|
|
|
validator_host() {
|
|
local vmid="$1"
|
|
if type get_host_for_vmid >/dev/null 2>&1; then
|
|
get_host_for_vmid "$vmid"
|
|
elif [ "$vmid" -le 1002 ]; then
|
|
echo "$PROXMOX_R630"
|
|
else
|
|
echo "${PROXMOX_HOST_ML110:-192.168.11.10}"
|
|
fi
|
|
}
|
|
|
|
validator_ip() {
|
|
local vmid="$1"
|
|
case "$vmid" in
|
|
1000) echo "${IP_VALIDATOR_0:-192.168.11.100}" ;;
|
|
1001) echo "${IP_VALIDATOR_1:-192.168.11.101}" ;;
|
|
1002) echo "${IP_VALIDATOR_2:-192.168.11.102}" ;;
|
|
1003) echo "${IP_VALIDATOR_3:-192.168.11.103}" ;;
|
|
1004) echo "${IP_VALIDATOR_4:-192.168.11.104}" ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
echo "=== Blockchain Health Monitor ==="
|
|
echo "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo ""
|
|
|
|
# Check RPC connectivity
|
|
log_section "RPC Node Status"
|
|
if timeout 5 cast chain-id --rpc-url "$RPC_URL" >/dev/null 2>&1; then
|
|
CHAIN_ID=$(cast chain-id --rpc-url "$RPC_URL" 2>/dev/null)
|
|
BLOCK_NUM=$(cast block-number --rpc-url "$RPC_URL" 2>/dev/null)
|
|
BLOCK_DEC=$(cast --to-dec "$BLOCK_NUM" 2>/dev/null || echo "0")
|
|
log_success "RPC accessible"
|
|
echo " Chain ID: $CHAIN_ID"
|
|
echo " Latest block: $BLOCK_DEC ($BLOCK_NUM)"
|
|
else
|
|
log_error "RPC not accessible"
|
|
exit 1
|
|
fi
|
|
|
|
# Check block production
|
|
log_section "Block Production"
|
|
BLOCK1=$(cast block-number --rpc-url "$RPC_URL" 2>/dev/null)
|
|
sleep 5
|
|
BLOCK2=$(cast block-number --rpc-url "$RPC_URL" 2>/dev/null)
|
|
BLOCK1_DEC=$(cast --to-dec "$BLOCK1" 2>/dev/null || echo "0")
|
|
BLOCK2_DEC=$(cast --to-dec "$BLOCK2" 2>/dev/null || echo "0")
|
|
BLOCK_DIFF=$((BLOCK2_DEC - BLOCK1_DEC))
|
|
|
|
if [ "$BLOCK_DIFF" -gt 0 ]; then
|
|
log_success "Blocks being produced ($BLOCK_DIFF blocks in 5s)"
|
|
else
|
|
log_error "Block production stalled (no new blocks in 5s)"
|
|
# If validators are all active, they may still be syncing (QBFT does not produce until sync completes)
|
|
SYNC_HINT=$(ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no "${PROXMOX_SSH_USER}@${PROXMOX_R630}" \
|
|
"pct exec 1000 -- journalctl -u besu-validator --no-pager -n 30 2>/dev/null" 2>/dev/null | grep -c "Full sync\|initial sync in progress\|QBFT mining coordinator not starting" || true)
|
|
if [ "${SYNC_HINT:-0}" -gt 0 ]; then
|
|
echo " → Validators may be syncing; block production will resume when sync completes (see docs/06-besu/CRITICAL_ISSUE_BLOCK_PRODUCTION_STOPPED.md)."
|
|
fi
|
|
fi
|
|
|
|
# Check transaction inclusion
|
|
log_section "Transaction Inclusion"
|
|
TX_COUNT_TOTAL=0
|
|
EMPTY_BLOCKS=0
|
|
for i in 0 1 2 3 4 5; do
|
|
BLOCK_NUM=$((BLOCK2_DEC - i))
|
|
BLOCK_HEX=$(printf '0x%x' $BLOCK_NUM)
|
|
TX_COUNT=$(cast rpc eth_getBlockTransactionCountByNumber "$BLOCK_HEX" --rpc-url "$RPC_URL" 2>/dev/null | tr -d '"')
|
|
TX_COUNT_DEC=$(cast --to-dec "$TX_COUNT" 2>/dev/null || echo "0")
|
|
TX_COUNT_TOTAL=$((TX_COUNT_TOTAL + TX_COUNT_DEC))
|
|
if [ "$TX_COUNT_DEC" -eq 0 ]; then
|
|
EMPTY_BLOCKS=$((EMPTY_BLOCKS + 1))
|
|
fi
|
|
done
|
|
|
|
if [ "$TX_COUNT_TOTAL" -gt 0 ]; then
|
|
log_success "Transactions being included ($TX_COUNT_TOTAL txs in last 6 blocks)"
|
|
else
|
|
echo " Recent blocks are empty ($EMPTY_BLOCKS empty blocks)"
|
|
fi
|
|
|
|
# Check pending transactions
|
|
log_section "Pending Transactions"
|
|
LATEST_HEX=$(cast rpc eth_getTransactionCount "$DEPLOYER" latest --rpc-url "$RPC_URL" 2>/dev/null | tr -d '"')
|
|
PENDING_HEX=$(cast rpc eth_getTransactionCount "$DEPLOYER" pending --rpc-url "$RPC_URL" 2>/dev/null | tr -d '"')
|
|
LATEST_DEC=$(cast --to-dec "$LATEST_HEX" 2>/dev/null || echo "0")
|
|
PENDING_DEC=$(cast --to-dec "$PENDING_HEX" 2>/dev/null || echo "0")
|
|
PENDING_COUNT=$((PENDING_DEC - LATEST_DEC))
|
|
|
|
if [ "$PENDING_COUNT" -eq 0 ]; then
|
|
log_success "No pending transactions"
|
|
else
|
|
log_warn "$PENDING_COUNT pending transactions (nonces $((LATEST_DEC + 1))-$PENDING_DEC)"
|
|
fi
|
|
|
|
GLOBAL_PENDING=0
|
|
TXPOOL_JSON=$(cast rpc txpool_besuTransactions --rpc-url "$RPC_URL" 2>/dev/null || echo "")
|
|
if [ -n "$TXPOOL_JSON" ]; then
|
|
GLOBAL_PENDING=$(echo "$TXPOOL_JSON" | jq 'length' 2>/dev/null || echo "0")
|
|
if [ "${GLOBAL_PENDING:-0}" -gt 0 ]; then
|
|
log_warn "Global txpool still has $GLOBAL_PENDING transaction(s)"
|
|
else
|
|
log_success "Global txpool is empty"
|
|
fi
|
|
fi
|
|
|
|
if [ "$TX_COUNT_TOTAL" -eq 0 ]; then
|
|
if [ "$PENDING_COUNT" -gt 0 ] || [ "${GLOBAL_PENDING:-0}" -gt 0 ]; then
|
|
log_warn "Recent blocks are empty while pending transactions still exist"
|
|
else
|
|
log_info "Recent blocks are empty and the txpool is empty"
|
|
fi
|
|
fi
|
|
|
|
# Check validator status
|
|
log_section "Validator Status"
|
|
VALIDATORS=(
|
|
"1000:$(validator_host 1000)"
|
|
"1001:$(validator_host 1001)"
|
|
"1002:$(validator_host 1002)"
|
|
"1003:$(validator_host 1003)"
|
|
"1004:$(validator_host 1004)"
|
|
)
|
|
|
|
ACTIVE_COUNT=0
|
|
P2P_MISMATCHES=0
|
|
for validator in "${VALIDATORS[@]}"; do
|
|
IFS=':' read -r VMID HOST <<< "$validator"
|
|
SSH_TARGET="${PROXMOX_SSH_USER}@${HOST}"
|
|
STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" \
|
|
"pct exec $VMID -- systemctl is-active besu-validator" 2>/dev/null || echo "unknown")
|
|
|
|
if [ "$STATUS" = "active" ]; then
|
|
ACTIVE_COUNT=$((ACTIVE_COUNT + 1))
|
|
echo " Validator $VMID: $STATUS"
|
|
else
|
|
log_warn "Validator $VMID: $STATUS"
|
|
fi
|
|
|
|
EXPECTED_P2P_HOST="$(validator_ip "$VMID" 2>/dev/null || echo "")"
|
|
if [ -n "$EXPECTED_P2P_HOST" ]; then
|
|
P2P_HOST_CFG=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" \
|
|
"pct exec $VMID -- bash -lc 'grep -E \"^p2p-host=\" /etc/besu/config-validator.toml 2>/dev/null | sed -E \"s/^[^\\\"]*\\\"([^\\\"]+)\\\".*/\\1/\"'" \
|
|
2>/dev/null || echo "")
|
|
if [ -n "$P2P_HOST_CFG" ] && [ "$P2P_HOST_CFG" != "$EXPECTED_P2P_HOST" ]; then
|
|
log_warn "Validator $VMID p2p-host mismatch: $P2P_HOST_CFG (expected $EXPECTED_P2P_HOST)"
|
|
P2P_MISMATCHES=$((P2P_MISMATCHES + 1))
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "$ACTIVE_COUNT" -eq 5 ]; then
|
|
log_success "All 5 validators active"
|
|
else
|
|
log_error "Only $ACTIVE_COUNT/5 validators active"
|
|
fi
|
|
|
|
# Check peer connections
|
|
log_section "Peer Connections"
|
|
PEER_COUNT=$(cast rpc admin_peers --rpc-url "$RPC_URL" 2>/dev/null | jq '. | length' 2>/dev/null || echo "N/A")
|
|
if [ "$PEER_COUNT" != "N/A" ] && [ "$PEER_COUNT" -ge 5 ]; then
|
|
log_success "RPC has $PEER_COUNT peer connections"
|
|
else
|
|
log_warn "RPC has $PEER_COUNT peer connections (expected >= 5)"
|
|
fi
|
|
|
|
# Summary
|
|
log_section "Health Summary"
|
|
ISSUES=0
|
|
|
|
if [ "$BLOCK_DIFF" -eq 0 ]; then
|
|
log_error "❌ Block production stalled"
|
|
ISSUES=$((ISSUES + 1))
|
|
# Hint when validators are syncing (all active but no new blocks yet)
|
|
if [ "$ACTIVE_COUNT" -eq 5 ]; then
|
|
echo " (If validators recently restarted: they are likely in full sync; blocks will resume when sync completes.)"
|
|
fi
|
|
else
|
|
log_success "✓ Block production active"
|
|
fi
|
|
|
|
if [ "$TX_COUNT_TOTAL" -eq 0 ] && [ "$PENDING_COUNT" -gt 0 ]; then
|
|
log_error "❌ Transactions not being included"
|
|
ISSUES=$((ISSUES + 1))
|
|
elif [ "$TX_COUNT_TOTAL" -gt 0 ]; then
|
|
log_success "✓ Transactions being included"
|
|
fi
|
|
|
|
if [ "$ACTIVE_COUNT" -lt 5 ]; then
|
|
log_error "❌ Not all validators active"
|
|
ISSUES=$((ISSUES + 1))
|
|
else
|
|
log_success "✓ All validators active"
|
|
fi
|
|
|
|
if [ "$P2P_MISMATCHES" -gt 0 ]; then
|
|
log_error "❌ $P2P_MISMATCHES validator(s) have the wrong p2p-host"
|
|
ISSUES=$((ISSUES + 1))
|
|
fi
|
|
|
|
if [ "$PENDING_COUNT" -gt 10 ]; then
|
|
log_warn "⚠ High number of pending transactions ($PENDING_COUNT)"
|
|
ISSUES=$((ISSUES + 1))
|
|
fi
|
|
|
|
if [ "${GLOBAL_PENDING:-0}" -gt 0 ] && [ "$TX_COUNT_TOTAL" -eq 0 ]; then
|
|
log_warn "⚠ Global txpool has $GLOBAL_PENDING transaction(s) but recent blocks are empty"
|
|
ISSUES=$((ISSUES + 1))
|
|
fi
|
|
|
|
echo ""
|
|
if [ "$ISSUES" -eq 0 ]; then
|
|
log_success "Overall Status: HEALTHY"
|
|
exit 0
|
|
else
|
|
log_error "Overall Status: $ISSUES issue(s) detected"
|
|
exit 1
|
|
fi
|