269 lines
7.6 KiB
Bash
Executable File
269 lines
7.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Cluster-wide Besu inventory audit.
|
|
# Uses Proxmox cluster resources so nodes hosted on r630-03 / r630-04 are not missed.
|
|
#
|
|
# Usage:
|
|
# bash scripts/verify/check-cluster-besu-inventory.sh
|
|
# PVE_CLUSTER_API_HOST=192.168.11.11 bash scripts/verify/check-cluster-besu-inventory.sh
|
|
# bash scripts/verify/check-cluster-besu-inventory.sh --json
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
CLUSTER_HOST="${PVE_CLUSTER_API_HOST:-192.168.11.11}"
|
|
JSON_ONLY=false
|
|
if [[ "${1:-}" == "--json" ]]; then
|
|
JSON_ONLY=true
|
|
fi
|
|
|
|
need_cmd() {
|
|
command -v "$1" >/dev/null 2>&1 || {
|
|
echo "missing required command: $1" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
need_cmd ssh
|
|
need_cmd jq
|
|
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
|
|
section() {
|
|
$JSON_ONLY || echo -e "\n${CYAN}━━━ $1 ━━━${NC}"
|
|
}
|
|
|
|
info() {
|
|
$JSON_ONLY || echo "$1"
|
|
}
|
|
|
|
canonical_vmids=(
|
|
1000 1001 1002 1003 1004
|
|
1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
|
|
2101 2102 2103 2201 2301 2303 2304 2305 2306 2307 2308
|
|
2400 2401 2402 2403
|
|
2500 2501 2502 2503 2504 2505
|
|
)
|
|
|
|
declare -A canonical_set=()
|
|
for vmid in "${canonical_vmids[@]}"; do
|
|
canonical_set["$vmid"]=1
|
|
done
|
|
|
|
declare -A expected_ip=(
|
|
[1000]=192.168.11.100
|
|
[1001]=192.168.11.101
|
|
[1002]=192.168.11.102
|
|
[1003]=192.168.11.103
|
|
[1004]=192.168.11.104
|
|
[1500]=192.168.11.150
|
|
[1501]=192.168.11.151
|
|
[1502]=192.168.11.152
|
|
[1503]=192.168.11.153
|
|
[1504]=192.168.11.154
|
|
[1505]=192.168.11.213
|
|
[1506]=192.168.11.214
|
|
[1507]=192.168.11.244
|
|
[1508]=192.168.11.245
|
|
[1509]=192.168.11.219
|
|
[1510]=192.168.11.220
|
|
[2101]=192.168.11.211
|
|
[2102]=192.168.11.212
|
|
[2103]=192.168.11.217
|
|
[2201]=192.168.11.221
|
|
[2301]=192.168.11.232
|
|
[2303]=192.168.11.233
|
|
[2304]=192.168.11.234
|
|
[2305]=192.168.11.235
|
|
[2306]=192.168.11.236
|
|
[2307]=192.168.11.237
|
|
[2308]=192.168.11.238
|
|
[2400]=192.168.11.240
|
|
[2401]=192.168.11.241
|
|
[2402]=192.168.11.242
|
|
[2403]=192.168.11.243
|
|
[2500]=192.168.11.172
|
|
[2501]=192.168.11.173
|
|
[2502]=192.168.11.174
|
|
[2503]=192.168.11.246
|
|
[2504]=192.168.11.247
|
|
[2505]=192.168.11.248
|
|
)
|
|
|
|
cluster_nodes_json="$(ssh -o BatchMode=yes -o ConnectTimeout=10 "root@${CLUSTER_HOST}" \
|
|
'pvesh get /nodes --output-format json')"
|
|
resources_json="$(ssh -o BatchMode=yes -o ConnectTimeout=10 "root@${CLUSTER_HOST}" \
|
|
'pvesh get /cluster/resources --output-format json')"
|
|
config_rows_text="$(ssh -o BatchMode=yes -o ConnectTimeout=10 "root@${CLUSTER_HOST}" '
|
|
shopt -s nullglob
|
|
for f in /etc/pve/nodes/*/lxc/*.conf /etc/pve/nodes/*/qemu-server/*.conf; do
|
|
[ -f "$f" ] || continue
|
|
node="$(basename "$(dirname "$(dirname "$f")")")"
|
|
kind="$(basename "$(dirname "$f")")"
|
|
vmid="$(basename "$f" .conf)"
|
|
net0="$(grep -m1 "^net0:" "$f" || true)"
|
|
printf "%s|%s|%s|%s\n" "$kind" "$node" "$vmid" "$net0"
|
|
done
|
|
')"
|
|
|
|
besu_rows_json="$(
|
|
jq -c '
|
|
[
|
|
.[]
|
|
| select((.type == "lxc" or .type == "qemu"))
|
|
| {
|
|
vmid: (.vmid | tostring),
|
|
type,
|
|
node,
|
|
status,
|
|
name
|
|
}
|
|
]
|
|
| sort_by(.vmid | tonumber)
|
|
' <<<"$resources_json"
|
|
)"
|
|
|
|
declare -A found_vmids=()
|
|
declare -A live_ip_to_vmid=()
|
|
declare -A config_ip=()
|
|
declare -a table_rows=()
|
|
declare -a unexpected_rows=()
|
|
|
|
while IFS='|' read -r kind node vmid net0; do
|
|
[[ -z "$vmid" ]] && continue
|
|
ip=""
|
|
if [[ "$net0" =~ ip=([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)/ ]]; then
|
|
ip="${BASH_REMATCH[1]}"
|
|
fi
|
|
config_ip["${kind}:${node}:${vmid}"]="$ip"
|
|
done <<< "$config_rows_text"
|
|
|
|
while IFS= read -r row; do
|
|
[[ -z "$row" ]] && continue
|
|
vmid="$(jq -r '.vmid' <<<"$row")"
|
|
type="$(jq -r '.type' <<<"$row")"
|
|
node="$(jq -r '.node' <<<"$row")"
|
|
status="$(jq -r '.status' <<<"$row")"
|
|
name="$(jq -r '.name' <<<"$row")"
|
|
ip="${config_ip[${type}:${node}:${vmid}]:-}"
|
|
|
|
if [[ -z "${canonical_set[$vmid]:-}" && ! "$name" =~ besu|thirdweb-rpc ]]; then
|
|
continue
|
|
fi
|
|
|
|
canonical="no"
|
|
note="extra/non-canonical"
|
|
if [[ -n "${canonical_set[$vmid]:-}" ]]; then
|
|
canonical="yes"
|
|
note="canonical"
|
|
found_vmids["$vmid"]=1
|
|
if [[ -n "$ip" ]]; then
|
|
live_ip_to_vmid["$ip"]="$vmid"
|
|
if [[ -n "${expected_ip[$vmid]:-}" && "${expected_ip[$vmid]}" != "$ip" ]]; then
|
|
note="canonical, ip-mismatch expected=${expected_ip[$vmid]}"
|
|
fi
|
|
else
|
|
note="canonical, ip-unresolved"
|
|
fi
|
|
fi
|
|
|
|
row_json="$(jq -cn \
|
|
--arg vmid "$vmid" \
|
|
--arg type "$type" \
|
|
--arg node "$node" \
|
|
--arg status "$status" \
|
|
--arg name "$name" \
|
|
--arg ip "$ip" \
|
|
--arg canonical "$canonical" \
|
|
--arg note "$note" \
|
|
'{vmid:$vmid,type:$type,node:$node,status:$status,name:$name,ip:$ip,canonical:$canonical,note:$note}')"
|
|
|
|
table_rows+=("$row_json")
|
|
if [[ "$canonical" == "no" ]]; then
|
|
unexpected_rows+=("$row_json")
|
|
fi
|
|
done < <(jq -c '.[]' <<<"$besu_rows_json")
|
|
|
|
missing_items=()
|
|
for vmid in "${canonical_vmids[@]}"; do
|
|
if [[ -z "${found_vmids[$vmid]:-}" ]]; then
|
|
expected="${expected_ip[$vmid]:-}"
|
|
missing_items+=("$(jq -cn --arg vmid "$vmid" --arg expected_ip "$expected" '{vmid:$vmid, expected_ip:$expected_ip}')")
|
|
fi
|
|
done
|
|
|
|
rows_json="$(printf '%s\n' "${table_rows[@]}" | jq -s '.')"
|
|
unexpected_json="$(printf '%s\n' "${unexpected_rows[@]:-}" | jq -s '.')"
|
|
missing_json="$(printf '%s\n' "${missing_items[@]:-}" | jq -s '.')"
|
|
|
|
nodes_summary_json="$(
|
|
jq -c '[.[] | {node, status}] | sort_by(.node)' <<<"$cluster_nodes_json"
|
|
)"
|
|
|
|
final_json="$(
|
|
jq -cn \
|
|
--arg cluster_host "$CLUSTER_HOST" \
|
|
--argjson nodes "$nodes_summary_json" \
|
|
--argjson rows "$rows_json" \
|
|
--argjson missing "$missing_json" \
|
|
--argjson unexpected "$unexpected_json" \
|
|
'{
|
|
cluster_host: $cluster_host,
|
|
nodes: $nodes,
|
|
besu_resources: $rows,
|
|
missing_canonical_vmids: $missing,
|
|
unexpected_besu_resources: $unexpected
|
|
}'
|
|
)"
|
|
|
|
if $JSON_ONLY; then
|
|
echo "$final_json"
|
|
exit 0
|
|
fi
|
|
|
|
section "Cluster Nodes"
|
|
jq -r '.nodes[] | " - \(.node): \(.status)"' <<<"$final_json"
|
|
|
|
section "Besu Resources"
|
|
printf '%-6s %-4s %-8s %-9s %-30s %-15s %-10s %s\n' "VMID" "TYPE" "NODE" "STATUS" "NAME" "IP" "CANON" "NOTE"
|
|
printf '%-6s %-4s %-8s %-9s %-30s %-15s %-10s %s\n' "------" "----" "--------" "---------" "------------------------------" "---------------" "----------" "------------------------------"
|
|
while IFS= read -r row; do
|
|
vmid="$(jq -r '.vmid' <<<"$row")"
|
|
type="$(jq -r '.type' <<<"$row")"
|
|
node="$(jq -r '.node' <<<"$row")"
|
|
status="$(jq -r '.status' <<<"$row")"
|
|
name="$(jq -r '.name' <<<"$row")"
|
|
ip="$(jq -r '.ip // "-"' <<<"$row")"
|
|
canonical="$(jq -r '.canonical' <<<"$row")"
|
|
note="$(jq -r '.note' <<<"$row")"
|
|
printf '%-6s %-4s %-8s %-9s %-30s %-15s %-10s %s\n' \
|
|
"$vmid" "$type" "$node" "$status" "$name" "${ip:--}" "$canonical" "$note"
|
|
done < <(jq -c '.besu_resources[]' <<<"$final_json")
|
|
|
|
section "Missing Canonical VMIDs"
|
|
if [[ "$(jq 'length' <<<"$missing_json")" -eq 0 ]]; then
|
|
echo -e "${GREEN}[✓]${NC} All canonical Besu VMIDs are present in cluster resources."
|
|
else
|
|
while IFS= read -r row; do
|
|
vmid="$(jq -r '.vmid' <<<"$row")"
|
|
ip="$(jq -r '.expected_ip' <<<"$row")"
|
|
echo -e "${YELLOW}[⚠]${NC} Missing canonical VMID ${vmid} (expected IP ${ip})"
|
|
done < <(jq -c '.[]' <<<"$missing_json")
|
|
fi
|
|
|
|
section "Unexpected / Extra Besu Resources"
|
|
if [[ "${#unexpected_rows[@]}" -eq 0 ]]; then
|
|
echo -e "${GREEN}[✓]${NC} No extra Besu resources found outside the canonical VMID set."
|
|
else
|
|
while IFS= read -r row; do
|
|
echo -e "${YELLOW}[⚠]${NC} Extra Besu resource VMID $(jq -r '.vmid' <<<"$row") on $(jq -r '.node' <<<"$row") ($(jq -r '.name' <<<"$row"))"
|
|
done < <(jq -c '.unexpected_besu_resources[]' <<<"$final_json")
|
|
fi
|
|
|
|
exit 0
|