proxmox/scripts/besu/install-besu-permanent-on-missing-nodes.sh

#!/usr/bin/env bash
# Install Besu permanently on nodes that don't have /opt/besu/bin/besu (1505-1508, 2501-2505).
# Uses install-besu-in-ct-standalone.sh inside each CT; deploys config, genesis, node lists; enables and starts service.
#
# Usage: bash scripts/besu/install-besu-permanent-on-missing-nodes.sh [--dry-run]

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true

DRY_RUN=false
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=true

SSH_OPTS="-o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
BESU_VERSION="${BESU_VERSION:-23.10.3}"
GENESIS_SRC="${PROJECT_ROOT}/smom-dbis-138-proxmox/config/genesis.json"
STATIC_SRC="${PROJECT_ROOT}/config/besu-node-lists/static-nodes.json"
PERMS_SRC="${PROJECT_ROOT}/config/besu-node-lists/permissions-nodes.toml"

# VMIDs that may lack Besu (sentries 1505-1508 on ml110; RPC 2500-2505 on r630-01)
SENTRY_VMIDS=(1505 1506 1507 1508)
RPC_VMIDS=(2500 2501 2502 2503 2504 2505)
declare -A HOST_BY_VMID
for v in 1505 1506 1507 1508; do HOST_BY_VMID[$v]="${PROXMOX_ML110:-192.168.11.10}"; done
for v in 2500 2501 2502 2503 2504 2505; do HOST_BY_VMID[$v]="${PROXMOX_R630_01:-192.168.11.11}"; done
declare -A IP_BY_VMID
IP_BY_VMID[1505]=192.168.11.213
IP_BY_VMID[1506]=192.168.11.214
IP_BY_VMID[1507]=192.168.11.244
IP_BY_VMID[1508]=192.168.11.245
IP_BY_VMID[2500]=192.168.11.172
IP_BY_VMID[2501]=192.168.11.173
IP_BY_VMID[2502]=192.168.11.174
IP_BY_VMID[2503]=192.168.11.246
IP_BY_VMID[2504]=192.168.11.247
IP_BY_VMID[2505]=192.168.11.248

[[ ! -f "$GENESIS_SRC" ]] && { echo "ERROR: $GENESIS_SRC not found"; exit 1; }
[[ ! -f "$STATIC_SRC" ]] && { echo "ERROR: $STATIC_SRC not found"; exit 1; }
[[ ! -f "$PERMS_SRC" ]] && { echo "ERROR: $PERMS_SRC not found"; exit 1; }

install_sentry() {
  local vmid=$1 host=${HOST_BY_VMID[$vmid]} ip=${IP_BY_VMID[$vmid]}
  echo "--- VMID $vmid (sentry @ $ip) ---"
  if $DRY_RUN; then echo "  [dry-run] would install Besu sentry"; return 0; fi
  ssh $SSH_OPTS "root@$host" "pct exec $vmid -- rm -rf /opt/besu 2>/dev/null; true"
  scp -q $SSH_OPTS "${PROJECT_ROOT}/scripts/install-besu-in-ct-standalone.sh" "root@${host}:/tmp/"
  ssh $SSH_OPTS "root@$host" "pct push $vmid /tmp/install-besu-in-ct-standalone.sh /tmp/install-besu-in-ct-standalone.sh && pct exec $vmid -- env NODE_TYPE=sentry BESU_VERSION=$BESU_VERSION bash /tmp/install-besu-in-ct-standalone.sh" || { echo "  install failed"; return 1; }
  # Deploy genesis + node lists
  scp -q $SSH_OPTS "$GENESIS_SRC" "$STATIC_SRC" "$PERMS_SRC" "root@${host}:/tmp/"
  ssh $SSH_OPTS "root@$host" "pct push $vmid /tmp/genesis.json /etc/besu/genesis.json && pct push $vmid /tmp/static-nodes.json /etc/besu/static-nodes.json && pct push $vmid /tmp/permissions-nodes.toml /etc/besu/permissions-nodes.toml && pct exec $vmid -- chown -R besu:besu /etc/besu"
  # Config: ensure paths point to /etc/besu (copy from 1504 or minimal)
  ssh $SSH_OPTS "root@$host" "pct exec 1504 -- cat /etc/besu/config-sentry.toml 2>/dev/null" 2>/dev/null | sed 's|/genesis/|/etc/besu/|g; s|/var/lib/besu/.*|/etc/besu/|g' > /tmp/config-sentry.toml 2>/dev/null || true
  if [[ -s /tmp/config-sentry.toml ]]; then
    scp -q $SSH_OPTS /tmp/config-sentry.toml "root@${host}:/tmp/"
    ssh $SSH_OPTS "root@$host" "pct push $vmid /tmp/config-sentry.toml /etc/besu/config-sentry.toml && pct exec $vmid -- sed -i 's|permissions-nodes-config-file=.*|permissions-nodes-config-file=\"/etc/besu/permissions-nodes.toml\"|; s|static-nodes-file=.*|static-nodes-file=\"/etc/besu/static-nodes.json\"|; s|genesis-file=.*|genesis-file=\"/etc/besu/genesis.json\"|' /etc/besu/config-sentry.toml && pct exec $vmid -- chown besu:besu /etc/besu/config-sentry.toml"
  fi
  ssh $SSH_OPTS "root@$host" "pct exec $vmid -- systemctl enable besu-sentry.service && pct exec $vmid -- systemctl start besu-sentry.service" && echo "  besu-sentry enabled and started" || echo "  start failed"
}

install_rpc() {
  local vmid=$1 host=${HOST_BY_VMID[$vmid]} ip=${IP_BY_VMID[$vmid]}
  echo "--- VMID $vmid (RPC @ $ip) ---"
  if $DRY_RUN; then echo "  [dry-run] would install Besu RPC"; return 0; fi
  if ! ssh $SSH_OPTS "root@$host" "pct exec $vmid -- bash -c 'touch /tmp/.w && rm -f /tmp/.w'" 2>/dev/null; then
    echo "  CT $vmid /tmp not writable. Make CT writable (docs/00-meta/502_DEEP_DIVE_ROOT_CAUSES_AND_FIXES.md §Read-only CT) and re-run."
    return 1
  fi
  ssh $SSH_OPTS "root@$host" "pct exec $vmid -- rm -rf /opt/besu 2>/dev/null; true"
  scp -q $SSH_OPTS "${PROJECT_ROOT}/scripts/install-besu-in-ct-standalone.sh" "root@${host}:/tmp/"
  ssh $SSH_OPTS "root@$host" "pct push $vmid /tmp/install-besu-in-ct-standalone.sh /tmp/install-besu-in-ct-standalone.sh && pct exec $vmid -- env NODE_TYPE=rpc BESU_VERSION=$BESU_VERSION bash /tmp/install-besu-in-ct-standalone.sh" || { echo "  install failed"; return 1; }
  # 2500-style: besu.service + config.toml (not besu-rpc.service). Use inline template so we don't depend on 2500 having Besu.
  cat << 'BESUSVC' > /tmp/besu.service
[Unit]
Description=Hyperledger Besu
After=network.target
Wants=network-online.target

[Service]
Type=simple
User=besu
Group=besu
WorkingDirectory=/opt/besu
Environment="BESU_OPTS=-Xmx2g -Xms1g -Djava.io.tmpdir=/data/besu/tmp"
ExecStart=/opt/besu/bin/besu --config-file=/etc/besu/config.toml
Restart=always
RestartSec=10
LimitNOFILE=65536
StandardOutput=journal
StandardError=journal
SyslogIdentifier=besu

[Install]
WantedBy=multi-user.target
BESUSVC
  scp -q $SSH_OPTS /tmp/besu.service "root@${host}:/tmp/"
  ssh $SSH_OPTS "root@$host" "pct push $vmid /tmp/besu.service /etc/systemd/system/besu.service"
  # config.toml with this node's p2p-host
  cat <<EOF > /tmp/config-rpc-${vmid}.toml
data-path="/data/besu"
genesis-file="/etc/besu/genesis.json"
network-id=138
p2p-host="$ip"
p2p-port=30303
max-peers=25
discovery-enabled=false
host-allowlist=["*"]
rpc-http-enabled=true
rpc-http-host="0.0.0.0"
rpc-http-port=8545
rpc-http-api=["ETH","NET","WEB3","ADMIN","DEBUG"]
rpc-http-cors-origins=["*"]
rpc-ws-enabled=true
rpc-ws-host="0.0.0.0"
rpc-ws-port=8546
rpc-ws-api=["ETH","NET","WEB3","ADMIN","DEBUG"]
metrics-enabled=true
metrics-host="0.0.0.0"
metrics-port=9545
logging="INFO"
sync-mode="FULL"
privacy-enabled=false
miner-enabled=false
permissions-nodes-config-file-enabled=true
permissions-nodes-config-file="/etc/besu/permissions-nodes.toml"
static-nodes-file="/etc/besu/static-nodes.json"
EOF
  scp -q $SSH_OPTS "/tmp/config-rpc-${vmid}.toml" "root@${host}:/tmp/config.toml"
  ssh $SSH_OPTS "root@$host" "pct push $vmid /tmp/config.toml /etc/besu/config.toml"
  scp -q $SSH_OPTS "$GENESIS_SRC" "$STATIC_SRC" "$PERMS_SRC" "root@${host}:/tmp/"
  ssh $SSH_OPTS "root@$host" "pct push $vmid /tmp/genesis.json /etc/besu/genesis.json && pct push $vmid /tmp/static-nodes.json /etc/besu/static-nodes.json && pct push $vmid /tmp/permissions-nodes.toml /etc/besu/permissions-nodes.toml && pct exec $vmid -- chown -R besu:besu /etc/besu"
  ssh $SSH_OPTS "root@$host" "pct exec $vmid -- systemctl disable besu-rpc.service 2>/dev/null; pct exec $vmid -- systemctl daemon-reload; pct exec $vmid -- systemctl enable besu.service && pct exec $vmid -- systemctl start besu.service" && echo "  besu.service enabled and started" || echo "  start failed"
}

echo "Installing Besu permanently on nodes missing /opt/besu/bin/besu (1505-1508, 2500-2505)"
echo ""

for vmid in "${SENTRY_VMIDS[@]}"; do
  host="${HOST_BY_VMID[$vmid]:-}"
  has_besu=$(ssh $SSH_OPTS "root@$host" "pct exec $vmid -- test -x /opt/besu/bin/besu 2>/dev/null" && echo yes || echo no)
  if [[ "$has_besu" == yes ]]; then
    echo "VMID $vmid: Besu already present (skip)"
  else
    install_sentry "$vmid"
  fi
done

for vmid in "${RPC_VMIDS[@]}"; do
  host="${HOST_BY_VMID[$vmid]:-}"
  has_besu=$(ssh $SSH_OPTS "root@$host" "pct exec $vmid -- test -x /opt/besu/bin/besu 2>/dev/null" && echo yes || echo no)
  if [[ "$has_besu" == yes ]]; then
    echo "VMID $vmid: Besu already present (skip)"
  else
    install_rpc "$vmid"
  fi
done

echo ""
echo "Done. Verify: bash scripts/besu/restart-besu-reload-node-lists.sh (optional); then check block production on RPCs."
rm -f /tmp/config-sentry.toml /tmp/besu.service /tmp/config.toml 2>/dev/null || true
for v in 2500 2501 2502 2503 2504 2505; do rm -f /tmp/config-rpc-${v}.toml 2>/dev/null; done