ops: add chain138 and rpc diagnostic tooling

2026-04-24 10:55:55 -07:00
parent 454aeda9d5
commit c65b896fad
15 changed files with 909 additions and 52 deletions
--- a/scripts/clear-besu-transaction-pools-complete.sh
+++ b/scripts/clear-besu-transaction-pools-complete.sh
@@ -9,10 +9,7 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
-
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true

 # Colors
 RED='\033[0;31m'
@@ -28,21 +25,29 @@ log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
 log_error() { echo -e "${RED}[✗]${NC} $1"; }
 log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; }

-# Proxmox hosts
-PROXMOX_HOSTS=("ml110" "r630-01")
+host_for_vmid() {
+    local vmid="$1"
+    if type get_host_for_vmid >/dev/null 2>&1; then
+        get_host_for_vmid "$vmid"
+    elif [ "$vmid" -le 1002 ]; then
+        echo "${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
+    else
+        echo "${PROXMOX_HOST_ML110:-192.168.11.10}"
+    fi
+}

 # RPC nodes (VMID:host)
 RPC_NODES=(
-    "2101:ml110"  # Core RPC
+    "2101:$(host_for_vmid 2101)"  # Core RPC
 )

 # Validators (VMID:host)
 VALIDATORS=(
-    "1000:r630-01"
-    "1001:r630-01"
-    "1002:r630-01"
-    "1003:ml110"
-    "1004:ml110"
+    "1000:$(host_for_vmid 1000)"
+    "1001:$(host_for_vmid 1001)"
+    "1002:$(host_for_vmid 1002)"
+    "1003:$(host_for_vmid 1003)"
+    "1004:$(host_for_vmid 1004)"
 )

 log_section "Clear Transaction Pools - Complete Method"
--- a/scripts/clear-rpc-2103-txpool.sh
+++ b/scripts/clear-rpc-2103-txpool.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Clear transaction pool on Thirdweb admin core RPC (VMID 2103 — 192.168.11.217).
+# Stops Besu, removes local tx-pool data under /data/besu|/var/lib/besu, restarts.
+# See also: clear-rpc-2201-txpool.sh, clear-all-transaction-pools.sh
+# Usage: ./scripts/clear-rpc-2103-txpool.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+PROXMOX_USER="${PROXMOX_USER:-root}"
+# 2101/2103 on r630-01 (load-project-env get_host_for_vmid)
+HOST="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
+VMID=2103
+SSH_TARGET="${PROXMOX_USER}@${HOST}"
+
+echo "=== Clear Thirdweb core RPC (VMID 2103) transaction pool ==="
+echo "Host: $HOST  LXC: $VMID  (http://192.168.11.217:8545)"
+echo ""
+
+if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" "pct list 2>/dev/null | grep -q '2103'"; then
+  echo "VMID 2103 not found on $HOST. Abort." >&2
+  exit 1
+fi
+
+echo "Stopping Besu (besu-rpc-core or besu-rpc)..."
+ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \
+  "pct exec $VMID -- systemctl stop besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc.service 2>/dev/null || true" 2>&1 | grep -v "Configuration file" || true
+sleep 2
+
+echo "Clearing tx pool database..."
+ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \
+  "pct exec $VMID -- bash -c '
+  for d in /data/besu /var/lib/besu; do
+    [ -d \"\$d\" ] && find \"\$d\" -type d -name \"*pool*\" -exec rm -rf {} \; 2>/dev/null; find \"\$d\" -type f -name \"*transaction*\" -delete 2>/dev/null; find \"\$d\" -type f -name \"*txpool*\" -delete 2>/dev/null; true
+  done
+  '" 2>&1 | grep -v "Configuration file" || true
+echo "Pool cleared."
+
+echo "Starting Besu..."
+ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \
+  "pct exec $VMID -- systemctl start besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc.service 2>/dev/null || true" 2>&1 | grep -v "Configuration file" || true
+sleep 3
+
+if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" "pct exec $VMID -- systemctl is-active besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl is-active besu-rpc 2>/dev/null" 2>/dev/null | grep -q active; then
+  echo "Service: active"
+else
+  echo "WARN: Check service status on $HOST: pct exec $VMID -- systemctl status besu-rpc-core besu-rpc" >&2
+fi
+
+echo ""
+echo "Done. wait ~30s, then verify: curl -s -X POST http://192.168.11.217:8545 -H Content-Type:application/json -d '{\"jsonrpc\":\"2.0\",\"method\":\"txpool_besuPendingTransactions\",\"params\":[],\"id\":1}' | head -c 200"
--- a/scripts/fix-validator-permissioning-toml.sh
+++ b/scripts/fix-validator-permissioning-toml.sh
@@ -12,6 +12,7 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 cd "$PROJECT_ROOT"

 [ -f config/ip-addresses.conf ] && source config/ip-addresses.conf 2>/dev/null || true
+[ -f scripts/lib/load-project-env.sh ] && source scripts/lib/load-project-env.sh 2>/dev/null || true

 SOURCE_TOML="$PROJECT_ROOT/config/besu-node-lists/permissions-nodes.toml"
 SOURCE_STATIC="$PROJECT_ROOT/config/besu-node-lists/static-nodes.json"
@@ -25,17 +26,39 @@ if [ ! -f "$SOURCE_STATIC" ]; then
 fi

 R630_01="${PROXMOX_HOST_R630_01:-${PROXMOX_R630_01:-192.168.11.11}}"
-R630_03="${PROXMOX_HOST_R630_03:-${PROXMOX_R630_03:-192.168.11.13}}"
-USER="${PROXMOX_USER:-root}"
+SSH_USER="${PROXMOX_SSH_USER:-root}"
 PERM_PATH="/var/lib/besu/permissions"
 CONFIG_GLOB="/etc/besu/config-validator.toml"

+validator_host() {
+  local vmid="$1"
+  if type get_host_for_vmid >/dev/null 2>&1; then
+    get_host_for_vmid "$vmid"
+  elif [[ "$vmid" -le 1002 ]]; then
+    echo "$R630_01"
+  else
+    echo "${PROXMOX_HOST_ML110:-192.168.11.10}"
+  fi
+}
+
+validator_ip() {
+  local vmid="$1"
+  case "$vmid" in
+    1000) echo "${IP_VALIDATOR_0:-192.168.11.100}" ;;
+    1001) echo "${IP_VALIDATOR_1:-192.168.11.101}" ;;
+    1002) echo "${IP_VALIDATOR_2:-192.168.11.102}" ;;
+    1003) echo "${IP_VALIDATOR_3:-192.168.11.103}" ;;
+    1004) echo "${IP_VALIDATOR_4:-192.168.11.104}" ;;
+    *) return 1 ;;
+  esac
+}
+
 VALIDATORS=(
-  "1000:$R630_01"
-  "1001:$R630_01"
-  "1002:$R630_01"
-  "1003:$R630_03"
-  "1004:$R630_03"
+  "1000:$(validator_host 1000)"
+  "1001:$(validator_host 1001)"
+  "1002:$(validator_host 1002)"
+  "1003:$(validator_host 1003)"
+  "1004:$(validator_host 1004)"
 )

 RED='\033[0;31m'
@@ -53,18 +76,23 @@ echo "  Both are essential: static-nodes = bootstrap peers, permissions-nodes =
 echo ""

 # Copy both files to each host once
-for host in "$R630_01" "$R630_03"; do
+for host in $(printf '%s\n' "${VALIDATORS[@]}" | cut -d: -f2 | sort -u); do
  log_info "Copying static-nodes.json and permissions-nodes.toml to $host"
-  scp -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SOURCE_STATIC" "$SOURCE_TOML" "$USER@$host:/tmp/" 2>/dev/null || { log_err "scp to $host failed"; exit 1; }
+  scp -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SOURCE_STATIC" "$SOURCE_TOML" "$SSH_USER@$host:/tmp/" 2>/dev/null || { log_err "scp to $host failed"; exit 1; }
  log_ok "  Copied"
 done

 FAILED=0
 for entry in "${VALIDATORS[@]}"; do
  IFS=: read -r vmid host <<< "$entry"
+  validator_ip="$(validator_ip "$vmid")" || {
+    log_err "  could not determine validator IP for VMID $vmid"
+    ((FAILED++)) || true
+    continue
+  }
  log_info "VMID $vmid @ $host"

-  status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct status $vmid 2>/dev/null" | awk '{print $2}' || echo "unknown")
+  status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct status $vmid 2>/dev/null" | awk '{print $2}' || echo "unknown")
  if [ "$status" != "running" ]; then
    log_info "  Skip (not running)"
    continue
@@ -72,22 +100,26 @@ for entry in "${VALIDATORS[@]}"; do

  # Push static-nodes.json to /var/lib/besu/ and permissions-nodes.toml to permissions/
  STATIC_PATH="/var/lib/besu/static-nodes.json"
-  if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct push $vmid /tmp/static-nodes.json ${STATIC_PATH} && pct push $vmid /tmp/permissions-nodes.toml ${PERM_PATH}/permissions-nodes.toml" 2>/dev/null; then
+  if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct push $vmid /tmp/static-nodes.json ${STATIC_PATH} && pct push $vmid /tmp/permissions-nodes.toml ${PERM_PATH}/permissions-nodes.toml" 2>/dev/null; then
    log_err "  pct push failed"
    ((FAILED++)) || true
    continue
  fi

  # Point config to TOML (not JSON) and ensure static-nodes-file and permissions path are set
-  if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct exec $vmid -- bash -c '
+  if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct exec $vmid -- bash -c '
    for f in /etc/besu/config-validator.toml /config/config-validator.toml; do
      [ -f \"\$f\" ] || continue
      sed -i \"s|permissioned-nodes\\.json|permissions-nodes.toml|g\" \"\$f\"
      sed -i \"s|\"/var/lib/besu/permissions/permissioned-nodes.json\"|\"/var/lib/besu/permissions/permissions-nodes.toml\"|g\" \"\$f\"
      sed -i \"s|^static-nodes-file=.*|static-nodes-file=\\\"/var/lib/besu/static-nodes.json\\\"|\" \"\$f\"
      sed -i \"s|^permissions-nodes-config-file=.*|permissions-nodes-config-file=\\\"/var/lib/besu/permissions/permissions-nodes.toml\\\"|\" \"\$f\"
+      sed -i \"s|^p2p-host=.*|p2p-host=\\\"${validator_ip}\\\"|\" \"\$f\"
+      sed -i \"s|^sync-mode=.*|sync-mode=\\\"FULL\\\"|\" \"\$f\"
      grep -q \"static-nodes-file\" \"\$f\" || echo \"static-nodes-file=\\\"/var/lib/besu/static-nodes.json\\\"\" >> \"\$f\"
      grep -q \"permissions-nodes-config-file\" \"\$f\" || echo \"permissions-nodes-config-file=\\\"/var/lib/besu/permissions/permissions-nodes.toml\\\"\" >> \"\$f\"
+      grep -q \"^p2p-host=\" \"\$f\" || echo \"p2p-host=\\\"${validator_ip}\\\"\" >> \"\$f\"
+      grep -q \"^sync-mode=\" \"\$f\" || echo \"sync-mode=\\\"FULL\\\"\" >> \"\$f\"
      break
    done
  '" 2>/dev/null; then
@@ -96,9 +128,16 @@ for entry in "${VALIDATORS[@]}"; do
    continue
  fi

-  ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct exec $vmid -- chown besu:besu ${STATIC_PATH} ${PERM_PATH}/permissions-nodes.toml 2>/dev/null || pct exec $vmid -- chown root:root ${STATIC_PATH} ${PERM_PATH}/permissions-nodes.toml" 2>/dev/null || true
+  ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct exec $vmid -- chown besu:besu ${STATIC_PATH} ${PERM_PATH}/permissions-nodes.toml 2>/dev/null || pct exec $vmid -- chown root:root ${STATIC_PATH} ${PERM_PATH}/permissions-nodes.toml" 2>/dev/null || true

-  if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct exec $vmid -- systemctl restart besu-validator" 2>/dev/null; then
+  if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct exec $vmid -- bash -lc '
+    timeout 30 systemctl restart besu-validator || {
+      systemctl kill -s SIGKILL besu-validator || true
+      sleep 2
+      systemctl reset-failed besu-validator || true
+      systemctl start besu-validator
+    }
+  '" 2>/dev/null; then
    log_err "  restart failed"
    ((FAILED++)) || true
    continue
@@ -108,8 +147,8 @@ for entry in "${VALIDATORS[@]}"; do
 done

 # Cleanup host /tmp
-for host in "$R630_01" "$R630_03"; do
-  ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "rm -f /tmp/permissions-nodes.toml /tmp/static-nodes.json" 2>/dev/null || true
+for host in $(printf '%s\n' "${VALIDATORS[@]}" | cut -d: -f2 | sort -u); do
+  ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "rm -f /tmp/permissions-nodes.toml /tmp/static-nodes.json" 2>/dev/null || true
 done

 echo "=== Summary ==="
--- a/scripts/generate-mcp-allowlist-from-chain138.sh
+++ b/scripts/generate-mcp-allowlist-from-chain138.sh
@@ -5,7 +5,8 @@
 # Usage:
 #   ./scripts/generate-mcp-allowlist-from-chain138.sh                    # print to stdout
 #   ./scripts/generate-mcp-allowlist-from-chain138.sh -o allowlist.json   # write file
-#   OUT_PATH=ai-mcp-pmm-controller/config/allowlist-138.json ./scripts/generate-mcp-allowlist-from-chain138.sh
+#   ./scripts/generate-mcp-allowlist-from-chain138.sh -o ai-mcp-pmm-controller/config/allowlist-138.json
+#   (Output path is only via -o; there is no OUT_PATH env var.)
 #
 # Requires: RPC_URL_138 (or RPC_URL), DODO_PMM_INTEGRATION_ADDRESS in env (or .env in smom-dbis-138).
 # Optional: MAX_POOLS (default 200), PROFILE (default dodo_pmm_v2_like).
--- a/scripts/inspect-rpc-2201-txpool-pct.sh
+++ b/scripts/inspect-rpc-2201-txpool-pct.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# From Proxmox host, use pct exec on VMID 2201 to inspect local tx-pool / Besu (Public RPC).
+# JSON-RPC from outside may not expose txpool_besuPendingTransactions; this runs checks inside the LXC.
+# Usage: ./scripts/inspect-rpc-2201-txpool-pct.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+PROXMOX_USER="${PROXMOX_USER:-root}"
+HOST="${PROXMOX_R630_02:-${PROXMOX_HOST_R630_02:-192.168.11.12}}"
+VMID=2201
+SSH_TARGET="${PROXMOX_USER}@${HOST}"
+
+echo "=== Inspect Public RPC (VMID 2201) from host via pct ==="
+echo "Host: $HOST  (http://192.168.11.221:8545)"
+echo ""
+
+if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" "pct list 2>/dev/null | grep -q '$VMID'"; then
+  echo "VMID $VMID not found on $HOST. Abort." >&2
+  exit 1
+fi
+
+ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" bash -s -- "$VMID" <<'REMOTE'
+VMID="${1:?}"
+set -e
+pct exec "$VMID" -- bash -s <<'INNER'
+set -e
+echo "--- systemctl ---"
+systemctl is-active besu-rpc 2>/dev/null || systemctl is-active besu-rpc.service 2>/dev/null || echo "unknown"
+echo ""
+echo "--- txpool_besuPendingTransactions (first 2k chars) ---"
+OUT=$(curl -sS --connect-timeout 5 -X POST http://127.0.0.1:8545 -H "Content-Type: application/json" \
+  -d '{"jsonrpc":"2.0","method":"txpool_besuPendingTransactions","params":[],"id":1}' 2>&1) || true
+printf '%s' "$OUT" | head -c 2000; echo; echo ""
+echo "--- eth_blockNumber ---"
+curl -sS --connect-timeout 5 -X POST http://127.0.0.1:8545 -H "Content-Type: application/json" \
+  -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}'
+echo; echo ""
+echo "--- du *pool* under Besu data dirs ---"
+for d in /data/besu /var/lib/besu; do
+  [ -d "$d" ] && find "$d" -maxdepth 4 -type d -name '*pool*' 2>/dev/null | while read -r p; do
+    du -sh "$p" 2>/dev/null || true
+  done
+done
+echo "(end)"
+echo ""
+echo "--- journal (besu, last 12) ---"
+journalctl -u besu-rpc -u besu-rpc.service -n 12 --no-pager 2>/dev/null || true
+INNER
+REMOTE
+
+echo ""
+echo "To purge 2201 pool: ./scripts/clear-rpc-2201-txpool.sh"
--- a/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh
+++ b/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Tighten Besu layered tx-pool on VMID 2103 (Thirdweb admin core) only, so a single
+# sender cannot fill the local pool with far-future / gap nonces (e.g. 15 when next is 1).
+#
+# - Sets tx-pool-max-future-by-sender=1 (do NOT use 0: Besu 24+ can log IndexOutOfBounds in
+#   SparseTransactions on new blocks when 0 is set; observed 2026-04-22 on 2103).
+# - Restarts besu-rpc. Optional: purge local pool after restart (arg --purge) via clear-rpc-2103-txpool.
+#
+# Usage: ./scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh [--purge] [--value N]
+#   --value N  default 1
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck disable=SC1091
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+VAL="${VAL:-1}"
+PURGE=false
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --purge) PURGE=true; shift ;;
+    --value) VAL="$2"; shift 2 ;;
+    *) echo "Unknown: $1" >&2; exit 2 ;;
+  esac
+done
+
+HOST="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
+VMID=2103
+CFG_PCT="/etc/besu/config-rpc.toml"
+SSH="root@${HOST}"
+
+echo "=== 2103 tx-pool: tx-pool-max-future-by-sender=$VAL (Thirdweb path only) ==="
+echo "Host: $HOST  LXC: $VMID  config: $CFG_PCT (inside LXC)"
+echo ""
+
+if ! ssh -o ConnectTimeout=8 -o StrictHostKeyChecking=no "$SSH" "pct list | grep -q '$VMID'"; then
+  echo "VMID $VMID not on $HOST" >&2
+  exit 1
+fi
+
+if [[ "$VAL" -le 0 ]]; then
+  echo "Refusing: value 0 is unsafe on current Besu (SparseTransactions/IndexOutOfBounds). Use 1 or more." >&2
+  exit 1
+fi
+
+ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH" "pct exec $VMID -- bash -c '
+set -e
+cp -a $CFG_PCT ${CFG_PCT}.bak.\$(date +%Y%m%d%H%M%S)
+if ! grep -qE \"^tx-pool-max-future-by-sender=\" $CFG_PCT; then
+  echo \"[ERROR] $CFG_PCT has no tx-pool-max-future-by-sender= line; add it manually.\" >&2
+  exit 1
+fi
+sed -i \"s/^tx-pool-max-future-by-sender=.*/tx-pool-max-future-by-sender=$VAL/\" $CFG_PCT
+grep -E \"^tx-pool-max-future-by-sender=\" $CFG_PCT
+systemctl restart besu-rpc
+sleep 3
+systemctl is-active besu-rpc
+'"
+
+if [[ "$PURGE" == "true" ]]; then
+  echo ""
+  echo "=== Purging 2103 local pool ==="
+  "${PROJECT_ROOT}/scripts/clear-rpc-2103-txpool.sh"
+else
+  echo "Done. Optional: --purge to also run clear-rpc-2103-txpool.sh"
+fi
+
+echo ""
+echo "Next: from Thirdweb, use the nonce returned by eth_getTransactionCount(deployer, 'pending'|'latest')."
+echo "A send at nonce 15 will be rejected (NONCE_TOO_FAR) until nonces 1,2,… are broadcast with the deployer key outside Thirdweb, if the chain is still behind."
--- a/scripts/maintenance/fix-block-production-staggered-restart.sh
+++ b/scripts/maintenance/fix-block-production-staggered-restart.sh
@@ -12,6 +12,7 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
 source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+[ -f "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" ] && source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true

 DRY_RUN=false
 [[ "${1:-}" == "--dry-run" ]] && DRY_RUN=true
@@ -25,14 +26,24 @@ log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
 log_ok() { echo -e "${GREEN}[✓]${NC} $1"; }
 log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }

+validator_host() {
+  local vmid="$1"
+  if type get_host_for_vmid >/dev/null 2>&1; then
+    get_host_for_vmid "$vmid"
+  elif [[ "$vmid" -le 1002 ]]; then
+    echo "${PROXMOX_HOST_R630_01:-192.168.11.11}"
+  else
+    echo "${PROXMOX_HOST_ML110:-192.168.11.10}"
+  fi
+}
+
 # Order: restart one at a time; wait between so restarted node can sync from others
-# VMID : host
 VALIDATORS=(
-  "1004:${PROXMOX_HOST_R630_03:-192.168.11.13}"
-  "1003:${PROXMOX_HOST_R630_03:-192.168.11.13}"
-  "1002:${PROXMOX_HOST_R630_01:-192.168.11.11}"
-  "1001:${PROXMOX_HOST_R630_01:-192.168.11.11}"
-  "1000:${PROXMOX_HOST_R630_01:-192.168.11.11}"
+  "1004:$(validator_host 1004)"
+  "1003:$(validator_host 1003)"
+  "1002:$(validator_host 1002)"
+  "1001:$(validator_host 1001)"
+  "1000:$(validator_host 1000)"
 )
 WAIT_BETWEEN=90
 RPC="${RPC_URL_138:-http://192.168.11.211:8545}"
--- a/scripts/maintenance/make-validator-vmids-writable-via-ssh.sh
+++ b/scripts/maintenance/make-validator-vmids-writable-via-ssh.sh
@@ -11,18 +11,28 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
 [[ -f "${PROJECT_ROOT}/config/ip-addresses.conf" ]] && source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+[[ -f "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" ]] && source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true

 R630_01="${PROXMOX_HOST_R630_01:-192.168.11.11}"
-ML110="${PROXMOX_ML110:-192.168.11.10}"
 SSH_OPTS="-o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"

-# Validators: 1000,1001,1002 on r630-01; 1003,1004 on ml110
+validator_host() {
+  local vmid="$1"
+  if type get_host_for_vmid >/dev/null 2>&1; then
+    get_host_for_vmid "$vmid"
+  elif [[ "$vmid" -le 1002 ]]; then
+    echo "$R630_01"
+  else
+    echo "${PROXMOX_HOST_ML110:-192.168.11.10}"
+  fi
+}
+
 VALIDATORS=(
-  "1000:$R630_01"
-  "1001:$R630_01"
-  "1002:$R630_01"
-  "1003:$ML110"
-  "1004:$ML110"
+  "1000:$(validator_host 1000)"
+  "1001:$(validator_host 1001)"
+  "1002:$(validator_host 1002)"
+  "1003:$(validator_host 1003)"
+  "1004:$(validator_host 1004)"
 )

 DRY_RUN=false
--- a/scripts/remove-tx-pool-min-score-validators.sh
+++ b/scripts/remove-tx-pool-min-score-validators.sh
@@ -7,18 +7,35 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 [ -f "$PROJECT_ROOT/config/ip-addresses.conf" ] && source "$PROJECT_ROOT/config/ip-addresses.conf" 2>/dev/null || true
+[ -f "$PROJECT_ROOT/scripts/lib/load-project-env.sh" ] && source "$PROJECT_ROOT/scripts/lib/load-project-env.sh" 2>/dev/null || true

 R630_01="${PROXMOX_R630_01:-192.168.11.11}"
-ML110="${PROXMOX_ML110:-192.168.11.10}"
-USER="${PROXMOX_USER:-root}"
+SSH_USER="${PROXMOX_SSH_USER:-root}"

-VALIDATORS=( "1000:$R630_01" "1001:$R630_01" "1002:$R630_01" "1003:$ML110" "1004:$ML110" )
+validator_host() {
+  local vmid="$1"
+  if type get_host_for_vmid >/dev/null 2>&1; then
+    get_host_for_vmid "$vmid"
+  elif [[ "$vmid" -le 1002 ]]; then
+    echo "$R630_01"
+  else
+    echo "${PROXMOX_HOST_ML110:-192.168.11.10}"
+  fi
+}
+
+VALIDATORS=(
+  "1000:$(validator_host 1000)"
+  "1001:$(validator_host 1001)"
+  "1002:$(validator_host 1002)"
+  "1003:$(validator_host 1003)"
+  "1004:$(validator_host 1004)"
+)
 FAILED=0

 for entry in "${VALIDATORS[@]}"; do
  IFS=: read -r vmid host <<< "$entry"
  echo "[$vmid @ $host] Removing tx-pool-min-score and restarting..."
-  if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct exec $vmid -- bash -c '
+  if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct exec $vmid -- bash -c '
    CFG=/etc/besu/config-validator.toml
    [ -f /config/config-validator.toml ] && CFG=/config/config-validator.toml
    [ ! -f \"\$CFG\" ] && exit 1
--- a/scripts/verify/decode-singleton-deploy-pending-2103.sh
+++ b/scripts/verify/decode-singleton-deploy-pending-2103.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# List pending txs on 2103 that target the EIP-2470 singleton and decode deploy(bytes) initcode.
+# Usage: ./scripts/verify/decode-singleton-deploy-pending-2103.sh
+# Optional: RPC_URL_2103=...  PENDING_TX_FROM=0xB2dE... (stuck tx signer; avoids operator DEPLOYER from dotenv) SINGLETON=0x4e59...
+
+set -euo pipefail
+RPC_URL="${RPC_URL_2103:-http://192.168.11.217:8545}"
+PENDING_TX_FROM="${PENDING_TX_FROM:-0xB2dEA0e264ddfFf91057A3415112e57A1a5Eac14}"
+SINGLETON="${SINGLETON:-0x4e59b44847b379578588920cA78FbF26c0B4956C}"
+OUT_DIR="${OUT_DIR:-/tmp/singleton-decodes-$(date +%Y%m%d%H%M%S)}"
+export RPC_URL PENDING_TX_FROM SINGLETON OUT_DIR
+DEPLOYER="${PENDING_TX_FROM}"
+export DEPLOYER
+mkdir -p "$OUT_DIR"
+echo "RPC: $RPC_URL  pending from filter: $PENDING_TX_FROM  singleton: $SINGLETON"
+echo "Output: $OUT_DIR"
+echo ""
+
+python3 <<'PY'
+import json, sys, subprocess, os, urllib.request
+
+rpc = os.environ["RPC_URL"]
+dep = os.environ["DEPLOYER"].lower()
+sing = os.environ["SINGLETON"].lower()
+out = os.environ["OUT_DIR"]
+sel_4 = bytes.fromhex("00774360")
+
+req = urllib.request.Request(
+    rpc,
+    data=json.dumps({"jsonrpc": "2.0", "method": "txpool_besuPendingTransactions", "params": [], "id": 1}).encode(),
+    headers={"Content-Type": "application/json"},
+)
+raw = urllib.request.urlopen(req, timeout=15).read()
+d = json.loads(raw.decode())
+if d.get("error"):
+    print("RPC error:", d["error"])
+    sys.exit(1)
+txs = d.get("result") or []
+print("Total pending on 2103:", len(txs))
+matched = []
+for t in txs:
+    to = (t.get("to") or "").lower()
+    fr = (t.get("from") or "").lower()
+    if to != sing:
+        continue
+    if fr != dep:
+        continue
+    matched.append(t)
+print("Matching deployer -> singleton:", len(matched))
+if not matched:
+    print("Nothing to decode. Re-run when those txs are in 2103's pool, or paste a raw tx into this script.")
+    sys.exit(0)
+
+for i, t in enumerate(matched):
+    h = t.get("hash", "unknown")
+    raw_hex = t.get("input") or "0x"
+    if not raw_hex.startswith("0x"):
+        raw_hex = "0x" + raw_hex
+    inp = raw_hex[2:]
+    b = bytes.fromhex(inp)
+    j = b.find(sel_4)
+    if j >= 0 and j != 0:
+        print("INFO", h, "deploy(bytes) selector 0x00774360 at byte offset", j, "(not at 0; Thirdweb / padding)")
+        b = b[j:]
+    if len(b) < 4 or b[:4] != sel_4:
+        p = os.path.join(out, f"full_calldata_{i+1}_{h[:10]}.hex")
+        with open(p, "w") as f:
+            f.write(raw_hex.rstrip() + "\n")
+        if j < 0:
+            print("WARN", h, "no 0x00774360 in this `input` — on-chain `eth_call` may still use full RPC `input` (see dry-run-thirdweb-singleton-2103.sh) ->", p)
+        else:
+            print("WARN", h, "not deploy(bytes) 0x00774360; saved for replay ->", p)
+        continue
+    if len(b) < 4 + 32 + 32:
+        print("Input too short for ABI deploy(bytes)", h, "len", len(b))
+        continue
+    off = int.from_bytes(b[4:36], "big")
+    if 4 + off + 32 > len(b):
+        print("Bad offset", off, h)
+        continue
+    ln = int.from_bytes(b[4 + off : 4 + off + 32], "big")
+    init = b[4 + off + 32 : 4 + off + 32 + ln]
+    if len(init) != ln:
+        print("Length mismatch", h, ln, len(init))
+        continue
+    path = os.path.join(out, f"initcode_{i+1}_{h[:10]}.hex")
+    with open(path, "w") as f:
+        f.write("0x" + init.hex())
+    print("---", h, "---")
+    print("  nonce:", t.get("nonce"), " gas:", t.get("gas"), " initcode bytes:", len(init), "->", path)
+    try:
+        r = subprocess.run(
+            ["cast", "keccak", "0x" + init.hex()],
+            capture_output=True, text=True, timeout=5, check=False,
+        )
+        if r.returncode == 0:
+            print("  keccak256(initcode):", r.stdout.strip())
+    except FileNotFoundError:
+        pass
+    # After slice, b is body starting at 0x00774360; full tx `input` for the node is still raw_hex
+    bl = len(b)
+    print("  Body from selector (for ABI decode only):", bl, "bytes")
+
+    def _parse_gas(x):
+        if isinstance(x, int):
+            return x
+        s = str(x).strip()
+        if s.lower().startswith("0x"):
+            return int(s, 16)
+        return int(s, 10)
+
+    g = t.get("gas", "0x500000")
+    try:
+        gl = _parse_gas(g)
+    except (ValueError, TypeError):
+        gl = 5242880
+    print("  Example: cast send -r", rpc, "--private-key <KEY> --gas-limit", gl, sing, raw_hex)
+    print()
+PY
+
+echo "Done. Files under $OUT_DIR"
--- a/scripts/verify/dry-run-thirdweb-singleton-2103.sh
+++ b/scripts/verify/dry-run-thirdweb-singleton-2103.sh
@@ -0,0 +1,255 @@
+#!/usr/bin/env bash
+# Thirdweb / EIP-2470 singleton: simulate full calldata on VM 2103 before any broadcast.
+# - eth_call  -> CREATE2 precomputed address (20 bytes, left-padded in 32-byte word)
+# - cast estimate  -> gas (same as eth_estimateGas)
+# - getCode on predicted  -> should be 0x if not yet deployed
+# - getTransactionCount  -> for operator nonce when planning a real send
+#
+# Default mode is DRY-RUN only (no transaction). Use --send to broadcast (requires
+# CONFIRM_BROADCAST=1 in the environment in addition to PRIVATE_KEY from dotenv).
+#
+# Usage (from repo root, operator LAN + load-project-env):
+#   source scripts/lib/load-project-env.sh
+#   export INPUT_FILE=/path/to/one-line-0x-hex
+#   ./scripts/verify/dry-run-thirdweb-singleton-2103.sh
+#
+# Optional: INPUT_FILE=...  RPC_URL_2103=...  GAS_LIM=  GAS_BUMP_PCT=20  CAST_NONCE=  SKIP_2103_POOL_SSH_CHECK=1
+# EIP-1559 (required on 2103):  cast send without fees used ~15/1 wei maxFee/priority; node often reports
+#   eth_gasPrice ≈ 1000 wei. Set explicitly:
+#   GAS_MAX_FEE_PER_GAS= (wei)  GAS_PRIORITY_FEE_PER_GAS= (wei)  — or we derive from baseFee + cast gas-price
+# Send tuning:  ETH_TIMEOUT= (sec, default 600)  CAST_ASYNC=1  (return hash, no receipt wait)
+# Post-mine:  verify-singleton-post-deploy-2103.sh 0xPred
+#
+# Ref: docs/04-configuration/RPC_ENDPOINTS_MASTER.md (2103 Thirdweb admin core)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck disable=SC1091
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+
+SEND=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+  --send) SEND=1; shift ;;
+  -h | --help)
+    sed -n '1,30p' "$0"
+    exit 0
+    ;;
+  *) echo "Unknown arg: $1 (use --help)" >&2; exit 2 ;;
+  esac
+done
+
+if [[ "$SEND" -eq 1 && "${CONFIRM_BROADCAST:-0}" != "1" ]]; then
+  echo "ERROR: --send requires CONFIRM_BROADCAST=1. Run this script with no args for a dry-run only; then:" >&2
+  echo "  CONFIRM_BROADCAST=1 $0 --send" >&2
+  exit 1
+fi
+
+# VM 2103 Thirdweb admin — do not inherit generic dotenv RPC_URL (often 2101 @ .211)
+RPC_2103="${RPC_URL_2103:-http://192.168.11.217:8545}"
+SINGLETON="${SINGLETON:-0x4e59b44847b379578588920cA78FbF26c0B4956C}"
+INPUT_FILE="${INPUT_FILE:-/tmp/thirdweb-factory-input-2103.hex}"
+
+if [[ -z "${PRIVATE_KEY:-}" ]]; then
+  echo "ERROR: PRIVATE_KEY not set (e.g. from smom-dbis-138/.env via load-project-env.sh)" >&2
+  exit 1
+fi
+if ! command -v cast &>/dev/null; then
+  echo "ERROR: foundry 'cast' not in PATH" >&2
+  exit 1
+fi
+if [[ ! -f "$INPUT_FILE" ]]; then
+  echo "ERROR: INPUT_FILE not found: $INPUT_FILE" >&2
+  echo "  Hint: run scripts/verify/decode-singleton-deploy-pending-2103.sh when pool has" >&2
+  echo "  Thirdweb deploys, or set INPUT_FILE= to a one-line 0x… file." >&2
+  exit 1
+fi
+
+DATA="$(tr -d ' \n\r' < "$INPUT_FILE")"
+if [[ "${#DATA}" -lt 10 || "${DATA:0:2}" != "0x" ]]; then
+  echo "ERROR: $INPUT_FILE must be a single line 0x-prefixed hex" >&2
+  exit 1
+fi
+
+FROM_ADDR="$(cast wallet address "$PRIVATE_KEY" 2>/dev/null || true)"
+if [[ -z "$FROM_ADDR" || "$FROM_ADDR" == "0x0000000000000000000000000000000000000000" ]]; then
+  echo "ERROR: could not derive address from PRIVATE_KEY" >&2
+  exit 1
+fi
+
+if [[ "$SEND" -eq 1 ]]; then
+  echo "=== 2103 singleton broadcast (CONFIRM_BROADCAST=1) ==="
+else
+  echo "=== 2103 singleton dry-run (no broadcast) ==="
+fi
+echo "RPC:       $RPC_2103  (set RPC_URL_2103= to override)"
+echo "Singleton: $SINGLETON"
+echo "Input:     $INPUT_FILE  (${#DATA} hex chars = $(( ( ${#DATA} - 2) / 2 )) bytes data)"
+echo "From:      $FROM_ADDR  (cast wallet address of PRIVATE_KEY)"
+echo ""
+
+if ! out=$(cast call -r "$RPC_2103" --from "$FROM_ADDR" "$SINGLETON" --data "$DATA" 2>&1); then
+  echo "eth_call (cast call) failed:" >&2
+  echo "$out" >&2
+  exit 1
+fi
+if [[ ${#out} -ge 66 ]]; then
+  PRED=0x${out: -40}
+elif [[ ${#out} -eq 42 && "$out" == 0x* ]]; then
+  PRED="$out"
+else
+  PRED="$out"
+fi
+echo "eth_call result: $out"
+if [[ "$PRED" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+  echo "Predicted CREATE2 address: $PRED"
+else
+  echo "WARN: could not parse 20-byte address from return (see raw above)" >&2
+fi
+
+if ! gas=$(cast estimate -r "$RPC_2103" --from "$FROM_ADDR" "$SINGLETON" "$DATA" 2>&1); then
+  echo "eth_estimateGas (cast estimate) failed: $gas" >&2
+  exit 1
+fi
+echo "eth_estimateGas:  $gas"
+if ! [[ "$gas" =~ ^[0-9]+$ ]]; then
+  echo "ERROR: could not parse gas estimate: $gas" >&2
+  exit 1
+fi
+GAS_BUMP_PCT="${GAS_BUMP_PCT:-20}"
+if ! [[ "$GAS_BUMP_PCT" =~ ^[0-9]+$ ]]; then
+  echo "ERROR: GAS_BUMP_PCT must be a non-negative integer, got: $GAS_BUMP_PCT" >&2
+  exit 1
+fi
+if [[ -z "${GAS_LIM:-}" ]]; then
+  GAS_LIM=$((( gas * (100 + GAS_BUMP_PCT) + 99) / 100))
+  echo "suggested gas limit: $GAS_LIM  (estimate * (100+${GAS_BUMP_PCT})/100; set GAS_LIM= to override, GAS_BUMP_PCT= to retune headroom)"
+else
+  if ! [[ "$GAS_LIM" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: GAS_LIM must be a non-negative integer, got: $GAS_LIM" >&2
+    exit 1
+  fi
+  echo "using GAS_LIM=$GAS_LIM (overrides headroom from estimate $gas)"
+fi
+
+# ---- EIP-1559 fee hint from RPC (cast send defaults are unsafe on this chain) ----
+base_fee_per_gas="${BASE_FEE_PER_GAS:-$(cast block latest --field baseFeePerGas -r "$RPC_2103" 2>/dev/null || echo 0)}"
+sugg_wei="${GAS_SUGGEST_WEI:-$(cast gas-price -r "$RPC_2103" 2>/dev/null || echo 0)}"
+[[ "$base_fee_per_gas" =~ ^[0-9]+$ ]] || base_fee_per_gas=0
+if ! [[ "$sugg_wei" =~ ^[0-9]+$ && "$sugg_wei" -ge 1 ]]; then
+  sugg_wei=1000
+  echo "WARN: could not read eth_gasPrice; using sugg_wei=$sugg_wei" >&2
+fi
+# Max fee: at least 1.2x suggested; override in wei with GAS_MAX_FEE_PER_GAS
+if [[ -n "${GAS_MAX_FEE_PER_GAS:-}" ]]; then
+  if ! [[ "$GAS_MAX_FEE_PER_GAS" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: GAS_MAX_FEE_PER_GAS must be integer wei, got: $GAS_MAX_FEE_PER_GAS" >&2
+    exit 1
+  fi
+  max_fee_per_gas="$GAS_MAX_FEE_PER_GAS"
+else
+  max_fee_per_gas=$(((sugg_wei * 12 + 9) / 10))
+fi
+# Priority: (sugg - base), min 1; override with GAS_PRIORITY_FEE_PER_GAS
+if [[ -n "${GAS_PRIORITY_FEE_PER_GAS:-}" ]]; then
+  if ! [[ "$GAS_PRIORITY_FEE_PER_GAS" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: GAS_PRIORITY_FEE_PER_GAS must be integer wei, got: $GAS_PRIORITY_FEE_PER_GAS" >&2
+    exit 1
+  fi
+  priority_fee_per_gas="$GAS_PRIORITY_FEE_PER_GAS"
+else
+  t=$((sugg_wei - base_fee_per_gas))
+  if [[ "$t" -lt 1 ]]; then t=1; fi
+  priority_fee_per_gas=$t
+fi
+need_max=$((base_fee_per_gas + priority_fee_per_gas))
+if [[ "$max_fee_per_gas" -lt "$need_max" ]]; then
+  max_fee_per_gas=$(((need_max * 12 + 9) / 10))
+fi
+echo "EIP-1559 (from RPC):  baseFeePerGas=$base_fee_per_gas wei  eth_gasPrice(sugg)=$sugg_wei wei"
+echo "  would send with:  --gas-price $max_fee_per_gas  (maxFeePerGas wei)  --priority-gas-price $priority_fee_per_gas  (priority wei)"
+echo "  (set GAS_MAX_FEE_PER_GAS / GAS_PRIORITY_FEE_PER_GAS in wei to override; see header comment)"
+
+if [[ -n "$PRED" && "$PRED" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+  c=$(cast code -r "$RPC_2103" "$PRED" 2>/dev/null || true)
+  if [[ -n "$c" && "$c" != "0x" && "$c" != "0x0" ]]; then
+    clen=$(((${#c} - 2) / 2))
+    echo "getCode($PRED) on 2103: $clen byte(s)  ${c:0:22}…"
+  else
+    echo "getCode($PRED) on 2103: empty (not yet deployed; a send would create runtime if pool/gas allow)"
+  fi
+fi
+
+nonce_l=$(cast nonce -r "$RPC_2103" --block latest "$FROM_ADDR" 2>&1) || true
+nonce_p=$(cast nonce -r "$RPC_2103" --block pending "$FROM_ADDR" 2>&1) || true
+echo "nonce (latest):  $nonce_l"
+echo "nonce (pending): $nonce_p"
+if [[ ! "$nonce_l" =~ ^[0-9]+$ || ! "$nonce_p" =~ ^[0-9]+$ ]]; then
+  echo "WARN: could not parse both nonces (RPC error?); for --send set CAST_NONCE= or fix RPC" >&2
+else
+  if [[ "$nonce_l" -lt "$nonce_p" ]]; then
+    echo "NOTE: mempool in-flight  (latest=$nonce_l < pending=$nonce_p) — a **new** first tx reuses/replaces"
+    echo "  nonce $nonce_l with higher maxFee+priority, not nonce $nonce_p. Default --send will use $nonce_l unless CAST_NONCE= is set."
+  else
+    echo "next send nonce: $nonce_p  (no gap between latest and pending; passed to cast send as --nonce)"
+  fi
+fi
+
+echo ""
+if [[ "$SEND" -eq 1 ]]; then
+  if ! [[ "$nonce_p" =~ ^[0-9]+$ && "$nonce_l" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: could not read both nonces (latest+pending) for $FROM_ADDR" >&2
+    exit 1
+  fi
+  if [[ -n "${CAST_NONCE:-}" ]]; then
+    if [[ ! "$CAST_NONCE" =~ ^[0-9]+$ ]]; then
+      echo "ERROR: CAST_NONCE must be a non-negative integer, got: $CAST_NONCE" >&2
+      exit 1
+    fi
+    NONCE_USE="$CAST_NONCE"
+    echo "using CAST_NONCE=$NONCE_USE"
+  elif [[ "$nonce_p" -gt "$nonce_l" ]]; then
+    NONCE_USE="$nonce_l"
+    echo "using --nonce $NONCE_USE  (mempool: replace / bump same in-flight; pending counter was $nonce_p)"
+  else
+    NONCE_USE="$nonce_p"
+    echo "using --nonce $NONCE_USE  (no extra in-flight per latest vs pending)"
+  fi
+  echo "Broadcasting:  --gas-price $max_fee_per_gas  --priority-gas-price $priority_fee_per_gas  --nonce $NONCE_USE  --gas-limit $GAS_LIM"
+  export ETH_TIMEOUT="${ETH_TIMEOUT:-600}"
+  send_args=(cast send -r "$RPC_2103" --private-key "$PRIVATE_KEY" --nonce "$NONCE_USE" --gas-limit "$GAS_LIM"
+    --gas-price "$max_fee_per_gas" --priority-gas-price "$priority_fee_per_gas" "$SINGLETON" "$DATA")
+  if [[ "${CAST_ASYNC:-0}" == "1" ]]; then
+    send_args+=(--async)
+    echo "ETH_TIMEOUT=$ETH_TIMEOUT  (CAST_ASYNC=1, no wait for receipt)"
+  else
+    echo "ETH_TIMEOUT=$ETH_TIMEOUT  (receipt wait; set CAST_ASYNC=1 for hash only)"
+  fi
+  if ! send_out=$("${send_args[@]}" 2>&1); then
+    echo "cast send failed:" >&2
+    echo "$send_out" >&2
+    exit 1
+  fi
+  echo "$send_out"
+  if [[ -n "$PRED" && "$PRED" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+    echo ""
+    echo "=== post-deploy (predicted address + admin hint; not the signer) ==="
+    if [[ -x "${SCRIPT_DIR}/verify-singleton-post-deploy-2103.sh" ]]; then
+      export RPC_URL_2103="$RPC_2103"
+      export DEPLOY_TX_SIGNER="$FROM_ADDR"
+      "${SCRIPT_DIR}/verify-singleton-post-deploy-2103.sh" "$PRED" || true
+    fi
+  fi
+  echo ""
+  echo "If the tx is still pending, re-run:  DEPLOY_TX_SIGNER=$FROM_ADDR ${SCRIPT_DIR}/verify-singleton-post-deploy-2103.sh $PRED"
+  exit 0
+else
+  echo "Dry-run complete. No transaction sent."
+  if [[ -z "${SKIP_2103_POOL_SSH_CHECK:-}" ]]; then
+    echo "Pool (live, optional):  ${PROJECT_ROOT}/scripts/verify/verify-2103-besu-txpool-config.sh"
+  fi
+  echo "  (or: ${PROJECT_ROOT}/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh --value 1  if the live LXC has 0)"
+  echo "To broadcast after review:  CONFIRM_BROADCAST=1 $0 --send"
+  exit 0
+fi
--- a/scripts/verify/verify-2103-besu-txpool-config.sh
+++ b/scripts/verify/verify-2103-besu-txpool-config.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Read-only: verify VM 2103 Besu config does not set tx-pool-max-future-by-sender=0
+# (0 is unsafe: SparseTransactions / IndexOutOfBounds on block import; use >= 1).
+# Requires LAN SSH to Proxmox host for VM 2103 (r630-01 by default).
+#
+# Usage: ./scripts/verify/verify-2103-besu-txpool-config.sh
+# Exit: 0 if value is >=1; 1 if 0; 2 if line missing; 3 SSH/unreachable
+# Set VERIFICATION_SKIP_2103_SSH=1 to print SKIP and exit 0 (e.g. CI with no Proxmox SSH)
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck disable=SC1091
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+HOST="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
+VMID=2103
+CFG="/etc/besu/config-rpc.toml"
+SSH="root@${HOST}"
+
+if ! ssh -o ConnectTimeout=8 -o StrictHostKeyChecking=no "$SSH" "pct list 2>/dev/null | grep -q '$VMID'"; then
+  if [[ "${VERIFICATION_SKIP_2103_SSH:-0}" == "1" ]]; then
+    echo "SKIP: no SSH/CT $VMID (VERIFICATION_SKIP_2103_SSH=1)"
+    exit 0
+  fi
+  echo "FAIL: cannot reach $SSH or no CT $VMID (set VERIFICATION_SKIP_2103_SSH=1 to skip, or use LAN/operator host)" >&2
+  exit 3
+fi
+
+line="$(
+  ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH" \
+    "pct exec $VMID -- grep -E '^tx-pool-max-future-by-sender=' $CFG 2>/dev/null | head -1" || true
+)"
+if [[ -z "$line" ]]; then
+  echo "FAIL: no tx-pool-max-future-by-sender= in $CFG on 2103" >&2
+  exit 2
+fi
+echo "2103 $CFG: $line"
+
+val="${line#*=}"
+if ! [[ "$val" =~ ^[0-9]+$ ]]; then
+  echo "FAIL: unparseable value: $val" >&2
+  exit 2
+fi
+if [[ "$val" -le 0 ]]; then
+  echo "FAIL: value must be >=1 (0 breaks Besu layered pool). Run:" >&2
+  echo "  ${PROJECT_ROOT}/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh --value 1" >&2
+  exit 1
+fi
+echo "OK: tx-pool-max-future-by-sender=$val (not 0)"
+exit 0
--- a/scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh
+++ b/scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# On-chain check: thirdweb deployer 0xB2dE... can use EIP-2470 singleton
+# 0x4e59b448... on Chain 138 (deploy(bytes) + minimal initcode; eth_call + gas estimate).
+# Does not send a real tx (use cast send with PRIVATE_KEY only if you need a live deploy).
+# Usage: ./scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh [--rpc URL]
+# Default RPC: Thirdweb public 138, then 2103 if set in env.
+#
+# Ref: docs/04-configuration/RPC_ENDPOINTS_MASTER.md (Thirdweb / CREATE2 path)
+# Re-broadcast / large calldata dry-run: scripts/verify/dry-run-thirdweb-singleton-2103.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+# Canonical addresses (see EIP-2470, MULTI_CHAIN_EXECUTION_DETERMINISTIC_DEPLOYMENT.md)
+SINGLETON=0x4e59b44847b379578588920cA78FbF26c0B4956C
+DEPLOYER=0xB2dEA0e264ddfFf91057A3415112e57A1a5Eac14
+# Minimal create bytecode that deploys empty runtime (standard test pattern for factories)
+TEST_INIT=0x600a600c6000396000f3fe
+CHAIN_WANT=138
+RPC_2103="http://192.168.11.217:8545"
+RPC_PUBLIC="https://138.rpc.thirdweb.com"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rpc) RPC_OVERRIDE="$2"; shift 2 ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+if [[ -n "${RPC_OVERRIDE:-}" ]]; then
+  RPCS=("$RPC_OVERRIDE")
+elif [[ -n "${TEST_RPC_138:-}" ]]; then
+  RPCS=("$TEST_RPC_138")
+else
+  RPCS=("$RPC_PUBLIC" "$RPC_2103")
+fi
+
+ok() { echo "[OK] $*"; }
+fail() { echo "[FAIL] $*" >&2; exit 1; }
+
+any_ok=0
+for RPC in "${RPCS[@]}"; do
+  echo "━━━━━━━━ RPC: $RPC"
+  if ! out=$(cast chain-id -r "$RPC" 2>&1); then
+    echo "[SKIP] unreachable: $out"
+    continue
+  fi
+  cid="$out"
+  if [[ "$cid" != "$CHAIN_WANT" ]]; then
+    fail "chainId $cid (expected $CHAIN_WANT)"
+  fi
+  ok "chainId = $cid"
+
+  code=$(cast code "$SINGLETON" -r "$RPC" 2>/dev/null || true)
+  [[ -n "$code" && "$code" != "0x" && ${#code} -gt 4 ]] || fail "no runtime code at singleton $SINGLETON"
+  ok "singleton has code (len $(((${#code}-2)/2)) bytes)"
+
+  calldata=$(cast calldata "deploy(bytes)" "$TEST_INIT")
+  sel=${calldata:0:10}
+  if [[ "$sel" != "0x00774360" ]]; then
+    echo "[WARN] deploy(bytes) selector is $sel (expected 0x00774360); continuing anyway" >&2
+  fi
+
+  if ! result=$(cast call -r "$RPC" --from "$DEPLOYER" "$SINGLETON" "$calldata" 2>&1); then
+    fail "eth_call deploy(bytes) failed: $result"
+  fi
+  if [[ ${#result} -eq 42 && "$result" == 0x* ]]; then
+    addr="$result"
+  else
+    addr=0x${result: -40}
+  fi
+  if [[ ! "$addr" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+    fail "unexpected eth_call return: $result"
+  fi
+  ok "eth_call (from $DEPLOYER) returns address $addr (CREATE2 precompute; not mined until a tx is sent)"
+
+  if ! gas=$(cast estimate -r "$RPC" --from "$DEPLOYER" "$SINGLETON" "$calldata" 2>&1); then
+    fail "eth_estimateGas failed: $gas"
+  fi
+  ok "eth_estimateGas = $gas"
+
+  bal=$(cast balance "$DEPLOYER" -r "$RPC" -e 2>/dev/null || echo "?")
+  echo "    deployer balance (est. ETH): $bal (needs >= gas*price to broadcast a real tx)"
+  echo ""
+  any_ok=1
+done
+
+if [[ "$any_ok" -eq 0 ]]; then
+  fail "no RPC succeeded (check --rpc or network)"
+fi
+
+echo "All reached RPCs passed. No transaction was sent."
+echo "To broadcast a minimal test deploy (optional, uses same calldata as this check):"
+echo "  calldata=\$(cast calldata 'deploy(bytes)' $TEST_INIT)"
+echo "  cast send -r $RPC_2103 --private-key \"\$PRIVATE_KEY\" $SINGLETON \"\$calldata\""
--- a/scripts/verify/verify-rpc-2101-approve-and-sync.sh
+++ b/scripts/verify/verify-rpc-2101-approve-and-sync.sh
@@ -8,12 +8,12 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
 source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true

 RPC_CORE_1="${RPC_CORE_1:-192.168.11.211}"
 RPC_URL="${RPC_URL_138:-http://${RPC_CORE_1}:8545}"
-PROXMOX_USER="${PROXMOX_USER:-root}"
+PROXMOX_SSH_USER="${PROXMOX_SSH_USER:-root}"
 PROXMOX_R630="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
-PROXMOX_ML110="${PROXMOX_ML110:-${PROXMOX_HOST_ML110:-192.168.11.10}}"

 # Five validator IPs (1000-1004)
 VALIDATOR_IPS=(192.168.11.100 192.168.11.101 192.168.11.102 192.168.11.103 192.168.11.104)
@@ -101,13 +101,29 @@ else
  ((FAIL++)) || true
 fi

-# 5. Optional: validator service status (requires SSH to r630-01 and ml110)
-log_info "4. Validator status (5/5 active) — requires SSH to r630-01 and ml110..."
+validator_host() {
+  local vmid="$1"
+  if type get_host_for_vmid >/dev/null 2>&1; then
+    get_host_for_vmid "$vmid"
+  elif [[ "$vmid" -le 1002 ]]; then
+    echo "$PROXMOX_R630"
+  else
+    echo "${PROXMOX_HOST_ML110:-192.168.11.10}"
+  fi
+}
+
+# 5. Optional: validator service status (requires SSH to validator hosts)
+log_info "4. Validator status (5/5 active) — requires SSH to validator hosts..."
 ACTIVE=0
 SSH_OK=false
-for entry in "1000:$PROXMOX_R630" "1001:$PROXMOX_R630" "1002:$PROXMOX_R630" "1003:$PROXMOX_ML110" "1004:$PROXMOX_ML110"; do
+for entry in \
+  "1000:$(validator_host 1000)" \
+  "1001:$(validator_host 1001)" \
+  "1002:$(validator_host 1002)" \
+  "1003:$(validator_host 1003)" \
+  "1004:$(validator_host 1004)"; do
  IFS=':' read -r VMID HOST <<< "$entry"
-  STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${HOST}" \
+  STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_SSH_USER}@${HOST}" \
    "pct exec $VMID -- systemctl is-active besu-validator 2>/dev/null" 2>/dev/null || echo "unknown")
  if [[ "$STATUS" = "active" ]]; then
    ((ACTIVE++)) || true
--- a/scripts/verify/verify-singleton-post-deploy-2103.sh
+++ b/scripts/verify/verify-singleton-post-deploy-2103.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# After a singleton (EIP-2470) deploy tx, verify the **predicted** address — not the signer.
+# 1) Non-empty runtime code on RPC (default: 2103 Thirdweb admin)
+# 2) Optional: owner() if the contract implements it (otherwise note proxy/custom initcode)
+#
+# Usage: RPC_URL_2103=... ./scripts/verify/verify-singleton-post-deploy-2103.sh 0xPredictedAddr
+# Optional: DEPLOY_TX_SIGNER=0x... (display only: gas payer need not be admin/owner in Thirdweb/AA)
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck disable=SC1091
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true
+
+ADDR="${1:-}"
+RPC_2103="${RPC_URL_2103:-http://192.168.11.217:8545}"
+if [[ ! "$ADDR" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+  echo "Usage: $0 0xPredictedContractAddress" >&2
+  exit 2
+fi
+if ! command -v cast &>/dev/null; then
+  echo "ERROR: cast not in PATH" >&2
+  exit 1
+fi
+
+code=$(cast code -r "$RPC_2103" "$ADDR" 2>/dev/null || true)
+if [[ -z "$code" || "$code" == "0x" || "$code" == "0x0" ]]; then
+  echo "FAIL: no runtime code at $ADDR on $RPC_2103 (not deployed or wrong chain/RPC)" >&2
+  exit 1
+fi
+blen=$(((${#code} - 2) / 2))
+echo "OK: code at $ADDR: $blen byte(s) on 2103"
+
+# Optional: compare signer to owner — informational only
+if [[ -n "${DEPLOY_TX_SIGNER:-}" && "$DEPLOY_TX_SIGNER" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+  echo "    (tx signer: $DEPLOY_TX_SIGNER — not necessarily the contract admin/owner in Thirdweb/AA initcode.)"
+fi
+
+# Prefer typed return; fall back to plain "owner()"
+o1="$(cast call -r "$RPC_2103" "$ADDR" "owner()(address)" 2>/dev/null | tr -d ' \n\r' || true)"
+o2="$(cast call -r "$RPC_2103" "$ADDR" "owner()" 2>/dev/null | tr -d ' \n\r' || true)"
+if [[ -n "$o1" && "$o1" == 0x* && ${#o1} -ge 42 ]]; then
+  own="$o1"
+else
+  own="$o2"
+fi
+if [[ -n "$own" && "$own" == 0x* && ${#own} -ge 42 ]]; then
+  echo "owner() -> $own"
+  echo "    Compare to your intended admin; the deployer/signer and owner() are independent here."
+else
+  echo "owner() n/a (revert) — verify admin via Blockscout, proxy storage, or initcode/roles; do not assume signer == owner"
+fi
+exit 0