From c65b896fade172fd0942e52bb8bab5dc46298c92 Mon Sep 17 00:00:00 2001 From: defiQUG Date: Fri, 24 Apr 2026 10:55:55 -0700 Subject: [PATCH] ops: add chain138 and rpc diagnostic tooling --- .../clear-besu-transaction-pools-complete.sh | 29 +- scripts/clear-rpc-2103-txpool.sh | 54 ++++ scripts/fix-validator-permissioning-toml.sh | 71 +++-- .../generate-mcp-allowlist-from-chain138.sh | 3 +- scripts/inspect-rpc-2201-txpool-pct.sh | 56 ++++ .../apply-2103-thirdweb-strict-tx-pool.sh | 72 +++++ .../fix-block-production-staggered-restart.sh | 23 +- .../make-validator-vmids-writable-via-ssh.sh | 24 +- .../remove-tx-pool-min-score-validators.sh | 25 +- .../decode-singleton-deploy-pending-2103.sh | 121 +++++++++ .../verify/dry-run-thirdweb-singleton-2103.sh | 255 ++++++++++++++++++ .../verify/verify-2103-besu-txpool-config.sh | 50 ++++ ...70-singleton-thirdweb-deployer-chain138.sh | 98 +++++++ .../verify-rpc-2101-approve-and-sync.sh | 28 +- .../verify-singleton-post-deploy-2103.sh | 52 ++++ 15 files changed, 909 insertions(+), 52 deletions(-) create mode 100755 scripts/clear-rpc-2103-txpool.sh create mode 100755 scripts/inspect-rpc-2201-txpool-pct.sh create mode 100755 scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh create mode 100755 scripts/verify/decode-singleton-deploy-pending-2103.sh create mode 100755 scripts/verify/dry-run-thirdweb-singleton-2103.sh create mode 100755 scripts/verify/verify-2103-besu-txpool-config.sh create mode 100755 scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh create mode 100755 scripts/verify/verify-singleton-post-deploy-2103.sh diff --git a/scripts/clear-besu-transaction-pools-complete.sh b/scripts/clear-besu-transaction-pools-complete.sh index f6e8a2d1..3aedd6c4 100755 --- a/scripts/clear-besu-transaction-pools-complete.sh +++ b/scripts/clear-besu-transaction-pools-complete.sh @@ -9,10 +9,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true - - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true # Colors RED='\033[0;31m' @@ -28,21 +25,29 @@ log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; } log_error() { echo -e "${RED}[✗]${NC} $1"; } log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; } -# Proxmox hosts -PROXMOX_HOSTS=("ml110" "r630-01") +host_for_vmid() { + local vmid="$1" + if type get_host_for_vmid >/dev/null 2>&1; then + get_host_for_vmid "$vmid" + elif [ "$vmid" -le 1002 ]; then + echo "${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}" + else + echo "${PROXMOX_HOST_ML110:-192.168.11.10}" + fi +} # RPC nodes (VMID:host) RPC_NODES=( - "2101:ml110" # Core RPC + "2101:$(host_for_vmid 2101)" # Core RPC ) # Validators (VMID:host) VALIDATORS=( - "1000:r630-01" - "1001:r630-01" - "1002:r630-01" - "1003:ml110" - "1004:ml110" + "1000:$(host_for_vmid 1000)" + "1001:$(host_for_vmid 1001)" + "1002:$(host_for_vmid 1002)" + "1003:$(host_for_vmid 1003)" + "1004:$(host_for_vmid 1004)" ) log_section "Clear Transaction Pools - Complete Method" diff --git a/scripts/clear-rpc-2103-txpool.sh b/scripts/clear-rpc-2103-txpool.sh new file mode 100755 index 00000000..336fbdbf --- /dev/null +++ b/scripts/clear-rpc-2103-txpool.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Clear transaction pool on Thirdweb admin core RPC (VMID 2103 — 192.168.11.217). +# Stops Besu, removes local tx-pool data under /data/besu|/var/lib/besu, restarts. +# See also: clear-rpc-2201-txpool.sh, clear-all-transaction-pools.sh +# Usage: ./scripts/clear-rpc-2103-txpool.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true + +PROXMOX_USER="${PROXMOX_USER:-root}" +# 2101/2103 on r630-01 (load-project-env get_host_for_vmid) +HOST="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}" +VMID=2103 +SSH_TARGET="${PROXMOX_USER}@${HOST}" + +echo "=== Clear Thirdweb core RPC (VMID 2103) transaction pool ===" +echo "Host: $HOST LXC: $VMID (http://192.168.11.217:8545)" +echo "" + +if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" "pct list 2>/dev/null | grep -q '2103'"; then + echo "VMID 2103 not found on $HOST. Abort." >&2 + exit 1 +fi + +echo "Stopping Besu (besu-rpc-core or besu-rpc)..." +ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \ + "pct exec $VMID -- systemctl stop besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc.service 2>/dev/null || true" 2>&1 | grep -v "Configuration file" || true +sleep 2 + +echo "Clearing tx pool database..." +ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \ + "pct exec $VMID -- bash -c ' + for d in /data/besu /var/lib/besu; do + [ -d \"\$d\" ] && find \"\$d\" -type d -name \"*pool*\" -exec rm -rf {} \; 2>/dev/null; find \"\$d\" -type f -name \"*transaction*\" -delete 2>/dev/null; find \"\$d\" -type f -name \"*txpool*\" -delete 2>/dev/null; true + done + '" 2>&1 | grep -v "Configuration file" || true +echo "Pool cleared." + +echo "Starting Besu..." +ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \ + "pct exec $VMID -- systemctl start besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc.service 2>/dev/null || true" 2>&1 | grep -v "Configuration file" || true +sleep 3 + +if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" "pct exec $VMID -- systemctl is-active besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl is-active besu-rpc 2>/dev/null" 2>/dev/null | grep -q active; then + echo "Service: active" +else + echo "WARN: Check service status on $HOST: pct exec $VMID -- systemctl status besu-rpc-core besu-rpc" >&2 +fi + +echo "" +echo "Done. wait ~30s, then verify: curl -s -X POST http://192.168.11.217:8545 -H Content-Type:application/json -d '{\"jsonrpc\":\"2.0\",\"method\":\"txpool_besuPendingTransactions\",\"params\":[],\"id\":1}' | head -c 200" diff --git a/scripts/fix-validator-permissioning-toml.sh b/scripts/fix-validator-permissioning-toml.sh index c6ef2ab2..5dac05ed 100755 --- a/scripts/fix-validator-permissioning-toml.sh +++ b/scripts/fix-validator-permissioning-toml.sh @@ -12,6 +12,7 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" cd "$PROJECT_ROOT" [ -f config/ip-addresses.conf ] && source config/ip-addresses.conf 2>/dev/null || true +[ -f scripts/lib/load-project-env.sh ] && source scripts/lib/load-project-env.sh 2>/dev/null || true SOURCE_TOML="$PROJECT_ROOT/config/besu-node-lists/permissions-nodes.toml" SOURCE_STATIC="$PROJECT_ROOT/config/besu-node-lists/static-nodes.json" @@ -25,17 +26,39 @@ if [ ! -f "$SOURCE_STATIC" ]; then fi R630_01="${PROXMOX_HOST_R630_01:-${PROXMOX_R630_01:-192.168.11.11}}" -R630_03="${PROXMOX_HOST_R630_03:-${PROXMOX_R630_03:-192.168.11.13}}" -USER="${PROXMOX_USER:-root}" +SSH_USER="${PROXMOX_SSH_USER:-root}" PERM_PATH="/var/lib/besu/permissions" CONFIG_GLOB="/etc/besu/config-validator.toml" +validator_host() { + local vmid="$1" + if type get_host_for_vmid >/dev/null 2>&1; then + get_host_for_vmid "$vmid" + elif [[ "$vmid" -le 1002 ]]; then + echo "$R630_01" + else + echo "${PROXMOX_HOST_ML110:-192.168.11.10}" + fi +} + +validator_ip() { + local vmid="$1" + case "$vmid" in + 1000) echo "${IP_VALIDATOR_0:-192.168.11.100}" ;; + 1001) echo "${IP_VALIDATOR_1:-192.168.11.101}" ;; + 1002) echo "${IP_VALIDATOR_2:-192.168.11.102}" ;; + 1003) echo "${IP_VALIDATOR_3:-192.168.11.103}" ;; + 1004) echo "${IP_VALIDATOR_4:-192.168.11.104}" ;; + *) return 1 ;; + esac +} + VALIDATORS=( - "1000:$R630_01" - "1001:$R630_01" - "1002:$R630_01" - "1003:$R630_03" - "1004:$R630_03" + "1000:$(validator_host 1000)" + "1001:$(validator_host 1001)" + "1002:$(validator_host 1002)" + "1003:$(validator_host 1003)" + "1004:$(validator_host 1004)" ) RED='\033[0;31m' @@ -53,18 +76,23 @@ echo " Both are essential: static-nodes = bootstrap peers, permissions-nodes = echo "" # Copy both files to each host once -for host in "$R630_01" "$R630_03"; do +for host in $(printf '%s\n' "${VALIDATORS[@]}" | cut -d: -f2 | sort -u); do log_info "Copying static-nodes.json and permissions-nodes.toml to $host" - scp -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SOURCE_STATIC" "$SOURCE_TOML" "$USER@$host:/tmp/" 2>/dev/null || { log_err "scp to $host failed"; exit 1; } + scp -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SOURCE_STATIC" "$SOURCE_TOML" "$SSH_USER@$host:/tmp/" 2>/dev/null || { log_err "scp to $host failed"; exit 1; } log_ok " Copied" done FAILED=0 for entry in "${VALIDATORS[@]}"; do IFS=: read -r vmid host <<< "$entry" + validator_ip="$(validator_ip "$vmid")" || { + log_err " could not determine validator IP for VMID $vmid" + ((FAILED++)) || true + continue + } log_info "VMID $vmid @ $host" - status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct status $vmid 2>/dev/null" | awk '{print $2}' || echo "unknown") + status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct status $vmid 2>/dev/null" | awk '{print $2}' || echo "unknown") if [ "$status" != "running" ]; then log_info " Skip (not running)" continue @@ -72,22 +100,26 @@ for entry in "${VALIDATORS[@]}"; do # Push static-nodes.json to /var/lib/besu/ and permissions-nodes.toml to permissions/ STATIC_PATH="/var/lib/besu/static-nodes.json" - if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct push $vmid /tmp/static-nodes.json ${STATIC_PATH} && pct push $vmid /tmp/permissions-nodes.toml ${PERM_PATH}/permissions-nodes.toml" 2>/dev/null; then + if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct push $vmid /tmp/static-nodes.json ${STATIC_PATH} && pct push $vmid /tmp/permissions-nodes.toml ${PERM_PATH}/permissions-nodes.toml" 2>/dev/null; then log_err " pct push failed" ((FAILED++)) || true continue fi # Point config to TOML (not JSON) and ensure static-nodes-file and permissions path are set - if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct exec $vmid -- bash -c ' + if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct exec $vmid -- bash -c ' for f in /etc/besu/config-validator.toml /config/config-validator.toml; do [ -f \"\$f\" ] || continue sed -i \"s|permissioned-nodes\\.json|permissions-nodes.toml|g\" \"\$f\" sed -i \"s|\"/var/lib/besu/permissions/permissioned-nodes.json\"|\"/var/lib/besu/permissions/permissions-nodes.toml\"|g\" \"\$f\" sed -i \"s|^static-nodes-file=.*|static-nodes-file=\\\"/var/lib/besu/static-nodes.json\\\"|\" \"\$f\" sed -i \"s|^permissions-nodes-config-file=.*|permissions-nodes-config-file=\\\"/var/lib/besu/permissions/permissions-nodes.toml\\\"|\" \"\$f\" + sed -i \"s|^p2p-host=.*|p2p-host=\\\"${validator_ip}\\\"|\" \"\$f\" + sed -i \"s|^sync-mode=.*|sync-mode=\\\"FULL\\\"|\" \"\$f\" grep -q \"static-nodes-file\" \"\$f\" || echo \"static-nodes-file=\\\"/var/lib/besu/static-nodes.json\\\"\" >> \"\$f\" grep -q \"permissions-nodes-config-file\" \"\$f\" || echo \"permissions-nodes-config-file=\\\"/var/lib/besu/permissions/permissions-nodes.toml\\\"\" >> \"\$f\" + grep -q \"^p2p-host=\" \"\$f\" || echo \"p2p-host=\\\"${validator_ip}\\\"\" >> \"\$f\" + grep -q \"^sync-mode=\" \"\$f\" || echo \"sync-mode=\\\"FULL\\\"\" >> \"\$f\" break done '" 2>/dev/null; then @@ -96,9 +128,16 @@ for entry in "${VALIDATORS[@]}"; do continue fi - ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct exec $vmid -- chown besu:besu ${STATIC_PATH} ${PERM_PATH}/permissions-nodes.toml 2>/dev/null || pct exec $vmid -- chown root:root ${STATIC_PATH} ${PERM_PATH}/permissions-nodes.toml" 2>/dev/null || true + ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct exec $vmid -- chown besu:besu ${STATIC_PATH} ${PERM_PATH}/permissions-nodes.toml 2>/dev/null || pct exec $vmid -- chown root:root ${STATIC_PATH} ${PERM_PATH}/permissions-nodes.toml" 2>/dev/null || true - if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct exec $vmid -- systemctl restart besu-validator" 2>/dev/null; then + if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct exec $vmid -- bash -lc ' + timeout 30 systemctl restart besu-validator || { + systemctl kill -s SIGKILL besu-validator || true + sleep 2 + systemctl reset-failed besu-validator || true + systemctl start besu-validator + } + '" 2>/dev/null; then log_err " restart failed" ((FAILED++)) || true continue @@ -108,8 +147,8 @@ for entry in "${VALIDATORS[@]}"; do done # Cleanup host /tmp -for host in "$R630_01" "$R630_03"; do - ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "rm -f /tmp/permissions-nodes.toml /tmp/static-nodes.json" 2>/dev/null || true +for host in $(printf '%s\n' "${VALIDATORS[@]}" | cut -d: -f2 | sort -u); do + ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "rm -f /tmp/permissions-nodes.toml /tmp/static-nodes.json" 2>/dev/null || true done echo "=== Summary ===" diff --git a/scripts/generate-mcp-allowlist-from-chain138.sh b/scripts/generate-mcp-allowlist-from-chain138.sh index e3e02b57..0a48b3a7 100755 --- a/scripts/generate-mcp-allowlist-from-chain138.sh +++ b/scripts/generate-mcp-allowlist-from-chain138.sh @@ -5,7 +5,8 @@ # Usage: # ./scripts/generate-mcp-allowlist-from-chain138.sh # print to stdout # ./scripts/generate-mcp-allowlist-from-chain138.sh -o allowlist.json # write file -# OUT_PATH=ai-mcp-pmm-controller/config/allowlist-138.json ./scripts/generate-mcp-allowlist-from-chain138.sh +# ./scripts/generate-mcp-allowlist-from-chain138.sh -o ai-mcp-pmm-controller/config/allowlist-138.json +# (Output path is only via -o; there is no OUT_PATH env var.) # # Requires: RPC_URL_138 (or RPC_URL), DODO_PMM_INTEGRATION_ADDRESS in env (or .env in smom-dbis-138). # Optional: MAX_POOLS (default 200), PROFILE (default dodo_pmm_v2_like). diff --git a/scripts/inspect-rpc-2201-txpool-pct.sh b/scripts/inspect-rpc-2201-txpool-pct.sh new file mode 100755 index 00000000..fd7ee7d2 --- /dev/null +++ b/scripts/inspect-rpc-2201-txpool-pct.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# From Proxmox host, use pct exec on VMID 2201 to inspect local tx-pool / Besu (Public RPC). +# JSON-RPC from outside may not expose txpool_besuPendingTransactions; this runs checks inside the LXC. +# Usage: ./scripts/inspect-rpc-2201-txpool-pct.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true + +PROXMOX_USER="${PROXMOX_USER:-root}" +HOST="${PROXMOX_R630_02:-${PROXMOX_HOST_R630_02:-192.168.11.12}}" +VMID=2201 +SSH_TARGET="${PROXMOX_USER}@${HOST}" + +echo "=== Inspect Public RPC (VMID 2201) from host via pct ===" +echo "Host: $HOST (http://192.168.11.221:8545)" +echo "" + +if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" "pct list 2>/dev/null | grep -q '$VMID'"; then + echo "VMID $VMID not found on $HOST. Abort." >&2 + exit 1 +fi + +ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" bash -s -- "$VMID" <<'REMOTE' +VMID="${1:?}" +set -e +pct exec "$VMID" -- bash -s <<'INNER' +set -e +echo "--- systemctl ---" +systemctl is-active besu-rpc 2>/dev/null || systemctl is-active besu-rpc.service 2>/dev/null || echo "unknown" +echo "" +echo "--- txpool_besuPendingTransactions (first 2k chars) ---" +OUT=$(curl -sS --connect-timeout 5 -X POST http://127.0.0.1:8545 -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"txpool_besuPendingTransactions","params":[],"id":1}' 2>&1) || true +printf '%s' "$OUT" | head -c 2000; echo; echo "" +echo "--- eth_blockNumber ---" +curl -sS --connect-timeout 5 -X POST http://127.0.0.1:8545 -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' +echo; echo "" +echo "--- du *pool* under Besu data dirs ---" +for d in /data/besu /var/lib/besu; do + [ -d "$d" ] && find "$d" -maxdepth 4 -type d -name '*pool*' 2>/dev/null | while read -r p; do + du -sh "$p" 2>/dev/null || true + done +done +echo "(end)" +echo "" +echo "--- journal (besu, last 12) ---" +journalctl -u besu-rpc -u besu-rpc.service -n 12 --no-pager 2>/dev/null || true +INNER +REMOTE + +echo "" +echo "To purge 2201 pool: ./scripts/clear-rpc-2201-txpool.sh" diff --git a/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh b/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh new file mode 100755 index 00000000..cc01f1c7 --- /dev/null +++ b/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Tighten Besu layered tx-pool on VMID 2103 (Thirdweb admin core) only, so a single +# sender cannot fill the local pool with far-future / gap nonces (e.g. 15 when next is 1). +# +# - Sets tx-pool-max-future-by-sender=1 (do NOT use 0: Besu 24+ can log IndexOutOfBounds in +# SparseTransactions on new blocks when 0 is set; observed 2026-04-22 on 2103). +# - Restarts besu-rpc. Optional: purge local pool after restart (arg --purge) via clear-rpc-2103-txpool. +# +# Usage: ./scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh [--purge] [--value N] +# --value N default 1 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# shellcheck disable=SC1091 +source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true + +VAL="${VAL:-1}" +PURGE=false +while [[ $# -gt 0 ]]; do + case "$1" in + --purge) PURGE=true; shift ;; + --value) VAL="$2"; shift 2 ;; + *) echo "Unknown: $1" >&2; exit 2 ;; + esac +done + +HOST="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}" +VMID=2103 +CFG_PCT="/etc/besu/config-rpc.toml" +SSH="root@${HOST}" + +echo "=== 2103 tx-pool: tx-pool-max-future-by-sender=$VAL (Thirdweb path only) ===" +echo "Host: $HOST LXC: $VMID config: $CFG_PCT (inside LXC)" +echo "" + +if ! ssh -o ConnectTimeout=8 -o StrictHostKeyChecking=no "$SSH" "pct list | grep -q '$VMID'"; then + echo "VMID $VMID not on $HOST" >&2 + exit 1 +fi + +if [[ "$VAL" -le 0 ]]; then + echo "Refusing: value 0 is unsafe on current Besu (SparseTransactions/IndexOutOfBounds). Use 1 or more." >&2 + exit 1 +fi + +ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH" "pct exec $VMID -- bash -c ' +set -e +cp -a $CFG_PCT ${CFG_PCT}.bak.\$(date +%Y%m%d%H%M%S) +if ! grep -qE \"^tx-pool-max-future-by-sender=\" $CFG_PCT; then + echo \"[ERROR] $CFG_PCT has no tx-pool-max-future-by-sender= line; add it manually.\" >&2 + exit 1 +fi +sed -i \"s/^tx-pool-max-future-by-sender=.*/tx-pool-max-future-by-sender=$VAL/\" $CFG_PCT +grep -E \"^tx-pool-max-future-by-sender=\" $CFG_PCT +systemctl restart besu-rpc +sleep 3 +systemctl is-active besu-rpc +'" + +if [[ "$PURGE" == "true" ]]; then + echo "" + echo "=== Purging 2103 local pool ===" + "${PROJECT_ROOT}/scripts/clear-rpc-2103-txpool.sh" +else + echo "Done. Optional: --purge to also run clear-rpc-2103-txpool.sh" +fi + +echo "" +echo "Next: from Thirdweb, use the nonce returned by eth_getTransactionCount(deployer, 'pending'|'latest')." +echo "A send at nonce 15 will be rejected (NONCE_TOO_FAR) until nonces 1,2,… are broadcast with the deployer key outside Thirdweb, if the chain is still behind." diff --git a/scripts/maintenance/fix-block-production-staggered-restart.sh b/scripts/maintenance/fix-block-production-staggered-restart.sh index eab75d78..c94cff06 100755 --- a/scripts/maintenance/fix-block-production-staggered-restart.sh +++ b/scripts/maintenance/fix-block-production-staggered-restart.sh @@ -12,6 +12,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true +[ -f "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" ] && source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true DRY_RUN=false [[ "${1:-}" == "--dry-run" ]] && DRY_RUN=true @@ -25,14 +26,24 @@ log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_ok() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +validator_host() { + local vmid="$1" + if type get_host_for_vmid >/dev/null 2>&1; then + get_host_for_vmid "$vmid" + elif [[ "$vmid" -le 1002 ]]; then + echo "${PROXMOX_HOST_R630_01:-192.168.11.11}" + else + echo "${PROXMOX_HOST_ML110:-192.168.11.10}" + fi +} + # Order: restart one at a time; wait between so restarted node can sync from others -# VMID : host VALIDATORS=( - "1004:${PROXMOX_HOST_R630_03:-192.168.11.13}" - "1003:${PROXMOX_HOST_R630_03:-192.168.11.13}" - "1002:${PROXMOX_HOST_R630_01:-192.168.11.11}" - "1001:${PROXMOX_HOST_R630_01:-192.168.11.11}" - "1000:${PROXMOX_HOST_R630_01:-192.168.11.11}" + "1004:$(validator_host 1004)" + "1003:$(validator_host 1003)" + "1002:$(validator_host 1002)" + "1001:$(validator_host 1001)" + "1000:$(validator_host 1000)" ) WAIT_BETWEEN=90 RPC="${RPC_URL_138:-http://192.168.11.211:8545}" diff --git a/scripts/maintenance/make-validator-vmids-writable-via-ssh.sh b/scripts/maintenance/make-validator-vmids-writable-via-ssh.sh index e8f424e7..5300c311 100755 --- a/scripts/maintenance/make-validator-vmids-writable-via-ssh.sh +++ b/scripts/maintenance/make-validator-vmids-writable-via-ssh.sh @@ -11,18 +11,28 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" [[ -f "${PROJECT_ROOT}/config/ip-addresses.conf" ]] && source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true +[[ -f "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" ]] && source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true R630_01="${PROXMOX_HOST_R630_01:-192.168.11.11}" -ML110="${PROXMOX_ML110:-192.168.11.10}" SSH_OPTS="-o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new" -# Validators: 1000,1001,1002 on r630-01; 1003,1004 on ml110 +validator_host() { + local vmid="$1" + if type get_host_for_vmid >/dev/null 2>&1; then + get_host_for_vmid "$vmid" + elif [[ "$vmid" -le 1002 ]]; then + echo "$R630_01" + else + echo "${PROXMOX_HOST_ML110:-192.168.11.10}" + fi +} + VALIDATORS=( - "1000:$R630_01" - "1001:$R630_01" - "1002:$R630_01" - "1003:$ML110" - "1004:$ML110" + "1000:$(validator_host 1000)" + "1001:$(validator_host 1001)" + "1002:$(validator_host 1002)" + "1003:$(validator_host 1003)" + "1004:$(validator_host 1004)" ) DRY_RUN=false diff --git a/scripts/remove-tx-pool-min-score-validators.sh b/scripts/remove-tx-pool-min-score-validators.sh index 7697e9f8..015691d6 100755 --- a/scripts/remove-tx-pool-min-score-validators.sh +++ b/scripts/remove-tx-pool-min-score-validators.sh @@ -7,18 +7,35 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" [ -f "$PROJECT_ROOT/config/ip-addresses.conf" ] && source "$PROJECT_ROOT/config/ip-addresses.conf" 2>/dev/null || true +[ -f "$PROJECT_ROOT/scripts/lib/load-project-env.sh" ] && source "$PROJECT_ROOT/scripts/lib/load-project-env.sh" 2>/dev/null || true R630_01="${PROXMOX_R630_01:-192.168.11.11}" -ML110="${PROXMOX_ML110:-192.168.11.10}" -USER="${PROXMOX_USER:-root}" +SSH_USER="${PROXMOX_SSH_USER:-root}" -VALIDATORS=( "1000:$R630_01" "1001:$R630_01" "1002:$R630_01" "1003:$ML110" "1004:$ML110" ) +validator_host() { + local vmid="$1" + if type get_host_for_vmid >/dev/null 2>&1; then + get_host_for_vmid "$vmid" + elif [[ "$vmid" -le 1002 ]]; then + echo "$R630_01" + else + echo "${PROXMOX_HOST_ML110:-192.168.11.10}" + fi +} + +VALIDATORS=( + "1000:$(validator_host 1000)" + "1001:$(validator_host 1001)" + "1002:$(validator_host 1002)" + "1003:$(validator_host 1003)" + "1004:$(validator_host 1004)" +) FAILED=0 for entry in "${VALIDATORS[@]}"; do IFS=: read -r vmid host <<< "$entry" echo "[$vmid @ $host] Removing tx-pool-min-score and restarting..." - if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$USER@$host" "pct exec $vmid -- bash -c ' + if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_USER@$host" "pct exec $vmid -- bash -c ' CFG=/etc/besu/config-validator.toml [ -f /config/config-validator.toml ] && CFG=/config/config-validator.toml [ ! -f \"\$CFG\" ] && exit 1 diff --git a/scripts/verify/decode-singleton-deploy-pending-2103.sh b/scripts/verify/decode-singleton-deploy-pending-2103.sh new file mode 100755 index 00000000..f8ee82a4 --- /dev/null +++ b/scripts/verify/decode-singleton-deploy-pending-2103.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# List pending txs on 2103 that target the EIP-2470 singleton and decode deploy(bytes) initcode. +# Usage: ./scripts/verify/decode-singleton-deploy-pending-2103.sh +# Optional: RPC_URL_2103=... PENDING_TX_FROM=0xB2dE... (stuck tx signer; avoids operator DEPLOYER from dotenv) SINGLETON=0x4e59... + +set -euo pipefail +RPC_URL="${RPC_URL_2103:-http://192.168.11.217:8545}" +PENDING_TX_FROM="${PENDING_TX_FROM:-0xB2dEA0e264ddfFf91057A3415112e57A1a5Eac14}" +SINGLETON="${SINGLETON:-0x4e59b44847b379578588920cA78FbF26c0B4956C}" +OUT_DIR="${OUT_DIR:-/tmp/singleton-decodes-$(date +%Y%m%d%H%M%S)}" +export RPC_URL PENDING_TX_FROM SINGLETON OUT_DIR +DEPLOYER="${PENDING_TX_FROM}" +export DEPLOYER +mkdir -p "$OUT_DIR" +echo "RPC: $RPC_URL pending from filter: $PENDING_TX_FROM singleton: $SINGLETON" +echo "Output: $OUT_DIR" +echo "" + +python3 <<'PY' +import json, sys, subprocess, os, urllib.request + +rpc = os.environ["RPC_URL"] +dep = os.environ["DEPLOYER"].lower() +sing = os.environ["SINGLETON"].lower() +out = os.environ["OUT_DIR"] +sel_4 = bytes.fromhex("00774360") + +req = urllib.request.Request( + rpc, + data=json.dumps({"jsonrpc": "2.0", "method": "txpool_besuPendingTransactions", "params": [], "id": 1}).encode(), + headers={"Content-Type": "application/json"}, +) +raw = urllib.request.urlopen(req, timeout=15).read() +d = json.loads(raw.decode()) +if d.get("error"): + print("RPC error:", d["error"]) + sys.exit(1) +txs = d.get("result") or [] +print("Total pending on 2103:", len(txs)) +matched = [] +for t in txs: + to = (t.get("to") or "").lower() + fr = (t.get("from") or "").lower() + if to != sing: + continue + if fr != dep: + continue + matched.append(t) +print("Matching deployer -> singleton:", len(matched)) +if not matched: + print("Nothing to decode. Re-run when those txs are in 2103's pool, or paste a raw tx into this script.") + sys.exit(0) + +for i, t in enumerate(matched): + h = t.get("hash", "unknown") + raw_hex = t.get("input") or "0x" + if not raw_hex.startswith("0x"): + raw_hex = "0x" + raw_hex + inp = raw_hex[2:] + b = bytes.fromhex(inp) + j = b.find(sel_4) + if j >= 0 and j != 0: + print("INFO", h, "deploy(bytes) selector 0x00774360 at byte offset", j, "(not at 0; Thirdweb / padding)") + b = b[j:] + if len(b) < 4 or b[:4] != sel_4: + p = os.path.join(out, f"full_calldata_{i+1}_{h[:10]}.hex") + with open(p, "w") as f: + f.write(raw_hex.rstrip() + "\n") + if j < 0: + print("WARN", h, "no 0x00774360 in this `input` — on-chain `eth_call` may still use full RPC `input` (see dry-run-thirdweb-singleton-2103.sh) ->", p) + else: + print("WARN", h, "not deploy(bytes) 0x00774360; saved for replay ->", p) + continue + if len(b) < 4 + 32 + 32: + print("Input too short for ABI deploy(bytes)", h, "len", len(b)) + continue + off = int.from_bytes(b[4:36], "big") + if 4 + off + 32 > len(b): + print("Bad offset", off, h) + continue + ln = int.from_bytes(b[4 + off : 4 + off + 32], "big") + init = b[4 + off + 32 : 4 + off + 32 + ln] + if len(init) != ln: + print("Length mismatch", h, ln, len(init)) + continue + path = os.path.join(out, f"initcode_{i+1}_{h[:10]}.hex") + with open(path, "w") as f: + f.write("0x" + init.hex()) + print("---", h, "---") + print(" nonce:", t.get("nonce"), " gas:", t.get("gas"), " initcode bytes:", len(init), "->", path) + try: + r = subprocess.run( + ["cast", "keccak", "0x" + init.hex()], + capture_output=True, text=True, timeout=5, check=False, + ) + if r.returncode == 0: + print(" keccak256(initcode):", r.stdout.strip()) + except FileNotFoundError: + pass + # After slice, b is body starting at 0x00774360; full tx `input` for the node is still raw_hex + bl = len(b) + print(" Body from selector (for ABI decode only):", bl, "bytes") + + def _parse_gas(x): + if isinstance(x, int): + return x + s = str(x).strip() + if s.lower().startswith("0x"): + return int(s, 16) + return int(s, 10) + + g = t.get("gas", "0x500000") + try: + gl = _parse_gas(g) + except (ValueError, TypeError): + gl = 5242880 + print(" Example: cast send -r", rpc, "--private-key --gas-limit", gl, sing, raw_hex) + print() +PY + +echo "Done. Files under $OUT_DIR" diff --git a/scripts/verify/dry-run-thirdweb-singleton-2103.sh b/scripts/verify/dry-run-thirdweb-singleton-2103.sh new file mode 100755 index 00000000..7a17f54d --- /dev/null +++ b/scripts/verify/dry-run-thirdweb-singleton-2103.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# Thirdweb / EIP-2470 singleton: simulate full calldata on VM 2103 before any broadcast. +# - eth_call -> CREATE2 precomputed address (20 bytes, left-padded in 32-byte word) +# - cast estimate -> gas (same as eth_estimateGas) +# - getCode on predicted -> should be 0x if not yet deployed +# - getTransactionCount -> for operator nonce when planning a real send +# +# Default mode is DRY-RUN only (no transaction). Use --send to broadcast (requires +# CONFIRM_BROADCAST=1 in the environment in addition to PRIVATE_KEY from dotenv). +# +# Usage (from repo root, operator LAN + load-project-env): +# source scripts/lib/load-project-env.sh +# export INPUT_FILE=/path/to/one-line-0x-hex +# ./scripts/verify/dry-run-thirdweb-singleton-2103.sh +# +# Optional: INPUT_FILE=... RPC_URL_2103=... GAS_LIM= GAS_BUMP_PCT=20 CAST_NONCE= SKIP_2103_POOL_SSH_CHECK=1 +# EIP-1559 (required on 2103): cast send without fees used ~15/1 wei maxFee/priority; node often reports +# eth_gasPrice ≈ 1000 wei. Set explicitly: +# GAS_MAX_FEE_PER_GAS= (wei) GAS_PRIORITY_FEE_PER_GAS= (wei) — or we derive from baseFee + cast gas-price +# Send tuning: ETH_TIMEOUT= (sec, default 600) CAST_ASYNC=1 (return hash, no receipt wait) +# Post-mine: verify-singleton-post-deploy-2103.sh 0xPred +# +# Ref: docs/04-configuration/RPC_ENDPOINTS_MASTER.md (2103 Thirdweb admin core) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# shellcheck disable=SC1091 +source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" + +SEND=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --send) SEND=1; shift ;; + -h | --help) + sed -n '1,30p' "$0" + exit 0 + ;; + *) echo "Unknown arg: $1 (use --help)" >&2; exit 2 ;; + esac +done + +if [[ "$SEND" -eq 1 && "${CONFIRM_BROADCAST:-0}" != "1" ]]; then + echo "ERROR: --send requires CONFIRM_BROADCAST=1. Run this script with no args for a dry-run only; then:" >&2 + echo " CONFIRM_BROADCAST=1 $0 --send" >&2 + exit 1 +fi + +# VM 2103 Thirdweb admin — do not inherit generic dotenv RPC_URL (often 2101 @ .211) +RPC_2103="${RPC_URL_2103:-http://192.168.11.217:8545}" +SINGLETON="${SINGLETON:-0x4e59b44847b379578588920cA78FbF26c0B4956C}" +INPUT_FILE="${INPUT_FILE:-/tmp/thirdweb-factory-input-2103.hex}" + +if [[ -z "${PRIVATE_KEY:-}" ]]; then + echo "ERROR: PRIVATE_KEY not set (e.g. from smom-dbis-138/.env via load-project-env.sh)" >&2 + exit 1 +fi +if ! command -v cast &>/dev/null; then + echo "ERROR: foundry 'cast' not in PATH" >&2 + exit 1 +fi +if [[ ! -f "$INPUT_FILE" ]]; then + echo "ERROR: INPUT_FILE not found: $INPUT_FILE" >&2 + echo " Hint: run scripts/verify/decode-singleton-deploy-pending-2103.sh when pool has" >&2 + echo " Thirdweb deploys, or set INPUT_FILE= to a one-line 0x… file." >&2 + exit 1 +fi + +DATA="$(tr -d ' \n\r' < "$INPUT_FILE")" +if [[ "${#DATA}" -lt 10 || "${DATA:0:2}" != "0x" ]]; then + echo "ERROR: $INPUT_FILE must be a single line 0x-prefixed hex" >&2 + exit 1 +fi + +FROM_ADDR="$(cast wallet address "$PRIVATE_KEY" 2>/dev/null || true)" +if [[ -z "$FROM_ADDR" || "$FROM_ADDR" == "0x0000000000000000000000000000000000000000" ]]; then + echo "ERROR: could not derive address from PRIVATE_KEY" >&2 + exit 1 +fi + +if [[ "$SEND" -eq 1 ]]; then + echo "=== 2103 singleton broadcast (CONFIRM_BROADCAST=1) ===" +else + echo "=== 2103 singleton dry-run (no broadcast) ===" +fi +echo "RPC: $RPC_2103 (set RPC_URL_2103= to override)" +echo "Singleton: $SINGLETON" +echo "Input: $INPUT_FILE (${#DATA} hex chars = $(( ( ${#DATA} - 2) / 2 )) bytes data)" +echo "From: $FROM_ADDR (cast wallet address of PRIVATE_KEY)" +echo "" + +if ! out=$(cast call -r "$RPC_2103" --from "$FROM_ADDR" "$SINGLETON" --data "$DATA" 2>&1); then + echo "eth_call (cast call) failed:" >&2 + echo "$out" >&2 + exit 1 +fi +if [[ ${#out} -ge 66 ]]; then + PRED=0x${out: -40} +elif [[ ${#out} -eq 42 && "$out" == 0x* ]]; then + PRED="$out" +else + PRED="$out" +fi +echo "eth_call result: $out" +if [[ "$PRED" =~ ^0x[0-9a-fA-F]{40}$ ]]; then + echo "Predicted CREATE2 address: $PRED" +else + echo "WARN: could not parse 20-byte address from return (see raw above)" >&2 +fi + +if ! gas=$(cast estimate -r "$RPC_2103" --from "$FROM_ADDR" "$SINGLETON" "$DATA" 2>&1); then + echo "eth_estimateGas (cast estimate) failed: $gas" >&2 + exit 1 +fi +echo "eth_estimateGas: $gas" +if ! [[ "$gas" =~ ^[0-9]+$ ]]; then + echo "ERROR: could not parse gas estimate: $gas" >&2 + exit 1 +fi +GAS_BUMP_PCT="${GAS_BUMP_PCT:-20}" +if ! [[ "$GAS_BUMP_PCT" =~ ^[0-9]+$ ]]; then + echo "ERROR: GAS_BUMP_PCT must be a non-negative integer, got: $GAS_BUMP_PCT" >&2 + exit 1 +fi +if [[ -z "${GAS_LIM:-}" ]]; then + GAS_LIM=$((( gas * (100 + GAS_BUMP_PCT) + 99) / 100)) + echo "suggested gas limit: $GAS_LIM (estimate * (100+${GAS_BUMP_PCT})/100; set GAS_LIM= to override, GAS_BUMP_PCT= to retune headroom)" +else + if ! [[ "$GAS_LIM" =~ ^[0-9]+$ ]]; then + echo "ERROR: GAS_LIM must be a non-negative integer, got: $GAS_LIM" >&2 + exit 1 + fi + echo "using GAS_LIM=$GAS_LIM (overrides headroom from estimate $gas)" +fi + +# ---- EIP-1559 fee hint from RPC (cast send defaults are unsafe on this chain) ---- +base_fee_per_gas="${BASE_FEE_PER_GAS:-$(cast block latest --field baseFeePerGas -r "$RPC_2103" 2>/dev/null || echo 0)}" +sugg_wei="${GAS_SUGGEST_WEI:-$(cast gas-price -r "$RPC_2103" 2>/dev/null || echo 0)}" +[[ "$base_fee_per_gas" =~ ^[0-9]+$ ]] || base_fee_per_gas=0 +if ! [[ "$sugg_wei" =~ ^[0-9]+$ && "$sugg_wei" -ge 1 ]]; then + sugg_wei=1000 + echo "WARN: could not read eth_gasPrice; using sugg_wei=$sugg_wei" >&2 +fi +# Max fee: at least 1.2x suggested; override in wei with GAS_MAX_FEE_PER_GAS +if [[ -n "${GAS_MAX_FEE_PER_GAS:-}" ]]; then + if ! [[ "$GAS_MAX_FEE_PER_GAS" =~ ^[0-9]+$ ]]; then + echo "ERROR: GAS_MAX_FEE_PER_GAS must be integer wei, got: $GAS_MAX_FEE_PER_GAS" >&2 + exit 1 + fi + max_fee_per_gas="$GAS_MAX_FEE_PER_GAS" +else + max_fee_per_gas=$(((sugg_wei * 12 + 9) / 10)) +fi +# Priority: (sugg - base), min 1; override with GAS_PRIORITY_FEE_PER_GAS +if [[ -n "${GAS_PRIORITY_FEE_PER_GAS:-}" ]]; then + if ! [[ "$GAS_PRIORITY_FEE_PER_GAS" =~ ^[0-9]+$ ]]; then + echo "ERROR: GAS_PRIORITY_FEE_PER_GAS must be integer wei, got: $GAS_PRIORITY_FEE_PER_GAS" >&2 + exit 1 + fi + priority_fee_per_gas="$GAS_PRIORITY_FEE_PER_GAS" +else + t=$((sugg_wei - base_fee_per_gas)) + if [[ "$t" -lt 1 ]]; then t=1; fi + priority_fee_per_gas=$t +fi +need_max=$((base_fee_per_gas + priority_fee_per_gas)) +if [[ "$max_fee_per_gas" -lt "$need_max" ]]; then + max_fee_per_gas=$(((need_max * 12 + 9) / 10)) +fi +echo "EIP-1559 (from RPC): baseFeePerGas=$base_fee_per_gas wei eth_gasPrice(sugg)=$sugg_wei wei" +echo " would send with: --gas-price $max_fee_per_gas (maxFeePerGas wei) --priority-gas-price $priority_fee_per_gas (priority wei)" +echo " (set GAS_MAX_FEE_PER_GAS / GAS_PRIORITY_FEE_PER_GAS in wei to override; see header comment)" + +if [[ -n "$PRED" && "$PRED" =~ ^0x[0-9a-fA-F]{40}$ ]]; then + c=$(cast code -r "$RPC_2103" "$PRED" 2>/dev/null || true) + if [[ -n "$c" && "$c" != "0x" && "$c" != "0x0" ]]; then + clen=$(((${#c} - 2) / 2)) + echo "getCode($PRED) on 2103: $clen byte(s) ${c:0:22}…" + else + echo "getCode($PRED) on 2103: empty (not yet deployed; a send would create runtime if pool/gas allow)" + fi +fi + +nonce_l=$(cast nonce -r "$RPC_2103" --block latest "$FROM_ADDR" 2>&1) || true +nonce_p=$(cast nonce -r "$RPC_2103" --block pending "$FROM_ADDR" 2>&1) || true +echo "nonce (latest): $nonce_l" +echo "nonce (pending): $nonce_p" +if [[ ! "$nonce_l" =~ ^[0-9]+$ || ! "$nonce_p" =~ ^[0-9]+$ ]]; then + echo "WARN: could not parse both nonces (RPC error?); for --send set CAST_NONCE= or fix RPC" >&2 +else + if [[ "$nonce_l" -lt "$nonce_p" ]]; then + echo "NOTE: mempool in-flight (latest=$nonce_l < pending=$nonce_p) — a **new** first tx reuses/replaces" + echo " nonce $nonce_l with higher maxFee+priority, not nonce $nonce_p. Default --send will use $nonce_l unless CAST_NONCE= is set." + else + echo "next send nonce: $nonce_p (no gap between latest and pending; passed to cast send as --nonce)" + fi +fi + +echo "" +if [[ "$SEND" -eq 1 ]]; then + if ! [[ "$nonce_p" =~ ^[0-9]+$ && "$nonce_l" =~ ^[0-9]+$ ]]; then + echo "ERROR: could not read both nonces (latest+pending) for $FROM_ADDR" >&2 + exit 1 + fi + if [[ -n "${CAST_NONCE:-}" ]]; then + if [[ ! "$CAST_NONCE" =~ ^[0-9]+$ ]]; then + echo "ERROR: CAST_NONCE must be a non-negative integer, got: $CAST_NONCE" >&2 + exit 1 + fi + NONCE_USE="$CAST_NONCE" + echo "using CAST_NONCE=$NONCE_USE" + elif [[ "$nonce_p" -gt "$nonce_l" ]]; then + NONCE_USE="$nonce_l" + echo "using --nonce $NONCE_USE (mempool: replace / bump same in-flight; pending counter was $nonce_p)" + else + NONCE_USE="$nonce_p" + echo "using --nonce $NONCE_USE (no extra in-flight per latest vs pending)" + fi + echo "Broadcasting: --gas-price $max_fee_per_gas --priority-gas-price $priority_fee_per_gas --nonce $NONCE_USE --gas-limit $GAS_LIM" + export ETH_TIMEOUT="${ETH_TIMEOUT:-600}" + send_args=(cast send -r "$RPC_2103" --private-key "$PRIVATE_KEY" --nonce "$NONCE_USE" --gas-limit "$GAS_LIM" + --gas-price "$max_fee_per_gas" --priority-gas-price "$priority_fee_per_gas" "$SINGLETON" "$DATA") + if [[ "${CAST_ASYNC:-0}" == "1" ]]; then + send_args+=(--async) + echo "ETH_TIMEOUT=$ETH_TIMEOUT (CAST_ASYNC=1, no wait for receipt)" + else + echo "ETH_TIMEOUT=$ETH_TIMEOUT (receipt wait; set CAST_ASYNC=1 for hash only)" + fi + if ! send_out=$("${send_args[@]}" 2>&1); then + echo "cast send failed:" >&2 + echo "$send_out" >&2 + exit 1 + fi + echo "$send_out" + if [[ -n "$PRED" && "$PRED" =~ ^0x[0-9a-fA-F]{40}$ ]]; then + echo "" + echo "=== post-deploy (predicted address + admin hint; not the signer) ===" + if [[ -x "${SCRIPT_DIR}/verify-singleton-post-deploy-2103.sh" ]]; then + export RPC_URL_2103="$RPC_2103" + export DEPLOY_TX_SIGNER="$FROM_ADDR" + "${SCRIPT_DIR}/verify-singleton-post-deploy-2103.sh" "$PRED" || true + fi + fi + echo "" + echo "If the tx is still pending, re-run: DEPLOY_TX_SIGNER=$FROM_ADDR ${SCRIPT_DIR}/verify-singleton-post-deploy-2103.sh $PRED" + exit 0 +else + echo "Dry-run complete. No transaction sent." + if [[ -z "${SKIP_2103_POOL_SSH_CHECK:-}" ]]; then + echo "Pool (live, optional): ${PROJECT_ROOT}/scripts/verify/verify-2103-besu-txpool-config.sh" + fi + echo " (or: ${PROJECT_ROOT}/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh --value 1 if the live LXC has 0)" + echo "To broadcast after review: CONFIRM_BROADCAST=1 $0 --send" + exit 0 +fi diff --git a/scripts/verify/verify-2103-besu-txpool-config.sh b/scripts/verify/verify-2103-besu-txpool-config.sh new file mode 100755 index 00000000..2e22c2c5 --- /dev/null +++ b/scripts/verify/verify-2103-besu-txpool-config.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Read-only: verify VM 2103 Besu config does not set tx-pool-max-future-by-sender=0 +# (0 is unsafe: SparseTransactions / IndexOutOfBounds on block import; use >= 1). +# Requires LAN SSH to Proxmox host for VM 2103 (r630-01 by default). +# +# Usage: ./scripts/verify/verify-2103-besu-txpool-config.sh +# Exit: 0 if value is >=1; 1 if 0; 2 if line missing; 3 SSH/unreachable +# Set VERIFICATION_SKIP_2103_SSH=1 to print SKIP and exit 0 (e.g. CI with no Proxmox SSH) +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# shellcheck disable=SC1091 +source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true + +HOST="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}" +VMID=2103 +CFG="/etc/besu/config-rpc.toml" +SSH="root@${HOST}" + +if ! ssh -o ConnectTimeout=8 -o StrictHostKeyChecking=no "$SSH" "pct list 2>/dev/null | grep -q '$VMID'"; then + if [[ "${VERIFICATION_SKIP_2103_SSH:-0}" == "1" ]]; then + echo "SKIP: no SSH/CT $VMID (VERIFICATION_SKIP_2103_SSH=1)" + exit 0 + fi + echo "FAIL: cannot reach $SSH or no CT $VMID (set VERIFICATION_SKIP_2103_SSH=1 to skip, or use LAN/operator host)" >&2 + exit 3 +fi + +line="$( + ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH" \ + "pct exec $VMID -- grep -E '^tx-pool-max-future-by-sender=' $CFG 2>/dev/null | head -1" || true +)" +if [[ -z "$line" ]]; then + echo "FAIL: no tx-pool-max-future-by-sender= in $CFG on 2103" >&2 + exit 2 +fi +echo "2103 $CFG: $line" + +val="${line#*=}" +if ! [[ "$val" =~ ^[0-9]+$ ]]; then + echo "FAIL: unparseable value: $val" >&2 + exit 2 +fi +if [[ "$val" -le 0 ]]; then + echo "FAIL: value must be >=1 (0 breaks Besu layered pool). Run:" >&2 + echo " ${PROJECT_ROOT}/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh --value 1" >&2 + exit 1 +fi +echo "OK: tx-pool-max-future-by-sender=$val (not 0)" +exit 0 diff --git a/scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh b/scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh new file mode 100755 index 00000000..fbd9ceaa --- /dev/null +++ b/scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# On-chain check: thirdweb deployer 0xB2dE... can use EIP-2470 singleton +# 0x4e59b448... on Chain 138 (deploy(bytes) + minimal initcode; eth_call + gas estimate). +# Does not send a real tx (use cast send with PRIVATE_KEY only if you need a live deploy). +# Usage: ./scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh [--rpc URL] +# Default RPC: Thirdweb public 138, then 2103 if set in env. +# +# Ref: docs/04-configuration/RPC_ENDPOINTS_MASTER.md (Thirdweb / CREATE2 path) +# Re-broadcast / large calldata dry-run: scripts/verify/dry-run-thirdweb-singleton-2103.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true + +# Canonical addresses (see EIP-2470, MULTI_CHAIN_EXECUTION_DETERMINISTIC_DEPLOYMENT.md) +SINGLETON=0x4e59b44847b379578588920cA78FbF26c0B4956C +DEPLOYER=0xB2dEA0e264ddfFf91057A3415112e57A1a5Eac14 +# Minimal create bytecode that deploys empty runtime (standard test pattern for factories) +TEST_INIT=0x600a600c6000396000f3fe +CHAIN_WANT=138 +RPC_2103="http://192.168.11.217:8545" +RPC_PUBLIC="https://138.rpc.thirdweb.com" + +while [[ $# -gt 0 ]]; do + case "$1" in + --rpc) RPC_OVERRIDE="$2"; shift 2 ;; + *) echo "Unknown arg: $1" >&2; exit 2 ;; + esac +done + +if [[ -n "${RPC_OVERRIDE:-}" ]]; then + RPCS=("$RPC_OVERRIDE") +elif [[ -n "${TEST_RPC_138:-}" ]]; then + RPCS=("$TEST_RPC_138") +else + RPCS=("$RPC_PUBLIC" "$RPC_2103") +fi + +ok() { echo "[OK] $*"; } +fail() { echo "[FAIL] $*" >&2; exit 1; } + +any_ok=0 +for RPC in "${RPCS[@]}"; do + echo "━━━━━━━━ RPC: $RPC" + if ! out=$(cast chain-id -r "$RPC" 2>&1); then + echo "[SKIP] unreachable: $out" + continue + fi + cid="$out" + if [[ "$cid" != "$CHAIN_WANT" ]]; then + fail "chainId $cid (expected $CHAIN_WANT)" + fi + ok "chainId = $cid" + + code=$(cast code "$SINGLETON" -r "$RPC" 2>/dev/null || true) + [[ -n "$code" && "$code" != "0x" && ${#code} -gt 4 ]] || fail "no runtime code at singleton $SINGLETON" + ok "singleton has code (len $(((${#code}-2)/2)) bytes)" + + calldata=$(cast calldata "deploy(bytes)" "$TEST_INIT") + sel=${calldata:0:10} + if [[ "$sel" != "0x00774360" ]]; then + echo "[WARN] deploy(bytes) selector is $sel (expected 0x00774360); continuing anyway" >&2 + fi + + if ! result=$(cast call -r "$RPC" --from "$DEPLOYER" "$SINGLETON" "$calldata" 2>&1); then + fail "eth_call deploy(bytes) failed: $result" + fi + if [[ ${#result} -eq 42 && "$result" == 0x* ]]; then + addr="$result" + else + addr=0x${result: -40} + fi + if [[ ! "$addr" =~ ^0x[0-9a-fA-F]{40}$ ]]; then + fail "unexpected eth_call return: $result" + fi + ok "eth_call (from $DEPLOYER) returns address $addr (CREATE2 precompute; not mined until a tx is sent)" + + if ! gas=$(cast estimate -r "$RPC" --from "$DEPLOYER" "$SINGLETON" "$calldata" 2>&1); then + fail "eth_estimateGas failed: $gas" + fi + ok "eth_estimateGas = $gas" + + bal=$(cast balance "$DEPLOYER" -r "$RPC" -e 2>/dev/null || echo "?") + echo " deployer balance (est. ETH): $bal (needs >= gas*price to broadcast a real tx)" + echo "" + any_ok=1 +done + +if [[ "$any_ok" -eq 0 ]]; then + fail "no RPC succeeded (check --rpc or network)" +fi + +echo "All reached RPCs passed. No transaction was sent." +echo "To broadcast a minimal test deploy (optional, uses same calldata as this check):" +echo " calldata=\$(cast calldata 'deploy(bytes)' $TEST_INIT)" +echo " cast send -r $RPC_2103 --private-key \"\$PRIVATE_KEY\" $SINGLETON \"\$calldata\"" diff --git a/scripts/verify/verify-rpc-2101-approve-and-sync.sh b/scripts/verify/verify-rpc-2101-approve-and-sync.sh index 3d25d4b1..c36d631a 100755 --- a/scripts/verify/verify-rpc-2101-approve-and-sync.sh +++ b/scripts/verify/verify-rpc-2101-approve-and-sync.sh @@ -8,12 +8,12 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true +source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true RPC_CORE_1="${RPC_CORE_1:-192.168.11.211}" RPC_URL="${RPC_URL_138:-http://${RPC_CORE_1}:8545}" -PROXMOX_USER="${PROXMOX_USER:-root}" +PROXMOX_SSH_USER="${PROXMOX_SSH_USER:-root}" PROXMOX_R630="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}" -PROXMOX_ML110="${PROXMOX_ML110:-${PROXMOX_HOST_ML110:-192.168.11.10}}" # Five validator IPs (1000-1004) VALIDATOR_IPS=(192.168.11.100 192.168.11.101 192.168.11.102 192.168.11.103 192.168.11.104) @@ -101,13 +101,29 @@ else ((FAIL++)) || true fi -# 5. Optional: validator service status (requires SSH to r630-01 and ml110) -log_info "4. Validator status (5/5 active) — requires SSH to r630-01 and ml110..." +validator_host() { + local vmid="$1" + if type get_host_for_vmid >/dev/null 2>&1; then + get_host_for_vmid "$vmid" + elif [[ "$vmid" -le 1002 ]]; then + echo "$PROXMOX_R630" + else + echo "${PROXMOX_HOST_ML110:-192.168.11.10}" + fi +} + +# 5. Optional: validator service status (requires SSH to validator hosts) +log_info "4. Validator status (5/5 active) — requires SSH to validator hosts..." ACTIVE=0 SSH_OK=false -for entry in "1000:$PROXMOX_R630" "1001:$PROXMOX_R630" "1002:$PROXMOX_R630" "1003:$PROXMOX_ML110" "1004:$PROXMOX_ML110"; do +for entry in \ + "1000:$(validator_host 1000)" \ + "1001:$(validator_host 1001)" \ + "1002:$(validator_host 1002)" \ + "1003:$(validator_host 1003)" \ + "1004:$(validator_host 1004)"; do IFS=':' read -r VMID HOST <<< "$entry" - STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${HOST}" \ + STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_SSH_USER}@${HOST}" \ "pct exec $VMID -- systemctl is-active besu-validator 2>/dev/null" 2>/dev/null || echo "unknown") if [[ "$STATUS" = "active" ]]; then ((ACTIVE++)) || true diff --git a/scripts/verify/verify-singleton-post-deploy-2103.sh b/scripts/verify/verify-singleton-post-deploy-2103.sh new file mode 100755 index 00000000..f3ce8db5 --- /dev/null +++ b/scripts/verify/verify-singleton-post-deploy-2103.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# After a singleton (EIP-2470) deploy tx, verify the **predicted** address — not the signer. +# 1) Non-empty runtime code on RPC (default: 2103 Thirdweb admin) +# 2) Optional: owner() if the contract implements it (otherwise note proxy/custom initcode) +# +# Usage: RPC_URL_2103=... ./scripts/verify/verify-singleton-post-deploy-2103.sh 0xPredictedAddr +# Optional: DEPLOY_TX_SIGNER=0x... (display only: gas payer need not be admin/owner in Thirdweb/AA) +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# shellcheck disable=SC1091 +source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true + +ADDR="${1:-}" +RPC_2103="${RPC_URL_2103:-http://192.168.11.217:8545}" +if [[ ! "$ADDR" =~ ^0x[0-9a-fA-F]{40}$ ]]; then + echo "Usage: $0 0xPredictedContractAddress" >&2 + exit 2 +fi +if ! command -v cast &>/dev/null; then + echo "ERROR: cast not in PATH" >&2 + exit 1 +fi + +code=$(cast code -r "$RPC_2103" "$ADDR" 2>/dev/null || true) +if [[ -z "$code" || "$code" == "0x" || "$code" == "0x0" ]]; then + echo "FAIL: no runtime code at $ADDR on $RPC_2103 (not deployed or wrong chain/RPC)" >&2 + exit 1 +fi +blen=$(((${#code} - 2) / 2)) +echo "OK: code at $ADDR: $blen byte(s) on 2103" + +# Optional: compare signer to owner — informational only +if [[ -n "${DEPLOY_TX_SIGNER:-}" && "$DEPLOY_TX_SIGNER" =~ ^0x[0-9a-fA-F]{40}$ ]]; then + echo " (tx signer: $DEPLOY_TX_SIGNER — not necessarily the contract admin/owner in Thirdweb/AA initcode.)" +fi + +# Prefer typed return; fall back to plain "owner()" +o1="$(cast call -r "$RPC_2103" "$ADDR" "owner()(address)" 2>/dev/null | tr -d ' \n\r' || true)" +o2="$(cast call -r "$RPC_2103" "$ADDR" "owner()" 2>/dev/null | tr -d ' \n\r' || true)" +if [[ -n "$o1" && "$o1" == 0x* && ${#o1} -ge 42 ]]; then + own="$o1" +else + own="$o2" +fi +if [[ -n "$own" && "$own" == 0x* && ${#own} -ge 42 ]]; then + echo "owner() -> $own" + echo " Compare to your intended admin; the deployer/signer and owner() are independent here." +else + echo "owner() n/a (revert) — verify admin via Blockscout, proxy storage, or initcode/roles; do not assume signer == owner" +fi +exit 0