ops: add chain138 and rpc diagnostic tooling

2026-04-24 10:55:55 -07:00
parent 454aeda9d5
commit c65b896fad
15 changed files with 909 additions and 52 deletions
--- a/scripts/verify/decode-singleton-deploy-pending-2103.sh
+++ b/scripts/verify/decode-singleton-deploy-pending-2103.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# List pending txs on 2103 that target the EIP-2470 singleton and decode deploy(bytes) initcode.
+# Usage: ./scripts/verify/decode-singleton-deploy-pending-2103.sh
+# Optional: RPC_URL_2103=...  PENDING_TX_FROM=0xB2dE... (stuck tx signer; avoids operator DEPLOYER from dotenv) SINGLETON=0x4e59...
+
+set -euo pipefail
+RPC_URL="${RPC_URL_2103:-http://192.168.11.217:8545}"
+PENDING_TX_FROM="${PENDING_TX_FROM:-0xB2dEA0e264ddfFf91057A3415112e57A1a5Eac14}"
+SINGLETON="${SINGLETON:-0x4e59b44847b379578588920cA78FbF26c0B4956C}"
+OUT_DIR="${OUT_DIR:-/tmp/singleton-decodes-$(date +%Y%m%d%H%M%S)}"
+export RPC_URL PENDING_TX_FROM SINGLETON OUT_DIR
+DEPLOYER="${PENDING_TX_FROM}"
+export DEPLOYER
+mkdir -p "$OUT_DIR"
+echo "RPC: $RPC_URL  pending from filter: $PENDING_TX_FROM  singleton: $SINGLETON"
+echo "Output: $OUT_DIR"
+echo ""
+
+python3 <<'PY'
+import json, sys, subprocess, os, urllib.request
+
+rpc = os.environ["RPC_URL"]
+dep = os.environ["DEPLOYER"].lower()
+sing = os.environ["SINGLETON"].lower()
+out = os.environ["OUT_DIR"]
+sel_4 = bytes.fromhex("00774360")
+
+req = urllib.request.Request(
+    rpc,
+    data=json.dumps({"jsonrpc": "2.0", "method": "txpool_besuPendingTransactions", "params": [], "id": 1}).encode(),
+    headers={"Content-Type": "application/json"},
+)
+raw = urllib.request.urlopen(req, timeout=15).read()
+d = json.loads(raw.decode())
+if d.get("error"):
+    print("RPC error:", d["error"])
+    sys.exit(1)
+txs = d.get("result") or []
+print("Total pending on 2103:", len(txs))
+matched = []
+for t in txs:
+    to = (t.get("to") or "").lower()
+    fr = (t.get("from") or "").lower()
+    if to != sing:
+        continue
+    if fr != dep:
+        continue
+    matched.append(t)
+print("Matching deployer -> singleton:", len(matched))
+if not matched:
+    print("Nothing to decode. Re-run when those txs are in 2103's pool, or paste a raw tx into this script.")
+    sys.exit(0)
+
+for i, t in enumerate(matched):
+    h = t.get("hash", "unknown")
+    raw_hex = t.get("input") or "0x"
+    if not raw_hex.startswith("0x"):
+        raw_hex = "0x" + raw_hex
+    inp = raw_hex[2:]
+    b = bytes.fromhex(inp)
+    j = b.find(sel_4)
+    if j >= 0 and j != 0:
+        print("INFO", h, "deploy(bytes) selector 0x00774360 at byte offset", j, "(not at 0; Thirdweb / padding)")
+        b = b[j:]
+    if len(b) < 4 or b[:4] != sel_4:
+        p = os.path.join(out, f"full_calldata_{i+1}_{h[:10]}.hex")
+        with open(p, "w") as f:
+            f.write(raw_hex.rstrip() + "\n")
+        if j < 0:
+            print("WARN", h, "no 0x00774360 in this `input` — on-chain `eth_call` may still use full RPC `input` (see dry-run-thirdweb-singleton-2103.sh) ->", p)
+        else:
+            print("WARN", h, "not deploy(bytes) 0x00774360; saved for replay ->", p)
+        continue
+    if len(b) < 4 + 32 + 32:
+        print("Input too short for ABI deploy(bytes)", h, "len", len(b))
+        continue
+    off = int.from_bytes(b[4:36], "big")
+    if 4 + off + 32 > len(b):
+        print("Bad offset", off, h)
+        continue
+    ln = int.from_bytes(b[4 + off : 4 + off + 32], "big")
+    init = b[4 + off + 32 : 4 + off + 32 + ln]
+    if len(init) != ln:
+        print("Length mismatch", h, ln, len(init))
+        continue
+    path = os.path.join(out, f"initcode_{i+1}_{h[:10]}.hex")
+    with open(path, "w") as f:
+        f.write("0x" + init.hex())
+    print("---", h, "---")
+    print("  nonce:", t.get("nonce"), " gas:", t.get("gas"), " initcode bytes:", len(init), "->", path)
+    try:
+        r = subprocess.run(
+            ["cast", "keccak", "0x" + init.hex()],
+            capture_output=True, text=True, timeout=5, check=False,
+        )
+        if r.returncode == 0:
+            print("  keccak256(initcode):", r.stdout.strip())
+    except FileNotFoundError:
+        pass
+    # After slice, b is body starting at 0x00774360; full tx `input` for the node is still raw_hex
+    bl = len(b)
+    print("  Body from selector (for ABI decode only):", bl, "bytes")
+
+    def _parse_gas(x):
+        if isinstance(x, int):
+            return x
+        s = str(x).strip()
+        if s.lower().startswith("0x"):
+            return int(s, 16)
+        return int(s, 10)
+
+    g = t.get("gas", "0x500000")
+    try:
+        gl = _parse_gas(g)
+    except (ValueError, TypeError):
+        gl = 5242880
+    print("  Example: cast send -r", rpc, "--private-key <KEY> --gas-limit", gl, sing, raw_hex)
+    print()
+PY
+
+echo "Done. Files under $OUT_DIR"
--- a/scripts/verify/dry-run-thirdweb-singleton-2103.sh
+++ b/scripts/verify/dry-run-thirdweb-singleton-2103.sh
@@ -0,0 +1,255 @@
+#!/usr/bin/env bash
+# Thirdweb / EIP-2470 singleton: simulate full calldata on VM 2103 before any broadcast.
+# - eth_call  -> CREATE2 precomputed address (20 bytes, left-padded in 32-byte word)
+# - cast estimate  -> gas (same as eth_estimateGas)
+# - getCode on predicted  -> should be 0x if not yet deployed
+# - getTransactionCount  -> for operator nonce when planning a real send
+#
+# Default mode is DRY-RUN only (no transaction). Use --send to broadcast (requires
+# CONFIRM_BROADCAST=1 in the environment in addition to PRIVATE_KEY from dotenv).
+#
+# Usage (from repo root, operator LAN + load-project-env):
+#   source scripts/lib/load-project-env.sh
+#   export INPUT_FILE=/path/to/one-line-0x-hex
+#   ./scripts/verify/dry-run-thirdweb-singleton-2103.sh
+#
+# Optional: INPUT_FILE=...  RPC_URL_2103=...  GAS_LIM=  GAS_BUMP_PCT=20  CAST_NONCE=  SKIP_2103_POOL_SSH_CHECK=1
+# EIP-1559 (required on 2103):  cast send without fees used ~15/1 wei maxFee/priority; node often reports
+#   eth_gasPrice ≈ 1000 wei. Set explicitly:
+#   GAS_MAX_FEE_PER_GAS= (wei)  GAS_PRIORITY_FEE_PER_GAS= (wei)  — or we derive from baseFee + cast gas-price
+# Send tuning:  ETH_TIMEOUT= (sec, default 600)  CAST_ASYNC=1  (return hash, no receipt wait)
+# Post-mine:  verify-singleton-post-deploy-2103.sh 0xPred
+#
+# Ref: docs/04-configuration/RPC_ENDPOINTS_MASTER.md (2103 Thirdweb admin core)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck disable=SC1091
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+
+SEND=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+  --send) SEND=1; shift ;;
+  -h | --help)
+    sed -n '1,30p' "$0"
+    exit 0
+    ;;
+  *) echo "Unknown arg: $1 (use --help)" >&2; exit 2 ;;
+  esac
+done
+
+if [[ "$SEND" -eq 1 && "${CONFIRM_BROADCAST:-0}" != "1" ]]; then
+  echo "ERROR: --send requires CONFIRM_BROADCAST=1. Run this script with no args for a dry-run only; then:" >&2
+  echo "  CONFIRM_BROADCAST=1 $0 --send" >&2
+  exit 1
+fi
+
+# VM 2103 Thirdweb admin — do not inherit generic dotenv RPC_URL (often 2101 @ .211)
+RPC_2103="${RPC_URL_2103:-http://192.168.11.217:8545}"
+SINGLETON="${SINGLETON:-0x4e59b44847b379578588920cA78FbF26c0B4956C}"
+INPUT_FILE="${INPUT_FILE:-/tmp/thirdweb-factory-input-2103.hex}"
+
+if [[ -z "${PRIVATE_KEY:-}" ]]; then
+  echo "ERROR: PRIVATE_KEY not set (e.g. from smom-dbis-138/.env via load-project-env.sh)" >&2
+  exit 1
+fi
+if ! command -v cast &>/dev/null; then
+  echo "ERROR: foundry 'cast' not in PATH" >&2
+  exit 1
+fi
+if [[ ! -f "$INPUT_FILE" ]]; then
+  echo "ERROR: INPUT_FILE not found: $INPUT_FILE" >&2
+  echo "  Hint: run scripts/verify/decode-singleton-deploy-pending-2103.sh when pool has" >&2
+  echo "  Thirdweb deploys, or set INPUT_FILE= to a one-line 0x… file." >&2
+  exit 1
+fi
+
+DATA="$(tr -d ' \n\r' < "$INPUT_FILE")"
+if [[ "${#DATA}" -lt 10 || "${DATA:0:2}" != "0x" ]]; then
+  echo "ERROR: $INPUT_FILE must be a single line 0x-prefixed hex" >&2
+  exit 1
+fi
+
+FROM_ADDR="$(cast wallet address "$PRIVATE_KEY" 2>/dev/null || true)"
+if [[ -z "$FROM_ADDR" || "$FROM_ADDR" == "0x0000000000000000000000000000000000000000" ]]; then
+  echo "ERROR: could not derive address from PRIVATE_KEY" >&2
+  exit 1
+fi
+
+if [[ "$SEND" -eq 1 ]]; then
+  echo "=== 2103 singleton broadcast (CONFIRM_BROADCAST=1) ==="
+else
+  echo "=== 2103 singleton dry-run (no broadcast) ==="
+fi
+echo "RPC:       $RPC_2103  (set RPC_URL_2103= to override)"
+echo "Singleton: $SINGLETON"
+echo "Input:     $INPUT_FILE  (${#DATA} hex chars = $(( ( ${#DATA} - 2) / 2 )) bytes data)"
+echo "From:      $FROM_ADDR  (cast wallet address of PRIVATE_KEY)"
+echo ""
+
+if ! out=$(cast call -r "$RPC_2103" --from "$FROM_ADDR" "$SINGLETON" --data "$DATA" 2>&1); then
+  echo "eth_call (cast call) failed:" >&2
+  echo "$out" >&2
+  exit 1
+fi
+if [[ ${#out} -ge 66 ]]; then
+  PRED=0x${out: -40}
+elif [[ ${#out} -eq 42 && "$out" == 0x* ]]; then
+  PRED="$out"
+else
+  PRED="$out"
+fi
+echo "eth_call result: $out"
+if [[ "$PRED" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+  echo "Predicted CREATE2 address: $PRED"
+else
+  echo "WARN: could not parse 20-byte address from return (see raw above)" >&2
+fi
+
+if ! gas=$(cast estimate -r "$RPC_2103" --from "$FROM_ADDR" "$SINGLETON" "$DATA" 2>&1); then
+  echo "eth_estimateGas (cast estimate) failed: $gas" >&2
+  exit 1
+fi
+echo "eth_estimateGas:  $gas"
+if ! [[ "$gas" =~ ^[0-9]+$ ]]; then
+  echo "ERROR: could not parse gas estimate: $gas" >&2
+  exit 1
+fi
+GAS_BUMP_PCT="${GAS_BUMP_PCT:-20}"
+if ! [[ "$GAS_BUMP_PCT" =~ ^[0-9]+$ ]]; then
+  echo "ERROR: GAS_BUMP_PCT must be a non-negative integer, got: $GAS_BUMP_PCT" >&2
+  exit 1
+fi
+if [[ -z "${GAS_LIM:-}" ]]; then
+  GAS_LIM=$((( gas * (100 + GAS_BUMP_PCT) + 99) / 100))
+  echo "suggested gas limit: $GAS_LIM  (estimate * (100+${GAS_BUMP_PCT})/100; set GAS_LIM= to override, GAS_BUMP_PCT= to retune headroom)"
+else
+  if ! [[ "$GAS_LIM" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: GAS_LIM must be a non-negative integer, got: $GAS_LIM" >&2
+    exit 1
+  fi
+  echo "using GAS_LIM=$GAS_LIM (overrides headroom from estimate $gas)"
+fi
+
+# ---- EIP-1559 fee hint from RPC (cast send defaults are unsafe on this chain) ----
+base_fee_per_gas="${BASE_FEE_PER_GAS:-$(cast block latest --field baseFeePerGas -r "$RPC_2103" 2>/dev/null || echo 0)}"
+sugg_wei="${GAS_SUGGEST_WEI:-$(cast gas-price -r "$RPC_2103" 2>/dev/null || echo 0)}"
+[[ "$base_fee_per_gas" =~ ^[0-9]+$ ]] || base_fee_per_gas=0
+if ! [[ "$sugg_wei" =~ ^[0-9]+$ && "$sugg_wei" -ge 1 ]]; then
+  sugg_wei=1000
+  echo "WARN: could not read eth_gasPrice; using sugg_wei=$sugg_wei" >&2
+fi
+# Max fee: at least 1.2x suggested; override in wei with GAS_MAX_FEE_PER_GAS
+if [[ -n "${GAS_MAX_FEE_PER_GAS:-}" ]]; then
+  if ! [[ "$GAS_MAX_FEE_PER_GAS" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: GAS_MAX_FEE_PER_GAS must be integer wei, got: $GAS_MAX_FEE_PER_GAS" >&2
+    exit 1
+  fi
+  max_fee_per_gas="$GAS_MAX_FEE_PER_GAS"
+else
+  max_fee_per_gas=$(((sugg_wei * 12 + 9) / 10))
+fi
+# Priority: (sugg - base), min 1; override with GAS_PRIORITY_FEE_PER_GAS
+if [[ -n "${GAS_PRIORITY_FEE_PER_GAS:-}" ]]; then
+  if ! [[ "$GAS_PRIORITY_FEE_PER_GAS" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: GAS_PRIORITY_FEE_PER_GAS must be integer wei, got: $GAS_PRIORITY_FEE_PER_GAS" >&2
+    exit 1
+  fi
+  priority_fee_per_gas="$GAS_PRIORITY_FEE_PER_GAS"
+else
+  t=$((sugg_wei - base_fee_per_gas))
+  if [[ "$t" -lt 1 ]]; then t=1; fi
+  priority_fee_per_gas=$t
+fi
+need_max=$((base_fee_per_gas + priority_fee_per_gas))
+if [[ "$max_fee_per_gas" -lt "$need_max" ]]; then
+  max_fee_per_gas=$(((need_max * 12 + 9) / 10))
+fi
+echo "EIP-1559 (from RPC):  baseFeePerGas=$base_fee_per_gas wei  eth_gasPrice(sugg)=$sugg_wei wei"
+echo "  would send with:  --gas-price $max_fee_per_gas  (maxFeePerGas wei)  --priority-gas-price $priority_fee_per_gas  (priority wei)"
+echo "  (set GAS_MAX_FEE_PER_GAS / GAS_PRIORITY_FEE_PER_GAS in wei to override; see header comment)"
+
+if [[ -n "$PRED" && "$PRED" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+  c=$(cast code -r "$RPC_2103" "$PRED" 2>/dev/null || true)
+  if [[ -n "$c" && "$c" != "0x" && "$c" != "0x0" ]]; then
+    clen=$(((${#c} - 2) / 2))
+    echo "getCode($PRED) on 2103: $clen byte(s)  ${c:0:22}…"
+  else
+    echo "getCode($PRED) on 2103: empty (not yet deployed; a send would create runtime if pool/gas allow)"
+  fi
+fi
+
+nonce_l=$(cast nonce -r "$RPC_2103" --block latest "$FROM_ADDR" 2>&1) || true
+nonce_p=$(cast nonce -r "$RPC_2103" --block pending "$FROM_ADDR" 2>&1) || true
+echo "nonce (latest):  $nonce_l"
+echo "nonce (pending): $nonce_p"
+if [[ ! "$nonce_l" =~ ^[0-9]+$ || ! "$nonce_p" =~ ^[0-9]+$ ]]; then
+  echo "WARN: could not parse both nonces (RPC error?); for --send set CAST_NONCE= or fix RPC" >&2
+else
+  if [[ "$nonce_l" -lt "$nonce_p" ]]; then
+    echo "NOTE: mempool in-flight  (latest=$nonce_l < pending=$nonce_p) — a **new** first tx reuses/replaces"
+    echo "  nonce $nonce_l with higher maxFee+priority, not nonce $nonce_p. Default --send will use $nonce_l unless CAST_NONCE= is set."
+  else
+    echo "next send nonce: $nonce_p  (no gap between latest and pending; passed to cast send as --nonce)"
+  fi
+fi
+
+echo ""
+if [[ "$SEND" -eq 1 ]]; then
+  if ! [[ "$nonce_p" =~ ^[0-9]+$ && "$nonce_l" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: could not read both nonces (latest+pending) for $FROM_ADDR" >&2
+    exit 1
+  fi
+  if [[ -n "${CAST_NONCE:-}" ]]; then
+    if [[ ! "$CAST_NONCE" =~ ^[0-9]+$ ]]; then
+      echo "ERROR: CAST_NONCE must be a non-negative integer, got: $CAST_NONCE" >&2
+      exit 1
+    fi
+    NONCE_USE="$CAST_NONCE"
+    echo "using CAST_NONCE=$NONCE_USE"
+  elif [[ "$nonce_p" -gt "$nonce_l" ]]; then
+    NONCE_USE="$nonce_l"
+    echo "using --nonce $NONCE_USE  (mempool: replace / bump same in-flight; pending counter was $nonce_p)"
+  else
+    NONCE_USE="$nonce_p"
+    echo "using --nonce $NONCE_USE  (no extra in-flight per latest vs pending)"
+  fi
+  echo "Broadcasting:  --gas-price $max_fee_per_gas  --priority-gas-price $priority_fee_per_gas  --nonce $NONCE_USE  --gas-limit $GAS_LIM"
+  export ETH_TIMEOUT="${ETH_TIMEOUT:-600}"
+  send_args=(cast send -r "$RPC_2103" --private-key "$PRIVATE_KEY" --nonce "$NONCE_USE" --gas-limit "$GAS_LIM"
+    --gas-price "$max_fee_per_gas" --priority-gas-price "$priority_fee_per_gas" "$SINGLETON" "$DATA")
+  if [[ "${CAST_ASYNC:-0}" == "1" ]]; then
+    send_args+=(--async)
+    echo "ETH_TIMEOUT=$ETH_TIMEOUT  (CAST_ASYNC=1, no wait for receipt)"
+  else
+    echo "ETH_TIMEOUT=$ETH_TIMEOUT  (receipt wait; set CAST_ASYNC=1 for hash only)"
+  fi
+  if ! send_out=$("${send_args[@]}" 2>&1); then
+    echo "cast send failed:" >&2
+    echo "$send_out" >&2
+    exit 1
+  fi
+  echo "$send_out"
+  if [[ -n "$PRED" && "$PRED" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+    echo ""
+    echo "=== post-deploy (predicted address + admin hint; not the signer) ==="
+    if [[ -x "${SCRIPT_DIR}/verify-singleton-post-deploy-2103.sh" ]]; then
+      export RPC_URL_2103="$RPC_2103"
+      export DEPLOY_TX_SIGNER="$FROM_ADDR"
+      "${SCRIPT_DIR}/verify-singleton-post-deploy-2103.sh" "$PRED" || true
+    fi
+  fi
+  echo ""
+  echo "If the tx is still pending, re-run:  DEPLOY_TX_SIGNER=$FROM_ADDR ${SCRIPT_DIR}/verify-singleton-post-deploy-2103.sh $PRED"
+  exit 0
+else
+  echo "Dry-run complete. No transaction sent."
+  if [[ -z "${SKIP_2103_POOL_SSH_CHECK:-}" ]]; then
+    echo "Pool (live, optional):  ${PROJECT_ROOT}/scripts/verify/verify-2103-besu-txpool-config.sh"
+  fi
+  echo "  (or: ${PROJECT_ROOT}/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh --value 1  if the live LXC has 0)"
+  echo "To broadcast after review:  CONFIRM_BROADCAST=1 $0 --send"
+  exit 0
+fi
--- a/scripts/verify/verify-2103-besu-txpool-config.sh
+++ b/scripts/verify/verify-2103-besu-txpool-config.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Read-only: verify VM 2103 Besu config does not set tx-pool-max-future-by-sender=0
+# (0 is unsafe: SparseTransactions / IndexOutOfBounds on block import; use >= 1).
+# Requires LAN SSH to Proxmox host for VM 2103 (r630-01 by default).
+#
+# Usage: ./scripts/verify/verify-2103-besu-txpool-config.sh
+# Exit: 0 if value is >=1; 1 if 0; 2 if line missing; 3 SSH/unreachable
+# Set VERIFICATION_SKIP_2103_SSH=1 to print SKIP and exit 0 (e.g. CI with no Proxmox SSH)
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck disable=SC1091
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+HOST="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
+VMID=2103
+CFG="/etc/besu/config-rpc.toml"
+SSH="root@${HOST}"
+
+if ! ssh -o ConnectTimeout=8 -o StrictHostKeyChecking=no "$SSH" "pct list 2>/dev/null | grep -q '$VMID'"; then
+  if [[ "${VERIFICATION_SKIP_2103_SSH:-0}" == "1" ]]; then
+    echo "SKIP: no SSH/CT $VMID (VERIFICATION_SKIP_2103_SSH=1)"
+    exit 0
+  fi
+  echo "FAIL: cannot reach $SSH or no CT $VMID (set VERIFICATION_SKIP_2103_SSH=1 to skip, or use LAN/operator host)" >&2
+  exit 3
+fi
+
+line="$(
+  ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH" \
+    "pct exec $VMID -- grep -E '^tx-pool-max-future-by-sender=' $CFG 2>/dev/null | head -1" || true
+)"
+if [[ -z "$line" ]]; then
+  echo "FAIL: no tx-pool-max-future-by-sender= in $CFG on 2103" >&2
+  exit 2
+fi
+echo "2103 $CFG: $line"
+
+val="${line#*=}"
+if ! [[ "$val" =~ ^[0-9]+$ ]]; then
+  echo "FAIL: unparseable value: $val" >&2
+  exit 2
+fi
+if [[ "$val" -le 0 ]]; then
+  echo "FAIL: value must be >=1 (0 breaks Besu layered pool). Run:" >&2
+  echo "  ${PROJECT_ROOT}/scripts/maintenance/apply-2103-thirdweb-strict-tx-pool.sh --value 1" >&2
+  exit 1
+fi
+echo "OK: tx-pool-max-future-by-sender=$val (not 0)"
+exit 0
--- a/scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh
+++ b/scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# On-chain check: thirdweb deployer 0xB2dE... can use EIP-2470 singleton
+# 0x4e59b448... on Chain 138 (deploy(bytes) + minimal initcode; eth_call + gas estimate).
+# Does not send a real tx (use cast send with PRIVATE_KEY only if you need a live deploy).
+# Usage: ./scripts/verify/verify-eip-2470-singleton-thirdweb-deployer-chain138.sh [--rpc URL]
+# Default RPC: Thirdweb public 138, then 2103 if set in env.
+#
+# Ref: docs/04-configuration/RPC_ENDPOINTS_MASTER.md (Thirdweb / CREATE2 path)
+# Re-broadcast / large calldata dry-run: scripts/verify/dry-run-thirdweb-singleton-2103.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+# Canonical addresses (see EIP-2470, MULTI_CHAIN_EXECUTION_DETERMINISTIC_DEPLOYMENT.md)
+SINGLETON=0x4e59b44847b379578588920cA78FbF26c0B4956C
+DEPLOYER=0xB2dEA0e264ddfFf91057A3415112e57A1a5Eac14
+# Minimal create bytecode that deploys empty runtime (standard test pattern for factories)
+TEST_INIT=0x600a600c6000396000f3fe
+CHAIN_WANT=138
+RPC_2103="http://192.168.11.217:8545"
+RPC_PUBLIC="https://138.rpc.thirdweb.com"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rpc) RPC_OVERRIDE="$2"; shift 2 ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+if [[ -n "${RPC_OVERRIDE:-}" ]]; then
+  RPCS=("$RPC_OVERRIDE")
+elif [[ -n "${TEST_RPC_138:-}" ]]; then
+  RPCS=("$TEST_RPC_138")
+else
+  RPCS=("$RPC_PUBLIC" "$RPC_2103")
+fi
+
+ok() { echo "[OK] $*"; }
+fail() { echo "[FAIL] $*" >&2; exit 1; }
+
+any_ok=0
+for RPC in "${RPCS[@]}"; do
+  echo "━━━━━━━━ RPC: $RPC"
+  if ! out=$(cast chain-id -r "$RPC" 2>&1); then
+    echo "[SKIP] unreachable: $out"
+    continue
+  fi
+  cid="$out"
+  if [[ "$cid" != "$CHAIN_WANT" ]]; then
+    fail "chainId $cid (expected $CHAIN_WANT)"
+  fi
+  ok "chainId = $cid"
+
+  code=$(cast code "$SINGLETON" -r "$RPC" 2>/dev/null || true)
+  [[ -n "$code" && "$code" != "0x" && ${#code} -gt 4 ]] || fail "no runtime code at singleton $SINGLETON"
+  ok "singleton has code (len $(((${#code}-2)/2)) bytes)"
+
+  calldata=$(cast calldata "deploy(bytes)" "$TEST_INIT")
+  sel=${calldata:0:10}
+  if [[ "$sel" != "0x00774360" ]]; then
+    echo "[WARN] deploy(bytes) selector is $sel (expected 0x00774360); continuing anyway" >&2
+  fi
+
+  if ! result=$(cast call -r "$RPC" --from "$DEPLOYER" "$SINGLETON" "$calldata" 2>&1); then
+    fail "eth_call deploy(bytes) failed: $result"
+  fi
+  if [[ ${#result} -eq 42 && "$result" == 0x* ]]; then
+    addr="$result"
+  else
+    addr=0x${result: -40}
+  fi
+  if [[ ! "$addr" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+    fail "unexpected eth_call return: $result"
+  fi
+  ok "eth_call (from $DEPLOYER) returns address $addr (CREATE2 precompute; not mined until a tx is sent)"
+
+  if ! gas=$(cast estimate -r "$RPC" --from "$DEPLOYER" "$SINGLETON" "$calldata" 2>&1); then
+    fail "eth_estimateGas failed: $gas"
+  fi
+  ok "eth_estimateGas = $gas"
+
+  bal=$(cast balance "$DEPLOYER" -r "$RPC" -e 2>/dev/null || echo "?")
+  echo "    deployer balance (est. ETH): $bal (needs >= gas*price to broadcast a real tx)"
+  echo ""
+  any_ok=1
+done
+
+if [[ "$any_ok" -eq 0 ]]; then
+  fail "no RPC succeeded (check --rpc or network)"
+fi
+
+echo "All reached RPCs passed. No transaction was sent."
+echo "To broadcast a minimal test deploy (optional, uses same calldata as this check):"
+echo "  calldata=\$(cast calldata 'deploy(bytes)' $TEST_INIT)"
+echo "  cast send -r $RPC_2103 --private-key \"\$PRIVATE_KEY\" $SINGLETON \"\$calldata\""
--- a/scripts/verify/verify-rpc-2101-approve-and-sync.sh
+++ b/scripts/verify/verify-rpc-2101-approve-and-sync.sh
@@ -8,12 +8,12 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
 source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true

 RPC_CORE_1="${RPC_CORE_1:-192.168.11.211}"
 RPC_URL="${RPC_URL_138:-http://${RPC_CORE_1}:8545}"
-PROXMOX_USER="${PROXMOX_USER:-root}"
+PROXMOX_SSH_USER="${PROXMOX_SSH_USER:-root}"
 PROXMOX_R630="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
-PROXMOX_ML110="${PROXMOX_ML110:-${PROXMOX_HOST_ML110:-192.168.11.10}}"

 # Five validator IPs (1000-1004)
 VALIDATOR_IPS=(192.168.11.100 192.168.11.101 192.168.11.102 192.168.11.103 192.168.11.104)
@@ -101,13 +101,29 @@ else
  ((FAIL++)) || true
 fi

-# 5. Optional: validator service status (requires SSH to r630-01 and ml110)
-log_info "4. Validator status (5/5 active) — requires SSH to r630-01 and ml110..."
+validator_host() {
+  local vmid="$1"
+  if type get_host_for_vmid >/dev/null 2>&1; then
+    get_host_for_vmid "$vmid"
+  elif [[ "$vmid" -le 1002 ]]; then
+    echo "$PROXMOX_R630"
+  else
+    echo "${PROXMOX_HOST_ML110:-192.168.11.10}"
+  fi
+}
+
+# 5. Optional: validator service status (requires SSH to validator hosts)
+log_info "4. Validator status (5/5 active) — requires SSH to validator hosts..."
 ACTIVE=0
 SSH_OK=false
-for entry in "1000:$PROXMOX_R630" "1001:$PROXMOX_R630" "1002:$PROXMOX_R630" "1003:$PROXMOX_ML110" "1004:$PROXMOX_ML110"; do
+for entry in \
+  "1000:$(validator_host 1000)" \
+  "1001:$(validator_host 1001)" \
+  "1002:$(validator_host 1002)" \
+  "1003:$(validator_host 1003)" \
+  "1004:$(validator_host 1004)"; do
  IFS=':' read -r VMID HOST <<< "$entry"
-  STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${HOST}" \
+  STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_SSH_USER}@${HOST}" \
    "pct exec $VMID -- systemctl is-active besu-validator 2>/dev/null" 2>/dev/null || echo "unknown")
  if [[ "$STATUS" = "active" ]]; then
    ((ACTIVE++)) || true
--- a/scripts/verify/verify-singleton-post-deploy-2103.sh
+++ b/scripts/verify/verify-singleton-post-deploy-2103.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# After a singleton (EIP-2470) deploy tx, verify the **predicted** address — not the signer.
+# 1) Non-empty runtime code on RPC (default: 2103 Thirdweb admin)
+# 2) Optional: owner() if the contract implements it (otherwise note proxy/custom initcode)
+#
+# Usage: RPC_URL_2103=... ./scripts/verify/verify-singleton-post-deploy-2103.sh 0xPredictedAddr
+# Optional: DEPLOY_TX_SIGNER=0x... (display only: gas payer need not be admin/owner in Thirdweb/AA)
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck disable=SC1091
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true
+
+ADDR="${1:-}"
+RPC_2103="${RPC_URL_2103:-http://192.168.11.217:8545}"
+if [[ ! "$ADDR" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+  echo "Usage: $0 0xPredictedContractAddress" >&2
+  exit 2
+fi
+if ! command -v cast &>/dev/null; then
+  echo "ERROR: cast not in PATH" >&2
+  exit 1
+fi
+
+code=$(cast code -r "$RPC_2103" "$ADDR" 2>/dev/null || true)
+if [[ -z "$code" || "$code" == "0x" || "$code" == "0x0" ]]; then
+  echo "FAIL: no runtime code at $ADDR on $RPC_2103 (not deployed or wrong chain/RPC)" >&2
+  exit 1
+fi
+blen=$(((${#code} - 2) / 2))
+echo "OK: code at $ADDR: $blen byte(s) on 2103"
+
+# Optional: compare signer to owner — informational only
+if [[ -n "${DEPLOY_TX_SIGNER:-}" && "$DEPLOY_TX_SIGNER" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
+  echo "    (tx signer: $DEPLOY_TX_SIGNER — not necessarily the contract admin/owner in Thirdweb/AA initcode.)"
+fi
+
+# Prefer typed return; fall back to plain "owner()"
+o1="$(cast call -r "$RPC_2103" "$ADDR" "owner()(address)" 2>/dev/null | tr -d ' \n\r' || true)"
+o2="$(cast call -r "$RPC_2103" "$ADDR" "owner()" 2>/dev/null | tr -d ' \n\r' || true)"
+if [[ -n "$o1" && "$o1" == 0x* && ${#o1} -ge 42 ]]; then
+  own="$o1"
+else
+  own="$o2"
+fi
+if [[ -n "$own" && "$own" == 0x* && ${#own} -ge 42 ]]; then
+  echo "owner() -> $own"
+  echo "    Compare to your intended admin; the deployer/signer and owner() are independent here."
+else
+  echo "owner() n/a (revert) — verify admin via Blockscout, proxy storage, or initcode/roles; do not assume signer == owner"
+fi
+exit 0