#!/usr/bin/env node /** * Keeper Monitoring Service * Monitors keeper performance and sends alerts on failures * * Usage: * node scripts/reserve/monitor-keeper.js * * Environment Variables: * RPC_URL_138 - ChainID 138 RPC endpoint * PRICE_FEED_KEEPER_ADDRESS - PriceFeedKeeper contract address * ALERT_WEBHOOK - Webhook URL for alerts (optional) * CHECK_INTERVAL - Check interval in seconds (default: 60) * ALERT_THRESHOLD - Number of consecutive failures before alert (default: 3) */ const { ethers } = require('ethers'); const http = require('http'); require('dotenv').config(); const CONFIG = { rpcUrl: process.env.RPC_URL_138 || 'http://localhost:8545', keeperAddress: process.env.PRICE_FEED_KEEPER_ADDRESS, alertWebhook: process.env.ALERT_WEBHOOK, checkInterval: parseInt(process.env.CHECK_INTERVAL || '60') * 1000, alertThreshold: parseInt(process.env.ALERT_THRESHOLD || '3'), }; const KEEPER_ABI = [ "function checkUpkeep() external view returns (bool needsUpdate, address[] memory assets)", "function getTrackedAssets() external view returns (address[] memory)", "function updateInterval() external view returns (uint256)", "function lastUpdateTime(address asset) external view returns (uint256)", "event PriceFeedsUpdated(address[] assets, uint256 timestamp)", ]; class KeeperMonitor { constructor() { if (!CONFIG.keeperAddress) { throw new Error('PRICE_FEED_KEEPER_ADDRESS is required'); } this.provider = new ethers.JsonRpcProvider(CONFIG.rpcUrl); this.keeper = new ethers.Contract(CONFIG.keeperAddress, KEEPER_ABI, this.provider); this.failureCount = 0; this.lastCheck = null; this.stats = { checks: 0, failures: 0, alerts: 0, lastUpdate: null, }; } async start() { console.log('=== Keeper Monitor Service ==='); console.log(`RPC URL: ${CONFIG.rpcUrl}`); console.log(`Keeper Address: ${CONFIG.keeperAddress}`); console.log(`Check Interval: ${CONFIG.checkInterval / 1000} seconds`); console.log(`Alert Threshold: ${CONFIG.alertThreshold} failures`); console.log(''); // Start monitoring loop this.monitorLoop(); // Start HTTP health endpoint this.startHealthServer(); } async monitorLoop() { while (true) { try { await this.checkKeeper(); } catch (error) { console.error(`Monitor error: ${error.message}`); this.failureCount++; } await this.sleep(CONFIG.checkInterval); } } async checkKeeper() { this.stats.checks++; const checkTime = new Date(); try { // Check if upkeep is needed const [needsUpdate, assets] = await this.keeper.checkUpkeep(); // Get tracked assets const trackedAssets = await this.keeper.getTrackedAssets(); const updateInterval = await this.keeper.updateInterval(); // Check last update times const staleAssets = []; for (const asset of trackedAssets) { const lastUpdate = await this.keeper.lastUpdateTime(asset); const timeSinceUpdate = checkTime.getTime() / 1000 - Number(lastUpdate); if (lastUpdate > 0 && timeSinceUpdate > Number(updateInterval) * 2) { staleAssets.push({ asset, lastUpdate: Number(lastUpdate), stale: timeSinceUpdate }); } } // Log status const status = { timestamp: checkTime.toISOString(), needsUpdate, assetsNeedingUpdate: assets.length, trackedAssets: trackedAssets.length, staleAssets: staleAssets.length, updateInterval: Number(updateInterval), }; console.log(`[${status.timestamp}] Status:`, { needsUpdate: status.needsUpdate, assetsNeedingUpdate: status.assetsNeedingUpdate, trackedAssets: status.trackedAssets, staleAssets: status.staleAssets, }); // Check for issues if (needsUpdate && assets.length > 0) { const timeSinceLastCheck = this.lastCheck ? (checkTime.getTime() - this.lastCheck.getTime()) / 1000 : 0; if (timeSinceLastCheck > Number(updateInterval) * 2 && this.lastCheck !== null) { // Keeper hasn't updated in time this.failureCount++; console.warn(`⚠ Keeper appears stalled. ${assets.length} assets need update.`); if (this.failureCount >= CONFIG.alertThreshold) { await this.sendAlert({ type: 'keeper_stalled', message: `Keeper has not updated for ${timeSinceLastCheck} seconds`, assetsNeedingUpdate: assets.length, staleAssets, }); } } else { this.failureCount = 0; // Reset on success } } // Check for stale assets if (staleAssets.length > 0) { console.warn(`⚠ Found ${staleAssets.length} stale assets:`); staleAssets.forEach(({ asset, stale }) => { console.warn(` - ${asset}: ${(stale / 60).toFixed(1)} minutes stale`); }); await this.sendAlert({ type: 'stale_assets', message: `Found ${staleAssets.length} stale assets`, staleAssets, }); } this.lastCheck = checkTime; this.stats.lastUpdate = checkTime; } catch (error) { this.stats.failures++; this.failureCount++; console.error(`✗ Check failed: ${error.message}`); if (this.failureCount >= CONFIG.alertThreshold) { await this.sendAlert({ type: 'monitor_error', message: `Monitor check failed: ${error.message}`, error: error.message, }); } } } async sendAlert(alert) { this.stats.alerts++; const alertData = { ...alert, timestamp: new Date().toISOString(), keeperAddress: CONFIG.keeperAddress, stats: this.stats, }; console.error('🚨 ALERT:', alertData); if (CONFIG.alertWebhook) { try { await fetch(CONFIG.alertWebhook, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(alertData), }); console.log('Alert sent to webhook'); } catch (error) { console.error('Failed to send alert:', error.message); } } } startHealthServer() { const server = http.createServer((req, res) => { if (req.url === '/health') { res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ status: 'healthy', stats: this.stats, lastCheck: this.lastCheck?.toISOString(), })); } else if (req.url === '/stats') { res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(this.stats)); } else { res.writeHead(404); res.end('Not found'); } }); server.listen(3000, () => { console.log('Health server listening on port 3000'); }); } sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } } // Main execution if (require.main === module) { const monitor = new KeeperMonitor(); process.on('SIGINT', () => { console.log('\nStopping monitor...'); console.log('Final stats:', monitor.stats); process.exit(0); }); monitor.start().catch(error => { console.error('Fatal error:', error); process.exit(1); }); } module.exports = KeeperMonitor;