smom-dbis-138/scripts/reserve/monitor-keeper.js

#!/usr/bin/env node
/**
 * Keeper Monitoring Service
 * Monitors keeper performance and sends alerts on failures
 *
 * Usage:
 *   node scripts/reserve/monitor-keeper.js
 *
 * Environment Variables:
 *   RPC_URL_138 - ChainID 138 RPC endpoint
 *   PRICE_FEED_KEEPER_ADDRESS - PriceFeedKeeper contract address
 *   ALERT_WEBHOOK - Webhook URL for alerts (optional)
 *   CHECK_INTERVAL - Check interval in seconds (default: 60)
 *   ALERT_THRESHOLD - Number of consecutive failures before alert (default: 3)
 */

const { ethers } = require('ethers');
const http = require('http');
require('dotenv').config();

const CONFIG = {
    rpcUrl: process.env.RPC_URL_138 || 'http://localhost:8545',
    keeperAddress: process.env.PRICE_FEED_KEEPER_ADDRESS,
    alertWebhook: process.env.ALERT_WEBHOOK,
    checkInterval: parseInt(process.env.CHECK_INTERVAL || '60') * 1000,
    alertThreshold: parseInt(process.env.ALERT_THRESHOLD || '3'),
};

const KEEPER_ABI = [
    "function checkUpkeep() external view returns (bool needsUpdate, address[] memory assets)",
    "function getTrackedAssets() external view returns (address[] memory)",
    "function updateInterval() external view returns (uint256)",
    "function lastUpdateTime(address asset) external view returns (uint256)",
    "event PriceFeedsUpdated(address[] assets, uint256 timestamp)",
];

class KeeperMonitor {
    constructor() {
        if (!CONFIG.keeperAddress) {
            throw new Error('PRICE_FEED_KEEPER_ADDRESS is required');
        }

        this.provider = new ethers.JsonRpcProvider(CONFIG.rpcUrl);
        this.keeper = new ethers.Contract(CONFIG.keeperAddress, KEEPER_ABI, this.provider);
        this.failureCount = 0;
        this.lastCheck = null;
        this.stats = {
            checks: 0,
            failures: 0,
            alerts: 0,
            lastUpdate: null,
        };
    }

    async start() {
        console.log('=== Keeper Monitor Service ===');
        console.log(`RPC URL: ${CONFIG.rpcUrl}`);
        console.log(`Keeper Address: ${CONFIG.keeperAddress}`);
        console.log(`Check Interval: ${CONFIG.checkInterval / 1000} seconds`);
        console.log(`Alert Threshold: ${CONFIG.alertThreshold} failures`);
        console.log('');

        // Start monitoring loop
        this.monitorLoop();

        // Start HTTP health endpoint
        this.startHealthServer();
    }

    async monitorLoop() {
        while (true) {
            try {
                await this.checkKeeper();
            } catch (error) {
                console.error(`Monitor error: ${error.message}`);
                this.failureCount++;
            }

            await this.sleep(CONFIG.checkInterval);
        }
    }

    async checkKeeper() {
        this.stats.checks++;
        const checkTime = new Date();

        try {
            // Check if upkeep is needed
            const [needsUpdate, assets] = await this.keeper.checkUpkeep();

            // Get tracked assets
            const trackedAssets = await this.keeper.getTrackedAssets();
            const updateInterval = await this.keeper.updateInterval();

            // Check last update times
            const staleAssets = [];
            for (const asset of trackedAssets) {
                const lastUpdate = await this.keeper.lastUpdateTime(asset);
                const timeSinceUpdate = checkTime.getTime() / 1000 - Number(lastUpdate);

                if (lastUpdate > 0 && timeSinceUpdate > Number(updateInterval) * 2) {
                    staleAssets.push({ asset, lastUpdate: Number(lastUpdate), stale: timeSinceUpdate });
                }
            }

            // Log status
            const status = {
                timestamp: checkTime.toISOString(),
                needsUpdate,
                assetsNeedingUpdate: assets.length,
                trackedAssets: trackedAssets.length,
                staleAssets: staleAssets.length,
                updateInterval: Number(updateInterval),
            };

            console.log(`[${status.timestamp}] Status:`, {
                needsUpdate: status.needsUpdate,
                assetsNeedingUpdate: status.assetsNeedingUpdate,
                trackedAssets: status.trackedAssets,
                staleAssets: status.staleAssets,
            });

            // Check for issues
            if (needsUpdate && assets.length > 0) {
                const timeSinceLastCheck = this.lastCheck
                    ? (checkTime.getTime() - this.lastCheck.getTime()) / 1000
                    : 0;

                if (timeSinceLastCheck > Number(updateInterval) * 2 && this.lastCheck !== null) {
                    // Keeper hasn't updated in time
                    this.failureCount++;
                    console.warn(`⚠ Keeper appears stalled. ${assets.length} assets need update.`);

                    if (this.failureCount >= CONFIG.alertThreshold) {
                        await this.sendAlert({
                            type: 'keeper_stalled',
                            message: `Keeper has not updated for ${timeSinceLastCheck} seconds`,
                            assetsNeedingUpdate: assets.length,
                            staleAssets,
                        });
                    }
                } else {
                    this.failureCount = 0; // Reset on success
                }
            }

            // Check for stale assets
            if (staleAssets.length > 0) {
                console.warn(`⚠ Found ${staleAssets.length} stale assets:`);
                staleAssets.forEach(({ asset, stale }) => {
                    console.warn(`  - ${asset}: ${(stale / 60).toFixed(1)} minutes stale`);
                });

                await this.sendAlert({
                    type: 'stale_assets',
                    message: `Found ${staleAssets.length} stale assets`,
                    staleAssets,
                });
            }

            this.lastCheck = checkTime;
            this.stats.lastUpdate = checkTime;

        } catch (error) {
            this.stats.failures++;
            this.failureCount++;
            console.error(`✗ Check failed: ${error.message}`);

            if (this.failureCount >= CONFIG.alertThreshold) {
                await this.sendAlert({
                    type: 'monitor_error',
                    message: `Monitor check failed: ${error.message}`,
                    error: error.message,
                });
            }
        }
    }

    async sendAlert(alert) {
        this.stats.alerts++;
        const alertData = {
            ...alert,
            timestamp: new Date().toISOString(),
            keeperAddress: CONFIG.keeperAddress,
            stats: this.stats,
        };

        console.error('🚨 ALERT:', alertData);

        if (CONFIG.alertWebhook) {
            try {
                await fetch(CONFIG.alertWebhook, {
                    method: 'POST',
                    headers: { 'Content-Type': 'application/json' },
                    body: JSON.stringify(alertData),
                });
                console.log('Alert sent to webhook');
            } catch (error) {
                console.error('Failed to send alert:', error.message);
            }
        }
    }

    startHealthServer() {
        const server = http.createServer((req, res) => {
            if (req.url === '/health') {
                res.writeHead(200, { 'Content-Type': 'application/json' });
                res.end(JSON.stringify({
                    status: 'healthy',
                    stats: this.stats,
                    lastCheck: this.lastCheck?.toISOString(),
                }));
            } else if (req.url === '/stats') {
                res.writeHead(200, { 'Content-Type': 'application/json' });
                res.end(JSON.stringify(this.stats));
            } else {
                res.writeHead(404);
                res.end('Not found');
            }
        });

        server.listen(3000, () => {
            console.log('Health server listening on port 3000');
        });
    }

    sleep(ms) {
        return new Promise(resolve => setTimeout(resolve, ms));
    }
}

// Main execution
if (require.main === module) {
    const monitor = new KeeperMonitor();

    process.on('SIGINT', () => {
        console.log('\nStopping monitor...');
        console.log('Final stats:', monitor.stats);
        process.exit(0);
    });

    monitor.start().catch(error => {
        console.error('Fatal error:', error);
        process.exit(1);
    });
}

module.exports = KeeperMonitor;