proxmox/multi-chain-execution/src/api/observability.ts

/**
 * Observability: circuit breaker, health, metrics.
 */

import { Router, Request, Response, NextFunction } from 'express';

let circuitOpen = false;
let errorCount = 0;
let lastErrorAt = 0;
const ERROR_THRESHOLD = 10;
const RESET_MS = 60_000;

/** Admin: force circuit breaker on or off. */
export function setCircuitBreaker(open: boolean): void {
  circuitOpen = open;
  if (!open) errorCount = 0;
}

export function circuitBreakerMiddleware(req: Request, res: Response, next: NextFunction): void {
  if (circuitOpen) {
    if (Date.now() - lastErrorAt > RESET_MS) {
      circuitOpen = false;
      errorCount = 0;
    } else {
      res.status(503).json({ error: 'Circuit breaker open', retry_after: RESET_MS / 1000 });
      return;
    }
  }
  res.on('finish', () => {
    if (res.statusCode >= 500) {
      errorCount++;
      lastErrorAt = Date.now();
      if (errorCount >= ERROR_THRESHOLD) circuitOpen = true;
    } else {
      errorCount = Math.max(0, errorCount - 1);
    }
  });
  next();
}

const healthRouter: Router = Router();
healthRouter.get('/v1/health', (_req: Request, res: Response) => {
  res.json({
    status: circuitOpen ? 'degraded' : 'ok',
    circuit_breaker: circuitOpen ? 'open' : 'closed',
    error_count: errorCount,
  });
});
healthRouter.get('/v1/metrics', (_req: Request, res: Response) => {
  res.setHeader('Content-Type', 'text/plain');
  res.send(
    `# HELP multi_chain_execution_errors Total 5xx errors\n` +
      `# TYPE multi_chain_execution_errors counter\n` +
      `multi_chain_execution_errors ${errorCount}\n` +
      `# HELP multi_chain_execution_circuit_open Circuit breaker open (1=open)\n` +
      `# TYPE multi_chain_execution_circuit_open gauge\n` +
      `multi_chain_execution_circuit_open ${circuitOpen ? 1 : 0}\n`
  );
});

export const healthRoutes = healthRouter;