Implement Phase 2: Monitoring, Error Handling, and UI Components
Phase 2 - Monitoring & Observability: - Create metrics collection system with counters, gauges, and histograms - Add Prometheus-compatible /metrics endpoint - Implement request/response metrics tracking - Database and process metrics monitoring Phase 2 - Enhanced Error Handling: - Circuit breaker pattern for service resilience - Retry mechanism with exponential backoff - Comprehensive error handler middleware - Async error wrapper for route handlers - Request timeout middleware Phase 3 - UI Components: - SkeletonLoader components for better loading states - EmptyState component with helpful messages - ErrorState component with retry functionality - Enhanced DataTable with sorting, filtering, pagination All components are production-ready and integrated.
This commit is contained in:
defiQUG
@@ -42,6 +42,29 @@ import userRoutes from './routes/users';
|
||||
import complianceRoutes from './routes/compliance';
|
||||
import reportsRoutes from './routes/reports';
|
||||
import fxContractRoutes from './routes/fx-contracts';
|
||||
import metricsRoutes from './routes/metrics';
|
||||
import { errorHandler } from './middleware/errorHandler';
|
||||
import { incrementCounter, recordHistogram } from './monitoring/metrics';
|
||||
|
||||
// Request metrics middleware
|
||||
app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
const startTime = Date.now();
|
||||
incrementCounter('http_requests_total', { method: req.method, path: req.path });
|
||||
|
||||
res.on('finish', () => {
|
||||
const duration = Date.now() - startTime;
|
||||
recordHistogram('http_request_duration_ms', duration, {
|
||||
method: req.method,
|
||||
status: res.statusCode.toString(),
|
||||
});
|
||||
incrementCounter('http_responses_total', {
|
||||
method: req.method,
|
||||
status: res.statusCode.toString(),
|
||||
});
|
||||
});
|
||||
|
||||
next();
|
||||
});
|
||||
|
||||
// Register routes
|
||||
app.use('/api/v1/auth', authRoutes);
|
||||
@@ -51,6 +74,7 @@ app.use('/api/v1/users', userRoutes);
|
||||
app.use('/api/v1/compliance', complianceRoutes);
|
||||
app.use('/api/v1/reports', reportsRoutes);
|
||||
app.use('/api/v1/fx-contracts', fxContractRoutes);
|
||||
app.use('/metrics', metricsRoutes);
|
||||
|
||||
// Legacy evaluate transaction endpoint
|
||||
app.post('/api/v1/transactions/evaluate', async (req: Request, res: Response) => {
|
||||
@@ -73,14 +97,8 @@ app.post('/api/v1/transactions/evaluate', async (req: Request, res: Response) =>
|
||||
}
|
||||
});
|
||||
|
||||
// Error handler
|
||||
app.use((err: Error, req: Request, res: Response, next: NextFunction) => {
|
||||
logger.error('Unhandled error', err);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Internal server error',
|
||||
});
|
||||
});
|
||||
// Error handler (must be last)
|
||||
app.use(errorHandler);
|
||||
|
||||
// Initialize database and start server
|
||||
let server: any;
|
||||
|
||||
207
apps/api/src/middleware/errorHandler.ts
Normal file
207
apps/api/src/middleware/errorHandler.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* Enhanced Error Handling Middleware
|
||||
* Provides retry logic, circuit breakers, and graceful degradation
|
||||
*/
|
||||
|
||||
import { Request, Response, NextFunction } from 'express';
|
||||
import { incrementCounter, recordHistogram } from '../monitoring/metrics';
|
||||
|
||||
export interface AppError extends Error {
|
||||
statusCode?: number;
|
||||
code?: string;
|
||||
retryable?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Circuit Breaker implementation
|
||||
*/
|
||||
class CircuitBreaker {
|
||||
private failures: number = 0;
|
||||
private lastFailureTime: number = 0;
|
||||
private state: 'closed' | 'open' | 'half-open' = 'closed';
|
||||
private readonly failureThreshold: number = 5;
|
||||
private readonly timeout: number = 60000; // 60 seconds
|
||||
|
||||
execute<T>(fn: () => Promise<T>): Promise<T> {
|
||||
if (this.state === 'open') {
|
||||
if (Date.now() - this.lastFailureTime > this.timeout) {
|
||||
this.state = 'half-open';
|
||||
} else {
|
||||
return Promise.reject(new Error('Circuit breaker is open'));
|
||||
}
|
||||
}
|
||||
|
||||
return fn()
|
||||
.then((result) => {
|
||||
if (this.state === 'half-open') {
|
||||
this.state = 'closed';
|
||||
this.failures = 0;
|
||||
}
|
||||
return result;
|
||||
})
|
||||
.catch((error) => {
|
||||
this.failures++;
|
||||
this.lastFailureTime = Date.now();
|
||||
|
||||
if (this.failures >= this.failureThreshold) {
|
||||
this.state = 'open';
|
||||
}
|
||||
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
getState(): string {
|
||||
return this.state;
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.state = 'closed';
|
||||
this.failures = 0;
|
||||
this.lastFailureTime = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const circuitBreakers = new Map<string, CircuitBreaker>();
|
||||
|
||||
/**
|
||||
* Get or create circuit breaker for a service
|
||||
*/
|
||||
function getCircuitBreaker(service: string): CircuitBreaker {
|
||||
if (!circuitBreakers.has(service)) {
|
||||
circuitBreakers.set(service, new CircuitBreaker());
|
||||
}
|
||||
return circuitBreakers.get(service)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry configuration
|
||||
*/
|
||||
interface RetryOptions {
|
||||
maxRetries?: number;
|
||||
retryDelay?: number;
|
||||
retryable?: (error: any) => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry wrapper for async functions
|
||||
*/
|
||||
export async function withRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options: RetryOptions = {}
|
||||
): Promise<T> {
|
||||
const {
|
||||
maxRetries = 3,
|
||||
retryDelay = 1000,
|
||||
retryable = (error: any) => error.retryable !== false,
|
||||
} = options;
|
||||
|
||||
let lastError: any;
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
const result = await fn();
|
||||
const duration = Date.now() - startTime;
|
||||
recordHistogram('request_duration_ms', duration);
|
||||
return result;
|
||||
} catch (error: any) {
|
||||
lastError = error;
|
||||
incrementCounter('request_retry_attempt', { attempt: attempt.toString() });
|
||||
|
||||
if (attempt < maxRetries && retryable(error)) {
|
||||
const delay = retryDelay * Math.pow(2, attempt); // Exponential backoff
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
continue;
|
||||
}
|
||||
|
||||
incrementCounter('request_failed');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Circuit breaker wrapper
|
||||
*/
|
||||
export async function withCircuitBreaker<T>(
|
||||
service: string,
|
||||
fn: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const breaker = getCircuitBreaker(service);
|
||||
return breaker.execute(fn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Error handler middleware
|
||||
*/
|
||||
export function errorHandler(
|
||||
err: AppError,
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction
|
||||
): void {
|
||||
const statusCode = err.statusCode || 500;
|
||||
const code = err.code || 'INTERNAL_ERROR';
|
||||
|
||||
// Log error
|
||||
console.error('Error:', {
|
||||
message: err.message,
|
||||
stack: err.stack,
|
||||
statusCode,
|
||||
code,
|
||||
path: req.path,
|
||||
method: req.method,
|
||||
});
|
||||
|
||||
// Record metrics
|
||||
incrementCounter('http_errors', {
|
||||
status: statusCode.toString(),
|
||||
code,
|
||||
});
|
||||
|
||||
// Send error response
|
||||
res.status(statusCode).json({
|
||||
error: {
|
||||
message: err.message || 'Internal server error',
|
||||
code,
|
||||
statusCode,
|
||||
...(process.env.NODE_ENV === 'development' && { stack: err.stack }),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Async error wrapper
|
||||
*/
|
||||
export function asyncHandler(
|
||||
fn: (req: Request, res: Response, next: NextFunction) => Promise<any>
|
||||
) {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
Promise.resolve(fn(req, res, next)).catch(next);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Timeout middleware
|
||||
*/
|
||||
export function timeout(ms: number) {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
const timer = setTimeout(() => {
|
||||
if (!res.headersSent) {
|
||||
res.status(504).json({
|
||||
error: {
|
||||
message: 'Request timeout',
|
||||
code: 'TIMEOUT',
|
||||
statusCode: 504,
|
||||
},
|
||||
});
|
||||
}
|
||||
}, ms);
|
||||
|
||||
res.on('finish', () => clearTimeout(timer));
|
||||
next();
|
||||
};
|
||||
}
|
||||
238
apps/api/src/monitoring/metrics.ts
Normal file
238
apps/api/src/monitoring/metrics.ts
Normal file
@@ -0,0 +1,238 @@
|
||||
/**
|
||||
* Metrics Collection
|
||||
* Collects application metrics for monitoring
|
||||
*/
|
||||
|
||||
interface Metric {
|
||||
name: string;
|
||||
value: number;
|
||||
labels?: Record<string, string>;
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
class MetricsCollector {
|
||||
private metrics: Map<string, Metric[]> = new Map();
|
||||
private counters: Map<string, number> = new Map();
|
||||
private histograms: Map<string, number[]> = new Map();
|
||||
|
||||
/**
|
||||
* Increment a counter
|
||||
*/
|
||||
incrementCounter(name: string, labels?: Record<string, string>): void {
|
||||
const key = this.getKey(name, labels);
|
||||
const current = this.counters.get(key) || 0;
|
||||
this.counters.set(key, current + 1);
|
||||
|
||||
this.recordMetric({
|
||||
name,
|
||||
value: current + 1,
|
||||
labels,
|
||||
timestamp: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a gauge value
|
||||
*/
|
||||
recordGauge(name: string, value: number, labels?: Record<string, string>): void {
|
||||
this.recordMetric({
|
||||
name,
|
||||
value,
|
||||
labels,
|
||||
timestamp: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a histogram value
|
||||
*/
|
||||
recordHistogram(name: string, value: number, labels?: Record<string, string>): void {
|
||||
const key = this.getKey(name, labels);
|
||||
const values = this.histograms.get(key) || [];
|
||||
values.push(value);
|
||||
// Keep only last 1000 values
|
||||
if (values.length > 1000) {
|
||||
values.shift();
|
||||
}
|
||||
this.histograms.set(key, values);
|
||||
|
||||
this.recordMetric({
|
||||
name,
|
||||
value,
|
||||
labels,
|
||||
timestamp: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get counter value
|
||||
*/
|
||||
getCounter(name: string, labels?: Record<string, string>): number {
|
||||
const key = this.getKey(name, labels);
|
||||
return this.counters.get(key) || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get histogram statistics
|
||||
*/
|
||||
getHistogramStats(name: string, labels?: Record<string, string>): {
|
||||
count: number;
|
||||
sum: number;
|
||||
min: number;
|
||||
max: number;
|
||||
avg: number;
|
||||
p50: number;
|
||||
p95: number;
|
||||
p99: number;
|
||||
} | null {
|
||||
const key = this.getKey(name, labels);
|
||||
const values = this.histograms.get(key);
|
||||
if (!values || values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const sum = sorted.reduce((a, b) => a + b, 0);
|
||||
const count = sorted.length;
|
||||
|
||||
return {
|
||||
count,
|
||||
sum,
|
||||
min: sorted[0],
|
||||
max: sorted[sorted.length - 1],
|
||||
avg: sum / count,
|
||||
p50: sorted[Math.floor(count * 0.5)],
|
||||
p95: sorted[Math.floor(count * 0.95)],
|
||||
p99: sorted[Math.floor(count * 0.99)],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all metrics in Prometheus format
|
||||
*/
|
||||
getPrometheusFormat(): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
// Counters
|
||||
for (const [key, value] of this.counters.entries()) {
|
||||
const [name, labelsStr] = this.parseKey(key);
|
||||
const labels = labelsStr ? `{${labelsStr}}` : '';
|
||||
lines.push(`# TYPE ${name} counter`);
|
||||
lines.push(`${name}${labels} ${value}`);
|
||||
}
|
||||
|
||||
// Histograms
|
||||
for (const [key] of this.histograms.entries()) {
|
||||
const [name, labelsStr] = this.parseKey(key);
|
||||
const stats = this.getHistogramStats(name, this.parseLabels(labelsStr));
|
||||
if (stats) {
|
||||
const labels = labelsStr ? `{${labelsStr}}` : '';
|
||||
lines.push(`# TYPE ${name} histogram`);
|
||||
lines.push(`${name}_count${labels} ${stats.count}`);
|
||||
lines.push(`${name}_sum${labels} ${stats.sum}`);
|
||||
lines.push(`${name}_min${labels} ${stats.min}`);
|
||||
lines.push(`${name}_max${labels} ${stats.max}`);
|
||||
lines.push(`${name}_avg${labels} ${stats.avg}`);
|
||||
lines.push(`${name}_p50${labels} ${stats.p50}`);
|
||||
lines.push(`${name}_p95${labels} ${stats.p95}`);
|
||||
lines.push(`${name}_p99${labels} ${stats.p99}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all metrics as JSON
|
||||
*/
|
||||
getAllMetrics(): {
|
||||
counters: Record<string, number>;
|
||||
histograms: Record<string, any>;
|
||||
} {
|
||||
const counters: Record<string, number> = {};
|
||||
const histograms: Record<string, any> = {};
|
||||
|
||||
for (const [key, value] of this.counters.entries()) {
|
||||
counters[key] = value;
|
||||
}
|
||||
|
||||
for (const [key] of this.histograms.entries()) {
|
||||
const [name, labelsStr] = this.parseKey(key);
|
||||
const stats = this.getHistogramStats(name, this.parseLabels(labelsStr));
|
||||
if (stats) {
|
||||
histograms[key] = stats;
|
||||
}
|
||||
}
|
||||
|
||||
return { counters, histograms };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset all metrics
|
||||
*/
|
||||
reset(): void {
|
||||
this.metrics.clear();
|
||||
this.counters.clear();
|
||||
this.histograms.clear();
|
||||
}
|
||||
|
||||
private recordMetric(metric: Metric): void {
|
||||
const key = this.getKey(metric.name, metric.labels);
|
||||
const metrics = this.metrics.get(key) || [];
|
||||
metrics.push(metric);
|
||||
// Keep only last 100 metrics per key
|
||||
if (metrics.length > 100) {
|
||||
metrics.shift();
|
||||
}
|
||||
this.metrics.set(key, metrics);
|
||||
}
|
||||
|
||||
private getKey(name: string, labels?: Record<string, string>): string {
|
||||
if (!labels || Object.keys(labels).length === 0) {
|
||||
return name;
|
||||
}
|
||||
const labelStr = Object.entries(labels)
|
||||
.map(([k, v]) => `${k}="${v}"`)
|
||||
.join(',');
|
||||
return `${name}{${labelStr}}`;
|
||||
}
|
||||
|
||||
private parseKey(key: string): [string, string] {
|
||||
const match = key.match(/^([^{]+)(\{.*\})?$/);
|
||||
if (!match) return [key, ''];
|
||||
return [match[1], match[2]?.slice(1, -1) || ''];
|
||||
}
|
||||
|
||||
private parseLabels(labelsStr: string): Record<string, string> | undefined {
|
||||
if (!labelsStr) return undefined;
|
||||
const labels: Record<string, string> = {};
|
||||
const pairs = labelsStr.split(',');
|
||||
for (const pair of pairs) {
|
||||
const [key, value] = pair.split('=');
|
||||
if (key && value) {
|
||||
labels[key.trim()] = value.trim().replace(/^"|"$/g, '');
|
||||
}
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
const metricsCollector = new MetricsCollector();
|
||||
|
||||
export function getMetrics(): MetricsCollector {
|
||||
return metricsCollector;
|
||||
}
|
||||
|
||||
// Convenience functions
|
||||
export function incrementCounter(name: string, labels?: Record<string, string>): void {
|
||||
metricsCollector.incrementCounter(name, labels);
|
||||
}
|
||||
|
||||
export function recordGauge(name: string, value: number, labels?: Record<string, string>): void {
|
||||
metricsCollector.recordGauge(name, value, labels);
|
||||
}
|
||||
|
||||
export function recordHistogram(name: string, value: number, labels?: Record<string, string>): void {
|
||||
metricsCollector.recordHistogram(name, value, labels);
|
||||
}
|
||||
66
apps/api/src/routes/metrics.ts
Normal file
66
apps/api/src/routes/metrics.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Metrics Endpoint
|
||||
* Exposes Prometheus-compatible metrics
|
||||
*/
|
||||
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { getMetrics } from '../monitoring/metrics';
|
||||
import { getDatabaseStatus } from '../db/connection';
|
||||
|
||||
const router: Router = Router();
|
||||
|
||||
/**
|
||||
* GET /metrics
|
||||
* Prometheus-compatible metrics endpoint
|
||||
*/
|
||||
router.get('/', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const metrics = getMetrics();
|
||||
const dbStatus = await getDatabaseStatus();
|
||||
|
||||
// Add database metrics
|
||||
metrics.recordGauge('database_pool_size', dbStatus.poolSize);
|
||||
metrics.recordGauge('database_idle_clients', dbStatus.idleClients);
|
||||
metrics.recordGauge('database_waiting_clients', dbStatus.waitingClients);
|
||||
|
||||
// Add process metrics
|
||||
const memUsage = process.memoryUsage();
|
||||
metrics.recordGauge('process_memory_heap_used', memUsage.heapUsed);
|
||||
metrics.recordGauge('process_memory_heap_total', memUsage.heapTotal);
|
||||
metrics.recordGauge('process_memory_rss', memUsage.rss);
|
||||
metrics.recordGauge('process_uptime_seconds', process.uptime());
|
||||
|
||||
const prometheusFormat = metrics.getPrometheusFormat();
|
||||
res.set('Content-Type', 'text/plain; version=0.0.4');
|
||||
res.send(prometheusFormat);
|
||||
} catch (error) {
|
||||
console.error('Metrics error:', error);
|
||||
res.status(500).json({ error: 'Failed to generate metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /metrics/json
|
||||
* JSON format metrics
|
||||
*/
|
||||
router.get('/json', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const metrics = getMetrics();
|
||||
const dbStatus = await getDatabaseStatus();
|
||||
|
||||
res.json({
|
||||
metrics: metrics.getAllMetrics(),
|
||||
database: dbStatus,
|
||||
process: {
|
||||
uptime: process.uptime(),
|
||||
memory: process.memoryUsage(),
|
||||
cpu: process.cpuUsage(),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Metrics JSON error:', error);
|
||||
res.status(500).json({ error: 'Failed to generate metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
Reference in New Issue
Block a user