273 lines
7.7 KiB
TypeScript
273 lines
7.7 KiB
TypeScript
// Alert Service
|
|
// Sends alerts for critical events and risk violations
|
|
|
|
import { logger } from '@/infrastructure/monitoring/logger';
|
|
import { metricsService } from './metrics.service';
|
|
|
|
export enum AlertSeverity {
|
|
LOW = 'low',
|
|
MEDIUM = 'medium',
|
|
HIGH = 'high',
|
|
CRITICAL = 'critical',
|
|
}
|
|
|
|
export interface Alert {
|
|
severity: AlertSeverity;
|
|
message: string;
|
|
dealId?: string;
|
|
violationType?: string;
|
|
metadata?: Record<string, any>;
|
|
timestamp: Date;
|
|
}
|
|
|
|
export class AlertService {
|
|
private alertChannels: AlertChannel[] = [];
|
|
|
|
constructor() {
|
|
// Initialize alert channels based on environment
|
|
if (process.env.SLACK_WEBHOOK_URL) {
|
|
this.alertChannels.push(new SlackAlertChannel(process.env.SLACK_WEBHOOK_URL));
|
|
}
|
|
|
|
if (process.env.PAGERDUTY_INTEGRATION_KEY) {
|
|
this.alertChannels.push(new PagerDutyAlertChannel(process.env.PAGERDUTY_INTEGRATION_KEY));
|
|
}
|
|
|
|
if (process.env.EMAIL_ALERT_RECIPIENTS) {
|
|
this.alertChannels.push(new EmailAlertChannel(process.env.EMAIL_ALERT_RECIPIENTS.split(',')));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send alert
|
|
*/
|
|
async sendAlert(alert: Alert): Promise<void> {
|
|
logger.error('Alert triggered', {
|
|
severity: alert.severity,
|
|
message: alert.message,
|
|
dealId: alert.dealId,
|
|
violationType: alert.violationType,
|
|
});
|
|
|
|
// Record in metrics
|
|
if (alert.violationType) {
|
|
metricsService.recordRiskViolation(alert.violationType, alert.severity);
|
|
}
|
|
|
|
// Send to all channels
|
|
const promises = this.alertChannels.map(channel =>
|
|
channel.send(alert).catch(err => {
|
|
logger.error('Failed to send alert via channel', {
|
|
channel: channel.constructor.name,
|
|
error: err instanceof Error ? err.message : 'Unknown error',
|
|
});
|
|
})
|
|
);
|
|
|
|
await Promise.allSettled(promises);
|
|
}
|
|
|
|
/**
|
|
* Alert on risk violation
|
|
*/
|
|
async alertRiskViolation(
|
|
violationType: string,
|
|
message: string,
|
|
dealId?: string,
|
|
severity: AlertSeverity = AlertSeverity.HIGH
|
|
): Promise<void> {
|
|
await this.sendAlert({
|
|
severity,
|
|
message: `Risk Violation: ${message}`,
|
|
dealId,
|
|
violationType,
|
|
timestamp: new Date(),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Alert on LTV threshold
|
|
*/
|
|
async alertLtvThreshold(dealId: string, currentLtv: number, maxLtv: number): Promise<void> {
|
|
const percentage = (currentLtv / maxLtv) * 100;
|
|
let severity = AlertSeverity.MEDIUM;
|
|
|
|
if (percentage >= 95) {
|
|
severity = AlertSeverity.CRITICAL;
|
|
} else if (percentage >= 85) {
|
|
severity = AlertSeverity.HIGH;
|
|
}
|
|
|
|
await this.alertRiskViolation(
|
|
'ltv_threshold',
|
|
`LTV at ${(currentLtv * 100).toFixed(2)}% (${percentage.toFixed(1)}% of max ${(maxLtv * 100).toFixed(2)}%)`,
|
|
dealId,
|
|
severity
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Alert on USDTz exposure
|
|
*/
|
|
async alertUsdtzExposure(dealId: string, exposure: number, maxExposure: number): Promise<void> {
|
|
const percentage = (exposure / maxExposure) * 100;
|
|
let severity = AlertSeverity.MEDIUM;
|
|
|
|
if (percentage >= 95) {
|
|
severity = AlertSeverity.CRITICAL;
|
|
} else if (percentage >= 85) {
|
|
severity = AlertSeverity.HIGH;
|
|
}
|
|
|
|
await this.alertRiskViolation(
|
|
'usdtz_exposure',
|
|
`USDTz exposure at $${exposure.toFixed(2)} (${percentage.toFixed(1)}% of max $${maxExposure.toFixed(2)})`,
|
|
dealId,
|
|
severity
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Alert on deal failure
|
|
*/
|
|
async alertDealFailure(dealId: string, error: string, step?: string): Promise<void> {
|
|
await this.sendAlert({
|
|
severity: AlertSeverity.HIGH,
|
|
message: `Deal execution failed: ${error}`,
|
|
dealId,
|
|
metadata: { step },
|
|
timestamp: new Date(),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Alert on system error
|
|
*/
|
|
async alertSystemError(error: string, metadata?: Record<string, any>): Promise<void> {
|
|
await this.sendAlert({
|
|
severity: AlertSeverity.CRITICAL,
|
|
message: `System error: ${error}`,
|
|
metadata,
|
|
timestamp: new Date(),
|
|
});
|
|
}
|
|
}
|
|
|
|
// Alert Channel Interfaces
|
|
interface AlertChannel {
|
|
send(alert: Alert): Promise<void>;
|
|
}
|
|
|
|
class SlackAlertChannel implements AlertChannel {
|
|
constructor(private webhookUrl: string) {}
|
|
|
|
async send(alert: Alert): Promise<void> {
|
|
const color = {
|
|
[AlertSeverity.LOW]: '#36a64f',
|
|
[AlertSeverity.MEDIUM]: '#ffa500',
|
|
[AlertSeverity.HIGH]: '#ff6600',
|
|
[AlertSeverity.CRITICAL]: '#ff0000',
|
|
}[alert.severity];
|
|
|
|
const fields: Array<{ title: string; value: string; short: boolean }> = [
|
|
...(alert.dealId ? [{ title: 'Deal ID', value: alert.dealId, short: true }] : []),
|
|
...(alert.violationType ? [{ title: 'Violation Type', value: alert.violationType, short: true }] : []),
|
|
{ title: 'Timestamp', value: alert.timestamp.toISOString(), short: true },
|
|
...(alert.metadata ? Object.entries(alert.metadata).map(([k, v]) => ({ title: k, value: String(v), short: true })) : []),
|
|
];
|
|
|
|
const payload = {
|
|
attachments: [{ color, title: `Arbitrage Alert: ${alert.severity.toUpperCase()}`, text: alert.message, fields }],
|
|
};
|
|
|
|
try {
|
|
const res = await fetch(this.webhookUrl, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(payload),
|
|
});
|
|
if (!res.ok) throw new Error(`Slack webhook failed: ${res.status}`);
|
|
} catch (err) {
|
|
logger.error('Slack alert delivery failed', { error: err instanceof Error ? err.message : err });
|
|
}
|
|
}
|
|
}
|
|
|
|
class PagerDutyAlertChannel implements AlertChannel {
|
|
constructor(private integrationKey: string) {}
|
|
|
|
async send(alert: Alert): Promise<void> {
|
|
const severity = {
|
|
[AlertSeverity.LOW]: 'info',
|
|
[AlertSeverity.MEDIUM]: 'warning',
|
|
[AlertSeverity.HIGH]: 'error',
|
|
[AlertSeverity.CRITICAL]: 'critical',
|
|
}[alert.severity];
|
|
|
|
const payload = {
|
|
routing_key: this.integrationKey,
|
|
event_action: 'trigger',
|
|
payload: {
|
|
summary: alert.message,
|
|
severity,
|
|
source: 'arbitrage-service',
|
|
custom_details: {
|
|
dealId: alert.dealId,
|
|
violationType: alert.violationType,
|
|
...alert.metadata,
|
|
},
|
|
},
|
|
};
|
|
|
|
try {
|
|
const res = await fetch('https://events.pagerduty.com/v2/enqueue', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(payload),
|
|
});
|
|
if (!res.ok) {
|
|
const text = await res.text();
|
|
throw new Error(`PagerDuty API ${res.status}: ${text}`);
|
|
}
|
|
} catch (err) {
|
|
logger.error('PagerDuty alert delivery failed', {
|
|
error: err instanceof Error ? err.message : String(err),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
class EmailAlertChannel implements AlertChannel {
|
|
constructor(private recipients: string[]) {}
|
|
|
|
async send(alert: Alert): Promise<void> {
|
|
if (alert.severity !== AlertSeverity.CRITICAL && alert.severity !== AlertSeverity.HIGH) {
|
|
return;
|
|
}
|
|
|
|
const emailApiUrl = process.env.EMAIL_ALERT_API_URL;
|
|
if (!emailApiUrl) {
|
|
logger.warn('Email alert skipped: Set EMAIL_ALERT_API_URL (e.g. SendGrid) to enable');
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const res = await fetch(emailApiUrl, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
to: this.recipients,
|
|
subject: `[${alert.severity.toUpperCase()}] Arbitrage Alert`,
|
|
text: alert.message,
|
|
html: `<p>${alert.message}</p><p>Deal ID: ${alert.dealId || 'N/A'}</p>`,
|
|
}),
|
|
});
|
|
if (!res.ok) throw new Error(`Email API failed: ${res.status}`);
|
|
} catch (err) {
|
|
logger.error('Email alert delivery failed', { error: err instanceof Error ? err.message : err });
|
|
}
|
|
}
|
|
}
|
|
|
|
export const alertService = new AlertService();
|