d-bis/smom-dbis-138
4
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

fix(relay): defer on inventory probe errors; optional retry without budget

Browse Source 
- MessageQueue: resetRetryCount and retry({ increment: false }) for shedder/inventory paths
- RelayService: treat bridge_inventory_probe like other soft-failure scopes; wrap inventory check in try/catch
- Token aggregation: catch DB pool lookup errors and fall back to live DODO path
- Mainnet WETH profile: START_BLOCK=latest; extend RELAY_SKIP_MESSAGE_IDS for backlog hygiene
- Extend relay test.js for deferred requeue behavior

Made-with: Cursor
This commit is contained in:
defiQUG
2026-04-12 11:35:18 -07:00
parent 06c4bebcb7
commit 8ec6af94d5
5 changed files with 67 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
services/relay/src/MessageQueue.js
View File

@@ -65,14 +65,22 @@ export class MessageQueue {
async getRetryCount(messageId) {
return this.retryCounts.get(messageId) || 0;
}
async resetRetryCount(messageId) {
this.retryCounts.delete(messageId);
}
async retry(messageId) {
async retry(messageId, options = {}) {
const { increment = true } = options;
const count = this.retryCounts.get(messageId) || 0;
this.retryCounts.set(messageId, count + 1);
if (increment) {
this.retryCounts.set(messageId, count + 1);
}
const existingIndex = this.queue.findIndex(m => m.messageId === messageId);
if (existingIndex >= 0) {
this.logger.info(`Message ${messageId} retry count: ${count + 1}`);
const nextCount = increment ? count + 1 : count;
this.logger.info(`Message ${messageId} retry count: ${nextCount}`);
return;
}
@@ -84,7 +92,8 @@ export class MessageQueue {
this.inFlight.delete(messageId);
this.queue.push(messageData);
this.logger.info(`Message ${messageId} requeued. Retry count: ${count + 1}. Queue size: ${this.queue.length}`);
const nextCount = increment ? count + 1 : count;
this.logger.info(`Message ${messageId} requeued. Retry count: ${nextCount}. Queue size: ${this.queue.length}`);
}
getStats() {

47
services/relay/src/RelayService.js
View File

@@ -161,7 +161,11 @@ export class RelayService {
const lastSuccessMs = this.lastRelaySuccess && this.lastRelaySuccess.at ? Date.parse(this.lastRelaySuccess.at) : 0;
if (
this.lastError &&
(this.lastError.scope === 'relay_message' || this.lastError.scope === 'bridge_inventory') &&
(
this.lastError.scope === 'relay_message' ||
this.lastError.scope === 'bridge_inventory' ||
this.lastError.scope === 'bridge_inventory_probe'
) &&
Number.isFinite(lastErrorMs) &&
lastErrorMs > 0 &&
lastErrorMs >= lastSuccessMs
@@ -731,7 +735,8 @@ export class RelayService {
this.logger.warn(
`Destination relay router is paused; deferring ${messageId} (enable RELAY_SHEDDING=1 to pause off-chain too)`
);
await this.messageQueue.retry(messageId);
await this.messageQueue.resetRetryCount(messageId);
await this.messageQueue.retry(messageId, { increment: false });
await new Promise((r) => setTimeout(r, 15000));
return null;
}
@@ -807,11 +812,27 @@ export class RelayService {
};
});
const inventoryCheck = await this.ensureTargetBridgeInventory(
messageId,
targetBridge,
mappedTokenAmounts
);
let inventoryCheck;
try {
inventoryCheck = await this.ensureTargetBridgeInventory(
messageId,
targetBridge,
mappedTokenAmounts
);
} catch (inventoryProbeError) {
this.logger.warn(
`Bridge inventory probe failed for ${messageId}; deferring message until the next cycle`,
inventoryProbeError
);
this.recordError('bridge_inventory_probe', inventoryProbeError, {
message_id: messageId,
target_bridge: targetBridge
});
await this.messageQueue.resetRetryCount(messageId);
await this.messageQueue.retry(messageId, { increment: false });
await new Promise(resolve => setTimeout(resolve, this.config.retry.retryDelay));
return null;
}
if (!inventoryCheck.ok) {
const inventoryError = new Error(inventoryCheck.message);
this.logger.warn(inventoryCheck.message);
@@ -823,7 +844,8 @@ export class RelayService {
required_amount: inventoryCheck.requiredAmount.toString(),
shortfall: inventoryCheck.shortfall.toString()
});
await this.messageQueue.retry(messageId);
await this.messageQueue.resetRetryCount(messageId);
await this.messageQueue.retry(messageId, { increment: false });
await new Promise(resolve => setTimeout(resolve, this.config.retry.retryDelay));
return null;
}
@@ -896,7 +918,14 @@ export class RelayService {
target_bridge: targetBridge,
tx_hash: receipt.hash
};
if (this.lastError && (this.lastError.scope === 'relay_message' || this.lastError.scope === 'bridge_inventory')) {
if (
this.lastError &&
(
this.lastError.scope === 'relay_message' ||
this.lastError.scope === 'bridge_inventory' ||
this.lastError.scope === 'bridge_inventory_probe'
)
) {
this.lastError = null;
}