262 lines
6.3 KiB
JavaScript
262 lines
6.3 KiB
JavaScript
const logger = require('./logger');
|
|
|
|
/**
|
|
* Retry configuration options
|
|
*/
|
|
const DEFAULT_RETRY_OPTIONS = {
|
|
maxAttempts: 3,
|
|
baseDelay: 1000, // 1 second
|
|
maxDelay: 10000, // 10 seconds
|
|
backoffFactor: 2,
|
|
jitter: true,
|
|
retryCondition: (error) => {
|
|
// Default retry condition - retry on network errors and 5xx responses
|
|
const retryableCodes = ['ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED'];
|
|
const retryableStatusCodes = [500, 502, 503, 504];
|
|
|
|
return retryableCodes.includes(error.code) ||
|
|
retryableStatusCodes.includes(error.statusCode) ||
|
|
(error.status && retryableStatusCodes.includes(error.status));
|
|
}
|
|
};
|
|
|
|
/**
|
|
* Calculate delay with exponential backoff and optional jitter
|
|
*/
|
|
const calculateDelay = (attempt, options) => {
|
|
const { baseDelay, maxDelay, backoffFactor, jitter } = options;
|
|
|
|
let delay = baseDelay * Math.pow(backoffFactor, attempt - 1);
|
|
|
|
// Apply maximum delay limit
|
|
delay = Math.min(delay, maxDelay);
|
|
|
|
// Add jitter to prevent thundering herd
|
|
if (jitter) {
|
|
delay = delay * (0.5 + Math.random() * 0.5);
|
|
}
|
|
|
|
return Math.floor(delay);
|
|
};
|
|
|
|
/**
|
|
* Sleep utility function
|
|
*/
|
|
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
|
|
/**
|
|
* Retry wrapper for async operations
|
|
*/
|
|
const withRetry = async (operation, options = {}) => {
|
|
const config = { ...DEFAULT_RETRY_OPTIONS, ...options };
|
|
const { maxAttempts, retryCondition } = config;
|
|
|
|
let lastError;
|
|
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
try {
|
|
logger.debug('Retry Attempt', {
|
|
operation: operation.name || 'anonymous',
|
|
attempt,
|
|
maxAttempts
|
|
});
|
|
|
|
const result = await operation();
|
|
|
|
if (attempt > 1) {
|
|
logger.info('Retry Successful', {
|
|
operation: operation.name || 'anonymous',
|
|
attempt,
|
|
totalAttempts: attempt
|
|
});
|
|
}
|
|
|
|
return result;
|
|
} catch (error) {
|
|
lastError = error;
|
|
|
|
logger.warn('Retry Attempt Failed', {
|
|
operation: operation.name || 'anonymous',
|
|
attempt,
|
|
maxAttempts,
|
|
error: error.message,
|
|
errorCode: error.code,
|
|
statusCode: error.statusCode
|
|
});
|
|
|
|
// Check if we should retry
|
|
if (attempt === maxAttempts || !retryCondition(error)) {
|
|
break;
|
|
}
|
|
|
|
// Calculate and apply delay
|
|
const delay = calculateDelay(attempt, config);
|
|
|
|
logger.debug('Retry Delay', {
|
|
operation: operation.name || 'anonymous',
|
|
attempt,
|
|
delay: `${delay}ms`
|
|
});
|
|
|
|
await sleep(delay);
|
|
}
|
|
}
|
|
|
|
// All attempts failed
|
|
logger.error('Retry Exhausted', {
|
|
operation: operation.name || 'anonymous',
|
|
totalAttempts: maxAttempts,
|
|
finalError: lastError.message
|
|
});
|
|
|
|
throw lastError;
|
|
};
|
|
|
|
/**
|
|
* Retry wrapper for database operations
|
|
*/
|
|
const withDatabaseRetry = async (operation, options = {}) => {
|
|
const dbRetryOptions = {
|
|
maxAttempts: 3,
|
|
baseDelay: 500,
|
|
maxDelay: 5000,
|
|
retryCondition: (error) => {
|
|
// Retry on database connection errors and lock timeouts
|
|
const retryableCodes = [
|
|
'SQLITE_BUSY',
|
|
'SQLITE_LOCKED',
|
|
'SQLITE_PROTOCOL',
|
|
'ECONNRESET',
|
|
'ETIMEDOUT'
|
|
];
|
|
|
|
return retryableCodes.includes(error.code) ||
|
|
error.message.includes('database is locked') ||
|
|
error.message.includes('connection') ||
|
|
error.message.includes('timeout');
|
|
},
|
|
...options
|
|
};
|
|
|
|
return withRetry(operation, dbRetryOptions);
|
|
};
|
|
|
|
/**
|
|
* Retry wrapper for file operations
|
|
*/
|
|
const withFileRetry = async (operation, options = {}) => {
|
|
const fileRetryOptions = {
|
|
maxAttempts: 2,
|
|
baseDelay: 100,
|
|
maxDelay: 1000,
|
|
retryCondition: (error) => {
|
|
// Retry on temporary file system errors
|
|
const retryableCodes = [
|
|
'EBUSY',
|
|
'EMFILE',
|
|
'ENFILE',
|
|
'ENOENT'
|
|
];
|
|
|
|
return retryableCodes.includes(error.code);
|
|
},
|
|
...options
|
|
};
|
|
|
|
return withRetry(operation, fileRetryOptions);
|
|
};
|
|
|
|
/**
|
|
* Circuit breaker pattern implementation
|
|
*/
|
|
class CircuitBreaker {
|
|
constructor(options = {}) {
|
|
this.failureThreshold = options.failureThreshold || 5;
|
|
this.resetTimeout = options.resetTimeout || 60000; // 1 minute
|
|
this.monitoringPeriod = options.monitoringPeriod || 10000; // 10 seconds
|
|
|
|
this.state = 'CLOSED'; // CLOSED, OPEN, HALF_OPEN
|
|
this.failureCount = 0;
|
|
this.lastFailureTime = null;
|
|
this.successCount = 0;
|
|
}
|
|
|
|
async execute(operation) {
|
|
if (this.state === 'OPEN') {
|
|
if (Date.now() - this.lastFailureTime >= this.resetTimeout) {
|
|
this.state = 'HALF_OPEN';
|
|
this.successCount = 0;
|
|
logger.info('Circuit Breaker Half-Open', {
|
|
operation: operation.name || 'anonymous'
|
|
});
|
|
} else {
|
|
const error = new Error('Circuit breaker is OPEN');
|
|
error.code = 'CIRCUIT_BREAKER_OPEN';
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
try {
|
|
const result = await operation();
|
|
|
|
if (this.state === 'HALF_OPEN') {
|
|
this.successCount++;
|
|
if (this.successCount >= 3) {
|
|
this.reset();
|
|
logger.info('Circuit Breaker Closed', {
|
|
operation: operation.name || 'anonymous'
|
|
});
|
|
}
|
|
} else {
|
|
this.reset();
|
|
}
|
|
|
|
return result;
|
|
} catch (error) {
|
|
this.recordFailure();
|
|
|
|
if (this.failureCount >= this.failureThreshold) {
|
|
this.state = 'OPEN';
|
|
this.lastFailureTime = Date.now();
|
|
|
|
logger.error('Circuit Breaker Opened', {
|
|
operation: operation.name || 'anonymous',
|
|
failureCount: this.failureCount,
|
|
threshold: this.failureThreshold
|
|
});
|
|
}
|
|
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
reset() {
|
|
this.state = 'CLOSED';
|
|
this.failureCount = 0;
|
|
this.lastFailureTime = null;
|
|
this.successCount = 0;
|
|
}
|
|
|
|
recordFailure() {
|
|
this.failureCount++;
|
|
}
|
|
|
|
getState() {
|
|
return {
|
|
state: this.state,
|
|
failureCount: this.failureCount,
|
|
lastFailureTime: this.lastFailureTime,
|
|
successCount: this.successCount
|
|
};
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
withRetry,
|
|
withDatabaseRetry,
|
|
withFileRetry,
|
|
CircuitBreaker,
|
|
calculateDelay,
|
|
sleep,
|
|
DEFAULT_RETRY_OPTIONS
|
|
}; |