diff --git a/utils/retry/.eslintrc.js b/utils/retry/.eslintrc.js new file mode 100644 index 0000000000..ce90fb9646 --- /dev/null +++ b/utils/retry/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/utils/retry/.npmignore b/utils/retry/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/utils/retry/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/utils/retry/config/rig.json b/utils/retry/config/rig.json new file mode 100644 index 0000000000..78cc5a1733 --- /dev/null +++ b/utils/retry/config/rig.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig", + "rigProfile": "node" +} diff --git a/utils/retry/jest.config.js b/utils/retry/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/utils/retry/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/utils/retry/package.json b/utils/retry/package.json new file mode 100644 index 0000000000..dfea4bc7e1 --- /dev/null +++ b/utils/retry/package.json @@ -0,0 +1,38 @@ +{ + "name": "@hcengineering/retry", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "author": "Copyright © Hardcore Engineering Inc.", + "template": "@hcengineering/node-package", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "test": "jest --passWithNoTests --silent --forceExit", + "format": "format src", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent --forceExit", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.6.0", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.3.3", + "@types/node": "~20.11.16", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5" + }, + "dependencies": { + } +} diff --git a/utils/retry/readme.md b/utils/retry/readme.md new file mode 100644 index 0000000000..55eda694b5 --- /dev/null +++ b/utils/retry/readme.md @@ -0,0 +1,252 @@ +# Retry Utility + +A comprehensive TypeScript utility for handling transient failures with exponential backoff, jitter, and customizable retry conditions. + +## Features + +- ✅ **Exponential backoff** with configurable parameters +- ✅ **Jitter support** to prevent thundering herd problems +- ✅ **Customizable retry conditions** to control which errors should be retried +- ✅ **TypeScript decorators** for clean, declarative retry logic +- ✅ **Function wrappers** for retrofitting existing code with retry capabilities +- ✅ **Comprehensive logging** of retry attempts and failures + +## Usage + +### Basic Usage + +Wrap any async operation with the `withRetry` function: + +```typescript +import { withRetry } from '@hcengineering/retry' + +async function fetchData() { + const data = await withRetry( + async () => { + // Your async operation that might fail transiently + return await api.getData() + }, + { maxRetries: 3 } + ) + return data +} +``` + +### Using Decorators + +For class methods, you can use the `@Retryable` decorator for clean, declarative retry logic: + +```typescript +import { Retryable } from '@hcengineering/retry' + +class UserService { + @Retryable({ maxRetries: 5 }) + async getUserProfile(userId: string): Promise { + // This method will automatically retry on failure + return await this.api.fetchUserProfile(userId) + } +} +``` + +### Delay Strategies + +The package provides several delay strategies to control the timing between retry attempts: + +Exponential Backoff +Increases the delay exponentially between retries, which is ideal for backing off from overloaded services: + +```typescript +import { withRetry, DelayStrategyFactory } from '@hcengineering/retry' + +await withRetry( + async () => await api.getData(), + { + maxRetries: 5, + delayStrategy: DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 100, // Start with 100ms + maxDelayMs: 10000, // Cap at 10 seconds + backoffFactor: 2, // Double the delay each time (100, 200, 400, 800, 1600) + jitter: 0.2 // Add ±20% randomness + }) + } +) +``` + +Fixed Delay +Uses the same delay for all retry attempts, useful when retrying after a fixed cooldown period: +```typescript +import { withRetry, DelayStrategyFactory } from '@hcengineering/retry' + +await withRetry( + async () => await api.getData(), + { + maxRetries: 3, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 1000, // Always wait 1 second between retries + jitter: 0.1 // Optional: add ±10% randomness + }) + } +) +``` +Fibonacci Delay +Uses the Fibonacci sequence to calculate delays, providing a more moderate growth rate than exponential backoff: +```typescript +import { withRetry, DelayStrategyFactory } from '@hcengineering/retry' + +await withRetry( + async () => await api.getData(), + { + maxRetries: 6, + delayStrategy: DelayStrategyFactory.fibonacci({ + baseDelayMs: 100, // Base unit for Fibonacci sequence + maxDelayMs: 10000, // Maximum delay cap + jitter: 0.2 // Add ±20% randomness + }) + } +) +// Delays follow Fibonacci sequence: 100ms, 200ms, 300ms, 500ms, 800ms, ... +``` + +### Custom Retry Conditions + +You can specify which errors should trigger retries: + +```typescript +import { withRetry, retryNetworkErrors } from '@platform/utils/retry' + +async function fetchData() { + return await withRetry( + async () => await api.getData(), + { + // Only retry network-related errors + isRetryable: retryNetworkErrors, + maxRetries: 5 + } + ) +} +``` + +Create your own custom retry condition: + +```typescript +import { type IsRetryable } from '@platform/utils/retry' + +// Custom retry condition +const retryDatabaseErrors: IsRetryable = (error: unknown): boolean => { + if (error instanceof DatabaseError) { + // Only retry specific database errors + return error.code === 'CONNECTION_LOST' || + error.code === 'DEADLOCK' || + error.code === 'TIMEOUT' + } + return false +} + +// Use it +await withRetry( + async () => await db.query('SELECT * FROM users'), + { isRetryable: retryDatabaseErrors } +) +``` + +## API Reference + +### `withRetry(operation, options?, operationName?): Promise` + +Executes an async operation with retry logic. + +- `operation`: `() => Promise` - The async operation to execute +- `options`: `Partial` - Retry configuration (optional) +- `operationName`: `string` - Name for logging (optional) +- Returns: `Promise` - The result of the operation + +### `createRetryableFunction(fn, options?, operationName?): T` + +Creates a retryable function from an existing function. + +- `fn`: `T extends (...args: any[]) => Promise` - The function to make retryable +- `options`: `Partial` - Retry configuration (optional) +- `operationName`: `string` - Name for logging (optional) +- Returns: `T` - A wrapped function with retry logic + +### `@Retryable(options?)` + +Method decorator for adding retry functionality to class methods. + +- `options`: `Partial` - Retry configuration (optional) + +### RetryOptions + +Configuration options for the retry mechanism: + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `initialDelayMs` | `number` | `1000` | Initial delay between retries in milliseconds | +| `maxDelayMs` | `number` | `30000` | Maximum delay between retries in milliseconds | +| `maxRetries` | `number` | `5` | Maximum number of retry attempts | +| `backoffFactor` | `number` | `1.5` | Backoff factor for exponential delay increase | +| `jitter` | `number` | `0.2` | Jitter factor (0-1) to add randomness to delay times | +| `isRetryable` | `IsRetryable` | `retryAllErrors` | Function to determine if an error is retriable | +| `logger` | `Logger` | `defaultLogger` | Logger to use | + +### Retry Condition Functions + +| Function | Description | +|----------|-------------| +| `retryAllErrors` | Retry on any error (default) | +| `retryNetworkErrors` | Retry only on network-related errors | + +## Examples + +### Basic Retry with Custom Options + +```typescript +import { withRetry } from '@platform/utils/retry' + +async function fetchDataWithRetry() { + return await withRetry( + async () => { + const response = await fetch('https://api.example.com/data') + if (!response.ok) { + throw new Error(`HTTP error: ${response.status}`) + } + return await response.json() + }, + { + initialDelayMs: 300, // Start with 300ms delay + maxDelayMs: 10000, // Max delay of 10 seconds + maxRetries: 4, // Try up to 4 times (1 initial + 3 retries) + backoffFactor: 2, // Double the delay each time + jitter: 0.25 // Add 25% randomness to delay + }, + 'fetchApiData' // Name for logging + ) +} +``` + +### Class with Multiple Retryable Methods + +```typescript +import { Retryable, retryNetworkErrors } from '@platform/utils/retry' + +class DataService { + @Retryable({ + maxRetries: 3, + initialDelayMs: 200 + }) + async fetchUsers(): Promise { + // Will retry up to 3 times with initial 200ms delay + return await this.api.getUsers() + } + + @Retryable({ + maxRetries: 5, + initialDelayMs: 1000, + isRetryable: retryNetworkErrors + }) + async uploadFile(file: File): Promise { + // Will retry up to 5 times, but only for network errors + return await this.api.uploadFile(file) + } +} +``` diff --git a/utils/retry/src/__test__/decorator.test.ts b/utils/retry/src/__test__/decorator.test.ts new file mode 100644 index 0000000000..1e1bdb4679 --- /dev/null +++ b/utils/retry/src/__test__/decorator.test.ts @@ -0,0 +1,388 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { DelayStrategyFactory } from '../delay' +import { Retryable } from '../decorator' +import { type RetryOptions } from '../retry' + +// Instead of mocking withRetry, we'll mock setTimeout to avoid waiting in tests +jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => { + fn() + return 1 as any +}) + +describe('Retryable decorator', () => { + const mockLogger = { + warn: jest.fn(), + error: jest.fn(), + info: jest.fn() + } + + // Update the mock options to use delay strategy + const mockOptions: Partial = { + maxRetries: 3, + delayStrategy: DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 10, + maxDelayMs: 100, + backoffFactor: 2 + }), + logger: mockLogger + } + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should retry failed operations', async () => { + // Create a test class with decorated method that fails initially then succeeds + const error = new Error('First attempt failed') + class TestService { + callCount = 0 + + @Retryable(mockOptions) + async testMethod (param1: string, param2: number): Promise { + this.callCount++ + if (this.callCount === 1) { + throw error + } + return `${param1}-${param2}` + } + } + + const service = new TestService() + const result = await service.testMethod('test', 123) + + // Check results + expect(result).toBe('test-123') + expect(service.callCount).toBe(2) // Called once, failed, then succeeded on retry + + // Check logs + expect(mockLogger.warn).toHaveBeenCalledTimes(1) + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('testMethod failed'), + expect.objectContaining({ + error, + attempt: 1 + }) + ) + expect(mockLogger.error).not.toHaveBeenCalled() + }) + + it('should work with default options', async () => { + class TestService { + callCount = 0 + + @Retryable() + async testMethod (): Promise { + this.callCount++ + if (this.callCount === 1) { + throw new Error('First attempt failed') + } + return 'success' + } + } + + const service = new TestService() + const result = await service.testMethod() + + expect(result).toBe('success') + expect(service.callCount).toBe(2) // Should have retried once + }) + + it('should preserve class instance context (this)', async () => { + class TestService { + private counter = 0 + + @Retryable(mockOptions) + async incrementAndGet (): Promise { + if (this.counter === 0) { + this.counter++ + throw new Error('First attempt failed') + } + this.counter++ + return this.counter + } + + getCounter (): number { + return this.counter + } + } + + const service = new TestService() + const result = await service.incrementAndGet() + + // Check that the class context was preserved across retries + expect(result).toBe(2) + expect(service.getCounter()).toBe(2) // Incremented once per attempt + }) + + it('should throw after max retries are exhausted', async () => { + class TestService { + @Retryable({ + maxRetries: 2, // Only try twice total (initial + 1 retry) + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 10 + }), + logger: mockLogger + }) + async alwaysFailingMethod (): Promise { + throw new Error('Persistent failure') + } + } + + const service = new TestService() + await expect(service.alwaysFailingMethod()).rejects.toThrow('Persistent failure') + + // Should have tried twice in total (initial + 1 retry) + expect(mockLogger.warn).toHaveBeenCalledTimes(1) + expect(mockLogger.error).toHaveBeenCalledTimes(1) + expect(mockLogger.error).toHaveBeenCalledWith( + expect.stringContaining('alwaysFailingMethod failed after 2 attempts'), + expect.any(Object) + ) + }) + + it('should handle async methods correctly', async () => { + let resolutionCount = 0 + + // Create a class with an async method that fails then resolves + class TestService { + @Retryable(mockOptions) + async delayedMethod (): Promise { + return await new Promise((resolve, reject) => { + resolutionCount++ + if (resolutionCount === 1) { + reject(new Error('Delayed error')) + } else { + resolve('delayed success') + } + }) + } + } + + const service = new TestService() + const result = await service.delayedMethod() + + expect(result).toBe('delayed success') + expect(resolutionCount).toBe(2) + expect(mockLogger.warn).toHaveBeenCalledTimes(1) + }) + + it('should retry according to the specified retry count', async () => { + let callCount = 0 + + class TestService { + @Retryable({ + maxRetries: 5, // Should try up to 5 times total + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 10 + }), + logger: mockLogger + }) + async unstableMethod (): Promise { + callCount++ + if (callCount < 4) { + // Succeed on the 4th attempt + throw new Error(`Failure #${callCount}`) + } + return 'success after retries' + } + } + + const service = new TestService() + const result = await service.unstableMethod() + + expect(result).toBe('success after retries') + expect(callCount).toBe(4) // Initial attempt + 3 retries = 4 total calls + expect(mockLogger.warn).toHaveBeenCalledTimes(3) // Should have logged 3 warnings + }) + + it('should respect different delay strategies', async () => { + // Override the setTimeout mock to capture delay values + const delayValues: number[] = [] + jest.spyOn(global, 'setTimeout').mockImplementation((fn: any, delay: number | undefined) => { + delayValues.push(delay ?? 0) + fn() + return 1 as any + }) + + let callCount = 0 + + class TestService { + @Retryable({ + maxRetries: 4, + delayStrategy: DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 100, + maxDelayMs: 500, + backoffFactor: 2, + jitter: 0 // Disable jitter for predictable tests + }) + }) + async delayingMethod (): Promise { + callCount++ + if (callCount < 4) { + throw new Error(`Attempt ${callCount} failed`) + } + return 'success' + } + } + + const service = new TestService() + await service.delayingMethod() + + // Should have recorded 3 delays: initial, 2x initial, 4x initial (capped at maxDelayMs) + expect(delayValues).toHaveLength(3) + expect(delayValues[0]).toBe(100) // initial delay + expect(delayValues[1]).toBe(200) // 2x initial + expect(delayValues[2]).toBe(400) // 4x initial + }) + + it('should test various delay strategies', async () => { + // Override the setTimeout mock to capture delay values + const delayValues: number[] = [] + jest.spyOn(global, 'setTimeout').mockImplementation((fn: any, delay: number | undefined) => { + delayValues.push(delay ?? 0) + fn() + return 1 as any + }) + + // Test Fixed strategy + let callCount = 0 + class FixedTestService { + @Retryable({ + maxRetries: 3, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 100, + jitter: 0 // Disable jitter for predictable tests + }) + }) + async method (): Promise { + callCount++ + if (callCount < 3) { + throw new Error(`Attempt ${callCount} failed`) + } + return 'success' + } + } + + delayValues.length = 0 // Reset captured delays + await new FixedTestService().method() + expect(delayValues).toEqual([100, 100]) // Should be constant + + // Test Fibonacci strategy + callCount = 0 + class FibonacciTestService { + @Retryable({ + maxRetries: 4, + delayStrategy: DelayStrategyFactory.fibonacci({ + baseDelayMs: 100, + maxDelayMs: 1000, + jitter: 0 // Disable jitter for predictable tests + }) + }) + async method (): Promise { + callCount++ + if (callCount < 4) { + throw new Error(`Attempt ${callCount} failed`) + } + return 'success' + } + } + + delayValues.length = 0 // Reset captured delays + await new FibonacciTestService().method() + // Fibonacci sequence delay pattern + expect(delayValues).toEqual([100, 200, 300]) // fib(2)=1, fib(3)=2, fib(4)=3 multiplied by baseDelayMs + }) + + it('should handle methods returning non-promises', async () => { + let callCount = 0 + + class TestService { + @Retryable(mockOptions) + nonAsyncMethod (input: string): string { + callCount++ + if (callCount === 1) { + throw new Error('Sync error') + } + return `processed-${input}` + } + } + + const service = new TestService() + // Even though the original method is not async, the decorated method returns a Promise + // eslint-disable-next-line @typescript-eslint/await-thenable + const result = await service.nonAsyncMethod('test') + + expect(result).toBe('processed-test') + expect(callCount).toBe(2) // Should have retried once + expect(mockLogger.warn).toHaveBeenCalledTimes(1) + }) + + it('should handle static methods', async () => { + let callCount = 0 + + // eslint-disable-next-line @typescript-eslint/no-extraneous-class + class TestService { + @Retryable(mockOptions) + static async staticMethod (input: string): Promise { + callCount++ + if (callCount === 1) { + throw new Error('Static method error') + } + return `static-${input}` + } + } + + const result = await TestService.staticMethod('test') + + expect(result).toBe('static-test') + expect(callCount).toBe(2) // Should have retried once + }) + + it('should respect isRetryable option', async () => { + class TestService { + callCount = 0 + + @Retryable({ + maxRetries: 3, + delayStrategy: DelayStrategyFactory.fixed({ delayMs: 10 }), + logger: mockLogger, + isRetryable: (err) => { + // Only retry errors with "retry" in the message + return err instanceof Error && err.message.includes('Please retry') + } + }) + async conditionalRetryMethod (): Promise { + this.callCount++ + if (this.callCount === 1) { + throw new Error('Please retry this') // should be retried + } + if (this.callCount === 2) { + throw new Error('Do not retry this') // should not be retried + } + return 'success' + } + } + + const service = new TestService() + // Should fail with the second error since it won't be retried + await expect(service.conditionalRetryMethod()).rejects.toThrow('Do not retry this') + + expect(service.callCount).toBe(2) // Should have called twice (original + 1 retry) + expect(mockLogger.warn).toHaveBeenCalledTimes(1) // Only first error logged as warning + expect(mockLogger.error).toHaveBeenCalledTimes(1) // Second error logged as error + }) +}) diff --git a/utils/retry/src/__test__/delay.test.ts b/utils/retry/src/__test__/delay.test.ts new file mode 100644 index 0000000000..67df348b8b --- /dev/null +++ b/utils/retry/src/__test__/delay.test.ts @@ -0,0 +1,296 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { DelayStrategyFactory, ExponentialBackoffStrategy, FibonacciDelayStrategy, FixedDelayStrategy } from '../delay' + +describe('Delay Strategies', () => { + // Mock Math.random to return fixed values in tests + let randomMock: jest.SpyInstance + + beforeEach(() => { + // Default mock returns 0.5 for Math.random, which creates 0 jitter effect + randomMock = jest.spyOn(Math, 'random').mockReturnValue(0.5) + }) + + afterEach(() => { + randomMock.mockRestore() + }) + + describe('FixedDelayStrategy', () => { + it('should return the same delay for all attempts without jitter', () => { + const delay = 1000 + const strategy = new FixedDelayStrategy({ delayMs: delay }) + + expect(strategy.getDelay(1)).toBe(delay) + expect(strategy.getDelay(2)).toBe(delay) + expect(strategy.getDelay(5)).toBe(delay) + expect(strategy.getDelay(10)).toBe(delay) + }) + + it('should apply jitter correctly', () => { + const delay = 1000 + const jitter = 0.2 // 20% jitter + const strategy = new FixedDelayStrategy({ delayMs: delay, jitter }) + + // Mock random to return different values + randomMock.mockReturnValueOnce(0.6) // => +0.2 * delay + randomMock.mockReturnValueOnce(0.4) // => -0.2 * delay + randomMock.mockReturnValueOnce(1.0) // => +1.0 * delay + + // With 0.6 random value, jitter effect is (0.6-0.5)*2*0.2 = +0.04 => 4% increase + expect(strategy.getDelay(1)).toBe(delay + delay * 0.2 * 0.2) + + // With 0.4 random value, jitter effect is (0.4-0.5)*2*0.2 = -0.04 => 4% decrease + expect(strategy.getDelay(2)).toBe(delay - delay * 0.2 * 0.2) + + // With 1.0 random value, jitter effect is (1.0-0.5)*2*0.2 = +0.2 => 20% increase + expect(strategy.getDelay(3)).toBe(delay + delay * 0.2 * 1.0) + }) + + it('should never return negative values even with high jitter', () => { + const delay = 100 + const jitter = 1.0 // 100% jitter, extreme case + const strategy = new FixedDelayStrategy({ delayMs: delay, jitter }) + + // Mock random to return min value (full negative jitter) + randomMock.mockReturnValue(0) // => -1.0 * delay + + // With 0 random value and 1.0 jitter, the effect would be -100%, but should be capped at 0 + expect(strategy.getDelay(1)).toBe(0) + }) + + it('should be creatable through factory method', () => { + const strategy = DelayStrategyFactory.fixed({ + delayMs: 1000, + jitter: 0.1 + }) + expect(strategy).toBeInstanceOf(FixedDelayStrategy) + expect(strategy.getDelay(1)).toBe(1000) // With 0.5 mock random, no jitter effect + }) + }) + + describe('ExponentialBackoffStrategy', () => { + it('should increase delay exponentially without jitter', () => { + const initial = 1000 + const max = 60000 + const factor = 2 + const strategy = new ExponentialBackoffStrategy({ + initialDelayMs: initial, + maxDelayMs: max, + backoffFactor: factor + }) + + expect(strategy.getDelay(1)).toBe(initial) // 1000 + expect(strategy.getDelay(2)).toBe(initial * factor) // 2000 + expect(strategy.getDelay(3)).toBe(initial * Math.pow(factor, 2)) // 4000 + expect(strategy.getDelay(4)).toBe(initial * Math.pow(factor, 3)) // 8000 + }) + + it('should respect maximum delay', () => { + const strategy = new ExponentialBackoffStrategy({ + initialDelayMs: 1000, + maxDelayMs: 5000, + backoffFactor: 2 + }) + + expect(strategy.getDelay(1)).toBe(1000) + expect(strategy.getDelay(2)).toBe(2000) + expect(strategy.getDelay(3)).toBe(4000) + // Should be capped at 5000 + expect(strategy.getDelay(4)).toBe(5000) + expect(strategy.getDelay(5)).toBe(5000) + }) + + it('should apply jitter correctly', () => { + const strategy = new ExponentialBackoffStrategy({ + initialDelayMs: 1000, + maxDelayMs: 60000, + backoffFactor: 2, + jitter: 0.2 + }) + + // Mock random to return different values + randomMock.mockReturnValueOnce(0.6) // => +0.2 * delay + randomMock.mockReturnValueOnce(0.4) // => -0.2 * delay + + // First attempt: 1000ms with jitter +4% + expect(strategy.getDelay(1)).toBe(1000 + 1000 * 0.2 * 0.2) + + // Second attempt: 2000ms with jitter -4% + expect(strategy.getDelay(2)).toBe(2000 - 2000 * 0.2 * 0.2) + }) + + it('should be creatable through factory method', () => { + const strategy = DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 1000, + maxDelayMs: 60000, + backoffFactor: 2, + jitter: 0.1 + }) + expect(strategy).toBeInstanceOf(ExponentialBackoffStrategy) + expect(strategy.getDelay(1)).toBe(1000) // With 0.5 mock random, no jitter effect + expect(strategy.getDelay(2)).toBe(2000) + }) + }) + + describe('FibonacciDelayStrategy', () => { + it('should follow Fibonacci sequence without jitter', () => { + const baseDelay = 100 + const maxDelay = 10000 + const strategy = new FibonacciDelayStrategy({ + baseDelayMs: baseDelay, + maxDelayMs: maxDelay + }) + + expect(strategy.getDelay(1)).toBe(baseDelay * 1) + expect(strategy.getDelay(2)).toBe(baseDelay * 2) + expect(strategy.getDelay(3)).toBe(baseDelay * 3) + expect(strategy.getDelay(4)).toBe(baseDelay * 5) + expect(strategy.getDelay(5)).toBe(baseDelay * 8) + expect(strategy.getDelay(6)).toBe(baseDelay * 13) + }) + + it('should respect maximum delay', () => { + const strategy = new FibonacciDelayStrategy({ + baseDelayMs: 100, + maxDelayMs: 500 + }) + + expect(strategy.getDelay(1)).toBe(100) + expect(strategy.getDelay(2)).toBe(200) + expect(strategy.getDelay(3)).toBe(300) + expect(strategy.getDelay(4)).toBe(500) + expect(strategy.getDelay(5)).toBe(500) // Capped at maxDelayMs + expect(strategy.getDelay(6)).toBe(500) // Capped at maxDelayMs + }) + + it('should cache Fibonacci calculations for performance', () => { + const strategy = new FibonacciDelayStrategy({ + baseDelayMs: 100, + maxDelayMs: 10000 + }) + + // Access private cache to verify it's working + const fibCache = (strategy as any).fibCache + expect(fibCache.size).toBe(2) // Initial cache has 0->0 and 1->1 + + strategy.getDelay(7) // Should calculate fib(8) = 21 + + expect(fibCache.size).toBeGreaterThan(2) + expect(fibCache.get(8)).toBe(21) + }) + + it('should apply jitter correctly', () => { + const strategy = new FibonacciDelayStrategy({ + baseDelayMs: 100, + maxDelayMs: 10000, + jitter: 0.2 + }) + + // Mock random to return different values + randomMock.mockReturnValueOnce(0.6) // => +0.2 * delay + randomMock.mockReturnValueOnce(0.4) // => -0.2 * delay + + // First attempt: 100ms (fib(2)=1 * 100) with jitter +4% + expect(strategy.getDelay(1)).toBe(100 + 100 * 0.2 * 0.2) + + // Second attempt: 100ms (fib(3)=1 * 100) with jitter -4% + expect(strategy.getDelay(2)).toBe(200 - 200 * 0.2 * 0.2) + }) + + it('should handle large Fibonacci numbers efficiently', () => { + const strategy = new FibonacciDelayStrategy({ + baseDelayMs: 1, + maxDelayMs: Number.MAX_SAFE_INTEGER + }) + + // This would be extremely slow without memoization + const start = performance.now() + const delay = strategy.getDelay(40) // fib(41) = 165580141 + const duration = performance.now() - start + + // Should be much faster than calculating naively + expect(duration).toBeLessThan(100) + expect(delay).toBe(165580141) // fib(41) * 1 + }) + + it('should be creatable through factory method', () => { + const strategy = DelayStrategyFactory.fibonacci({ + baseDelayMs: 100, + maxDelayMs: 10000, + jitter: 0.1 + }) + expect(strategy).toBeInstanceOf(FibonacciDelayStrategy) + expect(strategy.getDelay(1)).toBe(100) + expect(strategy.getDelay(2)).toBe(200) + expect(strategy.getDelay(3)).toBe(300) + expect(strategy.getDelay(4)).toBe(500) + }) + }) + + describe('DelayStrategyFactory', () => { + it('should create strategies with correct types', () => { + expect( + DelayStrategyFactory.fixed({ + delayMs: 1000 + }) + ).toBeInstanceOf(FixedDelayStrategy) + + expect( + DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 1000, + maxDelayMs: 60000, + backoffFactor: 2 + }) + ).toBeInstanceOf(ExponentialBackoffStrategy) + + expect( + DelayStrategyFactory.fibonacci({ + baseDelayMs: 100, + maxDelayMs: 10000 + }) + ).toBeInstanceOf(FibonacciDelayStrategy) + }) + + it('should pass parameters correctly to strategies', () => { + const fixed = DelayStrategyFactory.fixed({ + delayMs: 1000, + jitter: 0.1 + }) as FixedDelayStrategy + expect((fixed as any).delayMs).toBe(1000) + expect((fixed as any).jitter).toBe(0.1) + + const exponential = DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 1000, + maxDelayMs: 60000, + backoffFactor: 2.5, + jitter: 0.2 + }) as ExponentialBackoffStrategy + expect((exponential as any).initialDelayMs).toBe(1000) + expect((exponential as any).maxDelayMs).toBe(60000) + expect((exponential as any).backoffFactor).toBe(2.5) + expect((exponential as any).jitter).toBe(0.2) + + const fibonacci = DelayStrategyFactory.fibonacci({ + baseDelayMs: 100, + maxDelayMs: 10000, + jitter: 0.15 + }) as FibonacciDelayStrategy + expect((fibonacci as any).baseDelayMs).toBe(100) + expect((fibonacci as any).maxDelayMs).toBe(10000) + expect((fibonacci as any).jitter).toBe(0.15) + }) + }) +}) diff --git a/utils/retry/src/__test__/retry.test.ts b/utils/retry/src/__test__/retry.test.ts new file mode 100644 index 0000000000..07bc982f3e --- /dev/null +++ b/utils/retry/src/__test__/retry.test.ts @@ -0,0 +1,556 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { DelayStrategyFactory } from '../delay' +import { withRetry, createRetryableFunction, type RetryOptions } from '../retry' +import { type IsRetryable, retryAllErrors } from '../retryable' + +// Mock the sleep function to speed up tests +jest.mock('../delay', () => { + const originalModule = jest.requireActual('../delay') + return { + ...originalModule, + // Override the internal sleep function to resolve immediately + sleep: jest.fn().mockImplementation(() => Promise.resolve()) + } +}) + +describe('withRetry', () => { + // Create a mock logger to capture logs + const mockLogger = { + warn: jest.fn(), + error: jest.fn(), + info: jest.fn() + } + + // Use the new delayStrategy option + const mockOptions: Partial = { + maxRetries: 3, + delayStrategy: DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 10, + maxDelayMs: 100, + backoffFactor: 2, + jitter: 0 + }), + logger: mockLogger, + isRetryable: retryAllErrors + } + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should return the result when operation succeeds on first try', async () => { + const mockOperation = jest.fn().mockResolvedValue('success') + + const result = await withRetry(mockOperation, mockOptions) + + expect(result).toBe('success') + expect(mockOperation).toHaveBeenCalledTimes(1) + expect(mockLogger.warn).not.toHaveBeenCalled() + expect(mockLogger.error).not.toHaveBeenCalled() + }) + + it('should retry when operation fails and eventually succeed', async () => { + const mockOperation = jest + .fn() + .mockRejectedValueOnce(new Error('first failure')) + .mockRejectedValueOnce(new Error('second failure')) + .mockResolvedValueOnce('success after retries') + + const result = await withRetry(mockOperation, mockOptions) + + expect(result).toBe('success after retries') + expect(mockOperation).toHaveBeenCalledTimes(3) + expect(mockLogger.warn).toHaveBeenCalledTimes(2) + expect(mockLogger.error).not.toHaveBeenCalled() + }) + + it('should throw an error after maximum retries are exhausted', async () => { + const mockError = new Error('persistent failure') + const mockOperation = jest.fn().mockRejectedValue(mockError) + + await expect(withRetry(mockOperation, mockOptions)).rejects.toThrow('persistent failure') + + expect(mockOperation).toHaveBeenCalledTimes(mockOptions.maxRetries ?? -1) + expect(mockLogger.error).toHaveBeenCalledTimes(1) + }) + + it('should use default options when none are provided', async () => { + const mockOperation = jest.fn().mockResolvedValue('success') + + const result = await withRetry(mockOperation) + + expect(result).toBe('success') + expect(mockOperation).toHaveBeenCalledTimes(1) + }) + + it('should use provided operation name in log messages', async () => { + const mockOperation = jest.fn().mockRejectedValueOnce(new Error('failure')).mockResolvedValueOnce('success') + + await withRetry(mockOperation, mockOptions, 'custom-operation') + + expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('custom-operation failed'), expect.any(Object)) + }) + + it('should apply jitter to delay calculation', async () => { + const mockOperation = jest.fn().mockRejectedValueOnce(new Error('failure')).mockResolvedValueOnce('success') + + // Use Math.random mock to make jitter predictable + const mockRandom = jest.spyOn(Math, 'random').mockReturnValue(0.5) + + // Create options with jitter enabled + const jitterOptions = { + ...mockOptions, + delayStrategy: DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 10, + maxDelayMs: 100, + backoffFactor: 2, + jitter: 0.2 + }) + } + + await withRetry(mockOperation, jitterOptions) + + // With Math.random = 0.5, jitter should be 0 + // (since 0.5 * 2 - 1 = 0) + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ delayMs: 10 }) // Should still be the initial delay + ) + + mockRandom.mockRestore() + }) + + it('should respect maximum delay', async () => { + const mockOperation = jest + .fn() + .mockRejectedValueOnce(new Error('failure 1')) + .mockRejectedValueOnce(new Error('failure 2')) + .mockRejectedValueOnce(new Error('failure 3')) + .mockResolvedValueOnce('success') + + // Use high backoff factor to test maximum delay cap + const maxDelayOptions = { + maxRetries: 4, + delayStrategy: DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 50, + maxDelayMs: 1000, + backoffFactor: 10 // Would normally go 50 -> 500 -> 5000, but should cap at 1000 + }), + logger: mockLogger + } + + await withRetry(mockOperation, maxDelayOptions) + + // Check that delays are correctly calculated and capped + expect(mockLogger.warn).toHaveBeenNthCalledWith( + 1, + expect.any(String), + expect.objectContaining({ delayMs: expect.any(Number) }) + ) + + // Second retry delay + expect(mockLogger.warn).toHaveBeenNthCalledWith( + 2, + expect.any(String), + expect.objectContaining({ delayMs: 500 }) // 50 * 10 = 500 + ) + + // Third retry delay (should be capped) + expect(mockLogger.warn).toHaveBeenNthCalledWith( + 3, + expect.any(String), + expect.objectContaining({ delayMs: 1000 }) // 500 * 10 = 5000, capped at 1000 + ) + + // Function should have been called 4 times total + expect(mockOperation).toHaveBeenCalledTimes(4) + }) + + it('should work with different delay strategies', async () => { + // Test with fixed delay + const fixedDelayOperation = jest + .fn() + .mockRejectedValueOnce(new Error('failure 1')) + .mockRejectedValueOnce(new Error('failure 2')) + .mockResolvedValueOnce('success') + + const fixedDelayOptions = { + maxRetries: 3, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 200, + jitter: 0 + }), + logger: mockLogger + } + + await withRetry(fixedDelayOperation, fixedDelayOptions) + + // Both retries should have the same delay + expect(mockLogger.warn).toHaveBeenNthCalledWith(1, expect.any(String), expect.objectContaining({ delayMs: 200 })) + expect(mockLogger.warn).toHaveBeenNthCalledWith(2, expect.any(String), expect.objectContaining({ delayMs: 200 })) + + // Clear mocks for next test + jest.clearAllMocks() + + // Test with Fibonacci delay + const fibonacciDelayOperation = jest + .fn() + .mockRejectedValueOnce(new Error('failure 1')) + .mockRejectedValueOnce(new Error('failure 2')) + .mockResolvedValueOnce('success') + + const fibonacciDelayOptions = { + maxRetries: 3, + delayStrategy: DelayStrategyFactory.fibonacci({ + baseDelayMs: 100, + maxDelayMs: 10000, + jitter: 0 + }), + logger: mockLogger + } + + await withRetry(fibonacciDelayOperation, fibonacciDelayOptions) + + // Delays should follow Fibonacci sequence + expect(mockLogger.warn).toHaveBeenNthCalledWith( + 1, + expect.any(String), + expect.objectContaining({ delayMs: 100 }) // fib(2) = 1 * 100 + ) + expect(mockLogger.warn).toHaveBeenNthCalledWith( + 2, + expect.any(String), + expect.objectContaining({ delayMs: 200 }) // fib(3) = 2 * 100 + ) + }) +}) + +describe('createRetryableFunction', () => { + const mockLogger = { + warn: jest.fn(), + error: jest.fn(), + info: jest.fn() + } + + const mockOptions: Partial = { + maxRetries: 2, + delayStrategy: DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 10, + maxDelayMs: 100, + backoffFactor: 2 + }), + logger: mockLogger + } + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should create a function that applies retry logic', async () => { + const mockFn = jest.fn().mockRejectedValueOnce(new Error('first failure')).mockResolvedValueOnce('success') + + const retryableFn = createRetryableFunction(mockFn, mockOptions) + + const result = await retryableFn('arg1', 123) + + expect(result).toBe('success') + expect(mockFn).toHaveBeenCalledTimes(2) + expect(mockFn).toHaveBeenCalledWith('arg1', 123) + expect(mockLogger.warn).toHaveBeenCalledTimes(1) + }) + + it('should pass through function parameters correctly', async () => { + const mockFn = jest.fn().mockResolvedValue('success') + + const retryableFn = createRetryableFunction(mockFn, mockOptions) + + await retryableFn('arg1', 123, { complex: true }) + + expect(mockFn).toHaveBeenCalledWith('arg1', 123, { complex: true }) + }) + + it('should use custom operation name in logs', async () => { + const mockFn = jest.fn().mockRejectedValueOnce(new Error('failure')).mockResolvedValueOnce('success') + + const retryableFn = createRetryableFunction(mockFn, mockOptions, 'custom-function') + + await retryableFn() + + expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('custom-function failed'), expect.any(Object)) + }) + + it('should propagate the final error if all retries fail', async () => { + const mockError = new Error('persistent failure') + const mockFn = jest.fn().mockRejectedValue(mockError) + + const retryableFn = createRetryableFunction(mockFn, mockOptions) + + await expect(retryableFn()).rejects.toThrow('persistent failure') + + expect(mockOptions.maxRetries).toBeDefined() + expect(mockFn).toHaveBeenCalledTimes(mockOptions.maxRetries ?? -1) + }) +}) + +// Test with a decorated class +describe('Using retry in class methods', () => { + class TestService { + counter = 0 + + async unstableFunction (): Promise { + this.counter++ + if (this.counter < 3) { + throw new Error(`Failure attempt ${this.counter}`) + } + return 'success' + } + } + + it('should work with instance methods', async () => { + const service = new TestService() + + // Create a retryable version of the method that's bound to the service + const retryableMethod = createRetryableFunction(service.unstableFunction.bind(service), { + maxRetries: 3, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 10 + }) + }) + + const result = await retryableMethod() + + expect(result).toBe('success') + expect(service.counter).toBe(3) + }) +}) + +describe('withRetry with isRetryable option', () => { + // Create a mock logger to capture logs + const mockLogger = { + warn: jest.fn(), + error: jest.fn(), + info: jest.fn() + } + + beforeEach(() => { + jest.clearAllMocks() + // Mock the sleep function to speed up tests + jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => { + fn() + return 1 as any + }) + }) + + it('should retry errors that are marked as retriable', async () => { + // Custom isRetryable function that only retries certain errors + const customRetriable: IsRetryable = (err: any) => { + return err.message.includes('retriable') + } + + const mockOperation = jest + .fn() + .mockRejectedValueOnce(new Error('This is a retriable error')) + .mockRejectedValueOnce(new Error('This is a retriable error again')) + .mockResolvedValueOnce('success') + + const result = await withRetry(mockOperation, { + maxRetries: 5, + logger: mockLogger, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 10 + }), + isRetryable: customRetriable + }) + + expect(result).toBe('success') + expect(mockOperation).toHaveBeenCalledTimes(3) + expect(mockLogger.warn).toHaveBeenCalledTimes(2) + expect(mockLogger.error).not.toHaveBeenCalled() + }) + + it('should not retry errors that are not marked as retriable', async () => { + // Custom isRetryable function that never retries + const neverRetry: IsRetryable = (_err: any) => { + return false + } + + const mockOperation = jest + .fn() + .mockRejectedValueOnce(new Error('This error should not be retried')) + .mockResolvedValueOnce('success') + + await expect( + withRetry(mockOperation, { + maxRetries: 5, + logger: mockLogger, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 10 + }), + isRetryable: neverRetry + }) + ).rejects.toThrow('This error should not be retried') + + expect(mockOperation).toHaveBeenCalledTimes(1) + expect(mockLogger.warn).not.toHaveBeenCalled() + expect(mockLogger.error).toHaveBeenCalledWith( + expect.stringContaining('failed with non-retriable error'), + expect.any(Object) + ) + }) + + it('should have different behavior for different error types', async () => { + // Custom isRetryable function that retries only NetworkErrors + const retryOnlyNetworkErrors: IsRetryable = (err: any) => { + return err.name === 'NetworkError' + } + + // Create different error types + const networkError = new Error('Network failed') + networkError.name = 'NetworkError' + + const validationError = new Error('Validation failed') + validationError.name = 'ValidationError' + + // First test with network error - should be retried + const mockNetworkOp = jest.fn().mockRejectedValueOnce(networkError).mockResolvedValueOnce('network success') + + const result1 = await withRetry(mockNetworkOp, { + maxRetries: 3, + logger: mockLogger, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 10 + }), + isRetryable: retryOnlyNetworkErrors + }) + + expect(result1).toBe('network success') + expect(mockNetworkOp).toHaveBeenCalledTimes(2) + + // Reset mocks + jest.clearAllMocks() + + // Then test with validation error - should not be retried + const mockValidationOp = jest.fn().mockRejectedValueOnce(validationError) + + await expect( + withRetry(mockValidationOp, { + maxRetries: 3, + logger: mockLogger, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 10 + }), + isRetryable: retryOnlyNetworkErrors + }) + ).rejects.toThrow('Validation failed') + + expect(mockValidationOp).toHaveBeenCalledTimes(1) + expect(mockLogger.error).toHaveBeenCalledWith(expect.stringContaining('non-retriable error'), expect.any(Object)) + }) + + it('should use the default retryAllErrors if isRetryable is not provided', async () => { + // All errors should be retried by default + const mockOperation = jest + .fn() + .mockRejectedValueOnce(new Error('error 1')) + .mockRejectedValueOnce(new Error('error 2')) + .mockResolvedValueOnce('success') + + const result = await withRetry(mockOperation, { + maxRetries: 5, + logger: mockLogger, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 10 + }) + // isRetryable not provided, should use default + }) + + expect(result).toBe('success') + expect(mockOperation).toHaveBeenCalledTimes(3) + expect(mockLogger.warn).toHaveBeenCalledTimes(2) + }) + + it('should pass the error to isRetryable function', async () => { + // Mock isRetryable function to track calls + const mockisRetryable = jest.fn().mockReturnValue(true) + + const testError = new Error('test error') + const mockOperation = jest.fn().mockRejectedValueOnce(testError).mockResolvedValueOnce('success') + + await withRetry(mockOperation, { + maxRetries: 3, + logger: mockLogger, + delayStrategy: DelayStrategyFactory.fixed({ + delayMs: 10 + }), + isRetryable: mockisRetryable + }) + + // Verify isRetryable was called with the actual error + expect(mockisRetryable).toHaveBeenCalledTimes(1) + expect(mockisRetryable).toHaveBeenCalledWith(testError) + }) +}) + +describe('createRetryableFunction with isRetryable', () => { + const mockLogger = { + warn: jest.fn(), + error: jest.fn(), + info: jest.fn() + } + + beforeEach(() => { + jest.clearAllMocks() + // Mock the sleep function to speed up tests + jest.spyOn(global, 'setTimeout').mockImplementation((fn: any) => { + fn() + return 1 as any + }) + }) + + it('should respect isRetryable when applied to a function', async () => { + // Function to wrap + const unstableFunction = jest + .fn() + .mockRejectedValueOnce(new Error('retriable network error')) + .mockRejectedValueOnce(new Error('retriable network error')) + .mockResolvedValueOnce('success') + + // Custom isRetryable that only retries network errors + const customRetriable: IsRetryable = (err: any) => { + return err.message.includes('network') + } + + // Create retryable version with custom isRetryable + const retryableFunction = createRetryableFunction( + unstableFunction, + { + maxRetries: 3, + logger: mockLogger, + delayStrategy: DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 10, + maxDelayMs: 100, + backoffFactor: 2 + }), + isRetryable: customRetriable + }, + 'custom-operation' + ) + + const result = await retryableFunction() + expect(result).toBe('success') + }) +}) diff --git a/utils/retry/src/__test__/retryable.test.ts b/utils/retry/src/__test__/retryable.test.ts new file mode 100644 index 0000000000..8814cc8008 --- /dev/null +++ b/utils/retry/src/__test__/retryable.test.ts @@ -0,0 +1,158 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { retryAllErrors, retryNetworkErrors } from '../retryable' + +describe('retryAllErrors', () => { + it('should return true for any error', () => { + expect(retryAllErrors(new Error('any error'))).toBe(true) + expect(retryAllErrors(new TypeError('type error'))).toBe(true) + expect(retryAllErrors(null)).toBe(true) + expect(retryAllErrors(undefined)).toBe(true) + expect(retryAllErrors({ custom: 'error object' })).toBe(true) + expect(retryAllErrors('error string')).toBe(true) + }) +}) + +describe('retryNetworkErrors', () => { + it('should return false for null or undefined', () => { + expect(retryNetworkErrors(null)).toBe(false) + expect(retryNetworkErrors(undefined)).toBe(false) + }) + + it('should return true for errors with network-related names', () => { + const networkErrorNames = [ + 'NetworkError', + 'FetchError', + 'AbortError', + 'TimeoutError', + 'ConnectionError', + 'ConnectionRefusedError', + 'ETIMEDOUT', + 'ECONNREFUSED', + 'ECONNRESET', + 'ENOTFOUND', + 'EAI_AGAIN' + ] + + networkErrorNames.forEach((name) => { + // Create an error with the specified name + const error = new Error('Test error') + error.name = name + expect(retryNetworkErrors(error)).toBe(true) + }) + }) + + it('should return true for errors with network-related message patterns', () => { + const networkErrorMessages = [ + 'Network error occurred', + 'Connection timed out', + 'Connection refused', + 'Connection reset', + 'Socket hang up', + 'DNS lookup failed', + 'getaddrinfo ENOTFOUND api.example.com', + 'connect ECONNREFUSED 127.0.0.1:8080', + 'read ECONNRESET', + 'connect ETIMEDOUT 192.168.1.1:443', + 'getaddrinfo EAI_AGAIN myserver.local' + ] + + networkErrorMessages.forEach((message) => { + expect(retryNetworkErrors(new Error(message))).toBe(true) + }) + }) + + it('should return false for non-network related errors', () => { + const nonNetworkErrors = [ + new Error('Invalid input'), + new TypeError('Cannot read property of undefined'), + new RangeError('Value out of range'), + new SyntaxError('Unexpected token'), + new Error('File not found'), + new Error('Permission denied'), + new Error('Invalid state') + ] + + nonNetworkErrors.forEach((error) => { + expect(retryNetworkErrors(error)).toBe(false) + }) + }) + + it('should return true for errors with server error status codes (5xx)', () => { + const serverErrors = [ + createErrorWithStatus(500), + createErrorWithStatus(501), + createErrorWithStatus(502), + createErrorWithStatus(503), + createErrorWithStatus(504), + createErrorWithStatus(599) + ] + + serverErrors.forEach((error) => { + expect(retryNetworkErrors(error)).toBe(true) + }) + }) + + it('should return true for specific client error status codes', () => { + const retriableClientErrors = [ + createErrorWithStatus(408), // Request Timeout + createErrorWithStatus(423), // Locked + createErrorWithStatus(425), // Too Early + createErrorWithStatus(429), // Too Many Requests + createErrorWithStatus(449) // Retry With + ] + + retriableClientErrors.forEach((error) => { + expect(retryNetworkErrors(error)).toBe(true) + }) + }) + + it('should return false for non-retriable client error status codes', () => { + const nonRetriableClientErrors = [ + createErrorWithStatus(400), // Bad Request + createErrorWithStatus(401), // Unauthorized + createErrorWithStatus(403), // Forbidden + createErrorWithStatus(404), // Not Found + createErrorWithStatus(422) // Unprocessable Entity + ] + + nonRetriableClientErrors.forEach((error) => { + expect(retryNetworkErrors(error)).toBe(false) + }) + }) + + it('should return false for non-Error objects without network-related properties', () => { + const nonNetworkErrorObjects = [ + { code: 'INVALID_INPUT' }, + { code: 'AUTH_FAILED' }, + { message: 'Invalid credentials' }, + { error: 'Not found' } + ] + + nonNetworkErrorObjects.forEach((errorObj) => { + expect(retryNetworkErrors(errorObj)).toBe(false) + }) + }) +}) + +/** + * Helper function to create an Error object with a status property + */ +function createErrorWithStatus (status: number): Error { + const error: any = new Error(`HTTP Error ${status}`) + error.status = status + return error +} diff --git a/utils/retry/src/decorator.ts b/utils/retry/src/decorator.ts new file mode 100644 index 0000000000..cb6080e0ae --- /dev/null +++ b/utils/retry/src/decorator.ts @@ -0,0 +1,35 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { type RetryOptions, DEFAULT_RETRY_OPTIONS, withRetry } from './retry' + +/** + * Method decorator for adding retry functionality to class methods + * + * @param options - Retry configuration options + * @param operationName - Name of the operation for logging (defaults to method name) + * @returns Method decorator + */ +export function Retryable (options: Partial = DEFAULT_RETRY_OPTIONS): MethodDecorator { + return function (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value as (...args: any[]) => any + + descriptor.value = async function (...args: any[]) { + const methodName = propertyKey.toString() + return await withRetry(() => originalMethod.apply(this, args), options, methodName) + } + + return descriptor + } +} diff --git a/utils/retry/src/delay.ts b/utils/retry/src/delay.ts new file mode 100644 index 0000000000..5adbd17d36 --- /dev/null +++ b/utils/retry/src/delay.ts @@ -0,0 +1,171 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export const DelayStrategyFactory = { + /** + * Create a fixed delay strategy + */ + fixed (options: FixedDelayOptions): DelayStrategy { + return new FixedDelayStrategy(options) + }, + + /** + * Create an exponential backoff delay strategy + */ + exponentialBackoff (options: ExponentialBackoffOptions): DelayStrategy { + return new ExponentialBackoffStrategy(options) + }, + + /** + * Create a Fibonacci delay strategy + */ + fibonacci (options: FibonacciDelayOptions): DelayStrategy { + return new FibonacciDelayStrategy(options) + } +} + +export interface DelayStrategy { + getDelay: (attempt: number) => number +} + +export interface FixedDelayOptions { + /** Delay between retries in milliseconds */ + delayMs: number + /** Optional jitter factor (0-1) to add randomness to delay times */ + jitter?: number +} + +/** + * Fixed delay strategy - uses the same delay for all retry attempts + */ +export class FixedDelayStrategy implements DelayStrategy { + private readonly delayMs: number + private readonly jitter: number + + constructor (options: FixedDelayOptions) { + this.delayMs = options.delayMs + this.jitter = options.jitter ?? 0 + } + + getDelay (_attempt: number): number { + if (this.jitter > 0) { + const jitterAmount = this.delayMs * this.jitter * (Math.random() * 2 - 1) + return Math.max(0, this.delayMs + jitterAmount) + } + return this.delayMs + } +} + +export interface ExponentialBackoffOptions { + /** Initial delay between retries in milliseconds */ + initialDelayMs: number + /** Maximum delay between retries in milliseconds */ + maxDelayMs: number + /** Backoff factor for exponential delay increase */ + backoffFactor: number + /** Optional jitter factor (0-1) to add randomness to delay times */ + jitter?: number +} + +/** + * Exponential backoff delay strategy - increases delay exponentially with each attempt + */ +export class ExponentialBackoffStrategy implements DelayStrategy { + private readonly initialDelayMs: number + private readonly maxDelayMs: number + private readonly backoffFactor: number + private readonly jitter: number + + constructor (options: ExponentialBackoffOptions) { + this.initialDelayMs = options.initialDelayMs + this.maxDelayMs = options.maxDelayMs + this.backoffFactor = options.backoffFactor + this.jitter = options.jitter ?? 0 + } + + getDelay (attempt: number): number { + const baseDelay = Math.min(this.initialDelayMs * Math.pow(this.backoffFactor, attempt - 1), this.maxDelayMs) + + if (this.jitter > 0) { + const jitterAmount = baseDelay * this.jitter * (Math.random() * 2 - 1) + return Math.min(Math.max(0, baseDelay + jitterAmount), this.maxDelayMs) + } + + return baseDelay + } +} + +export interface FibonacciDelayOptions { + /** Base unit for calculating Fibonacci sequence in milliseconds */ + baseDelayMs: number + /** Maximum delay between retries in milliseconds */ + maxDelayMs: number + /** Optional jitter factor (0-1) to add randomness to delay times */ + jitter?: number +} + +export class FibonacciDelayStrategy implements DelayStrategy { + private readonly baseDelayMs: number + private readonly maxDelayMs: number + private readonly jitter: number + + // Cache for Fibonacci numbers to improve performance + private readonly fibCache: Map + + constructor (options: FibonacciDelayOptions) { + this.baseDelayMs = options.baseDelayMs + this.maxDelayMs = options.maxDelayMs + this.jitter = options.jitter ?? 0 + this.fibCache = new Map([ + [0, 0], + [1, 1] + ]) + } + + private fibonacci (n: number): number { + // Return from cache if available + if (this.fibCache.has(n)) { + return this.fibCache.get(n) as number + } + + if (n <= 1) { + return n + } + + // Calculate using recursion with memoization + const result = this.fibonacci(n - 1) + this.fibonacci(n - 2) + this.fibCache.set(n, result) + return result + } + + getDelay (attempt: number): number { + const fibNumber = this.fibonacci(attempt + 1) + const baseDelay = Math.min(fibNumber * this.baseDelayMs, this.maxDelayMs) + + if (this.jitter > 0) { + const jitterAmount = baseDelay * this.jitter * (Math.random() * 2 - 1) + return Math.min(Math.max(0, baseDelay + jitterAmount), this.maxDelayMs) + } + + return baseDelay + } +} + +/** + * Promise-based sleep function + */ +export function sleep (ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/utils/retry/src/index.ts b/utils/retry/src/index.ts new file mode 100644 index 0000000000..3ce12b411b --- /dev/null +++ b/utils/retry/src/index.ts @@ -0,0 +1,18 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export * from './retry' +export * from './decorator' +export * from './retryable' diff --git a/utils/retry/src/logger.ts b/utils/retry/src/logger.ts new file mode 100644 index 0000000000..6bf7b99066 --- /dev/null +++ b/utils/retry/src/logger.ts @@ -0,0 +1,31 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +export interface Logger { + warn: (message: string, meta?: Record) => void + error: (message: string, meta?: Record) => void + info: (message: string, meta?: Record) => void +} + +export const defaultLogger: Logger = { + warn: (message: string, meta?: Record) => { + console.warn(`[WARN] ${message}`, meta) + }, + error: (message: string, meta?: Record) => { + console.error(`[ERROR] ${message}`, meta) + }, + info: (message: string, meta?: Record) => { + console.info(`[INFO] ${message}`, meta) + } +} diff --git a/utils/retry/src/retry.ts b/utils/retry/src/retry.ts new file mode 100644 index 0000000000..15f8d4550c --- /dev/null +++ b/utils/retry/src/retry.ts @@ -0,0 +1,128 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { defaultLogger, type Logger } from './logger' +import { type IsRetryable, retryAllErrors } from './retryable' +import { type DelayStrategy, DelayStrategyFactory, sleep } from './delay' + +/** + * Configuration options for the retry mechanism + */ +export interface RetryOptions { + /** Maximum number of retry attempts */ + maxRetries: number + /** Function to determine if an error is retriable */ + isRetryable: IsRetryable + /** Strategy for calculating delay between retries */ + delayStrategy: DelayStrategy + /** Logger to use (defaults to console logger) */ + logger?: Logger +} + +/** + * Default retry options + */ +export const DEFAULT_RETRY_OPTIONS: RetryOptions = { + maxRetries: 5, + isRetryable: retryAllErrors, + delayStrategy: DelayStrategyFactory.exponentialBackoff({ + initialDelayMs: 1000, + maxDelayMs: 30000, + backoffFactor: 1.5, + jitter: 0.2 + }), + logger: defaultLogger +} + +/** + * Executes an operation with exponential backoff retry + * + * @param operation - Async operation to execute + * @param options - Retry configuration options + * @param operationName - Name of the operation for logging + * @returns The result of the operation + * @throws The last error encountered after all retries have been exhausted + */ +export async function withRetry ( + operation: () => Promise, + options: Partial = {}, + operationName: string = 'operation' +): Promise { + const config: RetryOptions = { ...DEFAULT_RETRY_OPTIONS, ...options } + const logger = config.logger ?? defaultLogger + let attempt = 1 + let lastError: Error | undefined + + while (attempt <= config.maxRetries) { + try { + return await operation() + } catch (error: any) { + lastError = error + const isLastAttempt = attempt >= config.maxRetries + + if (isLastAttempt) { + logger.error(`${operationName} failed after ${attempt} attempts`, { + error, + attempt, + maxRetries: config.maxRetries + }) + throw error + } + if (!config.isRetryable(error)) { + logger.error(`${operationName} failed with non-retriable error`, { + error, + attempt, + maxRetries: config.maxRetries + }) + throw error + } + + // Get delay for next attempt from strategy + const delayMs = Math.round(config.delayStrategy.getDelay(attempt)) + + logger.warn(`${operationName} failed, retrying in ${delayMs}ms`, { + error, + attempt, + nextAttempt: attempt + 1, + delayMs + }) + + // Wait before retry + await sleep(delayMs) + attempt++ + } + } + + // This should not be reached due to the throw in the last iteration + throw lastError ?? new Error(`${operationName} failed for unknown reason`) +} + +/** + * Creates a retryable function from a base function + * Returns a wrapped function that will apply retry logic automatically + * + * @param fn - The function to make retryable + * @param options - Retry configuration options + * @param operationName - Name of the operation for logging + * @returns A wrapped function that applies retry logic + */ +export function createRetryableFunction Promise> ( + fn: T, + options: Partial = {}, + operationName: string = 'operation' +): T { + return (async (...args: Parameters): Promise> => { + return await (withRetry(() => fn(...args), options, operationName) as Promise>) + }) as T +} diff --git a/utils/retry/src/retryable.ts b/utils/retry/src/retryable.ts new file mode 100644 index 0000000000..d7e5ac2efb --- /dev/null +++ b/utils/retry/src/retryable.ts @@ -0,0 +1,96 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export type IsRetryable = (error: Error | unknown) => boolean + +export const retryAllErrors: IsRetryable = (_error: Error | unknown): boolean => { + return true +} + +const NETWORK_ERROR_NAMES = new Set([ + 'NetworkError', + 'FetchError', + 'AbortError', + 'TimeoutError', + 'ConnectionError', + 'ConnectionRefusedError', + 'ETIMEDOUT', + 'ECONNREFUSED', + 'ECONNRESET', + 'ENOTFOUND', + 'EAI_AGAIN' +]) + +/** + * Patterns in error messages that suggest network issues + */ +const NETWORK_ERROR_PATTERNS = [ + /network/i, + /connection/i, + /timeout/i, + /unreachable/i, + /refused/i, + /reset/i, + /socket/i, + /DNS/i, + /ENOTFOUND/, + /ECONNREFUSED/, + /ECONNRESET/, + /ETIMEDOUT/, + /EAI_AGAIN/ +] + +/** + * Determine if an error is related to network issues + */ +export const retryNetworkErrors: IsRetryable = (error: Error | unknown): boolean => { + if (error == null) { + return false + } + + // Check for error name + if (error instanceof Error) { + // Check if the error name is in our set of network errors + if (NETWORK_ERROR_NAMES.has(error.name)) { + return true + } + + // Check if the error message matches our network error patterns + for (const pattern of NETWORK_ERROR_PATTERNS) { + if (pattern.test(error.message)) { + return true + } + } + + // Check for status codes in response errors + if ('status' in error && typeof (error as any).status === 'number') { + const status = (error as any).status + // Retry server errors (5xx) and some specific client errors + return ( + (status >= 500 && status < 600) || // Server errors + status === 429 || // Too Many Requests + status === 408 || // Request Timeout + status === 423 || // Locked + status === 425 || // Too Early + status === 449 || // Retry With + status === 503 || // Service Unavailable + status === 504 // Gateway Timeout + ) + } + } + + // If we couldn't identify it as a network error, don't retry + return false +} diff --git a/utils/retry/tsconfig.json b/utils/retry/tsconfig.json new file mode 100644 index 0000000000..c6a877cf6c --- /dev/null +++ b/utils/retry/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "lib", "dist", "types", "bundle"] +} \ No newline at end of file