/** * Example 5: Error Handling and Retry Logic * * This example demonstrates robust error handling patterns including: * - Exponential backoff retry logic * - Graceful degradation * - Timeout handling * - Connection recovery * * @example * // Start the network server first: * // cd pods/network-pod && rushx dev * * // Then run this example: * // cd examples && rushx run:retry */ import { AgentImpl, TickManagerImpl, NetworkImpl } from '@hcengineering/network-core' import { NetworkServer } from '@hcengineering/network-server' import { createNetworkClient, NetworkAgentServer } from '@hcengineering/network-client' import type { Container, ContainerUuid, ClientUuid, ContainerKind, GetOptions } from '@hcengineering/network-core' class UnreliableServiceContainer implements Container { private callCount = 0 private failureRate = 0.3 // 30% failure rate constructor(readonly uuid: ContainerUuid) { console.log(`[UnreliableService] Container ${uuid} created`) } async request(operation: string, data?: any, clientId?: ClientUuid): Promise { this.callCount++ // Simulate random failures if (Math.random() < this.failureRate) { console.log(`[UnreliableService] ❌ Request ${this.callCount} failed (simulated)`) throw new Error(`Service temporarily unavailable (call ${this.callCount})`) } // Simulate slow responses occasionally const delay = Math.random() > 0.7 ? 2000 : 100 await new Promise(resolve => setTimeout(resolve, delay)) console.log(`[UnreliableService] ✓ Request ${this.callCount} succeeded`) return { success: true, operation, callNumber: this.callCount, data: data?.value ? data.value * 2 : null } } async ping(): Promise {} async terminate(): Promise { console.log(`[UnreliableService] Terminated after ${this.callCount} calls`) } connect(clientId: ClientUuid, broadcast: (data: any) => Promise): void {} disconnect(clientId: ClientUuid): void {} } // Retry helper with exponential backoff async function retryWithBackoff( operation: () => Promise, maxRetries = 3, initialDelay = 1000, maxDelay = 10000 ): Promise { let lastError: Error | undefined for (let attempt = 1; attempt <= maxRetries; attempt++) { try { console.log(` Attempt ${attempt}/${maxRetries}...`) const result = await operation() console.log(` ✓ Success on attempt ${attempt}`) return result } catch (error: any) { lastError = error console.log(` ✗ Attempt ${attempt} failed: ${error.message}`) if (attempt < maxRetries) { const delay = Math.min(initialDelay * Math.pow(2, attempt - 1), maxDelay) console.log(` Waiting ${delay}ms before retry...`) await new Promise(resolve => setTimeout(resolve, delay)) } } } throw new Error(`Operation failed after ${maxRetries} attempts: ${lastError?.message}`) } // Robust container access with retry and cleanup async function robustContainerAccess( client: any, kind: ContainerKind, options: GetOptions, operation: string, data?: any, maxRetries = 3, acquireTimeoutMs = 5000 ): Promise { let containerRef: any try { // Get container with retry and timeout containerRef = await retryWithBackoff(async () => { return await withTimeout( client.get(kind, options), acquireTimeoutMs, 'Container acquisition' ) }, maxRetries) console.log(`✓ Container acquired: ${containerRef.uuid}`) // Perform operation with retry const result = await retryWithBackoff(async () => { return await containerRef.request(operation, data) }, maxRetries) return result } catch (error: any) { console.error(`✗ All attempts failed: ${error.message}`) throw error } finally { // Always cleanup if (containerRef) { try { await containerRef.close() console.log('✓ Container released') } catch (err: any) { console.error('✗ Failed to release container:', err.message) } } } } // Timeout wrapper async function withTimeout( promise: Promise, timeoutMs: number, operation: string ): Promise { return Promise.race([ promise, new Promise((_, reject) => setTimeout(() => reject(new Error(`${operation} timed out after ${timeoutMs}ms`)), timeoutMs) ) ]) } async function main(): Promise { console.log('=== Error Handling and Retry Logic Example ===\n') // 1. Setup infrastructure const tickManager = new TickManagerImpl(1) tickManager.start() const network = new NetworkImpl(tickManager) const server = new NetworkServer(network, tickManager, '*', 3737) console.log('✓ Network server started\n') // 2. Create agent const agent = new AgentImpl('unreliable-agent' as any, { ['unreliable-service' as ContainerKind]: async (options: GetOptions) => { const uuid = options.uuid ?? `unreliable-${Date.now()}` as ContainerUuid const container = new UnreliableServiceContainer(uuid) return { uuid, container, endpoint: `unreliable://localhost/${uuid}` as any } } }) const agentServer = new NetworkAgentServer(tickManager, 'localhost', '*', 3738) await agentServer.start(agent) console.log('✓ Agent server started\n') // 3. Connect client await using client = createNetworkClient('localhost:3737') try { await withTimeout( client.waitConnection(10000), 5000, 'Client connection' ) console.log('✓ Client connected\n') } catch (error: any) { console.error('✗ Failed to connect:', error.message) throw error } // 4. Register agent await client.register(agent) console.log('✓ Agent registered\n') // 5. Test basic retry logic console.log('=== Test 1: Basic Retry with Backoff ===\n') try { const result = await robustContainerAccess( client, 'unreliable-service' as ContainerKind, {}, 'process', { value: 42 } ) console.log('Final result:', result) } catch (error: any) { console.error('Operation failed:', error.message) } console.log() // 6. Test multiple concurrent requests with error handling console.log('=== Test 2: Concurrent Requests with Individual Error Handling ===\n') const promises = [] for (let i = 0; i < 5; i++) { promises.push( robustContainerAccess( client, 'unreliable-service' as ContainerKind, {}, 'process', { value: i + 1 } ).then(result => ({ success: true, result })) .catch(error => ({ success: false, error: error.message })) ) } const results = await Promise.all(promises) results.forEach((result, idx) => { if (result.success) { console.log(`Request ${idx + 1}: ✓ Success -`, result.result) } else { console.log(`Request ${idx + 1}: ✗ Failed -`, result.error) } }) console.log() // 7. Test timeout handling console.log('=== Test 3: Timeout Handling ===\n') try { const containerRef = await withTimeout( client.get('unreliable-service' as ContainerKind, {}), 3000, 'Container acquisition' ) console.log(`✓ Container acquired: ${containerRef.uuid}\n`) // Try with short timeout (likely to timeout on slow responses) for (let i = 0; i < 3; i++) { try { console.log(`Request ${i + 1} with 500ms timeout...`) const result = await withTimeout( containerRef.request('process', { value: i }), 500, 'Request' ) console.log(' ✓ Completed:', result) } catch (error: any) { console.log(' ✗', error.message) } } await containerRef.close() } catch (error: any) { console.log(' ✗ Could not acquire container:', error.message) } console.log() // 8. Test graceful degradation console.log('=== Test 4: Graceful Degradation ===\n') async function getDataWithFallback(id: number) { try { // Try primary service const result = await robustContainerAccess( client, 'unreliable-service' as ContainerKind, {}, 'process', { value: id }, 2 // Fewer retries for faster fallback ) return { source: 'primary', data: result } } catch (error) { console.log('Primary service failed, using fallback...') // Fallback to cached/default data return { source: 'fallback', data: { success: true, operation: 'process', value: id, cached: true } } } } const dataResults = [] for (let i = 0; i < 3; i++) { const data = await getDataWithFallback(i + 1) dataResults.push(data) console.log(`Data ${i + 1}: from ${data.source} -`, data.data) } console.log() // 9. Test circuit breaker pattern console.log('=== Test 5: Circuit Breaker Pattern ===\n') class CircuitBreaker { private failureCount = 0 private lastFailureTime = 0 private state: 'closed' | 'open' | 'half-open' = 'closed' constructor( private readonly failureThreshold = 3, private readonly resetTimeout = 5000 ) {} async execute(operation: () => Promise): Promise { // Check if circuit should reset if (this.state === 'open') { if (Date.now() - this.lastFailureTime > this.resetTimeout) { console.log(' [CircuitBreaker] Half-open: Trying again...') this.state = 'half-open' } else { throw new Error('Circuit breaker is OPEN - service unavailable') } } try { const result = await operation() // Success - reset circuit breaker if (this.state === 'half-open') { console.log(' [CircuitBreaker] Closed: Service recovered') } this.failureCount = 0 this.state = 'closed' return result } catch (error) { this.failureCount++ this.lastFailureTime = Date.now() if (this.failureCount >= this.failureThreshold) { console.log(` [CircuitBreaker] OPEN: ${this.failureCount} failures detected`) this.state = 'open' } throw error } } getState() { return { state: this.state, failures: this.failureCount } } } const breaker = new CircuitBreaker(2, 3000) for (let i = 0; i < 5; i++) { try { const result = await breaker.execute(async () => { return await robustContainerAccess( client, 'unreliable-service' as ContainerKind, {}, 'process', { value: i }, 1 // Single retry ) }) console.log(`Request ${i + 1}: ✓`, result) } catch (error: any) { console.log(`Request ${i + 1}: ✗`, error.message) } console.log(' Circuit breaker state:', breaker.getState()) await new Promise(resolve => setTimeout(resolve, 500)) } console.log() // 10. Cleanup console.log('--- Cleanup ---') client.close() await agentServer.close() await server.close() tickManager.stop() console.log('✓ Cleanup complete') console.log('\n✓ Example completed successfully!') console.log('\nKey patterns demonstrated:') console.log(' 1. Exponential backoff retry') console.log(' 2. Timeout handling') console.log(' 3. Graceful degradation with fallbacks') console.log(' 4. Circuit breaker pattern') console.log(' 5. Proper resource cleanup') // Exit cleanly await new Promise(resolve => setTimeout(resolve, 500)) process.exit(0) } main().catch(console.error) export { UnreliableServiceContainer, retryWithBackoff, withTimeout }