fix: Egress Gateway should keep retrying to connect to Redis

service/src/egress-ledger-reconnect.test.ts

@@ -4,7 +4,7 @@ import IORedis from 'ioredis';
 import { env } from './config';
 import { startTestRedis } from './test/redis';
 import {
-  EGRESS_LEDGER_REDIS_RETRY_OPTIONS, createEgressLedger, recordEgressRead,
+  EGRESS_LEDGER_REDIS_RETRY_OPTIONS, createEgressLedger, egressLedgerRedisOptions, recordEgressRead,
   assertEgressGrantActive, setEgressLedgerRedisForTest,
 } from './egress-ledger';
 import type { EgressGrantClaims } from './egress-grant';
@@ -61,3 +61,10 @@ test('a lost Redis mutation reply rejects without replaying its applied counter
     await redis.closeTestServer();
   }
 }, 10000);
+
+test('the ledger client keeps trying to reconnect using linear backoff with cap, however long Redis is unavailable', () => {
+  const { retryStrategy } = egressLedgerRedisOptions();
+  for (const attempt of [1, 6, 100, 10_000]) {
+    expect(retryStrategy!(attempt)).toBeGreaterThan(0);
+  }
+});

service/src/egress-ledger.ts

@@ -1,11 +1,11 @@
 import IORedis from 'ioredis';
-import type { CommonRedisOptions } from 'ioredis';
+import type { RedisOptions } from 'ioredis';
 import type * as tls from 'tls';
 import { env } from './config';
 import type { EgressGrantClaims } from './egress-grant';
 import { EgressGrantError } from './egress-grant';
 import logger from './logger';
-import { redisKeepAliveOptions } from './redis-options';
+import { redisKeepAliveOptions, redisReconnectDelay } from './redis-options';
 import { EGRESS_LEDGER_SCRIPT } from './egress-ledger-script';
 
 type LedgerStatus = 'active' | 'revoked';
@@ -53,18 +53,13 @@ function ttlSeconds(exp: number): number {
   return Math.max(1, exp - Math.floor(Date.now() / 1000) + env.EGRESS_LEDGER_TTL_GRACE_SECONDS);
 }
 
-function redisConnection(): IORedis {
-  if (redis) return redis;
-  const retryStrategy: CommonRedisOptions['retryStrategy'] = times => {
-    if (times > 5) return null;
-    return 2000;
-  };
-  redis = new IORedis({
+export function egressLedgerRedisOptions(): RedisOptions {
+  return {
     host: process.env.REDIS_HOST ?? 'redis',
     port: Number(process.env.REDIS_PORT) || 6379,
     password: process.env.REDIS_PASSWORD,
     ...EGRESS_LEDGER_REDIS_RETRY_OPTIONS,
-    retryStrategy,
+    retryStrategy: redisReconnectDelay,    // Use linear backoff with a cap for reconnect attempts, otherwise Redis outages do not recover automatically
     enableReadyCheck: true,
     connectTimeout: 10000,
     ...redisKeepAliveOptions(),
@@ -74,7 +69,12 @@ function redisConnection(): IORedis {
     ...(env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP
       ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) }
       : {}),
-  });
+  };
+}
+
+function redisConnection(): IORedis {
+  if (redis) return redis;
+  redis = new IORedis(egressLedgerRedisOptions());
   redis.on('error', error => logger.error('Egress ledger Redis error', { error }));
   return redis;
 }