Merge pull request #222 from blockfrost/chore/exit-on-healthcheckfail

.vscode/settings.json

@@ -5,6 +5,7 @@
     "blockfrost",
     "emurgo",
     "from",
+    "HEALTHCHECK",
     "keyhash",
     "tailwindcss",
     "to",

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Changed
+
+- process exits after health check fails for more than `HEALTHCHECK_FAIL_THRESHOLD_MS` (default 60s)
+
 ## [2.0.0] - 2023-04-05
 
 ### Added

src/constants/config.ts

@@ -23,3 +23,6 @@ export const BLOCKFROST_REQUEST_CONCURRENCY = 500;
 
 // How often should metrics be updated
 export const METRICS_COLLECTOR_INTERVAL_MS = 10_000;
+
+// If healthcheck repeatedly fails for duration longer than this constant the process exits
+export const HEALTHCHECK_FAIL_THRESHOLD_MS = 6 * METRICS_COLLECTOR_INTERVAL_MS; // 6 health checks

src/utils/blockfrost-api.ts

@@ -6,6 +6,7 @@ export const getBlockfrostClient = (options?: Partial<Options>) => {
   return new BlockFrostAPI({
     projectId: process.env.BLOCKFROST_PROJECT_ID,
     customBackend: process.env.BLOCKFROST_BACKEND_URL || '',
+    // @ts-expect-error passing string network type
     network: process.env.BLOCKFROST_NETWORK,
     userAgent: `${packageJson.name}@${packageJson.version}`,
     rateLimiter: false,

src/utils/prometheus.ts

@@ -3,6 +3,7 @@ import { healthCheck } from './health';
 import { assetMetadataLimiter, pLimiter, ratesLimiter } from './limiter';
 import { logger } from './logger';
 import { getPort } from './server';
+import { HEALTHCHECK_FAIL_THRESHOLD_MS } from '../constants/config';
 
 const port = getPort();
 
@@ -20,6 +21,7 @@ export class MetricsCollector {
   intervalId: NodeJS.Timer | undefined;
   wss: WebSocketServer;
   healthy = true;
+  healthCheckFailingSince: number | null = null;
 
   constructor(wss: WebSocketServer, interval: number) {
     this.wss = wss;
@@ -31,9 +33,23 @@ export class MetricsCollector {
     try {
       await healthCheck(`ws://localhost:${port}`);
       this.healthy = true;
+      this.healthCheckFailingSince = null;
     } catch (error) {
       logger.error(error);
       this.healthy = false;
+      if (this.healthCheckFailingSince) {
+        const failDurationMs = Date.now() - this.healthCheckFailingSince;
+
+        if (failDurationMs > HEALTHCHECK_FAIL_THRESHOLD_MS) {
+          logger.error(
+            `Healthcheck failing for longer than ${HEALTHCHECK_FAIL_THRESHOLD_MS} ms. Exiting process.`,
+          );
+          // eslint-disable-next-line unicorn/no-process-exit
+          process.exit(1);
+        }
+      } else {
+        this.healthCheckFailingSince = Date.now();
+      }
     }
     logger.info('[HealthCheck] Health check done');
     return {