Log DB query & environment values

Changelog.md

@@ -10,6 +10,8 @@
   exist ([#542](https://github.com/aws/graph-explorer/pull/542))
 - **Added** global error page if the React app crashes
   ([#547](https://github.com/aws/graph-explorer/pull/547))
+- **Added** server logging of database queries when using the proxy server
+  ([#574](https://github.com/aws/graph-explorer/pull/574))
 - **Improved** handling of server errors with more consistent logging
   ([#557](https://github.com/aws/graph-explorer/pull/557))
 - **Transition** to Tailwind instead of EmotionCSS for styles, which should make
@@ -22,6 +24,9 @@
   [#548](https://github.com/aws/graph-explorer/pull/548))
 - **Improved** SageMaker Lifecycle script handling of CloudWatch log driver
   failures ([#550](https://github.com/aws/graph-explorer/pull/550))
+- **Improved** parsing of environment values in proxy server resulting in an
+  error when the values are invalid
+  ([#574](https://github.com/aws/graph-explorer/pull/574))
 - **Changed** Node to run in production mode
   ([#558](https://github.com/aws/graph-explorer/pull/558))
 - **Removed** hosting production server using the client side Vite

packages/graph-explorer-proxy-server/src/env.ts

@@ -0,0 +1,45 @@
+import dotenv from "dotenv";
+import path from "path";
+import { clientRoot } from "./paths.js";
+import { z } from "zod";
+
+/** Coerces a string to a boolean value in a case insensitive way. */
+const BooleanStringSchema = z
+  .string()
+  .refine(s => s.toLowerCase() === "true" || s.toLowerCase() === "false")
+  .transform(s => s.toLowerCase() === "true");
+
+// Define a required schema for the values we expect along with their defaults
+const EnvironmentValuesSchema = z.object({
+  HOST: z.string().default("localhost"),
+  PROXY_SERVER_HTTPS_CONNECTION: BooleanStringSchema.default("false"),
+  PROXY_SERVER_HTTPS_PORT: z.coerce.number().default(443),
+  PROXY_SERVER_HTTP_PORT: z.coerce.number().default(80),
+  LOG_LEVEL: z
+    .enum(["fatal", "error", "warn", "info", "debug", "trace", "silent"])
+    .default("debug"),
+  LOG_STYLE: z.enum(["cloudwatch", "default"]).default("default"),
+});
+
+// Load environment variables from .env file.
+dotenv.config({
+  path: [path.join(clientRoot, ".env.local"), path.join(clientRoot, ".env")],
+});
+
+// Parse the environment values from the process
+const parsedEnvironmentValues = EnvironmentValuesSchema.safeParse(process.env);
+
+if (!parsedEnvironmentValues.success) {
+  console.error("Failed to parse environment values");
+  const flattenedErrors = parsedEnvironmentValues.error.flatten();
+  console.error(flattenedErrors.fieldErrors);
+  process.exit(1);
+}
+
+// eslint-disable-next-line no-console
+console.log("Parsed environment values:", parsedEnvironmentValues.data);
+
+// Adds all environment values to local object
+export const env = {
+  ...parsedEnvironmentValues.data,
+};

packages/graph-explorer-proxy-server/src/logging.ts

@@ -1,37 +1,16 @@
 import { NextFunction, Request, Response } from "express";
 import { pino } from "pino";
 import { PrettyOptions } from "pino-pretty";
-import { z } from "zod";
+import { env } from "./env.js";
 
 export type LogLevel = pino.LevelWithSilent;
 
-const LogLevelSchema = z.enum([
-  "fatal",
-  "error",
-  "warn",
-  "info",
-  "debug",
-  "trace",
-  "silent",
-]);
-
 export const logger = createLogger();
 
-/** Parses the log level from a given string. If the value is unrecognized or undefined, the default is "info". */
-function toLogLevel(value: string | undefined): LogLevel {
-  const parsed = LogLevelSchema.safeParse(value);
-
-  if (!parsed.success) {
-    return "info";
-  }
-
-  return parsed.data;
-}
-
 /** Create a logger instance with pino. */
 function createLogger() {
   // Check whether we are configured with CloudWatch style
-  const loggingInCloudWatch = process.env.LOG_STYLE === "cloudwatch";
+  const loggingInCloudWatch = env.LOG_STYLE === "cloudwatch";
   const options: PrettyOptions = loggingInCloudWatch
     ? {
         // Avoid colors
@@ -43,7 +22,7 @@ function createLogger() {
         colorize: true,
         translateTime: true,
       };
-  const level = toLogLevel(process.env.LOG_LEVEL);
+  const level = env.LOG_LEVEL;
 
   return pino({
     level,
@@ -88,6 +67,12 @@ export function logRequestAndResponse(req: Request, res: Response) {
 /** Creates the pino-http middleware with the given logger and appropriate options. */
 export function requestLoggingMiddleware() {
   return (req: Request, res: Response, next: NextFunction) => {
+    // Ignore requests to logger endpoint
+    if (req.path.includes("/logger")) {
+      next();
+      return;
+    }
+
     // Wait for the request to complete.
     req.on("end", () => {
       logRequestAndResponse(req, res);

packages/graph-explorer-proxy-server/src/node-server.ts

@@ -1,7 +1,6 @@
 import express, { NextFunction, Response } from "express";
 import cors from "cors";
 import compression from "compression";
-import dotenv from "dotenv";
 import fetch, { RequestInit } from "node-fetch";
 import https from "https";
 import bodyParser from "body-parser";
@@ -13,14 +12,10 @@ import { IncomingHttpHeaders } from "http";
 import { logger as proxyLogger, requestLoggingMiddleware } from "./logging.js";
 import { clientRoot, proxyServerRoot } from "./paths.js";
 import { errorHandlingMiddleware, handleError } from "./error-handler.js";
+import { env } from "./env.js";
 
 const app = express();
 
-// Load environment variables from .env file.
-dotenv.config({
-  path: [path.join(clientRoot, ".env.local"), path.join(clientRoot, ".env")],
-});
-
 const DEFAULT_SERVICE_TYPE = "neptune-db";
 
 interface DbQueryIncomingHttpHeaders extends IncomingHttpHeaders {
@@ -229,11 +224,13 @@ app.post("/sparql", (req, res, next) => {
   });
 
   // Validate the input before making any external calls.
-  if (!req.body.query) {
+  const queryString = req.body.query;
+  if (!queryString) {
     return res.status(400).send({ error: "[Proxy]SPARQL: Query not provided" });
   }
+  proxyLogger.debug("[SPARQL] Received database query:\n%s", queryString);
   const rawUrl = `${graphDbConnectionUrl}/sparql`;
-  let body = `query=${encodeURIComponent(req.body.query)}`;
+  let body = `query=${encodeURIComponent(queryString)}`;
   if (queryId) {
     body += `&queryId=${encodeURIComponent(queryId)}`;
   }
@@ -270,12 +267,15 @@ app.post("/gremlin", (req, res, next) => {
     : "";
 
   // Validate the input before making any external calls.
-  if (!req.body.query) {
+  const queryString = req.body.query;
+  if (!queryString) {
     return res
       .status(400)
       .send({ error: "[Proxy]Gremlin: query not provided" });
   }
 
+  proxyLogger.debug("[Gremlin] Received database query:\n%s", queryString);
+
   /// Function to cancel long running queries if the client disappears before completion
   async function cancelQuery() {
     if (!queryId) {
@@ -312,7 +312,7 @@ app.post("/gremlin", (req, res, next) => {
     await cancelQuery();
   });
 
-  const body = { gremlin: req.body.query, queryId };
+  const body = { gremlin: queryString, queryId };
   const rawUrl = `${graphDbConnectionUrl}/gremlin`;
   const requestOptions = {
     method: "POST",
@@ -336,13 +336,16 @@ app.post("/gremlin", (req, res, next) => {
 
 // POST endpoint for openCypher queries.
 app.post("/openCypher", (req, res, next) => {
+  const queryString = req.body.query;
   // Validate the input before making any external calls.
-  if (!req.body.query) {
+  if (!queryString) {
     return res
       .status(400)
       .send({ error: "[Proxy]OpenCypher: query not provided" });
   }
 
+  proxyLogger.debug("[openCypher] Received database query:\n%s", queryString);
+
   const headers = req.headers as DbQueryIncomingHttpHeaders;
   const rawUrl = `${headers["graph-db-connection-url"]}/openCypher`;
   const requestOptions = {
@@ -351,7 +354,7 @@ app.post("/openCypher", (req, res, next) => {
       "Content-Type": "application/x-www-form-urlencoded",
       Accept: "application/json",
     },
-    body: `query=${encodeURIComponent(req.body.query)}`,
+    body: `query=${encodeURIComponent(queryString)}`,
   };
 
   const isIamEnabled = !!headers["aws-neptune-region"];
@@ -498,11 +501,11 @@ const certificateKeyFilePath = path.join(
 const certificateFilePath = path.join(proxyServerRoot, "cert-info/server.crt");
 
 // Get the port numbers to listen on
-const host = process.env.HOST || "localhost";
-const httpPort = process.env.PROXY_SERVER_HTTP_PORT || 80;
-const httpsPort = process.env.PROXY_SERVER_HTTPS_PORT || 443;
+const host = env.HOST;
+const httpPort = env.PROXY_SERVER_HTTP_PORT;
+const httpsPort = env.PROXY_SERVER_HTTPS_PORT;
 const useHttps =
-  process.env.PROXY_SERVER_HTTPS_CONNECTION === "true" &&
+  env.PROXY_SERVER_HTTPS_CONNECTION &&
   fs.existsSync(certificateKeyFilePath) &&
   fs.existsSync(certificateFilePath);
 

packages/graph-explorer/src/connector/LoggerConnector.ts

@@ -13,29 +13,36 @@ export interface LoggerConnector {
 /** Sends log messages to the server in the connection configuration. */
 export class ServerLoggerConnector implements LoggerConnector {
   private readonly _baseUrl: string;
+  private readonly _clientLogger: ClientLoggerConnector;
 
   constructor(connectionUrl: string) {
     const url = connectionUrl.replace(/\/$/, "");
     this._baseUrl = `${url}/logger`;
+    this._clientLogger = new ClientLoggerConnector();
   }
 
   public error(message: unknown) {
+    this._clientLogger.error(message);
     return this._sendLog("error", message);
   }
 
   public warn(message: unknown) {
+    this._clientLogger.warn(message);
     return this._sendLog("warn", message);
   }
 
   public info(message: unknown) {
+    this._clientLogger.info(message);
     return this._sendLog("info", message);
   }
 
   public debug(message: unknown) {
+    this._clientLogger.info(message);
     return this._sendLog("debug", message);
   }
 
   public trace(message: unknown) {
+    this._clientLogger.trace(message);
     return this._sendLog("trace", message);
   }
 

packages/graph-explorer/src/connector/gremlin/gremlinExplorer.ts

@@ -9,6 +9,7 @@ import { GraphSummary } from "./types";
 import { v4 } from "uuid";
 import { Explorer } from "../useGEFetchTypes";
 import { logger } from "@/utils";
+import { createLoggerFromConnection } from "@/core/connector";
 
 function _gremlinFetch(connection: ConnectionConfig, options: any) {
   return async (queryTemplate: string) => {
@@ -54,30 +55,31 @@ async function fetchSummary(
 }
 
 export function createGremlinExplorer(connection: ConnectionConfig): Explorer {
+  const remoteLogger = createLoggerFromConnection(connection);
   return {
     connection: connection,
     async fetchSchema(options) {
-      logger.log("[Gremlin Explorer] Fetching schema...");
+      remoteLogger.info("[Gremlin Explorer] Fetching schema...");
       const summary = await fetchSummary(connection, options);
       return fetchSchema(_gremlinFetch(connection, options), summary);
     },
     async fetchVertexCountsByType(req, options) {
-      logger.log("[Gremlin Explorer] Fetching vertex counts by type...");
+      remoteLogger.info("[Gremlin Explorer] Fetching vertex counts by type...");
       return fetchVertexTypeCounts(_gremlinFetch(connection, options), req);
     },
     async fetchNeighbors(req, options) {
-      logger.log("[Gremlin Explorer] Fetching neighbors...");
+      remoteLogger.info("[Gremlin Explorer] Fetching neighbors...");
       return fetchNeighbors(_gremlinFetch(connection, options), req);
     },
     async fetchNeighborsCount(req, options) {
-      logger.log("[Gremlin Explorer] Fetching neighbors count...");
+      remoteLogger.info("[Gremlin Explorer] Fetching neighbors count...");
       return fetchNeighborsCount(_gremlinFetch(connection, options), req);
     },
     async keywordSearch(req, options) {
       options ??= {};
       options.queryId = v4();
 
-      logger.log("[Gremlin Explorer] Fetching keyword search...");
+      remoteLogger.info("[Gremlin Explorer] Fetching keyword search...");
       return keywordSearch(_gremlinFetch(connection, options), req);
     },
   };

packages/graph-explorer/src/connector/openCypher/openCypherExplorer.ts

@@ -9,6 +9,7 @@ import { ConnectionConfig } from "@shared/types";
 import { DEFAULT_SERVICE_TYPE } from "@/utils/constants";
 import { Explorer } from "../useGEFetchTypes";
 import { env, logger } from "@/utils";
+import { createLoggerFromConnection } from "@/core/connector";
 
 function _openCypherFetch(connection: ConnectionConfig, options: any) {
   return async (queryTemplate: string) => {
@@ -27,23 +28,31 @@ function _openCypherFetch(connection: ConnectionConfig, options: any) {
 export function createOpenCypherExplorer(
   connection: ConnectionConfig
 ): Explorer {
+  const remoteLogger = createLoggerFromConnection(connection);
   const serviceType = connection.serviceType || DEFAULT_SERVICE_TYPE;
   return {
     connection: connection,
     async fetchSchema(options) {
+      remoteLogger.info("[openCypher Explorer] Fetching schema...");
       const summary = await fetchSummary(serviceType, connection, options);
       return fetchSchema(_openCypherFetch(connection, options), summary);
     },
     async fetchVertexCountsByType(req, options) {
+      remoteLogger.info(
+        "[openCypher Explorer] Fetching vertex counts by type..."
+      );
       return fetchVertexTypeCounts(_openCypherFetch(connection, options), req);
     },
     async fetchNeighbors(req, options) {
+      remoteLogger.info("[openCypher Explorer] Fetching neighbors...");
       return fetchNeighbors(_openCypherFetch(connection, options), req);
     },
     async fetchNeighborsCount(req, options) {
+      remoteLogger.info("[openCypher Explorer] Fetching neighbors count...");
       return fetchNeighborsCount(_openCypherFetch(connection, options), req);
     },
     async keywordSearch(req, options) {
+      remoteLogger.info("[openCypher Explorer] Fetching keyword search...");
       return keywordSearch(_openCypherFetch(connection, options), req);
     },
   };