New investigation flow (#18)

* New investigation flow boilerplate

* Add logs capabilities

* Save demo changes

* Add a mapping between PagerDuty and Slack channel

* Fix coralogix hard coded value

* Finalize new investigation flow

* Remove old code

* Improve coralogix dataprime prompt

.env.example

@@ -111,4 +111,8 @@ DASHBOARD_PORT=5173
 APP_URL=http://localhost:${DASHBOARD_PORT}
 
 # PagerDuty
-PAGERDUTY_SECRET="put your secret"
+PAGERDUTY_SECRET="put your secret"
+
+# Log parser
+LOG_PARSER_PORT=3004
+LOG_PARSER_URL=http://log-parser:${LOG_PARSER_PORT}

.vscode/launch.json

@@ -38,12 +38,17 @@
       "type": "node",
       "request": "launch",
       "name": "API: Debug TypeScript",
-      "program": "${workspaceFolder}/services/api/node_modules/ts-node/dist/bin.js",
-      "args": ["${file}"],
+      "runtimeExecutable": "yarn",
+      "args": [
+        "runFile",
+        "${file}"
+        // "Q2HI9P83ZC9UXT",
+        // "66841ebc489b57392f102e08",
+        // "PagerDuty"
+      ],
       "sourceMaps": true,
-      "cwd": "${workspaceFolder}/services/api/",
-      "internalConsoleOptions": "neverOpen",
-      "envFile": "${workspaceFolder}/services/api/.env"
+      "cwd": "${workspaceFolder}/services/api",
+      "internalConsoleOptions": "neverOpen"
     },
     {
       "type": "node",
@@ -73,6 +78,14 @@
       "program": "src/main.py",
       "cwd": "${workspaceFolder}/services/data-processor/",
       "justMyCode": false
+    },
+    {
+      "type": "debugpy",
+      "name": "Log Parser: Debug",
+      "request": "launch",
+      "program": "src/main.py",
+      "cwd": "${workspaceFolder}/services/log-parser/",
+      "justMyCode": false
     }
   ]
 }

CONTRIBUTING.md

@@ -91,6 +91,7 @@ We work in a monorepo manner using [yarn](https://yarnpkg.com/) and [nx](https:/
 - `slackbot` - the slackbot code. Contains the logic of our Slack app.
 - `dashboard` - the web ui code. Contains the code of the web app, which allows users to configure their organization.
 - `data-processor` - contains the data processor code. This service is used to build the knowledge base that is needed in order to perform an extensive investigation with rich context.
+- `log-parser` - performs simple log aggregation/clustering.
 
 **Other important folders/files**
 

config/litellm/config.example.yaml

@@ -19,7 +19,7 @@ model_list:
   ###  OpenAI (Default) ####
   - model_name: chat-model
     litellm_params:
-      model: openai/gpt-3.5-turbo-1106
+      model: openai/gpt-4o-mini
       api_key: os.environ/OPENAI_API_KEY
   - model_name: embedding-model
     litellm_params:

docker-compose.common.yml

@@ -154,6 +154,7 @@ services:
       - DATA_PROCESSOR_URL
       - PAGERDUTY_SECRET
       - LITELLM_URL
+      - LOG_PARSER_URL
       - NODE_ENV=development
       - METADATA_ENCRYPTION_KEY=f78d8202-bb2a-49fe-bd63-a586cd921226
       - ENCRYPTION_SALT=1234567890123456
@@ -205,6 +206,14 @@ services:
       - ${DATA_DIR}/secrets:${SECRET_MANAGER_DIRECTORY}
     depends_on:
       - mongo
+  log-parser-common:
+    container_name: log-parser
+    profiles: ["app"]
+    environment:
+      - TELEMETRY_ENABLED
+      - PORT=${LOG_PARSER_PORT}
+    ports:
+      - "${LOG_PARSER_PORT}:${LOG_PARSER_PORT}"
   dashboard-common:
     container_name: dashboard
     profiles: ["app"]

docker-compose.images.yml

@@ -56,6 +56,11 @@ services:
       file: docker-compose.common.yml
       service: data-processor-common
     image: merlinnco/data-processor:latest
+  log-parser:
+    extends:
+      file: docker-compose.common.yml
+      service: log-parser-common
+    image: merlinnco/log-parser:latest
   dashboard:
     extends:
       file: docker-compose.common.yml

docker-compose.yml

@@ -61,6 +61,12 @@ services:
       service: data-processor-common
     build: 
       context: services/data-processor
+  log-parser:
+    extends:
+      file: docker-compose.common.yml
+      service: log-parser-common
+    build: 
+      context: services/log-parser
   dashboard:
     extends:
       file: docker-compose.common.yml

packages/db/src/types.ts

@@ -124,9 +124,7 @@ export interface BaseConnection {
   _id: Types.ObjectId;
   vendor: IVendor;
   organization: IOrganization;
-  settings?: {
-    tools?: Record<string, unknown>;
-  };
+  settings?: Record<string, unknown>;
   type: "basic" | "oauth";
   createdAt: Date;
   updatedAt: Date;
@@ -180,6 +178,9 @@ export interface PagerDutyIntegration extends BaseConnection {
     scope: string;
     expires_in: number;
   };
+  settings: {
+    slackChannelId: string;
+  };
 }
 
 export interface OpsgenieIntegration extends BaseConnection {

services/api/.env.dev

@@ -13,4 +13,5 @@ CHROMA_HOST="http://localhost:8000"
 CHROMA_API_KEY="secret-token"
 DATA_PROCESSOR_URL=http://localhost:3002
 LITELLM_URL=http://localhost:4000
+LOG_PARSER_URL=http://localhost:3004
 TELEMETRY_ENABLED=true

services/api/package.json

@@ -45,7 +45,8 @@
     "lint": "nx lint --ext ts",
     "test": "jest",
     "typecheck": "tsc -p tsconfig.json --noEmit",
-    "deploy": "npx ts-node ../../tools/scripts/deploy.ts api"
+    "deploy": "npx ts-node ../../tools/scripts/deploy.ts api",
+    "runFile": "npx dotenv-cli -o -e ../../.env -e .env.dev npx ts-node"
   },
   "devDependencies": {
     "@babel/preset-typescript": "^7.23.3",

services/api/src/agent/prompts.ts

@@ -20,6 +20,23 @@ export const prefix = {
   
   Begin!
   `,
+  investigationLean: `
+  You are an expert on-call engineer called Merlinn. Your mission is to investigate incidents in Production and provide findings to the responders, with as much information as possible.
+
+  Given the contextual information, produce a meaningful summarization about the data provided.
+  Don't write a report, write in free text, 5 sentences max. Be very concise and produce a short answer in a human readable format.
+
+  Begin! 
+
+  Incident:
+  {incident}
+
+  Additional investigation information:
+  {additionalInfo}
+  
+  Contextual information:
+  {context}
+  `,
   conversation: `
   You are a smart AI assistant called Merlinn. Your mission is to help on-call developers and SREs investigate production incidents.
   
@@ -50,6 +67,85 @@ export const prefix = {
   captionImage: `
   Please describe what you see in this image
   `,
+  generateQueries: `
+  Hi! You are tasked with retrieving information about the following production incident:
+
+  {incident}
+
+  You have access to a semantic search engine (VectorDB), where you can fetch historical & related information about this issue. Please create {nQueries} different queries that you would want to search against this search engine.
+
+  Your queries should be:
+  * Related to the incident above
+  * Short and simple. No long sentences.
+  * Abstract and expressive, with no IP addresses & numbers.
+
+
+  You should return your answer as JSON. It should contain 1 key called "queries", and it should be a list. For instance, here is an example response:
+  \`\`\`json
+  {{"queries": ["Service X issue", "500 error", "User could not pay issue"]}}
+  \`\`\`
+
+
+  IMPORTANT: Please respond only in JSON.
+
+  Begin!
+  `,
+  verifyDocument: `
+  Hi! You are tasked with verifying whether a document is actually relevant to a source information. The source information is a production incident information. You are tasked with judging whether a document is relevant. 
+
+  For example, given the following incident:
+  \`\`\`
+  Title: Coralogix Alert: Service data-processor has high CPU usage
+  Source: PagerDuty
+  Time: 5 months ago
+  Additional information: {{"Application":"demo-app","CompanyId":4014214}}
+  \`\`\`
+  
+  And the following document:
+  \`\`\`
+  Does someone know where do we save our finance reports?
+  \`\`\`
+
+  You should return false. IMPORTANT: return only true or false.
+  
+  Begin!
+
+  Incident:
+  {incident}
+
+  Document:
+  {document}
+  `,
+  extractLogStructureKeys: `  
+  Given a log record, return the key paths of the severity and message in a JSON format.
+
+  Examples:
+  Log records:
+  [{{"message": "Successfully updated user 123", "timestamp": "some-time", "severityText": "INFO"}},
+  {{"message": "Successfully updated user 456", "timestamp": "some-time", "severityText": "INFO"}}]
+
+  Output:
+  {{
+    "severityKey": "severityText",
+    "messageKey": "message"
+  }}
+
+  Log record:
+  [{{"timestamp": "some-time", "severity": {{"severityNumber": 3, "severityText": "INFO"}}, "logRecord": {{"body": "Successfully updated user 123"}}}},
+  {{"timestamp": "some-time2", "severity": {{"severityNumber": 3, "severityText": "INFO"}}, "logRecord": {{"body": "Successfully updated user 456"}}}}
+  ]
+
+  Output:
+  {{
+    "severityKey": "severity.severityText",
+    "messageKey": "logRecord.body"
+  }}
+
+  Begin!
+  
+  Log records:
+  {logRecords}
+  `,
 };
 
 export const investigationTemplate = ChatPromptTemplate.fromMessages([
@@ -76,3 +172,19 @@ export const conversationIssuesTemplate = ChatPromptTemplate.fromMessages([
 export const summarizeReadmePrompt = PromptTemplate.fromTemplate(
   prefix.summarizeReadme,
 );
+
+export const generateQueriesPrompt = PromptTemplate.fromTemplate(
+  prefix.generateQueries,
+);
+
+export const verifyDocumentPrompt = PromptTemplate.fromTemplate(
+  prefix.verifyDocument,
+);
+
+export const investigationLeanTemplate = PromptTemplate.fromTemplate(
+  prefix.investigationLean,
+);
+
+export const extractLogStructureKeysPrompt = PromptTemplate.fromTemplate(
+  prefix.extractLogStructureKeys,
+);

services/api/src/agent/rag/utils.ts

@@ -1,5 +1,6 @@
 import { PineconeVectorStore } from "./pinecone";
 import { ChromaDBVectorStore } from "./chromadb";
+import type { Document } from "./types";
 
 export function getVectorStore(
   indexName: string,
@@ -26,3 +27,15 @@ export function getVectorStore(
       throw new Error(`Invalid index source: ${indexType}`);
   }
 }
+
+export function nodesToText(documents: Document[]) {
+  const formattedNodes = documents.map(
+    (document, index) =>
+      `Document: ${index + 1}\n
+       Source: ${document.metadata.source}\n
+       Score: ${document.score}\n
+       Metadata: ${JSON.stringify(document.metadata)}\n
+       Text: ${document.text}`,
+  );
+  return formattedNodes.join("\n\n");
+}

services/api/src/agent/tools/base.ts

@@ -15,7 +15,7 @@ export const createAgent = async (
   tools: Tool[],
   template: ChatPromptTemplate,
 ) => {
-  const apiKey = process.env.OPENAI_API_KEY;
+  const apiKey = "secret-key"; // This is a dummy variable. LiteLLM is the one responsible for the actual API key
   const baseURL = process.env.LITELLM_URL;
 
   const model = new ChatOpenAI({