diff --git a/Makefile b/Makefile index 71ce39fb30318a..8660e052ec5f38 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ NODE_VERSION = $(shell cat .node-version) -KIBANA_VERSION ?= 8.1.x +KIBANA_VERSION ?= 8.2.x NETWORK ?= apm-integration-testing PORT ?= 5601 diff --git a/src/plugins/profiling/common/callercallee.test.ts b/src/plugins/profiling/common/callercallee.test.ts index 310ec4210a5a00..95dfd6b2185eaa 100644 --- a/src/plugins/profiling/common/callercallee.test.ts +++ b/src/plugins/profiling/common/callercallee.test.ts @@ -7,11 +7,172 @@ */ import { + createCallerCalleeDiagram, createCallerCalleeIntermediateNode, fromCallerCalleeIntermediateNode, } from './callercallee'; import { createStackFrameMetadata, hashFrameGroup } from './profiling'; +enum stackTraceID { + A = 'yU2Oct2ct0HkxJ7-pRcPkg==', + B = 'Xt8aKN70PDXpMDLCOmojzQ==', + C = '8OauxYq2WK4_tBqM4xkIwA==', + D = 'nQWGdRxvqVjwlLmQWH1Phw==', + E = '2KciEEWALlol3b6x95PHcw==', + F = 'BxRgiXa4h9Id6BjdPPHK8Q==', +} + +enum fileID { + A = 'Ncujji3wC1nL73TTEyFBhA==', + B = 'T2vdys5d7j85az1aP86zCg==', + C = 'jMaTVVjYv7cecd0C4HguGw==', + D = 'RLkjnlfcvSJN2Wph9WUuOQ==', + E = 'gnEsgxvvEODj6iFYMQWYlA==', + F = 'Gf4xoLc8QuAHU49Ch_CFOA==', + G = 'ZCOCZlls7r2cbG1HchkbVg==', + H = 'Og7kGWGe9qiCunkaXDffHQ==', + I = 'WAE6T1TeDsjDMOuwX4Ynxg==', +} + +enum frameID { + A = 'ZNiZco1zgh0nJI6hPllMaQAAAAABkPp6', + B = 'abl5r8Vvvb2Y7NaDZW1QLQAAAAAAZmzG', + C = 'gnEsgxvvEODj6iFYMQWYlAAAAAAGTnjJ', + D = 'gnEsgxvvEODj6iFYMQWYlAAAAAAGTnwG', + E = 'gnEsgxvvEODj6iFYMQWYlAAAAAAGYRMy', + F = 'gnEsgxvvEODj6iFYMQWYlAAAAAAGYV1J', + G = 'gnEsgxvvEODj6iFYMQWYlAAAAAAGEz_F', + H = 'Gf4xoLc8QuAHU49Ch_CFOAAAAAAABjhI', + I = 'Gf4xoLc8QuAHU49Ch_CFOAAAAAAAAcit', + J = 'Gf4xoLc8QuAHU49Ch_CFOAAAAAAAAfiT', + K = 'Gf4xoLc8QuAHU49Ch_CFOAAAAAAAAf7J', + L = 'ZCOCZlls7r2cbG1HchkbVgAAAAABGAwE', + M = 'Og7kGWGe9qiCunkaXDffHQAAAAAAAAvT', + N = 'WAE6T1TeDsjDMOuwX4YnxgAAAAAAABRR', + O = 'Gf4xoLc8QuAHU49Ch_CFOAAAAAAABloA', + P = 'Gf4xoLc8QuAHU49Ch_CFOAAAAAABV97Q', + Q = 'Gf4xoLc8QuAHU49Ch_CFOAAAAAABV9CG', + R = 'gnEsgxvvEODj6iFYMQWYlAAAAAAEBDLw', + S = 'gnEsgxvvEODj6iFYMQWYlAAAAAAD05_D', +} + +const events = new Map([ + [stackTraceID.A, 16], + [stackTraceID.B, 9], + [stackTraceID.C, 7], + [stackTraceID.D, 5], + [stackTraceID.E, 2], + [stackTraceID.F, 1], +]); + +const stackTraces = new Map([ + [ + stackTraceID.A, + { + FileID: [fileID.A, fileID.B, fileID.C, fileID.D], + FrameID: [frameID.A, frameID.A, frameID.A, frameID.B], + Type: [3, 3, 3, 3], + }, + ], + [ + stackTraceID.B, + { + FileID: [fileID.E, fileID.E, fileID.E, fileID.E, fileID.E], + FrameID: [frameID.C, frameID.D, frameID.E, frameID.F, frameID.G], + Type: [3, 3, 3, 3, 3], + }, + ], + [ + stackTraceID.C, + { + FileID: [fileID.F, fileID.F, fileID.F, fileID.F], + FrameID: [frameID.H, frameID.I, frameID.J, frameID.K], + Type: [3, 3, 3, 3], + }, + ], + [ + stackTraceID.D, + { + FileID: [fileID.G, fileID.H, fileID.I], + FrameID: [frameID.L, frameID.M, frameID.N], + Type: [3, 8, 8], + }, + ], + [ + stackTraceID.E, + { + FileID: [fileID.F, fileID.F, fileID.F], + FrameID: [frameID.O, frameID.P, frameID.Q], + Type: [3, 3, 3], + }, + ], + [ + stackTraceID.F, + { + FileID: [fileID.E, fileID.E], + FrameID: [frameID.R, frameID.S], + Type: [3, 3], + }, + ], +]); + +const defaultStackFrame = { + FileName: '', + FunctionName: '', + FunctionOffset: 0, + LineNumber: 0, + SourceType: 0, +}; + +const stackFrames = new Map([ + [ + frameID.A, + { + FileName: 'ThreadPoolExecutor.java', + FunctionName: 'java.lang.Runnable java.util.concurrent.ThreadPoolExecutor.getTask()', + FunctionOffset: 26, + LineNumber: 1061, + SourceType: 5, + }, + ], + [ + frameID.B, + { FileName: '', FunctionName: 'sock_sendmsg', FunctionOffset: 0, LineNumber: 0, SourceType: 0 }, + ], + [frameID.C, defaultStackFrame], + [frameID.D, defaultStackFrame], + [frameID.E, defaultStackFrame], + [frameID.F, defaultStackFrame], + [frameID.G, defaultStackFrame], + [frameID.H, defaultStackFrame], + [frameID.I, defaultStackFrame], + [frameID.J, defaultStackFrame], + [frameID.K, defaultStackFrame], + [ + frameID.L, + { FileName: '', FunctionName: 'udp_sendmsg', FunctionOffset: 0, LineNumber: 0, SourceType: 0 }, + ], + [frameID.M, defaultStackFrame], + [frameID.N, defaultStackFrame], + [frameID.O, defaultStackFrame], + [frameID.P, defaultStackFrame], + [frameID.Q, defaultStackFrame], + [frameID.R, defaultStackFrame], + [frameID.S, defaultStackFrame], +]); + +const executables = new Map([ + [fileID.A, { FileName: '' }], + [fileID.B, { FileName: '' }], + [fileID.C, { FileName: '' }], + [fileID.D, { FileName: 'libglapi.so.0.0.0' }], + [fileID.E, { FileName: '' }], + [fileID.F, { FileName: '' }], + [fileID.G, { FileName: '' }], + [fileID.H, { FileName: '' }], + [fileID.I, { FileName: '' }], +]); + describe('Caller-callee operations', () => { test('1', () => { const parentFrame = createStackFrameMetadata({ @@ -51,4 +212,9 @@ describe('Caller-callee operations', () => { expect(graph.Callees[0].Samples).toEqual(10); expect(graph.Callees[1].Samples).toEqual(10); }); + + test('2', () => { + const root = createCallerCalleeDiagram(events, stackTraces, stackFrames, executables); + expect(root.Samples).toEqual(40); + }); }); diff --git a/src/plugins/profiling/common/callercallee.ts b/src/plugins/profiling/common/callercallee.ts index 5f9358140df033..4f4cbd6ce888a9 100644 --- a/src/plugins/profiling/common/callercallee.ts +++ b/src/plugins/profiling/common/callercallee.ts @@ -10,9 +10,11 @@ import { clone } from 'lodash'; import { compareFrameGroup, + createStackFrameMetadata, defaultGroupBy, FrameGroup, FrameGroupID, + groupStackFrameMetadataByStackTrace, hashFrameGroup, StackFrameMetadata, StackTraceID, @@ -263,3 +265,19 @@ export function fromCallerCalleeIntermediateNode( return node; } + +export function createCallerCalleeDiagram( + events: Map, + stackTraces: Map, + stackFrames: Map, + executables: Map +): CallerCalleeNode { + const rootFrame = createStackFrameMetadata(); + const frameMetadataForTraces = groupStackFrameMetadataByStackTrace( + stackTraces, + stackFrames, + executables + ); + const root = createCallerCalleeIntermediateRoot(rootFrame, events, frameMetadataForTraces); + return fromCallerCalleeIntermediateNode(root); +} diff --git a/src/plugins/profiling/common/flamegraph.ts b/src/plugins/profiling/common/flamegraph.ts index 3308ad16ad8a6e..4963afee71dd1e 100644 --- a/src/plugins/profiling/common/flamegraph.ts +++ b/src/plugins/profiling/common/flamegraph.ts @@ -5,12 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import { Logger } from 'kibana/server'; -import { - CallerCalleeNode, - createCallerCalleeIntermediateRoot, - fromCallerCalleeIntermediateNode, -} from './callercallee'; +import { CallerCalleeNode, createCallerCalleeDiagram } from './callercallee'; import { StackTraceID, StackFrameID, @@ -18,23 +13,125 @@ import { StackTrace, StackFrame, Executable, - createStackFrameMetadata, - groupStackFrameMetadataByStackTrace, } from './profiling'; +interface ColumnarCallerCallee { + Label: string[]; + Value: number[]; + X: number[]; + Y: number[]; + Color: number[]; +} + +interface ElasticFlameGraph { + Label: string[]; + Value: number[]; + Position: number[]; + Size: number[]; + Color: number[]; +} + interface PixiFlameGraph extends CallerCalleeNode { TotalTraces: number; TotalSeconds: number; } +/* + * Helper to calculate the color of a given block to be drawn. The desirable outcomes of this are: + * Each of the following frame types should get a different set of color hues: + * + * 0 = Unsymbolized frame + * 1 = Python + * 2 = PHP + * 3 = Native + * 4 = Kernel + * 5 = JVM/Hotspot + * 6 = Ruby + * 7 = Perl + * 8 = JavaScript + * + * This is most easily achieved by mapping frame types to different color variations, using + * the x-position we can use different colors for adjacent blocks while keeping a similar hue + * + * Taken originally from prodfiler_ui/src/helpers/Pixi/frameTypeToColors.tsx + */ +const frameTypeToColors = [ + [0xfd8484, 0xfd9d9d, 0xfeb5b5, 0xfecece], + [0xfcae6b, 0xfdbe89, 0xfdcea6, 0xfedfc4], + [0xfcdb82, 0xfde29b, 0xfde9b4, 0xfef1cd], + [0x6dd0dc, 0x8ad9e3, 0xa7e3ea, 0xc5ecf1], + [0x7c9eff, 0x96b1ff, 0xb0c5ff, 0xcbd8ff], + [0x65d3ac, 0x84dcbd, 0xa3e5cd, 0xc1edde], + [0xd79ffc, 0xdfb2fd, 0xe7c5fd, 0xefd9fe], + [0xf98bb9, 0xfaa2c7, 0xfbb9d5, 0xfdd1e3], + [0xcbc3e3, 0xd5cfe8, 0xdfdbee, 0xeae7f3], +]; + +function frameTypeToRGB(frameType: number, x: number): number { + return frameTypeToColors[frameType][x % 4]; +} + +function normalizeColor(rgb: number): number[] { + return [ + Math.floor(rgb / 65536) / 255, + (Math.floor(rgb / 256) % 256) / 255, + (rgb % 256) / 255, + 1.0, + ]; +} + +function normalize(n: number, lower: number, upper: number): number { + return (n - lower) / (upper - lower); +} + function checkIfStringHasParentheses(s: string) { return /\(|\)/.test(s); } -function getFunctionName(frame: StackFrame) { - return frame.FunctionName !== '' && !checkIfStringHasParentheses(frame.FunctionName) - ? `${frame.FunctionName}()` - : frame.FunctionName; +function getFunctionName(node: CallerCalleeNode) { + return node.FunctionName !== '' && !checkIfStringHasParentheses(node.FunctionName) + ? `${node.FunctionName}()` + : node.FunctionName; +} + +function getExeFileName(node: CallerCalleeNode) { + if (node?.ExeFileName === undefined) { + return ''; + } + if (node.ExeFileName !== '') { + return node.ExeFileName; + } + switch (node.FrameType) { + case 0: + return ''; + case 1: + return 'Python'; + case 2: + return 'PHP'; + case 3: + return 'Native'; + case 4: + return 'Kernel'; + case 5: + return 'JVM/Hotspot'; + case 6: + return 'Ruby'; + case 7: + return 'Perl'; + case 8: + return 'JavaScript'; + default: + return ''; + } +} + +function getLabel(node: CallerCalleeNode) { + if (node.FunctionName !== '') { + const sourceFilename = node.SourceFilename; + const sourceURL = sourceFilename ? sourceFilename.split('/').pop() : ''; + return `${getExeFileName(node)}: ${getFunctionName(node)} in ${sourceURL} #${node.SourceLine}`; + } + return getExeFileName(node); } export class FlameGraph { @@ -55,16 +152,13 @@ export class FlameGraph { stackframes: Map; executables: Map; - private readonly logger: Logger; - constructor( sampleRate: number, totalCount: number, events: Map, stackTraces: Map, stackFrames: Map, - executables: Map, - logger: Logger + executables: Map ) { this.sampleRate = sampleRate; this.totalCount = totalCount; @@ -72,103 +166,101 @@ export class FlameGraph { this.stacktraces = stackTraces; this.stackframes = stackFrames; this.executables = executables; - this.logger = logger; } - private getExeFileName(exe: any, type: number) { - if (exe?.FileName === undefined) { - return ''; - } - if (exe.FileName !== '') { - return exe.FileName; - } - switch (type) { - case 0: - return ''; - case 1: - return 'Python'; - case 2: - return 'PHP'; - case 3: - return 'Native'; - case 4: - return 'Kernel'; - case 5: - return 'JVM/Hotspot'; - case 6: - return 'Ruby'; - case 7: - return 'Perl'; - case 8: - return 'JavaScript'; - default: - return ''; + // createColumnarCallerCallee flattens the intermediate representation of the diagram + // into a columnar format that is more compact than JSON. This representation will later + // need to be normalized into the response ultimately consumed by the flamegraph. + private createColumnarCallerCallee(root: CallerCalleeNode): ColumnarCallerCallee { + const columnar: ColumnarCallerCallee = { + Label: [], + Value: [], + X: [], + Y: [], + Color: [], + }; + const queue = [{ x: 0, depth: 1, node: root }]; + + while (queue.length > 0) { + const { x, depth, node } = queue.pop()!; + + if (x === 0 && depth === 1) { + columnar.Label.push('root: Represents 100% of CPU time.'); + } else { + columnar.Label.push(getLabel(node)); + } + columnar.Value.push(node.Samples); + columnar.X.push(x); + columnar.Y.push(depth); + columnar.Color.push(frameTypeToRGB(node.FrameType, x)); + + node.Callees.sort((a: CallerCalleeNode, b: CallerCalleeNode) => b.Samples - a.Samples); + + let delta = 0; + for (const callee of node.Callees) { + delta += callee.Samples; + } + + for (let i = node.Callees.length - 1; i >= 0; i--) { + delta -= node.Callees[i].Samples; + queue.push({ x: x + delta, depth: depth + 1, node: node.Callees[i] }); + } } + + return columnar; } - // Generates the label for a flamegraph node - // - // This is slightly modified from the original code in elastic/prodfiler_ui - private getLabel(frame: StackFrame, executable: Executable, type: number) { - if (frame.FunctionName !== '') { - return `${this.getExeFileName(executable, type)}: ${getFunctionName(frame)} in #${ - frame.LineNumber - }`; + // createElasticFlameGraph normalizes the intermediate columnar representation into the + // response ultimately consumed by the flamegraph. + private createElasticFlameGraph(columnar: ColumnarCallerCallee): ElasticFlameGraph { + const graph: ElasticFlameGraph = { + Label: [], + Value: [], + Position: [], + Size: [], + Color: [], + }; + + graph.Label = columnar.Label; + graph.Value = columnar.Value; + + const maxX = columnar.Value[0]; + const maxY = columnar.Y.reduce((max, n) => (n > max ? n : max), 0); + + for (let i = 0; i < columnar.X.length; i++) { + const x = normalize(columnar.X[i], 0, maxX); + const y = normalize(maxY - columnar.Y[i], 0, maxY); + graph.Position.push(x, y); } - return this.getExeFileName(executable, type); - } - toElastic() { - const leaves = []; - let n = 0; - - for (const trace of this.stacktraces.values()) { - const path = ['root']; - for (let i = 0; i < trace.FrameID.length; i++) { - const label = this.getLabel( - this.stackframes.get(trace.FrameID[i])!, - this.executables.get(trace.FileID[i])!, - trace.Type[i] - ); - - if (label.length === 0) { - path.push(trace.FrameID[i]); - } else { - path.push(label); - } - } - const leaf = { - id: path[0], - value: 1, - depth: trace.FrameID.length, - pathFromRoot: Object.fromEntries(path.map((item, i) => [i, item])), - }; - leaves.push(leaf); - - n++; - if (n >= 1000) { - // just don't overload the Kibana flamechart - break; - } + graph.Size = graph.Value.map((n) => normalize(n, 0, maxX)); + + for (const color of columnar.Color) { + graph.Color.push(...normalizeColor(color)); } - return { leaves }; + return graph; } - toPixi(): PixiFlameGraph { - const rootFrame = createStackFrameMetadata(); - const frameMetadataForTraces = groupStackFrameMetadataByStackTrace( + toElastic(): ElasticFlameGraph { + const root = createCallerCalleeDiagram( + this.events, this.stacktraces, this.stackframes, this.executables ); - const diagram = createCallerCalleeIntermediateRoot( - rootFrame, + return this.createElasticFlameGraph(this.createColumnarCallerCallee(root)); + } + + toPixi(): PixiFlameGraph { + const root = createCallerCalleeDiagram( this.events, - frameMetadataForTraces + this.stacktraces, + this.stackframes, + this.executables ); return { - ...fromCallerCalleeIntermediateNode(diagram), + ...root, TotalTraces: this.totalCount, TotalSeconds: 0, } as PixiFlameGraph; diff --git a/src/plugins/profiling/public/app.tsx b/src/plugins/profiling/public/app.tsx index 2872e98f68c3f5..62545a40e214c9 100644 --- a/src/plugins/profiling/public/app.tsx +++ b/src/plugins/profiling/public/app.tsx @@ -107,7 +107,7 @@ function App({ fetchTopN, fetchElasticFlamechart, fetchPixiFlamechart }: Props) series: new Map(), }); - const [elasticFlamegraph, setElasticFlamegraph] = useState({ leaves: [] }); + const [elasticFlamegraph, setElasticFlamegraph] = useState({}); const [pixiFlamegraph, setPixiFlamegraph] = useState({}); const handleTimeChange = (selectedTime: { start: string; end: string; isInvalid: boolean }) => { diff --git a/src/plugins/profiling/public/components/flamegraph.tsx b/src/plugins/profiling/public/components/flamegraph.tsx index cec7f88abd9965..003459f0644ad7 100644 --- a/src/plugins/profiling/public/components/flamegraph.tsx +++ b/src/plugins/profiling/public/components/flamegraph.tsx @@ -8,9 +8,7 @@ import React, { useContext, useEffect, useMemo } from 'react'; -import { Chart, Partition, PartitionLayout, PrimitiveValue, Settings } from '@elastic/charts'; - -import { EuiSpacer } from '@elastic/eui'; +import { Chart, ColumnarViewModel, Datum, Flame, PartialTheme, Settings } from '@elastic/charts'; import { FlameGraphContext } from './contexts/flamegraph'; @@ -19,6 +17,16 @@ export interface FlameGraphProps { height: number; } +const nullColumnarViewModel = { + label: [], + value: new Float64Array(), + color: new Float32Array(), + position0: new Float32Array(), + position1: new Float32Array(), + size0: new Float32Array(), + size1: new Float32Array(), +}; + export const FlameGraph: React.FC = ({ id, height }) => { const ctx = useContext(FlameGraphContext); @@ -26,44 +34,47 @@ export const FlameGraph: React.FC = ({ id, height }) => { console.log(new Date().toISOString(), 'updated flamegraph'); }); - const layers = useMemo(() => { - if (!ctx || !ctx.leaves || !ctx.leaves.length) { - return []; + const columnarData = useMemo(() => { + if (!ctx || !ctx.Label || ctx.Label.length === 0) { + return nullColumnarViewModel; } - const { leaves } = ctx; - const maxDepth = Math.max(...leaves.map((node: any) => node.depth)); + const value = new Float64Array(ctx.Value); + const position = new Float32Array(ctx.Position); + const size = new Float32Array(ctx.Size); + const color = new Float32Array(ctx.Color); - const result = [...new Array(maxDepth)].map((_, depth) => { - return { - groupByRollup: (d: any) => d.pathFromRoot[depth], - nodeLabel: (label: PrimitiveValue) => label, - showAccessor: (n: PrimitiveValue) => !!n, - shape: { - fillColor: (d: any) => '#FD8484', - }, - }; - }); - - return result; + return { + label: ctx.Label, + value: value, + color: color, + position0: position, + position1: position, + size0: size, + size1: size, + } as ColumnarViewModel; }, [ctx]); + const theme: PartialTheme = { + chartMargins: { top: 0, left: 0, bottom: 0, right: 0 }, + chartPaddings: { left: 0, right: 0, top: 0, bottom: 0 }, + }; + return ( <> - - - - d.value as number} - valueFormatter={() => ''} - /> - + {columnarData.label.length > 0 && ( + + + d.value as number} + valueFormatter={(value) => `${value}`} + animation={{ duration: 250 }} + controlProviderCallback={() => {}} + /> + + )} ); }; diff --git a/src/plugins/profiling/server/routes/downsampling.ts b/src/plugins/profiling/server/routes/downsampling.ts index dc33895333cb02..7baee6b276797e 100644 --- a/src/plugins/profiling/server/routes/downsampling.ts +++ b/src/plugins/profiling/server/routes/downsampling.ts @@ -8,7 +8,7 @@ import seedrandom from 'seedrandom'; import type { ElasticsearchClient, Logger } from 'kibana/server'; -import { ProjectTimeQuery } from './mappings'; +import { ProjectTimeQuery } from './query'; import { StackTraceID } from '../../common/profiling'; import { getHits } from './compat'; diff --git a/src/plugins/profiling/server/routes/flamechart.ts b/src/plugins/profiling/server/routes/flamechart.ts index 153d77f9778495..eda60278b4e88f 100644 --- a/src/plugins/profiling/server/routes/flamechart.ts +++ b/src/plugins/profiling/server/routes/flamechart.ts @@ -12,7 +12,7 @@ import { getRoutePaths } from '../../common'; import { FlameGraph } from '../../common/flamegraph'; import { StackTraceID } from '../../common/profiling'; import { logExecutionLatency } from './logger'; -import { newProjectTimeQuery, ProjectTimeQuery } from './mappings'; +import { newProjectTimeQuery, ProjectTimeQuery } from './query'; import { downsampleEventsRandomly, findDownsampledIndex } from './downsampling'; import { mgetExecutables, mgetStackFrames, mgetStackTraces, searchStackTraces } from './stacktrace'; import { getHitsItems, getAggs, getClient } from './compat'; @@ -149,8 +149,7 @@ async function queryFlameGraph( stackTraceEvents, stackTraces, stackFrames, - executables, - logger + executables ); }); } diff --git a/src/plugins/profiling/server/routes/mappings.ts b/src/plugins/profiling/server/routes/mappings.ts deleted file mode 100644 index dd24cec10df4ea..00000000000000 --- a/src/plugins/profiling/server/routes/mappings.ts +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; - -export interface ProjectTimeQuery { - bool: { - filter: Array< - | { - term: { - ProjectID: string; - }; - } - | { - range: { - '@timestamp': { - gte: string; - lt: string; - format: string; - boost: number; - }; - }; - } - >; - }; -} - -export function newProjectTimeQuery( - projectID: string, - timeFrom: string, - timeTo: string -): ProjectTimeQuery { - return { - bool: { - filter: [ - { - term: { - ProjectID: projectID, - }, - }, - { - range: { - '@timestamp': { - gte: timeFrom, - lt: timeTo, - format: 'epoch_second', - boost: 1.0, - }, - }, - }, - ], - }, - } as ProjectTimeQuery; -} - -export function autoHistogramSumCountOnGroupByField( - searchField: string, - topNItems: number -): AggregationsAggregationContainer { - return { - auto_date_histogram: { - field: '@timestamp', - buckets: 50, - }, - aggs: { - group_by: { - terms: { - field: searchField, - // We remove the ordering since we will rely directly on the natural - // ordering of Elasticsearch: by default this will be the descending count - // of matched documents. This is not equal to the ordering by sum of Count field, - // but it's a good-enough approximation given the distribution of Count. - size: topNItems, - // 'execution_hint: map' skips the slow building of ordinals that we don't need. - // Especially with high cardinality fields, this setting speeds up the aggregation. - execution_hint: 'map', - }, - aggs: { - count: { - sum: { - field: 'Count', - }, - }, - }, - }, - }, - }; -} - -function getExeFileName(obj: any) { - if (obj.ExeFileName === undefined) { - return ''; - } - if (obj.ExeFileName !== '') { - return obj.ExeFileName; - } - switch (obj.FrameType) { - case 0: - return ''; - case 1: - return 'Python'; - case 2: - return 'PHP'; - case 3: - return 'Native'; - case 4: - return 'Kernel'; - case 5: - return 'JVM/Hotspot'; - case 6: - return 'Ruby'; - case 7: - return 'Perl'; - case 8: - return 'JavaScript'; - default: - return ''; - } -} - -function checkIfStringHasParentheses(s: string) { - return /\(|\)/.test(s); -} - -function getFunctionName(obj: any) { - return obj.FunctionName !== '' && !checkIfStringHasParentheses(obj.FunctionName) - ? `${obj.FunctionName}()` - : obj.FunctionName; -} - -function getBlockName(obj: any) { - if (obj.FunctionName !== '') { - const sourceFileName = obj.SourceFilename; - const sourceURL = sourceFileName ? sourceFileName.split('/').pop() : ''; - return `${getExeFileName(obj)}: ${getFunctionName(obj)} in ${sourceURL}#${obj.SourceLine}`; - } - return getExeFileName(obj); -} - -const compareFlamechartSample = function (a: any, b: any) { - return b.Samples - a.Samples; -}; - -const sortFlamechart = function (data: any) { - data.Callees.sort(compareFlamechartSample); - return data; -}; - -const parseFlamechart = function (data: any) { - const parsedData = sortFlamechart(data); - parsedData.Callees = data.Callees.map(parseFlamechart); - return parsedData; -}; - -function extendFlameGraph(node: any, depth: any) { - node.id = getBlockName(node); - node.value = node.Samples; - node.depth = depth; - - for (const callee of node.Callees) { - extendFlameGraph(callee, depth + 1); - } -} - -function flattenTree(root: any, depth: any) { - if (root.Callees.length === 0) { - return [ - { - id: root.id, - value: root.value, - depth: root.depth, - pathFromRoot: { - [depth]: root.id, - }, - }, - ]; - } - - const children = root.Callees.flatMap((child: any) => flattenTree(child, depth + 1)); - - children.forEach((child: any) => { - child.pathFromRoot[depth] = root.id; - }); - - return children; -} - -export function mapFlamechart(src: any) { - src.ExeFileName = 'root'; - - const root = parseFlamechart(src); - - extendFlameGraph(root, 0); - - const newRoot = flattenTree(root, 0); - - return { - leaves: newRoot, - }; -} diff --git a/src/plugins/profiling/server/routes/query.ts b/src/plugins/profiling/server/routes/query.ts new file mode 100644 index 00000000000000..7b9113bb78a097 --- /dev/null +++ b/src/plugins/profiling/server/routes/query.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; + +export interface ProjectTimeQuery { + bool: { + filter: Array< + | { + term: { + ProjectID: string; + }; + } + | { + range: { + '@timestamp': { + gte: string; + lt: string; + format: string; + boost: number; + }; + }; + } + >; + }; +} + +export function newProjectTimeQuery( + projectID: string, + timeFrom: string, + timeTo: string +): ProjectTimeQuery { + return { + bool: { + filter: [ + { + term: { + ProjectID: projectID, + }, + }, + { + range: { + '@timestamp': { + gte: timeFrom, + lt: timeTo, + format: 'epoch_second', + boost: 1.0, + }, + }, + }, + ], + }, + } as ProjectTimeQuery; +} + +export function autoHistogramSumCountOnGroupByField( + searchField: string, + topNItems: number +): AggregationsAggregationContainer { + return { + auto_date_histogram: { + field: '@timestamp', + buckets: 50, + }, + aggs: { + group_by: { + terms: { + field: searchField, + // We remove the ordering since we will rely directly on the natural + // ordering of Elasticsearch: by default this will be the descending count + // of matched documents. This is not equal to the ordering by sum of Count field, + // but it's a good-enough approximation given the distribution of Count. + size: topNItems, + // 'execution_hint: map' skips the slow building of ordinals that we don't need. + // Especially with high cardinality fields, this setting speeds up the aggregation. + execution_hint: 'map', + }, + aggs: { + count: { + sum: { + field: 'Count', + }, + }, + }, + }, + }, + }; +} diff --git a/src/plugins/profiling/server/routes/stacktrace.test.ts b/src/plugins/profiling/server/routes/stacktrace.test.ts index d915de3ce21efd..812cdf4df3f26e 100644 --- a/src/plugins/profiling/server/routes/stacktrace.test.ts +++ b/src/plugins/profiling/server/routes/stacktrace.test.ts @@ -6,26 +6,81 @@ * Side Public License, v 1. */ -import { extractFileIDFromFrameID } from './stacktrace'; +import { StackTrace } from '../../common/profiling'; +import { decodeStackTrace, EncodedStackTrace, runLengthDecodeReverse } from './stacktrace'; -describe('Extract FileID from FrameID', () => { - test('extractFileIDFromFrameID', () => { +describe('Stack trace operations', () => { + test('decodeStackTrace', () => { const tests: Array<{ - frameID: string; - expected: string; + original: EncodedStackTrace; + expected: StackTrace; }> = [ { - frameID: 'aQpJmTLWydNvOapSFZOwKgAAAAAAB924', - expected: 'aQpJmTLWydNvOapSFZOwKg==', + original: { + FrameID: 'aQpJmTLWydNvOapSFZOwKgAAAAAAB924', + Type: Buffer.from([0x1, 0x0]).toString('base64url'), + } as EncodedStackTrace, + expected: { + FileID: ['aQpJmTLWydNvOapSFZOwKg=='], + FrameID: ['aQpJmTLWydNvOapSFZOwKgAAAAAAB924'], + Type: [0], + } as StackTrace, }, { - frameID: 'hz_u-HGyrN6qeIk6UIJeCAAAAAAAAAZZ', - expected: 'hz_u-HGyrN6qeIk6UIJeCA==', + original: { + FrameID: 'hz_u-HGyrN6qeIk6UIJeCAAAAAAAAAZZ', + Type: Buffer.from([0x1, 0x8]).toString('base64url'), + } as EncodedStackTrace, + expected: { + FileID: ['hz_u-HGyrN6qeIk6UIJeCA=='], + FrameID: ['hz_u-HGyrN6qeIk6UIJeCAAAAAAAAAZZ'], + Type: [8], + } as StackTrace, }, ]; for (const t of tests) { - expect(extractFileIDFromFrameID(t.frameID)).toEqual(t.expected); + expect(decodeStackTrace(t.original)).toEqual(t.expected); + } + }); + + test('runLengthDecodeReverse with optional parameter', () => { + const tests: Array<{ + bytes: Buffer; + expected: number[]; + }> = [ + { + bytes: Buffer.from([0x5, 0x0, 0x2, 0x2]), + expected: [2, 2, 0, 0, 0, 0, 0], + }, + { + bytes: Buffer.from([0x1, 0x8]), + expected: [8], + }, + ]; + + for (const t of tests) { + expect(runLengthDecodeReverse(t.bytes, t.expected.length)).toEqual(t.expected); + } + }); + + test('runLengthDecodeReverse without optional parameter', () => { + const tests: Array<{ + bytes: Buffer; + expected: number[]; + }> = [ + { + bytes: Buffer.from([0x5, 0x0, 0x2, 0x2]), + expected: [2, 2, 0, 0, 0, 0, 0], + }, + { + bytes: Buffer.from([0x1, 0x8]), + expected: [8], + }, + ]; + + for (const t of tests) { + expect(runLengthDecodeReverse(t.bytes)).toEqual(t.expected); } }); }); diff --git a/src/plugins/profiling/server/routes/stacktrace.ts b/src/plugins/profiling/server/routes/stacktrace.ts index 92e8cb9c8a64d8..5b656ca0c01cc3 100644 --- a/src/plugins/profiling/server/routes/stacktrace.ts +++ b/src/plugins/profiling/server/routes/stacktrace.ts @@ -21,38 +21,109 @@ import { logExecutionLatency } from './logger'; import { getHitsItems, getDocs } from './compat'; const traceLRU = new LRUCache({ max: 20000 }); -const frameIDToFileIDCache = new LRUCache({ max: 100000 }); - -// convertFrameIDToFileID extracts the FileID from the FrameID and returns as base64url string. -export function extractFileIDFromFrameID(frameID: string): string { - const fileIDChunk = frameID.slice(0, 23); - let fileID = frameIDToFileIDCache.get(fileIDChunk) as string; - if (fileID) return fileID; - - // Step 1: Convert the base64-encoded frameID to an array of 22 bytes. - // We use 'base64url' instead of 'base64' because frameID is encoded URL-friendly. - // The first 16 bytes contain the FileID. - const buf = Buffer.from(fileIDChunk, 'base64url'); - - // Convert the FileID bytes into base64 with URL-friendly encoding. - // We have to manually append '==' since we use the FileID string for - // comparing / looking up the FileID strings in the ES indices, which have - // the '==' appended. - // We may want to remove '==' in the future to reduce the uncompressed storage size by 10%. - fileID = buf.toString('base64url', 0, 16) + '=='; - frameIDToFileIDCache.set(fileIDChunk, fileID); - return fileID; +const fileIDChunkToFileIDCache = new LRUCache({ max: 100000 }); + +const BASE64_FILE_ID_LENGTH = 22; +const BASE64_FRAME_ID_LENGTH = 32; + +export interface EncodedStackTrace { + // This field is a base64-encoded byte string. The string represents a + // serialized list of frame IDs. Each frame ID is composed of two + // concatenated values: a 16-byte file ID and an 8-byte address or line + // number (depending on the context of the downstream reader). + // + // Frame ID #1 Frame ID #2 + // +----------------+--------+----------------+--------+---- + // | File ID | Addr | File ID | Addr | + // +----------------+--------+----------------+--------+---- + FrameID: string; + + // This field is a run-length encoding of a list of uint8s. The order is + // reversed from the original input. + Type: string; } -// extractFileIDArrayFromFrameIDArray extracts all FileIDs from the array of FrameIDs -// and returns them as an array of base64url encoded strings. The order of this array -// corresponds to the order of the input array. -function extractFileIDArrayFromFrameIDArray(frameIDs: string[]): string[] { - const fileIDs = Array(frameIDs.length); - for (let i = 0; i < frameIDs.length; i++) { - fileIDs[i] = extractFileIDFromFrameID(frameIDs[i]); +// runLengthDecodeReverse decodes a run-length encoding for the reversed input array. +// +// The input is a binary stream of 2-byte pairs (first byte is the length and the +// second byte is the binary representation of the object). The output is a list of +// uint8s in reverse order. +// +// E.g. byte array [5, 0, 2, 2] is converted into an uint8 array like +// [2, 2, 0, 0, 0, 0, 0]. +export function runLengthDecodeReverse(input: Buffer, outputSize?: number): number[] { + let size; + + if (typeof outputSize === 'undefined') { + size = 0; + for (let i = 0; i < input.length; i += 2) { + size += input[i]; + } + } else { + size = outputSize; } - return fileIDs; + + const output: number[] = new Array(size); + + let idx = 0; + for (let i = input.length - 1; i >= 1; i -= 2) { + for (let j = 0; j < input[i - 1]; j++) { + output[idx] = input[i]; + idx++; + } + } + + return output; +} + +// decodeStackTrace unpacks an encoded stack trace from Elasticsearch +export function decodeStackTrace(input: EncodedStackTrace): StackTrace { + const countsFrameIDs = input.FrameID.length / BASE64_FRAME_ID_LENGTH; + const fileIDs: string[] = new Array(countsFrameIDs); + const frameIDs: string[] = new Array(countsFrameIDs); + + // Step 1: Convert the base64-encoded frameID list into two separate + // lists (frame IDs and file IDs), both of which are also base64-encoded. + // + // To get the frame ID, we grab the next 32 bytes. + // + // To get the file ID, we grab the first 22 bytes of the frame ID. + // However, since the file ID is base64-encoded using 21.33 bytes + // (16 * 4 / 3), then the 22 bytes have an extra 4 bits from the + // address (see diagram in definition of EncodedStackTrace). + for (let i = 0; i < input.FrameID.length; i += BASE64_FRAME_ID_LENGTH) { + const frameID = input.FrameID.slice(i, i + BASE64_FRAME_ID_LENGTH); + const fileIDChunk = frameID.slice(0, BASE64_FILE_ID_LENGTH); + const fileID = fileIDChunkToFileIDCache.get(fileIDChunk) as string; + const j = Math.floor(i / BASE64_FRAME_ID_LENGTH); + + frameIDs[j] = frameID; + + if (fileID) { + fileIDs[j] = fileID; + } else { + const buf = Buffer.from(fileIDChunk, 'base64url'); + + // We have to manually append '==' since we use the FileID string for + // comparing / looking up the FileID strings in the ES indices, which have + // the '==' appended. + // + // We may want to remove '==' in the future to reduce the uncompressed + // storage size by 10%. + fileIDs[j] = buf.toString('base64url', 0, 16) + '=='; + fileIDChunkToFileIDCache.set(fileIDChunk, fileIDs[j]); + } + } + + // Step 2: Convert the run-length byte encoding into a list of uint8s. + const types = Buffer.from(input.Type, 'base64url'); + const typeIDs = runLengthDecodeReverse(types, countsFrameIDs); + + return { + FileID: fileIDs, + FrameID: frameIDs, + Type: typeIDs, + } as StackTrace; } export async function searchStackTraces( @@ -108,17 +179,12 @@ export async function searchStackTraces( await logExecutionLatency(logger, 'processing data', async () => { const traces = stackResponses.flatMap((response) => getHitsItems(response)); for (const trace of traces) { - const frameIDs = trace.fields.FrameID as string[]; - const fileIDs = extractFileIDArrayFromFrameIDArray(frameIDs); - stackTraces.set(trace._id, { - FileID: fileIDs, - FrameID: frameIDs, - Type: trace.fields.Type, - }); - for (const frameID of frameIDs) { + const stackTrace = decodeStackTrace(trace.fields as EncodedStackTrace); + stackTraces.set(trace._id, stackTrace); + for (const frameID of stackTrace.FrameID) { stackFrameDocIDs.add(frameID); } - for (const fileID of fileIDs) { + for (const fileID of stackTrace.FileID) { executableDocIDs.add(fileID); } } @@ -181,12 +247,7 @@ export async function mgetStackTraces( const traceid = trace._id as StackTraceID; let stackTrace = traceLRU.get(traceid) as StackTrace; if (!stackTrace) { - const frameIDs = trace._source.FrameID as string[]; - stackTrace = { - FileID: extractFileIDArrayFromFrameIDArray(frameIDs), - FrameID: frameIDs, - Type: trace._source.Type, - }; + stackTrace = decodeStackTrace(trace._source as EncodedStackTrace); traceLRU.set(traceid, stackTrace); } diff --git a/src/plugins/profiling/server/routes/topn.test.ts b/src/plugins/profiling/server/routes/topn.test.ts index ae01e38bdcb5c7..fea7b04219d315 100644 --- a/src/plugins/profiling/server/routes/topn.test.ts +++ b/src/plugins/profiling/server/routes/topn.test.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { loggerMock } from '@kbn/logging/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; import { topNElasticSearchQuery } from './topn'; import { ElasticsearchClient, kibanaResponseFactory } from '../../../../core/server'; import { coreMock } from '../../../../core/server/mocks'; @@ -16,7 +16,7 @@ const anyQuery = 'any::query'; const index = 'test'; const testAgg = { aggs: { test: {} } }; -jest.mock('./mappings', () => ({ +jest.mock('./query', () => ({ newProjectTimeQuery: (proj: string, from: string, to: string) => { return anyQuery; }, diff --git a/src/plugins/profiling/server/routes/topn.ts b/src/plugins/profiling/server/routes/topn.ts index 5bb084a0fec9cd..cc63f9f2f02554 100644 --- a/src/plugins/profiling/server/routes/topn.ts +++ b/src/plugins/profiling/server/routes/topn.ts @@ -18,7 +18,7 @@ import { groupStackFrameMetadataByStackTrace, StackTraceID } from '../../common/ import { createTopNBucketsByDate } from '../../common/topn'; import { findDownsampledIndex } from './downsampling'; import { logExecutionLatency } from './logger'; -import { autoHistogramSumCountOnGroupByField, newProjectTimeQuery } from './mappings'; +import { autoHistogramSumCountOnGroupByField, newProjectTimeQuery } from './query'; import { mgetExecutables, mgetStackFrames, mgetStackTraces } from './stacktrace'; import { getClient, getAggs } from './compat';