[APM] Synthtrace high cardinality scenarios (#174746)

While working on #127036 I
needed scenarios for creating a high number of services, transactions
and errors. I've removed some unnecessary stuff in the scenarios that we
already have elsewhere to make them run faster.

packages/kbn-apm-synthtrace/src/scenarios/helpers/random_names.ts

@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const randomGreekishNames = [
+  'Adonis',
+  'Agamemnon',
+  'Ajax',
+  'Alexander',
+  'Aphrodite',
+  'Apollo',
+  'Ares',
+  'Aristophanes',
+  'Artemis',
+  'Athena',
+  'Atlas',
+  'Boreas',
+  'Calliope',
+  'Cassandra',
+  'Clio',
+  'Daphne',
+  'Demeter',
+  'Dionysius',
+  'Dionysus',
+  'Eileithyia',
+  'Electra',
+  'Eos',
+  'Erato',
+  'Erebos',
+  'Eros',
+  'Euterpe',
+  'Gaia',
+  'Hades',
+  'Hecate',
+  'Helios',
+  'Hephaestus',
+  'Hera',
+  'Heracles (Hercules)',
+  'Hercules',
+  'Hermes',
+  'Hestia',
+  'Hypatia',
+  'Hypnos',
+  'Iphigenia',
+  'Iris',
+  'Kratos',
+  'Leonidas',
+  'Medusa',
+  'Mnemosyne',
+  'Morpheus',
+  'Narcissus',
+  'Nemesis',
+  'Nike',
+  'Notus',
+  'Nyx',
+  'Oceanus',
+  'Odysseus',
+  'Orpheus',
+  'Pan',
+  'Pandora',
+  'Penelope',
+  'Pericles',
+  'Persephone',
+  'Perseus',
+  'Phoebe',
+  'Polyphemus',
+  'Pontus',
+  'Poseidon',
+  'Priam',
+  'Prometheus',
+  'Rhea',
+  'Sappho',
+  'Selene',
+  'Terpsichore',
+  'Tethys',
+  'Thalia',
+  'Thanatos',
+  'Theia',
+  'Theseus',
+  'Thetis',
+  'Triton',
+  'Tyche',
+  'Uranus',
+  'Zephyrus',
+  'Zeus',
+  'Augustus',
+  'Bacchus',
+  'Brutus',
+  'Caesar',
+  'Caligula',
+  'Caracalla',
+  'Cassius',
+  'Cato',
+  'Cicero',
+  'Cleopatra',
+  'Commodus',
+  'Constantine',
+  'Diana',
+  'Domitian',
+  'Hadrian',
+  'Horace',
+  'Julius',
+  'Juno',
+  'Jupiter',
+  'Livy',
+  'Marcus Aurelius',
+  'Mars',
+  'Mercury',
+  'Minerva',
+  'Neptune',
+  'Nero',
+  'Octavian',
+  'Ovid',
+  'Pliny',
+  'Pluto',
+  'Pompey',
+  'Remus',
+  'Romulus',
+  'Saturn',
+  'Scipio',
+  'Seneca',
+  'Spartacus',
+  'Theodosius',
+  'Tiberius',
+  'Titus',
+  'Trajan',
+  'Venus',
+  'Vespasian',
+  'Vesta',
+  'Virgil',
+];
+
+export function getRandomNameForIndex(index: number) {
+  return randomGreekishNames[index % randomGreekishNames.length];
+}

packages/kbn-apm-synthtrace/src/scenarios/many_errors.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import { ApmFields, apm } from '@kbn/apm-synthtrace-client';
+import { Scenario } from '../cli/scenario';
+import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
+import { withClient } from '../lib/utils/with_client';
+import { getRandomNameForIndex } from './helpers/random_names';
+
+const ENVIRONMENT = getSynthtraceEnvironment(__filename);
+
+const scenario: Scenario<ApmFields> = async (runOptions) => {
+  const { logger } = runOptions;
+
+  const severities = ['critical', 'error', 'warning', 'info', 'debug', 'trace'];
+
+  return {
+    generate: ({ range, clients: { apmEsClient } }) => {
+      const transactionName = 'DELETE /api/orders/{id}';
+
+      const instance = apm
+        .service({ name: `synth-node`, environment: ENVIRONMENT, agentName: 'nodejs' })
+        .instance('instance');
+
+      const failedTraceEvents = range
+        .interval('1m')
+        .rate(2000)
+        .generator((timestamp, index) => {
+          const severity = severities[index % severities.length];
+          const errorMessage = `${severity}: ${getRandomNameForIndex(index)} ${index}`;
+          return instance
+            .transaction({ transactionName })
+            .timestamp(timestamp)
+            .duration(1000)
+            .failure()
+            .errors(
+              instance.error({ message: errorMessage, type: 'My Type' }).timestamp(timestamp + 50)
+            );
+        });
+
+      return withClient(
+        apmEsClient,
+        logger.perf('generating_apm_events', () => failedTraceEvents)
+      );
+    },
+  };
+};
+
+export default scenario;

packages/kbn-apm-synthtrace/src/scenarios/many_instances.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ApmFields, apm, Instance } from '@kbn/apm-synthtrace-client';
+import { random, times } from 'lodash';
+import { Scenario } from '../cli/scenario';
+import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
+import { withClient } from '../lib/utils/with_client';
+import { getRandomNameForIndex } from './helpers/random_names';
+
+const ENVIRONMENT = getSynthtraceEnvironment(__filename);
+
+const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts = { instances: 2000 } }) => {
+  const numInstances = scenarioOpts.instances;
+  const agentVersions = ['2.1.0', '2.0.0', '1.15.0', '1.14.0', '1.13.1'];
+  const language = 'go';
+  const serviceName = 'synth-many-instances';
+  const transactionName = 'GET /order/{id}';
+
+  return {
+    generate: ({ range, clients: { apmEsClient } }) => {
+      const instances = times(numInstances).map((index) => {
+        const agentVersion = agentVersions[index % agentVersions.length];
+        const randomName = getRandomNameForIndex(index);
+        return apm
+          .service({
+            name: serviceName,
+            environment: ENVIRONMENT,
+            agentName: language,
+          })
+          .instance(`instance-${randomName}-${index}`)
+          .defaults({ 'agent.version': agentVersion, 'service.language.name': language });
+      });
+
+      const instanceSpans = (instance: Instance) => {
+        const hasHighDuration = Math.random() > 0.5;
+        const throughput = random(1, 10);
+
+        const traces = range.ratePerMinute(throughput).generator((timestamp) => {
+          const parentDuration = hasHighDuration ? random(1000, 5000) : random(100, 1000);
+          const generateError = random(1, 4) % 3 === 0;
+          const span = instance
+            .transaction({ transactionName })
+            .timestamp(timestamp)
+            .duration(parentDuration);
+
+          return !generateError
+            ? span.success()
+            : span
+                .failure()
+                .errors(
+                  instance
+                    .error({ message: `No handler for ${transactionName}` })
+                    .timestamp(timestamp + 50)
+                );
+        });
+
+        const cpuPct = random(0, 1);
+        const memoryFree = random(0, 1000);
+        const metricsets = range
+          .interval('30s')
+          .rate(1)
+          .generator((timestamp) =>
+            instance
+              .appMetrics({
+                'system.memory.actual.free': memoryFree,
+                'system.memory.total': 1000,
+                'system.cpu.total.norm.pct': cpuPct,
+                'system.process.cpu.total.norm.pct': 0.7,
+              })
+              .timestamp(timestamp)
+          );
+
+        return [traces, metricsets];
+      };
+
+      return withClient(
+        apmEsClient,
+        logger.perf('generating_apm_events', () => instances.flatMap(instanceSpans))
+      );
+    },
+  };
+};
+
+export default scenario;