Add plugin list to docs that is generated from the repo

docs/developer/architecture/index.asciidoc

@@ -17,9 +17,12 @@ A few notable services are called out below.
 * <<development-security>>
 * <<add-data-tutorials>>
 * <<development-visualize-index>>
+* <<code-exploration>>
 
 include::add-data-tutorials.asciidoc[]
 
 include::development-visualize-index.asciidoc[]
 
 include::security/index.asciidoc[]
+
+include::code-exploration.asciidoc[]

packages/kbn-dev-utils/src/index.ts

@@ -39,3 +39,5 @@ export { KbnClient } from './kbn_client';
 export * from './axios';
 export * from './stdio';
 export * from './ci_stats_reporter';
+export * from './plugin_list';
+export * from './simple_kibana_platform_plugin_discovery';

packages/kbn-dev-utils/src/plugin_list/discover_plugins.ts

@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Path from 'path';
+import Fs from 'fs';
+
+import MarkdownIt from 'markdown-it';
+import cheerio from 'cheerio';
+
+import { REPO_ROOT } from '../repo_root';
+import { simpleKibanaPlatformPluginDiscovery } from '../simple_kibana_platform_plugin_discovery';
+
+export interface Plugin {
+  id: string;
+  relativeDir?: string;
+  relativeReadmePath?: string;
+  readmeSnippet?: string;
+}
+
+export type Plugins = Plugin[];
+
+const getReadmeName = (directory: string) =>
+  Fs.readdirSync(directory).find((name) => name.toLowerCase() === 'readme.md');
+
+export const discoverPlugins = (pluginsRootDir: string): Plugins =>
+  simpleKibanaPlatformPluginDiscovery([pluginsRootDir], []).map(
+    ({ directory, manifest: { id } }): Plugin => {
+      const readmeName = getReadmeName(directory);
+
+      let relativeReadmePath: string | undefined;
+      let readmeSnippet: string | undefined;
+      if (readmeName) {
+        const readmePath = Path.resolve(directory, readmeName);
+        relativeReadmePath = Path.relative(REPO_ROOT, readmePath);
+
+        const md = new MarkdownIt();
+        const parsed = md.render(Fs.readFileSync(readmePath, 'utf8'));
+        const $ = cheerio.load(parsed);
+
+        const firstParagraph = $('p')[0];
+        if (firstParagraph) {
+          readmeSnippet = $(firstParagraph).text();
+        }
+      }
+
+      return {
+        id,
+        relativeReadmePath,
+        relativeDir: relativeReadmePath || Path.relative(REPO_ROOT, directory),
+        readmeSnippet,
+      };
+    }
+  );

packages/kbn-dev-utils/src/plugin_list/generate_plugin_list.ts

@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Plugins } from './discover_plugins';
+
+function* printPlugins(plugins: Plugins) {
+  for (const plugin of plugins) {
+    const path = plugin.relativeReadmePath || plugin.relativeDir;
+    yield '';
+    yield `- {kib-repo}blob/{branch}/${path}[${plugin.id}]`;
+
+    if (!plugin.relativeReadmePath || plugin.readmeSnippet) {
+      yield '';
+      yield plugin.readmeSnippet || 'WARNING: Missing README.';
+      yield '';
+    }
+  }
+}
+
+export function generatePluginList(ossPlugins: Plugins, xpackPlugins: Plugins) {
+  return `
+[[code-exploration]]
+=== Exploring Kibana code
+
+The goals of our folder heirarchy are:
+
+- Easy for developers to know where to add new services, plugins and applications.
+- Easy for developers to know where to find the code from services, plugins and applications.
+- Easy to browse and understand our folder structure.
+
+To that aim, we strive to:
+
+- Avoid too many files in any given folder.
+- Choose clear, unambigious folder names.
+- Organize by domain.
+- Every folder should contain a README that describes the contents of that folder.
+
+[discrete]
+[[kibana-services-applications]]
+==== Services and Applications
+
+[discrete]
+===== src/plugins
+${Array.from(printPlugins(ossPlugins)).join('\n')}
+
+[discrete]
+===== x-pack/plugins
+${Array.from(printPlugins(xpackPlugins)).join('\n')}
+`;
+}

packages/kbn-dev-utils/src/plugin_list/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export * from './run_plugin_list_cli';

packages/kbn-dev-utils/src/plugin_list/run_plugin_list_cli.ts

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Path from 'path';
+import Fs from 'fs';
+
+import { run } from '../run';
+import { REPO_ROOT } from '../repo_root';
+
+import { discoverPlugins } from './discover_plugins';
+import { generatePluginList } from './generate_plugin_list';
+
+const OSS_PLUGIN_DIR = Path.resolve(REPO_ROOT, 'src/plugins');
+const XPACK_PLUGIN_DIR = Path.resolve(REPO_ROOT, 'x-pack/plugins');
+const OUTPUT_PATH = Path.resolve(
+  REPO_ROOT,
+  'docs/developer/architecture/code-exploration.asciidoc'
+);
+
+export function runPluginListCli() {
+  run(async ({ log }) => {
+    log.info('looking for oss plugins');
+    const ossPlugins = discoverPlugins(OSS_PLUGIN_DIR);
+    log.success(`found ${ossPlugins.length} plugins`);
+
+    log.info('looking for x-pack plugins');
+    const xpackPlugins = discoverPlugins(XPACK_PLUGIN_DIR);
+    log.success(`found ${xpackPlugins.length} plugins`);
+
+    log.info('writing plugin list to', OUTPUT_PATH);
+    Fs.writeFileSync(OUTPUT_PATH, generatePluginList(ossPlugins, xpackPlugins));
+  });
+}

packages/kbn-dev-utils/src/simple_kibana_platform_plugin_discovery.ts

@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Path from 'path';
+
+import globby from 'globby';
+import loadJsonFile from 'load-json-file';
+
+export interface KibanaPlatformPlugin {
+  readonly directory: string;
+  readonly manifestPath: string;
+  readonly manifest: {
+    id: string;
+    [key: string]: unknown;
+  };
+}
+
+/**
+ * Helper to find the new platform plugins.
+ */
+export function simpleKibanaPlatformPluginDiscovery(scanDirs: string[], paths: string[]) {
+  const patterns = Array.from(
+    new Set([
+      // find kibana.json files up to 5 levels within the scan dir
+      ...scanDirs.reduce(
+        (acc: string[], dir) => [
+          ...acc,
+          `${dir}/*/kibana.json`,
+          `${dir}/*/*/kibana.json`,
+          `${dir}/*/*/*/kibana.json`,
+          `${dir}/*/*/*/*/kibana.json`,
+          `${dir}/*/*/*/*/*/kibana.json`,
+        ],
+        []
+      ),
+      ...paths.map((path) => `${path}/kibana.json`),
+    ])
+  );
+
+  const manifestPaths = globby.sync(patterns, { absolute: true }).map((path) =>
+    // absolute paths returned from globby are using normalize or something so the path separators are `/` even on windows, Path.resolve solves this
+    Path.resolve(path)
+  );
+
+  return manifestPaths.map(
+    (manifestPath): KibanaPlatformPlugin => {
+      if (!Path.isAbsolute(manifestPath)) {
+        throw new TypeError('expected new platform manifest path to be absolute');
+      }
+
+      const manifest = loadJsonFile.sync(manifestPath);
+      if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) {
+        throw new TypeError('expected new platform plugin manifest to be a JSON encoded object');
+      }
+
+      if (typeof manifest.id !== 'string') {
+        throw new TypeError('expected new platform plugin manifest to have a string id');
+      }
+
+      return {
+        directory: Path.dirname(manifestPath),
+        manifestPath,
+        manifest: {
+          ...manifest,
+          id: manifest.id,
+        },
+      };
+    }
+  );
+}