[7.x] Add usage collection for savedObject tagging (#83160) (#83924)

* Add usage collection for savedObject tagging (#83160)

* add so tagging usage collection

* update telemetry mappings

* fix types

* remove check on esClient presence

* update schema and README

* fix mappings for backport

x-pack/plugins/saved_objects_tagging/README.md

@@ -1,3 +1,53 @@
 # SavedObjectsTagging
 
-Add tagging capability to saved objects
+Add tagging capability to saved objects
+
+## Integrating tagging on a new object type
+
+In addition to use the UI api to plug the tagging feature in your application, there is a couple
+things that needs to be done on the server:
+
+### Add read-access to the `tag` SO type to your feature's capabilities
+
+In order to be able to fetch the tags assigned to an object, the user must have read permission
+for the `tag` saved object type. Which is why all features relying on SO tagging must update
+their capabilities.
+
+```typescript
+features.registerKibanaFeature({
+  id: 'myFeature',
+  // ...
+  privileges: {
+    all: {
+      // ...
+      savedObject: {
+        all: ['some-type'],
+        read: ['tag'], // <-- HERE
+      },
+    },
+    read: {
+      // ...
+      savedObject: {
+        all: [],
+        read: ['some-type', 'tag'], // <-- AND HERE
+      },
+    },
+  },
+});
+```
+
+### Update the SOT telemetry collector schema to add the new type
+
+The schema is located here: `x-pack/plugins/saved_objects_tagging/server/usage/schema.ts`. You
+just need to add the name of the SO type you are adding.
+
+```ts
+export const tagUsageCollectorSchema: MakeSchemaFrom<TaggingUsageData> = {
+  // ...
+  types: {
+    dashboard: perTypeSchema,
+    visualization: perTypeSchema,
+    // <-- add your type here
+  },
+};
+```

x-pack/plugins/saved_objects_tagging/kibana.json

@@ -6,5 +6,6 @@
   "ui": true,
   "configPath": ["xpack", "saved_object_tagging"],
   "requiredPlugins": ["features", "management", "savedObjectsTaggingOss"],
-  "requiredBundles": ["kibanaReact"]
+  "requiredBundles": ["kibanaReact"],
+  "optionalPlugins": ["usageCollection"]
 }

x-pack/plugins/saved_objects_tagging/server/plugin.test.mocks.ts

@@ -8,3 +8,8 @@ export const registerRoutesMock = jest.fn();
 jest.doMock('./routes', () => ({
   registerRoutes: registerRoutesMock,
 }));
+
+export const createTagUsageCollectorMock = jest.fn();
+jest.doMock('./usage', () => ({
+  createTagUsageCollector: createTagUsageCollectorMock,
+}));

x-pack/plugins/saved_objects_tagging/server/plugin.test.ts

@@ -4,20 +4,32 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { registerRoutesMock } from './plugin.test.mocks';
+import { registerRoutesMock, createTagUsageCollectorMock } from './plugin.test.mocks';
 
 import { coreMock } from '../../../../src/core/server/mocks';
 import { featuresPluginMock } from '../../features/server/mocks';
+import { usageCollectionPluginMock } from '../../../../src/plugins/usage_collection/server/mocks';
 import { SavedObjectTaggingPlugin } from './plugin';
 import { savedObjectsTaggingFeature } from './features';
 
 describe('SavedObjectTaggingPlugin', () => {
   let plugin: SavedObjectTaggingPlugin;
   let featuresPluginSetup: ReturnType<typeof featuresPluginMock.createSetup>;
+  let usageCollectionSetup: ReturnType<typeof usageCollectionPluginMock.createSetupContract>;
 
   beforeEach(() => {
     plugin = new SavedObjectTaggingPlugin(coreMock.createPluginInitializerContext());
     featuresPluginSetup = featuresPluginMock.createSetup();
+    usageCollectionSetup = usageCollectionPluginMock.createSetupContract();
+    // `usageCollection` 'mocked' implementation use the real `CollectorSet` implementation
+    // that throws when registering things that are not collectors.
+    // We just want to assert that it was called here, so jest.fn is fine.
+    usageCollectionSetup.registerCollector = jest.fn();
+  });
+
+  afterEach(() => {
+    registerRoutesMock.mockReset();
+    createTagUsageCollectorMock.mockReset();
   });
 
   describe('#setup', () => {
@@ -43,5 +55,18 @@ describe('SavedObjectTaggingPlugin', () => {
         savedObjectsTaggingFeature
       );
     });
+
+    it('registers the usage collector if `usageCollection` is present', async () => {
+      const tagUsageCollector = Symbol('saved_objects_tagging');
+      createTagUsageCollectorMock.mockReturnValue(tagUsageCollector);
+
+      await plugin.setup(coreMock.createSetup(), {
+        features: featuresPluginSetup,
+        usageCollection: usageCollectionSetup,
+      });
+
+      expect(usageCollectionSetup.registerCollector).toHaveBeenCalledTimes(1);
+      expect(usageCollectionSetup.registerCollector).toHaveBeenCalledWith(tagUsageCollector);
+    });
   });
 });

x-pack/plugins/saved_objects_tagging/server/plugin.ts

@@ -4,22 +4,36 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { CoreSetup, CoreStart, PluginInitializerContext, Plugin } from 'src/core/server';
+import { Observable } from 'rxjs';
+import {
+  CoreSetup,
+  CoreStart,
+  PluginInitializerContext,
+  Plugin,
+  SharedGlobalConfig,
+} from 'src/core/server';
 import { PluginSetupContract as FeaturesPluginSetup } from '../../features/server';
+import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server';
 import { savedObjectsTaggingFeature } from './features';
 import { tagType } from './saved_objects';
 import { ITagsRequestHandlerContext } from './types';
-import { registerRoutes } from './routes';
 import { TagsRequestHandlerContext } from './request_handler_context';
+import { registerRoutes } from './routes';
+import { createTagUsageCollector } from './usage';
 
 interface SetupDeps {
   features: FeaturesPluginSetup;
+  usageCollection?: UsageCollectionSetup;
 }
 
 export class SavedObjectTaggingPlugin implements Plugin<{}, {}, SetupDeps, {}> {
-  constructor(context: PluginInitializerContext) {}
+  private readonly legacyConfig$: Observable<SharedGlobalConfig>;
 
-  public setup({ savedObjects, http }: CoreSetup, { features }: SetupDeps) {
+  constructor(context: PluginInitializerContext) {
+    this.legacyConfig$ = context.config.legacy.globalConfig$;
+  }
+
+  public setup({ savedObjects, http }: CoreSetup, { features, usageCollection }: SetupDeps) {
     savedObjects.registerType(tagType);
 
     const router = http.createRouter();
@@ -34,6 +48,15 @@ export class SavedObjectTaggingPlugin implements Plugin<{}, {}, SetupDeps, {}> {
 
     features.registerKibanaFeature(savedObjectsTaggingFeature);
 
+    if (usageCollection) {
+      usageCollection.registerCollector(
+        createTagUsageCollector({
+          usageCollection,
+          legacyConfig$: this.legacyConfig$,
+        })
+      );
+    }
+
     return {};
   }
 

x-pack/plugins/saved_objects_tagging/server/usage/fetch_tag_usage_data.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ElasticsearchClient } from 'src/core/server';
+import { TaggingUsageData, ByTypeTaggingUsageData } from './types';
+
+/**
+ * Manual type reflection of the `tagDataAggregations` resulting payload
+ */
+interface AggregatedTagUsageResponseBody {
+  aggregations: {
+    by_type: {
+      buckets: Array<{
+        key: string;
+        doc_count: number;
+        nested_ref: {
+          tag_references: {
+            doc_count: number;
+            tag_id: {
+              buckets: Array<{
+                key: string;
+                doc_count: number;
+              }>;
+            };
+          };
+        };
+      }>;
+    };
+  };
+}
+
+export const fetchTagUsageData = async ({
+  esClient,
+  kibanaIndex,
+}: {
+  esClient: ElasticsearchClient;
+  kibanaIndex: string;
+}): Promise<TaggingUsageData> => {
+  const { body } = await esClient.search<AggregatedTagUsageResponseBody>({
+    index: [kibanaIndex],
+    ignore_unavailable: true,
+    filter_path: 'aggregations',
+    body: {
+      size: 0,
+      query: {
+        bool: {
+          must: [hasTagReferenceClause],
+        },
+      },
+      aggs: tagDataAggregations,
+    },
+  });
+
+  const byTypeUsages: Record<string, ByTypeTaggingUsageData> = {};
+  const allUsedTags = new Set<string>();
+  let totalTaggedObjects = 0;
+
+  const typeBuckets = body.aggregations.by_type.buckets;
+  typeBuckets.forEach((bucket) => {
+    const type = bucket.key;
+    const taggedDocCount = bucket.doc_count;
+    const usedTagIds = bucket.nested_ref.tag_references.tag_id.buckets.map(
+      (tagBucket) => tagBucket.key
+    );
+
+    totalTaggedObjects += taggedDocCount;
+    usedTagIds.forEach((tagId) => allUsedTags.add(tagId));
+
+    byTypeUsages[type] = {
+      taggedObjects: taggedDocCount,
+      usedTags: usedTagIds.length,
+    };
+  });
+
+  return {
+    usedTags: allUsedTags.size,
+    taggedObjects: totalTaggedObjects,
+    types: byTypeUsages,
+  };
+};
+
+const hasTagReferenceClause = {
+  nested: {
+    path: 'references',
+    query: {
+      bool: {
+        must: [
+          {
+            term: {
+              'references.type': 'tag',
+            },
+          },
+        ],
+      },
+    },
+  },
+};
+
+const tagDataAggregations = {
+  by_type: {
+    terms: {
+      field: 'type',
+    },
+    aggs: {
+      nested_ref: {
+        nested: {
+          path: 'references',
+        },
+        aggs: {
+          tag_references: {
+            filter: {
+              term: {
+                'references.type': 'tag',
+              },
+            },
+            aggs: {
+              tag_id: {
+                terms: {
+                  field: 'references.id',
+                },
+              },
+            },
+          },
+        },
+      },
+    },
+  },
+};

x-pack/plugins/saved_objects_tagging/server/usage/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { createTagUsageCollector } from './tag_usage_collector';

x-pack/plugins/saved_objects_tagging/server/usage/schema.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { MakeSchemaFrom } from '../../../../../src/plugins/usage_collection/server';
+import { TaggingUsageData, ByTypeTaggingUsageData } from './types';
+
+const perTypeSchema: MakeSchemaFrom<ByTypeTaggingUsageData> = {
+  usedTags: { type: 'integer' },
+  taggedObjects: { type: 'integer' },
+};
+
+export const tagUsageCollectorSchema: MakeSchemaFrom<TaggingUsageData> = {
+  usedTags: { type: 'integer' },
+  taggedObjects: { type: 'integer' },
+
+  types: {
+    dashboard: perTypeSchema,
+    visualization: perTypeSchema,
+    map: perTypeSchema,
+  },
+};

x-pack/plugins/saved_objects_tagging/server/usage/tag_usage_collector.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Observable } from 'rxjs';
+import { take } from 'rxjs/operators';
+import { SharedGlobalConfig } from 'src/core/server';
+import { UsageCollectionSetup } from '../../../../../src/plugins/usage_collection/server';
+import { TaggingUsageData } from './types';
+import { fetchTagUsageData } from './fetch_tag_usage_data';
+import { tagUsageCollectorSchema } from './schema';
+
+export const createTagUsageCollector = ({
+  usageCollection,
+  legacyConfig$,
+}: {
+  usageCollection: UsageCollectionSetup;
+  legacyConfig$: Observable<SharedGlobalConfig>;
+}) => {
+  return usageCollection.makeUsageCollector<TaggingUsageData>({
+    type: 'saved_objects_tagging',
+    isReady: () => true,
+    schema: tagUsageCollectorSchema,
+    fetch: async ({ esClient }) => {
+      const { kibana } = await legacyConfig$.pipe(take(1)).toPromise();
+      return fetchTagUsageData({ esClient, kibanaIndex: kibana.index });
+    },
+  });
+};