From 6d223174bff9ab82a2777df820bdc2594c01d45a Mon Sep 17 00:00:00 2001 From: Kai Franz Date: Tue, 10 Sep 2024 13:41:41 -0700 Subject: [PATCH] [BACKPORT pg15-cherrypicks][#23706] YSQL: Add table-level catcache Prometheus metrics Summary: Original commit: 9889df753 / D37633 PG15 removes the YB pinned objects in bea1ffbfcbba5b8b56ef0275ffc2c702992987cc. YB master 9889df753be54d355f244c53b368aa9989a0a4b3 conflicts with this (false conflict, we keep the PG15 code here). There is also the following logical conflict: PG15 adds 6 new catcaches: - 2 on pg_parameter_acl - 2 on pg_publication_namespace - 1 on pg_range - 1 on pg_statistic_ext_data Of these, pg_parameter_acl, pg_publication_namespace, and pg_statistic_ext_data are new catalog tables introduced in pg15. Adjust catalog cache metrics to account for new catalog tables/caches introduced in pg15, adapt according to YB pg15 merge 602cb2de9d855b8dfd84346f3a5f618b93c3839c. - src/postgres/src/backend/utils/cache/syscache.h: - enum YbCatalogCacheTable: - YB master 9889df753be54d355f244c53b368aa9989a0a4b3 adds this enum representing the catalog tables that have catcaches. - PG15 adds three new catalog tables. - Resolution: add the three new catalog tables to this enum. - src/postgres/src/backend/utils/cache/syscache.c: - `static const char *yb_cache_table_name_table` / `static YbCatalogCacheTable yb_catalog_cache_tables`: - YB master 9889df753be54d355f244c53b368aa9989a0a4b3 adds this list of all the PG catalog tables there are catcaches for. - PG15 adds three new catalog tables and six new catcaches. - Resolution: add the six entries corresponding to the new catcaches to these lists. - src/postgres/yb-extensions/yb_pg_metrics/yb_pg_metrics.c: - enum statementType: - YB master 9889df753be54d355f244c53b368aa9989a0a4b3 adds one CatCacheTableMisses entry per table that has a catcache (total 47 entries). - PG15 adds three new catalog tables. - Resolution: add the three new CatCacheTableMisses entries corresponding to the new catcaches to this enum (total 50 entries). - src/yb/yql/pgwrapper/pg_libpq-test.cc: - GetCatalogTableNameFromIndexName: - YB master commit 9889df753be54d355f244c53b368aa9989a0a4b3 adds a test that has a list of all the PG catalog tables. - PG15 adds three new catalog tables. - Resolution: add the three new catalog tables to this list. Conflict is related to c2719446217c08f3b16302a660f30ac1943fc710. - `src/postgres/yb-extensions/yb_pg_metrics/yb_pg_metrics.c`: - In c2719446217c08f3b16302a660f30ac1943fc710, are correcting the `enum statementType`'s `CatCacheIdMisses` section so that the end-of-section market points to the last enum in the section rather than being its own enum after the end of the section. In this revision, we are adding a new section, `CatCacheTableMisses`, to `enum statementType`, so the changes are unrelated. We make sure, however, that the end-of-section marker here points to the last enum in the section to avoid the issue addressed in c2719446217c08f3b16302a660f30ac1943fc710. Adds table-level Prometheus metrics for the number of catcache misses in addition to the current index-level ones added in D35792. These metrics are node-level, i.e. they will include the cache misses from all backends running on this node, but not the other nodes. They accessible via the `:13000/prometheus-metrics` endpoint. Including the metrics added in this revision, we now have three YSQL metrics that track catalog cache misses: - `CatalogCacheMisses` (no `table_name` field): Total number of cache misses across all catcaches. - `CatalogCacheMisses` (`table_name` field specified): Number of cache misses for the index specified by `table_name`. - `CatalogCacheTableMisses` (**new**): Number of cache misses for all catcaches on the table specified by `table_name`. Jira: DB-12616 Test Plan: ``` ./yb_build.sh --cxx-test pgwrapper_pg_libpq-test --gtest_filter PgLibPqTest.CatalogCacheIdMissMetricsTest ``` Manual test: ``` bin/yb-ctl create bin/ysqlsh yugabyte=# create table test(a int); CREATE TABLE yugabyte=# \c You are now connected to database "yugabyte" as user "yugabyte". yugabyte=# select * from t; a --- (0 rows) yugabyte=# \q wget 'http://127.0.0.1:13000/prometheus-metrics?reset_histograms=false&show_help=false' -O - 2>/dev/null | grep CatalogCacheTableMisses | grep count handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_aggregate",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_am",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_amop",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_amproc",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_attribute",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_auth_members",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_authid",exported_instance="kf-mbp-v47vl:9000"} 2 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_cast",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_class",exported_instance="kf-mbp-v47vl:9000"} 3 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_collation",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_constraint",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_conversion",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_database",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_default_acl",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_enum",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_event_trigger",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_foreign_data_wrapper",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_foreign_server",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_foreign_table",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_index",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_language",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_namespace",exported_instance="kf-mbp-v47vl:9000"} 2 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_opclass",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_operator",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_opfamily",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_partitioned_table",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_proc",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_publication",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_publication_rel",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_range",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_replication_origin",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_rewrite",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_sequence",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_statistic",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_statistic_ext",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_subscription",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_subscription_rel",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_tablespace",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_transform",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_ts_config",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_ts_config_map",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_ts_dict",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_ts_parser",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_ts_template",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_type",exported_instance="kf-mbp-v47vl:9000"} 1 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_user_mapping",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count{metric_id="yb.ysqlserver",metric_type="server",table_name="pg_yb_tablegroup",exported_instance="kf-mbp-v47vl:9000"} 0 1724889458674 ``` Reviewers: jason, tfoucher Reviewed By: jason Subscribers: yql Differential Revision: https://phorge.dev.yugabyte.com/D37871 --- .../src/backend/utils/cache/catcache.c | 8 + .../src/backend/utils/cache/syscache.c | 187 +++++++++++++++++ src/postgres/src/include/utils/catcache.h | 1 + src/postgres/src/include/utils/lsyscache.h | 1 + src/postgres/src/include/utils/syscache.h | 59 ++++++ .../yb_pg_metrics/yb_pg_metrics.c | 195 ++++++++++++------ .../webserver/pgsql_webserver_wrapper.cc | 25 +-- .../webserver/pgsql_webserver_wrapper.h | 5 +- src/yb/yql/pgwrapper/pg_libpq-test.cc | 195 +++++++++++++++--- 9 files changed, 570 insertions(+), 106 deletions(-) diff --git a/src/postgres/src/backend/utils/cache/catcache.c b/src/postgres/src/backend/utils/cache/catcache.c index 0ef4c2d90c94..f7cced2192dc 100644 --- a/src/postgres/src/backend/utils/cache/catcache.c +++ b/src/postgres/src/backend/utils/cache/catcache.c @@ -76,6 +76,7 @@ static CatCacheHeader *CacheHdr = NULL; static long YbNumCatalogCacheMisses; static long YbNumCatalogCacheIdMisses[SysCacheSize] = {0}; +static long YbNumCatalogCacheTableMisses[YbNumCatalogCacheTables] = {0}; static inline HeapTuple SearchCatCacheInternal(CatCache *cache, int nkeys, @@ -1806,6 +1807,7 @@ SearchCatCacheMiss(CatCache *cache, { YbNumCatalogCacheMisses++; YbNumCatalogCacheIdMisses[cache->id]++; + YbNumCatalogCacheTableMisses[YbGetCatalogCacheTableIdFromCacheId(cache->id)]++; } if (yb_debug_log_catcache_events) @@ -2658,6 +2660,12 @@ YbGetCatCacheIdMisses() return YbNumCatalogCacheIdMisses; } +long * +YbGetCatCacheTableMisses() +{ + return YbNumCatalogCacheTableMisses; +} + YbCatCListIterator YbCatCListIteratorBegin(CatCList *list) { diff --git a/src/postgres/src/backend/utils/cache/syscache.c b/src/postgres/src/backend/utils/cache/syscache.c index 0d27d45d9322..2def34f5686b 100644 --- a/src/postgres/src/backend/utils/cache/syscache.c +++ b/src/postgres/src/backend/utils/cache/syscache.c @@ -1170,6 +1170,158 @@ static const char *yb_cache_index_name_table[] = { static_assert(SysCacheSize == sizeof(yb_cache_index_name_table) / sizeof(const char *), "Wrong catalog cache number"); +/* List of all the tables that have caches on them */ +static const char *yb_cache_table_name_table[] = { + "pg_aggregate", + "pg_am", + "pg_amop", + "pg_amproc", + "pg_attribute", + "pg_auth_members", + "pg_authid", + "pg_cast", + "pg_class", + "pg_collation", + "pg_constraint", + "pg_conversion", + "pg_database", + "pg_default_acl", + "pg_enum", + "pg_event_trigger", + "pg_foreign_data_wrapper", + "pg_foreign_server", + "pg_foreign_table", + "pg_index", + "pg_language", + "pg_namespace", + "pg_opclass", + "pg_operator", + "pg_opfamily", + "pg_parameter_acl", + "pg_partitioned_table", + "pg_proc", + "pg_publication", + "pg_publication_namespace", + "pg_publication_rel", + "pg_range", + "pg_replication_origin", + "pg_rewrite", + "pg_sequence", + "pg_statistic", + "pg_statistic_ext", + "pg_statistic_ext_data", + "pg_subscription", + "pg_subscription_rel", + "pg_tablespace", + "pg_transform", + "pg_ts_config", + "pg_ts_config_map", + "pg_ts_dict", + "pg_ts_parser", + "pg_ts_template", + "pg_type", + "pg_user_mapping", + "pg_yb_tablegroup", +}; + +static_assert(YbNumCatalogCacheTables == + sizeof(yb_cache_table_name_table) / sizeof(const char *), + "yb_catalog_cache_table_name_table size mismatch"); + + +/* Maps cache id to the table id in yb_cache_table_name_table */ +static YbCatalogCacheTable yb_catalog_cache_tables[] = { + YbCatalogCacheTable_pg_aggregate, + YbCatalogCacheTable_pg_am, + YbCatalogCacheTable_pg_am, + YbCatalogCacheTable_pg_amop, + YbCatalogCacheTable_pg_amop, + YbCatalogCacheTable_pg_amproc, + YbCatalogCacheTable_pg_attribute, + YbCatalogCacheTable_pg_attribute, + YbCatalogCacheTable_pg_auth_members, + YbCatalogCacheTable_pg_auth_members, + YbCatalogCacheTable_pg_authid, + YbCatalogCacheTable_pg_authid, + YbCatalogCacheTable_pg_cast, + YbCatalogCacheTable_pg_opclass, + YbCatalogCacheTable_pg_opclass, + YbCatalogCacheTable_pg_collation, + YbCatalogCacheTable_pg_collation, + YbCatalogCacheTable_pg_conversion, + YbCatalogCacheTable_pg_conversion, + YbCatalogCacheTable_pg_constraint, + YbCatalogCacheTable_pg_conversion, + YbCatalogCacheTable_pg_database, + YbCatalogCacheTable_pg_default_acl, + YbCatalogCacheTable_pg_enum, + YbCatalogCacheTable_pg_enum, + YbCatalogCacheTable_pg_event_trigger, + YbCatalogCacheTable_pg_event_trigger, + YbCatalogCacheTable_pg_foreign_data_wrapper, + YbCatalogCacheTable_pg_foreign_data_wrapper, + YbCatalogCacheTable_pg_foreign_server, + YbCatalogCacheTable_pg_foreign_server, + YbCatalogCacheTable_pg_foreign_table, + YbCatalogCacheTable_pg_index, + YbCatalogCacheTable_pg_language, + YbCatalogCacheTable_pg_language, + YbCatalogCacheTable_pg_namespace, + YbCatalogCacheTable_pg_namespace, + YbCatalogCacheTable_pg_operator, + YbCatalogCacheTable_pg_operator, + YbCatalogCacheTable_pg_opfamily, + YbCatalogCacheTable_pg_opfamily, + YbCatalogCacheTable_pg_parameter_acl, + YbCatalogCacheTable_pg_parameter_acl, + YbCatalogCacheTable_pg_partitioned_table, + YbCatalogCacheTable_pg_proc, + YbCatalogCacheTable_pg_proc, + YbCatalogCacheTable_pg_publication, + YbCatalogCacheTable_pg_publication_namespace, + YbCatalogCacheTable_pg_publication_namespace, + YbCatalogCacheTable_pg_publication, + YbCatalogCacheTable_pg_publication_rel, + YbCatalogCacheTable_pg_publication_rel, + YbCatalogCacheTable_pg_range, + YbCatalogCacheTable_pg_range, + YbCatalogCacheTable_pg_class, + YbCatalogCacheTable_pg_class, + YbCatalogCacheTable_pg_replication_origin, + YbCatalogCacheTable_pg_replication_origin, + YbCatalogCacheTable_pg_rewrite, + YbCatalogCacheTable_pg_sequence, + YbCatalogCacheTable_pg_statistic_ext_data, + YbCatalogCacheTable_pg_statistic_ext, + YbCatalogCacheTable_pg_statistic_ext, + YbCatalogCacheTable_pg_statistic, + YbCatalogCacheTable_pg_subscription, + YbCatalogCacheTable_pg_subscription, + YbCatalogCacheTable_pg_subscription_rel, + YbCatalogCacheTable_pg_tablespace, + YbCatalogCacheTable_pg_transform, + YbCatalogCacheTable_pg_transform, + YbCatalogCacheTable_pg_ts_config_map, + YbCatalogCacheTable_pg_ts_config, + YbCatalogCacheTable_pg_ts_config, + YbCatalogCacheTable_pg_ts_dict, + YbCatalogCacheTable_pg_ts_dict, + YbCatalogCacheTable_pg_ts_parser, + YbCatalogCacheTable_pg_ts_parser, + YbCatalogCacheTable_pg_ts_template, + YbCatalogCacheTable_pg_ts_template, + YbCatalogCacheTable_pg_type, + YbCatalogCacheTable_pg_type, + YbCatalogCacheTable_pg_user_mapping, + YbCatalogCacheTable_pg_user_mapping, + YbCatalogCacheTable_pg_yb_tablegroup, + YbCatalogCacheTable_pg_constraint, +}; + +static_assert(SysCacheSize == + sizeof(yb_catalog_cache_tables) / sizeof(YbCatalogCacheTable), + "yb_catalog_cache_tables size mismatch"); + static CatCache *SysCache[SysCacheSize]; static bool CacheInitialized = false; @@ -1992,6 +2144,20 @@ YbCheckCatalogCacheIndexNameTable() return false; } ReleaseSysCache(tuple); + + const char *table_name = YbGetCatalogCacheTableNameFromCacheId(cache_id); + Oid reloid = cacheinfo[cache_id].reloid; + tuple = SearchSysCache1(RELOID, reloid); + Assert(HeapTupleIsValid(tuple)); + classForm = (Form_pg_class) GETSTRUCT(tuple); + if (strcmp(NameStr(classForm->relname), table_name)) + { + ReleaseSysCache(tuple); + YBC_LOG_WARNING("Cache id %u has name mismatch: %s vs %s", cache_id, + NameStr(classForm->relname), table_name); + return false; + } + ReleaseSysCache(tuple); } return true; } @@ -2001,3 +2167,24 @@ const char* YbGetCatalogCacheIndexName(int cache_id) { return yb_cache_index_name_table[cache_id]; } + +const char * +YbGetCatalogCacheTableNameFromTableId(int table_id) +{ + Assert(table_id >= 0 && table_id < YbNumCatalogCacheTables); + return yb_cache_table_name_table[table_id]; +} + +int +YbGetCatalogCacheTableIdFromCacheId(int cache_id) +{ + int table_id = yb_catalog_cache_tables[cache_id]; + Assert(table_id >= 0 && table_id < YbNumCatalogCacheTables); + return table_id; +} + +const char * +YbGetCatalogCacheTableNameFromCacheId(int cache_id) +{ + return YbGetCatalogCacheTableNameFromTableId(YbGetCatalogCacheTableIdFromCacheId(cache_id)); +} diff --git a/src/postgres/src/include/utils/catcache.h b/src/postgres/src/include/utils/catcache.h index 226507c3f8fe..a5b39bccf059 100644 --- a/src/postgres/src/include/utils/catcache.h +++ b/src/postgres/src/include/utils/catcache.h @@ -243,6 +243,7 @@ extern void SetCatCacheList(CatCache *cache, int nkeys, List *fnlist); extern bool RelationHasCachedLists(Relation relation); extern long YbGetCatCacheMisses(); extern long* YbGetCatCacheIdMisses(); +extern long *YbGetCatCacheTableMisses(); extern YbCatCListIterator YbCatCListIteratorBegin(CatCList *list); extern HeapTuple YbCatCListIteratorGetNext(YbCatCListIterator *iterator); diff --git a/src/postgres/src/include/utils/lsyscache.h b/src/postgres/src/include/utils/lsyscache.h index db5f58c47f38..e683c6a8694f 100644 --- a/src/postgres/src/include/utils/lsyscache.h +++ b/src/postgres/src/include/utils/lsyscache.h @@ -1,3 +1,4 @@ + /*------------------------------------------------------------------------- * * lsyscache.h diff --git a/src/postgres/src/include/utils/syscache.h b/src/postgres/src/include/utils/syscache.h index 85d68248e5d5..ee2f4db2aad8 100644 --- a/src/postgres/src/include/utils/syscache.h +++ b/src/postgres/src/include/utils/syscache.h @@ -122,6 +122,62 @@ enum SysCacheIdentifier #define SysCacheSize (YBCONSTRAINTRELIDTYPIDNAME + 1) }; +typedef enum YbCatalogCacheTable +{ + YbCatalogCacheTable_pg_aggregate, + YbCatalogCacheTable_pg_am, + YbCatalogCacheTable_pg_amop, + YbCatalogCacheTable_pg_amproc, + YbCatalogCacheTable_pg_attribute, + YbCatalogCacheTable_pg_auth_members, + YbCatalogCacheTable_pg_authid, + YbCatalogCacheTable_pg_cast, + YbCatalogCacheTable_pg_class, + YbCatalogCacheTable_pg_collation, + YbCatalogCacheTable_pg_constraint, + YbCatalogCacheTable_pg_conversion, + YbCatalogCacheTable_pg_database, + YbCatalogCacheTable_pg_default_acl, + YbCatalogCacheTable_pg_enum, + YbCatalogCacheTable_pg_event_trigger, + YbCatalogCacheTable_pg_foreign_data_wrapper, + YbCatalogCacheTable_pg_foreign_server, + YbCatalogCacheTable_pg_foreign_table, + YbCatalogCacheTable_pg_index, + YbCatalogCacheTable_pg_language, + YbCatalogCacheTable_pg_namespace, + YbCatalogCacheTable_pg_opclass, + YbCatalogCacheTable_pg_operator, + YbCatalogCacheTable_pg_opfamily, + YbCatalogCacheTable_pg_parameter_acl, + YbCatalogCacheTable_pg_partitioned_table, + YbCatalogCacheTable_pg_proc, + YbCatalogCacheTable_pg_publication, + YbCatalogCacheTable_pg_publication_namespace, + YbCatalogCacheTable_pg_publication_rel, + YbCatalogCacheTable_pg_range, + YbCatalogCacheTable_pg_replication_origin, + YbCatalogCacheTable_pg_rewrite, + YbCatalogCacheTable_pg_sequence, + YbCatalogCacheTable_pg_statistic, + YbCatalogCacheTable_pg_statistic_ext, + YbCatalogCacheTable_pg_statistic_ext_data, + YbCatalogCacheTable_pg_subscription, + YbCatalogCacheTable_pg_subscription_rel, + YbCatalogCacheTable_pg_tablespace, + YbCatalogCacheTable_pg_transform, + YbCatalogCacheTable_pg_ts_config, + YbCatalogCacheTable_pg_ts_config_map, + YbCatalogCacheTable_pg_ts_dict, + YbCatalogCacheTable_pg_ts_parser, + YbCatalogCacheTable_pg_ts_template, + YbCatalogCacheTable_pg_type, + YbCatalogCacheTable_pg_user_mapping, + YbCatalogCacheTable_pg_yb_tablegroup + +#define YbNumCatalogCacheTables (YbCatalogCacheTable_pg_yb_tablegroup + 1) +} YbCatalogCacheTable; + /* Used in IsYugaByteEnabled() mode only */ extern void YbSetSysCacheTuple(Relation rel, HeapTuple tup); extern void YbPreloadCatalogCache(int cache_id, int idx_cache_id); @@ -129,6 +185,9 @@ extern void YbPreloadCatalogCache(int cache_id, int idx_cache_id); extern bool YbCheckCatalogCacheIndexNameTable(); #endif extern const char* YbGetCatalogCacheIndexName(int cache_id); +extern const char *YbGetCatalogCacheTableNameFromTableId(int table_id); +extern const char *YbGetCatalogCacheTableNameFromCacheId(int cache_id); +extern int YbGetCatalogCacheTableIdFromCacheId(int cache_id); extern void InitCatalogCache(void); extern void InitCatalogCachePhase2(void); diff --git a/src/postgres/yb-extensions/yb_pg_metrics/yb_pg_metrics.c b/src/postgres/yb-extensions/yb_pg_metrics/yb_pg_metrics.c index 88faaf9b3653..11cfdc88fbb1 100644 --- a/src/postgres/yb-extensions/yb_pg_metrics/yb_pg_metrics.c +++ b/src/postgres/yb-extensions/yb_pg_metrics/yb_pg_metrics.c @@ -149,6 +149,58 @@ typedef enum statementType CatCacheIdMisses_83, CatCacheIdMisses_84, CatCacheIdMisses_End = CatCacheIdMisses_84, + CatCacheTableMisses_Start, + CatCacheTableMisses_0 = CatCacheTableMisses_Start, + CatCacheTableMisses_1, + CatCacheTableMisses_2, + CatCacheTableMisses_3, + CatCacheTableMisses_4, + CatCacheTableMisses_5, + CatCacheTableMisses_6, + CatCacheTableMisses_7, + CatCacheTableMisses_8, + CatCacheTableMisses_9, + CatCacheTableMisses_10, + CatCacheTableMisses_11, + CatCacheTableMisses_12, + CatCacheTableMisses_13, + CatCacheTableMisses_14, + CatCacheTableMisses_15, + CatCacheTableMisses_16, + CatCacheTableMisses_17, + CatCacheTableMisses_18, + CatCacheTableMisses_19, + CatCacheTableMisses_20, + CatCacheTableMisses_21, + CatCacheTableMisses_22, + CatCacheTableMisses_23, + CatCacheTableMisses_24, + CatCacheTableMisses_25, + CatCacheTableMisses_26, + CatCacheTableMisses_27, + CatCacheTableMisses_28, + CatCacheTableMisses_29, + CatCacheTableMisses_30, + CatCacheTableMisses_31, + CatCacheTableMisses_32, + CatCacheTableMisses_33, + CatCacheTableMisses_34, + CatCacheTableMisses_35, + CatCacheTableMisses_36, + CatCacheTableMisses_37, + CatCacheTableMisses_38, + CatCacheTableMisses_39, + CatCacheTableMisses_40, + CatCacheTableMisses_41, + CatCacheTableMisses_42, + CatCacheTableMisses_43, + CatCacheTableMisses_44, + CatCacheTableMisses_45, + CatCacheTableMisses_46, + CatCacheTableMisses_47, + CatCacheTableMisses_48, + CatCacheTableMisses_49, + CatCacheTableMisses_End = CatCacheTableMisses_49, kMaxStatementType } statementType; int num_entries = kMaxStatementType; @@ -192,6 +244,7 @@ extern int MaxConnections; static long last_cache_misses_val = 0; static long last_cache_id_misses_val[SysCacheSize] = {0}; +static long last_cache_table_misses_val[YbNumCatalogCacheTables] = {0}; static volatile sig_atomic_t got_SIGHUP = false; static volatile sig_atomic_t got_SIGTERM = false; @@ -270,32 +323,45 @@ DecBlockNestingLevel(void) void set_metric_names(void) { - strcpy(ybpgm_table[Select].name, YSQL_METRIC_PREFIX "SelectStmt"); - strcpy(ybpgm_table[Insert].name, YSQL_METRIC_PREFIX "InsertStmt"); - strcpy(ybpgm_table[Delete].name, YSQL_METRIC_PREFIX "DeleteStmt"); - strcpy(ybpgm_table[Update].name, YSQL_METRIC_PREFIX "UpdateStmt"); - strcpy(ybpgm_table[Begin].name, YSQL_METRIC_PREFIX "BeginStmt"); - strcpy(ybpgm_table[Commit].name, YSQL_METRIC_PREFIX "CommitStmt"); - strcpy(ybpgm_table[Rollback].name, YSQL_METRIC_PREFIX "RollbackStmt"); - strcpy(ybpgm_table[Other].name, YSQL_METRIC_PREFIX "OtherStmts"); - // Deprecated. Names with "_"s may cause confusion to metric conumsers. - strcpy( - ybpgm_table[Single_Shard_Transaction].name, YSQL_METRIC_PREFIX "Single_Shard_Transactions"); - - strcpy(ybpgm_table[SingleShardTransaction].name, YSQL_METRIC_PREFIX "SingleShardTransactions"); - strcpy(ybpgm_table[Transaction].name, YSQL_METRIC_PREFIX "Transactions"); - strcpy(ybpgm_table[AggregatePushdown].name, YSQL_METRIC_PREFIX "AggregatePushdowns"); - strcpy(ybpgm_table[CatCacheMisses].name, YSQL_METRIC_PREFIX "CatalogCacheMisses"); - for (int i = CatCacheIdMisses_Start; i <= CatCacheIdMisses_End; ++i) - { - int cache_id = i - CatCacheIdMisses_Start; - char index_name[NAMEDATALEN + 16]; - strcpy(ybpgm_table[i].name, YSQL_METRIC_PREFIX "CatalogCacheMisses"); - sprintf(index_name, "_%s", YbGetCatalogCacheIndexName(cache_id)); - Assert(strlen(ybpgm_table[i].name) + strlen(index_name) < - sizeof(ybpgm_table[i].name)); - strcat(ybpgm_table[i].name, index_name); - } + for (int i = 0; i < kMaxStatementType; i++) + ybpgm_table[i].table_name[0] = '\0'; + + strcpy(ybpgm_table[Select].name, YSQL_METRIC_PREFIX "SelectStmt"); + strcpy(ybpgm_table[Insert].name, YSQL_METRIC_PREFIX "InsertStmt"); + strcpy(ybpgm_table[Delete].name, YSQL_METRIC_PREFIX "DeleteStmt"); + strcpy(ybpgm_table[Update].name, YSQL_METRIC_PREFIX "UpdateStmt"); + strcpy(ybpgm_table[Begin].name, YSQL_METRIC_PREFIX "BeginStmt"); + strcpy(ybpgm_table[Commit].name, YSQL_METRIC_PREFIX "CommitStmt"); + strcpy(ybpgm_table[Rollback].name, YSQL_METRIC_PREFIX "RollbackStmt"); + strcpy(ybpgm_table[Other].name, YSQL_METRIC_PREFIX "OtherStmts"); + // Deprecated. Names with "_"s may cause confusion to metric conumsers. + strcpy(ybpgm_table[Single_Shard_Transaction].name, + YSQL_METRIC_PREFIX "Single_Shard_Transactions"); + strcpy(ybpgm_table[SingleShardTransaction].name, + YSQL_METRIC_PREFIX "SingleShardTransactions"); + strcpy(ybpgm_table[Transaction].name, YSQL_METRIC_PREFIX "Transactions"); + strcpy(ybpgm_table[AggregatePushdown].name, + YSQL_METRIC_PREFIX "AggregatePushdowns"); + strcpy(ybpgm_table[CatCacheMisses].name, YSQL_METRIC_PREFIX "CatalogCacheMisses"); + for (int i = CatCacheIdMisses_Start; i <= CatCacheIdMisses_End; ++i) + { + int cache_id = i - CatCacheIdMisses_Start; + strcpy(ybpgm_table[i].name, YSQL_METRIC_PREFIX "CatalogCacheMisses"); + const char *index_name = YbGetCatalogCacheIndexName(cache_id); + Assert(strlen(index_name) < YB_PG_METRIC_NAME_LEN); + snprintf(ybpgm_table[i].table_name, YB_PG_METRIC_NAME_LEN, "%s", + index_name); + } + + for (int i = CatCacheTableMisses_Start; i <= CatCacheTableMisses_End; ++i) + { + int table_id = i - CatCacheTableMisses_Start; + strcpy(ybpgm_table[i].name, YSQL_METRIC_PREFIX "CatalogCacheTableMisses"); + const char *table_name = YbGetCatalogCacheTableNameFromTableId(table_id); + Assert(strlen(table_name) < YB_PG_METRIC_NAME_LEN); + snprintf(ybpgm_table[i].table_name, YB_PG_METRIC_NAME_LEN, "%s", + table_name); + } } /* @@ -776,43 +842,52 @@ ybpgm_ExecutorEnd(QueryDesc *queryDesc) * use this not-null check for now. */ if (isTopLevelStatement() && queryDesc->totaltime) { - InstrEndLoop(queryDesc->totaltime); - const uint64_t time = (uint64_t) (queryDesc->totaltime->total * 1000000.0); - const uint64 rows_count = queryDesc->estate->es_processed; + InstrEndLoop(queryDesc->totaltime); + const uint64_t time = (uint64_t) (queryDesc->totaltime->total * 1000000.0); + const uint64 rows_count = queryDesc->estate->es_processed; - ybpgm_Store(type, time, rows_count); + ybpgm_Store(type, time, rows_count); - if (queryDesc->estate->yb_es_is_single_row_modify_txn) - { - ybpgm_Store(Single_Shard_Transaction, time, rows_count); - ybpgm_Store(SingleShardTransaction, time, rows_count); - } + if (queryDesc->estate->yb_es_is_single_row_modify_txn) + { + ybpgm_Store(Single_Shard_Transaction, time, rows_count); + ybpgm_Store(SingleShardTransaction, time, rows_count); + } + + if (!is_inside_transaction_block) + ybpgm_Store(Transaction, time, rows_count); + + if (IsA(queryDesc->planstate, AggState) && + castNode(AggState, queryDesc->planstate)->yb_pushdown_supported) + ybpgm_Store(AggregatePushdown, time, rows_count); + + long current_cache_misses = YbGetCatCacheMisses(); + long* current_cache_id_misses = YbGetCatCacheIdMisses(); + long *current_cache_table_misses = YbGetCatCacheTableMisses(); + + long total_delta = current_cache_misses - last_cache_misses_val; + last_cache_misses_val = current_cache_misses; - if (!is_inside_transaction_block) - ybpgm_Store(Transaction, time, rows_count); - - if (IsA(queryDesc->planstate, AggState) && - castNode(AggState, queryDesc->planstate)->yb_pushdown_supported) - ybpgm_Store(AggregatePushdown, time, rows_count); - - long current_cache_misses = YbGetCatCacheMisses(); - long* current_cache_id_misses = YbGetCatCacheIdMisses(); - - long total_delta = current_cache_misses - last_cache_misses_val; - last_cache_misses_val = current_cache_misses; - - /* Currently we set the time parameter to 0 as we don't have metrics - * for that available - * TODO: Get timing metrics for catalog cache misses - */ - ybpgm_StoreCount(CatCacheMisses, 0, total_delta); - if (total_delta > 0) - for (int i = CatCacheIdMisses_Start; i <= CatCacheIdMisses_End; ++i) - { - int j = i - CatCacheIdMisses_Start; - ybpgm_StoreCount(i, 0, current_cache_id_misses[j] - last_cache_id_misses_val[j]); - last_cache_id_misses_val[j] = current_cache_id_misses[j]; - } + /* Currently we set the time parameter to 0 as we don't have metrics + * for that available + * TODO: Get timing metrics for catalog cache misses + */ + ybpgm_StoreCount(CatCacheMisses, 0, total_delta); + if (total_delta > 0) + for (int i = CatCacheIdMisses_Start; i <= CatCacheIdMisses_End; ++i) + { + int j = i - CatCacheIdMisses_Start; + ybpgm_StoreCount(i, 0, current_cache_id_misses[j] - last_cache_id_misses_val[j]); + last_cache_id_misses_val[j] = current_cache_id_misses[j]; + } + for (int i = CatCacheTableMisses_Start; i <= CatCacheTableMisses_End; ++i) + { + int j = i - CatCacheTableMisses_Start; + ybpgm_StoreCount(i, 0, + current_cache_table_misses[j] - + last_cache_table_misses_val[j]); + last_cache_table_misses_val[j] = current_cache_table_misses[j]; + } } IncStatementNestingLevel(); diff --git a/src/yb/yql/pggate/webserver/pgsql_webserver_wrapper.cc b/src/yb/yql/pggate/webserver/pgsql_webserver_wrapper.cc index 0c3e6b053535..b14a3d00cef0 100644 --- a/src/yb/yql/pggate/webserver/pgsql_webserver_wrapper.cc +++ b/src/yb/yql/pggate/webserver/pgsql_webserver_wrapper.cc @@ -459,34 +459,19 @@ static void PgPrometheusMetricsHandler( MetricPrometheusOptions opts; PrometheusWriter writer(output, opts); - auto cache_miss_prefix_len = sizeof("CatalogCacheMisses"); for (int i = 0; i < ybpgm_num_entries; ++i) { - bool is_cache_id_miss = false; - std::string name = ybpgm_table[i].name; - auto pos = name.find("CatalogCacheMisses"); - if (pos != std::string::npos) { - // Example of name: - // handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheMisses_pg_authid_oid_index - // In this case, we split the name into metric_name and the index table name: - // metric_name: handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheIdMisses - // table_name: pg_authid_oid_index - pos += cache_miss_prefix_len - 1; - if (name[pos] == '_') { - is_cache_id_miss = true; - DCHECK(!prometheus_attr.contains("table_name")); - prometheus_attr["table_name"] = name.substr(pos + 1); - } + std::string metric_name = ybpgm_table[i].name; + std::string table_name = ybpgm_table[i].table_name; + if (!table_name.empty()) { + prometheus_attr["table_name"] = table_name; } - auto metric_name = is_cache_id_miss ? name.substr(0, pos) : name; WARN_NOT_OK(writer.WriteSingleEntry(prometheus_attr, metric_name + "_count", ybpgm_table[i].calls, AggregationFunction::kSum, kServerLevel), "Couldn't write text metrics for Prometheus"); WARN_NOT_OK(writer.WriteSingleEntry(prometheus_attr, metric_name + "_sum", ybpgm_table[i].total_time, AggregationFunction::kSum, kServerLevel), "Couldn't write text metrics for Prometheus"); - if (is_cache_id_miss) { - prometheus_attr.erase("table_name"); - } + prometheus_attr.erase("table_name"); } // Publish sql server connection related metrics diff --git a/src/yb/yql/pggate/webserver/pgsql_webserver_wrapper.h b/src/yb/yql/pggate/webserver/pgsql_webserver_wrapper.h index 08afc8d648ae..2310c19c8504 100644 --- a/src/yb/yql/pggate/webserver/pgsql_webserver_wrapper.h +++ b/src/yb/yql/pggate/webserver/pgsql_webserver_wrapper.h @@ -33,8 +33,11 @@ extern "C" { struct WebserverWrapper; +#define YB_PG_METRIC_NAME_LEN 120 + typedef struct ybpgmEntry { - char name[120]; + char name[YB_PG_METRIC_NAME_LEN]; + char table_name[YB_PG_METRIC_NAME_LEN]; YB_ATOMIC_ULLONG calls; YB_ATOMIC_ULLONG total_time; YB_ATOMIC_ULLONG rows; diff --git a/src/yb/yql/pgwrapper/pg_libpq-test.cc b/src/yb/yql/pgwrapper/pg_libpq-test.cc index 700623fb7f43..f65930199282 100644 --- a/src/yb/yql/pgwrapper/pg_libpq-test.cc +++ b/src/yb/yql/pgwrapper/pg_libpq-test.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -3950,11 +3951,130 @@ TEST_F(PgLibPqTest, CatalogCacheMemoryLeak) { } } -static int GetCacheMissCount(const string& metric_instance) { - auto begin = metric_instance.find("} "); - int count; - std::istringstream(metric_instance.substr(begin + 2)) >> count; - return count; +static std::optional GetCatalogTableNameFromIndexName(const string& index_name) { + static const std::regex table_name_regex( + "(pg_publication_namespace|" + "pg_foreign_data_wrapper|" + "pg_largeobject_metadata|" + "pg_replication_origin|" + "pg_statistic_ext_data|" + "pg_yb_catalog_version|" + "pg_partitioned_table|" + "pg_subscription_rel|" + "pg_db_role_setting|" + "pg_publication_rel|" + "pg_yb_role_profile|" + "pg_foreign_server|" + "pg_event_trigger|" + "pg_foreign_table|" + "pg_parameter_acl|" + "pg_shdescription|" + "pg_statistic_ext|" + "pg_ts_config_map|" + "pg_yb_tablegroup|" + "pg_auth_members|" + "pg_subscription|" + "pg_user_mapping|" + "pg_yb_migration|" + "pg_default_acl|" + "pg_description|" + "pg_largeobject|" + "pg_publication|" + "pg_ts_template|" + "pg_constraint|" + "pg_conversion|" + "pg_init_privs|" + "pg_shseclabel|" + "pg_tablespace|" + "pg_yb_profile|" + "pg_aggregate|" + "pg_attribute|" + "pg_collation|" + "pg_extension|" + "pg_namespace|" + "pg_statistic|" + "pg_transform|" + "pg_ts_config|" + "pg_ts_parser|" + "pg_database|" + "pg_inherits|" + "pg_language|" + "pg_operator|" + "pg_opfamily|" + "pg_seclabel|" + "pg_sequence|" + "pg_shdepend|" + "pg_attrdef|" + "pg_opclass|" + "pg_rewrite|" + "pg_trigger|" + "pg_ts_dict|" + "pg_amproc|" + "pg_authid|" + "pg_depend|" + "pg_policy|" + "pg_class|" + "pg_index|" + "pg_range|" + "pg_amop|" + "pg_cast|" + "pg_enum|" + "pg_proc|" + "pg_type|" + "pg_am)_.*"); + + std::smatch match; + if (std::regex_search(index_name, match, table_name_regex)) { + return match[1].str(); + } + return std::nullopt; +} + +struct YsqlMetric { + std::string name; + std::unordered_map labels; + int64_t value; + int64_t time; + + YsqlMetric( + std::string name, std::unordered_map labels, int64_t value, + int64_t time) + : name(std::move(name)), labels(std::move(labels)), value(value), time(time) {} +}; + +static std::vector ParsePrometheusMetrics(const std::string& metrics_output) { + // Splits a metric line into name, labels, value and timestamp. + // Example line: + // metric_name{label_1="value_1",label_2="value_2"} 123 456 + const std::regex metric_regex(R"((\w+)\{([^}]+)\}\s+(\d+)\s+(\d+))"); + + // Splits the list of labels into individual label-value pairs. + const std::regex label_regex(R"((\w+)=\"([^\"]+)\")"); + + std::vector parsed_metrics; + std::istringstream stream(metrics_output); + std::string line; + + while (std::getline(stream, line)) { + std::smatch metric_match; + if (std::regex_search(line, metric_match, metric_regex)) { + std::unordered_map labels; + const std::string labels_str = metric_match[2].str(); + auto search_start = labels_str.cbegin(); + std::smatch label_match; + + while (std::regex_search(search_start, labels_str.cend(), label_match, label_regex)) { + labels[label_match[1].str()] = label_match[2].str(); + search_start = label_match.suffix().first; + } + + parsed_metrics.emplace_back( + metric_match[1].str(), std::move(labels), std::stoll(metric_match[3].str()), + std::stoll(metric_match[4].str())); + } + } + + return parsed_metrics; } TEST_F(PgLibPqTest, CatalogCacheIdMissMetricsTest) { @@ -3967,33 +4087,58 @@ TEST_F(PgLibPqTest, CatalogCacheIdMissMetricsTest) { auto result = ASSERT_RESULT(conn.Fetch("SELECT * FROM t")); ExternalTabletServer* ts = cluster_->tablet_server(0); auto hostport = Format("$0:$1", ts->bind_host(), ts->pgsql_http_port()); - auto pg_metrics_url = Substitute( - "http://$0/prometheus-metrics?reset_histograms=false&show_help=false", - hostport); + auto pg_metrics_url = + Substitute("http://$0/prometheus-metrics?reset_histograms=false&show_help=false", hostport); EasyCurl c; faststring buf; ASSERT_OK(c.FetchURL(pg_metrics_url, &buf)); string page_content = buf.ToString(); - auto cache_miss_metric = - "handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheMisses_count"; - auto begin = page_content.find(cache_miss_metric); - auto end = page_content.find("\n", begin); - auto expected = GetCacheMissCount(page_content.substr(begin, end - begin)); - ASSERT_GT(expected, 0); - LOG(INFO) << "Expected total cache misses: " << expected; - int total_cache_misses = 0; - while (true) { - auto cache_id_miss_metric = "handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheMisses_"; - begin = page_content.find(cache_id_miss_metric, end); - if (begin == std::string::npos) { + auto metrics = ParsePrometheusMetrics(page_content); + + int64_t expected_total_cache_misses = 0; + for (const auto& metric : metrics) { + if (metric.name == "handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheMisses_count" && + metric.labels.find("table_name") == metric.labels.end()) { + expected_total_cache_misses = metric.value; break; } - end = page_content.find("\n", begin); - auto cache_misses = GetCacheMissCount(page_content.substr(begin, end - begin)); - ASSERT_GE(cache_misses, 0); - total_cache_misses += cache_misses; } - ASSERT_EQ(total_cache_misses, expected); + ASSERT_GT(expected_total_cache_misses, 0); + LOG(INFO) << "Expected total cache misses: " << expected_total_cache_misses; + + // Sum the cache miss metrics for each index. + int64_t total_index_cache_misses = 0; + std::unordered_map per_table_index_cache_misses; + for (const auto& metric : metrics) { + if (metric.name == "handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheMisses_count" && + metric.labels.find("table_name") != metric.labels.end()) { + auto table_name = GetCatalogTableNameFromIndexName(metric.labels.at("table_name")); + ASSERT_TRUE(table_name) << "Failed to get table name from index name: " + << metric.labels.at("table_name"); + + per_table_index_cache_misses[*table_name] += metric.value; + total_index_cache_misses += metric.value; + LOG_IF(INFO, metric.value > 0) << "Index " << metric.labels.at("table_name") << " has " + << metric.value << " cache misses"; + } + } + ASSERT_EQ(expected_total_cache_misses, total_index_cache_misses); + + // Check that the sum of the cache misses for all the indexes on each table is equal to the + // table-level cache miss metric. + int64_t total_table_cache_misses = 0; + for (const auto& metric : metrics) { + if (metric.name == "handler_latency_yb_ysqlserver_SQLProcessor_CatalogCacheTableMisses_count") { + auto table_name = metric.labels.at("table_name"); + ASSERT_EQ(per_table_index_cache_misses[table_name], metric.value) + << "Expected sum of index cache misses for table " << table_name + << " to be equal to the table cache misses"; + total_table_cache_misses += metric.value; + LOG_IF(INFO, metric.value > 0) + << "Table " << table_name << " has " << metric.value << " cache misses"; + } + } + ASSERT_EQ(expected_total_cache_misses, total_table_cache_misses); } class PgLibPqCreateSequenceNamespaceRaceTest : public PgLibPqTest {