[#23645] docdb: Fix tests timing out on TSAN after 15786f3

Summary:
After 15786f3, a number of tests began regularly timing out on TSAN:
```
org.yb.pgsql.TestLoadBalance#TestWithBlacklistedServer
org.yb.pgsql.TestLoadBalance#TestWithBlacklistedServer
ForceMasterLookup/ClientTestForceMasterLookup.TestConcurrentLookups/1
LoadBalancerLegacyColocatedDBColocatedTablesTest.GlobalLoadBalancingWithLegacyColocatedDBColocatedTables
```

This diff adds point fixes for these tests.

Note `XClusterYSqlTestConsistentTransactionsTest.MasterLeaderRestart` was fixed by 8a0d6ff.
Jira: DB-12556

Test Plan:
```
./yb_build.sh tsan --java-test 'org.yb.pgsql.TestLoadBalance#TestWithBlacklistedServer' && \
  ./yb_build.sh tsan --java-test 'org.yb.pgsql.TestLoadBalance#TestWithBlacklistedServer' && \
  ./yb_build.sh tsan --cxx-test client-test --gtest_filter 'ForceMasterLookup/ClientTestForceMasterLookup.TestConcurrentLookups/1' && \
  ./yb_build.sh tsan --cxx-test load_balancer_colocated_tables-test --gtest_filter 'LoadBalancerLegacyColocatedDBColocatedTablesTest.GlobalLoadBalancingWithLegacyColocatedDBColocatedTables'
```

Reviewers: asrivastava

Reviewed By: asrivastava

Subscribers: ybase, slingam

Differential Revision: https://phorge.dev.yugabyte.com/D38139

java/yb-loadtester/src/test/java/org/yb/loadtester/TestClusterIsLoadBalancerIdle.java

@@ -16,9 +16,10 @@
 import org.junit.runner.RunWith;
 import org.yb.YBTestRunner;
 import org.yb.minicluster.MiniYBCluster;
+import org.yb.util.BuildTypeUtil;
 
-import static junit.framework.TestCase.assertTrue;
 import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertTrue;
 
 /**
  * This is an integration test that ensures we can fully move a YB cluster
@@ -49,9 +50,13 @@ public void testClusterIsLoadBalancerIdle() throws Exception {
     for (int i = 0; i < num_tables; i++) {
       setupTable("test_table_" + i, 2);
     }
+    long wait_time = num_tables * MiniYBCluster.CQL_NODE_LIST_REFRESH_SECS * 1000;
+    if (BuildTypeUtil.isTSAN()) {
+      wait_time *= 2;
+    }
 
     // Wait for the partition metadata to refresh.
-    Thread.sleep(num_tables * MiniYBCluster.CQL_NODE_LIST_REFRESH_SECS * 1000);
+    Thread.sleep(wait_time);
 
     // Wait for load to balance across the three tservers.
     assertTrue(client.waitForLoadBalance(LOADBALANCE_TIMEOUT_MS, NUM_TABLET_SERVERS));
@@ -65,7 +70,7 @@ public void testClusterIsLoadBalancerIdle() throws Exception {
     addNewTServers(1);
 
     // Wait for the partition metadata to refresh.
-    Thread.sleep(num_tables * MiniYBCluster.CQL_NODE_LIST_REFRESH_SECS * 1000);
+    Thread.sleep(wait_time);
 
     verifyClusterHealth(NUM_TABLET_SERVERS + 1);
 

java/yb-pgsql/src/test/java/org/yb/pgsql/TestLoadBalance.java

@@ -14,17 +14,17 @@
 package org.yb.pgsql;
 
 import com.google.common.net.HostAndPort;
-import com.yugabyte.ysql.ClusterAwareLoadBalancer;
 import com.yugabyte.jdbc.PgConnection;
-import org.yb.AssertionWrappers;
+import com.yugabyte.ysql.ClusterAwareLoadBalancer;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.yb.AssertionWrappers;
+import org.yb.YBTestRunner;
 import org.yb.client.TestUtils;
 import org.yb.minicluster.MiniYBClusterBuilder;
 import org.yb.minicluster.MiniYBDaemon;
-import org.yb.YBTestRunner;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
@@ -240,7 +240,7 @@ public void TestWithBlacklistedServer() throws Exception{
     AssertionWrappers.assertTrue("Expected 7 tservers, found " + rows, rows == 7);
 
     AssertionWrappers.assertTrue("Expected 6 tservers not found",
-      verifyResultUntil(10, 3000, e.getKey(), 6));
+      verifyResultUntil(50, 3000, e.getKey(), 6));
 
     runProcess(
       TestUtils.findBinary("yb-admin"),

src/yb/client/client-test.cc

@@ -611,7 +611,10 @@ class ClientTestForceMasterLookup :
 
 
   void PerformManyLookups(const std::shared_ptr<YBTable>& table, bool point_lookup) {
-    for (int i = 0; i < kNumIterations; i++) {
+    // With the cache disabled and tsan enabled, each lookup is extremely slow. So we greatly
+    // decrease the number of iterations.
+    int adjusted_num_iterations = yb::IsTsan() ? 10 : kNumIterations;
+    for (int i = 0; i < adjusted_num_iterations; i++) {
       if (point_lookup) {
           auto key_rt = ASSERT_RESULT(LookupFirstTabletFuture(client_.get(), table).get());
           ASSERT_NOTNULL(key_rt);
@@ -639,7 +642,6 @@ TEST_P(ClientTestForceMasterLookup, TestConcurrentLookups) {
       PerformManyLookups(table, true /* point_lookup */)); });
   auto t2 = std::thread([&]() { ASSERT_NO_FATALS(
       PerformManyLookups(table, false /* point_lookup */)); });
-
   t1.join();
   t2.join();
 }

src/yb/integration-tests/load_balancer_colocated_tables-test.cc

@@ -160,7 +160,8 @@ class LoadBalancerColocatedTablesTest : public YBTableTestBase {
         kDefaultTimeout));
 
     // Wait for load balancing to complete.
-    WaitForLoadBalanceCompletion();
+    auto base_timeout = MonoDelta::FromMilliseconds(kDefaultLoadBalanceTimeoutMs);
+    WaitForLoadBalanceCompletion(yb::IsTsan() ? base_timeout * 5 : base_timeout);
 
     // Assert that each table is balanced, and that we are globally balanced (Before global load
     // balancing, colocated parent tablets would not move) - Each colocated database or tablegroup

src/yb/master/ts_descriptor.cc

@@ -38,11 +38,11 @@
 #include "yb/common/wire_protocol.h"
 #include "yb/common/wire_protocol.pb.h"
 
-#include "yb/master/master_fwd.h"
-#include "yb/master/master_util.h"
 #include "yb/master/catalog_manager_util.h"
 #include "yb/master/master_cluster.pb.h"
+#include "yb/master/master_fwd.h"
 #include "yb/master/master_heartbeat.pb.h"
+#include "yb/master/master_util.h"
 
 #include "yb/util/atomic.h"
 #include "yb/util/flags.h"