From 87a936a64b6c25697d9b7f7fd6fd1ece235d1dc2 Mon Sep 17 00:00:00 2001 From: Muthu Chidambaram Date: Tue, 17 Sep 2024 12:39:52 +0000 Subject: [PATCH] [PLAT-15353] Consistency checks testing hooks Summary: This diff adds some testing hooks for consistency check testing. test_pending will shutdown YBA immediately after the update is applied to the DB, to test handling of pending tasks on YBA restart ysql_timeout_secs is a flag that allows configuration of ysql timeout for consistency check commands (default is 30s) update_delay_secs allows you to introduce a sleep after the DB commit occurs, simulating a slow/unreliable network connection. During this time YBA can be interrupted or ysql timeout may occur. Also adds some minor improvements around the consistency_check table (renamed to yba_consistency_check) and added yw_uuid and yw_ip columns that may be useful going forward. Only the yw_uuid is populated for now. Test Plan: consistency check with flags enabled, ensure failures are triggered regular task execution not impacted Reviewers: sanketh, dshubin, nsingh, anijhawan Reviewed By: sanketh, nsingh Subscribers: yugaware Differential Revision: https://phorge.dev.yugabyte.com/D37950 --- managed/RUNTIME-FLAGS.md | 3 +- .../commissioner/tasks/UniverseTaskBase.java | 38 +- .../subtasks/UpdateConsistencyCheck.java | 349 ++++++++++++------ .../com/yugabyte/yw/common/ConfigHelper.java | 8 + .../yw/common/CustomerTaskManager.java | 2 +- .../java/com/yugabyte/yw/common/Util.java | 2 +- .../yugabyte/yw/common/YsqlQueryExecutor.java | 24 +- .../yw/common/config/UniverseConfKeys.java | 28 +- .../yw/models/helpers/CommonUtils.java | 10 +- managed/src/main/resources/reference.conf | 7 +- .../yugabyte/yw/commissioner/MockUpgrade.java | 2 +- .../tasks/AddNodeToUniverseTest.java | 6 +- .../tasks/DeleteNodeFromUniverseTest.java | 4 +- .../tasks/DeleteXClusterConfigTest.java | 4 +- .../tasks/DestroyKubernetesUniverseTest.java | 2 - .../tasks/EditKubernetesUniverseTest.java | 6 - .../commissioner/tasks/EditUniverseTest.java | 4 +- .../tasks/EditXClusterConfigTest.java | 8 +- .../commissioner/tasks/PauseUniverseTest.java | 2 - .../tasks/ReadOnlyClusterCreateTest.java | 2 +- .../tasks/ReadOnlyClusterDeleteTest.java | 1 - .../tasks/RebootNodeInUniverseTest.java | 6 +- .../ReleaseInstanceFromUniverseTest.java | 2 +- .../tasks/RemoveNodeFromUniverseTest.java | 6 +- .../tasks/ResumeUniverseTest.java | 4 - .../tasks/StartMasterOnNodeTest.java | 2 +- .../tasks/StartNodeInUniverseTest.java | 4 +- .../tasks/StopNodeInUniverseTest.java | 10 +- .../tasks/UpgradeKubernetesUniverseTest.java | 4 +- .../local/LocalProviderUniverseTestBase.java | 8 +- .../local/UpdateConsistencyLocalTest.java | 6 +- .../tasks/upgrade/CertsRotateTest.java | 6 +- .../tasks/upgrade/FinalizeUpgradeTest.java | 4 +- .../upgrade/GFlagsKubernetesUpgradeTest.java | 2 +- .../RollbackKubernetesUpgradeTest.java | 2 +- .../tasks/upgrade/RollbackUpgradeTest.java | 6 +- .../SoftwareKubernetesUpgradeTest.java | 2 +- .../SoftwareKubernetesUpgradeYBTest.java | 2 +- .../tasks/upgrade/TlsToggleTest.java | 4 +- .../tasks/upgrade/VMImageUpgradeTest.java | 4 +- 40 files changed, 398 insertions(+), 198 deletions(-) diff --git a/managed/RUNTIME-FLAGS.md b/managed/RUNTIME-FLAGS.md index fa8a092dc58c..baf8f556a1e9 100644 --- a/managed/RUNTIME-FLAGS.md +++ b/managed/RUNTIME-FLAGS.md @@ -227,6 +227,7 @@ | "Use server broadcast address for yb_backup" | "yb.backup.use_server_broadcast_address_for_yb_backup" | "UNIVERSE" | "Controls whether server_broadcast_address entry should be used during yb_backup.py backup/restore" | "Boolean" | | "Slow Queries Timeout" | "yb.query_stats.slow_queries.timeout_secs" | "UNIVERSE" | "Timeout in secs for slow queries" | "Long" | | "YSQL Queries Timeout" | "yb.ysql_timeout_secs" | "UNIVERSE" | "Timeout in secs for YSQL queries" | "Long" | +| "YSQL Queries Timeout for Consistency Check Operations" | "yb.universe.consistency_check.ysql_timeout_secs" | "UNIVERSE" | "Timeout in secs for YSQL queries" | "Long" | | "Number of cores to keep" | "yb.num_cores_to_keep" | "UNIVERSE" | "Controls the configuration to set the number of cores to keep in the Ansible layer" | "Integer" | | "Whether to check YBA xCluster object is in sync with DB replication group" | "yb.xcluster.ensure_sync_get_replication_status" | "UNIVERSE" | "It ensures that the YBA XCluster object for tables that are in replication is in sync with replication group in DB. If they are not in sync and this is true, getting the xCluster object will throw an exception and the user has to resync the xCluster config." | "Boolean" | | "Network Load balancer health check ports" | "yb.universe.network_load_balancer.custom_health_check_ports" | "UNIVERSE" | "Ports to use for health checks performed by the network load balancer. Invalid and duplicate ports will be ignored. For GCP, only the first health check port would be used." | "Integer List" | @@ -256,6 +257,6 @@ | "Enable health checks for time drift between nodes" | "yb.health_checks.check_clock_time_drift" | "UNIVERSE" | "Enable health checks for time drift between nodes." | "Boolean" | | "Time drift threshold for warning health check" | "yb.health_checks.time_drift_wrn_threshold_ms" | "UNIVERSE" | "Threshold to raise a warning when time drift exceeds this amount" | "Integer" | | "Time drift threshold for error health check" | "yb.health_checks.time_drift_err_threshold_ms" | "UNIVERSE" | "Threshold to raise a error when time drift exceeds this amount" | "Integer" | -| "Enable consistency check for universe" | "yb.universe.consistency_check_enabled" | "UNIVERSE" | "When enabled, all universe operations will attempt consistency check validation before proceeding. Turn off in disaster scenarios to force perform actions." | "Boolean" | +| "Enable consistency check for universe" | "yb.universe.consistency_check.enabled" | "UNIVERSE" | "When enabled, all universe operations will attempt consistency check validation before proceeding. Turn off in disaster scenarios to force perform actions." | "Boolean" | | "Fail the the health check if no clock sync service is found" | "yb.health_checks.clock_sync_service_required" | "UNIVERSE" | "Require chrony or ntp(d) to be installed for health check to pass" | "Boolean" | | "Node Agent Enabler Installation Time-out" | "yb.node_agent.enabler.install_timeout" | "UNIVERSE" | "Node agent enabler installation time-out for the universe" | "Duration" | diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java index 22aad89ff450..5ebb12900287 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java @@ -389,6 +389,27 @@ public CustomBuilder taskTypes(Collection taskTypes) { TaskType.DeleteBackupScheduleKubernetes, TaskType.EnableNodeAgentInUniverse); + private static final Set SKIP_CONSISTENCY_CHECK_TASKS = + ImmutableSet.of( + TaskType.CreateBackup, + TaskType.CreateBackupSchedule, + TaskType.CreateBackupScheduleKubernetes, + TaskType.CreateKubernetesUniverse, + TaskType.CreateSupportBundle, + TaskType.CreateUniverse, + TaskType.BackupUniverse, + TaskType.DeleteBackupSchedule, + TaskType.DeleteBackupScheduleKubernetes, + TaskType.DeleteDrConfig, + TaskType.DeletePitrConfig, + TaskType.DeleteXClusterConfig, + TaskType.DestroyUniverse, + TaskType.DestroyKubernetesUniverse, + TaskType.EditBackupSchedule, + TaskType.EditBackupScheduleKubernetes, + TaskType.MultiTableBackup, + TaskType.ReadOnlyClusterDelete); + private static final Set RERUNNABLE_PLACEMENT_MODIFICATION_TASKS = ImmutableSet.of( TaskType.GFlagsUpgrade, @@ -1172,6 +1193,13 @@ public Universe lockAndFreezeUniverseForUpdate( Universe universe = lockUniverseForUpdate(universeUuid, updater); try { createPrecheckTasks(universe); + TaskType taskType = getTaskExecutor().getTaskType(getClass()); + if (!SKIP_CONSISTENCY_CHECK_TASKS.contains(taskType) + && confGetter.getConfForScope(universe, UniverseConfKeys.enableConsistencyCheck) + && universe.getUniverseDetails().getPrimaryCluster().userIntent.replicationFactor > 1) { + log.info("Creating consistency check task for task {}", taskType); + checkAndCreateConsistencyCheckTableTask(universe.getUniverseDetails().getPrimaryCluster()); + } if (isFirstTry()) { createFreezeUniverseTask(universeUuid, firstRunTxnCallback) .setSubTaskGroupType(SubTaskGroupType.ValidateConfigurations); @@ -1181,14 +1209,6 @@ public Universe lockAndFreezeUniverseForUpdate( createFreezeUniverseTask(universeUuid) .setSubTaskGroupType(SubTaskGroupType.ValidateConfigurations); } - if (confGetter.getConfForScope(universe, UniverseConfKeys.enableConsistencyCheck)) { - TaskType taskType = getTaskExecutor().getTaskType(getClass()); - if (taskType != TaskType.CreateUniverse && taskType != TaskType.CreateKubernetesUniverse) { - log.info("Creating consistency check task for task {}", taskType); - checkAndCreateConsistencyCheckTableTask( - universe.getUniverseDetails().getPrimaryCluster()); - } - } return Universe.getOrBadRequest(universeUuid); } catch (RuntimeException e) { unlockUniverseForUpdate(universeUuid); @@ -1462,7 +1482,7 @@ public void createDropSystemPlatformDBTablesTask( universe, CommonTypes.TableType.PGSQL_TABLE_TYPE, Util.SYSTEM_PLATFORM_DB, - Util.CONSISTENCY_CHECK) + Util.CONSISTENCY_CHECK_TABLE_NAME) .setSubTaskGroupType(subTaskGroupType); } diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/UpdateConsistencyCheck.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/UpdateConsistencyCheck.java index 9395ae4efea4..6f4e0ba23b64 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/UpdateConsistencyCheck.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/UpdateConsistencyCheck.java @@ -10,7 +10,7 @@ package com.yugabyte.yw.commissioner.tasks.subtasks; -import static com.yugabyte.yw.common.Util.CONSISTENCY_CHECK; +import static com.yugabyte.yw.common.Util.CONSISTENCY_CHECK_TABLE_NAME; import static com.yugabyte.yw.common.Util.SYSTEM_PLATFORM_DB; import static play.mvc.Http.Status.BAD_REQUEST; import static play.mvc.Http.Status.INTERNAL_SERVER_ERROR; @@ -23,23 +23,22 @@ import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.RecoverableException; import com.yugabyte.yw.common.ShellResponse; +import com.yugabyte.yw.common.Util; import com.yugabyte.yw.common.YsqlQueryExecutor.ConsistencyInfoResp; +import com.yugabyte.yw.common.config.UniverseConfKeys; import com.yugabyte.yw.forms.RunQueryFormData; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.forms.UniverseTaskParams; import com.yugabyte.yw.models.PendingConsistencyCheck; import com.yugabyte.yw.models.Universe; -import com.yugabyte.yw.models.helpers.ColumnDetails; -import com.yugabyte.yw.models.helpers.ColumnDetails.YQLDataType; import com.yugabyte.yw.models.helpers.CommonUtils; import com.yugabyte.yw.models.helpers.NodeDetails; -import com.yugabyte.yw.models.helpers.TableDetails; +import com.yugabyte.yw.models.helpers.TaskType; import java.time.Duration; -import java.util.ArrayList; +import java.util.UUID; import javax.inject.Inject; import lombok.extern.slf4j.Slf4j; -import org.yb.ColumnSchema.SortOrder; -import org.yb.CommonTypes.TableType; +import org.apache.commons.lang3.StringUtils; @Slf4j public class UpdateConsistencyCheck extends UniverseTaskBase { @@ -57,156 +56,292 @@ protected Params taskParams() { return (Params) taskParams; } - public void createConsistencyCheckTable() { - - CreateTable task = createTask(CreateTable.class); - ColumnDetails seqNumColumn = new ColumnDetails(); - seqNumColumn.isClusteringKey = true; - seqNumColumn.name = "seq_num"; - seqNumColumn.type = YQLDataType.INT; - seqNumColumn.sortOrder = SortOrder.ASC; - - ColumnDetails opUUIDColumn = new ColumnDetails(); - opUUIDColumn.name = "task_uuid"; - opUUIDColumn.type = YQLDataType.UUID; - - TableDetails details = new TableDetails(); - details.tableName = CONSISTENCY_CHECK; - details.keyspace = SYSTEM_PLATFORM_DB; - details.columns = new ArrayList<>(); - details.columns.add(seqNumColumn); - details.columns.add(opUUIDColumn); - - CreateTable.Params params = new CreateTable.Params(); - params.setUniverseUUID(taskParams().getUniverseUUID()); - params.tableType = TableType.PGSQL_TABLE_TYPE; - params.tableName = details.tableName; - params.tableDetails = details; - params.ifNotExist = true; + public void createConsistencyCheckTable(Universe universe, NodeDetails node) { + String createQuery = + String.format( + "CREATE TABLE IF NOT EXISTS %s (seq_num INT, task_uuid UUID, yw_uuid UUID, yw_host" + + " VARCHAR, PRIMARY KEY (task_uuid HASH, seq_num DESC)) SPLIT INTO 1 TABLETS;", + CONSISTENCY_CHECK_TABLE_NAME); + RunQueryFormData runQueryFormData = new RunQueryFormData(); + runQueryFormData.setDbName(SYSTEM_PLATFORM_DB); + runQueryFormData.setQuery(createQuery); + JsonNode ysqlResponse = + ysqlQueryExecutor.executeQueryInNodeShell( + universe, + runQueryFormData, + node, + confGetter.getConfForScope(universe, UniverseConfKeys.ysqlConsistencyTimeoutSecs)); + int retries = 0; + // Retry loop + while (ysqlResponse != null && ysqlResponse.has("error") && retries < 5) { + retries += 1; + ysqlResponse = + ysqlQueryExecutor.executeQueryInNodeShell( + universe, + runQueryFormData, + CommonUtils.getARandomLiveOrToBeRemovedTServer(universe), + confGetter.getConfForScope(universe, UniverseConfKeys.ysqlConsistencyTimeoutSecs)); + } + if (ysqlResponse != null && ysqlResponse.has("error")) { + TaskType taskType = getTaskExecutor().getTaskType(getClass()); + if (taskType == TaskType.CreateUniverse || taskType == TaskType.CreateKubernetesUniverse) { + log.error( + "Could not create initial consistency check table for new universe {}.", + universe.getName()); + throw new PlatformServiceException( + INTERNAL_SERVER_ERROR, ysqlResponse.get("error").asText()); + } else { + log.warn( + "Could not create consistency check table for existing universe {}, skipping. Is the" + + " universe healthy?", + universe.getName()); + return; + } + } + runQueryFormData.setQuery( + String.format( + "INSERT INTO %s (seq_num, task_uuid, yw_uuid, yw_host) VALUES (0, '%s', '%s'," + + " '%s')", + CONSISTENCY_CHECK_TABLE_NAME, + getTaskUUID(), + configHelper.getYugawareUUID(), + getYwHostname())); + ysqlResponse = + ysqlQueryExecutor.executeQueryInNodeShell( + universe, + runQueryFormData, + node, + confGetter.getConfForScope(universe, UniverseConfKeys.ysqlConsistencyTimeoutSecs)); + retries = 0; + // retry loop + while (ysqlResponse != null && ysqlResponse.has("error") && retries < 5) { + retries += 1; + node = CommonUtils.getARandomLiveOrToBeRemovedTServer(universe); + ysqlResponse = + ysqlQueryExecutor.executeQueryInNodeShell( + universe, + runQueryFormData, + node, + confGetter.getConfForScope(universe, UniverseConfKeys.ysqlConsistencyTimeoutSecs)); + } + if (ysqlResponse != null && ysqlResponse.has("error")) { + log.warn("Could not perform inital insert into consistency check table."); + throw new PlatformServiceException(INTERNAL_SERVER_ERROR, ysqlResponse.get("error").asText()); + } + // Update local YBA sequence number with initial value + updateUniverseSeqNum(universe, 0); + } - task.initialize(params); - task.setUserTaskUUID(getUserTaskUUID()); - task.run(); + private String getYwHostname() { + String host = Util.getHostname(); + if (StringUtils.isNotBlank(host) && !host.equalsIgnoreCase("localhost")) { + return host; + } + String ip = Util.getHostIP(); + if (!ip.equalsIgnoreCase("127.0.0.1")) { + return ip; + } + return ""; } @Override public void run() { Universe universe = Universe.getOrBadRequest(taskParams().getUniverseUUID()); + String universeName = universe.getName(); + UUID universeUUID = universe.getUniverseUUID(); + int ybaSeqNum = universe.getUniverseDetails().sequenceNumber; NodeDetails node; try { node = CommonUtils.getServerToRunYsqlQuery(universe, true); } catch (IllegalStateException e) { - log.warn("Could not find valid tserver, skipping consistency check."); - return; - } - ConsistencyInfoResp response; - try { - response = ysqlQueryExecutor.getConsistencyInfo(universe); - } catch (RecoverableException e) { - log.info("Creating consistency check table for the first time."); - createConsistencyCheckTable(); - RunQueryFormData runQueryFormData = new RunQueryFormData(); - runQueryFormData.setQuery( - String.format( - "INSERT INTO %s (seq_num, task_uuid) VALUES (0, '%s')", - CONSISTENCY_CHECK, getTaskUUID().toString())); - runQueryFormData.setDbName(SYSTEM_PLATFORM_DB); - JsonNode ysqlResponse = - ysqlQueryExecutor.executeQueryInNodeShell(universe, runQueryFormData, node); - if (ysqlResponse != null && ysqlResponse.has("error")) { - throw new PlatformServiceException( - INTERNAL_SERVER_ERROR, ysqlResponse.get("error").asText()); - } - // Update local YBA sequence number with initial value - updateUniverseSeqNum(0); + log.warn( + "Could not find valid tserver, skipping consistency check for universe {} ({}).", + universeName, + universeUUID); return; } - - if (response != null) { - int ybaSeqNum = universe.getUniverseDetails().sequenceNumber; - int dbSeqNum = response.getSeqNum(); - // Accept whatever is in the DB - if (ybaSeqNum == -1) { - log.info("Accepting whatever sequence number found in YBDB."); - updateUniverseSeqNum(dbSeqNum); + // Special case for -1 + if (ybaSeqNum == -1) { + log.info("YBA sequence number is -1 for universe {} ({})", universeName, universeUUID); + ConsistencyInfoResp response = null; + try { + response = ysqlQueryExecutor.getConsistencyInfo(universe); + } catch (RecoverableException e) { + // Table does not exist + log.info( + "Creating consistency check table for the first time for universe {} ({}).", + universeName, + universeUUID); + createConsistencyCheckTable(universe, node); return; - // Stale - } else if (dbSeqNum > ybaSeqNum) { - throw new PlatformServiceException( - BAD_REQUEST, "Can not operate on universe with stale metadata."); + } + if (response != null) { + log.info( + "Accepting db sequence number {} for universe {} ({})", + response.getSeqNum(), + universeName, + universeUUID); + updateUniverseSeqNum(universe, response.getSeqNum()); } else { - log.info("Validated YBA sequence number"); + log.warn( + "Could not read consistency info for universe {} ({}) and local is -1, skipping" + + " consistency check", + universeName, + universeUUID); } - } else { - log.warn( - "Could not read consistency info, skipping comparison validation but proceeding with" - + " update."); + return; } - // Update values + // Normal update int ybaSeqNumUpdate = universe.getUniverseDetails().sequenceNumber + 1; PendingConsistencyCheck pend = PendingConsistencyCheck.create(getTaskUUID(), universe); - String taskUUIDString = getTaskUUID().toString(); RunQueryFormData runQueryFormData = new RunQueryFormData(); String updateQuery = String.format( - "UPDATE %s SET seq_num = %d, task_uuid = '%s' WHERE seq_num < %d OR" - + " (seq_num = %d AND task_uuid = '%s') RETURNING seq_num, task_uuid", - CONSISTENCY_CHECK, + "WITH updated_rows AS (UPDATE %s SET seq_num = %d, task_uuid = '%s', yw_uuid = '%s'," + + " yw_host = '%s' WHERE seq_num < %d OR (seq_num = %d AND task_uuid = '%s')" + + " RETURNING seq_num, task_uuid) SELECT jsonb_agg(updated_rows) AS result FROM" + + " updated_rows;", + CONSISTENCY_CHECK_TABLE_NAME, ybaSeqNumUpdate, - taskUUIDString, + getTaskUUID(), + configHelper.getYugawareUUID(), + getYwHostname(), ybaSeqNumUpdate, ybaSeqNumUpdate, - taskUUIDString); - runQueryFormData.setQuery( - String.format( - "WITH updated_rows AS (%s) SELECT jsonb_agg(updated_rows) AS result FROM" - + " updated_rows;", - updateQuery)); + getTaskUUID()); + // Testing string that includes pg_sleep + if (confGetter.getConfForScope(universe, UniverseConfKeys.consistencyUpdateDelay) > 0) { + updateQuery = + String.format( + "BEGIN; UPDATE %s SET seq_num = %d, task_uuid = '%s', yw_uuid = '%s', yw_host = '%s'" + + " WHERE seq_num < %d OR (seq_num = %d AND task_uuid = '%s'); COMMIT; SELECT" + + " pg_sleep(%d); SELECT jsonb_agg(x) FROM (SELECT seq_num, task_uuid FROM %s" + + " ORDER BY seq_num DESC LIMIT 1) as x;", + CONSISTENCY_CHECK_TABLE_NAME, + ybaSeqNumUpdate, + getTaskUUID(), + configHelper.getYugawareUUID(), + getYwHostname(), + ybaSeqNumUpdate, + ybaSeqNumUpdate, + getTaskUUID(), + confGetter.getConfForScope(universe, UniverseConfKeys.consistencyUpdateDelay), + CONSISTENCY_CHECK_TABLE_NAME); + } + runQueryFormData.setQuery(updateQuery); runQueryFormData.setDbName(SYSTEM_PLATFORM_DB); + log.info( + "Attempting to update DB to sequence number {} for universe {} ({})", + ybaSeqNumUpdate, + universeName, + universeUUID); try { + // Attempt to update DB JsonNode ysqlResponse = - ysqlQueryExecutor.executeQueryInNodeShell(universe, runQueryFormData, node); + ysqlQueryExecutor.executeQueryInNodeShell( + universe, + runQueryFormData, + node, + confGetter.getConfForScope(universe, UniverseConfKeys.ysqlConsistencyTimeoutSecs)); int retries = 0; + // Retry loop while (ysqlResponse != null && ysqlResponse.has("error") && retries < 5) { - waitFor(Duration.ofMillis(2500)); + node = CommonUtils.getARandomLiveOrToBeRemovedTServer(universe); retries += 1; - ysqlResponse = ysqlQueryExecutor.executeQueryInNodeShell(universe, runQueryFormData, node); + ysqlResponse = + ysqlQueryExecutor.executeQueryInNodeShell( + universe, + runQueryFormData, + node, + confGetter.getConfForScope(universe, UniverseConfKeys.ysqlConsistencyTimeoutSecs)); + } + // Testing hook for CustomerTaskManager.handlePendingConsistencyTasks + if (confGetter.getConfForScope(universe, UniverseConfKeys.consistencyCheckPendingTest)) { + Util.shutdownYbaProcess(0); + waitFor(Duration.ofMillis(10000)); + } + // Error case with unexpected failure running remote query, unable to validate so reset to + // be safe + if (ysqlResponse != null && ysqlResponse.has("error")) { + log.warn( + "Consistency check is not active for universe {} due to error: {}. Resetting local" + + " sequence number.", + universeName, + ysqlResponse.get("error").asText()); + updateUniverseSeqNum(universe, -1); + return; } if (ysqlResponse != null && ysqlResponse.has("result")) { ShellResponse shellResponse = ShellResponse.create(0, ysqlResponse.get("result").asText()); ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode = objectMapper.readTree(CommonUtils.extractJsonisedSqlResponse(shellResponse)); - if (jsonNode != null && jsonNode.get(0) != null && jsonNode.get(0).has("seq_num")) { - // Update local YBA sequence number with update value - updateUniverseSeqNum(jsonNode.get(0).get("seq_num").asInt()); - } else { - // no rows updated, must be stale + // No rows updated, stale metadata + if (jsonNode == null + || jsonNode.get(0) == null + || !jsonNode.get(0).has("seq_num") + || !jsonNode.get(0).has("task_uuid")) { + // Best effort try to read what's in the DB for better error message. + ConsistencyInfoResp response = ysqlQueryExecutor.getConsistencyInfo(universe); + if (response != null) { + throw new PlatformServiceException( + BAD_REQUEST, + String.format( + "No rows updated performing consistency check, stale universe metadata. DB at" + + " version %d set by YBA %s (%s) during task %s. Task should be run from" + + " up to date YBA or contact Yugabyte Support to resolve.", + response.getSeqNum(), + response.getYwHost(), + response.getYwUUID(), + response.getTaskUUID())); + } throw new PlatformServiceException( BAD_REQUEST, - "No rows updated performing consistency check, potentially stale universe" - + " metadata."); + "No rows updated performing consistency check, stale universe metadata. Task should" + + " be run from up to date YBA or contact Yugabyte Support to resolve."); } - } else if (ysqlResponse != null && ysqlResponse.has("error")) { - log.warn( - "Consistency check is not active due to error: {}.", - ysqlResponse.get("error").asText()); + // Valid result, perform update + int dbSeqNum = jsonNode.get(0).get("seq_num").asInt(); + UUID dbTaskUuid = UUID.fromString(jsonNode.get(0).get("task_uuid").asText()); + + // Doubtful this should ever execute, but in case, treat as stale. + if (dbSeqNum != ybaSeqNumUpdate || !dbTaskUuid.equals(getTaskUUID())) { + throw new PlatformServiceException( + BAD_REQUEST, + String.format( + "Found different values (seq_num: %d, task_uuid: %s) in DB after intended update." + + " Expected seq_num: %d and task_uuid: %s.", + dbSeqNum, dbTaskUuid, ybaSeqNumUpdate, getTaskUUID())); + } + // Update local YBA sequence number with update value + log.info( + "Updated DB to seq_num: {} and task_uuid: {}, setting local sequence number for" + + " universe {}.", + dbSeqNum, + dbTaskUuid, + universeName); + updateUniverseSeqNum(universe, dbSeqNum); } } catch (JsonProcessingException e) { - log.warn("Error processing JSON response from update query: {}.", e.getMessage()); + log.warn( + "Error processing JSON response from update query: {}. Consistency check may not be" + + " active.", + e.getMessage()); + updateUniverseSeqNum(universe, -1); } finally { pend.delete(); } } - private void updateUniverseSeqNum(int seqNum) { + private void updateUniverseSeqNum(Universe universe, int seqNum) { Universe.UniverseUpdater updater = - universe -> { - UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); + u -> { + UniverseDefinitionTaskParams universeDetails = u.getUniverseDetails(); universeDetails.sequenceNumber = seqNum; - universe.setUniverseDetails(universeDetails); + u.setUniverseDetails(universeDetails); }; saveUniverseDetails(updater); + log.info("Updated {} universe details sequence number to {}.", universe.getName(), seqNum); } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/ConfigHelper.java b/managed/src/main/java/com/yugabyte/yw/common/ConfigHelper.java index 2572ac787bb4..c14c8a7d5181 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/ConfigHelper.java +++ b/managed/src/main/java/com/yugabyte/yw/common/ConfigHelper.java @@ -131,6 +131,14 @@ public void loadSoftwareVersiontoDB(Environment environment) { loadConfigToDB(ConfigType.YugawareMetadata, ywMetadata); } + public UUID getYugawareUUID() { + Object ywUUID = getConfig(ConfigHelper.ConfigType.YugawareMetadata).get("yugaware_uuid"); + if (ywUUID != null) { + return UUID.fromString(ywUUID.toString()); + } + return null; + } + public void loadConfigsToDB(Environment environment) { LoaderOptions loaderOptions = new LoaderOptions(); diff --git a/managed/src/main/java/com/yugabyte/yw/common/CustomerTaskManager.java b/managed/src/main/java/com/yugabyte/yw/common/CustomerTaskManager.java index b972a5bb51f3..30d890168272 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/CustomerTaskManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/CustomerTaskManager.java @@ -739,7 +739,7 @@ public void handlePendingConsistencyTasks() { Universe universe = Universe.getOrBadRequest(pending.getUniverse().getUniverseUUID()); ConsistencyInfoResp response = ysqlQueryExecutor.getConsistencyInfo(universe); if (response != null) { - UUID dbTaskUuid = response.getTaskUuid(); + UUID dbTaskUuid = response.getTaskUUID(); int dbSeqNum = response.getSeqNum(); if (dbTaskUuid.equals(pending.getTaskUuid())) { // Updated on DB side before crash, set ourselves to whatever is in the DB diff --git a/managed/src/main/java/com/yugabyte/yw/common/Util.java b/managed/src/main/java/com/yugabyte/yw/common/Util.java index 3f646977fa79..a1b01366529d 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/Util.java +++ b/managed/src/main/java/com/yugabyte/yw/common/Util.java @@ -108,7 +108,7 @@ public class Util { public static final String DEFAULT_YCQL_USERNAME = "cassandra"; public static final String DEFAULT_YCQL_PASSWORD = "cassandra"; public static final String YUGABYTE_DB = "yugabyte"; - public static final String CONSISTENCY_CHECK = "consistency_check"; + public static final String CONSISTENCY_CHECK_TABLE_NAME = "yba_consistency_check"; public static final int MIN_NUM_BACKUPS_TO_RETAIN = 3; public static final String REDACT = "REDACTED"; public static final String KEY_LOCATION_SUFFIX = "/backup_keys.json"; diff --git a/managed/src/main/java/com/yugabyte/yw/common/YsqlQueryExecutor.java b/managed/src/main/java/com/yugabyte/yw/common/YsqlQueryExecutor.java index cdce2f380ea7..93259759422b 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/YsqlQueryExecutor.java +++ b/managed/src/main/java/com/yugabyte/yw/common/YsqlQueryExecutor.java @@ -2,7 +2,7 @@ package com.yugabyte.yw.common; -import static com.yugabyte.yw.common.Util.CONSISTENCY_CHECK; +import static com.yugabyte.yw.common.Util.CONSISTENCY_CHECK_TABLE_NAME; import static com.yugabyte.yw.common.Util.SYSTEM_PLATFORM_DB; import static play.libs.Json.newObject; import static play.libs.Json.toJson; @@ -509,13 +509,27 @@ public static class ConsistencyInfoResp { @JsonProperty("seq_num") private int seqNum; - public UUID getTaskUuid() { + @JsonProperty("yw_uuid") + private UUID ywUuid; + + @JsonProperty("yw_host") + private String ywHost; + + public UUID getTaskUUID() { return taskUuid; } public int getSeqNum() { return seqNum; } + + public UUID getYwUUID() { + return ywUuid; + } + + public String getYwHost() { + return ywHost; + } } public ConsistencyInfoResp getConsistencyInfo(Universe universe) throws RecoverableException { @@ -530,14 +544,16 @@ public ConsistencyInfoResp getConsistencyInfo(Universe universe) throws Recovera ysqlQuery.setDbName(SYSTEM_PLATFORM_DB); ysqlQuery.setQuery( String.format( - "SELECT seq_num, task_uuid FROM %s ORDER BY seq_num DESC LIMIT 1", CONSISTENCY_CHECK)); + "SELECT seq_num, task_uuid, yw_uuid, yw_host FROM %s ORDER BY seq_num DESC LIMIT 1", + CONSISTENCY_CHECK_TABLE_NAME)); JsonNode response = executeQueryInNodeShell(universe, ysqlQuery, node); int retries = 0; while (response != null && response.has("error") && retries < 5) { - String match = String.format("relation \"%s\" does not exist", CONSISTENCY_CHECK); + String match = String.format("relation \"%s\" does not exist", CONSISTENCY_CHECK_TABLE_NAME); if (response.get("error").asText().contains(match)) { throw new RecoverableException("consistency_check table does not exist"); } + node = CommonUtils.getARandomLiveOrToBeRemovedTServer(universe); retries += 1; response = executeQueryInNodeShell(universe, ysqlQuery, node); } diff --git a/managed/src/main/java/com/yugabyte/yw/common/config/UniverseConfKeys.java b/managed/src/main/java/com/yugabyte/yw/common/config/UniverseConfKeys.java index 04a32963f36c..99efc6b6540e 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/config/UniverseConfKeys.java +++ b/managed/src/main/java/com/yugabyte/yw/common/config/UniverseConfKeys.java @@ -826,6 +826,14 @@ public class UniverseConfKeys extends RuntimeConfigKeysModule { "Timeout in secs for YSQL queries", ConfDataType.LongType, ImmutableList.of(ConfKeyTags.PUBLIC)); + public static final ConfKeyInfo ysqlConsistencyTimeoutSecs = + new ConfKeyInfo<>( + "yb.universe.consistency_check.ysql_timeout_secs", + ScopeType.UNIVERSE, + "YSQL Queries Timeout for Consistency Check Operations", + "Timeout in secs for YSQL queries", + ConfDataType.LongType, + ImmutableList.of(ConfKeyTags.PUBLIC)); public static final ConfKeyInfo numCoresToKeep = new ConfKeyInfo<>( "yb.num_cores_to_keep", @@ -1323,7 +1331,7 @@ public class UniverseConfKeys extends RuntimeConfigKeysModule { ImmutableList.of(ConfKeyTags.INTERNAL)); public static final ConfKeyInfo enableConsistencyCheck = new ConfKeyInfo<>( - "yb.universe.consistency_check_enabled", + "yb.universe.consistency_check.enabled", ScopeType.UNIVERSE, "Enable consistency check for universe", "When enabled, all universe operations will attempt consistency check validation before" @@ -1346,4 +1354,22 @@ public class UniverseConfKeys extends RuntimeConfigKeysModule { "Node agent enabler installation time-out for the universe", ConfDataType.DurationType, ImmutableList.of(ConfKeyTags.PUBLIC)); + public static final ConfKeyInfo consistencyUpdateDelay = + new ConfKeyInfo<>( + "yb.universe.consistency_check.update_delay_secs", + ScopeType.UNIVERSE, + "Delay after updating consistency check information in YBDB", + "Introduces a sleep in the PG update query to help simulate testing with unreliable YSQL" + + " connection", + ConfDataType.LongType, + ImmutableList.of(ConfKeyTags.INTERNAL)); + public static final ConfKeyInfo consistencyCheckPendingTest = + new ConfKeyInfo<>( + "yb.universe.consistency_check.test_pending", + ScopeType.UNIVERSE, + "Test handling of pending consistency check update", + "YBA will shutdown immediately after updating the DB sequence number but before saving to" + + " local universe details", + ConfDataType.BooleanType, + ImmutableList.of(ConfKeyTags.INTERNAL)); } diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/CommonUtils.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/CommonUtils.java index dc8ce8f21ba9..0ecc963e2b6b 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/helpers/CommonUtils.java +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/CommonUtils.java @@ -810,7 +810,7 @@ public static NodeDetails getServerToRunYsqlQuery(Universe universe, boolean use .toList(); NodeDetails sameRegionTServer; if (useToBeRemoved) { - sameRegionTServer = getARandomLiveOrRemovedTServer(sameRegionNodes); + sameRegionTServer = getARandomLiveOrToBeRemovedTServer(sameRegionNodes); } else { sameRegionTServer = getARandomLiveTServer(sameRegionNodes); } @@ -820,7 +820,7 @@ public static NodeDetails getServerToRunYsqlQuery(Universe universe, boolean use } } if (useToBeRemoved) { - return getARandomLiveOrRemovedTServer(universe); + return getARandomLiveOrToBeRemovedTServer(universe); } return getARandomLiveTServer(universe); } @@ -837,9 +837,9 @@ private static NodeDetails getARandomLiveTServer(Collection nodes) return tserverLiveNodes.get(new Random().nextInt(tserverLiveNodes.size())); } - public static NodeDetails getARandomLiveOrRemovedTServer(Universe universe) { + public static NodeDetails getARandomLiveOrToBeRemovedTServer(Universe universe) { NodeDetails randomLiveOrRemovedTServer = - getARandomLiveOrRemovedTServer(universe.getTServersInPrimaryCluster()); + getARandomLiveOrToBeRemovedTServer(universe.getTServersInPrimaryCluster()); if (randomLiveOrRemovedTServer == null) { throw new IllegalStateException( "No live or toBeRemoved TServers found for Universe UUID: " + universe.getUniverseUUID()); @@ -847,7 +847,7 @@ public static NodeDetails getARandomLiveOrRemovedTServer(Universe universe) { return randomLiveOrRemovedTServer; } - private static NodeDetails getARandomLiveOrRemovedTServer(Collection nodes) { + private static NodeDetails getARandomLiveOrToBeRemovedTServer(Collection nodes) { List tserverLiveNodes = nodes.stream() .filter(nodeDetails -> nodeDetails.isTserver) diff --git a/managed/src/main/resources/reference.conf b/managed/src/main/resources/reference.conf index e85740662326..32d72ca7d55b 100644 --- a/managed/src/main/resources/reference.conf +++ b/managed/src/main/resources/reference.conf @@ -245,8 +245,13 @@ yb { otel_collector_metrics_port = 8889 audit_logging_enabled = false allow_connection_pooling = false - consistency_check_enabled = false default_service_scope_for_k8s="AZ" # possible: Namespaced, AZ + consistency_check { + enabled = true + test_pending = false # NEVER SET IN PROD + update_delay_secs = 0 # NEVER SET IN PROD + ysql_timeout_secs = 30 + } } xcluster { diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/MockUpgrade.java b/managed/src/test/java/com/yugabyte/yw/commissioner/MockUpgrade.java index f7451cba95a0..43061fa04012 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/MockUpgrade.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/MockUpgrade.java @@ -176,10 +176,10 @@ public MockUpgrade precheckTasks(boolean enableYSQL, TaskType... taskTypes) { for (TaskType taskType : taskTypes) { addTask(taskType, null); } - addTask(TaskType.FreezeUniverse, null); if (enableYSQL) { addTask(TaskType.UpdateConsistencyCheck, null); } + addTask(TaskType.FreezeUniverse, null); addHookTasks(true); return this; } diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/AddNodeToUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/AddNodeToUniverseTest.java index 83ea33d1395a..8d52ecaeda93 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/AddNodeToUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/AddNodeToUniverseTest.java @@ -194,8 +194,8 @@ private TaskInfo submitTask(UUID universeUUID, Provider provider, String nodeNam ImmutableList.of( TaskType.InstanceExistCheck, // only if it wasn't decommissioned. TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, // to Adding TaskType.SetNodeStatus, // to Adding for 'To Be Added' TaskType.AnsibleCreateServer, @@ -252,8 +252,8 @@ private TaskInfo submitTask(UUID universeUUID, Provider provider, String nodeNam ImmutableList.of( TaskType.CheckLeaderlessTablets, TaskType.PreflightNodeCheck, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.SetNodeStatus, TaskType.AnsibleCreateServer, @@ -310,8 +310,8 @@ private TaskInfo submitTask(UUID universeUUID, Provider provider, String nodeNam ImmutableList.of( TaskType.InstanceExistCheck, TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.SetNodeStatus, TaskType.AnsibleCreateServer, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteNodeFromUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteNodeFromUniverseTest.java index a68f02833ffd..b3e25bdd6246 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteNodeFromUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteNodeFromUniverseTest.java @@ -50,8 +50,8 @@ public class DeleteNodeFromUniverseTest extends CommissionerBaseTest { private static final List DELETE_NODE_TASK_SEQUENCE_WITH_INSTANCE = ImmutableList.of( - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.CheckNodeSafeToDelete, TaskType.AnsibleDestroyServer, TaskType.DeleteNode, @@ -59,8 +59,8 @@ public class DeleteNodeFromUniverseTest extends CommissionerBaseTest { private static final List DELETE_NODE_TASK_SEQUENCE_WITHOUT_INSTANCE = ImmutableList.of( - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.RemoveNodeAgent, TaskType.DeleteNode, TaskType.UniverseUpdateSucceeded); diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfigTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfigTest.java index cacc70f1c48f..b2f8657100d9 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfigTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DeleteXClusterConfigTest.java @@ -87,9 +87,7 @@ public class DeleteXClusterConfigTest extends CommissionerBaseTest { List DELETE_XCLUSTER_CONFIG_TASK_SEQUENCE = ImmutableList.of( TaskType.FreezeUniverse, - TaskType.UpdateConsistencyCheck, TaskType.FreezeUniverse, - TaskType.UpdateConsistencyCheck, TaskType.XClusterConfigSetStatus, TaskType.DeleteReplication, TaskType.DeleteBootstrapIds, @@ -382,7 +380,7 @@ public void testDeleteXClusterFailure() { assertNotNull(subtaskGroup); assertEquals(DELETE_XCLUSTER_CONFIG_TASK_SEQUENCE.get(i), subtaskGroup.getTaskType()); } - String taskErrMsg = taskInfo.getSubTasks().get(5).getErrorMessage(); + String taskErrMsg = taskInfo.getSubTasks().get(3).getErrorMessage(); String expectedErrMsg = String.format( "Failed to delete replication for XClusterConfig(%s): %s", diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverseTest.java index fcfb8316c086..db00917d0f88 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/DestroyKubernetesUniverseTest.java @@ -191,8 +191,6 @@ private void assertTaskSequence( // Shift by 1 subtask due to FreezeUniverse. assertEquals( TaskType.FreezeUniverse, subTasksByPosition.get(position++).get(0).getTaskType()); - assertEquals( - TaskType.UpdateConsistencyCheck, subTasksByPosition.get(position++).get(0).getTaskType()); } for (int i = 0; i < KUBERNETES_DESTROY_UNIVERSE_TASKS.size(); i++) { TaskType taskType = KUBERNETES_DESTROY_UNIVERSE_TASKS.get(i); diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverseTest.java index aaeeab04a16d..7760948cfc3d 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditKubernetesUniverseTest.java @@ -214,7 +214,6 @@ private void setupUniverseMultiAZ(boolean setMasters, int numTservers) { ImmutableList.of( TaskType.CheckLeaderlessTablets, TaskType.FreezeUniverse, - TaskType.UpdateConsistencyCheck, TaskType.HandleKubernetesNamespacedServices, TaskType.KubernetesCommandExecutor, TaskType.KubernetesCheckNumPod, @@ -232,7 +231,6 @@ private List getExpectedAddPodTaskResults() { Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), - Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("commandType", HELM_UPGRADE.name())), Json.toJson(ImmutableMap.of("commandType", WAIT_FOR_PODS.name())), Json.toJson(ImmutableMap.of("commandType", POD_INFO.name())), @@ -249,7 +247,6 @@ private List getExpectedAddPodTaskResults() { ImmutableList.of( TaskType.CheckLeaderlessTablets, TaskType.FreezeUniverse, - TaskType.UpdateConsistencyCheck, TaskType.HandleKubernetesNamespacedServices, TaskType.UpdatePlacementInfo, TaskType.WaitForDataMove, @@ -272,7 +269,6 @@ private List getExpectedRemovePodTaskResults() { Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), - Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("commandType", HELM_UPGRADE.name())), Json.toJson(ImmutableMap.of("commandType", WAIT_FOR_PODS.name())), Json.toJson(ImmutableMap.of()), @@ -288,7 +284,6 @@ private List getExpectedRemovePodTaskResults() { ImmutableList.of( TaskType.CheckLeaderlessTablets, TaskType.FreezeUniverse, - TaskType.UpdateConsistencyCheck, TaskType.HandleKubernetesNamespacedServices, TaskType.UpdatePlacementInfo, TaskType.CheckUnderReplicatedTablets, @@ -325,7 +320,6 @@ private List getExpectedChangeInstaceTypeResults() { Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), - Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("commandType", HELM_UPGRADE.name())), Json.toJson(ImmutableMap.of("commandType", WAIT_FOR_POD.name())), Json.toJson(ImmutableMap.of()), diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditUniverseTest.java index 17219e4f1537..540439d7351b 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditUniverseTest.java @@ -75,8 +75,8 @@ public class EditUniverseTest extends UniverseModifyBaseTest { private static final List UNIVERSE_EXPAND_TASK_SEQUENCE = ImmutableList.of( TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeStatus, // ToBeAdded to Adding TaskType.AnsibleCreateServer, TaskType.AnsibleUpdateNodeInfo, @@ -125,8 +125,8 @@ public class EditUniverseTest extends UniverseModifyBaseTest { ImmutableList.of( TaskType.CheckLeaderlessTablets, TaskType.PreflightNodeCheck, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeStatus, // ToBeAdded to Adding TaskType.AnsibleCreateServer, TaskType.AnsibleUpdateNodeInfo, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfigTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfigTest.java index 8e289d9401b3..0d96a0aaae0a 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfigTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/EditXClusterConfigTest.java @@ -114,11 +114,11 @@ public class EditXClusterConfigTest extends CommissionerBaseTest { List RENAME_FAILURE_TASK_SEQUENCE = ImmutableList.of( // Freeze for source. - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, - // Freeze for target. TaskType.FreezeUniverse, + // Freeze for target. TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.XClusterConfigSetStatus, TaskType.XClusterConfigRename, TaskType.XClusterConfigSetStatus, @@ -128,11 +128,11 @@ public class EditXClusterConfigTest extends CommissionerBaseTest { List ADD_TABLE_IS_ALTER_DONE_FAILURE = ImmutableList.of( // Freeze for source. - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, - // Freeze for target. TaskType.FreezeUniverse, + // Freeze for target. TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.XClusterConfigSetStatus, TaskType.XClusterConfigSetStatusForTables, TaskType.BootstrapProducer, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/PauseUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/PauseUniverseTest.java index aaf69ccfb1be..4c1ff896f10c 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/PauseUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/PauseUniverseTest.java @@ -103,7 +103,6 @@ private void setupUniverse(boolean updateInProgress) { private static final List PAUSE_UNIVERSE_TASKS = ImmutableList.of( TaskType.FreezeUniverse, - TaskType.UpdateConsistencyCheck, TaskType.SetNodeState, TaskType.AnsibleClusterServerCtl, TaskType.SetNodeState, @@ -115,7 +114,6 @@ private void setupUniverse(boolean updateInProgress) { private static final List PAUSE_UNIVERSE_EXPECTED_RESULTS = ImmutableList.of( - Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("process", "tserver", "command", "stop")), diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterCreateTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterCreateTest.java index f143c312cde8..d7d4680353dc 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterCreateTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterCreateTest.java @@ -86,8 +86,8 @@ private TaskInfo submitTask(UniverseDefinitionTaskParams taskParams) { private static final List CLUSTER_CREATE_TASK_SEQUENCE = ImmutableList.of( TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeStatus, TaskType.AnsibleCreateServer, TaskType.AnsibleUpdateNodeInfo, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDeleteTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDeleteTest.java index 5d4a55100692..c66d96f85783 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDeleteTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReadOnlyClusterDeleteTest.java @@ -132,7 +132,6 @@ private Cluster addReadReplica(Region region) { private static final List CLUSTER_DELETE_TASK_SEQUENCE = ImmutableList.of( TaskType.FreezeUniverse, - TaskType.UpdateConsistencyCheck, TaskType.CheckLeaderlessTablets, TaskType.SetNodeState, TaskType.AnsibleDestroyServer, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/RebootNodeInUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/RebootNodeInUniverseTest.java index 6265f8715bf0..279599b19eb7 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/RebootNodeInUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/RebootNodeInUniverseTest.java @@ -108,8 +108,8 @@ public void setUp(boolean withMaster, int numNodes, int replicationFactor) { private List rebootNodeTaskSequence(boolean isHardReboot) { return ImmutableList.of( TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.AnsibleClusterServerCtl, isHardReboot ? TaskType.HardRebootServer : TaskType.RebootServer, @@ -139,8 +139,8 @@ private List rebootNodeTaskExpectedResults(boolean isHardReboot) { private List rebootNodeWithMaster(boolean isHardReboot) { return ImmutableList.of( TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.AnsibleClusterServerCtl, TaskType.AnsibleClusterServerCtl, @@ -180,8 +180,8 @@ private List rebootNodeWithMasterResults(boolean isHardReboot) { private List rebootNodeWithOnlyMaster(boolean isHardReboot) { return ImmutableList.of( TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.AnsibleClusterServerCtl, TaskType.WaitForMasterLeader, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReleaseInstanceFromUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReleaseInstanceFromUniverseTest.java index 5ab3e42d223c..4841ee03151f 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReleaseInstanceFromUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ReleaseInstanceFromUniverseTest.java @@ -153,8 +153,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName, int vers private static final List RELEASE_INSTANCE_TASK_SEQUENCE = ImmutableList.of( TaskType.CheckNodeSafeToDelete, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.WaitForMasterLeader, TaskType.SetNodeState, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/RemoveNodeFromUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/RemoveNodeFromUniverseTest.java index 782cd4e6b3cf..4514afdb5150 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/RemoveNodeFromUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/RemoveNodeFromUniverseTest.java @@ -173,8 +173,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { private static final List REMOVE_NODE_TASK_SEQUENCE = ImmutableList.of( TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.UpdatePlacementInfo, TaskType.WaitForDataMove, @@ -201,8 +201,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { private static final List REMOVE_NODE_WITH_MASTER_REPLACE = ImmutableList.of( TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.UpdatePlacementInfo, TaskType.WaitForDataMove, @@ -271,8 +271,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { private static final List REMOVE_NODE_WITH_MASTER = ImmutableList.of( TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.UpdatePlacementInfo, TaskType.WaitForDataMove, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverseTest.java index 4f661720d434..939d416153e2 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/ResumeUniverseTest.java @@ -113,7 +113,6 @@ private void setupUniverse(boolean updateInProgress, int numOfNodes) { private static final List RESUME_UNIVERSE_TASKS = ImmutableList.of( TaskType.FreezeUniverse, - TaskType.UpdateConsistencyCheck, TaskType.ResumeServer, TaskType.WaitForClockSync, // Ensure clock skew is low enough TaskType.AnsibleClusterServerCtl, @@ -131,7 +130,6 @@ private void setupUniverse(boolean updateInProgress, int numOfNodes) { private static final List RESUME_ENCRYPTION_AT_REST_UNIVERSE_TASKS = ImmutableList.of( TaskType.FreezeUniverse, - TaskType.UpdateConsistencyCheck, TaskType.ResumeServer, TaskType.WaitForClockSync, // Ensure clock skew is low enough TaskType.AnsibleClusterServerCtl, @@ -152,7 +150,6 @@ private void setupUniverse(boolean updateInProgress, int numOfNodes) { Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), - Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("process", "master", "command", "start")), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), @@ -170,7 +167,6 @@ private void setupUniverse(boolean updateInProgress, int numOfNodes) { Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), - Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of("process", "master", "command", "start")), Json.toJson(ImmutableMap.of()), Json.toJson(ImmutableMap.of()), diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNodeTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNodeTest.java index a8533160be96..8faa7e02be6b 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNodeTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNodeTest.java @@ -153,8 +153,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { private static final List START_MASTER_TASK_SEQUENCE = ImmutableList.of( TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.WaitForClockSync, // Ensure clock skew is low enough TaskType.AnsibleConfigureServers, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartNodeInUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartNodeInUniverseTest.java index e944a5cf7674..b4280fa029b3 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartNodeInUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartNodeInUniverseTest.java @@ -173,8 +173,8 @@ private Universe setMasters(Universe universe, String... nodeNames) { List START_NODE_TASK_SEQUENCE = ImmutableList.of( - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.WaitForClockSync, // Ensure clock skew is low enough TaskType.AnsibleConfigureServers, @@ -203,8 +203,8 @@ private Universe setMasters(Universe universe, String... nodeNames) { List WITH_MASTER_UNDER_REPLICATED = ImmutableList.of( - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.WaitForClockSync, // Ensure clock skew is low enough TaskType.AnsibleConfigureServers, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StopNodeInUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StopNodeInUniverseTest.java index 0997cd303508..32a6b70be32e 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StopNodeInUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StopNodeInUniverseTest.java @@ -164,8 +164,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { TaskType.CheckUnderReplicatedTablets, TaskType.CheckNodesAreSafeToTakeDown, TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.ModifyBlackList, TaskType.SetNodeState, TaskType.ModifyBlackList, @@ -200,8 +200,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { TaskType.CheckUnderReplicatedTablets, TaskType.CheckNodesAreSafeToTakeDown, TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.ModifyBlackList, TaskType.SetNodeState, TaskType.ModifyBlackList, @@ -238,8 +238,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { TaskType.CheckUnderReplicatedTablets, TaskType.CheckNodesAreSafeToTakeDown, TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.ModifyBlackList, TaskType.SetNodeState, TaskType.ModifyBlackList, @@ -292,8 +292,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { TaskType.CheckUnderReplicatedTablets, TaskType.CheckNodesAreSafeToTakeDown, TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.ModifyBlackList, TaskType.SetNodeState, TaskType.ModifyBlackList, @@ -347,8 +347,8 @@ private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) { ImmutableList.of( TaskType.CheckNodesAreSafeToTakeDown, TaskType.CheckLeaderlessTablets, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.SetNodeState, TaskType.ChangeMasterConfig, TaskType.AnsibleClusterServerCtl, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverseTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverseTest.java index 3c675c6b36d3..e93c71f9f97c 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverseTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/UpgradeKubernetesUniverseTest.java @@ -243,8 +243,8 @@ private void setupUniverseMultiAZ(boolean setMasters) { private static final List KUBERNETES_UPGRADE_SOFTWARE_TASKS = ImmutableList.of( - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.KubernetesCommandExecutor, TaskType.CheckNodesAreSafeToTakeDown, TaskType.KubernetesCommandExecutor, @@ -299,8 +299,8 @@ private void setupUniverseMultiAZ(boolean setMasters) { private static final List KUBERNETES_UPGRADE_GFLAG_TASKS = ImmutableList.of( - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.UpdateAndPersistGFlags, TaskType.KubernetesCommandExecutor, TaskType.CheckNodesAreSafeToTakeDown, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/LocalProviderUniverseTestBase.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/LocalProviderUniverseTestBase.java index 58f4db5128a3..d1d414fe77bf 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/LocalProviderUniverseTestBase.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/LocalProviderUniverseTestBase.java @@ -24,6 +24,7 @@ import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase.ServerType; import com.yugabyte.yw.commissioner.tasks.subtasks.CheckClusterConsistency; import com.yugabyte.yw.common.ApiUtils; +import com.yugabyte.yw.common.ConfigHelper; import com.yugabyte.yw.common.LocalNodeManager; import com.yugabyte.yw.common.LocalNodeUniverseManager; import com.yugabyte.yw.common.ModelFactory; @@ -432,7 +433,7 @@ public void setUp() { settableRuntimeConfigFactory.globalRuntimeConf().setValue("yb.releases.use_redesign", "false"); settableRuntimeConfigFactory .globalRuntimeConf() - .setValue("yb.universe.consistency_check_enabled", "true"); + .setValue("yb.universe.consistency_check.enabled", "true"); Pair ipRange = getIpRange(); localNodeManager.setIpRangeStart(ipRange.getFirst()); localNodeManager.setIpRangeEnd(ipRange.getSecond()); @@ -455,6 +456,11 @@ public void setUp() { ReleaseManager.YBC_CONFIG_TYPE.name(), getMetadataJson("ybc-" + YBC_VERSION, true), "release"); + ObjectNode ywMetadata = Json.newObject(); + ywMetadata.put("yugaware_uuid", UUID.randomUUID().toString()); + ywMetadata.put("version", ybVersion); + YugawareProperty.addConfigProperty( + ConfigHelper.ConfigType.YugawareMetadata.name(), ywMetadata, "Yugaware Metadata"); customer = ModelFactory.testCustomer(); user = ModelFactory.testUser(customer); diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/UpdateConsistencyLocalTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/UpdateConsistencyLocalTest.java index ed2fce242cfe..1caf8360bc01 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/UpdateConsistencyLocalTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/UpdateConsistencyLocalTest.java @@ -2,7 +2,7 @@ package com.yugabyte.yw.commissioner.tasks.local; -import static com.yugabyte.yw.common.Util.CONSISTENCY_CHECK; +import static com.yugabyte.yw.common.Util.CONSISTENCY_CHECK_TABLE_NAME; import static com.yugabyte.yw.common.Util.SYSTEM_PLATFORM_DB; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; @@ -40,7 +40,7 @@ public void testUpdateStale() throws InterruptedException { details, universe, SYSTEM_PLATFORM_DB, - String.format("update %s set seq_num = 10", CONSISTENCY_CHECK), + String.format("update %s set seq_num = 10", CONSISTENCY_CHECK_TABLE_NAME), 10); assertTrue(ysqlResponse.isSuccess()); UniverseDefinitionTaskParams.Cluster cluster = @@ -59,6 +59,6 @@ public void testUpdateStale() throws InterruptedException { TaskInfo taskInfo = waitForTask(taskID, universe); assertEquals(TaskInfo.State.Failure, taskInfo.getTaskState()); String error = getAllErrorsStr(taskInfo); - assertThat(error, containsString("stale metadata")); + assertThat(error, containsString("stale universe metadata")); } } diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/CertsRotateTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/CertsRotateTest.java index 9096eb44077d..75539344677c 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/CertsRotateTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/CertsRotateTest.java @@ -517,8 +517,8 @@ public void testCertsRotateNonRollingUpgrade( subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); int position = 0; - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); // RootCA update task int expectedPosition = 16; if (rotateRootCA) { @@ -626,8 +626,8 @@ public void testCertsRotateRollingUpgrade( int expectedPosition = 81; int expectedNumberOfInvocations = 21; assertTaskType(subTasksByPosition.get(position++), TaskType.CheckNodesAreSafeToTakeDown); - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); if (rotateRootCA) { expectedPosition += 150; expectedNumberOfInvocations += 30; @@ -739,8 +739,8 @@ public void testCertsRotateSelfSignedServerCert( if (isRolling) { assertTaskType(subTasksByPosition.get(position++), TaskType.CheckNodesAreSafeToTakeDown); } - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); // RootCA update task int expectedPosition = isRolling ? 81 : 17; // Cert update tasks diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/FinalizeUpgradeTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/FinalizeUpgradeTest.java index 84addefafd80..d4f715690bff 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/FinalizeUpgradeTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/FinalizeUpgradeTest.java @@ -80,8 +80,8 @@ public void testFinalizeWithUpgradeSystemCatalog() throws Exception { subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); assertEquals(7, subTasks.size()); int position = 0; - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateUniverseState); assertTaskType(subTasksByPosition.get(position++), TaskType.RunYsqlUpgrade); assertTaskType(subTasksByPosition.get(position++), TaskType.PromoteAutoFlags); @@ -108,8 +108,8 @@ public void testFinalizeWithNoSystemCatalog() { subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); assertEquals(6, subTasks.size()); int position = 0; - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateUniverseState); assertTaskType(subTasksByPosition.get(position++), TaskType.PromoteAutoFlags); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateUniverseState); diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsKubernetesUpgradeTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsKubernetesUpgradeTest.java index aa3b8b0dd491..2014688cecca 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsKubernetesUpgradeTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsKubernetesUpgradeTest.java @@ -38,8 +38,8 @@ public class GFlagsKubernetesUpgradeTest extends KubernetesUpgradeTaskTest { private static final List UPGRADE_TASK_SEQUENCE = ImmutableList.of( TaskType.CheckNodesAreSafeToTakeDown, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.KubernetesCommandExecutor, TaskType.CheckNodesAreSafeToTakeDown, TaskType.KubernetesCommandExecutor, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackKubernetesUpgradeTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackKubernetesUpgradeTest.java index daefc9dbbba4..8f1c7f04d510 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackKubernetesUpgradeTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackKubernetesUpgradeTest.java @@ -46,8 +46,8 @@ public class RollbackKubernetesUpgradeTest extends KubernetesUpgradeTaskTest { private static final List UPGRADE_TASK_SEQUENCE = ImmutableList.of( TaskType.CheckNodesAreSafeToTakeDown, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.UpdateUniverseState, TaskType.RollbackAutoFlags, TaskType.KubernetesCommandExecutor, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackUpgradeTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackUpgradeTest.java index 106921197de7..d72e6bfd1c3a 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackUpgradeTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/RollbackUpgradeTest.java @@ -320,8 +320,8 @@ public void testRollbackUpgradeInRollingManner() { int position = 0; assertTaskType(subTasksByPosition.get(position++), TaskType.CheckNodesAreSafeToTakeDown); - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateUniverseState); assertTaskType(subTasksByPosition.get(position++), TaskType.RollbackAutoFlags); @@ -362,8 +362,8 @@ public void testRollbackUpgradeInNonRollingManner() { subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); int position = 0; - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateUniverseState); assertTaskType(subTasksByPosition.get(position++), TaskType.RollbackAutoFlags); @@ -406,8 +406,8 @@ public void testRollbackPartialUpgrade() { int position = 0; assertTaskType(subTasksByPosition.get(position++), TaskType.CheckNodesAreSafeToTakeDown); - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateUniverseState); assertTaskType(subTasksByPosition.get(position++), TaskType.RollbackAutoFlags); List downloadTasks = subTasksByPosition.get(position++); diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareKubernetesUpgradeTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareKubernetesUpgradeTest.java index 360807e35f4c..b9bd5ca29936 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareKubernetesUpgradeTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareKubernetesUpgradeTest.java @@ -51,8 +51,8 @@ public class SoftwareKubernetesUpgradeTest extends KubernetesUpgradeTaskTest { ImmutableList.of( TaskType.CheckNodesAreSafeToTakeDown, TaskType.CheckUpgrade, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.KubernetesCommandExecutor, TaskType.CheckNodesAreSafeToTakeDown, TaskType.KubernetesCommandExecutor, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareKubernetesUpgradeYBTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareKubernetesUpgradeYBTest.java index d952eaa85770..46d32e627596 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareKubernetesUpgradeYBTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareKubernetesUpgradeYBTest.java @@ -56,8 +56,8 @@ public class SoftwareKubernetesUpgradeYBTest extends KubernetesUpgradeTaskTest { ImmutableList.of( TaskType.CheckNodesAreSafeToTakeDown, TaskType.CheckUpgrade, - TaskType.FreezeUniverse, TaskType.UpdateConsistencyCheck, + TaskType.FreezeUniverse, TaskType.UpdateUniverseState, TaskType.KubernetesCommandExecutor, TaskType.CheckNodesAreSafeToTakeDown, diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/TlsToggleTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/TlsToggleTest.java index 07bfc81774f3..08427fc07e65 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/TlsToggleTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/TlsToggleTest.java @@ -500,8 +500,8 @@ public void testTlsNonRollingUpgrade( subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); int position = 0; - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); if (taskParams.enableNodeToNodeEncrypt || taskParams.enableClientToNodeEncrypt) { // Cert update tasks will be non rolling List certUpdateTasks = subTasksByPosition.get(position++); @@ -656,8 +656,8 @@ public void testTlsRollingUpgrade( subTasks.stream().collect(Collectors.groupingBy(TaskInfo::getPosition)); int position = 0; - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); if (taskParams.enableNodeToNodeEncrypt || taskParams.enableClientToNodeEncrypt) { // Cert update tasks will be non rolling List certUpdateTasks = subTasksByPosition.get(position++); diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/VMImageUpgradeTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/VMImageUpgradeTest.java index 9bfcacf99eb4..d48d16ae12d0 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/VMImageUpgradeTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/VMImageUpgradeTest.java @@ -211,8 +211,8 @@ public void testVMImageUpgrade() { int position = 0; assertTaskType(subTasksByPosition.get(position++), TaskType.CheckNodesAreSafeToTakeDown); - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); List createRootVolumeTasks = subTasksByPosition.get(position++); assertTaskType(createRootVolumeTasks, TaskType.CreateRootVolumes); assertEquals(expectedRootVolumeCreationTasks, createRootVolumeTasks.size()); @@ -400,8 +400,8 @@ public void testVMImageUpgradeWithImageBundle() { int position = 0; assertTaskType(subTasksByPosition.get(position++), TaskType.CheckNodesAreSafeToTakeDown); - assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); assertTaskType(subTasksByPosition.get(position++), TaskType.UpdateConsistencyCheck); + assertTaskType(subTasksByPosition.get(position++), TaskType.FreezeUniverse); List createRootVolumeTasks = subTasksByPosition.get(position++); assertTaskType(createRootVolumeTasks, TaskType.CreateRootVolumes); assertEquals(expectedRootVolumeCreationTasks, createRootVolumeTasks.size());