Skip to content

Commit

Permalink
[BACKPORT pg15-cherrypicks] all: Bulk port from master - 100
Browse files Browse the repository at this point in the history
Summary:
 32572f7 [#23629]yugabyted: 2nd node doesn't join if webserver_port is changed
 b2d16eb [PLAT-15182]: Reset config working incorrectly for kubernetes universe
 dd60793 [#23843] YSQL: Do not retry schema mismatch errors that occur as part of a batched execution.
 22b6a47 [#23661] YSQL: Change the default wal_level to logical
 5eb925b [#23410] YSQL: Fix CREATE TABLE with pg_default tablespace in colocated databases
 2c0c0eb [#23913] build: fix ysql conn mgr ASAN issue
 b98984a Disable failing DR local provider tests
 48e3c40 Update YBA HA pre-reqs (#23865)
 9baa0be [docs] Release notes for 2.20.2.4 (#23880)
 a2a74d0 [Docs][Voyager] Added a known-issue in PG and some missing entries from PG compatibility doc (#23906)
 0a738ee [#23336] YSQL: Fix tablespace id update in pg_yb_tablegroup after ALTER TABLE CASCADE
 a0e2c4a [PLAT-15156] Pass pg_max_mem_mb during configure phase
 Excluded: 7f59bd8 [#22519] YSQL: Substitute YBSetRowLockPolicy function with YBGetDocDBWaitPolicy

MERGE:

- yb_pg_publication.out:
  - CREATE PUBLICATION testpub_foralltables FOR ALL TABLES;: YB master
    22b6a47 causes WARNING about
    wal_level not being "logical" to go away.  This warning is only
    present in YB pg15 because YB pg15
    5c5a2a0 was not able to suppress
    this WARNING for this one case (see that commit message for more
    details).
- yb_publication.out:
  - CREATE PUBLICATION testpub FOR ALL TABLES;: (same)
- yb_pg_pg_stat_statements.out
  - Check WAL is generated for the above statements: "DROP TABLE
    pgss_test" row's wal column values change.  Likely related to YB
    master 22b6a47.  Not sure how
    exactly it relates, but since it makes the test closer to the
    original pg_stat_statements.out, take it without question.
- yb_colocated_tables_with_tablespaces.{sql,out}:
  - Explicitly specifying pg_default tablespace must not create a new
    tablegroup: YB master 5eb925b adds
    these queries with SELECT * from system table, and upstream PG
    changes that to include the oid column.  We don't want to show the
    oid column since that can easily change, so replace * with explicit
    column names.

Test Plan: Jenkins: rebase: pg15-cherrypicks

Reviewers: tfoucher, jenkins-bot, stiwary, utkarsh.munjal

Subscribers: yql

Differential Revision: https://phorge.dev.yugabyte.com/D38035
  • Loading branch information
yugabyte-ci authored and jaki committed Sep 16, 2024
1 parent 9ef4d9e commit 1a5c22b
Show file tree
Hide file tree
Showing 32 changed files with 320 additions and 74 deletions.
24 changes: 14 additions & 10 deletions bin/yugabyted
Original file line number Diff line number Diff line change
Expand Up @@ -6243,9 +6243,8 @@ class ControlScript(object):

# Get the current master leader known to a tserver using api/v1/masters endpoint
# of the tserver.
def get_current_master_leader_from_api(self, tserverIP, timeout=60):
tserver_addr = "{}:{}".format(tserverIP,
self.configs.saved_data.get("tserver_webserver_port"))
def get_current_master_leader_from_api(self, tserverIP, tserver_webserver_port, timeout=60):
tserver_addr = "{}:{}".format(tserverIP, tserver_webserver_port)
try:
tserverMastersAPI = "http://{}/api/v1/masters".format(tserver_addr)
Output.log("Trying to get masters information from {}".format(tserverMastersAPI) +
Expand Down Expand Up @@ -6961,6 +6960,13 @@ class ControlScript(object):
self.setup_env_init.set_ycql_keyspace(args.keyspace)

if args.parser == "start":
master_webserver_port = self.configs.saved_data.get("master_webserver_port")
if args.master_webserver_port is not None:
master_webserver_port = args.master_webserver_port

tserver_webserver_port = self.configs.saved_data.get("tserver_webserver_port")
if args.tserver_webserver_port is not None:
tserver_webserver_port = args.tserver_webserver_port
if args.read_replica and not args.join:
Output.print_out(Output.make_red("ERROR") + ": read_replica neds to be " +
"started with --join flag.")
Expand Down Expand Up @@ -7044,12 +7050,12 @@ class ControlScript(object):
if not cluster_member:
# Check if tserver webserver at join_IP is reachable or not
# Also get the leader master(used to get the info of all tservers)
master_leader = self.get_current_master_leader_from_api(args.join)
master_leader = self.get_current_master_leader_from_api(args.join,
tserver_webserver_port)
args.join = master_leader

# Get info on all tservers
master_leader_hostport = "{}:{}".format(master_leader,
self.configs.saved_data.get("master_webserver_port"))
master_leader_hostport = "{}:{}".format(master_leader, master_webserver_port)
tservers_info = dict(self.get_all_tserver_info(master_leader_hostport))

# Check if any existing node has the same IP as advertise address
Expand Down Expand Up @@ -7125,8 +7131,7 @@ class ControlScript(object):

if args.insecure:
if args.join and not cluster_member:
master_hostport = "{}:{}".format(args.join,
self.configs.saved_data.get("master_webserver_port"))
master_hostport = "{}:{}".format(args.join, master_webserver_port)
if self.is_leader_master_secure(master_hostport):
# Case Scenario: When a User starts the 1st node in secure mode and tries
# to start the second node in insecure mode
Expand All @@ -7136,8 +7141,7 @@ class ControlScript(object):
"secure and an insecure node.")
elif args.secure:
if args.join and not cluster_member:
master_hostport = "{}:{}".format(args.join,
self.configs.saved_data.get("master_webserver_port"))
master_hostport = "{}:{}".format(args.join, master_webserver_port)
if not self.is_leader_master_secure(master_hostport):
# Case Scenario: When the user starts the 1st node in insecure mode and
# tries to start the second node in secure mode.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ The following PostgreSQL features are not supported in YugabyteDB:
| Table inheritance | [5956](https://github.com/yugabyte/yugabyte-db/issues/5956)|
| Exclusion constraints | [3944](https://github.com/yugabyte/yugabyte-db/issues/3944)|
| Deferrable constraints | [1709](https://github.com/yugabyte/yugabyte-db/issues/1709)|
| Constraint Triggers|[4700](https://github.com/yugabyte/yugabyte-db/issues/4700)|
| GiST indexes | [1337](https://github.com/yugabyte/yugabyte-db/issues/1337)|
| Events (Listen/Notify) | [1872](https://github.com/yugabyte/yugabyte-db/issues/1872)|
| XML Functions | [1043](https://github.com/yugabyte/yugabyte-db/issues/1043)|
Expand All @@ -197,3 +198,7 @@ The following PostgreSQL features are not supported in YugabyteDB:
| transaction ids (xid) <br/> YugabyteDB uses [Hybrid logical clocks](../../../architecture/transactions/transactions-overview/#hybrid-logical-clocks) instead of transaction ids. | [15638](https://github.com/yugabyte/yugabyte-db/issues/15638)|
| DDL operations within transaction| [1404](https://github.com/yugabyte/yugabyte-db/issues/1404)|
| Some ALTER TABLE variants| [1124](https://github.com/yugabyte/yugabyte-db/issues/1124)|
| UNLOGGED table | [1129](https://github.com/yugabyte/yugabyte-db/issues/1129) |
| Indexes on complex datatypes such as INET, CITEXT, JSONB, ARRAYs, and so on.| [9698](https://github.com/yugabyte/yugabyte-db/issues/9698), [23829](https://github.com/yugabyte/yugabyte-db/issues/23829), [17017](https://github.com/yugabyte/yugabyte-db/issues/17017) |
| %TYPE syntax in Functions/Procedures/Triggers|[23619](https://github.com/yugabyte/yugabyte-db/issues/23619)|
| Storage parameters on indexes or constraints|[23467](https://github.com/yugabyte/yugabyte-db/issues/23467)|
20 changes: 20 additions & 0 deletions docs/content/preview/releases/yba-releases/v2.20.md
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,26 @@ This is a [YugabyteDB-only release](../../ybdb-releases/v2.20/#v2.20.3.1), with
* Prevents universe chain upgrade failure from 2.0 to 2.18 and 2.20 by ensuring the clock-script doesn't run during yb-process start command if it's not present or executable. PLAT-13444
* Prevents 'Edit Universe' modal from wrongly displaying master placement as 'Place Masters on the same nodes as T-Servers' for a dedicated universe, providing accurate universe creation details. PLAT-13445

## v2.20.2.4 - September 12, 2024 {#v2.20.2.4}

**Build:** `2.20.2.4-b3`

**Third-party licenses:** [YugabyteDB](https://downloads.yugabyte.com/releases/2.20.2.4/yugabytedb-2.20.2.4-b3-third-party-licenses.html), [YugabyteDB Anywhere](https://downloads.yugabyte.com/releases/2.20.2.4/yugabytedb-anywhere-2.20.2.4-b3-third-party-licenses.html)

### Download

<ul class="nav yb-pills">
<li>
<a href="https://downloads.yugabyte.com/releases/2.20.2.4/yba_installer_full-2.20.2.4-b3-linux-x86_64.tar.gz">
<span>Linux x86</span>
</a>
</li>
</ul>

### Bug fix

* Allows configuration of `wait_for_server_ready.timeout` to adjust the default 10-minute wait time to suit user requirements. PLAT-8632

## v2.20.2.3 - June 27, 2024 {#v2.20.2.3}

**Build:** `2.20.2.3-b2`
Expand Down
41 changes: 41 additions & 0 deletions docs/content/preview/releases/ybdb-releases/v2.20.md
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,47 @@ docker pull yugabytedb/yugabyte:2.20.3.1-b2
* Updates the condition for HT lease reporting to ensure accurate leaderless tablet detection in RF-1 setup, preventing false alarms. {{<issue 20919>}}
* Reduces disruptions by throttling the master process log messages related to "tablet server has a pending delete" into 20-second intervals. {{<issue 19331>}}

## v2.20.2.4 - September 12, 2024 {#v2.20.2.4}

**Build:** `2.20.2.4-b3`

**Third-party licenses:** [YugabyteDB](https://downloads.yugabyte.com/releases/2.20.2.4/yugabytedb-2.20.2.4-b3-third-party-licenses.html), [YugabyteDB Anywhere](https://downloads.yugabyte.com/releases/2.20.2.4/yugabytedb-anywhere-2.20.2.4-b3-third-party-licenses.html)

### Downloads

<ul class="nav yb-pills">
<li>
<a href="https://downloads.yugabyte.com/releases/2.20.2.4/yugabyte-2.20.2.4-b3-darwin-x86_64.tar.gz">
<i class="fa-brands fa-apple"></i>
<span>macOS</span>
</a>
</li>
<li>
<a href="https://downloads.yugabyte.com/releases/2.20.2.4/yugabyte-2.20.2.4-b3-linux-x86_64.tar.gz">
<i class="fa-brands fa-linux"></i>
<span>Linux x86</span>
</a>
</li>
<li>
<a href="https://downloads.yugabyte.com/releases/2.20.2.4/yugabyte-2.20.2.4-b3-el8-aarch64.tar.gz">
<i class="fa-brands fa-linux"></i>
<span>Linux ARM</span>
</a>
</li>
</ul>

### Docker

```sh
docker pull yugabytedb/yugabyte:2.20.2.4-b3
```

### Bug fix

#### DocDB

* Reduces the risk of deadlocks when making active read/write requests to a table right after a tablet-split. {{<issue 23747>}}

## v2.20.2.3 - June 27, 2024 {#v2.20.2.3}

**Build:** `2.20.2.3-b2`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ Before configuring a HA cluster for your YBA instances, ensure that you have the
- [Two or more YBA instances](../../install-yugabyte-platform/) to be used in the HA cluster.
- The YBA instances can connect to each other over the port where the YBA UI is reachable (443 by default).
- Communication is open in both directions over port 443 and 9090 on all YBA instances.
- The YBA instances were installed using the same installation method (YBA Installer, Replicated, or Helm (Kubernetes)).
- The YBA instances are configured to use the same path for the installation root.
- If you are using custom ports for Prometheus, all YBA instances are using the same custom port. (The default Prometheus port for YugabyteDB Anywhere is 9090.)
- All YBA instances are running the same version of YBA software. (The YBA instances in a HA cluster should always be upgraded at approximately the same time.)
- The YBA instances have the same login credentials.
Expand Down
75 changes: 38 additions & 37 deletions docs/content/preview/yugabyte-voyager/known-issues/postgresql.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ Review limitations and implement suggested workarounds to successfully migrate d
- [Adding primary key to a partitioned table results in an error](#adding-primary-key-to-a-partitioned-table-results-in-an-error)
- [Index creation on partitions fail for some YugabyteDB builds](#index-creation-on-partitions-fail-for-some-yugabytedb-builds)
- [Creation of certain views in the rule.sql file](#creation-of-certain-views-in-the-rule-sql-file)
- [Indexes on INET type are not supported](#indexes-on-inet-type-are-not-supported)
- [Create or alter conversion is not supported](#create-or-alter-conversion-is-not-supported)
- [GENERATED ALWAYS AS STORED type column is not supported](#generated-always-as-stored-type-column-is-not-supported)
- [Unsupported ALTER TABLE DDL variants in source schema](#unsupported-alter-table-ddl-variants-in-source-schema)
Expand All @@ -38,6 +37,7 @@ Review limitations and implement suggested workarounds to successfully migrate d
- [GIN indexes on multiple columns are not supported](#gin-indexes-on-multiple-columns-are-not-supported)
- [Policies on users in source require manual user creation](#policies-on-users-in-source-require-manual-user-creation)
- [VIEW WITH CHECK OPTION is not supported](#view-with-check-option-is-not-supported)
- [UNLOGGED table is not supported](#unlogged-table-is-not-supported)

### Adding primary key to a partitioned table results in an error

Expand Down Expand Up @@ -156,40 +156,6 @@ CREATE OR REPLACE VIEW public.v1 AS
GROUP BY foo.n1;
```

---

### Indexes on INET type are not supported

**GitHub**: [Issue #17017](https://github.com/yugabyte/yb-voyager/issues/17017)

**Description**: If there is an index on a column of the INET type, it errors out during import.

**Workaround**: Modify the column to a TEXT type.

**Example**

An example schema on the source database is as follows:

```sql
create table test( id int primary key, f1 inet);
create index test_index on test(f1);
```

The import schema error is as follows:

```sql
INDEXES_table.sql: CREATE INDEX test_index ON public.test USING btree (f1);
ERROR: INDEX on column of type 'INET' not yet supported (SQLSTATE 0A000)
```

Suggested workaround is to change the INET column to TEXT for the index creation to succeed as follows:

```sql
create table test( id int primary key, f1 text);
```

---

### Create or alter conversion is not supported

**GitHub**: [Issue #10866](https://github.com/yugabyte/yugabyte-db/issues/10866)
Expand Down Expand Up @@ -590,9 +556,9 @@ CREATE INDEX gist_idx ON public.ts_query_table USING gist (query);

### Indexes on some complex data types are not supported

**GitHub**: [Issue #9698](https://github.com/yugabyte/yugabyte-db/issues/9698), [Issue #23829](https://github.com/yugabyte/yugabyte-db/issues/23829)
**GitHub**: [Issue #9698](https://github.com/yugabyte/yugabyte-db/issues/9698), [Issue #23829](https://github.com/yugabyte/yugabyte-db/issues/23829), [Issue #17017](https://github.com/yugabyte/yugabyte-db/issues/17017)

**Description**: If you have indexes on some complex types such as TSQUERY, TSVECTOR, JSONB, UDTs, citext, and so on, those will error out in import schema phase with the following error:
**Description**: If you have indexes on some complex types such as TSQUERY, TSVECTOR, JSONB, ARRAYs, INET, UDTs, citext, and so on, those will error out in import schema phase with the following error:

```output
ERROR: INDEX on column of type '<TYPE_NAME>' not yet supported
Expand Down Expand Up @@ -911,3 +877,38 @@ CREATE TRIGGER trigger_modify_employee_12000
```

---


### UNLOGGED table is not supported

**GitHub**: [Issue #1129](https://github.com/yugabyte/yugabyte-db/issues/1129)

**Description**: If there are UNLOGGED tables in the source schema, they will error out during the import schema with the following error as it is not supported in target YugabyteDB.

```output
ERROR: UNLOGGED database object not supported yet
```

**Workaround**: Convert it to a LOGGED table.

**Example**

An example schema on the source database is as follows:

```sql
CREATE UNLOGGED TABLE tbl_unlogged (
id int,
val text
);
```

Suggested change to the schema is as follows:

```sql
CREATE TABLE tbl_unlogged (
id int,
val text
);
```

---
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ Before configuring a HA cluster for your YBA instances, ensure that you have the
- [Two or more YBA instances](../../install-yugabyte-platform/) to be used in the HA cluster.
- The YBA instances can connect to each other over the port where the YBA UI is reachable (443 by default).
- Communication is open in both directions over port 443 and 9090 on all YBA instances.
- The YBA instances were installed using the same installation method (YBA Installer, Replicated, or Helm (Kubernetes)).
- The YBA instances are configured to use the same path for the installation root.
- If you are using custom ports for Prometheus, all YBA instances are using the same custom port. (The default Prometheus port for YugabyteDB Anywhere is 9090.)
- All YBA instances are running the same version of YBA software. (The YBA instances in a HA cluster should always be upgraded at approximately the same time.)
- The YBA instances have the same login credentials.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ Before configuring a HA cluster for your YBA instances, ensure that you have the
- [Multiple YBA instances](../../install-yugabyte-platform/) to be used in the HA cluster.
- The YBA instances can connect to each other over the port where the YBA UI is reachable (typically 443).
- Communication is open in both directions over ports 9000 and 9090 on all YBA instances.
- The YBA instances were installed using the same installation method (YBA Installer, Replicated, or Helm (Kubernetes)).
- The YBA instances are configured to use the same path for the installation root.
- If you are using custom ports for Prometheus, all YBA instances are using the same custom port. The default Prometheus port for YugabyteDB Anywhere is 9090.
- All YBA instances are running the same version of YBA software. (The YBA instances in a HA cluster should always be upgraded at approximately the same time.)

Expand Down
36 changes: 36 additions & 0 deletions java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgBatch.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import static org.yb.AssertionWrappers.assertGreaterThan;
import static org.yb.AssertionWrappers.assertNotEquals;
import static org.yb.AssertionWrappers.assertTrue;
import static org.yb.AssertionWrappers.fail;

import com.yugabyte.util.PSQLException;
import java.sql.BatchUpdateException;
Expand Down Expand Up @@ -49,6 +50,8 @@ protected Map<String, String> getTServerFlags() {
flagMap.put("yb_enable_read_committed_isolation", "true");
// TODO: Remove this override when wait queues are enabled by default.
flagMap.put("enable_wait_queues", "true");
// Easier debugging.
flagMap.put("ysql_log_statement", "all");
return flagMap;
}

Expand Down Expand Up @@ -145,6 +148,39 @@ public void testTransparentRestart() throws Throwable {
testTransparentRestartHelper(5, SR);
}

@Test
public void testSchemaMismatchRetry() throws Throwable {
setUpTable(2, RR);
try (Connection c1 = getConnectionBuilder().connect();
Connection c2 = getConnectionBuilder().connect();
Statement s1 = c1.createStatement();
Statement s2 = c2.createStatement()) {
// Run UPDATE statement for the sole purpose of a caching catalog version.
s1.execute("UPDATE t SET v=2 WHERE k=0");
// Add more than one statement to the batch to ensure that
// YB treats this as batched execution mode.
for (int i = 1; i <= 2; i++) {
s1.addBatch(String.format("UPDATE t SET v=2 WHERE k=%d", i));
}
// Causes a schema version mismatch error on the next UPDATE statement.
// Execute ALTER in a different session c2 so as not to invalidate
// the catalog cache of c1 until the next heartbeat with the master.
s2.execute("ALTER TABLE t ALTER COLUMN v SET NOT NULL");
try {
// This uses the cached catalog version but the schema is changed
// by the ALTER TABLE statement above. This should cause a schema
// mismatch error. The schema mismatch error is not retried internally
// in batched execution mode.
s1.executeBatch();
// Should not reach here since we do not support retries in batched
// execution mode for schema mismatch errors.
fail("Internal retries are not supported in batched execution mode");
} catch (BatchUpdateException e) {
LOG.info(e.toString());
}
}
}

@Test
public void testTransparentRestartFirstStatement() throws Throwable {
// Params: conflicting statement, isolation level
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2890,4 +2890,12 @@ public void testDynamicTableWithReplicaIdentityChangeWithPgoutput() throws Excep
+ " plugin pgoutput. Consider using output plugin yboutput instead."));
}
}

@Test
public void testDefaultWalLevel() throws Exception {
try (Statement stmt = connection.createStatement()) {
Row row = getSingleRow(stmt, "SHOW wal_level");
assertEquals("logical", row.getString(0));
}
}
}
5 changes: 5 additions & 0 deletions managed/devops/opscli/ybops/cloud/common/method.py
Original file line number Diff line number Diff line change
Expand Up @@ -1231,6 +1231,8 @@ def prepare(self):
self.parser.add_argument('--gcs_remote_download', action="store_true")
self.parser.add_argument('--gcs_credentials_json')
self.parser.add_argument('--http_remote_download', action="store_true")
self.parser.add_argument("--pg_max_mem_mb", type=int, default=0,
help="Max memory for postgress process.")
self.parser.add_argument('--http_package_checksum', default='')
self.parser.add_argument('--install_third_party_packages',
action="store_true",
Expand Down Expand Up @@ -1309,6 +1311,9 @@ def callback(self, args):

if args.yb_process_type:
self.extra_vars["yb_process_type"] = args.yb_process_type.lower()

if args.pg_max_mem_mb:
self.extra_vars.update({"pg_max_mem_mb": args.pg_max_mem_mb})
else:
raise YBOpsRuntimeError("Supported types for this command are only: {}".format(
self.supported_types))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1458,6 +1458,7 @@ public SubTaskGroup createConfigureServerTasks(
universe.getUniverseDetails().clusters);
}
}
params.cgroupSize = getCGroupSize(node);
// Create the Ansible task to get the server info.
AnsibleConfigureServers task = createTask(AnsibleConfigureServers.class);
task.initialize(params);
Expand Down
Loading

0 comments on commit 1a5c22b

Please sign in to comment.