diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt index b3b6587d0d64..77a98edae313 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt @@ -4,11 +4,14 @@ package io.airbyte.cdk.integrations import io.airbyte.cdk.integrations.base.Integration +import io.airbyte.commons.features.EnvVariableFeatureFlags +import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.json.Jsons import io.airbyte.commons.resources.MoreResources import io.airbyte.protocol.models.v0.ConnectorSpecification abstract class BaseConnector : Integration { + override var featureFlags: FeatureFlags = EnvVariableFeatureFlags() /** * By convention the spec is stored as a resource for java connectors. That resource is called * spec.json. diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/Integration.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/Integration.kt index 0ed88f7207aa..e027f7c4eeff 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/Integration.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/Integration.kt @@ -4,6 +4,8 @@ package io.airbyte.cdk.integrations.base import com.fasterxml.jackson.databind.JsonNode +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner +import io.airbyte.commons.features.FeatureFlags import io.airbyte.protocol.models.v0.AirbyteConnectionStatus import io.airbyte.protocol.models.v0.ConnectorSpecification @@ -29,4 +31,12 @@ interface Integration { * - any exception. */ @Throws(Exception::class) fun check(config: JsonNode): AirbyteConnectionStatus? + + abstract var featureFlags: FeatureFlags + fun isCloudDeployment(): Boolean { + return AdaptiveSourceRunner.CLOUD_MODE.equals( + featureFlags.deploymentMode(), + ignoreCase = true + ) + } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingDestination.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingDestination.kt index e06ae860e189..dc9b7d336ba1 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingDestination.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingDestination.kt @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer import io.airbyte.cdk.integrations.base.Destination import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer +import io.airbyte.commons.features.FeatureFlags import io.airbyte.protocol.models.v0.AirbyteConnectionStatus import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog @@ -15,6 +16,7 @@ import java.util.function.Consumer abstract class SpecModifyingDestination(private val destination: Destination) : Destination { override val isV2Destination: Boolean = destination.isV2Destination + override var featureFlags: FeatureFlags = destination.featureFlags @Throws(Exception::class) abstract fun modifySpec(originalSpec: ConnectorSpecification): ConnectorSpecification diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingSource.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingSource.kt index f9c81b8c21a1..c883813e292e 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingSource.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingSource.kt @@ -5,6 +5,7 @@ package io.airbyte.cdk.integrations.base.spec_modification import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.integrations.base.Source +import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.util.AutoCloseableIterator import io.airbyte.protocol.models.v0.* @@ -14,6 +15,7 @@ import io.airbyte.protocol.models.v0.* * want to allow users to send data unencrypted. */ abstract class SpecModifyingSource(private val source: Source) : Source { + override var featureFlags: FeatureFlags = source.featureFlags @Throws(Exception::class) abstract fun modifySpec(originalSpec: ConnectorSpecification): ConnectorSpecification diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.kt index a346098f6852..2572963e6537 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.kt @@ -10,6 +10,7 @@ import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility import io.airbyte.cdk.integrations.base.Destination import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer import io.airbyte.commons.concurrency.VoidCallable +import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.json.Jsons import io.airbyte.commons.resources.MoreResources import io.airbyte.protocol.models.v0.AirbyteConnectionStatus @@ -25,17 +26,19 @@ private val LOGGER = KotlinLogging.logger {} * Decorates a Destination with an SSH Tunnel using the standard configuration that Airbyte uses for * configuring SSH. */ -class SshWrappedDestination : Destination { +open class SshWrappedDestination : Destination { private val delegate: Destination private val hostKey: List? private val portKey: List? private val endPointKey: String? + final override var featureFlags: FeatureFlags constructor(delegate: Destination, hostKey: List, portKey: List) { this.delegate = delegate this.hostKey = hostKey this.portKey = portKey this.endPointKey = null + this.featureFlags = delegate.featureFlags } constructor(delegate: Destination, endPointKey: String) { @@ -43,6 +46,7 @@ class SshWrappedDestination : Destination { this.endPointKey = endPointKey this.portKey = null this.hostKey = null + this.featureFlags = delegate.featureFlags } @Throws(Exception::class) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/ssh/SshWrappedSource.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/ssh/SshWrappedSource.kt index 30dcd2b36095..062f15fffb17 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/ssh/SshWrappedSource.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/base/ssh/SshWrappedSource.kt @@ -6,6 +6,7 @@ package io.airbyte.cdk.integrations.base.ssh import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility import io.airbyte.cdk.integrations.base.Source +import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.util.AutoCloseableIterator import io.airbyte.commons.util.AutoCloseableIterators import io.airbyte.protocol.models.v0.* @@ -19,12 +20,14 @@ class SshWrappedSource : Source { private val hostKey: List private val portKey: List private val sshGroup: Optional + final override var featureFlags: FeatureFlags constructor(delegate: Source, hostKey: List, portKey: List) { this.delegate = delegate this.hostKey = hostKey this.portKey = portKey this.sshGroup = Optional.empty() + this.featureFlags = delegate.featureFlags } constructor(delegate: Source, hostKey: List, portKey: List, sshGroup: String) { @@ -32,6 +35,7 @@ class SshWrappedSource : Source { this.hostKey = hostKey this.portKey = portKey this.sshGroup = Optional.of(sshGroup) + this.featureFlags = delegate.featureFlags } @Throws(Exception::class) @@ -53,6 +57,10 @@ class SshWrappedSource : Source { } } + override fun isCloudDeployment(): Boolean { + return delegate.isCloudDeployment() + } + @Throws(Exception::class) override fun discover(config: JsonNode): AirbyteCatalog { return SshTunnel.sshWrap(config, hostKey, portKey) { c: JsonNode -> delegate.discover(c) } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt index fd5f9e967692..b9606b01c8d0 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt @@ -16,6 +16,7 @@ import io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.Ar import io.airbyte.cdk.integrations.standardtest.destination.comparator.BasicTestDataComparator import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator import io.airbyte.commons.features.EnvVariableFeatureFlags +import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.jackson.MoreMappers import io.airbyte.commons.json.Jsons import io.airbyte.commons.lang.Exceptions @@ -87,6 +88,8 @@ abstract class DestinationAcceptanceTest { protected var localRoot: Path? = null open protected var _testDataComparator: TestDataComparator = getTestDataComparator() + protected open var featureFlags: FeatureFlags = EnvVariableFeatureFlags() + protected open fun getTestDataComparator(): TestDataComparator { return BasicTestDataComparator { @Suppress("deprecated") this.resolveIdentifier(it) } } @@ -1499,7 +1502,7 @@ abstract class DestinationAcceptanceTest { null, null, false, - EnvVariableFeatureFlags() + featureFlags ) ) .run(JobGetSpecConfig().withDockerImage(imageName), jobRoot) @@ -1519,7 +1522,7 @@ abstract class DestinationAcceptanceTest { null, null, false, - EnvVariableFeatureFlags() + featureFlags ), mConnectorConfigUpdater ) @@ -1541,7 +1544,7 @@ abstract class DestinationAcceptanceTest { null, null, false, - EnvVariableFeatureFlags() + featureFlags ), mConnectorConfigUpdater ) @@ -1569,7 +1572,7 @@ abstract class DestinationAcceptanceTest { null, null, false, - EnvVariableFeatureFlags() + featureFlags ) ) } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt index 1e67982a5e94..6372c1be6ded 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt @@ -22,8 +22,6 @@ import io.airbyte.cdk.integrations.util.ApmTraceUtils.addExceptionToTrace import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil import io.airbyte.commons.exceptions.ConfigErrorException import io.airbyte.commons.exceptions.ConnectionErrorException -import io.airbyte.commons.features.EnvVariableFeatureFlags -import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.functional.CheckedConsumer import io.airbyte.commons.lang.Exceptions import io.airbyte.commons.stream.AirbyteStreamUtils @@ -49,8 +47,6 @@ private val LOGGER = KotlinLogging.logger {} abstract class AbstractDbSource protected constructor(driverClassName: String) : JdbcConnector(driverClassName), Source, AutoCloseable { - // TODO: Remove when the flag is not use anymore - var featureFlags: FeatureFlags = EnvVariableFeatureFlags() @Trace(operationName = CHECK_TRACE_OPERATION_NAME) @Throws(Exception::class) diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.kt b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.kt index afd75dde9758..0243f80194ba 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/kotlin/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.kt @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import com.google.common.collect.ImmutableMap import io.airbyte.commons.features.EnvVariableFeatureFlags +import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.json.Jsons import io.airbyte.commons.lang.Exceptions import io.airbyte.commons.resources.MoreResources @@ -65,6 +66,7 @@ abstract class BaseTypingDedupingTest { protected var streamNamespace: String? = null protected var streamName: String = "dummy" private var streamsToTearDown: MutableList? = null + protected open var featureFlags: FeatureFlags = EnvVariableFeatureFlags() protected abstract val imageName: String /** @return the docker image to run, e.g. `"airbyte/destination-bigquery:dev"`. */ @@ -1206,7 +1208,7 @@ abstract class BaseTypingDedupingTest { null, null, false, - EnvVariableFeatureFlags() + featureFlags ) ) diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle deleted file mode 100644 index ee5786d11d91..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle +++ /dev/null @@ -1,20 +0,0 @@ -plugins { - id 'airbyte-java-connector' -} - -airbyteJavaConnector { - cdkVersionRequired = '0.33.2' - features = ['db-destinations', 'typing-deduping', 'datastore-postgres'] - useLocalCdk = true -} - -application { - mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestinationStrictEncrypt' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-integrations:connectors:destination-postgres') - - integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:destination-postgres')) -} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties deleted file mode 100644 index 23da4989675e..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties +++ /dev/null @@ -1,3 +0,0 @@ -# our testcontainer has issues with too much concurrency. -# 4 threads seems to be the sweet spot. -testExecutionConcurrency=4 diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/icon.svg b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/icon.svg deleted file mode 100644 index 0c88b0ec1aad..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml deleted file mode 100644 index 5ff14bc82031..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml +++ /dev/null @@ -1,34 +0,0 @@ -data: - connectorSubtype: database - connectorType: destination - definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 2.1.0 - dockerRepository: airbyte/destination-postgres-strict-encrypt - documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres - githubIssueLabel: destination-postgres - icon: postgresql.svg - license: ELv2 - name: Postgres - registries: - cloud: - enabled: false - oss: - enabled: false - releases: - breakingChanges: - 2.0.0: - message: > - This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. - To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). - These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). - Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. For more controlled upgrade [see instructions](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#upgrading-connections-one-by-one-with-dual-writing). - upgradeDeadline: "2024-05-31" - releaseStage: generally_available - supportLevel: certified - supportsDbt: true - tags: - - language:java - connectorTestSuitesOptions: - - suite: unitTests - - suite: integrationTests -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.kt b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.kt deleted file mode 100644 index 54bbcc5dcd31..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.kt +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ -package io.airbyte.integrations.destination.postgres - -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.node.ObjectNode -import io.airbyte.cdk.db.jdbc.JdbcUtils -import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler -import io.airbyte.cdk.integrations.base.Destination -import io.airbyte.cdk.integrations.base.IntegrationRunner -import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination -import io.airbyte.commons.json.Jsons -import io.airbyte.integrations.destination.postgres.PostgresDestination.Companion.sshWrappedDestination -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus -import io.airbyte.protocol.models.v0.ConnectorSpecification -import org.postgresql.util.PSQLException -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -class PostgresDestinationStrictEncrypt : - SpecModifyingDestination(sshWrappedDestination()), Destination { - override fun modifySpec(originalSpec: ConnectorSpecification): ConnectorSpecification { - val spec: ConnectorSpecification = Jsons.clone(originalSpec) - (spec.connectionSpecification[PROPERTIES] as ObjectNode).remove(JdbcUtils.SSL_KEY) - return spec - } - - @Throws(Exception::class) - override fun check(config: JsonNode): AirbyteConnectionStatus? { - if ( - (config.has(TUNNEL_METHOD) && config[TUNNEL_METHOD].has(TUNNEL_METHOD)) && - config[TUNNEL_METHOD][TUNNEL_METHOD].asText() == NO_TUNNEL - ) { - // If no SSH tunnel - if (config.has(SSL_MODE) && config[SSL_MODE].has(MODE)) { - if ( - setOf(SSL_MODE_DISABLE, SSL_MODE_ALLOW, SSL_MODE_PREFER) - .contains(config[SSL_MODE][MODE].asText()) - ) { - // Fail in case SSL mode is disable, allow or prefer - return AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.FAILED) - .withMessage( - "Unsecured connection not allowed. If no SSH Tunnel set up, please use one of the following SSL modes: require, verify-ca, verify-full" - ) - } - } - } - return super.check(config) - } - - override val isV2Destination: Boolean - get() = true - - companion object { - private val LOGGER: Logger = - LoggerFactory.getLogger(PostgresDestinationStrictEncrypt::class.java) - private const val PROPERTIES = "properties" - const val TUNNEL_METHOD: String = "tunnel_method" - const val NO_TUNNEL: String = "NO_TUNNEL" - const val SSL_MODE: String = "ssl_mode" - const val MODE: String = "mode" - const val SSL_MODE_ALLOW: String = "allow" - const val SSL_MODE_PREFER: String = "prefer" - const val SSL_MODE_DISABLE: String = "disable" - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - AirbyteExceptionHandler.addThrowableForDeinterpolation(PSQLException::class.java) - val destination: Destination = PostgresDestinationStrictEncrypt() - LOGGER.info("starting destination: {}", PostgresDestinationStrictEncrypt::class.java) - IntegrationRunner(destination).run(args) - LOGGER.info("completed destination: {}", PostgresDestinationStrictEncrypt::class.java) - } - } -} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl deleted file mode 100644 index 9b6deaec5a83..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl deleted file mode 100644 index f6aaee67abb2..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl deleted file mode 100644 index 183223e4c4cd..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -// Keep the Alice record with more recent updated_at -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}],"sync_id":42}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?", "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl deleted file mode 100644 index a63c0d304081..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00.000000Z", "name": "Someone completely different"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl deleted file mode 100644 index 1a384f7ea1e8..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}, "_airbyte_generation_id": 43} -// Invalid columns are nulled out (i.e. SQL null, not JSON null) -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}],"sync_id":42}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?", "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl deleted file mode 100644 index b8d7ac7a1403..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} -// Invalid data is still allowed in the raw table. -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes": [{"field": "address", "change": "NULLED", "reason": "SOURCE_RETRIEVAL_ERROR"}],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl deleted file mode 100644 index 6e13b7f7c07f..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_messages_before_meta.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_messages_before_meta.jsonl deleted file mode 100644 index 5a40a7cd1574..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync1_messages_before_meta.jsonl +++ /dev/null @@ -1,14 +0,0 @@ -// emitted_at:1000 is equal to 1970-01-01 00:00:01Z, which is what you'll see in the expected records. -// This obviously makes no sense in relation to updated_at being in the year 2000, but that's OK -// because (from destinations POV) updated_at has no relation to emitted_at. -{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}} -// Emit a second record for id=(1,200) with a different updated_at. This generally doesn't happen -// in full refresh syncs - but if T+D is implemented correctly, it shouldn't matter -// (i.e. both records should be written to the final table). -{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}} -// Emit a record with no _ab_cdc_deleted_at field. CDC sources typically emit an explicit null, but we should handle both cases. -{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}} -// Emit a record with an invalid age & address nulled at source. -{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "meta": {"changes": [{"field": "address", "change": "NULLED", "reason": "SOURCE_RETRIEVAL_ERROR"}]}}} -// Emit a record with interesting characters in one of the values. -{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl deleted file mode 100644 index e9ead64b3e12..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -// Charlie wasn't reemitted with updated_at, so it still has a null cursor -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 2, "id2": 200, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl deleted file mode 100644 index d559f696c8a8..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl +++ /dev/null @@ -1,7 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl deleted file mode 100644 index e2ee9c8aa0c6..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl +++ /dev/null @@ -1,9 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}],"sync_id":42}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?", "_airbyte_generation_id": 43} - -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"},"_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z", "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_mixed_meta_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_mixed_meta_final.jsonl deleted file mode 100644 index 4ba91c1f8586..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_mixed_meta_final.jsonl +++ /dev/null @@ -1,10 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} - -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[],"sync_id":42}, "_airbyte_generation_id": 42, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[],"sync_id":42}, "_airbyte_generation_id": 42, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[],"sync_id":42}, "_airbyte_generation_id": 42, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}],"sync_id":42}, "_airbyte_generation_id": 42, "id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "address": {"city": "San Francisco", "state": "CA"}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_with_new_gen_id_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_with_new_gen_id_final.jsonl deleted file mode 100644 index d14394d3eccb..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_with_new_gen_id_final.jsonl +++ /dev/null @@ -1,9 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} - -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl deleted file mode 100644 index f71e5d40e4f6..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[], "sync_id":42}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[], "sync_id":42}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[], "sync_id":42}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z", "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl deleted file mode 100644 index 55d0a3af5582..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes": []}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes": []}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_final.jsonl deleted file mode 100644 index bc2849224699..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_final.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_raw.jsonl deleted file mode 100644 index f2e286786eaa..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_with_new_gen_id_raw.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl deleted file mode 100644 index e83893d70e48..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -// Delete Bob, keep Charlie -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl deleted file mode 100644 index 7c8417d72351..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[],"sync_id":42}, "_airbyte_generation_id": 43, "id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00.000000Z", "name": "Someone completely different v2"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_meta_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_meta_final.jsonl deleted file mode 100644 index 2514934ab369..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_meta_final.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[],"sync_id":13}, "_airbyte_generation_id": 42, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -// Delete Bob, updated Charlie -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}],"sync_id":13}, "_airbyte_generation_id": 42, "id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "address": {"city": "San Francisco", "state": "CA"}} -// Record before meta in raw table will continue to have errors. -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_mixed_meta_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_mixed_meta_raw.jsonl deleted file mode 100644 index f1ac9c0d5b8e..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_mixed_meta_raw.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -// We keep the records from the first sync -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}} -// And append the records from the second sync -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 42} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 42} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 42} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00Z", "name":"Charlie", "age": 42, "registration_date": "2023-12-23", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta":{"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}],"sync_id":42}, "_airbyte_generation_id": 42} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl deleted file mode 100644 index d51b7e3c8aea..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl +++ /dev/null @@ -1,10 +0,0 @@ -// We keep the records from the first sync -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes":[{"field":"address","change":"NULLED","reason":"SOURCE_RETRIEVAL_ERROR"}],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} -// And append the records from the second sync -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes": [],"sync_id":42}, "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl deleted file mode 100644 index 31901e53b9e8..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_with_new_gen_id_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_with_new_gen_id_raw.jsonl deleted file mode 100644 index 023a15b98ac9..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_expectedrecords_with_new_gen_id_raw.jsonl +++ /dev/null @@ -1,10 +0,0 @@ -// We keep the records from the first sync -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 43} -// And append the records from the second sync -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes":[],"sync_id":42}, "_airbyte_generation_id": 44} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_messages_after_meta.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_messages_after_meta.jsonl deleted file mode 100644 index 057165ce8f41..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_messages_after_meta.jsonl +++ /dev/null @@ -1,8 +0,0 @@ -{"type": "RECORD", "record": {"emitted_at": 2000, "data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}}} -{"type": "RECORD", "record": {"emitted_at": 2000, "data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}}} -// Set deleted_at to something non-null. Again, T+D doesn't check the actual _value_ of deleted_at (i.e. the fact that it's in the past is irrelevant). -// It only cares whether deleted_at is non-null. So this should delete Bob from the final table (in dedup mode). -{"type": "RECORD", "record": {"emitted_at": 2000, "data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}}} -// Emit earlier message with _airbyte_meta again with one fixed column. -// Emit a record with an invalid age & address nulled at source. -{"type": "RECORD", "record": {"emitted_at": 2000, "data": {"id1": 2, "id2": 200, "updated_at": "2000-01-02T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "address": {"city": "San Francisco", "state": "CA"}}, "meta": {"changes": [{"field": "address", "change": "NULLED", "reason": "SOURCE_RETRIEVAL_ERROR"}]}}} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_mixedcase_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_mixedcase_expectedrecords_fullrefresh_append_final.jsonl deleted file mode 100644 index f65e848fc6f3..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_mixedcase_expectedrecords_fullrefresh_append_final.jsonl +++ /dev/null @@ -1,9 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}, "_airbyte_generation_id": 0} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}, "_airbyte_generation_id": 0} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}, "_airbyte_generation_id": 0} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23", "_airbyte_generation_id": 0} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00.000000Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?", "_airbyte_generation_id": 0} - -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta": {"changes":[],"sync_id":null}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z", "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_mixedcase_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_mixedcase_expectedrecords_raw.jsonl deleted file mode 100644 index 26c6f21f8f1a..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/resources/dat/sync2_mixedcase_expectedrecords_raw.jsonl +++ /dev/null @@ -1,10 +0,0 @@ -// We keep the records from the first sync, _airbyte_meta in raw didn't exist in that version -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}, "_airbyte_generation_id": 0} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}, "_airbyte_generation_id": 0} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}, "_airbyte_generation_id": 0} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": 42, "registration_date": "2023-12-23"}, "_airbyte_generation_id": 0} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 3, "id2": 200, "updated_at": "2000-01-01T00:04:00Z", "name": "a\bb\fc\nd\re\tf`~!@#$%^&*()_+-=[]\\{}|'\",./<>?"}, "_airbyte_generation_id": 0} -// And append the records from the second sync, _airbyte_meta was added in this version -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}, "_airbyte_meta": {"changes": [],"sync_id":13}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}, "_airbyte_meta": {"changes": [],"sync_id":13}, "_airbyte_generation_id": 43} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}, "_airbyte_meta": {"changes": [],"sync_id":13}, "_airbyte_generation_id": 43} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptTest.kt b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptTest.kt deleted file mode 100644 index 79499f7326a6..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptTest.kt +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ -package io.airbyte.integrations.destination.postgres - -import io.airbyte.commons.json.Jsons.deserialize -import io.airbyte.commons.resources.MoreResources.readResource -import io.airbyte.protocol.models.v0.ConnectorSpecification -import org.junit.jupiter.api.Assertions -import org.junit.jupiter.api.Test - -internal class PostgresDestinationStrictEncryptTest { - @Test - @Throws(Exception::class) - fun testGetSpec() { - println(PostgresDestinationStrictEncrypt().spec().connectionSpecification) - Assertions.assertEquals( - deserialize(readResource("expected_spec.json"), ConnectorSpecification::class.java), - PostgresDestinationStrictEncrypt().spec() - ) - } -} diff --git a/airbyte-integrations/connectors/destination-postgres/metadata.yaml b/airbyte-integrations/connectors/destination-postgres/metadata.yaml index f4f32074ddb6..6b6246509994 100644 --- a/airbyte-integrations/connectors/destination-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 2.1.0 + dockerImageTag: 2.2.0 dockerRepository: airbyte/destination-postgres documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres githubIssueLabel: destination-postgres @@ -14,7 +14,6 @@ data: name: Postgres registries: cloud: - dockerRepository: airbyte/destination-postgres-strict-encrypt enabled: true oss: enabled: true diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestination.kt b/airbyte-integrations/connectors/destination-postgres/src/main/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestination.kt index db27fe6d5a62..e6c79f06d55f 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestination.kt +++ b/airbyte-integrations/connectors/destination-postgres/src/main/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestination.kt @@ -4,14 +4,17 @@ package io.airbyte.integrations.destination.postgres import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode import com.google.common.collect.ImmutableMap import io.airbyte.cdk.db.factory.DataSourceFactory import io.airbyte.cdk.db.factory.DatabaseDriver import io.airbyte.cdk.db.jdbc.JdbcDatabase +import io.airbyte.cdk.db.jdbc.JdbcSSLConnectionUtils import io.airbyte.cdk.db.jdbc.JdbcUtils import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler.Companion.addThrowableForDeinterpolation import io.airbyte.cdk.integrations.base.Destination import io.airbyte.cdk.integrations.base.IntegrationRunner +import io.airbyte.cdk.integrations.base.ssh.SshTunnel import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination import io.airbyte.cdk.integrations.destination.async.deser.StreamAwareDataTransformer import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination @@ -19,12 +22,16 @@ import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinat import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator import io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils import io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.obtainConnectionOptions +import io.airbyte.commons.json.JsonSchemas +import io.airbyte.commons.json.Jsons import io.airbyte.commons.json.Jsons.jsonNode import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator import io.airbyte.integrations.base.destination.typing_deduping.migrators.Migration import io.airbyte.integrations.destination.postgres.typing_deduping.* +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus +import io.airbyte.protocol.models.v0.ConnectorSpecification import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.time.Duration @@ -40,6 +47,43 @@ class PostgresDestination : PostgresSqlOperations() ), Destination { + + override fun check(config: JsonNode): AirbyteConnectionStatus? { + if ( + (config.has(SshTunnel.TUNNEL_METHOD_KEY) && + config[SshTunnel.TUNNEL_METHOD_KEY].has(SshTunnel.TUNNEL_METHOD_KEY)) && + config[SshTunnel.TUNNEL_METHOD_KEY][SshTunnel.TUNNEL_METHOD_KEY].asText() == + SshTunnel.TunnelMethod.NO_TUNNEL.name + ) { + // If no SSH tunnel + if ( + config.has(JdbcUtils.SSL_MODE_KEY) && + config[JdbcUtils.SSL_MODE_KEY].has(JdbcUtils.SSL_MODE_KEY) + ) { + if ( + setOf( + JdbcSSLConnectionUtils.SslMode.DISABLED, + JdbcSSLConnectionUtils.SslMode.ALLOWED, + JdbcSSLConnectionUtils.SslMode.PREFERRED + ) + .contains( + JdbcSSLConnectionUtils.SslMode.bySpec( + config[JdbcUtils.SSL_MODE_KEY][JdbcUtils.SSL_MODE_KEY].asText() + ) + .get() + ) + ) { + // Fail in case SSL mode is disable, allow or prefer + return AirbyteConnectionStatus() + .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage( + "Unsecured connection not allowed. If no SSH Tunnel set up, please use one of the following SSL modes: require, verify-ca, verify-full" + ) + } + } + } + return super.check(config) + } override fun modifyDataSourceBuilder( builder: DataSourceFactory.DataSourceBuilder ): DataSourceFactory.DataSourceBuilder { @@ -192,11 +236,30 @@ class PostgresDestination : @JvmStatic fun sshWrappedDestination(): Destination { - return SshWrappedDestination( - PostgresDestination(), - JdbcUtils.HOST_LIST_KEY, - JdbcUtils.PORT_LIST_KEY - ) + return object : + SshWrappedDestination( + PostgresDestination(), + JdbcUtils.HOST_LIST_KEY, + JdbcUtils.PORT_LIST_KEY + ) { + override fun spec(): ConnectorSpecification { + val originalSpec = super.spec() + if (isCloudDeployment()) { + println("in cloud deployment mode. Editing spec") + val spec: ConnectorSpecification = Jsons.clone(originalSpec) + println( + (spec.connectionSpecification[JsonSchemas.JSON_SCHEMA_PROPERTIES_KEY] + as ObjectNode) + .get(JdbcUtils.SSL_KEY) + ) + (spec.connectionSpecification[JsonSchemas.JSON_SCHEMA_PROPERTIES_KEY] + as ObjectNode) + .remove(JdbcUtils.SSL_KEY) + return spec + } + return originalSpec + } + } } @Throws(Exception::class) diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresGenerationIdMigrator.kt b/airbyte-integrations/connectors/destination-postgres/src/main/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresGenerationIdMigrator.kt index c94be7454b90..a900dbe10b42 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresGenerationIdMigrator.kt +++ b/airbyte-integrations/connectors/destination-postgres/src/main/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresGenerationIdMigrator.kt @@ -12,6 +12,7 @@ import io.airbyte.integrations.base.destination.typing_deduping.DestinationIniti import io.airbyte.integrations.base.destination.typing_deduping.Sql import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig import io.airbyte.integrations.base.destination.typing_deduping.migrators.Migration +import io.airbyte.integrations.destination.postgres.LOGGER import io.github.oshai.kotlinlogging.KotlinLogging import org.jooq.conf.ParamType import org.jooq.impl.DSL @@ -30,6 +31,7 @@ class PostgresGenerationIdMigration( stream: StreamConfig, state: DestinationInitialStatus ): Migration.MigrationResult { + LOGGER.info { "starting generationId migration" } if (state.initialRawTableStatus.rawTableExists) { // The table should exist because we checked for it above, so safe to get it. val existingRawTable = diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.kt b/airbyte-integrations/connectors/destination-postgres/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.kt similarity index 95% rename from airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.kt rename to airbyte-integrations/connectors/destination-postgres/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.kt index 522d0af3a527..c00ec451a3ea 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.kt +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.kt @@ -6,6 +6,7 @@ package io.airbyte.integrations.destination.postgres import com.fasterxml.jackson.databind.JsonNode import com.google.common.collect.ImmutableMap import io.airbyte.cdk.integrations.base.ssh.SshTunnel +import io.airbyte.commons.features.FeatureFlagsWrapper import io.airbyte.configoss.StandardCheckConnectionOutput import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Disabled @@ -14,9 +15,11 @@ import org.junit.jupiter.api.Test @Disabled("Disabled after DV2 migration. Re-enable with fixtures updated to DV2.") class PostgresDestinationStrictEncryptAcceptanceTest : AbstractPostgresDestinationAcceptanceTest() { private var testDb: PostgresTestDatabase? = null + override var featureFlags = + FeatureFlagsWrapper.overridingDeploymentMode(super.featureFlags, "CLOUD") override val imageName: String - get() = "airbyte/destination-postgres-strict-encrypt:dev" + get() = "airbyte/destination-postgres:dev" override fun getConfig(): JsonNode { return testDb!! diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.kt b/airbyte-integrations/connectors/destination-postgres/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.kt similarity index 92% rename from airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.kt rename to airbyte-integrations/connectors/destination-postgres/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.kt index 188394eb679d..a3e32181754d 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.kt +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/kotlin/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresStrictEncryptTypingDedupingTest.kt @@ -6,6 +6,7 @@ package io.airbyte.integrations.destination.postgres.typing_deduping import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import com.google.common.collect.ImmutableMap +import io.airbyte.commons.features.FeatureFlagsWrapper import io.airbyte.integrations.destination.postgres.PostgresDestination import io.airbyte.integrations.destination.postgres.PostgresTestDatabase import javax.sql.DataSource @@ -15,6 +16,8 @@ import org.junit.jupiter.api.BeforeAll // TODO: This test is added to ensure coverage missed by disabling DATs. Redundant when DATs // enabled. class PostgresStrictEncryptTypingDedupingTest : AbstractPostgresTypingDedupingTest() { + override var featureFlags = + FeatureFlagsWrapper.overridingDeploymentMode(super.featureFlags, "CLOUD") override fun getBaseConfig(): ObjectNode { return testContainer!! .configBuilder() @@ -55,7 +58,7 @@ class PostgresStrictEncryptTypingDedupingTest : AbstractPostgresTypingDedupingTe } override val imageName: String - get() = "airbyte/destination-postgres-strict-encrypt:dev" + get() = "airbyte/destination-postgres:dev" companion object { protected var testContainer: PostgresTestDatabase? = null diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptTest.kt b/airbyte-integrations/connectors/destination-postgres/src/test/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptTest.kt new file mode 100644 index 000000000000..df114fad2c5c --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test/kotlin/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptTest.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres + +import io.airbyte.commons.features.EnvVariableFeatureFlags +import io.airbyte.commons.features.FeatureFlagsWrapper +import io.airbyte.commons.json.Jsons +import io.airbyte.commons.resources.MoreResources +import io.airbyte.protocol.models.v0.ConnectorSpecification +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +internal class PostgresDestinationStrictEncryptTest { + @Test + @Throws(Exception::class) + fun testGetSpec() { + val dest = PostgresDestination.sshWrappedDestination() + dest.featureFlags = + FeatureFlagsWrapper.overridingDeploymentMode(EnvVariableFeatureFlags(), "CLOUD") + Assertions.assertEquals( + Jsons.deserialize( + MoreResources.readResource("expected_spec_strict_encrypt.json"), + ConnectorSpecification::class.java + ), + dest.spec() + ) + } +} diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-postgres/src/test/resources/expected_spec_strict_encrypt.json similarity index 100% rename from airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test/resources/expected_spec.json rename to airbyte-integrations/connectors/destination-postgres/src/test/resources/expected_spec_strict_encrypt.json