chore: update fixtures scope that are loading data into the analytica…

…l db (#17780)

* Update db setup fixture scope

* Load data into db only once

* Update fixture scopes for data loading

* cleanup imports

* try import

* Update scope

* fix cleanup

* Import all fixtures

* Separate data log and metadata creation

* Retain table description

* fix dtype

Co-authored-by: Bogdan Kyryliuk <bogdankyryliuk@dropbox.com>

tests/integration_tests/access_tests.py

@@ -25,12 +25,15 @@
 
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.test_app import app  # isort:skip
 from superset import db, security_manager

tests/integration_tests/base_api_tests.py

@@ -18,6 +18,7 @@
 import json
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 import pytest

tests/integration_tests/cache_tests.py

@@ -24,6 +24,7 @@
 from superset.extensions import cache_manager
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 from .base_tests import SupersetTestCase

tests/integration_tests/celery_tests.py

@@ -25,6 +25,7 @@
 from typing import Optional
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 import pytest

tests/integration_tests/charts/api_tests.py

@@ -40,9 +40,11 @@
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.fixtures.importexport import (
     chart_config,
@@ -53,9 +55,11 @@
 )
 from tests.integration_tests.fixtures.unicode_dashboard import (
     load_unicode_dashboard_with_slice,
+    load_unicode_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from tests.integration_tests.insert_chart_mixin import InsertChartMixin
 from tests.integration_tests.test_app import app

tests/integration_tests/charts/commands_tests.py

@@ -34,6 +34,7 @@
 from superset.models.slice import Slice
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 from tests.integration_tests.fixtures.importexport import (

tests/integration_tests/charts/data/api_tests.py

@@ -32,6 +32,7 @@
 from tests.integration_tests.annotation_layers.fixtures import create_annotation_layers
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.test_app import app
 

tests/integration_tests/charts/schema_tests.py

@@ -26,6 +26,7 @@
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.query_context import get_query_context
 

tests/integration_tests/cli_tests.py

@@ -30,6 +30,7 @@
 from superset import app
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 logger = logging.getLogger(__name__)

tests/integration_tests/core_tests.py

@@ -27,6 +27,7 @@
 from urllib.parse import quote
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 import pytest
@@ -44,6 +45,7 @@
 from tests.integration_tests.conftest import with_feature_flags
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.test_app import app
 import superset.views.utils
@@ -73,6 +75,7 @@
 from .base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 logger = logging.getLogger(__name__)

tests/integration_tests/dashboard_tests.py

@@ -33,16 +33,20 @@
 from superset.models.slice import Slice
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.fixtures.public_role import public_role_like_gamma
 from tests.integration_tests.fixtures.unicode_dashboard import (
     load_unicode_dashboard_with_position,
+    load_unicode_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 from .base_tests import SupersetTestCase

tests/integration_tests/dashboard_utils.py

@@ -29,35 +29,30 @@
 from superset.utils.core import get_example_default_schema
 
 
-def create_table_for_dashboard(
-    df: DataFrame,
+def get_table(
+    table_name: str, database: Database, schema: Optional[str] = None,
+):
+    schema = schema or get_example_default_schema()
+    table_source = ConnectorRegistry.sources["table"]
+    return (
+        db.session.query(table_source)
+        .filter_by(database_id=database.id, schema=schema, table_name=table_name)
+        .one_or_none()
+    )
+
+
+def create_table_metadata(
     table_name: str,
     database: Database,
-    dtype: Dict[str, Any],
     table_description: str = "",
     fetch_values_predicate: Optional[str] = None,
     schema: Optional[str] = None,
 ) -> SqlaTable:
     schema = schema or get_example_default_schema()
 
-    df.to_sql(
-        table_name,
-        database.get_sqla_engine(),
-        if_exists="replace",
-        chunksize=500,
-        dtype=dtype,
-        index=False,
-        method="multi",
-        schema=schema,
-    )
-
-    table_source = ConnectorRegistry.sources["table"]
-    table = (
-        db.session.query(table_source)
-        .filter_by(database_id=database.id, schema=schema, table_name=table_name)
-        .one_or_none()
-    )
+    table = get_table(table_name, database, schema)
     if not table:
+        table_source = ConnectorRegistry.sources["table"]
         table = table_source(schema=schema, table_name=table_name)
     if fetch_values_predicate:
         table.fetch_values_predicate = fetch_values_predicate

tests/integration_tests/dashboards/api_tests.py

@@ -53,9 +53,11 @@
 from tests.integration_tests.utils.get_dashboards import get_dashboards_ids
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 DASHBOARDS_FIXTURE_COUNT = 10

tests/integration_tests/dashboards/commands_tests.py

@@ -48,6 +48,7 @@
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 

tests/integration_tests/dashboards/dao_tests.py

@@ -28,6 +28,7 @@
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 

tests/integration_tests/dashboards/filter_state/api_tests.py

@@ -30,6 +30,7 @@
 from tests.integration_tests.base_tests import login
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from tests.integration_tests.test_app import app
 

tests/integration_tests/dashboards/security/security_dataset_tests.py

@@ -28,6 +28,7 @@
 from tests.integration_tests.dashboards.dashboard_test_utils import *
 from tests.integration_tests.dashboards.superset_factory_util import *
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 

tests/integration_tests/dashboards/security/security_rbac_tests.py

@@ -32,6 +32,7 @@
 )
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.public_role import public_role_like_gamma
 from tests.integration_tests.fixtures.query_context import get_query_context

tests/integration_tests/databases/api_tests.py

@@ -47,13 +47,16 @@
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.certificates import ssl_certificate
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from tests.integration_tests.fixtures.importexport import (
     database_config,
@@ -63,6 +66,7 @@
 )
 from tests.integration_tests.fixtures.unicode_dashboard import (
     load_unicode_dashboard_with_position,
+    load_unicode_data,
 )
 from tests.integration_tests.test_app import app
 

tests/integration_tests/databases/commands_tests.py

@@ -44,8 +44,10 @@
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 from tests.integration_tests.fixtures.importexport import (

tests/integration_tests/datasets/api_tests.py

@@ -46,8 +46,10 @@
 from tests.integration_tests.conftest import CTAS_SCHEMA_NAME
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 from tests.integration_tests.fixtures.importexport import (

tests/integration_tests/datasets/commands_tests.py

@@ -33,6 +33,7 @@
 from superset.utils.core import get_example_database, get_example_default_schema
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 from tests.integration_tests.fixtures.importexport import (
@@ -45,6 +46,7 @@
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 

tests/integration_tests/datasource_tests.py

@@ -31,6 +31,7 @@
 from tests.integration_tests.base_tests import db_insert_temp_object, SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.datasource import get_datasource_post
 

tests/integration_tests/db_engine_specs/base_engine_spec_tests.py

@@ -35,8 +35,14 @@
 from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
 from tests.integration_tests.test_app import app
 
-from ..fixtures.birth_names_dashboard import load_birth_names_dashboard_with_slices
-from ..fixtures.energy_dashboard import load_energy_table_with_slice
+from ..fixtures.birth_names_dashboard import (
+    load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
+)
+from ..fixtures.energy_dashboard import (
+    load_energy_table_data,
+    load_energy_table_with_slice,
+)
 from ..fixtures.pyodbcRow import Row
 
 

tests/integration_tests/db_engine_specs/bigquery_tests.py

@@ -29,6 +29,7 @@
 from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 

tests/integration_tests/fixtures/__init__.py

@@ -19,7 +19,7 @@
     load_birth_names_dashboard_with_slices,
     load_birth_names_dashboard_with_slices_module_scope,
 )
-from .energy_dashboard import load_energy_table_with_slice
+from .energy_dashboard import load_energy_table_data, load_energy_table_with_slice
 from .public_role import public_role_like_gamma, public_role_like_test_role
 from .unicode_dashboard import (
     load_unicode_dashboard_with_position,