From afd2e9f2ca73b4c7fa27c84156d2e87bd844bcf2 Mon Sep 17 00:00:00 2001 From: Bogdan Date: Thu, 16 Dec 2021 16:11:47 -0800 Subject: [PATCH] chore: update fixtures scope that are loading data into the analytical db (#17780) * Update db setup fixture scope * Load data into db only once * Update fixture scopes for data loading * cleanup imports * try import * Update scope * fix cleanup * Import all fixtures * Separate data log and metadata creation * Retain table description * fix dtype Co-authored-by: Bogdan Kyryliuk --- tests/integration_tests/access_tests.py | 3 + tests/integration_tests/base_api_tests.py | 1 + tests/integration_tests/cache_tests.py | 1 + tests/integration_tests/celery_tests.py | 1 + tests/integration_tests/charts/api_tests.py | 4 + .../charts/commands_tests.py | 1 + .../charts/data/api_tests.py | 1 + .../integration_tests/charts/schema_tests.py | 1 + tests/integration_tests/cli_tests.py | 1 + tests/integration_tests/core_tests.py | 3 + tests/integration_tests/dashboard_tests.py | 4 + tests/integration_tests/dashboard_utils.py | 35 +++---- .../integration_tests/dashboards/api_tests.py | 2 + .../dashboards/commands_tests.py | 1 + .../integration_tests/dashboards/dao_tests.py | 1 + .../dashboards/filter_state/api_tests.py | 1 + .../security/security_dataset_tests.py | 1 + .../security/security_rbac_tests.py | 1 + .../integration_tests/databases/api_tests.py | 4 + .../databases/commands_tests.py | 2 + tests/integration_tests/datasets/api_tests.py | 2 + .../datasets/commands_tests.py | 2 + tests/integration_tests/datasource_tests.py | 1 + .../db_engine_specs/base_engine_spec_tests.py | 10 +- .../db_engine_specs/bigquery_tests.py | 1 + tests/integration_tests/fixtures/__init__.py | 2 +- .../fixtures/birth_names_dashboard.py | 91 ++++++++++--------- .../fixtures/energy_dashboard.py | 54 ++++++----- .../fixtures/unicode_dashboard.py | 52 ++++++----- .../fixtures/world_bank_dashboard.py | 64 ++++++++----- .../integration_tests/import_export_tests.py | 3 + tests/integration_tests/model_tests.py | 6 +- .../integration_tests/query_context_tests.py | 1 + tests/integration_tests/reports/api_tests.py | 1 + .../reports/commands_tests.py | 2 + tests/integration_tests/schedules_test.py | 1 + tests/integration_tests/security_tests.py | 4 + tests/integration_tests/sqla_models_tests.py | 1 + tests/integration_tests/sqllab_tests.py | 1 + tests/integration_tests/strategy_tests.py | 8 +- .../tasks/async_queries_tests.py | 1 + tests/integration_tests/utils_tests.py | 2 + tox.ini | 2 +- 43 files changed, 245 insertions(+), 136 deletions(-) diff --git a/tests/integration_tests/access_tests.py b/tests/integration_tests/access_tests.py index 6bf6cac25f538..06a3ed23809d2 100644 --- a/tests/integration_tests/access_tests.py +++ b/tests/integration_tests/access_tests.py @@ -25,12 +25,15 @@ from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.test_app import app # isort:skip from superset import db, security_manager diff --git a/tests/integration_tests/base_api_tests.py b/tests/integration_tests/base_api_tests.py index a76346149ef66..09a754e392382 100644 --- a/tests/integration_tests/base_api_tests.py +++ b/tests/integration_tests/base_api_tests.py @@ -18,6 +18,7 @@ import json from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) import pytest diff --git a/tests/integration_tests/cache_tests.py b/tests/integration_tests/cache_tests.py index b600ab5579975..62edb514b35cc 100644 --- a/tests/integration_tests/cache_tests.py +++ b/tests/integration_tests/cache_tests.py @@ -24,6 +24,7 @@ from superset.extensions import cache_manager from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from .base_tests import SupersetTestCase diff --git a/tests/integration_tests/celery_tests.py b/tests/integration_tests/celery_tests.py index bfcb73271b634..1fd076f25bcc1 100644 --- a/tests/integration_tests/celery_tests.py +++ b/tests/integration_tests/celery_tests.py @@ -25,6 +25,7 @@ from typing import Optional from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import pytest diff --git a/tests/integration_tests/charts/api_tests.py b/tests/integration_tests/charts/api_tests.py index ba49c3e380381..c688044eb51aa 100644 --- a/tests/integration_tests/charts/api_tests.py +++ b/tests/integration_tests/charts/api_tests.py @@ -40,9 +40,11 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.fixtures.importexport import ( chart_config, @@ -53,9 +55,11 @@ ) from tests.integration_tests.fixtures.unicode_dashboard import ( load_unicode_dashboard_with_slice, + load_unicode_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.insert_chart_mixin import InsertChartMixin from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/charts/commands_tests.py b/tests/integration_tests/charts/commands_tests.py index caedb6a0b3d54..4e8ffa1af7706 100644 --- a/tests/integration_tests/charts/commands_tests.py +++ b/tests/integration_tests/charts/commands_tests.py @@ -34,6 +34,7 @@ from superset.models.slice import Slice from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) from tests.integration_tests.fixtures.importexport import ( diff --git a/tests/integration_tests/charts/data/api_tests.py b/tests/integration_tests/charts/data/api_tests.py index 2831c291c7097..1e6aa5601d034 100644 --- a/tests/integration_tests/charts/data/api_tests.py +++ b/tests/integration_tests/charts/data/api_tests.py @@ -32,6 +32,7 @@ from tests.integration_tests.annotation_layers.fixtures import create_annotation_layers from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/charts/schema_tests.py b/tests/integration_tests/charts/schema_tests.py index 821a8d57f582a..c28699f3302e4 100644 --- a/tests/integration_tests/charts/schema_tests.py +++ b/tests/integration_tests/charts/schema_tests.py @@ -26,6 +26,7 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.query_context import get_query_context diff --git a/tests/integration_tests/cli_tests.py b/tests/integration_tests/cli_tests.py index 40dfa749b8ded..58616272606cd 100644 --- a/tests/integration_tests/cli_tests.py +++ b/tests/integration_tests/cli_tests.py @@ -30,6 +30,7 @@ from superset import app from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) logger = logging.getLogger(__name__) diff --git a/tests/integration_tests/core_tests.py b/tests/integration_tests/core_tests.py index 87dca2d7b5aa5..67e5e5da0a2df 100644 --- a/tests/integration_tests/core_tests.py +++ b/tests/integration_tests/core_tests.py @@ -27,6 +27,7 @@ from urllib.parse import quote from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import pytest @@ -44,6 +45,7 @@ from tests.integration_tests.conftest import with_feature_flags from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.test_app import app import superset.views.utils @@ -73,6 +75,7 @@ from .base_tests import SupersetTestCase from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) logger = logging.getLogger(__name__) diff --git a/tests/integration_tests/dashboard_tests.py b/tests/integration_tests/dashboard_tests.py index feac2c60708c4..63453d85ee4fc 100644 --- a/tests/integration_tests/dashboard_tests.py +++ b/tests/integration_tests/dashboard_tests.py @@ -33,16 +33,20 @@ from superset.models.slice import Slice from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.fixtures.public_role import public_role_like_gamma from tests.integration_tests.fixtures.unicode_dashboard import ( load_unicode_dashboard_with_position, + load_unicode_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from .base_tests import SupersetTestCase diff --git a/tests/integration_tests/dashboard_utils.py b/tests/integration_tests/dashboard_utils.py index 39032c923165b..3c314312b9c6a 100644 --- a/tests/integration_tests/dashboard_utils.py +++ b/tests/integration_tests/dashboard_utils.py @@ -29,35 +29,30 @@ from superset.utils.core import get_example_default_schema -def create_table_for_dashboard( - df: DataFrame, +def get_table( + table_name: str, database: Database, schema: Optional[str] = None, +): + schema = schema or get_example_default_schema() + table_source = ConnectorRegistry.sources["table"] + return ( + db.session.query(table_source) + .filter_by(database_id=database.id, schema=schema, table_name=table_name) + .one_or_none() + ) + + +def create_table_metadata( table_name: str, database: Database, - dtype: Dict[str, Any], table_description: str = "", fetch_values_predicate: Optional[str] = None, schema: Optional[str] = None, ) -> SqlaTable: schema = schema or get_example_default_schema() - df.to_sql( - table_name, - database.get_sqla_engine(), - if_exists="replace", - chunksize=500, - dtype=dtype, - index=False, - method="multi", - schema=schema, - ) - - table_source = ConnectorRegistry.sources["table"] - table = ( - db.session.query(table_source) - .filter_by(database_id=database.id, schema=schema, table_name=table_name) - .one_or_none() - ) + table = get_table(table_name, database, schema) if not table: + table_source = ConnectorRegistry.sources["table"] table = table_source(schema=schema, table_name=table_name) if fetch_values_predicate: table.fetch_values_predicate = fetch_values_predicate diff --git a/tests/integration_tests/dashboards/api_tests.py b/tests/integration_tests/dashboards/api_tests.py index 071cb8beef001..755eb3776016d 100644 --- a/tests/integration_tests/dashboards/api_tests.py +++ b/tests/integration_tests/dashboards/api_tests.py @@ -53,9 +53,11 @@ from tests.integration_tests.utils.get_dashboards import get_dashboards_ids from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) DASHBOARDS_FIXTURE_COUNT = 10 diff --git a/tests/integration_tests/dashboards/commands_tests.py b/tests/integration_tests/dashboards/commands_tests.py index 63d56eed0b8db..7c7a2046f0592 100644 --- a/tests/integration_tests/dashboards/commands_tests.py +++ b/tests/integration_tests/dashboards/commands_tests.py @@ -48,6 +48,7 @@ ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) diff --git a/tests/integration_tests/dashboards/dao_tests.py b/tests/integration_tests/dashboards/dao_tests.py index ba35166248452..4a02cd7107acd 100644 --- a/tests/integration_tests/dashboards/dao_tests.py +++ b/tests/integration_tests/dashboards/dao_tests.py @@ -28,6 +28,7 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) diff --git a/tests/integration_tests/dashboards/filter_state/api_tests.py b/tests/integration_tests/dashboards/filter_state/api_tests.py index 02af14c5fabc9..bd0e2e7d97753 100644 --- a/tests/integration_tests/dashboards/filter_state/api_tests.py +++ b/tests/integration_tests/dashboards/filter_state/api_tests.py @@ -30,6 +30,7 @@ from tests.integration_tests.base_tests import login from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/dashboards/security/security_dataset_tests.py b/tests/integration_tests/dashboards/security/security_dataset_tests.py index de5a14ef375f1..01bce014afdc5 100644 --- a/tests/integration_tests/dashboards/security/security_dataset_tests.py +++ b/tests/integration_tests/dashboards/security/security_dataset_tests.py @@ -28,6 +28,7 @@ from tests.integration_tests.dashboards.dashboard_test_utils import * from tests.integration_tests.dashboards.superset_factory_util import * from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) diff --git a/tests/integration_tests/dashboards/security/security_rbac_tests.py b/tests/integration_tests/dashboards/security/security_rbac_tests.py index 2a8ab9921f496..5a1f02f1e3243 100644 --- a/tests/integration_tests/dashboards/security/security_rbac_tests.py +++ b/tests/integration_tests/dashboards/security/security_rbac_tests.py @@ -32,6 +32,7 @@ ) from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.public_role import public_role_like_gamma from tests.integration_tests.fixtures.query_context import get_query_context diff --git a/tests/integration_tests/databases/api_tests.py b/tests/integration_tests/databases/api_tests.py index a0e8bc3fdc116..a64eb3f7dcfec 100644 --- a/tests/integration_tests/databases/api_tests.py +++ b/tests/integration_tests/databases/api_tests.py @@ -47,13 +47,16 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.certificates import ssl_certificate from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.fixtures.importexport import ( database_config, @@ -63,6 +66,7 @@ ) from tests.integration_tests.fixtures.unicode_dashboard import ( load_unicode_dashboard_with_position, + load_unicode_data, ) from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/databases/commands_tests.py b/tests/integration_tests/databases/commands_tests.py index 0ec96abf9b444..5860b23fd91bc 100644 --- a/tests/integration_tests/databases/commands_tests.py +++ b/tests/integration_tests/databases/commands_tests.py @@ -44,8 +44,10 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) from tests.integration_tests.fixtures.importexport import ( diff --git a/tests/integration_tests/datasets/api_tests.py b/tests/integration_tests/datasets/api_tests.py index 09984a3dbac85..03209fd94fdf2 100644 --- a/tests/integration_tests/datasets/api_tests.py +++ b/tests/integration_tests/datasets/api_tests.py @@ -46,8 +46,10 @@ from tests.integration_tests.conftest import CTAS_SCHEMA_NAME from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) from tests.integration_tests.fixtures.importexport import ( diff --git a/tests/integration_tests/datasets/commands_tests.py b/tests/integration_tests/datasets/commands_tests.py index d3493a4d13fc6..67ff3a7918ae1 100644 --- a/tests/integration_tests/datasets/commands_tests.py +++ b/tests/integration_tests/datasets/commands_tests.py @@ -33,6 +33,7 @@ from superset.utils.core import get_example_database, get_example_default_schema from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) from tests.integration_tests.fixtures.importexport import ( @@ -45,6 +46,7 @@ ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) diff --git a/tests/integration_tests/datasource_tests.py b/tests/integration_tests/datasource_tests.py index 4c772d317cb7a..c923f2ebbfdcb 100644 --- a/tests/integration_tests/datasource_tests.py +++ b/tests/integration_tests/datasource_tests.py @@ -31,6 +31,7 @@ from tests.integration_tests.base_tests import db_insert_temp_object, SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.datasource import get_datasource_post diff --git a/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py b/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py index 294c6801e7532..e568b59a2ba21 100644 --- a/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py +++ b/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py @@ -35,8 +35,14 @@ from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec from tests.integration_tests.test_app import app -from ..fixtures.birth_names_dashboard import load_birth_names_dashboard_with_slices -from ..fixtures.energy_dashboard import load_energy_table_with_slice +from ..fixtures.birth_names_dashboard import ( + load_birth_names_dashboard_with_slices, + load_birth_names_data, +) +from ..fixtures.energy_dashboard import ( + load_energy_table_data, + load_energy_table_with_slice, +) from ..fixtures.pyodbcRow import Row diff --git a/tests/integration_tests/db_engine_specs/bigquery_tests.py b/tests/integration_tests/db_engine_specs/bigquery_tests.py index d3dd66df31bbb..b7405092c5446 100644 --- a/tests/integration_tests/db_engine_specs/bigquery_tests.py +++ b/tests/integration_tests/db_engine_specs/bigquery_tests.py @@ -29,6 +29,7 @@ from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) diff --git a/tests/integration_tests/fixtures/__init__.py b/tests/integration_tests/fixtures/__init__.py index d3531aa29e894..0e06be352ece5 100644 --- a/tests/integration_tests/fixtures/__init__.py +++ b/tests/integration_tests/fixtures/__init__.py @@ -19,7 +19,7 @@ load_birth_names_dashboard_with_slices, load_birth_names_dashboard_with_slices_module_scope, ) -from .energy_dashboard import load_energy_table_with_slice +from .energy_dashboard import load_energy_table_data, load_energy_table_with_slice from .public_role import public_role_like_gamma, public_role_like_test_role from .unicode_dashboard import ( load_unicode_dashboard_with_position, diff --git a/tests/integration_tests/fixtures/birth_names_dashboard.py b/tests/integration_tests/fixtures/birth_names_dashboard.py index 70a120bcf4c54..56d1bc2870288 100644 --- a/tests/integration_tests/fixtures/birth_names_dashboard.py +++ b/tests/integration_tests/fixtures/birth_names_dashboard.py @@ -23,7 +23,7 @@ import pandas as pd import pytest from pandas import DataFrame -from sqlalchemy import DateTime, String, TIMESTAMP +from sqlalchemy import DateTime, String from superset import ConnectorRegistry, db from superset.connectors.sqla.models import SqlaTable @@ -31,29 +31,14 @@ from superset.models.dashboard import Dashboard from superset.models.slice import Slice from superset.utils.core import get_example_database, get_example_default_schema -from tests.integration_tests.dashboard_utils import create_table_for_dashboard +from tests.integration_tests.dashboard_utils import create_table_metadata from tests.integration_tests.test_app import app - -@pytest.fixture() -def load_birth_names_dashboard_with_slices(): - dash_id_to_delete, slices_ids_to_delete = _load_data() - yield - with app.app_context(): - _cleanup(dash_id_to_delete, slices_ids_to_delete) - - -@pytest.fixture(scope="module") -def load_birth_names_dashboard_with_slices_module_scope(): - dash_id_to_delete, slices_ids_to_delete = _load_data() - yield - with app.app_context(): - _cleanup(dash_id_to_delete, slices_ids_to_delete) +BIRTH_NAMES_TBL_NAME = "birth_names" -def _load_data(): - table_name = "birth_names" - +@pytest.fixture(scope="session") +def load_birth_names_data(): with app.app_context(): database = get_example_database() df = _get_dataframe(database) @@ -63,35 +48,61 @@ def _load_data(): "state": String(10), "name": String(255), } - table = _create_table( - df=df, - table_name=table_name, - database=database, + + df.to_sql( + BIRTH_NAMES_TBL_NAME, + database.get_sqla_engine(), + if_exists="replace", + chunksize=500, dtype=dtype, - fetch_values_predicate="123 = 123", + index=False, + method="multi", + schema=get_example_default_schema(), ) + yield + with app.app_context(): + engine = get_example_database().get_sqla_engine() + engine.execute("DROP TABLE IF EXISTS birth_names") + + +@pytest.fixture() +def load_birth_names_dashboard_with_slices(load_birth_names_data): + with app.app_context(): + dash_id_to_delete, slices_ids_to_delete = _create_dashboards() + yield + _cleanup(dash_id_to_delete, slices_ids_to_delete) + + +@pytest.fixture(scope="module") +def load_birth_names_dashboard_with_slices_module_scope(load_birth_names_data): + with app.app_context(): + dash_id_to_delete, slices_ids_to_delete = _create_dashboards() + yield + _cleanup(dash_id_to_delete, slices_ids_to_delete) + + +def _create_dashboards(): + table = _create_table( + table_name=BIRTH_NAMES_TBL_NAME, + database=get_example_database(), + fetch_values_predicate="123 = 123", + ) - from superset.examples.birth_names import create_dashboard, create_slices + from superset.examples.birth_names import create_dashboard, create_slices - slices, _ = create_slices(table, admin_owner=False) - dash = create_dashboard(slices) - slices_ids_to_delete = [slice.id for slice in slices] - dash_id_to_delete = dash.id - return dash_id_to_delete, slices_ids_to_delete + slices, _ = create_slices(table, admin_owner=False) + dash = create_dashboard(slices) + slices_ids_to_delete = [slice.id for slice in slices] + dash_id_to_delete = dash.id + return dash_id_to_delete, slices_ids_to_delete def _create_table( - df: DataFrame, - table_name: str, - database: "Database", - dtype: Dict[str, Any], - fetch_values_predicate: Optional[str] = None, + table_name: str, database: "Database", fetch_values_predicate: Optional[str] = None, ): - table = create_table_for_dashboard( - df=df, + table = create_table_metadata( table_name=table_name, database=database, - dtype=dtype, fetch_values_predicate=fetch_values_predicate, ) from superset.examples.birth_names import _add_table_metrics, _set_table_metadata @@ -115,8 +126,6 @@ def _cleanup(dash_id: int, slices_ids: List[int]) -> None: columns = [column for column in datasource.columns] metrics = [metric for metric in datasource.metrics] - engine = get_example_database().get_sqla_engine() - engine.execute("DROP TABLE IF EXISTS birth_names") for column in columns: db.session.delete(column) for metric in metrics: diff --git a/tests/integration_tests/fixtures/energy_dashboard.py b/tests/integration_tests/fixtures/energy_dashboard.py index 14471052543c7..f4362ea7463af 100644 --- a/tests/integration_tests/fixtures/energy_dashboard.py +++ b/tests/integration_tests/fixtures/energy_dashboard.py @@ -15,34 +15,50 @@ # specific language governing permissions and limitations # under the License. import random -import textwrap from typing import Dict, Set import pandas as pd import pytest -from pandas import DataFrame from sqlalchemy import column, Float, String from superset import db from superset.connectors.sqla.models import SqlaTable, SqlMetric -from superset.models.dashboard import Dashboard from superset.models.slice import Slice -from superset.utils.core import get_example_database -from tests.integration_tests.dashboard_utils import ( - create_slice, - create_table_for_dashboard, -) +from superset.utils.core import get_example_database, get_example_default_schema +from tests.integration_tests.dashboard_utils import create_slice, create_table_metadata from tests.integration_tests.test_app import app misc_dash_slices: Set[str] = set() +ENERGY_USAGE_TBL_NAME = "energy_usage" + + +@pytest.fixture(scope="session") +def load_energy_table_data(): + with app.app_context(): + database = get_example_database() + df = _get_dataframe() + df.to_sql( + ENERGY_USAGE_TBL_NAME, + database.get_sqla_engine(), + if_exists="replace", + chunksize=500, + index=False, + dtype={"source": String(255), "target": String(255), "value": Float()}, + method="multi", + schema=get_example_default_schema(), + ) + yield + with app.app_context(): + engine = get_example_database().get_sqla_engine() + engine.execute("DROP TABLE IF EXISTS energy_usage") + + @pytest.fixture() -def load_energy_table_with_slice(): - table_name = "energy_usage" - df = _get_dataframe() +def load_energy_table_with_slice(load_energy_table_data): with app.app_context(): - _create_energy_table(df, table_name) + _create_energy_table() yield _cleanup() @@ -52,13 +68,11 @@ def _get_dataframe(): return pd.DataFrame.from_dict(data) -def _create_energy_table(df: DataFrame, table_name: str): - database = get_example_database() - - table_description = "Energy consumption" - schema = {"source": String(255), "target": String(255), "value": Float()} - table = create_table_for_dashboard( - df, table_name, database, schema, table_description +def _create_energy_table(): + table = create_table_metadata( + table_name=ENERGY_USAGE_TBL_NAME, + database=get_example_database(), + table_description="Energy consumption", ) table.fetch_metadata() @@ -96,8 +110,6 @@ def _create_and_commit_energy_slice( def _cleanup() -> None: - engine = get_example_database().get_sqla_engine() - engine.execute("DROP TABLE IF EXISTS energy_usage") for slice_data in _get_energy_slices(): slice = ( db.session.query(Slice) diff --git a/tests/integration_tests/fixtures/unicode_dashboard.py b/tests/integration_tests/fixtures/unicode_dashboard.py index c670e086b7e6b..1f1218c8f7dbc 100644 --- a/tests/integration_tests/fixtures/unicode_dashboard.py +++ b/tests/integration_tests/fixtures/unicode_dashboard.py @@ -16,42 +16,58 @@ # under the License. import pandas as pd import pytest -from pandas import DataFrame from sqlalchemy import String from superset import db from superset.connectors.sqla.models import SqlaTable from superset.models.dashboard import Dashboard from superset.models.slice import Slice -from superset.utils.core import get_example_database +from superset.utils.core import get_example_database, get_example_default_schema from tests.integration_tests.dashboard_utils import ( create_dashboard, create_slice, - create_table_for_dashboard, + create_table_metadata, ) from tests.integration_tests.test_app import app +UNICODE_TBL_NAME = "unicode_test" + + +@pytest.fixture(scope="session") +def load_unicode_data(): + with app.app_context(): + _get_dataframe().to_sql( + UNICODE_TBL_NAME, + get_example_database().get_sqla_engine(), + if_exists="replace", + chunksize=500, + dtype={"phrase": String(500)}, + index=False, + method="multi", + schema=get_example_default_schema(), + ) + + yield + with app.app_context(): + engine = get_example_database().get_sqla_engine() + engine.execute("DROP TABLE IF EXISTS unicode_test") + @pytest.fixture() -def load_unicode_dashboard_with_slice(): - table_name = "unicode_test" +def load_unicode_dashboard_with_slice(load_unicode_data): slice_name = "Unicode Cloud" - df = _get_dataframe() with app.app_context(): - dash = _create_unicode_dashboard(df, table_name, slice_name, None) + dash = _create_unicode_dashboard(slice_name, None) yield - _cleanup(dash, slice_name) @pytest.fixture() -def load_unicode_dashboard_with_position(): - table_name = "unicode_test" +def load_unicode_dashboard_with_position(load_unicode_data): slice_name = "Unicode Cloud" - df = _get_dataframe() position = "{}" with app.app_context(): - dash = _create_unicode_dashboard(df, table_name, slice_name, position) + dash = _create_unicode_dashboard(slice_name, position) yield _cleanup(dash, slice_name) @@ -74,14 +90,8 @@ def _get_unicode_data(): ] -def _create_unicode_dashboard( - df: DataFrame, table_name: str, slice_title: str, position: str -) -> Dashboard: - database = get_example_database() - dtype = { - "phrase": String(500), - } - table = create_table_for_dashboard(df, table_name, database, dtype) +def _create_unicode_dashboard(slice_title: str, position: str) -> Dashboard: + table = create_table_metadata(UNICODE_TBL_NAME, get_example_database()) table.fetch_metadata() if slice_title: @@ -101,8 +111,6 @@ def _create_and_commit_unicode_slice(table: SqlaTable, title: str): def _cleanup(dash: Dashboard, slice_name: str) -> None: - engine = get_example_database().get_sqla_engine() - engine.execute("DROP TABLE IF EXISTS unicode_test") db.session.delete(dash) if slice_name: slice = db.session.query(Slice).filter_by(slice_name=slice_name).one_or_none() diff --git a/tests/integration_tests/fixtures/world_bank_dashboard.py b/tests/integration_tests/fixtures/world_bank_dashboard.py index 96190c4b1d723..0782c8bdf543b 100644 --- a/tests/integration_tests/fixtures/world_bank_dashboard.py +++ b/tests/integration_tests/fixtures/world_bank_dashboard.py @@ -22,7 +22,7 @@ import pandas as pd import pytest from pandas import DataFrame -from sqlalchemy import DateTime, String, TIMESTAMP +from sqlalchemy import DateTime, String from superset import db from superset.connectors.sqla.models import SqlaTable @@ -32,43 +32,59 @@ from superset.utils.core import get_example_database, get_example_default_schema from tests.integration_tests.dashboard_utils import ( create_dashboard, - create_table_for_dashboard, + create_table_metadata, ) from tests.integration_tests.test_app import app +WB_HEALTH_POPULATION = "wb_health_population" + + +@pytest.fixture(scope="session") +def load_world_bank_data(): + with app.app_context(): + database = get_example_database() + dtype = { + "year": DateTime if database.backend != "presto" else String(255), + "country_code": String(3), + "country_name": String(255), + "region": String(255), + } + _get_dataframe(database).to_sql( + WB_HEALTH_POPULATION, + get_example_database().get_sqla_engine(), + if_exists="replace", + chunksize=500, + dtype=dtype, + index=False, + method="multi", + schema=get_example_default_schema(), + ) -@pytest.fixture() -def load_world_bank_dashboard_with_slices(): - dash_id_to_delete, slices_ids_to_delete = _load_data() yield with app.app_context(): + engine = get_example_database().get_sqla_engine() + engine.execute("DROP TABLE IF EXISTS wb_health_population") + + +@pytest.fixture() +def load_world_bank_dashboard_with_slices(load_world_bank_data): + with app.app_context(): + dash_id_to_delete, slices_ids_to_delete = create_dashboard_for_loaded_data() + yield _cleanup(dash_id_to_delete, slices_ids_to_delete) @pytest.fixture(scope="module") -def load_world_bank_dashboard_with_slices_module_scope(): - dash_id_to_delete, slices_ids_to_delete = _load_data() - yield +def load_world_bank_dashboard_with_slices_module_scope(load_world_bank_data): with app.app_context(): + dash_id_to_delete, slices_ids_to_delete = create_dashboard_for_loaded_data() + yield _cleanup(dash_id_to_delete, slices_ids_to_delete) -def _load_data(): - table_name = "wb_health_population" - +def create_dashboard_for_loaded_data(): with app.app_context(): - database = get_example_database() - schema = get_example_default_schema() - df = _get_dataframe(database) - dtype = { - "year": DateTime if database.backend != "presto" else String(255), - "country_code": String(3), - "country_name": String(255), - "region": String(255), - } - table = create_table_for_dashboard( - df, table_name, database, dtype, schema=schema - ) + table = create_table_metadata(WB_HEALTH_POPULATION, get_example_database()) slices = _create_world_bank_slices(table) dash = _create_world_bank_dashboard(table, slices) slices_ids_to_delete = [slice.id for slice in slices] @@ -112,8 +128,6 @@ def _create_world_bank_dashboard(table: SqlaTable, slices: List[Slice]) -> Dashb def _cleanup(dash_id: int, slices_ids: List[int]) -> None: - engine = get_example_database().get_sqla_engine() - engine.execute("DROP TABLE IF EXISTS wb_health_population") dash = db.session.query(Dashboard).filter_by(id=dash_id).first() db.session.delete(dash) for slice_id in slices_ids: diff --git a/tests/integration_tests/import_export_tests.py b/tests/integration_tests/import_export_tests.py index 42adcb851b8a6..1745f5e01c2fe 100644 --- a/tests/integration_tests/import_export_tests.py +++ b/tests/integration_tests/import_export_tests.py @@ -20,6 +20,7 @@ import unittest from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import pytest @@ -28,6 +29,7 @@ from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.test_app import app from superset.dashboards.commands.importers.v0 import decode_dashboards @@ -47,6 +49,7 @@ from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from .base_tests import SupersetTestCase diff --git a/tests/integration_tests/model_tests.py b/tests/integration_tests/model_tests.py index 7ffb173cd35fd..bcd57726ff63e 100644 --- a/tests/integration_tests/model_tests.py +++ b/tests/integration_tests/model_tests.py @@ -22,6 +22,7 @@ from superset.exceptions import SupersetException from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import pytest @@ -38,7 +39,10 @@ from superset.utils.core import get_example_database from .base_tests import SupersetTestCase -from .fixtures.energy_dashboard import load_energy_table_with_slice +from .fixtures.energy_dashboard import ( + load_energy_table_with_slice, + load_energy_table_data, +) class TestDatabaseModel(SupersetTestCase): diff --git a/tests/integration_tests/query_context_tests.py b/tests/integration_tests/query_context_tests.py index 6df246db74e96..91ce67bb84681 100644 --- a/tests/integration_tests/query_context_tests.py +++ b/tests/integration_tests/query_context_tests.py @@ -34,6 +34,7 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.query_context import get_query_context diff --git a/tests/integration_tests/reports/api_tests.py b/tests/integration_tests/reports/api_tests.py index 8df8b4bc8bee5..fa9e36d104a8c 100644 --- a/tests/integration_tests/reports/api_tests.py +++ b/tests/integration_tests/reports/api_tests.py @@ -43,6 +43,7 @@ from tests.integration_tests.conftest import with_feature_flags from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.reports.utils import insert_report_schedule diff --git a/tests/integration_tests/reports/commands_tests.py b/tests/integration_tests/reports/commands_tests.py index 1ed3f35a16512..3a7cc873e3131 100644 --- a/tests/integration_tests/reports/commands_tests.py +++ b/tests/integration_tests/reports/commands_tests.py @@ -59,9 +59,11 @@ from superset.utils.core import get_example_database from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices_module_scope, + load_world_bank_data, ) from tests.integration_tests.reports.utils import insert_report_schedule from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/schedules_test.py b/tests/integration_tests/schedules_test.py index c47bf5fc86966..b5cfe716051ae 100644 --- a/tests/integration_tests/schedules_test.py +++ b/tests/integration_tests/schedules_test.py @@ -26,6 +26,7 @@ from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.test_app import app from superset import db diff --git a/tests/integration_tests/security_tests.py b/tests/integration_tests/security_tests.py index e96ed67456b56..9863e1b316c35 100644 --- a/tests/integration_tests/security_tests.py +++ b/tests/integration_tests/security_tests.py @@ -48,9 +48,11 @@ from .base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.fixtures.public_role import ( public_role_like_gamma, @@ -58,9 +60,11 @@ ) from tests.integration_tests.fixtures.unicode_dashboard import ( load_unicode_dashboard_with_slice, + load_unicode_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) NEW_SECURITY_CONVERGE_VIEWS = ( diff --git a/tests/integration_tests/sqla_models_tests.py b/tests/integration_tests/sqla_models_tests.py index 53bf031bf53be..00d38590091c6 100644 --- a/tests/integration_tests/sqla_models_tests.py +++ b/tests/integration_tests/sqla_models_tests.py @@ -44,6 +44,7 @@ ) from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from .base_tests import SupersetTestCase diff --git a/tests/integration_tests/sqllab_tests.py b/tests/integration_tests/sqllab_tests.py index 8a3297f05e6dd..2dce2c11f3799 100644 --- a/tests/integration_tests/sqllab_tests.py +++ b/tests/integration_tests/sqllab_tests.py @@ -58,6 +58,7 @@ from .conftest import CTAS_SCHEMA_NAME from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) QUERY_1 = "SELECT * FROM birth_names LIMIT 1" diff --git a/tests/integration_tests/strategy_tests.py b/tests/integration_tests/strategy_tests.py index 1ba41d65bf35a..59430f80b856a 100644 --- a/tests/integration_tests/strategy_tests.py +++ b/tests/integration_tests/strategy_tests.py @@ -21,6 +21,7 @@ from unittest.mock import MagicMock from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from sqlalchemy import String, Date, Float @@ -42,8 +43,11 @@ ) from .base_tests import SupersetTestCase -from .dashboard_utils import create_dashboard, create_slice, create_table_for_dashboard -from .fixtures.unicode_dashboard import load_unicode_dashboard_with_slice +from .dashboard_utils import create_dashboard, create_slice, create_table_metadata +from .fixtures.unicode_dashboard import ( + load_unicode_dashboard_with_slice, + load_unicode_data, +) URL_PREFIX = "http://0.0.0.0:8081" diff --git a/tests/integration_tests/tasks/async_queries_tests.py b/tests/integration_tests/tasks/async_queries_tests.py index 6acdd88b78838..388185a54853d 100644 --- a/tests/integration_tests/tasks/async_queries_tests.py +++ b/tests/integration_tests/tasks/async_queries_tests.py @@ -35,6 +35,7 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.query_context import get_query_context from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/utils_tests.py b/tests/integration_tests/utils_tests.py index 3fcbe959e023a..fc9dba4c07214 100644 --- a/tests/integration_tests/utils_tests.py +++ b/tests/integration_tests/utils_tests.py @@ -26,6 +26,7 @@ from unittest.mock import Mock, patch from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import numpy as np @@ -81,6 +82,7 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from .fixtures.certificates import ssl_certificate diff --git a/tox.ini b/tox.ini index 839bce55867ec..88a51278ec2ae 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ setenv = postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test sqlite: SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db mysql-presto: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 - # docker run -p 8080:8080 --name presto prestosql/presto + # docker run -p 8080:8080 --name presto starburstdata/presto mysql-presto: SUPERSET__SQLALCHEMY_EXAMPLES_URI = presto://localhost:8080/memory/default # based on https://github.com/big-data-europe/docker-hadoop # clone the repo & run docker-compose up -d to test locally