NAS-127251 / 24.10 / Add pytest markers for more intelligent runs

tests/api2/pytest.ini

@@ -0,0 +1,23 @@
+[pytest]
+markers =
+    slow: mark test as slow
+    base: mark a test or module as requried for all runs
+    boot: include boot-volume-related tests
+    accounts: include account (user / group) tests
+    alerts: include alerts tests
+    audit: include audit tests
+    apps: include apps tests
+    certs: include cert-related tests
+    cloudsync: include cloudsync tests
+    core: include core plugin tests
+    cron: include cron-related tests
+    disk: include disk / storage-related tests
+    ds: include tests for directory services
+    fs: include filesystem plugin tests
+    ftp: include FTP tests
+    iscsi: include iSCSI tests
+    nfs: include NFS tests
+    rbac: include privilege / role tests
+    smb: include SMB tests
+    vm: include VM tests
+    zfs: include extra ZFS-related tests

tests/api2/test_001_ssh.py

@@ -10,6 +10,8 @@
 from middlewared.test.integration.utils import call, fail
 from middlewared.test.integration.utils.client import client
 
+pytestmark = pytest.mark.base
+
 
 @pytest.fixture(scope='module')
 def ip_to_use():

tests/api2/test_003_network_global.py

@@ -9,6 +9,8 @@
 from auto_config import ha, interface, hostname, domain, ip
 from middlewared.test.integration.utils.client import client
 
+pytestmark = pytest.mark.base
+
 
 @pytest.fixture(scope='module')
 def ip_to_use():

tests/api2/test_005_interface.py

@@ -9,6 +9,8 @@
 from auto_config import ip, interface, ha
 from middlewared.test.integration.utils.client import client
 
+pytestmark = pytest.mark.base
+
 
 @pytest.fixture(scope='module')
 def ip_to_use():

tests/api2/test_006_pool_and_sysds.py

@@ -12,6 +12,8 @@
 from middlewared.test.integration.utils import fail
 from middlewared.test.integration.utils.client import client
 
+pytestmark = pytest.mark.base
+
 
 @pytest.fixture(scope='module')
 def ws_client():

tests/api2/test_007_early_settings.py

@@ -1,10 +1,12 @@
+import pytest
+
 from middlewared.test.integration.utils import call
 
 # this is found in middlewared.plugins.sysctl.sysctl_info
 # but the client running the tests isn't guaranteed to have
 # the middlewared application installed locally
 DEFAULT_ARC_MAX_FILE = '/var/run/middleware/default_arc_max'
-
+pytestmark = pytest.mark.base
 
 def test_sysctl_arc_max_is_set():
     """Middleware should have created this file and written a number

tests/api2/test_011_user.py

@@ -20,6 +20,8 @@
 from middlewared.test.integration.assets.pool import dataset as dataset_asset
 from middlewared.test.integration.utils import call, ssh
 
+pytestmark = pytest.mark.accounts
+
 apifolder = os.getcwd()
 sys.path.append(apifolder)
 from functions import POST, GET, DELETE, PUT, SSH_TEST, wait_on_job

tests/api2/test_012_directory_service_ssh.py

@@ -11,6 +11,8 @@
 from middlewared.test.integration.assets.directory_service import active_directory, ldap
 from middlewared.test.integration.utils import call
 
+pytestmark = pytest.mark.ds
+
 try:
     from config import AD_DOMAIN, ADPASSWORD, ADUSERNAME
 except ImportError:

tests/api2/test_020_account.py

@@ -4,12 +4,15 @@
 # License: BSD
 # Location for tests into REST API of FreeNAS
 
+import pytest
 import sys
 import os
 apifolder = os.getcwd()
 sys.path.append(apifolder)
 from functions import DELETE, GET, POST
 
+pytest.mark.accounts
+
 
 def delete_group_delete_users(delete_users):
     results = POST("/user/", {

tests/api2/test_023_kubernetes.py

@@ -6,6 +6,7 @@
 from auto_config import ha, pool_name, interface, ip
 from middlewared.test.integration.utils import call
 
+pytestmark = pytest.mark.apps
 
 # Read all the test below only on non-HA
 if not ha:

tests/api2/test_024_container.py

@@ -25,6 +25,7 @@
 except ImportError:
     skip_container_image = pytest.mark.skipif(True, reason=container_reason)
 
+pytestmark = pytest.mark.apps
 
 # Read all the test below only on non-HA
 if not ha:

tests/api2/test_026_kubernetes_backup_chart_releases.py

@@ -15,7 +15,7 @@
 from middlewared.test.integration.assets.kubernetes import backup
 from middlewared.test.integration.utils import file_exists_and_perms_check
 
-
+pytestmark = [pytest.mark.apps, pytest.mark.slow]
 backup_release_name = 'backupsyncthing'
 
 # Read all the test below only on non-HA

tests/api2/test_027_kubernetes_logs.py

@@ -5,6 +5,7 @@
 from pytest_dependency import depends
 from time import sleep
 
+pytestmark = pytest.mark.apps
 
 @contextlib.contextmanager
 def official_chart_release(chart_name, release_name):

tests/api2/test_030_activedirectory.py

@@ -29,11 +29,11 @@
 try:
     from config import AD_DOMAIN, ADPASSWORD, ADUSERNAME
     AD_USER = fr"AD02\{ADUSERNAME.lower()}"
+    pytestmark = pytest.mark.ds
 except ImportError:
     Reason = 'ADNameServer AD_DOMAIN, ADPASSWORD, or/and ADUSERNAME are missing in config.py"'
     pytestmark = pytest.mark.skip(reason=Reason)
 
-
 SMB_NAME = "TestADShare"
 
 

tests/api2/test_032_ad_kerberos.py

@@ -20,6 +20,7 @@
 
 try:
     from config import AD_DOMAIN, ADPASSWORD, ADUSERNAME, ADNameServer, AD_COMPUTER_OU
+    pytestmark = pytest.mark.ds
 except ImportError:
     Reason = 'ADNameServer AD_DOMAIN, ADPASSWORD, or/and ADUSERNAME are missing in config.py"'
     pytestmark = pytest.mark.skip(reason=Reason)

tests/api2/test_035_ad_idmap.py

@@ -26,6 +26,7 @@
         LDAPBINDPASSWORD,
         LDAPHOSTNAME
     )
+    pytestmark = pytest.mark.ds
 except ImportError:
     Reason = 'ADNameServer AD_DOMAIN, ADPASSWORD, or/and ADUSERNAME are missing in config.py"'
     pytestmark = pytest.mark.skip(reason=Reason)

tests/api2/test_040_ad_user_group_cache.py

@@ -16,6 +16,7 @@
 
 try:
     from config import AD_DOMAIN, ADPASSWORD, ADUSERNAME, ADNameServer, AD_COMPUTER_OU
+    pytestmark = pytest.mark.ds
 except ImportError:
     Reason = 'ADNameServer AD_DOMAIN, ADPASSWORD, or/and ADUSERNAME are missing in config.py"'
     pytestmark = pytest.mark.skip(reason=Reason)

tests/api2/test_050_alert.py

@@ -11,6 +11,7 @@
 from auto_config import ip, password, user, pool_name
 from middlewared.test.integration.utils import call
 
+pytestmark = pytest.mark.alerts
 
 
 def test_01_get_alert_list():

tests/api2/test_070_alertservice.py

@@ -7,6 +7,8 @@
 sys.path.append(apifolder)
 from functions import GET, POST, PUT, DELETE
 
+pytestmark = pytest.mark.alerts
+
 
 def test_01_get_alertservice():
     results = GET("/alertservice/")

tests/api2/test_090_boot.py

@@ -12,6 +12,8 @@
 sys.path.append(apifolder)
 from functions import GET
 
+pytestmark = pytest.mark.boot
+
 
 @pytest.mark.dependency(name='BOOT_DISKS')
 def test_01_get_boot_disks():

tests/api2/test_100_bootenv.py

@@ -12,6 +12,8 @@
 sys.path.append(apifolder)
 from functions import POST, DELETE, GET, PUT, wait_on_job
 
+pytestmark = pytest.mark.boot
+
 
 def test_01_get_the_activated_bootenv():
     global active_be_id

tests/api2/test_110_certificate.py

@@ -20,6 +20,7 @@
         LDAPBINDPASSWORD,
         LDAPHOSTNAME,
     )
+    pytestmark = pytest.mark.certs
 except ImportError:
     Reason = 'LDAP* variable are not setup in config.py'
     # comment pytestmark for development testing with --dev-test

tests/api2/test_120_certificateauthority.py

@@ -10,6 +10,8 @@
 sys.path.append(apifolder)
 from functions import GET
 
+pytestmark = pytest.mark.certs
+
 
 def test_01_get_certificateauthority_query():
     results = GET('/certificateauthority/')

tests/api2/test_130_cloudsync.py

@@ -20,6 +20,7 @@
         AWS_SECRET_ACCESS_KEY,
         AWS_BUCKET
     )
+    pytestmark = pytest.mark.cloudsync
 except ImportError:
     Reason = 'AWS credential are missing in config.py'
     pytestmark = pytest.mark.skip(reason=Reason)

tests/api2/test_150_cronjob.py

@@ -12,6 +12,7 @@
 from auto_config import user, password, ip
 
 TESTFILE = '/tmp/.testFileCreatedViaCronjob'
+pytestmark = pytest.mark.cron
 
 
 @pytest.fixture(scope='module')

tests/api2/test_160_device.py

@@ -12,6 +12,8 @@
 from auto_config import ha
 global all_results
 all_results = {}
+pytestmark = pytest.mark.disk
+
 disk_list = list(POST('/device/get_info/', 'DISK', controller_a=ha).json().keys())
 
 

tests/api2/test_170_disk.py

@@ -13,6 +13,7 @@
 RunTest = True
 TestName = "get disk information"
 DISK_ID = None
+pytestmark = pytest.mark.disk
 
 disk_list = list(POST('/device/get_info/', 'DISK', controller_a=ha).json().keys())
 

tests/api2/test_190_filesystem.py

@@ -24,6 +24,7 @@
 path = '/etc'
 path_list = ['default', 'kernel', 'zfs', 'ssh']
 random_path = ['/boot/grub', '/root', '/bin', '/usr/bin']
+pytestmark = pytest.mark.fs
 
 
 def test_01_get_filesystem_listdir():

tests/api2/test_200_ftp.py

@@ -33,6 +33,8 @@
 else:
     from auto_config import ip
 
+pytestmark = pytest.mark.ftp
+
 FTP_DEFAULT = {}
 DB_DFLT = {}
 INIT_DIRS_AND_FILES = {

tests/api2/test_210_group.py

@@ -15,6 +15,7 @@
 from middlewared.test.integration.utils import call
 from pytest_dependency import depends
 GroupIdFile = "/tmp/.ixbuild_test_groupid"
+pytestmark = pytest.mark.accounts
 
 
 def test_01_get_next_gid():

tests/api2/test_230_idmap.py

@@ -15,6 +15,7 @@
         LDAPBINDPASSWORD,
         LDAPHOSTNAME,
     )
+    pytestmark = pytest.mark.certs
 except ImportError:
     Reason = 'LDAP* variable are not setup in config.py'
     # comment pytestmark for development testing with --dev-test

tests/api2/test_25_kubernetes_passthrough_test.py

@@ -15,6 +15,7 @@
 
 
 APP_NAME = 'syncthing'
+pytestmark = pytest.mark.apps
 
 
 @pytest.mark.dependency(name='default_kubernetes_cluster')

tests/api2/test_260_iscsi.py

@@ -30,6 +30,7 @@
 zvol_name = f"ds{digit}"
 zvol = f'{pool_name}/{zvol_name}'
 zvol_url = zvol.replace('/', '%2F')
+pytestmark = pytest.mark.iscsi
 
 
 def waiting_for_iscsi_to_disconnect(base_target, wait):

tests/api2/test_261_iscsi_cmd.py

@@ -26,6 +26,8 @@
 from protocols import (initiator_name_supported, iscsi_scsi_connection,
                        isns_connection)
 
+pytestmark = pytest.mark.iscsi
+
 if ha and "virtual_ip" in os.environ:
     from auto_config import password, user
     ip = os.environ["virtual_ip"]

tests/api2/test_275_ldap.py

@@ -28,6 +28,7 @@
         LDAPUSER,
         LDAPPASSWORD
     )
+    pytestmark = pytest.mark.ds
 except ImportError:
     Reason = 'LDAP* variable are not setup in config.py'
     pytestmark = pytest.mark.skipif(True, reason=Reason)

tests/api2/test_278_freeipa.py

@@ -24,6 +24,7 @@
         FREEIPA_BINDPW,
         FREEIPA_HOSTNAME,
     )
+    pytestmark = pytest.mark.ds
 except ImportError:
     Reason = 'FREEIPA* variable are not setup in config.py'
     pytestmark = pytest.mark.skipif(True, reason=Reason)

tests/api2/test_300_nfs.py

@@ -29,6 +29,8 @@
     ip = os.environ["virtual_ip"]
 else:
     from auto_config import ip
+
+pytestmark = pytest.mark.nfs
 MOUNTPOINT = f"/tmp/nfs-{hostname}"
 dataset = f"{pool_name}/nfs"
 dataset_url = dataset.replace('/', '%2F')

tests/api2/test_330_pool_acltype.py

@@ -12,6 +12,8 @@
 test1_dataset = f'{pool_name}/test1'
 dataset_url = test1_dataset.replace("/", "%2F")
 
+pytestmark = pytest.mark.zfs
+
 
 def test_01_verify_default_acltype_from_pool_dataset_with_api(request):
     results = GET(f'/pool/dataset/id/{pool_name}/')

tests/api2/test_340_pool_dataset.py

@@ -15,6 +15,7 @@
 from middlewared.test.integration.assets.pool import dataset as dataset_asset
 from middlewared.test.integration.utils import call
 
+pytestmark = pytest.mark.zfs
 dataset = f'{pool_name}/dataset1'
 dataset_url = dataset.replace('/', '%2F')
 zvol = f'{pool_name}/zvol1'

tests/api2/test_341_pool_dataset_encryption.py

@@ -22,6 +22,7 @@
 dataset_url = dataset.replace('/', '%2F')
 child_dataset = f'{dataset}/child'
 child_dataset_url = child_dataset.replace('/', '%2F')
+pytestmark = pytest.mark.zfs
 
 
 @pytest.mark.dependency(name="CREATED_POOL")

tests/api2/test_344_acl_templates.py

@@ -9,6 +9,7 @@
 from functions import POST, GET, PUT, DELETE
 from auto_config import pool_name
 
+pytestmark = pytest.mark.fs
 
 @pytest.mark.dependency(name="ACLTEMPLATE_DATASETS_CREATED")
 @pytest.mark.parametrize('acltype', ['NFSV4', 'POSIX'])

tests/api2/test_345_acl_nfs4.py

@@ -32,6 +32,7 @@
 
 ACL_USER = 'acluser'
 ACL_PWD = ''.join(secrets.choice(string.ascii_letters + string.digits) for i in range(10))
+pytestmark = [pytest.mark.fs, pytest.mark.slow]
 
 base_permset = {
     "READ_DATA": False,

tests/api2/test_347_posix_mode.py

@@ -12,6 +12,7 @@
 from auto_config import ip, pool_name, user, password
 from pytest_dependency import depends
 
+pytestmark = [pytest.mark.fs, pytest.mark.slow]
 MODE_DATASET = f'{pool_name}/modetest'
 dataset_url = MODE_DATASET.replace('/', '%2F')