NAS-131387 / 25.04 / Add integration tests for catalogs (#14575)

* Add basic test to verify catalog

* Add some tests to check catalog sync

* Add some more sanity tests

* Minor tweaks

* Properly wait for catalog to sync on docker.update

src/middlewared/middlewared/plugins/docker/fs_manage.py

@@ -18,7 +18,7 @@ async def common_func(self, mount):
                     await self.middleware.call('zfs.dataset.mount', docker_ds, {'recursive': True, 'force_mount': True})
                 else:
                     await self.middleware.call('zfs.dataset.umount', docker_ds, {'force': True})
-                await self.middleware.call('catalog.sync')
+                return await self.middleware.call('catalog.sync')
             except Exception as e:
                 await self.middleware.call(
                     'docker.state.set_status', Status.FAILED.value,
@@ -27,10 +27,10 @@ async def common_func(self, mount):
                 raise
 
     async def mount(self):
-        await self.common_func(True)
+        return await self.common_func(True)
 
     async def umount(self):
-        await self.common_func(False)
+        return await self.common_func(False)
 
     async def ix_apps_is_mounted(self, dataset_to_check=None):
         """

src/middlewared/middlewared/plugins/docker/state_setup.py

@@ -75,7 +75,9 @@ async def status_change(self):
 
         await self.create_update_docker_datasets(config['dataset'])
         # Docker dataset would not be mounted at this point, so we will explicitly mount them now
-        await self.middleware.call('docker.fs_manage.mount')
+        catalog_sync_job = await self.middleware.call('docker.fs_manage.mount')
+        if catalog_sync_job:
+            await catalog_sync_job.wait()
         await self.middleware.call('docker.state.start_service')
         self.middleware.create_task(self.middleware.call('docker.state.periodic_check'))
 

src/middlewared/middlewared/plugins/docker/update.py

@@ -96,15 +96,19 @@ async def do_update(self, job, data):
                 except Exception as e:
                     raise CallError(f'Failed to stop docker service: {e}')
 
+                catalog_sync_job = None
                 try:
-                    await self.middleware.call('docker.fs_manage.umount')
+                    catalog_sync_job = await self.middleware.call('docker.fs_manage.umount')
                 except CallError as e:
                     # We handle this specially, if for whatever reason ix-apps dataset is not there,
                     # we don't make it fatal to change pools etc - however if some dataset other then
                     # boot pool is mounted at ix-apps dir, then we will error out as it's a problem
                     # and needs to be fixed before we can proceed
                     if e.errno != errno.ENOENT or await self.middleware.call('docker.fs_manage.ix_apps_is_mounted'):
                         raise
+                finally:
+                    if catalog_sync_job:
+                        await catalog_sync_job.wait()
 
                 await self.middleware.call('docker.state.set_status', Status.UNCONFIGURED.value)
 

src/middlewared/middlewared/test/integration/utils/docker.py

@@ -3,6 +3,7 @@
 
 IX_APPS_DIR_NAME = '.ix-apps'
 IX_APPS_MOUNT_PATH: str = os.path.join('/mnt', IX_APPS_DIR_NAME)
+IX_APPS_CATALOG_PATH: str = os.path.join(IX_APPS_MOUNT_PATH, 'truenas_catalog')
 
 DOCKER_DATASET_PROPS = {
     'aclmode': 'discard',

tests/api2/test_catalogs.py

@@ -0,0 +1,86 @@
+import os.path
+
+import pytest
+
+from middlewared.test.integration.assets.pool import another_pool
+from middlewared.test.integration.utils import call
+from middlewared.test.integration.utils.docker import IX_APPS_CATALOG_PATH
+
+
+@pytest.fixture(scope='module')
+def docker_pool(request):
+    with another_pool() as pool:
+        yield pool['name']
+
+
+@pytest.mark.dependency(name='unconfigure_apps')
+def test_unconfigure_apps():
+    config = call('docker.update', {'pool': None}, job=True)
+    assert config['pool'] is None, config
+
+
+@pytest.mark.dependency(depends=['unconfigure_apps'])
+def test_catalog_sync():
+    call('catalog.sync', job=True)
+    assert call('catalog.synced') is True
+
+
+@pytest.mark.dependency(depends=['unconfigure_apps'])
+def test_catalog_cloned_location():
+    config = call('catalog.config')
+    assert config['location'] == '/var/run/middleware/ix-apps/catalogs', config
+
+
+@pytest.mark.dependency(depends=['unconfigure_apps'])
+def test_apps_are_being_reported():
+    assert call('app.available', [], {'count': True}) != 0
+
+
+@pytest.mark.dependency(name='docker_setup')
+def test_docker_setup(docker_pool):
+    config = call('docker.update', {'pool': docker_pool}, job=True)
+    assert config['pool'] == docker_pool, config
+
+
+@pytest.mark.dependency(depends=['docker_setup'])
+def test_catalog_synced_properly():
+    assert call('catalog.synced') is True
+
+
+@pytest.mark.dependency(depends=['docker_setup'])
+def test_catalog_sync_location():
+    assert call('catalog.config')['location'] == IX_APPS_CATALOG_PATH
+
+
+@pytest.mark.dependency(depends=['docker_setup'])
+def test_catalog_location_existence():
+    docker_config = call('docker.config')
+    assert docker_config['pool'] is not None
+
+    assert call('filesystem.statfs', IX_APPS_CATALOG_PATH)['source'] == os.path.join(
+        docker_config['dataset'], 'truenas_catalog'
+    )
+
+
+@pytest.mark.dependency(depends=['docker_setup'])
+def test_apps_are_being_reported_after_docker_setup():
+    assert call('app.available', [], {'count': True}) != 0
+
+
+@pytest.mark.dependency(depends=['docker_setup'])
+def test_categories_are_being_reported():
+    assert len(call('app.categories')) != 0
+
+
+@pytest.mark.dependency(depends=['docker_setup'])
+def test_app_version_details():
+    app_details = call('catalog.get_app_details', 'plex', {'train': 'stable'})
+    assert app_details['name'] == 'plex', app_details
+
+    assert len(app_details['versions']) != 0, app_details
+
+
+@pytest.mark.dependency(depends=['docker_setup'])
+def test_unconfigure_apps_after_setup():
+    config = call('docker.update', {'pool': None}, job=True)
+    assert config['pool'] is None, config