From a9e112a1c0f7767f81aca6a9dcb4a7fd857dce9f Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 11 Oct 2021 13:49:01 +0100 Subject: [PATCH] Made GeonodeUnifiedHarvesterWorker the default harvester for geonode remotes --- geonode/harvesting/config.py | 2 - ...7_convert_geonode_harvesters_to_unified.py | 37 +++++++++++++++++++ .../0048_alter_harvester_harvester_type.py | 18 +++++++++ geonode/harvesting/tests/test_admin.py | 2 +- .../harvesting/tests/test_api_serializers.py | 11 +++--- 5 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 geonode/harvesting/migrations/0047_convert_geonode_harvesters_to_unified.py create mode 100644 geonode/harvesting/migrations/0048_alter_harvester_harvester_type.py diff --git a/geonode/harvesting/config.py b/geonode/harvesting/config.py index 2a60fa777bb..d03bb7a04d5 100644 --- a/geonode/harvesting/config.py +++ b/geonode/harvesting/config.py @@ -30,8 +30,6 @@ _DEFAULT_HARVESTERS: typing.Final = [ "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker", - "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester", - "geonode.harvesting.harvesters.geonodeharvester.GeonodeCurrentHarvester", "geonode.harvesting.harvesters.wms.OgcWmsHarvester", ] diff --git a/geonode/harvesting/migrations/0047_convert_geonode_harvesters_to_unified.py b/geonode/harvesting/migrations/0047_convert_geonode_harvesters_to_unified.py new file mode 100644 index 00000000000..c15a94dd437 --- /dev/null +++ b/geonode/harvesting/migrations/0047_convert_geonode_harvesters_to_unified.py @@ -0,0 +1,37 @@ +# Generated by Django 3.2.7 on 2021-10-11 11:51 + +from django.db import migrations + +_INDIVIDUAL_HARVESTER_PATHS = [ + "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester", + "geonode.harvesting.harvesters.geonodeharvester.GeonodeCurrentHarvester", +] +_UNIFIED_HARVESTER_PATH = "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" + + +def convert_geonode_harvesters_to_unified(apps, schema_editor): + """Convert GeoNode harvesters to use the unified harvester""" + harvester_model = apps.get_model('harvesting', 'Harvester') + for harvester in harvester_model.objects.all(): + if harvester.harvester_type in _INDIVIDUAL_HARVESTER_PATHS: + harvester.harvester_type = _UNIFIED_HARVESTER_PATH + harvester.save() + + +def reverse_harvester_conversion(apps, schema_editor): + harvester_model = apps.get_model('harvesting', 'Harvester') + for harvester in harvester_model.objects.all(): + if harvester.harvester_type == _UNIFIED_HARVESTER_PATH: + harvester.harvester_type = _INDIVIDUAL_HARVESTER_PATHS[0] + harvester.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0046_alter_harvester_harvester_type'), + ] + + operations = [ + migrations.RunPython(convert_geonode_harvesters_to_unified, reverse_code=reverse_harvester_conversion), + ] diff --git a/geonode/harvesting/migrations/0048_alter_harvester_harvester_type.py b/geonode/harvesting/migrations/0048_alter_harvester_harvester_type.py new file mode 100644 index 00000000000..f891cffaded --- /dev/null +++ b/geonode/harvesting/migrations/0048_alter_harvester_harvester_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.7 on 2021-10-11 12:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('harvesting', '0047_convert_geonode_harvesters_to_unified'), + ] + + operations = [ + migrations.AlterField( + model_name='harvester', + name='harvester_type', + field=models.CharField(choices=[('geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker', 'geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker'), ('geonode.harvesting.harvesters.wms.OgcWmsHarvester', 'geonode.harvesting.harvesters.wms.OgcWmsHarvester')], default='geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker', help_text='Harvester class used to perform harvesting sessions. New harvester types can be added by an admin by changing the main GeoNode `settings.py` file', max_length=255), + ), + ] diff --git a/geonode/harvesting/tests/test_admin.py b/geonode/harvesting/tests/test_admin.py index 6768073c2b0..fd3975a4ad4 100644 --- a/geonode/harvesting/tests/test_admin.py +++ b/geonode/harvesting/tests/test_admin.py @@ -15,7 +15,7 @@ class HarvesterAdminTestCase(GeoNodeBaseTestSupport): - harvester_type = 'geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester' + harvester_type = 'geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker' def setUp(self): self.factory = RequestFactory() diff --git a/geonode/harvesting/tests/test_api_serializers.py b/geonode/harvesting/tests/test_api_serializers.py index 7bfb5a405b7..c6b74b96cf4 100644 --- a/geonode/harvesting/tests/test_api_serializers.py +++ b/geonode/harvesting/tests/test_api_serializers.py @@ -21,7 +21,7 @@ class BriefHarvesterSerializerTestCase(GeoNodeBaseTestSupport): remote_url = 'test.com' name = 'This is geonode harvester' user = get_user_model().objects.get(username='AnonymousUser') - harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" @classmethod def setUpTestData(cls): @@ -48,7 +48,7 @@ class HarvesterSerializerTestCase(GeoNodeBaseTestSupport): remote_url = 'test.com' name = 'This is geonode harvester' user = get_user_model().objects.get(username='AnonymousUser') - harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" @classmethod def setUpTestData(cls): @@ -76,7 +76,7 @@ def test_validate_also_validates_worker_specific_config(self, mock_validate_conf "name": "phony", "remote_url": "http://fake.com", "user": 1, - "harvester_type": "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester", + "harvester_type": "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker", "harvester_type_specific_configuration": {"something": "fake config"}, } @@ -125,7 +125,6 @@ def test_create(self): "remote_url": "http://fake.com", "user": 1, } - request = _REQUEST_FACTORY.post("/api/v2/harvesters/") request.user = self.user @@ -237,7 +236,7 @@ def setUpTestData(cls): remote_url = 'test.com' name = 'This is geonode harvester' user = get_user_model().objects.get(username='AnonymousUser') - harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + harvester_type = "geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" cls.harvester = models.Harvester.objects.create( remote_url=remote_url, name=name, @@ -270,7 +269,7 @@ def setUpTestData(cls): remote_url='test.com', name='This is geonode harvester', default_owner=get_user_model().objects.get(username='AnonymousUser'), - harvester_type="geonode.harvesting.harvesters.geonodeharvester.GeonodeLegacyHarvester" + harvester_type="geonode.harvesting.harvesters.geonodeharvester.GeonodeUnifiedHarvesterWorker" ) cls.harvestable_resource = models.HarvestableResource.objects.create( unique_identifier=cls.unique_identifier,