From c13e63ec63977e3c2404271d145f4a9643b583ec Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 22 Apr 2024 09:49:35 -0400 Subject: [PATCH 01/22] base import export code in admin --- src/Pipfile | 1 + src/registrar/admin.py | 52 +++++++++++++++++++++++++++++--- src/registrar/config/settings.py | 2 ++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/Pipfile b/src/Pipfile index 9366423f1a..0c0e845003 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -32,6 +32,7 @@ fred-epplib = {git = "https://github.com/cisagov/epplib.git", ref = "master"} pyzipper="*" tblib = "*" django-admin-multiple-choice-list-filter = "*" +django-import-export = "*" [dev-packages] django-debug-toolbar = "*" diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 31f031456c..7fbf4dff4a 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -28,12 +28,20 @@ from django.utils.html import escape from django.contrib.auth.forms import UserChangeForm, UsernameField from django_admin_multiple_choice_list_filter.list_filters import MultipleChoiceListFilter +from import_export import resources +from import_export.admin import ImportExportModelAdmin from django.utils.translation import gettext_lazy as _ logger = logging.getLogger(__name__) +class UserResource(resources.ModelResource): + + class Meta: + model = models.User + + class MyUserAdminForm(UserChangeForm): """This form utilizes the custom widget for its class's ManyToMany UIs. @@ -468,9 +476,11 @@ class UserContactInline(admin.StackedInline): model = models.Contact -class MyUserAdmin(BaseUserAdmin): +class MyUserAdmin(BaseUserAdmin, ImportExportModelAdmin): """Custom user admin class to use our inlines.""" + resource_classes = [UserResource] + form = MyUserAdminForm class Meta: @@ -848,9 +858,17 @@ def response_change(self, request, obj): return response -class UserDomainRoleAdmin(ListHeaderAdmin): +class UserDomainRoleResource(resources.ModelResource): + + class Meta: + model = models.UserDomainRole + + +class UserDomainRoleAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom user domain role admin class.""" + resource_classes = [UserDomainRoleResource] + class Meta: """Contains meta information about this class""" @@ -931,9 +949,17 @@ class Meta: change_form_template = "django/admin/email_clipboard_change_form.html" -class DomainInformationAdmin(ListHeaderAdmin): +class DomainInformationResource(resources.ModelResource): + + class Meta: + model = models.DomainInformation + + +class DomainInformationAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Customize domain information admin class.""" + resource_classes = [DomainInformationResource] + form = DomainInformationAdminForm # Columns @@ -1060,9 +1086,17 @@ def get_readonly_fields(self, request, obj=None): return readonly_fields # Read-only fields for analysts -class DomainRequestAdmin(ListHeaderAdmin): +class DomainRequestResource(resources.ModelResource): + + class Meta: + model = models.DomainRequest + + +class DomainRequestAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom domain requests admin class.""" + resource_classes = [DomainRequestResource] + form = DomainRequestAdminForm change_form_template = "django/admin/domain_request_change_form.html" @@ -1590,9 +1624,17 @@ def get_readonly_fields(self, request, obj=None): return DomainInformationAdmin.get_readonly_fields(self, request, obj=None) -class DomainAdmin(ListHeaderAdmin): +class DomainResource(resources.ModelResource): + + class Meta: + model = models.Domain + + +class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom domain admin class to add extra buttons.""" + resource_classes = [DomainResource] + class ElectionOfficeFilter(admin.SimpleListFilter): """Define a custom filter for is_election_board""" diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 54b65e83e2..62e6768390 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -148,6 +148,8 @@ "corsheaders", # library for multiple choice filters in django admin "django_admin_multiple_choice_list_filter", + # library for export and import of data + 'import_export', ] # Middleware are routines for processing web requests. From 2514caa3d838afa0ed785730fc5409f1d7ac2ec6 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 22 Apr 2024 10:35:40 -0400 Subject: [PATCH 02/22] merged freezing of django-fsm --- src/Pipfile.lock | 133 ++++++++++++++++++++++++++++++++++++++++++- src/requirements.txt | 10 ++++ 2 files changed, 141 insertions(+), 2 deletions(-) diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 5940f455ed..e485a5202a 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ce10883aef7e1ce10421d99b3ac35ebf419857a3fe468f0e2d93785f4323eaa8" + "sha256": "99cf9a4f3912639c02105889046a9eede7a29822fab6f9a04ca25f95e29513c0" }, "pipfile-spec": 6, "requires": {}, @@ -272,6 +272,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.7.1" }, + "diff-match-patch": { + "hashes": [ + "sha256:953019cdb9c9d2c9e47b5b12bcff3cf4746fc4598eb406076fa1fc27e6a1f15c", + "sha256:dce43505fb7b1b317de7195579388df0746d90db07015ed47a85e5e44930ef93" + ], + "markers": "python_version >= '3.7'", + "version": "==20230430" + }, "dj-database-url": { "hashes": [ "sha256:04bc34b248d4c21aaa13e4ab419ae6575ef5f10f3df735ce7da97722caa356e0", @@ -352,6 +360,15 @@ "index": "pypi", "version": "==2.8.1" }, + "django-import-export": { + "hashes": [ + "sha256:2eac09e8cec8670f36e24314760448011ad23c51e8fb930d55f50d0c3c926da0", + "sha256:4deabc557801d368093608c86fd0f4831bc9540e2ea41ca2f023e2efb3eb6f48" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==3.3.8" + }, "django-login-required-middleware": { "hashes": [ "sha256:847ae9a69fd7a07618ed53192b3c06946af70a0caf6d0f4eb40a8f37593cd970" @@ -390,6 +407,14 @@ "markers": "python_version >= '3.8'", "version": "==11.0.0" }, + "et-xmlfile": { + "hashes": [ + "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c", + "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada" + ], + "markers": "python_version >= '3.6'", + "version": "==1.1.0" + }, "faker": { "hashes": [ "sha256:34b947581c2bced340c39b35f89dbfac4f356932cfff8fe893bde854903f0e6e", @@ -725,6 +750,12 @@ "markers": "python_version >= '3.8'", "version": "==1.3.3" }, + "markuppy": { + "hashes": [ + "sha256:1adee2c0a542af378fe84548ff6f6b0168f3cb7f426b46961038a2bcfaad0d5f" + ], + "version": "==1.14" + }, "markupsafe": { "hashes": [ "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", @@ -799,6 +830,13 @@ "markers": "python_version >= '3.8'", "version": "==3.21.1" }, + "odfpy": { + "hashes": [ + "sha256:db766a6e59c5103212f3cc92ec8dd50a0f3a02790233ed0b52148b70d3c438ec", + "sha256:fc3b8d1bc098eba4a0fda865a76d9d1e577c4ceec771426bcb169a82c5e9dfe0" + ], + "version": "==1.4.1" + }, "oic": { "hashes": [ "sha256:385a1f64bb59519df1e23840530921bf416740240f505ea6d161e331d3d39fad", @@ -808,6 +846,13 @@ "markers": "python_version ~= '3.7'", "version": "==1.6.1" }, + "openpyxl": { + "hashes": [ + "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184", + "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5" + ], + "version": "==3.1.2" + }, "orderedmultidict": { "hashes": [ "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", @@ -1080,6 +1125,62 @@ "markers": "python_version >= '3.8'", "version": "==1.0.1" }, + "pyyaml": { + "hashes": [ + "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", + "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", + "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", + "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", + "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", + "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", + "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", + "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", + "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", + "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", + "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", + "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", + "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", + "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", + "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", + "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", + "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", + "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", + "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", + "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", + "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", + "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", + "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", + "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", + "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", + "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", + "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", + "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", + "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", + "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef", + "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", + "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", + "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", + "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", + "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", + "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", + "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", + "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", + "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", + "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", + "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", + "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", + "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", + "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", + "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", + "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", + "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", + "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", + "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", + "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", + "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" + ], + "version": "==6.0.1" + }, "pyzipper": { "hashes": [ "sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc", @@ -1130,6 +1231,21 @@ "markers": "python_version >= '3.8'", "version": "==0.5.0" }, + "tablib": { + "extras": [ + "html", + "ods", + "xls", + "xlsx", + "yaml" + ], + "hashes": [ + "sha256:9821caa9eca6062ff7299fa645e737aecff982e6b2b42046928a6413c8dabfd9", + "sha256:f6661dfc45e1d4f51fa8a6239f9c8349380859a5bfaa73280645f046d6c96e33" + ], + "markers": "python_version >= '3.8'", + "version": "==3.5.0" + }, "tblib": { "hashes": [ "sha256:80a6c77e59b55e83911e1e607c649836a69c103963c5f28a46cbeef44acf8129", @@ -1165,6 +1281,20 @@ "markers": "python_version >= '3.8'", "version": "==6.6.0" }, + "xlrd": { + "hashes": [ + "sha256:6a33ee89877bd9abc1158129f6e94be74e2679636b8a205b43b85206c3f0bbdd", + "sha256:f72f148f54442c6b056bf931dbc34f986fd0c3b0b6b5a58d013c9aef274d0c88" + ], + "version": "==2.0.1" + }, + "xlwt": { + "hashes": [ + "sha256:a082260524678ba48a297d922cc385f58278b8aa68741596a87de01a9c628b2e", + "sha256:c59912717a9b28f1a3c2a98fd60741014b06b043936dcecbc113eaaada156c88" + ], + "version": "==1.3.0" + }, "zope.event": { "hashes": [ "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26", @@ -1589,7 +1719,6 @@ "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" ], - "markers": "python_version >= '3.6'", "version": "==6.0.1" }, "rich": { diff --git a/src/requirements.txt b/src/requirements.txt index 3c005f1622..067a174e9e 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -10,6 +10,7 @@ cffi==1.16.0; platform_python_implementation != 'PyPy' charset-normalizer==3.3.2; python_full_version >= '3.7.0' cryptography==42.0.5; python_version >= '3.7' defusedxml==0.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +diff-match-patch==20230430; python_version >= '3.7' dj-database-url==2.1.0 dj-email-url==1.0.6 django==4.2.10; python_version >= '3.8' @@ -20,10 +21,12 @@ django-cache-url==3.4.5 django-cors-headers==4.3.1; python_version >= '3.8' django-csp==3.8 django-fsm==2.8.1 +django-import-export==3.3.8; python_version >= '3.8' django-login-required-middleware==0.9.0 django-phonenumber-field[phonenumberslite]==7.3.0; python_version >= '3.8' django-widget-tweaks==1.5.0; python_version >= '3.8' environs[django]==11.0.0; python_version >= '3.8' +et-xmlfile==1.1.0; python_version >= '3.6' faker==24.11.0; python_version >= '3.8' fred-epplib@ git+https://github.com/cisagov/epplib.git@d56d183f1664f34c40ca9716a3a9a345f0ef561c furl==2.1.3 @@ -35,9 +38,12 @@ idna==3.7; python_version >= '3.5' jmespath==1.0.1; python_version >= '3.7' lxml==5.2.1; python_version >= '3.6' mako==1.3.3; python_version >= '3.8' +markuppy==1.14 markupsafe==2.1.5; python_version >= '3.7' marshmallow==3.21.1; python_version >= '3.8' +odfpy==1.4.1 oic==1.6.1; python_version ~= '3.7' +openpyxl==3.1.2 orderedmultidict==1.0.1 packaging==24.0; python_version >= '3.7' phonenumberslite==8.13.35 @@ -50,15 +56,19 @@ pydantic-settings==2.2.1; python_version >= '3.8' pyjwkest==1.4.2 python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' python-dotenv==1.0.1; python_version >= '3.8' +pyyaml==6.0.1 pyzipper==0.3.6; python_version >= '3.4' requests==2.31.0; python_version >= '3.7' s3transfer==0.10.1; python_version >= '3.8' setuptools==69.5.1; python_version >= '3.8' six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' sqlparse==0.5.0; python_version >= '3.8' +tablib[html,ods,xls,xlsx,yaml]==3.5.0; python_version >= '3.8' tblib==3.0.0; python_version >= '3.8' typing-extensions==4.11.0; python_version >= '3.8' urllib3==2.2.1; python_version >= '3.8' whitenoise==6.6.0; python_version >= '3.8' +xlrd==2.0.1 +xlwt==1.3.0 zope.event==5.0; python_version >= '3.7' zope.interface==6.3; python_version >= '3.7' From 18658d027b2c885d92a8965a008501cf76dbb497 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 2 May 2024 08:29:38 -0400 Subject: [PATCH 03/22] wip, hardcoded with lots of debugging, needs to be generalized --- src/registrar/admin.py | 313 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 311 insertions(+), 2 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 40cecc3670..3219be625c 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,17 +1,21 @@ from datetime import date import logging import copy +import traceback +import warnings from django import forms from django.db.models import Value, CharField, Q from django.db.models.functions import Concat, Coalesce +from django.db.transaction import TransactionManagementError from django.http import HttpResponseRedirect from django.shortcuts import redirect -from django_fsm import get_available_FIELD_transitions +from django_fsm import get_available_FIELD_transitions, FSMField from django.contrib import admin, messages from django.contrib.auth.admin import UserAdmin as BaseUserAdmin from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError from django.urls import reverse from dateutil.relativedelta import relativedelta # type: ignore from epplibwrapper.errors import ErrorCode, RegistryError @@ -28,7 +32,8 @@ from django.utils.html import escape from django.contrib.auth.forms import UserChangeForm, UsernameField from django_admin_multiple_choice_list_filter.list_filters import MultipleChoiceListFilter -from import_export import resources +from import_export import resources, widgets +from import_export.results import RowResult from import_export.admin import ImportExportModelAdmin from django.utils.translation import gettext_lazy as _ @@ -40,6 +45,42 @@ class UserResource(resources.ModelResource): class Meta: model = models.User + import_id_fields = ('id', 'username',) + + def import_data( + self, + dataset, + dry_run=False, + raise_errors=False, + use_transactions=None, + collect_failed_rows=False, + rollback_on_validation_errors=False, + **kwargs + ): + logger.info("in import_data") + logger.info(dataset) + return super().import_data(dataset,dry_run,raise_errors,use_transactions,collect_failed_rows,rollback_on_validation_errors,**kwargs) + + + def before_import(self, dataset, using_transactions, dry_run, **kwargs): + logger.info("in before_import") + + def import_row( + self, + row, + instance_loader, + using_transactions=True, + dry_run=False, + raise_errors=None, + **kwargs + ): + logger.info("in import_row") + logger.info(row) + return super().import_row(row,instance_loader,using_transactions,dry_run,raise_errors,**kwargs) + + def after_import_row(self, row, row_result, **kwargs): + logger.info(row_result.original) + logger.info(row_result.instance) class MyUserAdminForm(UserChangeForm): @@ -1630,8 +1671,276 @@ def get_readonly_fields(self, request, obj=None): class DomainResource(resources.ModelResource): + WIDGETS_MAP = { + "ManyToManyField": "get_m2m_widget", + "OneToOneField": "get_fk_widget", + "ForeignKey": "get_fk_widget", + "CharField": widgets.CharWidget, + "DecimalField": widgets.DecimalWidget, + "DateTimeField": widgets.DateTimeWidget, + "DateField": widgets.DateWidget, + "TimeField": widgets.TimeWidget, + "DurationField": widgets.DurationWidget, + "FloatField": widgets.FloatWidget, + "IntegerField": widgets.IntegerWidget, + "PositiveIntegerField": widgets.IntegerWidget, + "BigIntegerField": widgets.IntegerWidget, + "PositiveSmallIntegerField": widgets.IntegerWidget, + "SmallIntegerField": widgets.IntegerWidget, + "SmallAutoField": widgets.IntegerWidget, + "AutoField": widgets.IntegerWidget, + "BigAutoField": widgets.IntegerWidget, + "NullBooleanField": widgets.BooleanWidget, + "BooleanField": widgets.BooleanWidget, + "JSONField": widgets.JSONWidget, + } + class Meta: model = models.Domain + # exclude = ('state') + + + def import_row( + self, + row, + instance_loader, + using_transactions=True, + dry_run=False, + raise_errors=None, + **kwargs + ): + """ + Imports data from ``tablib.Dataset``. Refer to :doc:`import_workflow` + for a more complete description of the whole import process. + + :param row: A ``dict`` of the row to import + + :param instance_loader: The instance loader to be used to load the row + + :param using_transactions: If ``using_transactions`` is set, a transaction + is being used to wrap the import + + :param dry_run: If ``dry_run`` is set, or error occurs, transaction + will be rolled back. + """ + if raise_errors is not None: + warnings.warn( + "raise_errors argument is deprecated and " + "will be removed in a future release.", + category=DeprecationWarning, + ) + + logger.info("in import_row") + skip_diff = self._meta.skip_diff + row_result = self.get_row_result_class()() + if self._meta.store_row_values: + row_result.row_values = row + original = None + try: + self.before_import_row(row, **kwargs) + logger.info("after before_import_row") + instance, new = self.get_or_init_instance(instance_loader, row) + logger.info("after get_or_init_instance") + logger.info(type(instance)) + for f in sorted(instance._meta.fields): + if callable(getattr(f, "get_internal_type", None)): + internal_type = f.get_internal_type() + logger.info(f"{f.name} {internal_type} {type(f)}") + logger.info(f"type == fsmfield: {isinstance(f, FSMField)}") + for attr_name, attr_value in vars(instance).items(): + logger.info(f"{attr_name}: {attr_value}") + if isinstance(attr_value, FSMField): + logger.info(f"FSMField: {attr_name}: {attr_value}") + self.after_import_instance(instance, new, **kwargs) + if new: + row_result.import_type = RowResult.IMPORT_TYPE_NEW + else: + row_result.import_type = RowResult.IMPORT_TYPE_UPDATE + row_result.new_record = new + if not skip_diff: + original = copy.deepcopy(instance) + diff = self.get_diff_class()(self, original, new) + if self.for_delete(row, instance): + if new: + row_result.import_type = RowResult.IMPORT_TYPE_SKIP + if not skip_diff: + diff.compare_with(self, None, dry_run) + else: + row_result.import_type = RowResult.IMPORT_TYPE_DELETE + row_result.add_instance_info(instance) + if self._meta.store_instance: + row_result.instance = instance + self.delete_instance(instance, using_transactions, dry_run) + if not skip_diff: + diff.compare_with(self, None, dry_run) + else: + import_validation_errors = {} + try: + logger.info("about to import_obj") + self.import_obj(instance, row, dry_run, **kwargs) + logger.info("got past import_obj") + except ValidationError as e: + # Validation errors from import_obj() are passed on to + # validate_instance(), where they can be combined with model + # instance validation errors if necessary + import_validation_errors = e.update_error_dict( + import_validation_errors + ) + + if self.skip_row(instance, original, row, import_validation_errors): + row_result.import_type = RowResult.IMPORT_TYPE_SKIP + else: + self.validate_instance(instance, import_validation_errors) + self.save_instance(instance, new, using_transactions, dry_run) + self.save_m2m(instance, row, using_transactions, dry_run) + row_result.add_instance_info(instance) + if self._meta.store_instance: + row_result.instance = instance + if not skip_diff: + diff.compare_with(self, instance, dry_run) + if not new: + row_result.original = original + + if not skip_diff and not self._meta.skip_html_diff: + row_result.diff = diff.as_html() + self.after_import_row(row, row_result, **kwargs) + + except ValidationError as e: + row_result.import_type = RowResult.IMPORT_TYPE_INVALID + row_result.validation_error = e + except Exception as e: + row_result.import_type = RowResult.IMPORT_TYPE_ERROR + # There is no point logging a transaction error for each row + # when only the original error is likely to be relevant + if not isinstance(e, TransactionManagementError): + logger.debug(e, exc_info=e) + tb_info = traceback.format_exc() + row_result.errors.append(self.get_error_result_class()(e, tb_info, row)) + + return row_result + + # def get_fsm_fields_from_model(self): + # logger.info("in get_fsm_fields_from_model") + # fsm_fields = [] + # for f in sorted(self._meta.model._meta.fields): + # # if callable(getattr(f, "get_internal_type", None)): + # # internal_type = f.get_internal_type() + # # logger.info(f"{f.name} {internal_type} {type(f)}") + # # logger.info(f"type == fsmfield: {isinstance(f, FSMField)}") + # if isinstance(f, FSMField): + # fsm_fields.append(f.name) + # return fsm_fields + + # def get_fsm_fields_from_row(self, fsm_fields, row=None): + # fields_and_values = [] + # for f in fsm_fields: + # if f in row: + # fields_and_values.append((f, row[f])) + # return fields_and_values + + # def init_instance(self, row=None): + # logger.info("initializing instance") + # logger.info(f"row: {row}") + # #get fields which are fsm fields + # fsm_fields = self.get_fsm_fields_from_model() + # #then get field values from row and return an array of tuples + # fields_and_values = self.get_fsm_fields_from_row(fsm_fields, row) + # logger.info(fields_and_values) + # #then set those tuples to kwargs + # kwargs = dict(fields_and_values) + # #then pass kwargs to model() + # #return self._meta.model(state='ready') + # return self._meta.model(**kwargs) + # #return super().init_instance(row) + + def init_instance(self, row=None): + logger.info("Initializing instance") + logger.info(f"Row: {row}") + + # Get fields which are fsm fields + fsm_fields = [] + + for f in sorted(self._meta.model._meta.fields): + if isinstance(f, FSMField): + if row and f.name in row: + fsm_fields.append((f.name, row[f.name])) + + logger.info(f"Fsm fields: {fsm_fields}") + + # Convert fields_and_values to kwargs + kwargs = dict(fsm_fields) + + # Initialize model instance with kwargs + return self._meta.model(**kwargs) + + def get_instance(self, instance_loader, row): + """ + If all 'import_id_fields' are present in the dataset, calls + the :doc:`InstanceLoader `. Otherwise, + returns `None`. + """ + logger.info("get_instance is called") + import_id_fields = [self.fields[f] for f in self.get_import_id_fields()] + for field in import_id_fields: + if field.column_name not in row: + return + return instance_loader.get_instance(row) + + def import_data( + self, + dataset, + dry_run=False, + raise_errors=False, + use_transactions=None, + collect_failed_rows=False, + rollback_on_validation_errors=False, + **kwargs + ): + logger.info("in import_data") + logger.info(dataset) + return super().import_data(dataset,dry_run,raise_errors,use_transactions,collect_failed_rows,rollback_on_validation_errors,**kwargs) + + + def before_import(self, dataset, using_transactions, dry_run, **kwargs): + logger.info("in before_import") + + # def import_row( + # self, + # row, + # instance_loader, + # using_transactions=True, + # dry_run=False, + # raise_errors=None, + # **kwargs + # ): + # logger.info("in import_row") + # logger.info(row) + # return super().import_row(row,instance_loader,using_transactions,dry_run,raise_errors,**kwargs) + + def after_import_row(self, row, row_result, **kwargs): + logger.info(row_result.original) + logger.info(row_result.instance) + + def import_field(self, field, obj, data, is_m2m=False, **kwargs): + logger.info(f"import_field: ${field}") + logger.info(field.attribute) + logger.info(field.widget) + logger.info(type(obj)) + logger.info(obj._meta.fields) + is_fsm = False + for f in obj._meta.fields: + logger.info(f.name) + logger.info(type(f)) + if field.attribute == f.name and isinstance(f, FSMField): + is_fsm = True + # if field.attribute in sorted(obj._meta.fields): + # logger.info("in fields") + # if not isinstance(obj._meta.fields[field.attribute], FSMField): + if not is_fsm: + logger.info("not fsm field") + #if (field.attribute != 'state'): + super().import_field(field, obj, data, is_m2m, **kwargs) + logger.info("finished importing") class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin): From 011a974a1e39c3365e53fbeb68312d2e472a97bb Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 2 May 2024 13:17:04 -0400 Subject: [PATCH 04/22] abstracted into FsmModelResource --- src/registrar/admin.py | 316 ++++++----------------------------------- 1 file changed, 46 insertions(+), 270 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 3219be625c..1be5131e1f 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -41,6 +41,50 @@ logger = logging.getLogger(__name__) +class FsmModelResource(resources.ModelResource): + + def init_instance(self, row=None): + logger.info("Initializing instance") + logger.info(f"Row: {row}") + + # Get fields which are fsm fields + fsm_fields = [] + + for f in sorted(self._meta.model._meta.fields): + if isinstance(f, FSMField): + if row and f.name in row: + fsm_fields.append((f.name, row[f.name])) + + logger.info(f"Fsm fields: {fsm_fields}") + + # Convert fields_and_values to kwargs + kwargs = dict(fsm_fields) + + # Initialize model instance with kwargs + return self._meta.model(**kwargs) + + def import_field(self, field, obj, data, is_m2m=False, **kwargs): + # logger.info(f"import_field: ${field}") + # logger.info(field.attribute) + # logger.info(field.widget) + # logger.info(type(obj)) + # logger.info(obj._meta.fields) + is_fsm = False + for f in obj._meta.fields: + # logger.info(f.name) + # logger.info(type(f)) + if field.attribute == f.name and isinstance(f, FSMField): + is_fsm = True + # if field.attribute in sorted(obj._meta.fields): + # logger.info("in fields") + # if not isinstance(obj._meta.fields[field.attribute], FSMField): + if not is_fsm: + # logger.info("not fsm field") + #if (field.attribute != 'state'): + super().import_field(field, obj, data, is_m2m, **kwargs) + # logger.info("finished importing") + + class UserResource(resources.ModelResource): class Meta: @@ -1129,7 +1173,7 @@ def get_readonly_fields(self, request, obj=None): return readonly_fields # Read-only fields for analysts -class DomainRequestResource(resources.ModelResource): +class DomainRequestResource(FsmModelResource): class Meta: model = models.DomainRequest @@ -1669,278 +1713,10 @@ def get_readonly_fields(self, request, obj=None): return DomainInformationAdmin.get_readonly_fields(self, request, obj=None) -class DomainResource(resources.ModelResource): - - WIDGETS_MAP = { - "ManyToManyField": "get_m2m_widget", - "OneToOneField": "get_fk_widget", - "ForeignKey": "get_fk_widget", - "CharField": widgets.CharWidget, - "DecimalField": widgets.DecimalWidget, - "DateTimeField": widgets.DateTimeWidget, - "DateField": widgets.DateWidget, - "TimeField": widgets.TimeWidget, - "DurationField": widgets.DurationWidget, - "FloatField": widgets.FloatWidget, - "IntegerField": widgets.IntegerWidget, - "PositiveIntegerField": widgets.IntegerWidget, - "BigIntegerField": widgets.IntegerWidget, - "PositiveSmallIntegerField": widgets.IntegerWidget, - "SmallIntegerField": widgets.IntegerWidget, - "SmallAutoField": widgets.IntegerWidget, - "AutoField": widgets.IntegerWidget, - "BigAutoField": widgets.IntegerWidget, - "NullBooleanField": widgets.BooleanWidget, - "BooleanField": widgets.BooleanWidget, - "JSONField": widgets.JSONWidget, - } +class DomainResource(FsmModelResource): class Meta: model = models.Domain - # exclude = ('state') - - - def import_row( - self, - row, - instance_loader, - using_transactions=True, - dry_run=False, - raise_errors=None, - **kwargs - ): - """ - Imports data from ``tablib.Dataset``. Refer to :doc:`import_workflow` - for a more complete description of the whole import process. - - :param row: A ``dict`` of the row to import - - :param instance_loader: The instance loader to be used to load the row - - :param using_transactions: If ``using_transactions`` is set, a transaction - is being used to wrap the import - - :param dry_run: If ``dry_run`` is set, or error occurs, transaction - will be rolled back. - """ - if raise_errors is not None: - warnings.warn( - "raise_errors argument is deprecated and " - "will be removed in a future release.", - category=DeprecationWarning, - ) - - logger.info("in import_row") - skip_diff = self._meta.skip_diff - row_result = self.get_row_result_class()() - if self._meta.store_row_values: - row_result.row_values = row - original = None - try: - self.before_import_row(row, **kwargs) - logger.info("after before_import_row") - instance, new = self.get_or_init_instance(instance_loader, row) - logger.info("after get_or_init_instance") - logger.info(type(instance)) - for f in sorted(instance._meta.fields): - if callable(getattr(f, "get_internal_type", None)): - internal_type = f.get_internal_type() - logger.info(f"{f.name} {internal_type} {type(f)}") - logger.info(f"type == fsmfield: {isinstance(f, FSMField)}") - for attr_name, attr_value in vars(instance).items(): - logger.info(f"{attr_name}: {attr_value}") - if isinstance(attr_value, FSMField): - logger.info(f"FSMField: {attr_name}: {attr_value}") - self.after_import_instance(instance, new, **kwargs) - if new: - row_result.import_type = RowResult.IMPORT_TYPE_NEW - else: - row_result.import_type = RowResult.IMPORT_TYPE_UPDATE - row_result.new_record = new - if not skip_diff: - original = copy.deepcopy(instance) - diff = self.get_diff_class()(self, original, new) - if self.for_delete(row, instance): - if new: - row_result.import_type = RowResult.IMPORT_TYPE_SKIP - if not skip_diff: - diff.compare_with(self, None, dry_run) - else: - row_result.import_type = RowResult.IMPORT_TYPE_DELETE - row_result.add_instance_info(instance) - if self._meta.store_instance: - row_result.instance = instance - self.delete_instance(instance, using_transactions, dry_run) - if not skip_diff: - diff.compare_with(self, None, dry_run) - else: - import_validation_errors = {} - try: - logger.info("about to import_obj") - self.import_obj(instance, row, dry_run, **kwargs) - logger.info("got past import_obj") - except ValidationError as e: - # Validation errors from import_obj() are passed on to - # validate_instance(), where they can be combined with model - # instance validation errors if necessary - import_validation_errors = e.update_error_dict( - import_validation_errors - ) - - if self.skip_row(instance, original, row, import_validation_errors): - row_result.import_type = RowResult.IMPORT_TYPE_SKIP - else: - self.validate_instance(instance, import_validation_errors) - self.save_instance(instance, new, using_transactions, dry_run) - self.save_m2m(instance, row, using_transactions, dry_run) - row_result.add_instance_info(instance) - if self._meta.store_instance: - row_result.instance = instance - if not skip_diff: - diff.compare_with(self, instance, dry_run) - if not new: - row_result.original = original - - if not skip_diff and not self._meta.skip_html_diff: - row_result.diff = diff.as_html() - self.after_import_row(row, row_result, **kwargs) - - except ValidationError as e: - row_result.import_type = RowResult.IMPORT_TYPE_INVALID - row_result.validation_error = e - except Exception as e: - row_result.import_type = RowResult.IMPORT_TYPE_ERROR - # There is no point logging a transaction error for each row - # when only the original error is likely to be relevant - if not isinstance(e, TransactionManagementError): - logger.debug(e, exc_info=e) - tb_info = traceback.format_exc() - row_result.errors.append(self.get_error_result_class()(e, tb_info, row)) - - return row_result - - # def get_fsm_fields_from_model(self): - # logger.info("in get_fsm_fields_from_model") - # fsm_fields = [] - # for f in sorted(self._meta.model._meta.fields): - # # if callable(getattr(f, "get_internal_type", None)): - # # internal_type = f.get_internal_type() - # # logger.info(f"{f.name} {internal_type} {type(f)}") - # # logger.info(f"type == fsmfield: {isinstance(f, FSMField)}") - # if isinstance(f, FSMField): - # fsm_fields.append(f.name) - # return fsm_fields - - # def get_fsm_fields_from_row(self, fsm_fields, row=None): - # fields_and_values = [] - # for f in fsm_fields: - # if f in row: - # fields_and_values.append((f, row[f])) - # return fields_and_values - - # def init_instance(self, row=None): - # logger.info("initializing instance") - # logger.info(f"row: {row}") - # #get fields which are fsm fields - # fsm_fields = self.get_fsm_fields_from_model() - # #then get field values from row and return an array of tuples - # fields_and_values = self.get_fsm_fields_from_row(fsm_fields, row) - # logger.info(fields_and_values) - # #then set those tuples to kwargs - # kwargs = dict(fields_and_values) - # #then pass kwargs to model() - # #return self._meta.model(state='ready') - # return self._meta.model(**kwargs) - # #return super().init_instance(row) - - def init_instance(self, row=None): - logger.info("Initializing instance") - logger.info(f"Row: {row}") - - # Get fields which are fsm fields - fsm_fields = [] - - for f in sorted(self._meta.model._meta.fields): - if isinstance(f, FSMField): - if row and f.name in row: - fsm_fields.append((f.name, row[f.name])) - - logger.info(f"Fsm fields: {fsm_fields}") - - # Convert fields_and_values to kwargs - kwargs = dict(fsm_fields) - - # Initialize model instance with kwargs - return self._meta.model(**kwargs) - - def get_instance(self, instance_loader, row): - """ - If all 'import_id_fields' are present in the dataset, calls - the :doc:`InstanceLoader `. Otherwise, - returns `None`. - """ - logger.info("get_instance is called") - import_id_fields = [self.fields[f] for f in self.get_import_id_fields()] - for field in import_id_fields: - if field.column_name not in row: - return - return instance_loader.get_instance(row) - - def import_data( - self, - dataset, - dry_run=False, - raise_errors=False, - use_transactions=None, - collect_failed_rows=False, - rollback_on_validation_errors=False, - **kwargs - ): - logger.info("in import_data") - logger.info(dataset) - return super().import_data(dataset,dry_run,raise_errors,use_transactions,collect_failed_rows,rollback_on_validation_errors,**kwargs) - - - def before_import(self, dataset, using_transactions, dry_run, **kwargs): - logger.info("in before_import") - - # def import_row( - # self, - # row, - # instance_loader, - # using_transactions=True, - # dry_run=False, - # raise_errors=None, - # **kwargs - # ): - # logger.info("in import_row") - # logger.info(row) - # return super().import_row(row,instance_loader,using_transactions,dry_run,raise_errors,**kwargs) - - def after_import_row(self, row, row_result, **kwargs): - logger.info(row_result.original) - logger.info(row_result.instance) - - def import_field(self, field, obj, data, is_m2m=False, **kwargs): - logger.info(f"import_field: ${field}") - logger.info(field.attribute) - logger.info(field.widget) - logger.info(type(obj)) - logger.info(obj._meta.fields) - is_fsm = False - for f in obj._meta.fields: - logger.info(f.name) - logger.info(type(f)) - if field.attribute == f.name and isinstance(f, FSMField): - is_fsm = True - # if field.attribute in sorted(obj._meta.fields): - # logger.info("in fields") - # if not isinstance(obj._meta.fields[field.attribute], FSMField): - if not is_fsm: - logger.info("not fsm field") - #if (field.attribute != 'state'): - super().import_field(field, obj, data, is_m2m, **kwargs) - logger.info("finished importing") class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin): From 780504c9f26c96672c920d449d62073bf7225cce Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 3 May 2024 07:14:06 -0400 Subject: [PATCH 05/22] update to allow DomainRequest to import properly --- src/registrar/admin.py | 24 +------- .../models/utility/generic_helper.py | 59 ++++++++++--------- 2 files changed, 32 insertions(+), 51 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 1be5131e1f..7231c634ee 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -1,13 +1,10 @@ from datetime import date import logging import copy -import traceback -import warnings from django import forms from django.db.models import Value, CharField, Q from django.db.models.functions import Concat, Coalesce -from django.db.transaction import TransactionManagementError from django.http import HttpResponseRedirect from django.shortcuts import redirect from django_fsm import get_available_FIELD_transitions, FSMField @@ -15,7 +12,6 @@ from django.contrib.auth.admin import UserAdmin as BaseUserAdmin from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType -from django.core.exceptions import ValidationError from django.urls import reverse from dateutil.relativedelta import relativedelta # type: ignore from epplibwrapper.errors import ErrorCode, RegistryError @@ -32,8 +28,7 @@ from django.utils.html import escape from django.contrib.auth.forms import UserChangeForm, UsernameField from django_admin_multiple_choice_list_filter.list_filters import MultipleChoiceListFilter -from import_export import resources, widgets -from import_export.results import RowResult +from import_export import resources from import_export.admin import ImportExportModelAdmin from django.utils.translation import gettext_lazy as _ @@ -44,8 +39,6 @@ class FsmModelResource(resources.ModelResource): def init_instance(self, row=None): - logger.info("Initializing instance") - logger.info(f"Row: {row}") # Get fields which are fsm fields fsm_fields = [] @@ -55,8 +48,6 @@ def init_instance(self, row=None): if row and f.name in row: fsm_fields.append((f.name, row[f.name])) - logger.info(f"Fsm fields: {fsm_fields}") - # Convert fields_and_values to kwargs kwargs = dict(fsm_fields) @@ -64,25 +55,12 @@ def init_instance(self, row=None): return self._meta.model(**kwargs) def import_field(self, field, obj, data, is_m2m=False, **kwargs): - # logger.info(f"import_field: ${field}") - # logger.info(field.attribute) - # logger.info(field.widget) - # logger.info(type(obj)) - # logger.info(obj._meta.fields) is_fsm = False for f in obj._meta.fields: - # logger.info(f.name) - # logger.info(type(f)) if field.attribute == f.name and isinstance(f, FSMField): is_fsm = True - # if field.attribute in sorted(obj._meta.fields): - # logger.info("in fields") - # if not isinstance(obj._meta.fields[field.attribute], FSMField): if not is_fsm: - # logger.info("not fsm field") - #if (field.attribute != 'state'): super().import_field(field, obj, data, is_m2m, **kwargs) - # logger.info("finished importing") class UserResource(resources.ModelResource): diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index 892298967a..12ec85e57d 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -102,34 +102,37 @@ def _handle_new_instance(self): def _handle_existing_instance(self, force_update_when_no_are_changes_found=False): # == Init variables == # - # Instance is already in the database, fetch its current state - current_instance = self.sender.objects.get(id=self.instance.id) - - # Check the new and old values - generic_org_type_changed = self.instance.generic_org_type != current_instance.generic_org_type - is_election_board_changed = self.instance.is_election_board != current_instance.is_election_board - organization_type_changed = self.instance.organization_type != current_instance.organization_type - - # == Check for invalid conditions before proceeding == # - if organization_type_changed and (generic_org_type_changed or is_election_board_changed): - # Since organization type is linked with generic_org_type and election board, - # we have to update one or the other, not both. - # This will not happen in normal flow as it is not possible otherwise. - raise ValueError("Cannot update organization_type and generic_org_type simultaneously.") - elif not organization_type_changed and (not generic_org_type_changed and not is_election_board_changed): - # No changes found - if force_update_when_no_are_changes_found: - # If we want to force an update anyway, we can treat this record like - # its a new one in that we check for "None" values rather than changes. - self._handle_new_instance() - else: - # == Update the linked values == # - # Find out which field needs updating - organization_type_needs_update = generic_org_type_changed or is_election_board_changed - generic_org_type_needs_update = organization_type_changed - - # Update the field - self._update_fields(organization_type_needs_update, generic_org_type_needs_update) + try: + # Instance is already in the database, fetch its current state + current_instance = self.sender.objects.get(id=self.instance.id) + + # Check the new and old values + generic_org_type_changed = self.instance.generic_org_type != current_instance.generic_org_type + is_election_board_changed = self.instance.is_election_board != current_instance.is_election_board + organization_type_changed = self.instance.organization_type != current_instance.organization_type + + # == Check for invalid conditions before proceeding == # + if organization_type_changed and (generic_org_type_changed or is_election_board_changed): + # Since organization type is linked with generic_org_type and election board, + # we have to update one or the other, not both. + # This will not happen in normal flow as it is not possible otherwise. + raise ValueError("Cannot update organization_type and generic_org_type simultaneously.") + elif not organization_type_changed and (not generic_org_type_changed and not is_election_board_changed): + # No changes found + if force_update_when_no_are_changes_found: + # If we want to force an update anyway, we can treat this record like + # its a new one in that we check for "None" values rather than changes. + self._handle_new_instance() + else: + # == Update the linked values == # + # Find out which field needs updating + organization_type_needs_update = generic_org_type_changed or is_election_board_changed + generic_org_type_needs_update = organization_type_changed + + # Update the field + self._update_fields(organization_type_needs_update, generic_org_type_needs_update) + except: + pass def _update_fields(self, organization_type_needs_update, generic_org_type_needs_update): """ From b5fb28e1568105a0c96b63a2af76de3e19a0eeb2 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 3 May 2024 09:29:20 -0400 Subject: [PATCH 06/22] added Contacts --- src/registrar/admin.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 744babb95a..da630c04ce 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -785,9 +785,17 @@ class MyHostAdmin(AuditedAdmin): inlines = [HostIPInline] -class ContactAdmin(ListHeaderAdmin): +class ContactResource(resources.ModelResource): + + class Meta: + model = models.Contact + + +class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom contact admin class to add search.""" + resource_classes = [ContactResource] + search_fields = ["email", "first_name", "last_name"] search_help_text = "Search by first name, last name or email." list_display = [ From e32525563ec4dcc3ae71c095f2cca9b1f216780f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 3 May 2024 16:46:18 -0400 Subject: [PATCH 07/22] commented and reformatted --- src/registrar/admin.py | 59 +++++++++++--------------------- src/registrar/config/settings.py | 2 +- 2 files changed, 21 insertions(+), 40 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index da630c04ce..89f8391b3c 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -37,26 +37,43 @@ class FsmModelResource(resources.ModelResource): + """ModelResource is extended to support importing of tables which + have FSMFields. ModelResource is extended with the following changes + to existing behavior: + When new objects are to be imported, FSMFields are initialized before + the object is initialized. This is because FSMFields do not allow + direct modification. + When objects, which are to be imported, are updated, the FSMFields + are skipped.""" def init_instance(self, row=None): + """Overrides the init_instance method of ModelResource. Returns + an instance of the model, with the FSMFields already initialized + from data in the row.""" # Get fields which are fsm fields fsm_fields = [] - for f in sorted(self._meta.model._meta.fields): + for f in self._meta.model._meta.fields: if isinstance(f, FSMField): if row and f.name in row: fsm_fields.append((f.name, row[f.name])) - # Convert fields_and_values to kwargs + # Convert fsm_fields fields_and_values to kwargs kwargs = dict(fsm_fields) # Initialize model instance with kwargs return self._meta.model(**kwargs) def import_field(self, field, obj, data, is_m2m=False, **kwargs): + """Overrides the import_field method of ModelResource. If the + field being imported is an FSMField, it is not imported.""" + is_fsm = False + + # check each field in the object for f in obj._meta.fields: + # if the field is an instance of FSMField if field.attribute == f.name and isinstance(f, FSMField): is_fsm = True if not is_fsm: @@ -67,42 +84,6 @@ class UserResource(resources.ModelResource): class Meta: model = models.User - import_id_fields = ('id', 'username',) - - def import_data( - self, - dataset, - dry_run=False, - raise_errors=False, - use_transactions=None, - collect_failed_rows=False, - rollback_on_validation_errors=False, - **kwargs - ): - logger.info("in import_data") - logger.info(dataset) - return super().import_data(dataset,dry_run,raise_errors,use_transactions,collect_failed_rows,rollback_on_validation_errors,**kwargs) - - - def before_import(self, dataset, using_transactions, dry_run, **kwargs): - logger.info("in before_import") - - def import_row( - self, - row, - instance_loader, - using_transactions=True, - dry_run=False, - raise_errors=None, - **kwargs - ): - logger.info("in import_row") - logger.info(row) - return super().import_row(row,instance_loader,using_transactions,dry_run,raise_errors,**kwargs) - - def after_import_row(self, row, row_result, **kwargs): - logger.info(row_result.original) - logger.info(row_result.instance) class MyUserAdminForm(UserChangeForm): @@ -1761,7 +1742,7 @@ class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom domain admin class to add extra buttons.""" resource_classes = [DomainResource] - + class ElectionOfficeFilter(admin.SimpleListFilter): """Define a custom filter for is_election_board""" diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 062e2492e8..a18adabde5 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -149,7 +149,7 @@ # library for multiple choice filters in django admin "django_admin_multiple_choice_list_filter", # library for export and import of data - 'import_export', + "import_export", ] # Middleware are routines for processing web requests. From 80aca78194a6a63785c97580e0a5522f4df7f582 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 3 May 2024 17:16:16 -0400 Subject: [PATCH 08/22] only show import on non_production --- .../admin/import_export/change_list_import_item.html | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/registrar/templates/admin/import_export/change_list_import_item.html diff --git a/src/registrar/templates/admin/import_export/change_list_import_item.html b/src/registrar/templates/admin/import_export/change_list_import_item.html new file mode 100644 index 0000000000..b640331cba --- /dev/null +++ b/src/registrar/templates/admin/import_export/change_list_import_item.html @@ -0,0 +1,8 @@ +{% load i18n %} +{% load admin_urls %} + +{% if has_import_permission %} + {% if not IS_PRODUCTION %} +
  • {% trans "Import" %}
    • + {% endif %} +{% endif %} \ No newline at end of file From c5748c6bc1bbd896ce63c7b440c398d48aadcbf0 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 3 May 2024 18:04:02 -0400 Subject: [PATCH 09/22] added documentation --- docs/operations/import_export.md | 57 ++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 docs/operations/import_export.md diff --git a/docs/operations/import_export.md b/docs/operations/import_export.md new file mode 100644 index 0000000000..3e57e5152e --- /dev/null +++ b/docs/operations/import_export.md @@ -0,0 +1,57 @@ +# Export / Import Tables + +A means is provided to export and import individual tables from +one environment to another. This allows for replication of +production data in a development environment. Import and export +are provided through the django admin interface, through a modified +library, django-import-export. Each supported model has an Import +and an Export button on the list view. + +### Export + +When exporting models from the source environment, make sure that +no filters are selected. This will ensure that all rows of the model +are exported. Due to database dependencies, the following models +need to be exported: + +* User +* Contact +* Domain +* DomainRequest +* DomainInformation +* DomainUserRole + +### Import + +When importing into the target environment, if the target environment +is different than the source environment, it must be prepared for the +import. This involves clearing out rows in the appropriate tables so +that there are no database conflicts on import. + +#### Preparing Target Environment + +Delete all rows from tables in the following order through django admin: + +* DomainInformation +* DomainRequest +* Domain +* User (all but the current user) +* Contact + +It may not be necessary, but advisable to also remove rows from these tables: + +* Websites +* DraftDomain +* Host + +#### Importing into Target Environment + +Once target environment is prepared, files can be imported in the following +order: + +* User +* Contact +* Domain +* DomainRequest +* DomainInformation +* UserDomainRole \ No newline at end of file From 54c65a6d85ea49c9e9da101f32fc25aaf65f652d Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 6 May 2024 12:16:09 -0400 Subject: [PATCH 10/22] fixed failing tests, formatted for linter --- src/registrar/models/utility/generic_helper.py | 2 +- src/registrar/tests/test_admin.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index d1d890da4c..7d3586770d 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -131,7 +131,7 @@ def _handle_existing_instance(self, force_update_when_no_are_changes_found=False # Update the field self._update_fields(organization_type_needs_update, generic_org_type_needs_update) - except: + except self.sender.DoesNotExist: pass def _update_fields(self, organization_type_needs_update, generic_org_type_needs_update): diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index cf00994be9..97e48279dc 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -508,10 +508,9 @@ def test_short_org_name_in_domains_list(self): domain_request.approve() response = self.client.get("/admin/registrar/domain/") - # There are 4 template references to Federal (4) plus four references in the table # for our actual domain_request - self.assertContains(response, "Federal", count=36) + self.assertContains(response, "Federal", count=48) # This may be a bit more robust self.assertContains(response, 'Federal', count=1) # Now let's make sure the long description does not exist @@ -1267,7 +1266,7 @@ def test_short_org_name_in_domain_requests_list(self): response = self.client.get("/admin/registrar/domainrequest/?generic_org_type__exact=federal") # There are 2 template references to Federal (4) and two in the results data # of the request - self.assertContains(response, "Federal", count=34) + self.assertContains(response, "Federal", count=46) # This may be a bit more robust self.assertContains(response, 'Federal', count=1) # Now let's make sure the long description does not exist From af3877f2e351d2df4b25d8a98b116a1b17582d71 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 May 2024 07:35:02 -0400 Subject: [PATCH 11/22] unit tests completed --- src/registrar/tests/test_admin.py | 48 ++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 97e48279dc..d8f4975e88 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -21,6 +21,7 @@ MyHostAdmin, UserDomainRoleAdmin, VerifiedByStaffAdmin, + FsmModelResource, ) from registrar.models import ( Domain, @@ -52,7 +53,7 @@ ) from django.contrib.sessions.backends.db import SessionStore from django.contrib.auth import get_user_model -from unittest.mock import ANY, call, patch +from unittest.mock import ANY, call, patch, Mock from unittest import skip from django.conf import settings @@ -62,6 +63,42 @@ logger = logging.getLogger(__name__) +class TestFsmModelResource(TestCase): + def setUp(self): + self.resource = FsmModelResource() + + def test_init_instance(self): + """Test initializing an instance of a class with a FSM field""" + + # Mock a row with FSMField data + row_data = {'state': 'ready'} + + self.resource._meta.model = Domain + + instance = self.resource.init_instance(row=row_data) + + # Assert that the instance is initialized correctly + self.assertIsInstance(instance, Domain) + self.assertEqual(instance.state, 'ready') + + def test_import_field(self): + """Test that importing a field does not import FSM field""" + # Mock a field and object + field_mock = Mock(attribute='state') + obj_mock = Mock(_meta=Mock(fields=[Mock(name='state', spec=['name'], __class__=Mock)])) + + # Mock the super() method + super_mock = Mock() + self.resource.import_field = super_mock + + # Call the method with FSMField and non-FSMField + self.resource.import_field(field_mock, obj_mock, data={}, is_m2m=False) + + # Assert that super().import_field() is called only for non-FSMField + super_mock.assert_called_once_with(field_mock, obj_mock, data={}, is_m2m=False) + + + class TestDomainAdmin(MockEppLib, WebTest): # csrf checks do not work with WebTest. # We disable them here. TODO for another ticket. @@ -759,6 +796,15 @@ def test_analyst_deletes_domain_idempotent(self): def test_place_and_remove_hold_epp(self): raise + @override_settings(IS_PRODUCTION=True) + def test_prod_only_shows_export(self): + """Test that production environment only displays export""" + with less_console_noise(): + response = self.client.get("/admin/registrar/domain/") + self.assertContains(response, ">Export<") + # Now let's make sure the long description does not exist + self.assertNotContains(response, ">Import<") + def tearDown(self): super().tearDown() PublicContact.objects.all().delete() From 2d9f96c6d01b4759a35af31953215dc168466ef9 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 May 2024 08:10:38 -0400 Subject: [PATCH 12/22] formatted for linter --- src/registrar/tests/test_admin.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 6dbff5d53e..98ba994d88 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -80,7 +80,7 @@ def test_init_instance(self): """Test initializing an instance of a class with a FSM field""" # Mock a row with FSMField data - row_data = {'state': 'ready'} + row_data = {"state": "ready"} self.resource._meta.model = Domain @@ -88,13 +88,13 @@ def test_init_instance(self): # Assert that the instance is initialized correctly self.assertIsInstance(instance, Domain) - self.assertEqual(instance.state, 'ready') + self.assertEqual(instance.state, "ready") def test_import_field(self): """Test that importing a field does not import FSM field""" # Mock a field and object - field_mock = Mock(attribute='state') - obj_mock = Mock(_meta=Mock(fields=[Mock(name='state', spec=['name'], __class__=Mock)])) + field_mock = Mock(attribute="state") + obj_mock = Mock(_meta=Mock(fields=[Mock(name="state", spec=["name"], __class__=Mock)])) # Mock the super() method super_mock = Mock() @@ -107,7 +107,6 @@ def test_import_field(self): super_mock.assert_called_once_with(field_mock, obj_mock, data={}, is_m2m=False) - class TestDomainAdmin(MockEppLib, WebTest): # csrf checks do not work with WebTest. # We disable them here. TODO for another ticket. From aca85227ace3ef0152122e14052505333c70e73c Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 May 2024 08:18:26 -0400 Subject: [PATCH 13/22] fixed tests --- src/registrar/tests/test_admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 98ba994d88..f901d5ce0e 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -638,7 +638,7 @@ def test_short_org_name_in_domains_list(self): response = self.client.get("/admin/registrar/domain/") # There are 4 template references to Federal (4) plus four references in the table # for our actual domain_request - self.assertContains(response, "Federal", count=48) + self.assertContains(response, "Federal", count=54) # This may be a bit more robust self.assertContains(response, 'Federal', count=1) # Now let's make sure the long description does not exist @@ -1420,7 +1420,7 @@ def test_short_org_name_in_domain_requests_list(self): response = self.client.get("/admin/registrar/domainrequest/?generic_org_type__exact=federal") # There are 2 template references to Federal (4) and two in the results data # of the request - self.assertContains(response, "Federal", count=46) + self.assertContains(response, "Federal", count=52) # This may be a bit more robust self.assertContains(response, 'Federal', count=1) # Now let's make sure the long description does not exist From fcef18658cb1335e5d0802aca9bf8016974e469f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 May 2024 11:08:46 -0400 Subject: [PATCH 14/22] adding newline to a html template --- .../templates/admin/import_export/change_list_import_item.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/templates/admin/import_export/change_list_import_item.html b/src/registrar/templates/admin/import_export/change_list_import_item.html index b640331cba..8255a8ba7c 100644 --- a/src/registrar/templates/admin/import_export/change_list_import_item.html +++ b/src/registrar/templates/admin/import_export/change_list_import_item.html @@ -5,4 +5,4 @@ {% if not IS_PRODUCTION %}
  • {% trans "Import" %}
    • {% endif %} -{% endif %} \ No newline at end of file +{% endif %} From 8b1d692457a6ce66d1a5d4f2dac277d04dd4720d Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 May 2024 12:18:57 -0400 Subject: [PATCH 15/22] updated unit test --- src/registrar/tests/test_admin.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index f901d5ce0e..b14016cf88 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -92,19 +92,27 @@ def test_init_instance(self): def test_import_field(self): """Test that importing a field does not import FSM field""" - # Mock a field and object - field_mock = Mock(attribute="state") - obj_mock = Mock(_meta=Mock(fields=[Mock(name="state", spec=["name"], __class__=Mock)])) - # Mock the super() method - super_mock = Mock() - self.resource.import_field = super_mock + # Mock a FSMField and a non-FSM-field + fsm_field_mock = Mock(attribute="state", column_name="state") + field_mock = Mock(attribute="name", column_name="name") + # Mock the data + data_mock = {"state": "unknown", "name": "test"} + # Define a mock Domain + obj = Domain(state=Domain.State.UNKNOWN, name="test") + + # Mock the save() method of fields so that we can test if save is called + # save() is only supposed to be called for non FSM fields + field_mock.save = Mock() + fsm_field_mock.save = Mock() # Call the method with FSMField and non-FSMField - self.resource.import_field(field_mock, obj_mock, data={}, is_m2m=False) + self.resource.import_field(fsm_field_mock, obj, data=data_mock, is_m2m=False) + self.resource.import_field(field_mock, obj, data=data_mock, is_m2m=False) - # Assert that super().import_field() is called only for non-FSMField - super_mock.assert_called_once_with(field_mock, obj_mock, data={}, is_m2m=False) + # Assert that field.save() in super().import_field() is called only for non-FSMField + field_mock.save.assert_called_once() + fsm_field_mock.save.assert_not_called() class TestDomainAdmin(MockEppLib, WebTest): @@ -893,7 +901,6 @@ def test_prod_only_shows_export(self): with less_console_noise(): response = self.client.get("/admin/registrar/domain/") self.assertContains(response, ">Export<") - # Now let's make sure the long description does not exist self.assertNotContains(response, ">Import<") def tearDown(self): From ab5b0be46641957c2985af3dff9e8296ab844517 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 May 2024 13:23:30 -0400 Subject: [PATCH 16/22] included DraftDomain, Websites and Host --- docs/operations/import_export.md | 11 +++++---- src/registrar/admin.py | 41 ++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/docs/operations/import_export.md b/docs/operations/import_export.md index 3e57e5152e..1b413809c4 100644 --- a/docs/operations/import_export.md +++ b/docs/operations/import_export.md @@ -20,6 +20,9 @@ need to be exported: * DomainRequest * DomainInformation * DomainUserRole +* DraftDomain +* Websites +* Host ### Import @@ -37,9 +40,6 @@ Delete all rows from tables in the following order through django admin: * Domain * User (all but the current user) * Contact - -It may not be necessary, but advisable to also remove rows from these tables: - * Websites * DraftDomain * Host @@ -49,8 +49,11 @@ It may not be necessary, but advisable to also remove rows from these tables: Once target environment is prepared, files can be imported in the following order: -* User +* User (After importing User table, you need to delete all rows from Contact table before importing Contacts) * Contact +* Host +* DraftDomain +* Websites * Domain * DomainRequest * DomainInformation diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 3562869361..040039e1e5 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -54,18 +54,15 @@ def init_instance(self, row=None): from data in the row.""" # Get fields which are fsm fields - fsm_fields = [] + fsm_fields = {} for f in self._meta.model._meta.fields: if isinstance(f, FSMField): if row and f.name in row: - fsm_fields.append((f.name, row[f.name])) + fsm_fields[f.name] = row[f.name] - # Convert fsm_fields fields_and_values to kwargs - kwargs = dict(fsm_fields) - - # Initialize model instance with kwargs - return self._meta.model(**kwargs) + # Initialize model instance with fsm_fields + return self._meta.model(**fsm_fields) def import_field(self, field, obj, data, is_m2m=False, **kwargs): """Overrides the import_field method of ModelResource. If the @@ -760,9 +757,17 @@ class HostIPInline(admin.StackedInline): model = models.HostIP -class MyHostAdmin(AuditedAdmin): +class HostResource(resources.ModelResource): + + class Meta: + model = models.Host + + +class MyHostAdmin(AuditedAdmin, ImportExportModelAdmin): """Custom host admin class to use our inlines.""" + resource_classes = [HostResource] + search_fields = ["name", "domain__name"] search_help_text = "Search by domain or host name." inlines = [HostIPInline] @@ -899,9 +904,17 @@ def change_view(self, request, object_id, form_url="", extra_context=None): return super().change_view(request, object_id, form_url, extra_context=extra_context) -class WebsiteAdmin(ListHeaderAdmin): +class WebsiteResource(resources.ModelResource): + + class Meta: + model = models.Website + + +class WebsiteAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom website admin class.""" + resource_classes = [WebsiteResource] + # Search search_fields = [ "website", @@ -2139,9 +2152,17 @@ def has_change_permission(self, request, obj=None): return super().has_change_permission(request, obj) -class DraftDomainAdmin(ListHeaderAdmin): +class DraftDomainResource(resources.ModelResource): + + class Meta: + model = models.DraftDomain + + +class DraftDomainAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Custom draft domain admin class.""" + resource_classes = [DraftDomainResource] + search_fields = ["name"] search_help_text = "Search by draft domain name." From 5ee316b7afda229eb12d3bc82277e6d0ba65347b Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 May 2024 13:28:04 -0400 Subject: [PATCH 17/22] reformatted for linter --- src/registrar/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 040039e1e5..bcda7e0486 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -762,7 +762,7 @@ class HostResource(resources.ModelResource): class Meta: model = models.Host - + class MyHostAdmin(AuditedAdmin, ImportExportModelAdmin): """Custom host admin class to use our inlines.""" From 943b14fe010a4a03aa5c31bb86911355969c7221 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 7 May 2024 13:51:03 -0400 Subject: [PATCH 18/22] updated documentation --- docs/operations/import_export.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/import_export.md b/docs/operations/import_export.md index 1b413809c4..897e8a01a8 100644 --- a/docs/operations/import_export.md +++ b/docs/operations/import_export.md @@ -51,10 +51,10 @@ order: * User (After importing User table, you need to delete all rows from Contact table before importing Contacts) * Contact +* Domain * Host * DraftDomain * Websites -* Domain * DomainRequest * DomainInformation * UserDomainRole \ No newline at end of file From 55e47d03cddf7c9c6201a9b241a9cbedba57e910 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 9 May 2024 11:31:11 -0400 Subject: [PATCH 19/22] added HostIP --- docs/operations/import_export.md | 3 +++ src/registrar/admin.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/operations/import_export.md b/docs/operations/import_export.md index 897e8a01a8..e04a79cd2b 100644 --- a/docs/operations/import_export.md +++ b/docs/operations/import_export.md @@ -23,6 +23,7 @@ need to be exported: * DraftDomain * Websites * Host +* HostIP ### Import @@ -42,6 +43,7 @@ Delete all rows from tables in the following order through django admin: * Contact * Websites * DraftDomain +* HostIP * Host #### Importing into Target Environment @@ -53,6 +55,7 @@ order: * Contact * Domain * Host +* HostIP * DraftDomain * Websites * DomainRequest diff --git a/src/registrar/admin.py b/src/registrar/admin.py index bcda7e0486..ae6e02c286 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -773,6 +773,19 @@ class MyHostAdmin(AuditedAdmin, ImportExportModelAdmin): inlines = [HostIPInline] +class HostIpResource(resources.ModelResource): + + class Meta: + model = models.HostIP + + +class HostIpAdmin(AuditedAdmin, ImportExportModelAdmin): + """Custom host ip admin class""" + + resource_classes = [HostIpResource] + model = models.HostIP + + class ContactResource(resources.ModelResource): class Meta: @@ -2298,9 +2311,8 @@ class Meta: admin.site.register(models.Domain, DomainAdmin) admin.site.register(models.DraftDomain, DraftDomainAdmin) admin.site.register(models.FederalAgency, FederalAgencyAdmin) -# Host and HostIP removed from django admin because changes in admin -# do not propagate to registry and logic not applied admin.site.register(models.Host, MyHostAdmin) +admin.site.register(models.HostIP, HostIpAdmin) admin.site.register(models.Website, WebsiteAdmin) admin.site.register(models.PublicContact, PublicContactAdmin) admin.site.register(models.DomainRequest, DomainRequestAdmin) From 5673889a55e6b11f73ef968010d7f0dab4beac9e Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 10 May 2024 07:21:31 -0400 Subject: [PATCH 20/22] updated templates with css --- src/registrar/assets/sass/_theme/_admin.scss | 4 + .../templates/admin/import_export/import.html | 191 ++++++++++++++++++ .../import_export/resource_fields_list.html | 21 ++ 3 files changed, 216 insertions(+) create mode 100644 src/registrar/templates/admin/import_export/import.html create mode 100644 src/registrar/templates/admin/import_export/resource_fields_list.html diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 9f5ea7a973..c716ad49c2 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -717,6 +717,10 @@ div.dja__model-description{ } +.import_export_text { + color: var(--secondary); +} + .text-underline { text-decoration: underline !important; } diff --git a/src/registrar/templates/admin/import_export/import.html b/src/registrar/templates/admin/import_export/import.html new file mode 100644 index 0000000000..ef1160a2dd --- /dev/null +++ b/src/registrar/templates/admin/import_export/import.html @@ -0,0 +1,191 @@ +{% extends "admin/import_export/base.html" %} +{% load i18n %} +{% load admin_urls %} +{% load import_export_tags %} +{% load static %} + +{% block extrastyle %}{{ block.super }}{% endblock %} + +{% block extrahead %}{{ block.super }} + + {% if confirm_form %} + {{ confirm_form.media }} + {% else %} + {{ form.media }} + {% endif %} +{% endblock %} + +{% block breadcrumbs_last %} +{% trans "Import" %} +{% endblock %} + +{% block content %} + + {% if confirm_form %} + {% block confirm_import_form %} +
    + {% csrf_token %} + {{ confirm_form.as_p }} +

    + {% trans "Below is a preview of data to be imported. If you are satisfied with the results, click 'Confirm import'" %} +

    +
    + +
    +
    + {% endblock %} + {% else %} + {% block import_form %} +
    + {% csrf_token %} + + {% include "admin/import_export/resource_fields_list.html" with import_or_export="import" %} + {% block import_form_additional_info %}{% endblock %} + + {% block form_detail %} +
    + {% for field in form %} +
    + {{ field.errors }} + + {{ field.label_tag }} + + {{ field }} + + {% if field.field.help_text %} +

    {{ field.field.help_text|safe }}

    + {% endif %} +
    + {% endfor %} +
    + {% endblock %} + + {% block form_submit_button %} +
    + +
    + {% endblock %} +
    + {% endblock %} + {% endif %} + + {% if result %} + + {% if result.has_errors %} + {% block errors %} +

    {% trans "Errors" %}

    +
      + {% for error in result.base_errors %} +
    • + {{ error.error }} +
      {{ error.traceback|linebreaks }}
      +
      • + {% endfor %} + {% for line, errors in result.row_errors %} + {% for error in errors %} +
    • + {% trans "Line number" %}: {{ line }} - {{ error.error }} +
      {{ error.row.values|join:", " }}
      +
      {{ error.traceback|linebreaks }}
      +
      • + {% endfor %} + {% endfor %} +
    + {% endblock %} + + {% elif result.has_validation_errors %} + + {% block validation_errors %} +

    {% trans "Some rows failed to validate" %}

    + +

    {% trans "Please correct these errors in your data where possible, then reupload it using the form above." %}

    + + + + + + + {% for field in result.diff_headers %} + + {% endfor %} + + + + {% for row in result.invalid_rows %} + + + + {% for field in row.values %} + + {% endfor %} + + {% endfor %} + +
    {% trans "Row" %}{% trans "Errors" %}{{ field }}
    {{ row.number }} + {{ row.error_count }} +
    +
      + {% for field_name, error_list in row.field_specific_errors.items %} +
    • + {{ field_name }} +
        + {% for error in error_list %} +
      • {{ error }}
        • + {% endfor %} +
      +
      • + {% endfor %} + {% if row.non_field_specific_errors %} +
    • + {% trans "Non field specific" %} +
        + {% for error in row.non_field_specific_errors %} +
      • {{ error }}
        • + {% endfor %} +
      +
      • + {% endif %} +
    +
    +     		{{ field }}
    + {% endblock %} + + {% else %} + + {% block preview %} +

    {% trans "Preview" %}

    + + + + + + {% for field in result.diff_headers %} + + {% endfor %} + + + {% for row in result.valid_rows %} + + + {% for field in row.diff %} + + {% endfor %} + + {% endfor %} +
    {{ field }}
    + {% if row.import_type == 'new' %} + {% trans "New" %} + {% elif row.import_type == 'skip' %} + {% trans "Skipped" %} + {% elif row.import_type == 'delete' %} + {% trans "Delete" %} + {% elif row.import_type == 'update' %} + {% trans "Update" %} + {% endif %} + {{ field }}
    + {% endblock %} + + {% endif %} + + {% endif %} +{% endblock %} diff --git a/src/registrar/templates/admin/import_export/resource_fields_list.html b/src/registrar/templates/admin/import_export/resource_fields_list.html new file mode 100644 index 0000000000..3f5483ea7c --- /dev/null +++ b/src/registrar/templates/admin/import_export/resource_fields_list.html @@ -0,0 +1,21 @@ +{% load i18n %} +{% block fields_help %} +

    + {% if import_or_export == "export" %} + {% trans "This exporter will export the following fields: " %} + {% elif import_or_export == "import" %} + {% trans "This importer will import the following fields: " %} + {% endif %} + + {% if fields_list|length <= 1 %} + {{ fields_list.0.1|join:", " }} + {% else %} +

    + {% for resource, fields in fields_list %} +
    {{ resource }}
    +
    {{ fields|join:", " }}
    + {% endfor %} +
    + {% endif %} +

    +{% endblock %} \ No newline at end of file From 645c85e1476c152e0a636b934eea855801f7544d Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 10 May 2024 07:36:23 -0400 Subject: [PATCH 21/22] updated documentation and code comments --- docs/operations/import_export.md | 5 ++++- src/registrar/admin.py | 10 ++++++++++ src/registrar/models/utility/generic_helper.py | 6 ++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/operations/import_export.md b/docs/operations/import_export.md index e04a79cd2b..7c3ee11591 100644 --- a/docs/operations/import_export.md +++ b/docs/operations/import_export.md @@ -60,4 +60,7 @@ order: * Websites * DomainRequest * DomainInformation -* UserDomainRole \ No newline at end of file +* UserDomainRole + +Optional step: +* Run fixtures to load fixture users back in \ No newline at end of file diff --git a/src/registrar/admin.py b/src/registrar/admin.py index ae6e02c286..49574496de 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -80,6 +80,7 @@ def import_field(self, field, obj, data, is_m2m=False, **kwargs): class UserResource(resources.ModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.User @@ -758,6 +759,7 @@ class HostIPInline(admin.StackedInline): class HostResource(resources.ModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.Host @@ -774,6 +776,7 @@ class MyHostAdmin(AuditedAdmin, ImportExportModelAdmin): class HostIpResource(resources.ModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.HostIP @@ -787,6 +790,7 @@ class HostIpAdmin(AuditedAdmin, ImportExportModelAdmin): class ContactResource(resources.ModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.Contact @@ -918,6 +922,7 @@ def change_view(self, request, object_id, form_url="", extra_context=None): class WebsiteResource(resources.ModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.Website @@ -976,6 +981,7 @@ def response_change(self, request, obj): class UserDomainRoleResource(resources.ModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.UserDomainRole @@ -1067,6 +1073,7 @@ class Meta: class DomainInformationResource(resources.ModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.DomainInformation @@ -1208,6 +1215,7 @@ def get_readonly_fields(self, request, obj=None): class DomainRequestResource(FsmModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.DomainRequest @@ -1761,6 +1769,7 @@ def get_readonly_fields(self, request, obj=None): class DomainResource(FsmModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.Domain @@ -2166,6 +2175,7 @@ def has_change_permission(self, request, obj=None): class DraftDomainResource(resources.ModelResource): + """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" class Meta: model = models.DraftDomain diff --git a/src/registrar/models/utility/generic_helper.py b/src/registrar/models/utility/generic_helper.py index 7d3586770d..0befd66270 100644 --- a/src/registrar/models/utility/generic_helper.py +++ b/src/registrar/models/utility/generic_helper.py @@ -100,7 +100,7 @@ def _handle_new_instance(self): # Update the field self._update_fields(organization_type_needs_update, generic_org_type_needs_update) - def _handle_existing_instance(self, force_update_when_no_are_changes_found=False): + def _handle_existing_instance(self, force_update_when_no_changes_are_found=False): # == Init variables == # try: # Instance is already in the database, fetch its current state @@ -119,7 +119,7 @@ def _handle_existing_instance(self, force_update_when_no_are_changes_found=False raise ValueError("Cannot update organization_type and generic_org_type simultaneously.") elif not organization_type_changed and (not generic_org_type_changed and not is_election_board_changed): # No changes found - if force_update_when_no_are_changes_found: + if force_update_when_no_changes_are_found: # If we want to force an update anyway, we can treat this record like # its a new one in that we check for "None" values rather than changes. self._handle_new_instance() @@ -132,6 +132,8 @@ def _handle_existing_instance(self, force_update_when_no_are_changes_found=False # Update the field self._update_fields(organization_type_needs_update, generic_org_type_needs_update) except self.sender.DoesNotExist: + # this exception should only be raised when import_export utility attempts to import + # a new row and already has an id pass def _update_fields(self, organization_type_needs_update, generic_org_type_needs_update): From b95c70ff83718382ee88113ec09bf4a78008acc1 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 10 May 2024 07:43:49 -0400 Subject: [PATCH 22/22] formatted for readability --- src/registrar/admin.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 49574496de..ab281c32f7 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -80,7 +80,8 @@ def import_field(self, field, obj, data, is_m2m=False, **kwargs): class UserResource(resources.ModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.User @@ -759,7 +760,8 @@ class HostIPInline(admin.StackedInline): class HostResource(resources.ModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.Host @@ -776,7 +778,8 @@ class MyHostAdmin(AuditedAdmin, ImportExportModelAdmin): class HostIpResource(resources.ModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.HostIP @@ -790,7 +793,8 @@ class HostIpAdmin(AuditedAdmin, ImportExportModelAdmin): class ContactResource(resources.ModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.Contact @@ -922,7 +926,8 @@ def change_view(self, request, object_id, form_url="", extra_context=None): class WebsiteResource(resources.ModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.Website @@ -981,7 +986,8 @@ def response_change(self, request, obj): class UserDomainRoleResource(resources.ModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.UserDomainRole @@ -1073,7 +1079,8 @@ class Meta: class DomainInformationResource(resources.ModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.DomainInformation @@ -1215,7 +1222,8 @@ def get_readonly_fields(self, request, obj=None): class DomainRequestResource(FsmModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.DomainRequest @@ -1769,7 +1777,8 @@ def get_readonly_fields(self, request, obj=None): class DomainResource(FsmModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.Domain @@ -2175,7 +2184,8 @@ def has_change_permission(self, request, obj=None): class DraftDomainResource(resources.ModelResource): - """defines how each field in the referenced model should be mapped to the corresponding fields in the import/export file""" + """defines how each field in the referenced model should be mapped to the corresponding fields in the + import/export file""" class Meta: model = models.DraftDomain