Merge pull request #2266 from cisagov/za/2166-domain-request-csv-report

(getgov-za) Ticket #2166: Domain request csv report

src/registrar/assets/sass/_theme/_admin.scss

@@ -773,3 +773,12 @@ div.dja__model-description{
 .module caption, .inline-group h2 {
     text-transform: capitalize;
 }
+
+.wrapped-button-group {
+    // This button group has too many items
+    flex-wrap: wrap;
+    // Fix a weird spacing issue with USWDS a buttons in DJA
+    a.button {
+        padding: 6px 8px 10px 8px;
+    }
+}

src/registrar/config/urls.py

@@ -18,6 +18,7 @@
     ExportDataType,
     ExportDataUnmanagedDomains,
     AnalyticsView,
+    ExportDomainRequestDataFull,
 )
 
 from registrar.views.domain_request import Step
@@ -66,6 +67,11 @@
         ExportDataType.as_view(),
         name="export_data_type",
     ),
+    path(
+        "admin/analytics/export_data_domain_requests_full/",
+        ExportDomainRequestDataFull.as_view(),
+        name="export_data_domain_requests_full",
+    ),
     path(
         "admin/analytics/export_data_full/",
         ExportDataFull.as_view(),

src/registrar/models/domain_request.py

@@ -52,6 +52,11 @@ class DomainRequestStatus(models.TextChoices):
         WITHDRAWN = "withdrawn", "Withdrawn"
         STARTED = "started", "Started"
 
+        @classmethod
+        def get_status_label(cls, status_name: str):
+            """Returns the associated label for a given status name"""
+            return cls(status_name).label if status_name else None
+
     class StateTerritoryChoices(models.TextChoices):
         ALABAMA = "AL", "Alabama (AL)"
         ALASKA = "AK", "Alaska (AK)"
@@ -133,6 +138,14 @@ class OrganizationChoices(models.TextChoices):
         SPECIAL_DISTRICT = "special_district", "Special district"
         SCHOOL_DISTRICT = "school_district", "School district"
 
+        @classmethod
+        def get_org_label(cls, org_name: str):
+            """Returns the associated label for a given org name"""
+            org_names = org_name.split("_election")
+            if len(org_names) > 0:
+                org_name = org_names[0]
+            return cls(org_name).label if org_name else None
+
     class OrgChoicesElectionOffice(models.TextChoices):
         """
         Primary organization choices for Django admin:

src/registrar/models/utility/generic_helper.py

@@ -298,3 +298,26 @@ def replace_url_queryparams(url_to_modify: str, query_params, convert_list_to_cs
     new_url = urlunparse(url_parts)
 
     return new_url
+
+
+def convert_queryset_to_dict(queryset, is_model=True, key="id"):
+    """
+    Transforms a queryset into a dictionary keyed by a specified key (like "id").
+
+    Parameters:
+        requests (QuerySet or list of dicts): Input data.
+        is_model (bool): Indicates if each item in 'queryset' are model instances (True) or dictionaries (False).
+        key (str): Key or attribute to use for the resulting dictionary's keys.
+
+    Returns:
+        dict: Dictionary with keys derived from 'key' and values corresponding to items in 'queryset'.
+    """
+
+    if is_model:
+        request_dict = {getattr(value, key): value for value in queryset}
+    else:
+        # Querysets sometimes contain sets of dictionaries.
+        # Calling .values is an example of this.
+        request_dict = {value[key]: value for value in queryset}
+
+    return request_dict

src/registrar/templates/admin/analytics.html

@@ -27,28 +27,35 @@ <h2>At a glance</h2>
       <div class="module height-full">
         <h2>Current domains</h2>
         <div class="padding-top-2 padding-x-2">
-          <ul class="usa-button-group">
+          <ul class="usa-button-group wrapped-button-group">
             <li class="usa-button-group__item">
-                <a href="{% url 'export_data_type' %}" class="button" role="button">
+                <a href="{% url 'export_data_type' %}" class="button text-no-wrap" role="button">
                   <svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
                     <use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
                   </svg><span class="margin-left-05">All domain metadata</span>
                 </a>
             </li>
             <li class="usa-button-group__item">
-              <a href="{% url 'export_data_full' %}" class="button" role="button">
+              <a href="{% url 'export_data_full' %}" class="button text-no-wrap" role="button">
                 <svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
                   <use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
                 </svg><span class="margin-left-05">Current full</span>
               </a>
             </li>
             <li class="usa-button-group__item">
-                <a href="{% url 'export_data_federal' %}" class="button" role="button">
+                <a href="{% url 'export_data_federal' %}" class="button text-no-wrap" role="button">
                   <svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
                     <use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
                   </svg><span class="margin-left-05">Current federal</span>
                 </a>
             </li>
+            <li class="usa-button-group__item">
+              <a href="{% url 'export_data_domain_requests_full' %}" class="button text-no-wrap" role="button">
+                <svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24" height="24">
+                  <use xlink:href="{%static 'img/sprite.svg'%}#file_download"></use>
+                </svg><span class="margin-left-05">All domain requests metadata</span>
+              </a>
+            </li>
           </ul>
         </div>
       </div>

src/registrar/tests/common.py

@@ -735,19 +735,53 @@ def setUp(self):
             self.domain_request_4 = completed_domain_request(
                 status=DomainRequest.DomainRequestStatus.STARTED,
                 name="city4.gov",
+                is_election_board=True,
+                generic_org_type="city",
             )
             self.domain_request_5 = completed_domain_request(
                 status=DomainRequest.DomainRequestStatus.APPROVED,
                 name="city5.gov",
             )
+            self.domain_request_6 = completed_domain_request(
+                status=DomainRequest.DomainRequestStatus.STARTED,
+                name="city6.gov",
+            )
             self.domain_request_3.submit()
             self.domain_request_4.submit()
-
+            self.domain_request_6.submit()
+
+            other, _ = Contact.objects.get_or_create(
+                first_name="Testy1232",
+                last_name="Tester24",
+                title="Another Tester",
+                email="te2@town.com",
+                phone="(555) 555 5557",
+            )
+            other_2, _ = Contact.objects.get_or_create(
+                first_name="Meow",
+                last_name="Tester24",
+                title="Another Tester",
+                email="te2@town.com",
+                phone="(555) 555 5557",
+            )
+            website, _ = Website.objects.get_or_create(website="igorville.gov")
+            website_2, _ = Website.objects.get_or_create(website="cheeseville.gov")
+            website_3, _ = Website.objects.get_or_create(website="https://www.example.com")
+            website_4, _ = Website.objects.get_or_create(website="https://www.example2.com")
+
+            self.domain_request_3.other_contacts.add(other, other_2)
+            self.domain_request_3.alternative_domains.add(website, website_2)
+            self.domain_request_3.current_websites.add(website_3, website_4)
+            self.domain_request_3.cisa_representative_email = "test@igorville.com"
             self.domain_request_3.submission_date = get_time_aware_date(datetime(2024, 4, 2))
-            self.domain_request_4.submission_date = get_time_aware_date(datetime(2024, 4, 2))
             self.domain_request_3.save()
+
+            self.domain_request_4.submission_date = get_time_aware_date(datetime(2024, 4, 2))
             self.domain_request_4.save()
 
+            self.domain_request_6.submission_date = get_time_aware_date(datetime(2024, 4, 2))
+            self.domain_request_6.save()
+
     def tearDown(self):
         super().tearDown()
         PublicContact.objects.all().delete()

src/registrar/tests/test_reports.py

@@ -4,6 +4,7 @@
 from io import StringIO
 from registrar.models.domain_request import DomainRequest
 from registrar.models.domain import Domain
+from registrar.models.utility.generic_helper import convert_queryset_to_dict
 from registrar.utility.csv_export import (
     export_data_managed_domains_to_csv,
     export_data_unmanaged_domains_to_csv,
@@ -12,7 +13,7 @@
     write_csv_for_domains,
     get_default_start_date,
     get_default_end_date,
-    write_csv_for_requests,
+    DomainRequestExport,
 )
 
 from django.core.management import call_command
@@ -23,6 +24,7 @@
 import boto3_mocking
 from registrar.utility.s3_bucket import S3ClientError, S3ClientErrorCodes  # type: ignore
 from django.utils import timezone
+from api.tests.common import less_console_noise_decorator
 from .common import MockDb, MockEppLib, less_console_noise, get_time_aware_date
 
 
@@ -667,10 +669,7 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self):
             # Define columns, sort fields, and filter condition
             # We'll skip submission date because it's dynamic and therefore
             # impossible to set in expected_content
-            columns = [
-                "Requested domain",
-                "Organization type",
-            ]
+            columns = ["Domain request", "Domain type", "Federal type"]
             sort_fields = [
                 "requested_domain__name",
             ]
@@ -679,17 +678,23 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self):
                 "submission_date__lte": self.end_date,
                 "submission_date__gte": self.start_date,
             }
-            write_csv_for_requests(writer, columns, sort_fields, filter_condition, should_write_header=True)
+
+            additional_values = ["requested_domain__name"]
+            all_requests = DomainRequest.objects.filter(**filter_condition).order_by(*sort_fields).distinct()
+            annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(all_requests, {}, additional_values)
+            requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False)
+            DomainRequestExport.write_csv_for_requests(writer, columns, requests_dict)
             # Reset the CSV file's position to the beginning
             csv_file.seek(0)
             # Read the content into a variable
             csv_content = csv_file.read()
             # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name
             # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name
             expected_content = (
-                "Requested domain,Organization type\n"
-                "city3.gov,Federal - Executive\n"
-                "city4.gov,Federal - Executive\n"
+                "Domain request,Domain type,Federal type\n"
+                "city3.gov,Federal,Executive\n"
+                "city4.gov,City,Executive\n"
+                "city6.gov,Federal,Executive\n"
             )
 
             # Normalize line endings and remove commas,
@@ -699,6 +704,72 @@ def test_write_requests_body_with_date_filter_pulls_requests_in_range(self):
 
             self.assertEqual(csv_content, expected_content)
 
+    @less_console_noise_decorator
+    def test_full_domain_request_report(self):
+        """Tests the full domain request report."""
+
+        # Create a CSV file in memory
+        csv_file = StringIO()
+        writer = csv.writer(csv_file)
+
+        # Call the report. Get existing fields from the report itself.
+        annotations = DomainRequestExport._full_domain_request_annotations()
+        additional_values = [
+            "requested_domain__name",
+            "federal_agency__agency",
+            "authorizing_official__first_name",
+            "authorizing_official__last_name",
+            "authorizing_official__email",
+            "authorizing_official__title",
+            "creator__first_name",
+            "creator__last_name",
+            "creator__email",
+            "investigator__email",
+        ]
+        requests = DomainRequest.objects.exclude(status=DomainRequest.DomainRequestStatus.STARTED)
+        annotated_requests = DomainRequestExport.annotate_and_retrieve_fields(requests, annotations, additional_values)
+        requests_dict = convert_queryset_to_dict(annotated_requests, is_model=False)
+        DomainRequestExport.write_csv_for_requests(writer, DomainRequestExport.all_columns, requests_dict)
+
+        # Reset the CSV file's position to the beginning
+        csv_file.seek(0)
+        # Read the content into a variable
+        csv_content = csv_file.read()
+        print(csv_content)
+        self.maxDiff = None
+        expected_content = (
+            # Header
+            "Domain request,Submitted at,Status,Domain type,Federal type,"
+            "Federal agency,Organization name,Election office,City,State/territory,"
+            "Region,Creator first name,Creator last name,Creator email,Creator approved domains count,"
+            "Creator active requests count,Alternative domains,AO first name,AO last name,AO email,"
+            "AO title/role,Request purpose,Request additional details,Other contacts,"
+            "CISA regional representative,Current websites,Investigator\n"
+            # Content
+            "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com,"
+            "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n"
+            "city3.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"
+            "cheeseville.gov | city1.gov | igorville.gov,Testy,Tester,testy@town.com,Chief Tester,"
+            "Purpose of the site,CISA-first-name CISA-last-name | There is more,Meow Tester24 te2@town.com | "
+            "Testy1232 Tester24 te2@town.com | Testy Tester testy2@town.com,test@igorville.com,"
+            "city.com | https://www.example2.com | https://www.example.com,\n"
+            "city4.gov,2024-04-02,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,"
+            "testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,"
+            "Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com,\n"
+            "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com,"
+            "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n"
+            "city6.gov,2024-04-02,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,"
+            "testy@town.com,Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,"
+            "Testy Tester testy2@town.com,cisaRep@igorville.gov,city.com,"
+        )
+
+        # Normalize line endings and remove commas,
+        # spaces and leading/trailing whitespace
+        csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip()
+        expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip()
+
+        self.assertEqual(csv_content, expected_content)
+
 
 class HelperFunctions(MockDb):
     """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy."""
@@ -741,5 +812,5 @@ def test_get_sliced_requests(self):
                 "submission_date__lte": self.end_date,
             }
             submitted_requests_sliced_at_end_date = get_sliced_requests(filter_condition)
-            expected_content = [2, 2, 0, 0, 0, 0, 0, 0, 0, 0]
+            expected_content = [3, 2, 0, 0, 0, 0, 1, 0, 0, 1]
             self.assertEqual(submitted_requests_sliced_at_end_date, expected_content)

src/registrar/utility/constants.py

@@ -5,3 +5,8 @@ class BranchChoices(models.TextChoices):
     EXECUTIVE = "executive", "Executive"
     JUDICIAL = "judicial", "Judicial"
     LEGISLATIVE = "legislative", "Legislative"
+
+    @classmethod
+    def get_branch_label(cls, branch_name: str):
+        """Returns the associated label for a given org name"""
+        return cls(branch_name).label if branch_name else None