Merge pull request #30 from IQSS/develop

Update

doc/sphinx-guides/source/_static/installation/files/etc/maintenance/maintenance.xhtml

@@ -13,19 +13,14 @@
 
         <link type="image/png" rel="image_src" href="https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/images/dataverseproject.png.xhtml" />
 
-        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css" />
+        <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css" />
 
-        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap-theme.min.css" />
-
-        <link rel="stylesheet" href="https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/css/fontcustom.css" />
-
-        <link rel="stylesheet" href="https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/css/socicon.css" />
+        <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap-theme.min.css" />
 
         <link rel="stylesheet" href="https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/css/structure.css" />
 
-        <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js"></script>
-
         <style type="text/css">
+            /* FontCustom CSS */
             @font-face {
               font-family: "fontcustom";
               src: url("https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.eot");
@@ -37,25 +32,32 @@
               font-style: normal;
             }
 
-            @font-face {
-                font-family: 'socicon';
-                src: url('https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/socicon-font/socicon-webfont.eot');
-                src: url('https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/socicon-font/socicon-webfont.eot?#iefix') format('embedded-opentype'),
-                     url('https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/socicon-font/socicon-webfont.woff') format('woff'),
-                     url('https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/socicon-font/socicon-webfont.woff2') format('woff2'),
-                     url('https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/socicon-font/socicon-webfont.ttf') format('truetype'),
-                     url('https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/socicon-font/socicon-webfont.svg#sociconregular') format('svg');
-                font-weight: normal;
-                font-style: normal;
-                text-transform: initial;
-            }
-
             @media screen and (-webkit-min-device-pixel-ratio:0) {
               @font-face {
                 font-family: "fontcustom";
                 src: url("https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.svg#fontcustom") format("svg");
               }
             }
+
+            [data-icon]:before { content: attr(data-icon); }
+
+            [data-icon]:before,
+            .icon-dataverse:before {
+              display: inline-block;
+              font-family: "fontcustom";
+              font-style: normal;
+              font-weight: normal;
+              font-variant: normal;
+              line-height: 1;
+              text-decoration: inherit;
+              text-rendering: optimizeLegibility;
+              text-transform: none;
+              -moz-osx-font-smoothing: grayscale;
+              -webkit-font-smoothing: antialiased;
+              font-smoothing: antialiased;
+            }
+
+            .icon-dataverse:before { content: "\f100"; }
         </style>
     </head>
     <body>
@@ -98,15 +100,17 @@
                 <p>If you have any comments, questions or concerns, please reach out to support@dataverse.org.</p>
             </div>
         </div>
-        <div class="container" id="footer">
-            <div class="row">
-                <div class="col-sm-8 small">
-                    <p>Copyright &#169; 2018</p>
-                </div>
-                <div class="col-sm-4 text-right">
-                    <div class="poweredbylogo">
-                        <span>Powered by</span> 
-                        <a href="http://dataverse.org/" title="The Dataverse Project" target="_blank"><img src="https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/images/dataverseproject_logo.jpg" alt="The Dataverse Project" /></a>
+        <div id="footer">
+            <div class="container">
+                <div class="row">
+                    <div class="col-sm-8 small">
+                        <p>Copyright &#169; 2019</p>
+                    </div>
+                    <div class="col-sm-4 text-right">
+                        <div class="poweredbylogo">
+                            <span>Powered by</span> 
+                            <a href="http://dataverse.org/" title="The Dataverse Project" target="_blank"><img src="https://cdn.rawgit.com/IQSS/dataverse/87cf58b5c3e8e01cf0f4891465df02334852eed3/src/main/webapp/resources/images/dataverseproject_logo.jpg" alt="The Dataverse Project" /></a>
+                        </div>
                     </div>
                 </div>
             </div>

doc/sphinx-guides/source/user/dataset-management.rst

@@ -16,7 +16,7 @@ A dataset contains three levels of metadata:
 
 #. **Citation Metadata**: any metadata that would be needed for generating a data citation and other general metadata that could be applied to any dataset;
 #. **Domain Specific Metadata**: with specific support currently for Social Science, Life Science, Geospatial, and Astronomy datasets; and
-#. **File-level Metadata**: varies depending on the type of data file - see *File Handling and Uploading* section below for more details). 
+#. **File-level Metadata**: varies depending on the type of data file - for more details see :ref:`File Handling <file-handling>` section below).
 
 For more details about what Citation and Domain Specific Metadata is supported please see our :ref:`user-appendix`.
 
@@ -67,7 +67,7 @@ If there are multiple upload options available, then you must choose which one t
 
 You can upload files to a dataset while first creating that dataset. You can also upload files after creating a dataset by clicking the "Edit" button at the top of the dataset page and from the dropdown list selecting "Files (Upload)" or clicking the "Upload Files" button above the files table in the Files tab. From either option you will be brought to the Upload Files page for that dataset.
 
-Certain file types in Dataverse are supported by additional functionality, which can include downloading in different formats, subsets, file-level metadata preservation, file-level data citation with UNFs, and exploration through data visualization and analysis. See the File Handling section of this page for more information.
+Certain file types in Dataverse are supported by additional functionality, which can include downloading in different formats, file-level metadata preservation, file-level data citation with UNFs, and exploration through data visualization and analysis. See the :ref:`File Handling <file-handling>` section of this page for more information.
 
 
 HTTP Upload
@@ -132,20 +132,22 @@ Basic usage is to run the command: ::
 
     java -jar DVUploader-v1.0.0.jar -server=<Dataverse server URL> -did=<Dataset DOI> -key=<User's API Key> <file or directory list>
 
-Additional command line arguments are available to make the DVUploader list what it would do without uploading, limit the number of files it uploads, recurse through sub-directories, verify fixity, exclude files with specific extensions or name patterns, and/or wait longer than 60 seconds for any Dataverse ingest lock to clear (e.g. while the previously uploaded file is processed, as discussed in the File Handling section below). 
+Additional command line arguments are available to make the DVUploader list what it would do without uploading, limit the number of files it uploads, recurse through sub-directories, verify fixity, exclude files with specific extensions or name patterns, and/or wait longer than 60 seconds for any Dataverse ingest lock to clear (e.g. while the previously uploaded file is processed, as discussed in the :ref:`File Handling <file-handling>` section below). 
 
 DVUploader is a community-developed tool, and its creation was primarily supported by the Texas Digital Library. Further information and support for DVUploader can be sought at `the project's GitHub repository <https://github.com/IQSS/dataverse-uploader>`_ . 
 
+.. _file-handling:
+
 File Handling
 =============
 
-Certain file types in Dataverse are supported by additional functionality, which can include downloading in different formats, subsets, file-level metadata preservation, file-level data citation; and exploration through data visualization and analysis. See the sections below for information about special functionality for specific file types.
+Certain file types in Dataverse are supported by additional functionality, which can include downloading in different formats, file-level metadata preservation, file-level data citation; and exploration through data visualization and analysis. See the sections below for information about special functionality for specific file types.
 
 
 Tabular Data Files
 ------------------
 
-Files in certain formats - Stata, SPSS, R, Excel(xlsx) and CSV - may be ingested as tabular data (see "Tabular Data Ingest" section of the User Guide for details). Tabular data files can be further explored and manipulated with `TwoRavens <../user/data-exploration/tworavens.html>`_ - a statistical data exploration application integrated with Dataverse, as well as other :doc:`/installation/external-tools` if they have been enabled in the installation of Dataverse you are using. TwoRavens allows the user to run statistical models, view summary statistics, download subsets of variable vectors and more. To start, click on the "Explore" button, found next to each relevant tabular file (the application will be opened in a new window). To download subsets of variables click on the "Download" button found next to a relevant tabular file and select "Data Subset" in the dropdown menu. You will then be able to create your subset using the interface opened in a new window (this functionality is also provided by the `TwoRavens <../user/data-exploration/tworavens.html>`_ project). See the `TwoRavens documentation section <../user/data-exploration/tworavens.html>`_ for more information.
+Files in certain formats - Stata, SPSS, R, Excel(xlsx) and CSV - may be ingested as tabular data (see :doc:`/user/tabulardataingest/index` section of the User Guide for details). Tabular data files can be further explored and manipulated with `TwoRavens <../user/data-exploration/tworavens.html>`_ - a statistical data exploration application integrated with Dataverse, as well as other :doc:`/installation/external-tools` if they have been enabled in the installation of Dataverse you are using. TwoRavens allows the user to run statistical models, view summary statistics, download subsets of variable vectors and more. To start, click on the "Explore" button, found next to each relevant tabular file (the application will be opened in a new window). Create and download your subset using `TwoRavens <../user/data-exploration/tworavens.html>`_. See the `TwoRavens documentation section <../user/data-exploration/tworavens.html>`_ for more information.
 
 Additional download options available for tabular data (found in the same drop-down menu under the "Download" button): 
 
@@ -156,7 +158,6 @@ Additional download options available for tabular data (found in the same drop-d
 - Data File Citation (currently in either RIS, EndNote XML, or BibTeX format); 
 - All of the above, as a zipped bundle. 
 
-|image2|
 
 Geospatial
 ----------
@@ -478,7 +479,7 @@ Once you have more than one version (this can simply be version 1 and a draft),
 .. _dataset-metrics-user:
 
 Dataset Metrics and Make Data Count
-===============
+===================================
 
 All installations of Dataverse count file downloads. These file download counts are aggregated and reported at the Dataset level as well as at the file level.
 
@@ -522,8 +523,6 @@ If you deaccession the most recently published version of the dataset but not al
 
 .. |image1| image:: ./img/DatasetDiagram.png
    :class: img-responsive
-.. |image2| image:: ./img/data-download.png
-   :class: img-responsive
 .. |image3| image:: ./img/data_publishing_version_workflow.png
    :class: img-responsive
 .. |file-upload-prov-button| image:: ./img/prov0.png

src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java

@@ -12,18 +12,17 @@
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.impl.DestroyDatasetCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.FinalizeDatasetPublicationCommand;
 import edu.harvard.iq.dataverse.export.ExportService;
 import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean;
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.workflows.WorkflowComment;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
@@ -48,9 +47,6 @@
 import javax.persistence.Query;
 import javax.persistence.StoredProcedureQuery;
 import javax.persistence.TypedQuery;
-import javax.xml.stream.XMLOutputFactory;
-import javax.xml.stream.XMLStreamException;
-import javax.xml.stream.XMLStreamWriter;
 import org.apache.commons.lang.RandomStringUtils;
 import org.ocpsoft.common.util.Strings;
 
@@ -928,4 +924,35 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws I
 
         return total; 
     }
+
+    /**
+     * An optimized method for deleting a harvested dataset. 
+     * 
+     * @param dataset
+     * @param request DataverseRequest (for initializing the DestroyDatasetCommand)
+     * @param hdLogger logger object (in practice, this will be a separate log file created for a specific harvesting job)
+     */
+    @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+    public void deleteHarvestedDataset(Dataset dataset, DataverseRequest request, Logger hdLogger) {
+        // Purge all the SOLR documents associated with this client from the 
+        // index server: 
+        indexService.deleteHarvestedDocuments(dataset);
+
+        try {
+            // files from harvested datasets are removed unceremoniously, 
+            // directly in the database. no need to bother calling the 
+            // DeleteFileCommand on them.
+            for (DataFile harvestedFile : dataset.getFiles()) {
+                DataFile merged = em.merge(harvestedFile);
+                em.remove(merged);
+                harvestedFile = null; 
+            }
+            dataset.setFiles(null);
+            Dataset merged = em.merge(dataset);
+            commandEngine.submit(new DestroyDatasetCommand(merged, request));
+            hdLogger.info("Successfully destroyed the dataset");
+        } catch (Exception ex) {
+            hdLogger.warning("Failed to destroy the dataset");
+        } 
+    }
 }

src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java

@@ -254,9 +254,9 @@ public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse)
      public void writeGuestbookAndOpenSubset(GuestbookResponse guestbookResponse) {
         RequestContext requestContext = RequestContext.getCurrentInstance();
         boolean valid = validateGuestbookResponse(guestbookResponse);
-                  
+
          if (!valid) {
-             
+
          } else {
              requestContext.execute("PF('downloadPopup').hide()"); 
              requestContext.execute("PF('downloadDataSubsetPopup').show()");

src/main/java/edu/harvard/iq/dataverse/FileMetadata.java

@@ -546,6 +546,17 @@ public boolean contentEquals(FileMetadata other) {
         } else if (other.getDescription() != null) {
             return false;
         }
+        List<String> categoryNames =this.getCategoriesByName();
+        List<String> otherCategoryNames =other.getCategoriesByName();
+        if(!categoryNames.isEmpty()) {
+            categoryNames.sort(null);
+            otherCategoryNames.sort(null);
+            if (!categoryNames.equals(otherCategoryNames)) {
+                return false;
+            }
+        } else if(!otherCategoryNames.isEmpty()) {
+            return false;
+        }
 
         return true;
     }

src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java

@@ -42,6 +42,7 @@
 import java.io.StringReader;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 import java.util.Set;
 import java.util.logging.Formatter;
@@ -197,7 +198,7 @@ public JsonObjectBuilder handleFile(DataverseRequest dataverseRequest, Dataverse
     }
 
     @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
-    public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, String harvestIdentifier, String metadataFormat, File metadataFile, PrintWriter cleanupLog) throws ImportException, IOException {
+    public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, String harvestIdentifier, String metadataFormat, File metadataFile, Date oaiDateStamp, PrintWriter cleanupLog) throws ImportException, IOException {
         if (harvestingClient == null || harvestingClient.getDataverse() == null) {
             throw new ImportException("importHarvestedDataset called wiht a null harvestingClient, or an invalid harvestingClient.");
         }
@@ -275,6 +276,10 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
             ds.setOwner(owner);
             ds.getLatestVersion().setDatasetFields(ds.getLatestVersion().initDatasetFields());
 
+            if (ds.getVersions().get(0).getReleaseTime() == null) {
+                ds.getVersions().get(0).setReleaseTime(oaiDateStamp);
+            }
+
             // Check data against required contraints
             List<ConstraintViolation<DatasetField>> violations = ds.getVersions().get(0).validateRequired();
             if (!violations.isEmpty()) {