Merge branch 'develop' into 8012-5.6-release-notes

conf/docker-aio/0prep_deps.sh

@@ -4,10 +4,10 @@ if [ ! -d dv/deps ]; then
 fi
 wdir=`pwd`
 
-if [ ! -e dv/deps/payara-5.2021.4.zip ]; then
+if [ ! -e dv/deps/payara-5.2021.5.zip ]; then
 	echo "payara dependency prep"
 	# no more fiddly patching :)
-	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip  -O dv/deps/payara-5.2021.4.zip
+	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip  -O dv/deps/payara-5.2021.5.zip
 fi
 
 if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then

conf/docker-aio/c8.dockerfile

@@ -24,7 +24,7 @@ COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
 COPY httpd.conf /etc/httpd/conf 
 RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz 
-RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.4.zip ; ln -s /opt/payara5 /opt/glassfish4
+RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.5.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
 # this dies under Java 11, do we keep it?

doc/sphinx-guides/source/admin/dataverses-datasets.rst

@@ -41,7 +41,7 @@ Recursively assigns the users and groups having a role(s),that are in the set co
     curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/addRoleAssignmentsToChildren
     
 Configure a Dataverse Collection to store all new files in a specific file store
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To direct new files (uploaded when datasets are created or edited) for all datasets in a given Dataverse collection, the store can be specified via the API as shown below, or by editing the 'General Information' for a Dataverse collection on the Dataverse collection page. Only accessible to superusers. ::
  
@@ -110,6 +110,8 @@ Mints a new identifier for a dataset previously registered with a handle. Only a
 
     curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/admin/$dataset-id/reregisterHDLToPID
     
+.. _send-metadata-to-pid-provider:
+
 Send Dataset metadata to PID provider
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

doc/sphinx-guides/source/developers/dataset-migration-api.rst

@@ -7,6 +7,7 @@ This experimental migration API offers an additional option with some potential
 
 * metadata can be specified using the json-ld format used in the OAI-ORE metadata export
 * existing publication dates and PIDs are maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the software is configured for)
+* updating the PID at the provider can be done immediately or later (with other existing APIs)
 * adding files can be done via the standard APIs, including using direct-upload to S3
 
 This API consists of 2 calls: one to create an initial Dataset version, and one to 'republish' the dataset through Dataverse with a specified publication date.
@@ -44,6 +45,14 @@ The call above creates a Dataset. Once it is created, other APIs can be used to
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
  
-  curl -H 'Content-Type: application/jsonld' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated"
+  curl -H 'Content-Type: application/ld+json' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated"
 
 datePublished is the only metadata supported in this call.
+
+An optional query parameter: updatepidatprovider (default is false) can be set to true to automatically update the metadata and targetUrl of the PID at the provider. With this set true, the result of this call will be that the PID redirects to this dataset rather than the dataset in the source repository.
+
+.. code-block:: bash
+
+  curl -H 'Content-Type: application/ld+json' -H X-Dataverse-key:$API_TOKEN -X POST -d '{"schema:datePublished": "2020-10-26","@context":{ "schema":"http://schema.org/"}}' "$SERVER_URL/api/datasets/{id}/actions/:releasemigrated?updatepidatprovider=true"
+
+If the parameter is not added and set to true, other existing APIs can be used to update the PID at the provider later, e.g. :ref:`send-metadata-to-pid-provider`

doc/sphinx-guides/source/developers/dev-environment.rst

@@ -85,9 +85,9 @@ To install Payara, run the following commands:
 
 ``cd /usr/local``
 
-``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip``
+``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip``
 
-``sudo unzip payara-5.2021.4.zip``
+``sudo unzip payara-5.2021.5.zip``
 
 ``sudo chown -R $USER /usr/local/payara5``
 

doc/sphinx-guides/source/installation/prerequisites.rst

@@ -44,7 +44,7 @@ On RHEL/derivative you can make Java 11 the default with the ``alternatives`` co
 Payara
 ------
 
-Payara 5.2021.4 is recommended. Newer versions might work fine, regular updates are recommended.
+Payara 5.2021.5 is recommended. Newer versions might work fine, regular updates are recommended.
 
 Installing Payara
 =================
@@ -55,8 +55,8 @@ Installing Payara
 
 - Download and install Payara (installed in ``/usr/local/payara5`` in the example commands below)::
 
-	# wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip
-	# unzip payara-5.2021.4.zip
+	# wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip
+	# unzip payara-5.2021.5.zip
 	# mv payara5 /usr/local
 
 If you intend to install and run Payara under a service account (and we hope you do), chown -R the Payara hierarchy to root to protect it but give the service account access to the below directories:

downloads/download.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.4/payara-5.2021.4.zip
+curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip
 curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz
 curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar
 curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar

pom.xml

@@ -26,7 +26,7 @@
         <skipUnitTests>false</skipUnitTests>
 
         <jakartaee-api.version>8.0.0</jakartaee-api.version>
-        <payara.version>5.2021.4</payara.version>
+        <payara.version>5.2021.5</payara.version>
         <postgresql.version>42.2.19</postgresql.version>
         <aws.version>1.11.762</aws.version>
         <commons.logging.version>1.2</commons.logging.version>

scripts/vagrant/setup.sh

@@ -52,7 +52,7 @@ SOLR_USER=solr
 echo "Ensuring Unix user '$SOLR_USER' exists"
 useradd $SOLR_USER || :
 DOWNLOAD_DIR='/dataverse/downloads'
-PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.4.zip"
+PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.5.zip"
 SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz"
 if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then
     echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...."

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -2777,14 +2777,22 @@ public String editFileMetadata(){
 
     public String deleteDatasetVersion() {
         DeleteDatasetVersionCommand cmd;
+
+        Map<Long, String> deleteStorageLocations = datafileService.getPhysicalFilesToDelete(dataset.getLatestVersion());
+        boolean deleteCommandSuccess = false;
         try {
             cmd = new DeleteDatasetVersionCommand(dvRequestService.getDataverseRequest(), dataset);
             commandEngine.submit(cmd);
             JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("datasetVersion.message.deleteSuccess"));
+            deleteCommandSuccess = true;
         } catch (CommandException ex) {
             JH.addMessage(FacesMessage.SEVERITY_FATAL, BundleUtil.getStringFromBundle("dataset.message.deleteFailure"));
             logger.severe(ex.getMessage());
         }
+
+        if (deleteCommandSuccess && !deleteStorageLocations.isEmpty()) {
+            datafileService.finalizeFileDeletes(deleteStorageLocations);
+        }
 
         return returnToDatasetOnly();
     }

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -1206,7 +1206,7 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S
     @POST
     @Path("{id}/actions/:releasemigrated")
     @Consumes("application/ld+json, application/json-ld")
-    public Response publishMigratedDataset(String jsonldBody, @PathParam("id") String id) {
+    public Response publishMigratedDataset(String jsonldBody, @PathParam("id") String id, @DefaultValue("false") @QueryParam ("updatepidatprovider") boolean contactPIDProvider) {
         try {
             AuthenticatedUser user = findAuthenticatedUserOrDie();
             if (!user.isSuperuser()) {
@@ -1270,11 +1270,11 @@ public Response publishMigratedDataset(String jsonldBody, @PathParam("id") Strin
                 if (prePubWf.isPresent()) {
                     // Start the workflow, the workflow will call FinalizeDatasetPublication later
                     wfService.start(prePubWf.get(),
-                            new WorkflowContext(createDataverseRequest(user), ds, TriggerType.PrePublishDataset, false),
+                            new WorkflowContext(createDataverseRequest(user), ds, TriggerType.PrePublishDataset, !contactPIDProvider),
                             false);
                 } else {
                     FinalizeDatasetPublicationCommand cmd = new FinalizeDatasetPublicationCommand(ds,
-                            createDataverseRequest(user), false);
+                            createDataverseRequest(user), !contactPIDProvider);
                     ds = commandEngine.submit(cmd);
                 }
             } catch (CommandException ex) {

src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java

@@ -1422,10 +1422,10 @@ public static void createDataDscr(XMLStreamWriter xmlw, DatasetVersion datasetVe
              * included for restricted files but that meant that summary
              * statistics were exposed. (To get at these statistics, API users
              * should instead use the "Data Variable Metadata Access" endpoint.)
-             * These days we return early to avoid this exposure.
+             * These days we skip restricted files to avoid this exposure.
              */
             if (dataFile.isRestricted()) {
-                return;
+                continue;
             }
 
             if (dataFile != null && dataFile.isTabularData()) {

src/main/java/edu/harvard/iq/dataverse/util/DataSourceProducer.java

@@ -38,7 +38,7 @@
 //})
 //
 // ... but at this time we don't think we need any. The full list
-// of properties can be found at https://docs.payara.fish/community/docs/5.2021.4/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties
+// of properties can be found at https://docs.payara.fish/community/docs/5.2021.5/documentation/payara-server/jdbc/advanced-connection-pool-properties.html#full-list-of-properties
 //
 // All these properties cannot be configured via MPCONFIG as Payara doesn't support this (yet). To be enhanced.
 // See also https://github.com/payara/Payara/issues/5024

src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java

@@ -104,12 +104,6 @@ public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody,
                 LocalDateTime dateTime = getDateTimeFrom(dateString);
                 ds.setModificationTime(Timestamp.valueOf(dateTime));
             }
-            try {
-                logger.fine("Output dsv: " + new OREMap(dsv, false).getOREMap().toString());
-            } catch (Exception e) {
-                // TODO Auto-generated catch block
-                e.printStackTrace();
-            }
         }
         return ds;
     }