8889 file-level PIDs configuration in individual collections

doc/release-notes/8889-filepids-in-collections.md

@@ -0,0 +1,3 @@
+It is now possible to configure registering PIDs for files in individual collections.
+
+For example, registration of PIDs for files can be enabled in a specific collection when it is disabled instance-wide. Or it can be disabled in specific collections where it is enabled by default. See the [:FilePIDsEnabled](https://guides.dataverse.org/en/latest/installation/config.html#filepidsenabled) section of the Configuration guide for details.

doc/sphinx-guides/source/api/native-api.rst

@@ -736,6 +736,24 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1 -o myResponses.csv
 
+.. _collection-attributes-api:
+
+Change Collection Attributes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: 
+
+  curl -H  X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/attribute/$ATTRIBUTE?value=$VALUE
+
+The following attributes are supported:
+
+* ``alias``  Collection alias
+* ``name`` Name
+* ``description`` Description
+* ``affiliation`` Affiliation
+* ``filePIDsEnabled`` ("true" or "false") Enables or disables registraion of file-level PIDs in datasets within the collection (overriding the instance-wide setting).
+
+
 Datasets
 --------
 

doc/sphinx-guides/source/installation/config.rst

@@ -2756,14 +2756,16 @@ timestamps.
 :FilePIDsEnabled
 ++++++++++++++++
 
-Toggles publishing of file-based PIDs for the entire installation. By default this setting is absent and Dataverse Software assumes it to be true. If enabled, the registration will be performed asynchronously (in the background) during publishing of a dataset.
+Toggles publishing of file-level PIDs for the entire installation. By default this setting is absent and Dataverse Software assumes it to be true. If enabled, the registration will be performed asynchronously (in the background) during publishing of a dataset.
 
 If you don't want to register file-based PIDs for your installation, set:
 
 ``curl -X PUT -d 'false' http://localhost:8080/api/admin/settings/:FilePIDsEnabled``
 
 Note: File-level PID registration was added in Dataverse Software 4.9; it could not be disabled until Dataverse Software 4.9.3.
 
+It is possible to override the installation-wide setting for specific collections. For example, registration of PIDs for files can be enabled in a specific collection when it is disabled instance-wide. Or it can be disabled in specific collections where it is enabled by default. See :ref:`collection-attributes-api` for details. 
+
 .. _:IndependentHandleService:
 
 :IndependentHandleService

src/main/java/edu/harvard/iq/dataverse/Dataverse.java

@@ -590,8 +590,17 @@ public void setCitationDatasetFieldTypes(List<DatasetFieldType> citationDatasetF
         this.citationDatasetFieldTypes = citationDatasetFieldTypes;
     }
 
-
+    @Column(nullable = true)
+    private Boolean filePIDsEnabled;
 
+    public Boolean getFilePIDsEnabled() {
+        return filePIDsEnabled;
+    }
+
+    public void setFilePIDsEnabled(boolean filePIDsEnabled) {
+        this.filePIDsEnabled = filePIDsEnabled;
+    }
+
     public List<DataverseFacet> getDataverseFacets() {
         return getDataverseFacets(false);
     }

src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java

@@ -82,6 +82,7 @@
 
 import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
+import edu.harvard.iq.dataverse.util.json.JsonPrinter;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief;
 import java.io.StringReader;
 import java.util.Collections;
@@ -129,6 +130,7 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 import javax.servlet.http.HttpServletResponse;
+import javax.validation.constraints.NotNull;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.StreamingOutput;
@@ -166,7 +168,7 @@ public class Dataverses extends AbstractApiBean {
 
     @EJB
     SwordServiceBean swordService;
-
+    
     @POST
     @AuthRequired
     public Response addRoot(@Context ContainerRequestContext crc, String body) {
@@ -590,6 +592,69 @@ public Response deleteDataverse(@Context ContainerRequestContext crc, @PathParam
         }, getRequestUser(crc));
     }
 
+    /**
+     * Endpoint to change attributes of a Dataverse collection.
+     *
+     * @apiNote Example curl command:
+     *          <code>curl -X PUT -d "test" http://localhost:8080/api/dataverses/$ALIAS/attribute/alias</code>
+     *          to change the alias of the collection named $ALIAS to "test".
+     */
+    @PUT
+    @AuthRequired
+    @Path("{identifier}/attribute/{attribute}")
+    public Response updateAttribute(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier,
+                                    @PathParam("attribute") String attribute, @QueryParam("value") String value) {
+        try {
+            Dataverse collection = findDataverseOrDie(identifier);
+            User user = getRequestUser(crc);
+            DataverseRequest dvRequest = createDataverseRequest(user);
+
+            // TODO: The cases below use hard coded strings, because we have no place for definitions of those!
+            //       They are taken from util.json.JsonParser / util.json.JsonPrinter. This shall be changed.
+            //       This also should be extended to more attributes, like the type, theme, contacts, some booleans, etc.
+            switch (attribute) {
+                case "alias":
+                    collection.setAlias(value);
+                    break;
+                case "name":
+                    collection.setName(value);
+                    break;
+                case "description":
+                    collection.setDescription(value);
+                    break;
+                case "affiliation":
+                    collection.setAffiliation(value);
+                    break;
+                /* commenting out the code from the draft pr #9462:
+                case "versionPidsConduct":
+                    CollectionConduct conduct = CollectionConduct.findBy(value);
+                    if (conduct == null) {
+                        return badRequest("'" + value + "' is not one of [" +
+                            String.join(",", CollectionConduct.asList()) + "]");
+                    }
+                    collection.setDatasetVersionPidConduct(conduct);
+                    break;
+                 */
+                case "filePIDsEnabled":
+                    collection.setFilePIDsEnabled(parseBooleanOrDie(value));
+                    break;
+                default:
+                    return badRequest("'" + attribute + "' is not a supported attribute");
+            }
+
+            // Off to persistence layer
+            execCommand(new UpdateDataverseCommand(collection, null, null, dvRequest, null));
+
+            // Also return modified collection to user
+            return ok("Update successful", JsonPrinter.json(collection));
+
+        // TODO: This is an anti-pattern, necessary due to this bean being an EJB, causing very noisy and unnecessary
+        //       logging by the EJB container for bubbling exceptions. (It would be handled by the error handlers.)
+        } catch (WrappedResponse e) {
+            return e.getResponse();
+        }
+    }
+
     @DELETE
     @AuthRequired
     @Path("{linkingDataverseId}/deleteLink/{linkedDataverseId}")

src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java

@@ -643,7 +643,7 @@ private boolean runAddReplacePhase1(Dataset owner,
                 df.setRootDataFileId(fileToReplace.getRootDataFileId());
             }
             // Reuse any file PID during a replace operation (if File PIDs are in use)
-            if (systemConfig.isFilePIDsEnabled()) {
+            if (systemConfig.isFilePIDsEnabledForCollection(owner.getOwner())) {
                 df.setGlobalId(fileToReplace.getGlobalId());
                 df.setGlobalIdCreateTime(fileToReplace.getGlobalIdCreateTime());
                 // Should be true or fileToReplace wouldn't have an identifier (since it's not

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java

@@ -373,7 +373,7 @@ private void publicizeExternalIdentifier(Dataset dataset, CommandContext ctxt) t
                 String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, "");
                 String currentGlobalAuthority = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, "");
                 String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
-                boolean isFilePIDsEnabled = ctxt.systemConfig().isFilePIDsEnabled();
+                boolean isFilePIDsEnabled = ctxt.systemConfig().isFilePIDsEnabledForCollection(getDataset().getOwner());
                 // We will skip trying to register the global identifiers for datafiles 
                 // if "dependent" file-level identifiers are requested, AND the naming 
                 // protocol, or the authority of the dataset global id is different from 

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java

@@ -135,7 +135,7 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException
             String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
             boolean registerGlobalIdsForFiles = 
                     (currentGlobalIdProtocol.equals(theDataset.getProtocol()) || dataFilePIDFormat.equals("INDEPENDENT")) 
-                    && ctxt.systemConfig().isFilePIDsEnabled();
+                    && ctxt.systemConfig().isFilePIDsEnabledForCollection(theDataset.getOwner());
 
             if ( registerGlobalIdsForFiles ){
                 registerGlobalIdsForFiles = currentGlobalAuthority.equals( theDataset.getAuthority() );

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java

@@ -57,7 +57,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException {
                 // didn't need updating.
                 String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, "");
                 String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
-                boolean isFilePIDsEnabled = ctxt.systemConfig().isFilePIDsEnabled();
+                boolean isFilePIDsEnabled = ctxt.systemConfig().isFilePIDsEnabledForCollection(target.getOwner());
                 // We will skip trying to update the global identifiers for datafiles if they
                 // aren't being used.
                 // If they are, we need to assure that there's an existing PID or, as when

src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java

@@ -2,6 +2,7 @@
 
 import com.ocpsoft.pretty.PrettyContext;
 import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.DvObjectContainer;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
@@ -995,9 +996,29 @@ public boolean isAllowCustomTerms() {
         return settingsService.isTrueForKey(SettingsServiceBean.Key.AllowCustomTermsOfUse, safeDefaultIfKeyNotFound);
     }
 
-    public boolean isFilePIDsEnabled() {
-        boolean safeDefaultIfKeyNotFound = true;
-        return settingsService.isTrueForKey(SettingsServiceBean.Key.FilePIDsEnabled, safeDefaultIfKeyNotFound);
+    public boolean isFilePIDsEnabledForCollection(Dataverse collection) {
+        if (collection == null) {
+            return false;
+        }
+
+        Dataverse thisCollection = collection; 
+
+        // If neither enabled nor disabled specifically for this collection,
+        // the parent collection setting is inhereted (recursively): 
+        while (thisCollection.getFilePIDsEnabled() == null) {
+            if (thisCollection.getOwner() == null) {
+                // We've reached the root collection, and file PIDs registration
+                // hasn't been explicitly enabled, therefore we presume that it is
+                // subject to how the registration is configured for the 
+                // entire instance:
+                return settingsService.isTrueForKey(SettingsServiceBean.Key.FilePIDsEnabled, true); 
+            }
+            thisCollection = thisCollection.getOwner();
+        }
+
+        // If present, the setting of the first direct ancestor collection 
+        // takes precedent:
+        return thisCollection.getFilePIDsEnabled();
     }
 
     public boolean isIndependentHandleService() {

src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java

@@ -153,6 +153,10 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
                 }
             }
         }
+
+        if (jobj.containsKey("filePIDsEnabled")) {
+            dv.setFilePIDsEnabled(jobj.getBoolean("filePIDsEnabled"));
+        }
 
         /*  We decided that subject is not user set, but gotten from the subject of the dataverse's
             datasets - leavig this code in for now, in case we need to go back to it at some point

src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java

@@ -294,6 +294,9 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail) {
         if(dv.getStorageDriverId() != null) {
         	bld.add("storageDriverLabel", DataAccess.getStorageDriverLabelFor(dv.getStorageDriverId()));
         }
+        if (dv.getFilePIDsEnabled() != null) {
+            bld.add("filePIDsEnabled", dv.getFilePIDsEnabled());
+        }
 
         return bld;
     }

src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java

@@ -394,7 +394,7 @@ private void workflowCompleted(Workflow wf, WorkflowContext ctxt) {
                 String dataFilePIDFormat = settings.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
                 boolean registerGlobalIdsForFiles = 
                         (currentGlobalIdProtocol.equals(ctxt.getDataset().getProtocol()) || dataFilePIDFormat.equals("INDEPENDENT")) 
-                        && systemConfig.isFilePIDsEnabled();
+                        && systemConfig.isFilePIDsEnabledForCollection(ctxt.getDataset().getOwner());
                 if ( registerGlobalIdsForFiles ){
                     registerGlobalIdsForFiles = currentGlobalAuthority.equals( ctxt.getDataset().getAuthority() );
                 }

src/main/resources/db/migration/V5.13.0.2__8889-filepids-in-collections.sql

@@ -0,0 +1 @@
+ALTER TABLE dataverse ADD COLUMN IF NOT EXISTS filePIDsEnabled bool;

src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java

@@ -13,6 +13,7 @@
 import static edu.harvard.iq.dataverse.api.AccessIT.apiToken;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import java.io.File;
 import java.io.IOException;
@@ -2018,4 +2019,76 @@ public void testDeleteFile() {
                 .body("data.files[0]", equalTo(null))
                 .statusCode(OK.getStatusCode());
     }
+
+    // The following specifically tests file-level PIDs configuration in 
+    // individual collections (#8889/#9614)
+    @Test
+    public void testFilePIDsBehavior() {
+        // Create user
+        String apiToken = createUserGetToken();
+
+        // Create Dataverse
+        String collectionAlias = createDataverseGetAlias(apiToken);
+
+        // Create Initial Dataset with 1 file:
+        Integer datasetId = createDatasetGetId(collectionAlias, apiToken);
+        String pathToFile = "scripts/search/data/replace_test/003.txt";
+        Response addResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
+
+        addResponse.then().assertThat()
+                .body("data.files[0].dataFile.contentType", equalTo("text/plain"))
+                .body("data.files[0].label", equalTo("003.txt"))
+                .statusCode(OK.getStatusCode());
+
+        Long origFileId = JsonPath.from(addResponse.body().asString()).getLong("data.files[0].dataFile.id");
+
+        // -------------------------
+        // Publish dataverse and dataset
+        // -------------------------
+        msg("Publish dataverse and dataset");
+        Response publishCollectionResp = UtilIT.publishDataverseViaSword(collectionAlias, apiToken);
+        publishCollectionResp.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        Response publishDatasetResp = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResp.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        // The file in this dataset should have been assigned a PID when it was published:
+        Response fileInfoResponse = UtilIT.getFileData(origFileId.toString(), apiToken);
+        fileInfoResponse.then().assertThat().statusCode(OK.getStatusCode());
+        String fileInfoResponseString = fileInfoResponse.body().asString();
+        msg(fileInfoResponseString);
+
+        String origFilePersistentId = JsonPath.from(fileInfoResponseString).getString("data.dataFile.persistentId");
+        assertNotNull("The file did not get a persistent identifier assigned (check that file PIDs are enabled instance-wide!)", origFilePersistentId);
+
+        // Now change the file PIDs registration configuration for the collection:
+
+        Response changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "filePIDsEnabled", "false", apiToken);
+
+        // ... And do the whole thing with creating another dataset with a file:
+
+        datasetId = createDatasetGetId(collectionAlias, apiToken);
+        addResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
+        addResponse.then().assertThat().statusCode(OK.getStatusCode());                
+        Long newFileId = JsonPath.from(addResponse.body().asString()).getLong("data.files[0].dataFile.id");
+
+        // And publish this dataset:
+        msg("Publish second dataset");
+
+        publishDatasetResp = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResp.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        // And confirm that the file didn't get a PID:
+
+        fileInfoResponse = UtilIT.getFileData(newFileId.toString(), apiToken);
+        fileInfoResponse.then().assertThat().statusCode(OK.getStatusCode());
+        fileInfoResponseString = fileInfoResponse.body().asString();
+        msg(fileInfoResponseString);
+
+        org.junit.Assert.assertEquals("The file was NOT supposed to be issued a PID", "", JsonPath.from(fileInfoResponseString).getString("data.dataFile.persistentId"));
+
+    }
 }

src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java

@@ -2091,6 +2091,12 @@ static Response setDataverseLogo(String dataverseAlias, String pathToImageFile,
                 .multiPart("file", new File(pathToImageFile))
                 .put("/api/dataverses/" + dataverseAlias + "/logo");
     }
+
+    static Response setCollectionAttribute(String dataverseAlias, String attribute, String value, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .put("/api/dataverses/" + dataverseAlias + "/attribute/" + attribute + "?value=" + value);
+    }
 
     /**
      * Deprecated as the apiToken is not used by the call.