Merge branch 'develop' into 9663-add-AskDataverse

doc/release-notes/5042-add-mydata-doc-api.md

@@ -0,0 +1,2 @@
+An API named 'MyData' is supported by Dataverse. A documentation has been added describing its use (PR #9596)
+This API is used to get a list of only the objects (datasets, dataverses or datafiles) that an authenticated user can modify.

doc/sphinx-guides/source/api/native-api.rst

@@ -4680,6 +4680,7 @@ A curl example using allowing access to a dataset's metadata
 Please see :ref:`dataverse.api.signature-secret` for the configuration option to add a shared secret, enabling extra
 security.
 
+
 .. _send-feedback:
 
 Send Feedback To Contact(s)
@@ -4705,3 +4706,42 @@ A curl example using an ``ID``
   curl -X POST -H 'Content-Type:application/json' -d "$JSON" $SERVER_URL/api/admin/feedback
 
 Note that this call could be useful in coordinating with dataset authors (assuming they are also contacts) as an alternative/addition to the functionality provided by :ref:`return-a-dataset`.
+
+
+MyData
+------
+
+The MyData API is used to get a list of just the datasets, dataverses or datafiles an authenticated user can edit.
+
+A curl example listing objects
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ROLE_IDS=6
+  export DVOBJECT_TYPES=Dataset
+  export PUBLISHED_STATES=Unpublished
+  export PER_PAGE=10
+
+  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve?role_ids=$ROLE_IDS&dvobject_types=$DVOBJECT_TYPES&published_states=$PUBLISHED_STATES&per_page=$PER_PAGE"
+
+Parameters:
+
+``role_id`` Roles are customizable. Standard roles include:
+
+- ``1`` = Admin
+- ``2`` = File Downloader
+- ``3`` = Dataverse + Dataset Creator
+- ``4`` = Dataverse Creator
+- ``5`` = Dataset Creator
+- ``6`` = Contributor
+- ``7`` = Curator
+- ``8`` = Member
+
+``dvobject_types`` Type of object, several possible values among: ``DataFile`` , ``Dataset`` & ``Dataverse`` .
+
+``published_states`` State of the object, several possible values among:``Published`` , ``Unpublished`` , ``Draft`` , ``Deaccessioned`` & ``In+Review`` .
+
+``per_page`` Number of results returned per page.
+

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -2849,10 +2849,15 @@ public String refresh() {
 
         if (persistentId != null) {
             setIdByPersistentId();
-        }
-        if (dataset.getId() != null) {
-            //retrieveDatasetVersionResponse = datasetVersionService.retrieveDatasetVersionById(dataset.getId(), version);
-            dataset = datasetService.findDeep(dataset.getId());
+            if (this.getId() == null) {
+                logger.warning("No such dataset: "+persistentId);
+                return permissionsWrapper.notFound();
+            }
+            dataset = datasetService.findDeep(this.getId());
+            if (dataset == null) {
+                logger.warning("No such dataset: "+persistentId);
+                return permissionsWrapper.notFound();
+            }
             retrieveDatasetVersionResponse = datasetVersionService.selectRequestedVersion(dataset.getVersions(), version);
         } else if (versionId != null) {
             retrieveDatasetVersionResponse = datasetVersionService.retrieveDatasetVersionByVersionId(versionId);

src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java

@@ -77,7 +77,7 @@ public JsonArray getDatasetFileDetails() {
         JsonArrayBuilder jab = Json.createArrayBuilder();
         for (FileMetadata fileMetadata : dv.getFileMetadatas()) {
             DataFile dataFile = fileMetadata.getDataFile();
-            jab.add(JsonPrinter.json(dataFile, fileMetadata));
+            jab.add(JsonPrinter.json(dataFile, fileMetadata, true));
         }
         return jab.build();
     }

src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java

@@ -1712,7 +1712,7 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t
         // we're not writing the opening <dataDscr> tag until we find an actual 
         // tabular datafile.
         for (int i=0;i<fileDetails.size();i++) {
-            JsonObject fileJson = fileDetails.getJsonObject(0);
+            JsonObject fileJson = fileDetails.getJsonObject(i);
 
             /**
              * Previously (in Dataverse 5.3 and below) the dataDscr section was
@@ -1721,7 +1721,7 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t
              * should instead use the "Data Variable Metadata Access" endpoint.)
              * These days we skip restricted files to avoid this exposure.
              */
-            if (fileJson.getBoolean("restricted")) {
+            if (fileJson.containsKey("restricted") && fileJson.getBoolean("restricted")) {
                 continue;
             }
             if(fileJson.containsKey("embargo")) {
@@ -1733,7 +1733,6 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t
              }
             }
 
-
             if (fileJson.containsKey("dataTables")) {
                 if (!tabularData) {
                     xmlw.writeStartElement("dataDscr");
@@ -2025,9 +2024,12 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails)
         String dataverseUrl = SystemConfig.getDataverseSiteUrlStatic();
         for (int i =0;i<fileDetails.size();i++) {
             JsonObject fileJson = fileDetails.getJsonObject(i);
-
-            if (fileJson.containsKey("dataTables")) {
-                JsonObject dt = fileJson.getJsonArray("dataTables").getJsonObject(0);
+            //originalFileFormat is one of several keys that only exist for tabular data
+            if (fileJson.containsKey("originalFileFormat")) {
+                JsonObject dt = null;
+                if (fileJson.containsKey("dataTables")) {
+                    dt = fileJson.getJsonArray("dataTables").getJsonObject(0);
+                }
                 xmlw.writeStartElement("fileDscr");
                 String fileId = fileJson.getJsonNumber("id").toString();
                 writeAttribute(xmlw, "ID", "f" + fileId);
@@ -2038,7 +2040,8 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails)
                 xmlw.writeCharacters(fileJson.getString("filename"));
                 xmlw.writeEndElement(); // fileName
 
-                if (dt.containsKey("caseQuantity") || dt.containsKey("varQuantity") || dt.containsKey("recordsPerCase")) {
+                if (dt != null && (dt.containsKey("caseQuantity") || dt.containsKey("varQuantity")
+                        || dt.containsKey("recordsPerCase"))) {
                     xmlw.writeStartElement("dimensns");
 
                     if (dt.containsKey("caseQuantity")) {
@@ -2071,7 +2074,7 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails)
                 // various notes:
                 // this specially formatted note section is used to store the UNF
                 // (Universal Numeric Fingerprint) signature:
-                if (dt.containsKey("UNF") && !dt.getString("UNF").isBlank()) {
+                if ((dt!=null) && (dt.containsKey("UNF") && !dt.getString("UNF").isBlank())) {
                     xmlw.writeStartElement("notes");
                     writeAttribute(xmlw, "level", LEVEL_FILE);
                     writeAttribute(xmlw, "type", NOTE_TYPE_UNF);

src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java

@@ -617,7 +617,7 @@ public static JsonObjectBuilder json(FileMetadata fmd) {
                 .add("version", fmd.getVersion())
                 .add("datasetVersionId", fmd.getDatasetVersion().getId())
                 .add("categories", getFileCategories(fmd))
-                .add("dataFile", JsonPrinter.json(fmd.getDataFile(), fmd));
+                .add("dataFile", JsonPrinter.json(fmd.getDataFile(), fmd, false));
     }
 
       public static JsonObjectBuilder json(AuxiliaryFile auxFile) {
@@ -633,10 +633,10 @@ public static JsonObjectBuilder json(AuxiliaryFile auxFile) {
                 .add("dataFile", JsonPrinter.json(auxFile.getDataFile()));
     }
     public static JsonObjectBuilder json(DataFile df) {
-        return JsonPrinter.json(df, null);
+        return JsonPrinter.json(df, null, false);
     }
 
-    public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata) {
+    public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider) {
         // File names are no longer stored in the DataFile entity; 
         // (they are instead in the FileMetadata (as "labels") - this way 
         // the filename can change between versions... 
@@ -661,7 +661,7 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata) {
 
         JsonObjectBuilder embargo = df.getEmbargo() != null ? JsonPrinter.json(df.getEmbargo()) : null;
 
-        return jsonObjectBuilder()
+        NullSafeJsonBuilder builder = jsonObjectBuilder()
                 .add("id", df.getId())
                 .add("persistentId", pidString)
                 .add("pidURL", pidURL)
@@ -672,7 +672,6 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata) {
                 .add("categories", getFileCategories(fileMetadata))
                 .add("embargo", embargo)
                 //.add("released", df.isReleased())
-                .add("restricted", df.isRestricted())
                 .add("storageIdentifier", df.getStorageIdentifier())
                 .add("originalFileFormat", df.getOriginalFileFormat())
                 .add("originalFormatLabel", df.getOriginalFormatLabel())
@@ -691,12 +690,17 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata) {
                 //---------------------------------------------
                 .add("md5", getMd5IfItExists(df.getChecksumType(), df.getChecksumValue()))
                 .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue()))
-                .add("fileMetadataId", fileMetadata.getId())
                 .add("tabularTags", getTabularFileTags(df))
-                .add("creationDate",  df.getCreateDateFormattedYYYYMMDD())
-                .add("dataTables", df.getDataTables().isEmpty() ? null : JsonPrinter.jsonDT(df.getDataTables()))
-                .add("varGroups", fileMetadata.getVarGroups().isEmpty()
-                        ? null: JsonPrinter.jsonVarGroup(fileMetadata.getVarGroups()));
+                .add("creationDate", df.getCreateDateFormattedYYYYMMDD());
+        /*
+         * The restricted state was not included prior to #9175 so to avoid backward
+         * incompatability, it is now only added when generating json for the
+         * InternalExportDataProvider fileDetails.
+         */
+        if (forExportDataProvider) {
+            builder.add("restricted", df.isRestricted());
+        }
+        return builder;
     }
 
     //Started from https://github.com/RENCI-NRIG/dataverse/, i.e. https://github.com/RENCI-NRIG/dataverse/commit/2b5a1225b42cf1caba85e18abfeb952171c6754a