Merge pull request #6 from IQSS/develop

merge upstream develop into branch pull request

conf/docker-aio/readme.md

@@ -34,7 +34,7 @@ Also usable for normal development and system evaluation; not intended for produ
 - Installation (integration test): `docker exec dv /opt/dv/setupIT.bash` 
   (Note that it's possible to customize the installation by editing `conf/docker-aio/default.config` and running `docker exec dv /opt/dv/install.bash` but for the purposes of integration testing, the `setupIT.bash` script above works fine.)
 
-- update `dataverse.siteUrl` (appears only necessary for `DatasetsIT.testPrivateUrl`): `docker exec -it dv /usr/local/glassfish4/bin/asadmin create-jvm-options "-Ddataverse.siteUrl=http\://localhost\:8084"` (or use the provided `seturl.bash`)
+- update `dataverse.siteUrl` (appears only necessary for `DatasetsIT.testPrivateUrl`): `docker exec dv /usr/local/glassfish4/bin/asadmin create-jvm-options "-Ddataverse.siteUrl=http\://localhost\:8084"` (or use the provided `seturl.bash`)
 
 #### Run integration tests: 
 

conf/docker-aio/run-test-suite.sh

@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT -Ddataverse.test.baseurl=$dvurl
+mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT -Ddataverse.test.baseurl=$dvurl

conf/docker-aio/seturl.bash

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-docker exec -it dv /usr/local/glassfish4/bin/asadmin create-jvm-options "\"-Ddataverse.siteUrl=http\://localhost\:8084\""
+docker exec dv /usr/local/glassfish4/bin/asadmin create-jvm-options "\"-Ddataverse.siteUrl=http\://localhost\:8084\""

conf/jhove/jhove.conf

@@ -40,4 +40,15 @@
  <module>
   <class>edu.harvard.hul.ois.jhove.module.Utf8Module</class>
  </module>
+ <!-- New modules for application/gzip and application/warc: -->
+ <module>
+  <class>edu.harvard.hul.ois.jhove.module.GzipModule</class>
+ </module>
+ <module>
+  <class>edu.harvard.hul.ois.jhove.module.WarcModule</class>
+ </module>
+ <!-- A new 3rd-party module for image/png from mcgauth.com: -->
+ <module>
+  <class>com.mcgath.jhove.module.PngModule</class>
+ </module>
 </jhoveConfig>

doc/release-notes/2202-improved-file-detection.md

@@ -0,0 +1,5 @@
+Upgrade instructions:
+
+A new version of file type detection software, Jhove, is added in this release. It requires an update of its configuration file: ``jhove.conf``. Download the new configuration file from the Dataverse release page on GitHub, or from the source tree at https://github.com/IQSS/dataverse/blob/master/conf/jhove/jhove.conf, and place it in ``<GLASSFISH_DOMAIN_DIRECTORY>/config/``. For example: ``/usr/local/glassfish4/glassfish/domains/domain1/config/jhove.conf``. 
+
+**Important:** If your Glassfish installation directory is different from ``/usr/local/glassfish4``, make sure to edit the header of the config file, to reflect the correct location. 

doc/sphinx-guides/source/admin/troubleshooting.rst

@@ -71,3 +71,8 @@ In real life production use, it may be possible to end up in a situation where s
 (contrary to what the message suggests, there are no specific "details" anywhere in the stack trace that would explain what values violate which constraints)  
 
 To identifiy the specific invalid values in the affected datasets, or to check all the datasets in the Dataverse for constraint violations, see :ref:`Dataset Validation <dataset-validation-api>` in the :doc:`/api/native-api` section of the User Guide.
+
+Many Files with a File Type of "Unknown", "Application", or "Binary"
+--------------------------------------------------------------------
+
+From the home page of a Dataverse installation you can get a count of files by file type by clicking "Files" and then scrolling down to "File Type". If you see a lot of files that are "Unknown", "Application", or "Binary" you can have Dataverse attempt to redetect the file type by using the :ref:`Redetect File Type <redetect-file-type>` API endpoint.

doc/sphinx-guides/source/api/native-api.rst

@@ -66,6 +66,18 @@ Show Contents of a Dataverse
 
     GET http://$SERVER/api/dataverses/$id/contents
 
+
+Report the data (file) size of a Dataverse
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Shows the combined size in bytes of all the files uploaded into the dataverse ``id``. ::
+
+    GET http://$SERVER/api/dataverses/$id/storagesize
+
+Both published and unpublished files will be counted, in the dataverse specified, and in all its sub-dataverses, recursively. 
+By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by Dataverse - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. 
+
+
 List Roles Defined in a Dataverse
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -446,6 +458,8 @@ A more detailed "add" example using curl::
 
     curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F 'file=@data.tsv' -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"true"}' "https://example.dataverse.edu/api/datasets/:persistentId/add?persistentId=$PERSISTENT_ID"
 
+Please note that it's possible to "trick" Dataverse into giving a file a content type (MIME type) of your choosing. For example, you can make a text file be treated like a video file with ``-F 'file=@README.txt;type=video/mpeg4'``, for example. If Dataverse does not properly detect a file type, specifying the content type via API like this a potential workaround.
+
 Example python code to add a file. This may be run by changing these parameters in the sample code:
 
 * ``dataverse_server`` - e.g. https://demo.dataverse.org
@@ -740,6 +754,25 @@ Note that this requires "superuser" credentials::
 
 Note: at present, the API cannot be used on a file that's already successfully ingested as tabular.
 
+.. _redetect-file-type:
+
+Redetect File Type
+~~~~~~~~~~~~~~~~~~
+
+Dataverse uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have Dataverse attempt to redetect the file type.
+
+When using the curl command below, you can pass ``dryRun=true`` if you don't want any changes to be saved to the database. Change this to ``dryRun=false`` (or omit it) to save the change. In the example below, the file is identified by database id "42".
+
+``export FILE_ID=42``
+
+``curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/files/$FILE_ID/redetect?dryRun=true``
+
+Currently the following methods are used to detect file types:
+
+- The file type detected by the browser (or sent via API).
+- JHOVE: http://jhove.openpreservation.org
+- As a last resort the file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``.
+
 Replacing Files
 ~~~~~~~~~~~~~~~
 

doc/sphinx-guides/source/conf.py

@@ -65,9 +65,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '4.14'
+version = '4.15'
 # The full version, including alpha/beta/rc tags.
-release = '4.14'
+release = '4.15'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/sphinx-guides/source/developers/tips.rst

@@ -150,6 +150,14 @@ Git on Mac
 
 On a Mac, you won't have git installed unless you have "Command Line Developer Tools" installed but running ``git clone`` for the first time will prompt you to install them.
 
+Sample Data
+-----------
+
+You may want to populate your installation of Dataverse with sample data. You have a couple options:
+
+- Code in https://github.com/IQSS/dataverse-sample-data (recommended).
+- Scripts called from ``scripts/deploy/phoenix.dataverse.org/post``.
+
 ----
 
 Previous: :doc:`dev-environment` | Next: :doc:`troubleshooting`

doc/sphinx-guides/source/installation/prerequisites.rst

@@ -228,26 +228,27 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo
 
 	# yum install lsof
 
-Finally, you may start Solr and create the core that will be used to manage search information::
+Finally, you need to tell Solr to create the core "collection1" on startup:
 
-        cd /usr/local/solr/solr-7.3.1
-        bin/solr start
-        bin/solr create_core -c collection1 -d server/solr/collection1/conf/
-	
+        echo "name=collection1" > /usr/local/solr/solr-7.3.1/server/solr/collection1/core.properties
 
 Solr Init Script
 ================
 
-For systems running systemd, as root, download :download:`solr.service<../_static/installation/files/etc/systemd/solr.service>` and place it in ``/tmp``. Then start Solr and configure it to start at boot with the following commands::
+Please choose the right option for your underlying Linux operating system.
+It will not be necessary to execute both!
+
+For systems running systemd (like CentOS/RedHat since 7, Debian since 9, Ubuntu since 15.04), as root, download :download:`solr.service<../_static/installation/files/etc/systemd/solr.service>` and place it in ``/tmp``. Then start Solr and configure it to start at boot with the following commands::
 
-        cp /tmp/solr.service /usr/lib/systemd/system
+        cp /tmp/solr.service /etc/systemd/system
+        systemctl daemon-reload
         systemctl start solr.service
         systemctl enable solr.service
 
-For systems using init.d, download this :download:`Solr init script <../_static/installation/files/etc/init.d/solr>` and place it in ``/tmp``. Then start Solr and configure it to start at boot with the following commands::
+For systems using init.d (like CentOS 6), download this :download:`Solr init script <../_static/installation/files/etc/init.d/solr>` and place it in ``/tmp``. Then start Solr and configure it to start at boot with the following commands::
 
         cp /tmp/solr /etc/init.d
-        systemctl restart solr.service
+        service start solr
         chkconfig solr on
 
 Securing Solr

doc/sphinx-guides/source/style/patterns.rst

@@ -378,7 +378,7 @@ For action buttons on a page, we include an icon and text label. Action buttons
 Form Buttons
 ------------
 
-Form buttons typically appear at the bottom of a form, aligned to the left. They do not have icons, just text labels.
+Form buttons typically appear at the bottom of a form, aligned to the left. They do not have icons, just text labels. The primary button is styled differently.
 
 .. raw:: html
 
@@ -388,7 +388,7 @@ Form buttons typically appear at the bottom of a form, aligned to the left. They
 	  		<button id="datasetForm:save" name="datasetForm:save" class="ui-button ui-widget ui-state-default ui-corner-all ui-button-text-only btn btn-default" onclick="return false;" tabindex="1000" type="submit" role="button" aria-disabled="false">
 	  			<span class="ui-button-text ui-c">Save Changes</span>
 	  		</button>
-	  		<button id="datasetForm:cancel" name="datasetForm:cancel" class="ui-button ui-widget ui-state-default ui-corner-all ui-button-text-only btn btn-default" onclick="return false;" tabindex="1000" type="submit" role="button" aria-disabled="false">
+	  		<button id="datasetForm:cancel" name="datasetForm:cancel" class="btn btn-link" onclick="return false;" tabindex="1000" type="submit" role="button" aria-disabled="false">
 	  			<span class="ui-button-text ui-c">Cancel</span>
 	  		</button>
   		</div>
@@ -399,7 +399,7 @@ Form buttons typically appear at the bottom of a form, aligned to the left. They
 
     <div class="button-block">
       <p:commandButton id="save" styleClass="btn btn-default" value="#{bundle.saveChanges}" action="#{page.save}" update="@form,:messagePanel" />
-      <p:commandButton id="cancel" styleClass="btn btn-default" value="#{bundle.cancel}" action="#{page.cancel}" process="@this" update="@form">
+      <p:commandButton id="cancel" styleClass="btn btn-link" value="#{bundle.cancel}" action="#{page.cancel}" process="@this" update="@form">
         <p:resetInput target="@form" />
       </p:commandButton>
     </div>

doc/sphinx-guides/source/versions.rst

@@ -6,9 +6,9 @@ Dataverse Guides Versions
 
 This list provides a way to refer to previous versions of the Dataverse guides, which we still host. In order to learn more about the updates delivered from one version to another, visit the `Releases <https://github.com/IQSS/dataverse/releases>`__ page in our GitHub repo.
 
-- 4.14
-
+- 4.15
 
+- `4.14 </en/4.14/>`__
 - `4.13 </en/4.13/>`__
 - `4.12 </en/4.12/>`__
 - `4.11 </en/4.11/>`__

pom.xml

@@ -7,7 +7,7 @@
     -->
     <groupId>edu.harvard.iq</groupId>
     <artifactId>dataverse</artifactId>
-    <version>4.14</version>
+    <version>4.15</version>
     <packaging>war</packaging>
     <name>dataverse</name>
     <properties>
@@ -34,6 +34,7 @@
         <junit.platform.version>1.3.1</junit.platform.version>
         <mockito.version>2.22.0</mockito.version>
         <flyway.version>5.2.4</flyway.version>
+        <jhove.version>1.20.1</jhove.version>
         <!--
         Jacoco 0.8.2 seems to break Netbeans code coverage integration so we'll use 0.8.1 instead.
         See https://github.com/jacoco/jacoco/issues/772 for discussion of how the XML changed.
@@ -376,19 +377,19 @@
             <version>4.0.0</version>
         </dependency>
         <dependency>
-            <groupId>edu.harvard.hul.ois.jhove</groupId>
-            <artifactId>jhove</artifactId>
-            <version>1.11.0</version>
+            <groupId>org.openpreservation.jhove</groupId>
+            <artifactId>jhove-core</artifactId>
+            <version>${jhove.version}</version>
         </dependency>
         <dependency>
-            <groupId>edu.harvard.hul.ois.jhove</groupId>
-            <artifactId>jhove-module</artifactId>
-            <version>1.11.0</version>
+            <groupId>org.openpreservation.jhove</groupId>
+            <artifactId>jhove-modules</artifactId>
+            <version>${jhove.version}</version>
         </dependency>
         <dependency>
-            <groupId>edu.harvard.hul.ois.jhove</groupId>
-            <artifactId>jhove-handler</artifactId>
-            <version>1.11.0</version>
+            <groupId>org.openpreservation.jhove</groupId>
+            <artifactId>jhove-ext-modules</artifactId>
+            <version>${jhove.version}</version>
         </dependency>
         <!-- JAI (Java Advanced Imaging) jars: -->
         <dependency>

src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java

@@ -66,20 +66,6 @@ public class DataFileServiceBean implements java.io.Serializable {
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
 
-    // File type "classes" tags:
-
-    private static final String FILE_CLASS_AUDIO = "audio";
-    private static final String FILE_CLASS_CODE = "code";
-    private static final String FILE_CLASS_DOCUMENT = "document";
-    private static final String FILE_CLASS_ASTRO = "astro";
-    private static final String FILE_CLASS_IMAGE = "image";
-    private static final String FILE_CLASS_NETWORK = "network";
-    private static final String FILE_CLASS_GEO = "geodata";
-    private static final String FILE_CLASS_TABULAR = "tabular";
-    private static final String FILE_CLASS_VIDEO = "video";
-    private static final String FILE_CLASS_PACKAGE = "package";
-    private static final String FILE_CLASS_OTHER = "other";
-
     // Assorted useful mime types:
 
     // 3rd-party and/or proprietary tabular data formasts that we know
@@ -1151,51 +1137,25 @@ public String getFileClassById (Long fileId) {
             return null; 
         }
 
-        return getFileClass(file);
+        return getFileThumbnailClass(file);
     }
 
-    public String getFileClass (DataFile file) {
-        if (isFileClassImage(file)) {
-            return FILE_CLASS_IMAGE;
-        }
-
-        if (isFileClassVideo(file)) {
-            return FILE_CLASS_VIDEO;
-        }
-
-        if (isFileClassAudio(file)) {
-            return FILE_CLASS_AUDIO;
-        }
-
-        if (isFileClassCode(file)) {
-            return FILE_CLASS_CODE;
-        }
-
-        if (isFileClassDocument(file)) {
-            return FILE_CLASS_DOCUMENT;
-        }
-
-        if (isFileClassAstro(file)) {
-            return FILE_CLASS_ASTRO;
-        }
-
-        if (isFileClassNetwork(file)) {
-            return FILE_CLASS_NETWORK;
-        }
-
-        if (isFileClassGeo(file)) {
-            return FILE_CLASS_GEO;
+    public String getFileThumbnailClass (DataFile file) {
+        // there's no solr search facet for "package files", but
+        // there is a special thumbnail icon:
+        if (isFileClassPackage(file)) {
+            return FileUtil.FILE_THUMBNAIL_CLASS_PACKAGE;
         }
 
-        if (isFileClassTabularData(file)) {
-            return FILE_CLASS_TABULAR;
-        }
+        if (file != null) {
+            String fileTypeFacet = FileUtil.getFacetFileType(file);
 
-        if (isFileClassPackage(file)) {
-            return FILE_CLASS_PACKAGE;
+            if (fileTypeFacet != null && FileUtil.FILE_THUMBNAIL_CLASSES.containsKey(fileTypeFacet)) {
+                return FileUtil.FILE_THUMBNAIL_CLASSES.get(fileTypeFacet);
+            }
         }
 
-        return FILE_CLASS_OTHER;
+        return FileUtil.FILE_THUMBNAIL_CLASS_OTHER;
     }