diff --git a/doc/release-notes/8732-date-in-citation-harvested-datasets.md b/doc/release-notes/8732-date-in-citation-harvested-datasets.md new file mode 100644 index 00000000000..85f2d24a8a9 --- /dev/null +++ b/doc/release-notes/8732-date-in-citation-harvested-datasets.md @@ -0,0 +1,7 @@ +Fix the year displayed in citation for harvested dataset, specialy for oai_dc format. + +For normal datasets, the date used is the "citation date" which is by default the publication date (the first release date) (https://guides.dataverse.org/en/latest/api/native-api.html?highlight=citationdate#set-citation-date-field-type-for-a-dataset). + +But for a harvested dataset, the distribution date is used instead and this date is not always present in the harvested metadata. With oai_dc format the date tag if used as production date. + +Now, the production date is used for harvested dataset in addition to distribution date. \ No newline at end of file diff --git a/doc/release-notes/8733-oai_dc-date.md b/doc/release-notes/8733-oai_dc-date.md new file mode 100644 index 00000000000..a2a09f361d3 --- /dev/null +++ b/doc/release-notes/8733-oai_dc-date.md @@ -0,0 +1,4 @@ +For exports and harvesting in `oai_dc` format, if "Production Date" is not set, "Publication Date" is now used instead. This change is reflected in the [Dataverse 4+ Metadata Crosswalk][] linked from the [Appendix][] of the User Guide. + +[Dataverse 4+ Metadata Crosswalk]: https://docs.google.com/spreadsheets/d/10Luzti7svVTVKTA-px27oq3RxCUM-QbiTkm8iMd5C54/edit#gid=1901625433&range=K7 +[Appendix]: https://guides.dataverse.org/en/latest/user/appendix.html diff --git a/doc/sphinx-guides/source/_static/util/clear_timer.sh b/doc/sphinx-guides/source/_static/util/clear_timer.sh index 3fcd9e8a387..1d9966e4e07 100755 --- a/doc/sphinx-guides/source/_static/util/clear_timer.sh +++ b/doc/sphinx-guides/source/_static/util/clear_timer.sh @@ -17,7 +17,7 @@ DV_DIR=${PAYARA_DIR}/glassfish/domains/domain1 ${PAYARA_DIR}/bin/asadmin stop-domain rm -rf ${PAYARA_DIR}/${DV_DIR}/generated/ -rm -rf ${PAYARA_DIR}/${DV_DIR}/osgi-cache/felix +rm -rf ${PAYARA_DIR}/${DV_DIR}/osgi-cache/ # restart the domain (also generates a warning if app server is stopped) ${PAYARA_DIR}/bin/asadmin start-domain diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index 4e3f6ac0038..88004d9dc5f 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -26,7 +26,7 @@ The email portion of :ref:`systemEmail` will be visible via OAI-PMH (from the "I How does it work? ----------------- -Only the published, unrestricted datasets in your Dataverse installation can +Only the published datasets in your Dataverse installation can be made harvestable. Remote clients normally keep their records in sync through scheduled incremental updates, daily or weekly, thus minimizing the load on your server. Note that it is only the metadata diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index 79ce98322a8..9f085ba90cd 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -57,7 +57,7 @@ Ingest is both CPU- and memory-intensive, and depending on your system resources ``/usr/local/payara5/mq/bin/imqcmd -u admin purge dst -t q -n DataverseIngest`` will purge the DataverseIngest queue, and prompt for your confirmation. -Finally, list destinations to verify that the purge was successful:: +Finally, list destinations to verify that the purge was successful: ``/usr/local/payara5/mq/bin/imqcmd -u admin list dst`` diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index c8175f21591..880ed561720 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -66,9 +66,9 @@ # built documents. # # The short X.Y version. -version = '5.11.1' +version = '5.12' # The full version, including alpha/beta/rc tags. -release = '5.11.1' +release = '5.12' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst index 7abae535276..0d16a299fce 100644 --- a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst @@ -1,5 +1,8 @@ Dataset Semantic Metadata API ============================= +.. contents:: |toctitle| + :local: + The OAI_ORE metadata export format represents Dataset metadata using json-ld (see the :doc:`/admin/metadataexport` section). As part of an RDA-supported effort to allow import of Datasets exported as Bags with an included OAI_ORE metadata file, an experimental API has been created that provides a json-ld alternative to the v1.0 API calls to get/set/delete Dataset metadata in the :doc:`/api/native-api`. diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index f46b9477d92..1cbd785b5dd 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -6,7 +6,8 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. -- 5.11.1 +- 5.12 +- `5.11.1 `__ - `5.11 `__ - `5.10.1 `__ - `5.10 `__ diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar deleted file mode 100644 index a23530b895c..00000000000 Binary files a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.md5 b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.md5 deleted file mode 100644 index d8b1ce2fa75..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -f578d8ec91811d5d72981355cb7a1f0f diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.sha1 b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.sha1 deleted file mode 100644 index 4c7d114634b..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -523abaf48b4423eb874dbc086b876aa917930a04 diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom deleted file mode 100644 index 2915745c27d..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom +++ /dev/null @@ -1,77 +0,0 @@ - - - - xoai - com.lyncode - 4.1.0 - - 4.0.0 - - XOAI Commons - xoai-common - 4.1.0-header-patch - - - - com.lyncode - xml-io - - - com.lyncode - test-support - - - - commons-codec - commons-codec - - - - commons-io - commons-io - - - - com.google.guava - guava - - - - xml-apis - xml-apis - - - - org.hamcrest - hamcrest-all - - - - org.codehaus.woodstox - stax2-api - - - - javax.xml.stream - stax-api - - - - org.apache.commons - commons-lang3 - - - - stax - stax-api - - - - junit - junit - test - - - - - diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.md5 b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.md5 deleted file mode 100644 index 15f47f4140a..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.md5 +++ /dev/null @@ -1 +0,0 @@ -346e9f235523e52256006bbe8eba60bb diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.sha1 b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.sha1 deleted file mode 100644 index 88668a4d49c..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.sha1 +++ /dev/null @@ -1 +0,0 @@ -cd83d08c097d6aa1b27b20ef4742c7e4fa47e6b5 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar deleted file mode 100644 index 28e5da7b0d6..00000000000 Binary files a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.md5 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.md5 deleted file mode 100644 index 67dda34a6c9..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -546f1ab3f3f654280f88e429ba3471ae diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.sha1 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.sha1 deleted file mode 100644 index 50e8f2f42cd..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f4da7ebc3fda69e1e7db12bda6d7b5fb4aecc7a4 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar deleted file mode 100644 index bdec990e2c6..00000000000 Binary files a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.md5 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.md5 deleted file mode 100644 index 77416d80f87..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -ec87ba7cb8e7396fc903acdbacd31ff6 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.sha1 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.sha1 deleted file mode 100644 index a6157b4dc0b..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -370c2955550a42b11fe7b9007771c506f5769639 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar deleted file mode 100644 index 331c9a80cd1..00000000000 Binary files a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.md5 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.md5 deleted file mode 100644 index 27381662d09..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -544e9b97062d054370695b9b09d4bb1c diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.sha1 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.sha1 deleted file mode 100644 index 37dd4e47c2c..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -67c505461f3c190894bb036cc866eb640c2f6a48 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom deleted file mode 100644 index 87d67b8c4a7..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom +++ /dev/null @@ -1,54 +0,0 @@ - - - - xoai - com.lyncode - 4.1.0-header-patch - - - 4.0.0 - - XOAI Data Provider - xoai-data-provider - 4.1.0-header-patch - - - - com.lyncode - xoai-common - ${project.version} - - - - log4j - log4j - - - - com.google.guava - guava - - - - com.lyncode - builder-commons - - - - org.apache.commons - commons-lang3 - - - - org.mockito - mockito-all - test - - - - junit - junit - test - - - diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.md5 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.md5 deleted file mode 100644 index 5959ea476c7..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.md5 +++ /dev/null @@ -1 +0,0 @@ -a1b49c13fcf448de9628798f8682fcaa diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.sha1 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.sha1 deleted file mode 100644 index 87fd86c23e0..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.sha1 +++ /dev/null @@ -1 +0,0 @@ -41be98af31f8d17d83ab6c38bd7939ba212eab8d diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar deleted file mode 100644 index 4382b3ded5d..00000000000 Binary files a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.md5 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.md5 deleted file mode 100644 index c9e720e6039..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -21bc45a29b715720f4b77f51bf9f1754 diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.sha1 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.sha1 deleted file mode 100644 index 756955d2840..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b544162e82d322116b87d99f2fbb6ddd4c4745e1 diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar deleted file mode 100644 index 314dad81872..00000000000 Binary files a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.md5 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.md5 deleted file mode 100644 index 27c55f9af58..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -88dc05805672ebe01ded1197a582cd60 diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.sha1 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.sha1 deleted file mode 100644 index 1098e506b93..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fe41289cb74c56e9282dd09c22df2eda47c68a0d diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar deleted file mode 100644 index 781fc1ce1e2..00000000000 Binary files a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.md5 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.md5 deleted file mode 100644 index 84891040047..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -52f8b446f78009757d593312778f428c diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.sha1 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.sha1 deleted file mode 100644 index dbded3dd83f..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -feb6903ad32d4b42461b7ca1b3fae6146740bb31 diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom deleted file mode 100644 index c45e15a91f9..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom +++ /dev/null @@ -1,67 +0,0 @@ - - - - xoai - com.lyncode - 4.1.0-header-patch - - 4.0.0 - - XOAI Service Provider - xoai-service-provider - 4.1.0-header-patch - - - - com.lyncode - xoai-common - ${project.version} - - - - com.lyncode - xml-io - - - - log4j - log4j - - - - org.apache.commons - commons-lang3 - - - - org.apache.httpcomponents - httpclient - - - - org.codehaus.woodstox - wstx-asl - - - - - com.lyncode - xoai-data-provider - ${project.version} - test - - - - org.mockito - mockito-all - test - - - - junit - junit - test - - - - diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.md5 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.md5 deleted file mode 100644 index 5e51f198572..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.md5 +++ /dev/null @@ -1 +0,0 @@ -b97b8ee92daa5fc4fd87004465f9ad2b diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.sha1 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.sha1 deleted file mode 100644 index 2c6dc74f02b..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.sha1 +++ /dev/null @@ -1 +0,0 @@ -f772583549263bd72ea4d5268d9db0a84c27cb9f diff --git a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom b/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom deleted file mode 100644 index 89a14d88c51..00000000000 --- a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom +++ /dev/null @@ -1,273 +0,0 @@ - - 4.0.0 - pom - - - xoai-common - xoai-data-provider - xoai-service-provider - - - - org.sonatype.oss - oss-parent - 7 - - - com.lyncode - xoai - 4.1.0-header-patch - - XOAI : OAI-PMH Java Toolkit - http://www.lyncode.com - - - 1.9.5 - 15.0 - 3.1 - 1.2.14 - 4.2.1 - 4.0.0 - - 1.0.2 - 1.0.3 - 1.0.4 - - - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - scm:git:git@github.com:lyncode/xoai.git - scm:git:git@github.com:lyncode/xoai.git - git@github.com:lyncode/xoai.git - xoai-4.1.0 - - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.8.1 - - - org.apache.maven.plugins - maven-source-plugin - 2.2.1 - - - org.apache.maven.plugins - maven-release-plugin - 2.5 - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - - - - - org.apache.maven.plugins - maven-release-plugin - - true - false - release - deploy - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.6 - 1.6 - false - false - true - - - - org.apache.maven.plugins - maven-javadoc-plugin - true - - - attach-javadocs - - jar - - - - - - org.apache.maven.plugins - maven-source-plugin - true - - - attach-sources - - jar - - - - - - - - - - - com.lyncode - xml-io - ${lyncode.xml-io} - - - - com.lyncode - test-support - ${lyncode.test-support} - - - - - log4j - log4j - ${log4j.version} - - - - org.apache.commons - commons-lang3 - ${commons.lang3.version} - - - - org.apache.httpcomponents - httpclient - ${http-commons.version} - - - - org.codehaus.woodstox - wstx-asl - ${woodstox.version} - - - - org.codehaus.woodstox - stax2-api - 3.0.4 - - - - commons-codec - commons-codec - 1.3 - - - org.hamcrest - hamcrest-all - 1.3 - - - xalan - xalan - 2.7.2 - - - dom4j - dom4j - 1.6.1 - - - - javax.xml.stream - stax-api - 1.0-2 - - - jaxen - jaxen - 1.1.4 - - - junit - junit - 4.11 - - - commons-io - commons-io - 2.4 - - - - xml-apis - xml-apis - 1.0.b2 - - - - stax - stax-api - 1.0.1 - - - - org.mockito - mockito-all - ${mockito.version} - - - - com.google.guava - guava - ${guava.version} - - - - com.lyncode - builder-commons - ${lyncode.builder-commons} - - - - - - - - DSpace @ Lyncode - dspace@lyncode.com - Lyncode - http://www.lyncode.com - - - - diff --git a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.md5 b/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.md5 deleted file mode 100644 index d2fdadd114f..00000000000 --- a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.md5 +++ /dev/null @@ -1 +0,0 @@ -b50966bebe8cfdcb58478cf029b08aa3 diff --git a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.sha1 b/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.sha1 deleted file mode 100644 index b142cd649e8..00000000000 --- a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.sha1 +++ /dev/null @@ -1 +0,0 @@ -28a5d65399cbc25b29b270caebbb86e292c5ba18 diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 6303fc17a60..ccc0a9a7f60 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -129,7 +129,7 @@ - 5.11.1 + 5.12 11 UTF-8 @@ -161,6 +161,9 @@ 1.21 4.5.13 4.4.14 + + + 5.0.0-RC1 1.15.0 @@ -301,7 +304,7 @@ Local repository for hosting jars not available from network repositories. file://${project.basedir}/local_lib - - - - - - - - - - - + - com.lyncode - xoai-common - 4.1.0-header-patch - - - com.lyncode + io.gdcc xoai-data-provider - 4.1.0-header-patch - - - log4j - log4j - - + ${gdcc.xoai.version} - com.lyncode + io.gdcc xoai-service-provider - 4.1.0-header-patch - - - log4j - log4j - - - - - - - - ch.qos.reload4j - reload4j - ${reload4j.version} - runtime + ${gdcc.xoai.version} diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index 9027be1350b..abe3cc3e6d7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -33,6 +33,7 @@ import javax.xml.stream.XMLStreamWriter; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.DateUtil; import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; @@ -713,25 +714,24 @@ private String flattenHtml(String html) { private Date getDateFrom(DatasetVersion dsv) { Date citationDate = null; - SimpleDateFormat sdf = new SimpleDateFormat("yyyy"); - if (!dsv.getDataset().isHarvested()) { - citationDate = dsv.getCitationDate(); + + if (dsv.getDataset().isHarvested()) { + citationDate = DateUtil.parseDate(dsv.getProductionDate()); if (citationDate == null) { - if (dsv.getDataset().getCitationDate() != null) { - citationDate = dsv.getDataset().getCitationDate(); - } else { // for drafts - citationDate = dsv.getLastUpdateTime(); - } + citationDate = DateUtil.parseDate(dsv.getDistributionDate()); } - } else { - try { - citationDate= sdf.parse(dsv.getDistributionDate()); - } catch (ParseException ex) { - // ignore - } catch (Exception ex) { - // ignore + } + + if (citationDate == null) { + if (dsv.getCitationDate() != null) { + citationDate = dsv.getCitationDate(); + } else if (dsv.getDataset().getCitationDate() != null) { + citationDate = dsv.getDataset().getCitationDate(); + } else { // for drafts + citationDate = dsv.getLastUpdateTime(); } } + if (citationDate == null) { //As a last resort, pick the current date logger.warning("Unable to find citation date for datasetversion: " + dsv.getId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java index 9278faf1f8f..4409d2340b1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java @@ -122,7 +122,12 @@ private static void createDC(XMLStreamWriter xmlw, DatasetDTO datasetDto, String writeFullElementList(xmlw, dcFlavor+":"+"language", dto2PrimitiveList(version, DatasetFieldConstant.language)); writeRelPublElement(xmlw, version, dcFlavor); - writeFullElement(xmlw, dcFlavor+":"+"date", dto2Primitive(version, DatasetFieldConstant.productionDate)); + + String date = dto2Primitive(version, DatasetFieldConstant.productionDate); + if (date == null) { + date = datasetDto.getPublicationDate(); + } + writeFullElement(xmlw, dcFlavor+":"+"date", date); writeFullElement(xmlw, dcFlavor+":"+"contributor", dto2Primitive(version, DatasetFieldConstant.depositor)); @@ -172,7 +177,11 @@ private static void createOAIDC(XMLStreamWriter xmlw, DatasetDTO datasetDto, Str writeFullElementList(xmlw, dcFlavor+":"+"language", dto2PrimitiveList(version, DatasetFieldConstant.language)); - writeFullElement(xmlw, dcFlavor+":"+"date", dto2Primitive(version, DatasetFieldConstant.productionDate)); + String date = dto2Primitive(version, DatasetFieldConstant.productionDate); + if (date == null) { + date = datasetDto.getPublicationDate(); + } + writeFullElement(xmlw, dcFlavor+":"+"date", date); writeFullElement(xmlw, dcFlavor+":"+"contributor", dto2Primitive(version, DatasetFieldConstant.depositor)); diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java index 60abc97bccd..5b3e4df331d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java @@ -72,7 +72,6 @@ public class FastGetRecord { - private static final String DATAVERSE_EXTENDED_METADATA = "dataverse_json"; private static final String XML_METADATA_TAG = "metadata"; private static final String XML_METADATA_TAG_OPEN = "<"+XML_METADATA_TAG+">"; private static final String XML_METADATA_TAG_CLOSE = ""; @@ -222,13 +221,7 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref //metadataOut.println(""); /* ? */ metadataFlag = true; - } else if (line.matches(".*<"+XML_METADATA_TAG+" [^>]*>.*")) { - if (metadataPrefix.equals(DATAVERSE_EXTENDED_METADATA)) { - oaiResponseHeader = oaiResponseHeader.concat(line); - metadataWritten = true; - metadataFlag = true; - } - } + } } //System.out.println(line); @@ -380,19 +373,12 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref try { StringReader reader = new StringReader(oaiResponseHeader); xmlr = xmlInputFactory.createXMLStreamReader(reader); - processOAIheader(xmlr, metadataPrefix.equals(DATAVERSE_EXTENDED_METADATA)); + processOAIheader(xmlr); } catch (XMLStreamException ex) { - //Logger.getLogger("global").log(Level.SEVERE, null, ex); if (this.errorMessage == null) { this.errorMessage = "Malformed GetRecord response; baseURL=" + baseURL + ", identifier=" + identifier + ", metadataPrefix=" + metadataPrefix; } - - // delete the temp metadata file; we won't need it: - if (savedMetadataFile != null) { - //savedMetadataFile.delete(); - } - } try { @@ -414,14 +400,8 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref if (!(metadataWritten) && !(this.isDeleted())) { this.errorMessage = "Failed to parse GetRecord response; baseURL=" + baseURL + ", identifier=" + identifier + ", metadataPrefix=" + metadataPrefix; - //savedMetadataFile.delete(); - } - - if (this.isDeleted()) { - //savedMetadataFile.delete(); } - } else { this.errorMessage = "GetRecord request failed. HTTP error code "+responseCode; } @@ -445,16 +425,16 @@ private static String getRequestURL(String baseURL, return requestURL.toString(); } - private void processOAIheader (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException { + private void processOAIheader (XMLStreamReader xmlr) throws XMLStreamException, IOException { // is this really a GetRecord response? xmlr.nextTag(); xmlr.require(XMLStreamConstants.START_ELEMENT, null, "OAI-PMH"); - processOAIPMH(xmlr, extensionMode); + processOAIPMH(xmlr); } - private void processOAIPMH (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException { + private void processOAIPMH (XMLStreamReader xmlr) throws XMLStreamException, IOException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { @@ -477,7 +457,7 @@ else if (xmlr.getLocalName().equals("error")) { } else if (xmlr.getLocalName().equals("GetRecord")) { - processGetRecordSection(xmlr, extensionMode); + processGetRecordSection(xmlr); } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("OAI-PMH")) return; @@ -485,11 +465,11 @@ else if (xmlr.getLocalName().equals("GetRecord")) { } } - private void processGetRecordSection (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException { + private void processGetRecordSection (XMLStreamReader xmlr) throws XMLStreamException, IOException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("record")) { - processRecord(xmlr, extensionMode); + processRecord(xmlr); } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("GetRecord")) return; @@ -498,7 +478,7 @@ private void processGetRecordSection (XMLStreamReader xmlr, boolean extensionMod } - private void processRecord (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException { + private void processRecord (XMLStreamReader xmlr) throws XMLStreamException, IOException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("header")) { @@ -506,11 +486,6 @@ private void processRecord (XMLStreamReader xmlr, boolean extensionMode) throws this.recordDeleted = true; } processHeader(xmlr); - } else if (xmlr.getLocalName().equals("metadata")) { - if (extensionMode) { - String extendedMetadataApiUrl = xmlr.getAttributeValue(null, "directApiCall"); - processMetadataExtended(extendedMetadataApiUrl); - } } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("record")) return; @@ -532,67 +507,6 @@ else if (xmlr.getLocalName().equals("setSpec")) {/*do nothing*/} } } - private void processMetadataExtended (String extendedApiUrl) throws IOException { - InputStream in = null; - int responseCode = 0; - HttpURLConnection con = null; - - - - try { - URL url = new URL(extendedApiUrl.replaceAll("&", "&")); // is this necessary? - - con = (HttpURLConnection) url.openConnection(); - con.setRequestProperty("User-Agent", "DataverseHarvester/3.0"); - responseCode = con.getResponseCode(); - } catch (MalformedURLException mue) { - throw new IOException ("Bad API URL: "+extendedApiUrl); - } catch (FileNotFoundException e) { - responseCode = HttpURLConnection.HTTP_UNAVAILABLE; - } - - - - - if (responseCode == 200) { - in = con.getInputStream(); - // TODO: - /* we should probably still support gzip/compress encoding here - ? - String contentEncoding = con.getHeaderField("Content-Encoding"); - - // support for the standard compress/gzip/deflate compression - // schemes: - - if ("compress".equals(contentEncoding)) { - ZipInputStream zis = new ZipInputStream(con.getInputStream()); - zis.getNextEntry(); - in = zis; - } else if ("gzip".equals(contentEncoding)) { - in = new GZIPInputStream(con.getInputStream()); - } else if ("deflate".equals(contentEncoding)) { - in = new InflaterInputStream(con.getInputStream()); - } ... - */ - FileOutputStream tempOut = new FileOutputStream(savedMetadataFile); - - int bufsize; - byte[] buffer = new byte[4 * 8192]; - - while ((bufsize = in.read(buffer)) != -1) { - tempOut.write(buffer, 0, bufsize); - tempOut.flush(); - } - - in.close(); - tempOut.close(); - return; - } - - throw new IOException("Failed to download extended metadata."); - - } - - // (from Gustavo's ddiServiceBean -- L.A.) // /* We had to add this method because the ref getElementText has a bug where it diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index 71cc23e242b..e7156dfe9aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -5,6 +5,7 @@ */ package edu.harvard.iq.dataverse.harvest.client; +import static java.net.HttpURLConnection.HTTP_OK; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.Dataverse; @@ -17,7 +18,6 @@ import java.util.Date; import java.util.Iterator; import java.util.List; -//import java.net.URLEncoder; import java.util.logging.FileHandler; import java.util.logging.Level; import java.util.logging.Logger; @@ -28,22 +28,26 @@ import javax.ejb.Stateless; import javax.ejb.Timer; import javax.inject.Named; -//import javax.xml.bind.Unmarshaller; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import org.apache.commons.lang3.mutable.MutableBoolean; -import org.apache.commons.lang3.mutable.MutableLong; import org.xml.sax.SAXException; -import com.lyncode.xoai.model.oaipmh.Header; +import io.gdcc.xoai.model.oaipmh.results.record.Header; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.api.imports.ImportServiceBean; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.harvest.client.oai.OaiHandler; import edu.harvard.iq.dataverse.harvest.client.oai.OaiHandlerException; import edu.harvard.iq.dataverse.search.IndexServiceBean; +import java.io.FileOutputStream; import java.io.FileWriter; +import java.io.InputStream; import java.io.PrintWriter; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -75,13 +79,12 @@ public class HarvesterServiceBean { IndexServiceBean indexService; private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean"); - private static final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss"); public static final String HARVEST_RESULT_SUCCESS="success"; public static final String HARVEST_RESULT_FAILED="failed"; - private static final Long INDEXING_CONTENT_BATCH_SIZE = 10000000L; - + public static final String DATAVERSE_PROPRIETARY_METADATA_FORMAT="dataverse_json"; + public static final String DATAVERSE_PROPRIETARY_METADATA_API="/api/datasets/export?exporter="+DATAVERSE_PROPRIETARY_METADATA_FORMAT+"&persistentId="; public HarvesterServiceBean() { @@ -183,24 +186,7 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingClientConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingClientConfig.getMetadataPrefix()); hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: " + deletedIdentifiers.size() + ", datasets failed: " + failedIdentifiers.size()); - // now index all the datasets we have harvested - created, modified or deleted: - /* (TODO: may not be needed at all. In Dataverse4, we may be able to get away with the normal - reindexing after every import. See the rest of the comments about batch indexing throughout - this service bean) - if (this.processedSizeThisBatch > 0) { - hdLogger.log(Level.INFO, "POST HARVEST, reindexing the remaining studies."); - if (this.harvestedDatasetIdsThisBatch != null) { - hdLogger.log(Level.INFO, this.harvestedDatasetIdsThisBatch.size()+" studies in the batch"); - } - hdLogger.log(Level.INFO, this.processedSizeThisBatch + " bytes of content"); - indexService.updateIndexList(this.harvestedDatasetIdsThisBatch); - hdLogger.log(Level.INFO, "POST HARVEST, calls to index finished."); - } else { - hdLogger.log(Level.INFO, "(All harvested content already reindexed)"); - } - */ } - //mailService.sendHarvestNotification(...getSystemEmail(), harvestingDataverse.getName(), logFileName, logTimestamp, harvestErrorOccurred.booleanValue(), harvestedDatasetIds.size(), failedIdentifiers); } catch (Throwable e) { harvestErrorOccurred.setValue(true); String message = "Exception processing harvest, server= " + harvestingClientConfig.getHarvestingUrl() + ",format=" + harvestingClientConfig.getMetadataPrefix() + " " + e.getClass().getName() + " " + e.getMessage(); @@ -235,8 +221,8 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien logBeginOaiHarvest(hdLogger, harvestingClient); List harvestedDatasetIds = new ArrayList(); - MutableLong processedSizeThisBatch = new MutableLong(0L); OaiHandler oaiHandler; + HttpClient httpClient = null; try { oaiHandler = new OaiHandler(harvestingClient); @@ -248,22 +234,35 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien hdLogger.log(Level.SEVERE, errorMessage); throw new IOException(errorMessage); } - + + if (DATAVERSE_PROPRIETARY_METADATA_FORMAT.equals(oaiHandler.getMetadataPrefix())) { + // If we are harvesting native Dataverse json, we'll also need this + // jdk http client to make direct calls to the remote Dataverse API: + httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.ALWAYS).build(); + } + try { for (Iterator
idIter = oaiHandler.runListIdentifiers(); idIter.hasNext();) { Header h = idIter.next(); String identifier = h.getIdentifier(); - Date dateStamp = h.getDatestamp(); + Date dateStamp = Date.from(h.getDatestamp()); hdLogger.info("processing identifier: " + identifier + ", date: " + dateStamp); + + if (h.isDeleted()) { + hdLogger.info("Deleting harvesting dataset for " + identifier + ", per ListIdentifiers."); + + deleteHarvestedDatasetIfExists(identifier, oaiHandler.getHarvestingClient().getDataverse(), dataverseRequest, deletedIdentifiers, hdLogger); + continue; + } MutableBoolean getRecordErrorOccurred = new MutableBoolean(false); // Retrieve and process this record with a separate GetRecord call: - Long datasetId = processRecord(dataverseRequest, hdLogger, importCleanupLog, oaiHandler, identifier, getRecordErrorOccurred, processedSizeThisBatch, deletedIdentifiers, dateStamp); - hdLogger.info("Total content processed in this batch so far: "+processedSizeThisBatch); + Long datasetId = processRecord(dataverseRequest, hdLogger, importCleanupLog, oaiHandler, identifier, getRecordErrorOccurred, deletedIdentifiers, dateStamp, httpClient); + if (datasetId != null) { harvestedDatasetIds.add(datasetId); @@ -280,20 +279,6 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien //temporary: //throw new IOException("Exception occured, stopping harvest"); } - - // reindexing in batches? - this is from DVN 3; - // we may not need it anymore. - if ( processedSizeThisBatch.longValue() > INDEXING_CONTENT_BATCH_SIZE ) { - - hdLogger.log(Level.INFO, "REACHED CONTENT BATCH SIZE LIMIT; calling index ("+ harvestedDatasetIdsThisBatch.size()+" datasets in the batch)."); - //indexService.updateIndexList(this.harvestedDatasetIdsThisBatch); - hdLogger.log(Level.INFO, "REINDEX DONE."); - - - processedSizeThisBatch.setValue(0L); - harvestedDatasetIdsThisBatch = null; - } - } } catch (OaiHandlerException e) { throw new IOException("Failed to run ListIdentifiers: " + e.getMessage()); @@ -303,57 +288,56 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien return harvestedDatasetIds; - } - + } - - private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, PrintWriter importCleanupLog, OaiHandler oaiHandler, String identifier, MutableBoolean recordErrorOccurred, MutableLong processedSizeThisBatch, List deletedIdentifiers, Date dateStamp) { + private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, PrintWriter importCleanupLog, OaiHandler oaiHandler, String identifier, MutableBoolean recordErrorOccurred, List deletedIdentifiers, Date dateStamp, HttpClient httpClient) { String errMessage = null; Dataset harvestedDataset = null; logGetRecord(hdLogger, oaiHandler, identifier); File tempFile = null; - try { - FastGetRecord record = oaiHandler.runGetRecord(identifier); - errMessage = record.getErrorMessage(); + try { + boolean deleted = false; + + if (DATAVERSE_PROPRIETARY_METADATA_FORMAT.equals(oaiHandler.getMetadataPrefix())) { + // Make direct call to obtain the proprietary Dataverse metadata + // in JSON from the remote Dataverse server: + String metadataApiUrl = oaiHandler.getProprietaryDataverseMetadataURL(identifier); + logger.info("calling "+metadataApiUrl); + tempFile = retrieveProprietaryDataverseMetadata(httpClient, metadataApiUrl); + + } else { + FastGetRecord record = oaiHandler.runGetRecord(identifier); + errMessage = record.getErrorMessage(); + deleted = record.isDeleted(); + tempFile = record.getMetadataFile(); + } if (errMessage != null) { hdLogger.log(Level.SEVERE, "Error calling GetRecord - " + errMessage); - } else if (record.isDeleted()) { - hdLogger.info("Deleting harvesting dataset for "+identifier+", per the OAI server's instructions."); - Dataset dataset = datasetService.getDatasetByHarvestInfo(oaiHandler.getHarvestingClient().getDataverse(), identifier); - if (dataset != null) { - hdLogger.info("Deleting dataset " + dataset.getGlobalIdString()); - datasetService.deleteHarvestedDataset(dataset, dataverseRequest, hdLogger); - // TODO: - // check the status of that Delete - see if it actually succeeded - deletedIdentifiers.add(identifier); - } else { - hdLogger.info("No dataset found for "+identifier+", skipping delete. "); - } - + } else if (deleted) { + hdLogger.info("Deleting harvesting dataset for "+identifier+", per GetRecord."); + + deleteHarvestedDatasetIfExists(identifier, oaiHandler.getHarvestingClient().getDataverse(), dataverseRequest, deletedIdentifiers, hdLogger); } else { hdLogger.info("Successfully retrieved GetRecord response."); - tempFile = record.getMetadataFile(); PrintWriter cleanupLog; harvestedDataset = importService.doImportHarvestedDataset(dataverseRequest, oaiHandler.getHarvestingClient(), identifier, oaiHandler.getMetadataPrefix(), - record.getMetadataFile(), + tempFile, dateStamp, importCleanupLog); hdLogger.fine("Harvest Successful for identifier " + identifier); - hdLogger.fine("Size of this record: " + record.getMetadataFile().length()); - processedSizeThisBatch.add(record.getMetadataFile().length()); + hdLogger.fine("Size of this record: " + tempFile.length()); } } catch (Throwable e) { logGetRecordException(hdLogger, oaiHandler, identifier, e); errMessage = "Caught exception while executing GetRecord on "+identifier; - //logException(e, hdLogger); } finally { if (tempFile != null) { @@ -364,14 +348,12 @@ private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, P } } - // TODO: the message below is taken from DVN3; - figure out what it means... - // // If we got an Error from the OAI server or an exception happened during import, then // set recordErrorOccurred to true (if recordErrorOccurred is being used) // otherwise throw an exception (if recordErrorOccurred is not used, i.e null) if (errMessage != null) { - if (recordErrorOccurred != null) { + if (recordErrorOccurred != null) { recordErrorOccurred.setValue(true); } else { throw new EJBException(errMessage); @@ -380,7 +362,55 @@ private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, P return harvestedDataset != null ? harvestedDataset.getId() : null; } - + + File retrieveProprietaryDataverseMetadata (HttpClient client, String remoteApiUrl) throws IOException { + + if (client == null) { + throw new IOException("Null Http Client, cannot make a call to obtain native metadata."); + } + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(remoteApiUrl)) + .GET() + .header("User-Agent", "DataverseHarvester/6.0") + .build(); + + HttpResponse response; + + try { + response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IOException("Failed to connect to the remote dataverse server to obtain native metadata"); + } + + int responseCode = response.statusCode(); + + if (responseCode == HTTP_OK) { + File tempMetadataFile = File.createTempFile("meta", ".tmp"); + + try (InputStream inputStream = response.body(); + FileOutputStream outputStream = new FileOutputStream(tempMetadataFile);) { + inputStream.transferTo(outputStream); + return tempMetadataFile; + } + } + + throw new IOException("Failed to download native metadata from the remote dataverse server."); + } + + private void deleteHarvestedDatasetIfExists(String persistentIdentifier, Dataverse harvestingDataverse, DataverseRequest dataverseRequest, List deletedIdentifiers, Logger hdLogger) { + Dataset dataset = datasetService.getDatasetByHarvestInfo(harvestingDataverse, persistentIdentifier); + if (dataset != null) { + datasetService.deleteHarvestedDataset(dataset, dataverseRequest, hdLogger); + // TODO: + // check the status of that Delete - see if it actually succeeded + deletedIdentifiers.add(persistentIdentifier); + return; + } + hdLogger.info("No dataset found for " + persistentIdentifier + ", skipping delete. "); + } + private void logBeginOaiHarvest(Logger hdLogger, HarvestingClient harvestingClient) { hdLogger.log(Level.INFO, "BEGIN HARVEST, oaiUrl=" +harvestingClient.getHarvestingUrl() @@ -448,47 +478,5 @@ private void logException(Throwable e, Logger logger) { } while ((e = e.getCause()) != null); logger.severe(fullMessage); } - - /* - some dead code below: - this functionality has been moved into OaiHandler. - TODO: test that harvesting is still working and remove. - - private ServiceProvider getServiceProvider(String baseOaiUrl, Granularity oaiGranularity) { - Context context = new Context(); - - context.withBaseUrl(baseOaiUrl); - context.withGranularity(oaiGranularity); - context.withOAIClient(new HttpOAIClient(baseOaiUrl)); - - ServiceProvider serviceProvider = new ServiceProvider(context); - return serviceProvider; - } - */ - - /** - * Creates an XOAI parameters object for the ListIdentifiers call - * - * @param metadataPrefix - * @param set - * @param from - * @return ListIdentifiersParameters - */ - /* - private ListIdentifiersParameters buildParams(String metadataPrefix, String set, Date from) { - ListIdentifiersParameters mip = ListIdentifiersParameters.request(); - mip.withMetadataPrefix(metadataPrefix); - - if (from != null) { - mip.withFrom(from); - } - - if (set != null) { - mip.withSetSpec(set); - } - return mip; - } - */ - - + } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java index d1aaea50793..c0a039e2d2b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java @@ -1,33 +1,27 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.harvest.client.oai; -import com.lyncode.xoai.model.oaipmh.Description; -import com.lyncode.xoai.model.oaipmh.Granularity; -import com.lyncode.xoai.model.oaipmh.Header; -import com.lyncode.xoai.model.oaipmh.MetadataFormat; -import com.lyncode.xoai.model.oaipmh.Set; -import com.lyncode.xoai.serviceprovider.ServiceProvider; -import com.lyncode.xoai.serviceprovider.client.HttpOAIClient; -import com.lyncode.xoai.serviceprovider.exceptions.BadArgumentException; -import com.lyncode.xoai.serviceprovider.exceptions.InvalidOAIResponse; -import com.lyncode.xoai.serviceprovider.exceptions.NoSetHierarchyException; -import com.lyncode.xoai.serviceprovider.model.Context; -import com.lyncode.xoai.serviceprovider.parameters.ListIdentifiersParameters; +import io.gdcc.xoai.model.oaipmh.Granularity; +import io.gdcc.xoai.model.oaipmh.results.record.Header; +import io.gdcc.xoai.model.oaipmh.results.MetadataFormat; +import io.gdcc.xoai.model.oaipmh.results.Set; +import io.gdcc.xoai.serviceprovider.ServiceProvider; +import io.gdcc.xoai.serviceprovider.client.JdkHttpOaiClient; +import io.gdcc.xoai.serviceprovider.exceptions.BadArgumentException; +import io.gdcc.xoai.serviceprovider.exceptions.InvalidOAIResponse; +import io.gdcc.xoai.serviceprovider.exceptions.NoSetHierarchyException; +import io.gdcc.xoai.serviceprovider.exceptions.IdDoesNotExistException; +import io.gdcc.xoai.serviceprovider.model.Context; +import io.gdcc.xoai.serviceprovider.parameters.ListIdentifiersParameters; import edu.harvard.iq.dataverse.harvest.client.FastGetRecord; +import static edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.DATAVERSE_PROPRIETARY_METADATA_API; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import java.io.IOException; import java.io.Serializable; -import java.io.UnsupportedEncodingException; import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.lang3.StringUtils; import org.xml.sax.SAXException; import javax.xml.transform.TransformerException; -import java.net.URLEncoder; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; @@ -74,8 +68,9 @@ public OaiHandler(HarvestingClient harvestingClient) throws OaiHandlerException this.harvestingClient = harvestingClient; } - private String baseOaiUrl; //= harvestingClient.getHarvestingUrl(); - private String metadataPrefix; // = harvestingClient.getMetadataPrefix(); + private String baseOaiUrl; + private String dataverseApiUrl; // if the remote server is a Dataverse and we access its native metadata + private String metadataPrefix; private String setName; private Date fromDate; private Boolean setListTruncated = false; @@ -124,7 +119,7 @@ public boolean isSetListTruncated() { return setListTruncated; } - private ServiceProvider getServiceProvider() throws OaiHandlerException { + public ServiceProvider getServiceProvider() throws OaiHandlerException { if (serviceProvider == null) { if (baseOaiUrl == null) { throw new OaiHandlerException("Could not instantiate Service Provider, missing OAI server URL."); @@ -133,8 +128,8 @@ private ServiceProvider getServiceProvider() throws OaiHandlerException { context.withBaseUrl(baseOaiUrl); context.withGranularity(Granularity.Second); - context.withOAIClient(new HttpOAIClient(baseOaiUrl)); - + // builds the client with the default parameters and the JDK http client: + context.withOAIClient(JdkHttpOaiClient.newBuilder().withBaseUrl(baseOaiUrl).build()); serviceProvider = new ServiceProvider(context); } @@ -199,6 +194,16 @@ public List runListMetadataFormats() throws OaiHandlerException { try { mfIter = sp.listMetadataFormats(); + } catch (IdDoesNotExistException idnee) { + // TODO: + // not sure why this exception is now thrown by List Metadata Formats (?) + // but looks like it was added in xoai 4.2. + // It appears that the answer is, they added it because you can + // call ListMetadataFormats on a specific identifier, optionally, + // and therefore it is possible to get back that response. Of course + // it will never be the case when calling it on an entire repository. + // But it's ok. + throw new OaiHandlerException("Id does not exist exception"); } catch (InvalidOAIResponse ior) { throw new OaiHandlerException("No valid response received from the OAI server."); } @@ -261,7 +266,7 @@ private ListIdentifiersParameters buildListIdentifiersParams() throws OaiHandler mip.withMetadataPrefix(metadataPrefix); if (this.fromDate != null) { - mip.withFrom(this.fromDate); + mip.withFrom(this.fromDate.toInstant()); } if (!StringUtils.isEmpty(this.setName)) { @@ -271,6 +276,18 @@ private ListIdentifiersParameters buildListIdentifiersParams() throws OaiHandler return mip; } + public String getProprietaryDataverseMetadataURL(String identifier) { + + if (dataverseApiUrl == null) { + dataverseApiUrl = baseOaiUrl.replaceFirst("/oai", ""); + } + + StringBuilder requestURL = new StringBuilder(dataverseApiUrl); + requestURL.append(DATAVERSE_PROPRIETARY_METADATA_API).append(identifier); + + return requestURL.toString(); + } + public void runIdentify() { // not implemented yet // (we will need it, both for validating the remote server, diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java index 057903d506a..6cdc4e5c277 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.export.ExportService; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import java.time.Instant; import java.io.File; import java.io.IOException; import java.sql.Timestamp; @@ -286,15 +287,15 @@ public List findOaiRecordsBySetName(String setName) { return findOaiRecordsBySetName(setName, null, null); } - public List findOaiRecordsBySetName(String setName, Date from, Date until) { + public List findOaiRecordsBySetName(String setName, Instant from, Instant until) { return findOaiRecordsBySetName(setName, from, until, false); } - public List findOaiRecordsNotInThisSet(String setName, Date from, Date until) { + public List findOaiRecordsNotInThisSet(String setName, Instant from, Instant until) { return findOaiRecordsBySetName(setName, from, until, true); } - public List findOaiRecordsBySetName(String setName, Date from, Date until, boolean excludeSet) { + public List findOaiRecordsBySetName(String setName, Instant from, Instant until, boolean excludeSet) { if (setName == null) { setName = ""; @@ -314,35 +315,18 @@ public List findOaiRecordsBySetName(String setName, Date from, Date u logger.fine("Query: "+queryString); TypedQuery query = em.createQuery(queryString, OAIRecord.class); - if (setName != null) { query.setParameter("setName",setName); } - if (from != null) { query.setParameter("from",from,TemporalType.TIMESTAMP); } - // In order to achieve inclusivity on the "until" matching, we need to do - // the following (if the "until" parameter is supplied): - // 1) if the supplied "until" parameter has the time portion (and is not just - // a date), we'll increment it by one second. This is because the time stamps we - // keep in the database also have fractional thousands of a second. - // So, a record may be shown as "T17:35:45", but in the database it is - // actually "17:35:45.356", so "<= 17:35:45" isn't going to work on this - // time stamp! - So we want to try "<= 17:35:45" instead. - // 2) if it's just a date, we'll increment it by a *full day*. Otherwise - // our database time stamp of 2016-10-23T17:35:45.123Z is NOT going to - // match " <= 2016-10-23" - which is really going to be interpreted as - // "2016-10-23T00:00:00.000". - // -- L.A. 4.6 + if (setName != null) { + query.setParameter("setName",setName); + } + // TODO: review and phase out the use of java.util.Date throughout this service. + + if (from != null) { + query.setParameter("from",Date.from(from),TemporalType.TIMESTAMP); + } if (until != null) { - // 24 * 3600 * 1000 = number of milliseconds in a day. - - if (until.getTime() % (24 * 3600 * 1000) == 0) { - // The supplied "until" parameter is a date, with no time - // portion. - logger.fine("plain date. incrementing by one day"); - until.setTime(until.getTime()+(24 * 3600 * 1000)); - } else { - logger.fine("date and time. incrementing by one second"); - until.setTime(until.getTime()+1000); - } - query.setParameter("until",until,TemporalType.TIMESTAMP); + Date untilDate = Date.from(until); + query.setParameter("until",untilDate,TemporalType.TIMESTAMP); } try { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java index f300f02f70c..6b28c8808a0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java @@ -67,7 +67,7 @@ public OAISet find(Object pk) { return em.find(OAISet.class, pk); } - public boolean specExists(String spec) { + public boolean setExists(String spec) { boolean specExists = false; OAISet set = findBySpec(spec); diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index d8619c42dfa..5eacb1addb6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -5,26 +5,19 @@ */ package edu.harvard.iq.dataverse.harvest.server.web.servlet; -import com.lyncode.xml.exceptions.XmlWriteException; -import com.lyncode.xoai.dataprovider.builder.OAIRequestParametersBuilder; -import com.lyncode.xoai.dataprovider.exceptions.OAIException; -import com.lyncode.xoai.dataprovider.repository.Repository; -import com.lyncode.xoai.dataprovider.repository.RepositoryConfiguration; -import com.lyncode.xoai.dataprovider.model.Context; -import com.lyncode.xoai.dataprovider.model.MetadataFormat; -import com.lyncode.xoai.services.impl.SimpleResumptionTokenFormat; -import com.lyncode.xoai.dataprovider.repository.ItemRepository; -import com.lyncode.xoai.dataprovider.repository.SetRepository; -import com.lyncode.xoai.model.oaipmh.DeletedRecord; -import com.lyncode.xoai.model.oaipmh.Granularity; -import com.lyncode.xoai.model.oaipmh.OAIPMH; -import static com.lyncode.xoai.model.oaipmh.OAIPMH.NAMESPACE_URI; -import static com.lyncode.xoai.model.oaipmh.OAIPMH.SCHEMA_LOCATION; -import com.lyncode.xoai.model.oaipmh.Verb; -import com.lyncode.xoai.xml.XSISchema; - -import com.lyncode.xoai.xml.XmlWriter; -import static com.lyncode.xoai.xml.XmlWriter.defaultContext; +import io.gdcc.xoai.dataprovider.DataProvider; +import io.gdcc.xoai.dataprovider.repository.Repository; +import io.gdcc.xoai.dataprovider.repository.RepositoryConfiguration; +import io.gdcc.xoai.dataprovider.model.Context; +import io.gdcc.xoai.dataprovider.model.MetadataFormat; +import io.gdcc.xoai.dataprovider.request.RequestBuilder; +import io.gdcc.xoai.dataprovider.request.RequestBuilder.RawRequest; +import io.gdcc.xoai.dataprovider.repository.ItemRepository; +import io.gdcc.xoai.dataprovider.repository.SetRepository; +import io.gdcc.xoai.model.oaipmh.DeletedRecord; +import io.gdcc.xoai.model.oaipmh.OAIPMH; + +import io.gdcc.xoai.xml.XmlWriter; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.export.ExportException; @@ -32,24 +25,20 @@ import edu.harvard.iq.dataverse.export.spi.Exporter; import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean; import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean; -import edu.harvard.iq.dataverse.harvest.server.xoai.XdataProvider; -import edu.harvard.iq.dataverse.harvest.server.xoai.XgetRecord; -import edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository; -import edu.harvard.iq.dataverse.harvest.server.xoai.XsetRepository; -import edu.harvard.iq.dataverse.harvest.server.xoai.XlistRecords; +import edu.harvard.iq.dataverse.harvest.server.xoai.DataverseXoaiItemRepository; +import edu.harvard.iq.dataverse.harvest.server.xoai.DataverseXoaiSetRepository; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.MailUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import io.gdcc.xoai.exceptions.OAIException; import org.apache.commons.lang3.StringUtils; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.util.Date; -import java.util.HashMap; import java.util.logging.Logger; import javax.ejb.EJB; +import javax.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; import javax.mail.internet.InternetAddress; import javax.servlet.ServletConfig; import javax.servlet.ServletException; @@ -57,12 +46,14 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.xml.stream.XMLStreamException; +import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.config.ConfigProvider; /** * * @author Leonid Andreev * Dedicated servlet for handling OAI-PMH requests. - * Uses lyncode XOAI data provider implementation for serving content. + * Uses lyncode/Dspace/gdcc XOAI data provider implementation for serving content. * The servlet itself is somewhat influenced by the older OCLC OAIcat implementation. */ public class OAIServlet extends HttpServlet { @@ -79,23 +70,42 @@ public class OAIServlet extends HttpServlet { @EJB SystemConfig systemConfig; + + @Inject + @ConfigProperty(name = "dataverse.oai.server.maxidentifiers", defaultValue="100") + private Integer maxListIdentifiers; + + @Inject + @ConfigProperty(name = "dataverse.oai.server.maxsets", defaultValue="100") + private Integer maxListSets; + + @Inject + @ConfigProperty(name = "dataverse.oai.server.maxrecords", defaultValue="10") + private Integer maxListRecords; private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.web.servlet.OAIServlet"); - protected HashMap attributesMap = new HashMap(); - private static final String OAI_PMH = "OAI-PMH"; - private static final String RESPONSEDATE_FIELD = "responseDate"; - private static final String REQUEST_FIELD = "request"; + // If we are going to stick with this solution - of providing a minimalist + // xml record containing a link to the proprietary json metadata API for + // "dataverse json harvesting", we'll probably want to create minimalistic, + // but valid schemas for this format as well. + // UPDATE: we are keeping this hack on the server side purely for backward + // compatibility with older (pre v6) Dataverses who may still be using the + // format. Once v6 has been around for a while, we will get rid of it completely. + // Starting this version, harvesting clients will not be making GetRecord + // calls at all when using harvesting dataverse_json; instead they will only + // be calling ListIdentifiers, and then making direct calls to the export + // API of the remote Dataverse, to obtain the records in native json. This + // is how we should have implemented this in the first place, really. private static final String DATAVERSE_EXTENDED_METADATA_FORMAT = "dataverse_json"; - private static final String DATAVERSE_EXTENDED_METADATA_INFO = "Custom Dataverse metadata in JSON format (Dataverse4 to Dataverse4 harvesting only)"; - private static final String DATAVERSE_EXTENDED_METADATA_SCHEMA = "JSON schema pending"; - + private static final String DATAVERSE_EXTENDED_METADATA_NAMESPACE = "Custom Dataverse metadata in JSON format (Dataverse4 to Dataverse4 harvesting only)"; + private static final String DATAVERSE_EXTENDED_METADATA_SCHEMA = "JSON schema pending"; private Context xoaiContext; private SetRepository setRepository; private ItemRepository itemRepository; private RepositoryConfiguration repositoryConfiguration; private Repository xoaiRepository; - private XdataProvider dataProvider; + private DataProvider dataProvider; public void init(ServletConfig config) throws ServletException { super.init(config); @@ -106,18 +116,17 @@ public void init(ServletConfig config) throws ServletException { xoaiContext = addDataverseJsonMetadataFormat(xoaiContext); } - setRepository = new XsetRepository(setService); - itemRepository = new XitemRepository(recordService, datasetService); + setRepository = new DataverseXoaiSetRepository(setService); + itemRepository = new DataverseXoaiItemRepository(recordService, datasetService, systemConfig.getDataverseSiteUrl()); repositoryConfiguration = createRepositoryConfiguration(); - + xoaiRepository = new Repository() .withSetRepository(setRepository) .withItemRepository(itemRepository) - .withResumptionTokenFormatter(new SimpleResumptionTokenFormat()) .withConfiguration(repositoryConfiguration); - dataProvider = new XdataProvider(getXoaiContext(), getXoaiRepository()); + dataProvider = new DataProvider(getXoaiContext(), getXoaiRepository()); } private Context createContext() { @@ -145,6 +154,7 @@ private void addSupportedMetadataFormats(Context context) { metadataFormat = MetadataFormat.metadataFormat(formatName); metadataFormat.withNamespace(exporter.getXMLNameSpace()); metadataFormat.withSchemaLocation(exporter.getXMLSchemaLocation()); + } catch (ExportException ex) { metadataFormat = null; } @@ -153,12 +163,11 @@ private void addSupportedMetadataFormats(Context context) { } } } - //return context; } private Context addDataverseJsonMetadataFormat(Context context) { MetadataFormat metadataFormat = MetadataFormat.metadataFormat(DATAVERSE_EXTENDED_METADATA_FORMAT); - metadataFormat.withNamespace(DATAVERSE_EXTENDED_METADATA_INFO); + metadataFormat.withNamespace(DATAVERSE_EXTENDED_METADATA_NAMESPACE); metadataFormat.withSchemaLocation(DATAVERSE_EXTENDED_METADATA_SCHEMA); context.withMetadataFormat(metadataFormat); return context; @@ -169,26 +178,30 @@ private boolean isDataverseOaiExtensionsSupported() { } private RepositoryConfiguration createRepositoryConfiguration() { - // TODO: - // some of the settings below - such as the max list numbers - - // need to be configurable! + Config config = ConfigProvider.getConfig(); + String repositoryName = config.getOptionalValue("dataverse.oai.server.repositoryname", String.class).orElse(""); - String dataverseName = dataverseService.getRootDataverseName(); - String repositoryName = StringUtils.isEmpty(dataverseName) || "Root".equals(dataverseName) ? "Test Dataverse OAI Archive" : dataverseName + " Dataverse OAI Archive"; - InternetAddress internetAddress = MailUtil.parseSystemAddress(settingsService.getValueForKey(SettingsServiceBean.Key.SystemEmail)); - - RepositoryConfiguration repositoryConfiguration = new RepositoryConfiguration() + if (repositoryName == null || repositoryName.isEmpty()) { + String dataverseName = dataverseService.getRootDataverseName(); + repositoryName = StringUtils.isEmpty(dataverseName) || "Root".equals(dataverseName) ? "Test Dataverse OAI Archive" : dataverseName + " Dataverse OAI Archive"; + } + // The admin email address associated with this installation: + // (Note: if the setting does not exist, we are going to assume that they + // have a reason not to want to advertise their email address, so no + // email will be shown in the output of Identify. + InternetAddress systemEmailAddress = MailUtil.parseSystemAddress(settingsService.getValueForKey(SettingsServiceBean.Key.SystemEmail)); + + RepositoryConfiguration repositoryConfiguration = RepositoryConfiguration.defaults() + .withEnableMetadataAttributes(true) .withRepositoryName(repositoryName) .withBaseUrl(systemConfig.getDataverseSiteUrl()+"/oai") - .withCompression("gzip") // ? - .withCompression("deflate") // ? - .withAdminEmail(internetAddress != null ? internetAddress.getAddress() : null) + .withCompression("gzip") + .withCompression("deflate") + .withAdminEmail(systemEmailAddress != null ? systemEmailAddress.getAddress() : null) .withDeleteMethod(DeletedRecord.TRANSIENT) - .withGranularity(Granularity.Second) - .withMaxListIdentifiers(100) - .withMaxListRecords(100) - .withMaxListSets(100) - .withEarliestDate(new Date()); + .withMaxListIdentifiers(maxListIdentifiers) + .withMaxListRecords(maxListRecords) + .withMaxListSets(maxListSets); return repositoryConfiguration; } @@ -221,9 +234,9 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { processRequest(request, response); } - + - private void processRequest(HttpServletRequest request, HttpServletResponse response) + private void processRequest(HttpServletRequest httpServletRequest, HttpServletResponse response) throws ServletException, IOException { try { @@ -233,150 +246,22 @@ private void processRequest(HttpServletRequest request, HttpServletResponse resp "Sorry. OAI Service is disabled on this Dataverse node."); return; } + + RawRequest rawRequest = RequestBuilder.buildRawRequest(httpServletRequest.getParameterMap()); - OAIRequestParametersBuilder parametersBuilder = newXoaiRequest(); - - for (Object p : request.getParameterMap().keySet()) { - String parameterName = (String)p; - String parameterValue = request.getParameter(parameterName); - parametersBuilder = parametersBuilder.with(parameterName, parameterValue); - - } - - OAIPMH handle = dataProvider.handle(parametersBuilder); + OAIPMH handle = dataProvider.handle(rawRequest); response.setContentType("text/xml;charset=UTF-8"); - - if (isGetRecord(request) && !handle.hasErrors()) { - writeGetRecord(response, handle); - } else if (isListRecords(request) && !handle.hasErrors()) { - writeListRecords(response, handle); - } else { - XmlWriter xmlWriter = new XmlWriter(response.getOutputStream()); + + try (XmlWriter xmlWriter = new XmlWriter(response.getOutputStream(), repositoryConfiguration);) { xmlWriter.write(handle); - xmlWriter.flush(); - xmlWriter.close(); } - } catch (IOException ex) { - logger.warning("IO exception in Get; "+ex.getMessage()); - throw new ServletException ("IO Exception in Get", ex); - } catch (OAIException oex) { - logger.warning("OAI exception in Get; "+oex.getMessage()); - throw new ServletException ("OAI Exception in Get", oex); - } catch (XMLStreamException xse) { - logger.warning("XML Stream exception in Get; "+xse.getMessage()); - throw new ServletException ("XML Stream Exception in Get", xse); - } catch (XmlWriteException xwe) { - logger.warning("XML Write exception in Get; "+xwe.getMessage()); - throw new ServletException ("XML Write Exception in Get", xwe); - } catch (Exception e) { - logger.warning("Unknown exception in Get; "+e.getMessage()); - throw new ServletException ("Unknown servlet exception in Get.", e); + } catch (XMLStreamException | OAIException e) { + throw new ServletException (e); } } - // Custom methods for the potentially expensive GetRecord and ListRecords requests: - - private void writeListRecords(HttpServletResponse response, OAIPMH handle) throws IOException { - OutputStream outputStream = response.getOutputStream(); - - outputStream.write(oaiPmhResponseToString(handle).getBytes()); - - Verb verb = handle.getVerb(); - - if (verb == null) { - throw new IOException("An error or a valid response must be set"); - } - - if (!verb.getType().equals(Verb.Type.ListRecords)) { - throw new IOException("writeListRecords() called on a non-ListRecords verb"); - } - - outputStream.write(("<" + verb.getType().displayName() + ">").getBytes()); - - outputStream.flush(); - - ((XlistRecords) verb).writeToStream(outputStream); - - outputStream.write(("").getBytes()); - outputStream.write(("\n").getBytes()); - - outputStream.flush(); - outputStream.close(); - - } - - private void writeGetRecord(HttpServletResponse response, OAIPMH handle) throws IOException, XmlWriteException, XMLStreamException { - OutputStream outputStream = response.getOutputStream(); - - outputStream.write(oaiPmhResponseToString(handle).getBytes()); - - Verb verb = handle.getVerb(); - - if (verb == null) { - throw new IOException("An error or a valid response must be set"); - } - - if (!verb.getType().equals(Verb.Type.GetRecord)) { - throw new IOException("writeListRecords() called on a non-GetRecord verb"); - } - - outputStream.write(("<" + verb.getType().displayName() + ">").getBytes()); - - outputStream.flush(); - - ((XgetRecord) verb).writeToStream(outputStream); - - outputStream.write(("").getBytes()); - outputStream.write(("\n").getBytes()); - - outputStream.flush(); - outputStream.close(); - - } - - // This function produces the string representation of the top level, - // "service" record of an OAIPMH response (i.e., the header that precedes - // the actual "payload" record, such as , , - // , etc. - - private String oaiPmhResponseToString(OAIPMH handle) { - try { - ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(byteOutputStream, defaultContext()); - - writer.writeStartElement(OAI_PMH); - writer.writeDefaultNamespace(NAMESPACE_URI); - writer.writeNamespace(XSISchema.PREFIX, XSISchema.NAMESPACE_URI); - writer.writeAttribute(XSISchema.PREFIX, XSISchema.NAMESPACE_URI, "schemaLocation", - NAMESPACE_URI + " " + SCHEMA_LOCATION); - - writer.writeElement(RESPONSEDATE_FIELD, handle.getResponseDate(), Granularity.Second); - writer.writeElement(REQUEST_FIELD, handle.getRequest()); - writer.writeEndElement(); - writer.flush(); - writer.close(); - - String ret = byteOutputStream.toString().replaceFirst("", ""); - - return ret; - } catch (Exception ex) { - logger.warning("caught exception trying to convert an OAIPMH response header to string: " + ex.getMessage()); - ex.printStackTrace(); - return null; - } - } - - private boolean isGetRecord(HttpServletRequest request) { - return "GetRecord".equals(request.getParameter("verb")); - - } - - private boolean isListRecords(HttpServletRequest request) { - return "ListRecords".equals(request.getParameter("verb")); - } - protected Context getXoaiContext () { return xoaiContext; } @@ -385,11 +270,6 @@ protected Repository getXoaiRepository() { return xoaiRepository; } - protected OAIRequestParametersBuilder newXoaiRequest() { - return new OAIRequestParametersBuilder(); - } - - public boolean isHarvestingServerEnabled() { return systemConfig.isOAIServerEnabled(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xitem.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItem.java similarity index 63% rename from src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xitem.java rename to src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItem.java index bd7a35ddb79..ecfd2f82369 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xitem.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItem.java @@ -1,18 +1,14 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.harvest.server.xoai; -import com.lyncode.xoai.dataprovider.model.Item; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.model.oaipmh.About; +import io.gdcc.xoai.dataprovider.model.Item; +import io.gdcc.xoai.dataprovider.model.Set; +import io.gdcc.xoai.model.oaipmh.results.record.Metadata; +import io.gdcc.xoai.model.oaipmh.results.record.About; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.harvest.server.OAIRecord; import edu.harvard.iq.dataverse.util.StringUtil; +import java.time.Instant; import java.util.ArrayList; -import java.util.Date; import java.util.List; @@ -20,13 +16,13 @@ * * @author Leonid Andreev * - * This is an implemention of an Lyncode XOAI Item; + * This is an implemention of a Lyncode/DSpace/gdcc XOAI Item. * You can think of it as an XOAI Item wrapper around the * Dataverse OAIRecord entity. */ -public class Xitem implements Item { +public final class DataverseXoaiItem implements Item { - public Xitem(OAIRecord oaiRecord) { + public DataverseXoaiItem(OAIRecord oaiRecord) { super(); this.oaiRecord = oaiRecord; oaisets = new ArrayList<>(); @@ -34,7 +30,7 @@ public Xitem(OAIRecord oaiRecord) { oaisets.add(new Set(oaiRecord.getSetName())); } } - + private OAIRecord oaiRecord; public OAIRecord getOaiRecord() { @@ -51,19 +47,21 @@ public Dataset getDataset() { return dataset; } - public Xitem withDataset(Dataset dataset) { + public DataverseXoaiItem withDataset(Dataset dataset) { this.dataset = dataset; return this; } + private Metadata metadata; + @Override - public List getAbout() { - return null; + public Metadata getMetadata() { + return metadata; } - - @Override - public Xmetadata getMetadata() { - return new Xmetadata((String)null); + + public DataverseXoaiItem withMetadata(Metadata metadata) { + this.metadata = metadata; + return this; } @Override @@ -72,8 +70,8 @@ public String getIdentifier() { } @Override - public Date getDatestamp() { - return oaiRecord.getLastUpdateTime(); + public Instant getDatestamp() { + return oaiRecord.getLastUpdateTime().toInstant(); } private List oaisets; @@ -82,12 +80,10 @@ public Date getDatestamp() { public List getSets() { return oaisets; - } @Override public boolean isDeleted() { return oaiRecord.isRemoved(); } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java new file mode 100644 index 00000000000..faf3cf9ddc4 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java @@ -0,0 +1,270 @@ +package edu.harvard.iq.dataverse.harvest.server.xoai; + +import io.gdcc.xoai.dataprovider.exceptions.handler.IdDoesNotExistException; +import io.gdcc.xoai.dataprovider.filter.ScopedFilter; +import io.gdcc.xoai.dataprovider.model.Item; +import io.gdcc.xoai.dataprovider.model.ItemIdentifier; +import io.gdcc.xoai.dataprovider.model.Set; +import io.gdcc.xoai.dataprovider.model.MetadataFormat; +import io.gdcc.xoai.dataprovider.repository.ItemRepository; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.export.ExportException; +import edu.harvard.iq.dataverse.export.ExportService; +import edu.harvard.iq.dataverse.harvest.server.OAIRecord; +import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean; +import edu.harvard.iq.dataverse.util.StringUtil; +import io.gdcc.xoai.dataprovider.exceptions.handler.HandlerException; +import io.gdcc.xoai.dataprovider.exceptions.handler.NoMetadataFormatsException; +import io.gdcc.xoai.dataprovider.repository.ResultsPage; +import io.gdcc.xoai.model.oaipmh.ResumptionToken; +import io.gdcc.xoai.model.oaipmh.results.record.Metadata; +import io.gdcc.xoai.xml.EchoElement; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.time.Instant; +import java.util.List; +import java.util.logging.Logger; + +/** + * + * @author Leonid Andreev + * Implements an XOAI "Item Repository". Retrieves Dataverse "OAIRecords" + * representing harvestable local datasets and translates them into + * XOAI "items". + */ + +public class DataverseXoaiItemRepository implements ItemRepository { + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.DataverseXoaiItemRepository"); + + private final OAIRecordServiceBean recordService; + private final DatasetServiceBean datasetService; + private final String serverUrl; + + public DataverseXoaiItemRepository (OAIRecordServiceBean recordService, DatasetServiceBean datasetService, String serverUrl) { + this.recordService = recordService; + this.datasetService = datasetService; + this.serverUrl = serverUrl; + } + + @Override + public ItemIdentifier getItem(String identifier) throws IdDoesNotExistException { + // This method is called when ListMetadataFormats request specifies + // the identifier, requesting the formats available for this specific record. + // In our case, under the current implementation, we need to simply look + // up if the record exists; if it does, all the OAI formats that we serve + // should be available for this record. + + List oaiRecords = recordService.findOaiRecordsByGlobalId(identifier); + if (oaiRecords != null && !oaiRecords.isEmpty()) { + for (OAIRecord oaiRecord : oaiRecords) { + // We can return the first *active* record we find for this identifier. + if (!oaiRecord.isRemoved()) { + return new DataverseXoaiItem(oaiRecord); + } + } + } + + throw new IdDoesNotExistException(); + } + + @Override + public Item getItem(String identifier, MetadataFormat metadataFormat) throws HandlerException { + logger.fine("getItem; calling findOaiRecordsByGlobalId, identifier " + identifier); + + if (metadataFormat == null) { + throw new NoMetadataFormatsException("Metadata Format is Required"); + } + + List oaiRecords = recordService.findOaiRecordsByGlobalId(identifier); + if (oaiRecords != null && !oaiRecords.isEmpty()) { + DataverseXoaiItem xoaiItem = null; + for (OAIRecord oaiRecord : oaiRecords) { + if (xoaiItem == null) { + xoaiItem = new DataverseXoaiItem(oaiRecord); + xoaiItem = addMetadata(xoaiItem, metadataFormat); + } else { + // Adding extra set specs to the XOAI Item, if this oaiRecord + // is part of multiple sets: + if (!StringUtil.isEmpty(oaiRecord.getSetName())) { + xoaiItem.getSets().add(new Set(oaiRecord.getSetName())); + } + } + } + if (xoaiItem != null) { + return xoaiItem; + } + } + + throw new IdDoesNotExistException(); + } + + @Override + public ResultsPage getItemIdentifiers(List filters, MetadataFormat metadataFormat, int maxResponseLength, ResumptionToken.Value resumptionToken) throws HandlerException { + + return (ResultsPage)getRepositoryRecords(metadataFormat, maxResponseLength, resumptionToken, false); + + } + + @Override + public ResultsPage getItems(List filters, MetadataFormat metadataFormat, int maxResponseLength, ResumptionToken.Value resumptionToken) throws HandlerException { + + return (ResultsPage)getRepositoryRecords(metadataFormat, maxResponseLength, resumptionToken, true); + } + + private ResultsPage getRepositoryRecords ( + MetadataFormat metadataFormat, + int maxResponseLength, + ResumptionToken.Value resumptionToken, + boolean fullItems) throws HandlerException { + + int offset = Long.valueOf(resumptionToken.getOffset()).intValue(); + String setSpec = resumptionToken.getSetSpec(); + Instant from = resumptionToken.getFrom(); + Instant until = resumptionToken.getUntil(); + + boolean hasMore = false; + + logger.fine("calling " + (fullItems ? "getItems" : "getItemIdentifiers") + + "; offset=" + offset + + ", length=" + maxResponseLength + + ", setSpec=" + setSpec + + ", from=" + from + + ", until=" + until); + + List oaiRecords = recordService.findOaiRecordsBySetName(setSpec, from, until); + + List xoaiItems = new ArrayList<>(); + + if (oaiRecords != null && !oaiRecords.isEmpty()) { + logger.fine("total " + oaiRecords.size() + " records returned"); + + for (int i = offset; i < offset + maxResponseLength && i < oaiRecords.size(); i++) { + OAIRecord record = oaiRecords.get(i); + DataverseXoaiItem xoaiItem = new DataverseXoaiItem(record); + + if (fullItems) { + // If we are cooking "full" Items (for the ListRecords verb), + // add the metadata to the item object (if not a deleted + // record, if available, etc.): + xoaiItem = addMetadata(xoaiItem, metadataFormat); + } + + xoaiItems.add(xoaiItem); + } + + // Run a second pass, looking for records in this set that occur + // in *other* sets. Then we'll add these multiple sets to the + // formatted output in the header: + addExtraSets(xoaiItems, setSpec, from, until); + + hasMore = offset + maxResponseLength < oaiRecords.size(); + + ResultsPage result = new ResultsPage(resumptionToken, hasMore, xoaiItems, oaiRecords.size()); + logger.fine("returning result with " + xoaiItems.size() + " items."); + return result; + } + + return new ResultsPage(resumptionToken, false, xoaiItems, 0); + } + + private void addExtraSets(Object xoaiItemsList, String setSpec, Instant from, Instant until) { + + List xoaiItems = (List)xoaiItemsList; + + List oaiRecords = recordService.findOaiRecordsNotInThisSet(setSpec, from, until); + + if (oaiRecords == null || oaiRecords.isEmpty()) { + return; + } + + // Make a second pass through the list of xoaiItems already found for this set, + // and add any other sets in which this item occurs: + + int j = 0; + for (int i = 0; i < xoaiItems.size(); i++) { + // fast-forward the second list, until we find a oaiRecord with this identifier, + // or until we are past this oaiRecord (both lists are sorted alphabetically by + // the identifier: + DataverseXoaiItem xitem = xoaiItems.get(i); + + while (j < oaiRecords.size() && xitem.getIdentifier().compareTo(oaiRecords.get(j).getGlobalId()) > 0) { + j++; + } + + while (j < oaiRecords.size() && xitem.getIdentifier().equals(oaiRecords.get(j).getGlobalId())) { + xoaiItems.get(i).getSets().add(new Set(oaiRecords.get(j).getSetName())); + j++; + } + } + } + + private DataverseXoaiItem addMetadata(DataverseXoaiItem xoaiItem, MetadataFormat metadataFormat) { + // This may be a "deleted" record - i.e., a oaiRecord kept in + // the OAI set for a dataset that's no longer in this Dataverse. + // (it serves to tell the remote client to delete it from their + // holdings too). + // If this is the case here, there's nothing we need to do for this item. + // If not, if it's a live record, let's try to look up the dataset and + // open the pre-generated metadata stream. + + if (!xoaiItem.isDeleted()) { + Dataset dataset = datasetService.findByGlobalId(xoaiItem.getIdentifier()); + if (dataset != null) { + try { + Metadata metadata = getDatasetMetadata(dataset, metadataFormat.getPrefix()); + xoaiItem.withDataset(dataset).withMetadata(metadata); + } catch (ExportException | IOException ex) { + // This is not supposed to happen in normal operations; + // since by design only the datasets for which the metadata + // records have been pre-generated ("exported") should be + // served as "OAI Record". But, things happen. If for one + // reason or another that cached metadata file is no longer there, + // we are not going to serve any metadata for this oaiRecord, + // BUT we are going to include it marked as "deleted" + // (because skipping it could potentially mess up the + // counts and offsets, in a resumption token scenario. + xoaiItem.getOaiRecord().setRemoved(true); + } + } else { + // If dataset (somehow) no longer exists (again, this is + // not supposed to happen), we will serve the oaiRecord, + // marked as "deleted" and without any metadata. + // We can't just skip it, because that could mess up the + // counts and offsets, in a resumption token scenario. + xoaiItem.getOaiRecord().setRemoved(true); + } + } + return xoaiItem; + } + + private Metadata getDatasetMetadata(Dataset dataset, String metadataPrefix) throws ExportException, IOException { + Metadata metadata; + + if ("dataverse_json".equals(metadataPrefix)) { + // Solely for backward compatibility, for older Dataverse harvesting clients + // that may still be relying on harvesting "dataverse_json"; + // we will want to eventually get rid of this hack! + // @Deprecated(since = "5.0") + metadata = new Metadata( + new EchoElement("custom metadata")) + .withAttribute("directApiCall", customDataverseJsonApiUri(dataset.getGlobalId().asString())); + + } else { + InputStream pregeneratedMetadataStream; + pregeneratedMetadataStream = ExportService.getInstance().getExport(dataset, metadataPrefix); + + metadata = Metadata.copyFromStream(pregeneratedMetadataStream); + } + return metadata; + } + + private String customDataverseJsonApiUri(String identifier) { + String ret = serverUrl + + "/api/datasets/export?exporter=dataverse_json&persistentId=" + + identifier; + + return ret; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XsetRepository.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiSetRepository.java similarity index 56% rename from src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XsetRepository.java rename to src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiSetRepository.java index 8e58e1bbf9a..b4e275b6059 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XsetRepository.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiSetRepository.java @@ -1,15 +1,9 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.harvest.server.xoai; -import com.lyncode.xoai.model.xoai.Element; -import com.lyncode.xoai.dataprovider.repository.SetRepository; -import com.lyncode.xoai.dataprovider.handlers.results.ListSetsResult; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.model.xoai.XOAIMetadata; +import io.gdcc.xoai.model.xoai.Element; +import io.gdcc.xoai.dataprovider.repository.SetRepository; +import io.gdcc.xoai.dataprovider.model.Set; +import io.gdcc.xoai.model.xoai.XOAIMetadata; import edu.harvard.iq.dataverse.harvest.server.OAISet; import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean; @@ -21,13 +15,12 @@ * * @author Leonid Andreev */ -public class XsetRepository implements SetRepository { - private static Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.XsetRepository"); +public class DataverseXoaiSetRepository implements SetRepository { + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.DataverseXoaiSetRepository"); private OAISetServiceBean setService; - public XsetRepository (OAISetServiceBean setService) { - super(); + public DataverseXoaiSetRepository (OAISetServiceBean setService) { this.setService = setService; } @@ -42,7 +35,6 @@ public void setSetService(OAISetServiceBean setService) { @Override public boolean supportSets() { - logger.fine("calling supportSets()"); List dataverseOAISets = setService.findAllNamedSets(); if (dataverseOAISets == null || dataverseOAISets.isEmpty()) { @@ -52,7 +44,7 @@ public boolean supportSets() { } @Override - public ListSetsResult retrieveSets(int offset, int length) { + public List getSets() { logger.fine("calling retrieveSets()"); List dataverseOAISets = setService.findAllNamedSets(); List XOAISets = new ArrayList(); @@ -62,25 +54,21 @@ public ListSetsResult retrieveSets(int offset, int length) { OAISet dataverseSet = dataverseOAISets.get(i); Set xoaiSet = new Set(dataverseSet.getSpec()); xoaiSet.withName(dataverseSet.getName()); - XOAIMetadata xMetadata = new XOAIMetadata(); + XOAIMetadata xoaiMetadata = new XOAIMetadata(); Element element = new Element("description"); element.withField("description", dataverseSet.getDescription()); - xMetadata.getElements().add(element); - xoaiSet.withDescription(xMetadata); + xoaiMetadata.getElements().add(element); + xoaiSet.withDescription(xoaiMetadata); XOAISets.add(xoaiSet); } } - return new ListSetsResult(offset + length < XOAISets.size(), XOAISets.subList(offset, Math.min(offset + length, XOAISets.size()))); + return XOAISets; } @Override public boolean exists(String setSpec) { - //for (Set s : this.sets) - // if (s.getSpec().equals(setSpec)) - // return true; - - return false; + return setService.setExists(setSpec); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XdataProvider.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XdataProvider.java deleted file mode 100644 index 8ba8fe96bec..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XdataProvider.java +++ /dev/null @@ -1,116 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - - -import com.lyncode.builder.Builder; -import com.lyncode.xoai.dataprovider.exceptions.*; -import com.lyncode.xoai.dataprovider.handlers.*; -import com.lyncode.xoai.exceptions.InvalidResumptionTokenException; -import com.lyncode.xoai.dataprovider.model.Context; -import com.lyncode.xoai.model.oaipmh.Identify; -import com.lyncode.xoai.model.oaipmh.OAIPMH; -import com.lyncode.xoai.model.oaipmh.Request; -import com.lyncode.xoai.dataprovider.parameters.OAICompiledRequest; -import com.lyncode.xoai.dataprovider.parameters.OAIRequest; -import com.lyncode.xoai.dataprovider.repository.Repository; -import com.lyncode.xoai.services.api.DateProvider; -import com.lyncode.xoai.services.impl.UTCDateProvider; -import static com.lyncode.xoai.dataprovider.parameters.OAIRequest.Parameter.*; - -import java.util.logging.Logger; - -/** - * - * @author Leonid Andreev - */ -public class XdataProvider { - private static Logger log = Logger.getLogger(XdataProvider.class.getCanonicalName()); - - public static XdataProvider dataProvider (Context context, Repository repository) { - return new XdataProvider(context, repository); - } - - private Repository repository; - private DateProvider dateProvider; - - private final IdentifyHandler identifyHandler; - private final XgetRecordHandler getRecordHandler; - private final ListSetsHandler listSetsHandler; - private final XlistRecordsHandler listRecordsHandler; - private final ListIdentifiersHandler listIdentifiersHandler; - private final ListMetadataFormatsHandler listMetadataFormatsHandler; - private final ErrorHandler errorsHandler; - - public XdataProvider (Context context, Repository repository) { - this.repository = repository; - this.dateProvider = new UTCDateProvider(); - - this.identifyHandler = new IdentifyHandler(context, repository); - this.listSetsHandler = new ListSetsHandler(context, repository); - this.listMetadataFormatsHandler = new ListMetadataFormatsHandler(context, repository); - this.listRecordsHandler = new XlistRecordsHandler(context, repository); - this.listIdentifiersHandler = new ListIdentifiersHandler(context, repository); - //this.getRecordHandler = new GetRecordHandler(context, repository); - this.getRecordHandler = new XgetRecordHandler(context, repository); - this.errorsHandler = new ErrorHandler(); - } - - public OAIPMH handle (Builder builder) throws OAIException { - return handle(builder.build()); - } - - public OAIPMH handle (OAIRequest requestParameters) throws OAIException { - log.fine("Handling OAI request"); - Request request = new Request(repository.getConfiguration().getBaseUrl()) - .withVerbType(requestParameters.get(Verb)) - .withResumptionToken(requestParameters.get(ResumptionToken)) - .withIdentifier(requestParameters.get(Identifier)) - .withMetadataPrefix(requestParameters.get(MetadataPrefix)) - .withSet(requestParameters.get(Set)) - .withFrom(requestParameters.get(From)) - .withUntil(requestParameters.get(Until)); - - OAIPMH response = new OAIPMH() - .withRequest(request) - .withResponseDate(dateProvider.now()); - try { - OAICompiledRequest parameters = compileParameters(requestParameters); - - switch (request.getVerbType()) { - case Identify: - Identify identify = identifyHandler.handle(parameters); - identify.getDescriptions().clear(); // We don't want to use the default description - response.withVerb(identify); - break; - case ListSets: - response.withVerb(listSetsHandler.handle(parameters)); - break; - case ListMetadataFormats: - response.withVerb(listMetadataFormatsHandler.handle(parameters)); - break; - case GetRecord: - response.withVerb(getRecordHandler.handle(parameters)); - break; - case ListIdentifiers: - response.withVerb(listIdentifiersHandler.handle(parameters)); - break; - case ListRecords: - response.withVerb(listRecordsHandler.handle(parameters)); - break; - } - } catch (HandlerException e) { - log.fine("HandlerException when executing "+request.getVerbType()+": " + e.getMessage()); - response.withError(errorsHandler.handle(e)); - } - - return response; - } - - private OAICompiledRequest compileParameters(OAIRequest requestParameters) throws IllegalVerbException, UnknownParameterException, BadArgumentException, DuplicateDefinitionException, BadResumptionToken { - try { - return requestParameters.compile(); - } catch (InvalidResumptionTokenException e) { - throw new BadResumptionToken("The resumption token is invalid"); - } - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecord.java deleted file mode 100644 index d86f555d105..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecord.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xoai.model.oaipmh.GetRecord; -import com.lyncode.xoai.model.oaipmh.Record; -import java.io.IOException; -import java.io.OutputStream; - -/** - * - * @author Leonid Andreev - * - * This is the Dataverse extension of XOAI GetRecord, - * optimized to stream individual records to the output directly - */ - -public class XgetRecord extends GetRecord { - private static final String RECORD_FIELD = "record"; - private static final String RECORD_START_ELEMENT = "<"+RECORD_FIELD+">"; - private static final String RECORD_CLOSE_ELEMENT = ""; - private static final String RESUMPTION_TOKEN_FIELD = "resumptionToken"; - private static final String EXPIRATION_DATE_ATTRIBUTE = "expirationDate"; - private static final String COMPLETE_LIST_SIZE_ATTRIBUTE = "completeListSize"; - private static final String CURSOR_ATTRIBUTE = "cursor"; - - - public XgetRecord(Xrecord record) { - super(record); - } - - public void writeToStream(OutputStream outputStream) throws IOException { - - if (this.getRecord() == null) { - throw new IOException("XgetRecord: null Record"); - } - Xrecord xrecord = (Xrecord) this.getRecord(); - - outputStream.write(RECORD_START_ELEMENT.getBytes()); - outputStream.flush(); - - xrecord.writeToStream(outputStream); - - outputStream.write(RECORD_CLOSE_ELEMENT.getBytes()); - outputStream.flush(); - - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecordHandler.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecordHandler.java deleted file mode 100644 index ba28894482a..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecordHandler.java +++ /dev/null @@ -1,92 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xml.exceptions.XmlWriteException; -import com.lyncode.xoai.dataprovider.exceptions.BadArgumentException; -import com.lyncode.xoai.dataprovider.exceptions.CannotDisseminateFormatException; -import com.lyncode.xoai.dataprovider.parameters.OAICompiledRequest; -import com.lyncode.xoai.dataprovider.exceptions.CannotDisseminateRecordException; -import com.lyncode.xoai.dataprovider.exceptions.HandlerException; -import com.lyncode.xoai.dataprovider.exceptions.IdDoesNotExistException; -import com.lyncode.xoai.dataprovider.exceptions.NoMetadataFormatsException; -import com.lyncode.xoai.dataprovider.exceptions.OAIException; -import com.lyncode.xoai.dataprovider.handlers.VerbHandler; -import com.lyncode.xoai.dataprovider.handlers.helpers.ItemHelper; -import com.lyncode.xoai.dataprovider.model.Context; -import com.lyncode.xoai.dataprovider.model.Item; -import com.lyncode.xoai.dataprovider.model.MetadataFormat; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.model.oaipmh.*; -import com.lyncode.xoai.dataprovider.repository.Repository; -import com.lyncode.xoai.xml.XSLPipeline; -import com.lyncode.xoai.xml.XmlWriter; -import edu.harvard.iq.dataverse.Dataset; - -import javax.xml.stream.XMLStreamException; -import javax.xml.transform.TransformerException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.logging.Logger; - -/* - * @author Leonid Andreev -*/ -public class XgetRecordHandler extends VerbHandler { - private static Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.XgetRecordHandler"); - public XgetRecordHandler(Context context, Repository repository) { - super(context, repository); - } - - @Override - public GetRecord handle(OAICompiledRequest parameters) throws OAIException, HandlerException { - - MetadataFormat format = getContext().formatForPrefix(parameters.getMetadataPrefix()); - Item item = getRepository().getItemRepository().getItem(parameters.getIdentifier()); - - if (getContext().hasCondition() && - !getContext().getCondition().getFilter(getRepository().getFilterResolver()).isItemShown(item)) - throw new IdDoesNotExistException("This context does not include this item"); - - if (format.hasCondition() && - !format.getCondition().getFilter(getRepository().getFilterResolver()).isItemShown(item)) - throw new CannotDisseminateRecordException("Format not applicable to this item"); - - - Xrecord record = this.createRecord(parameters, item); - GetRecord result = new XgetRecord(record); - - return result; - } - - private Xrecord createRecord(OAICompiledRequest parameters, Item item) - throws BadArgumentException, CannotDisseminateRecordException, - OAIException, NoMetadataFormatsException, CannotDisseminateFormatException { - MetadataFormat format = getContext().formatForPrefix(parameters.getMetadataPrefix()); - Header header = new Header(); - - Dataset dataset = ((Xitem)item).getDataset(); - Xrecord xrecord = new Xrecord().withFormatName(parameters.getMetadataPrefix()).withDataset(dataset); - header.withIdentifier(item.getIdentifier()); - - ItemHelper itemHelperWrap = new ItemHelper(item); - header.withDatestamp(item.getDatestamp()); - for (Set set : itemHelperWrap.getSets(getContext(), getRepository().getFilterResolver())) - header.withSetSpec(set.getSpec()); - if (item.isDeleted()) - header.withStatus(Header.Status.DELETED); - - xrecord.withHeader(header); - xrecord.withMetadata(item.getMetadata()); - - return xrecord; - } - - private XSLPipeline toPipeline(Item item) throws XmlWriteException, XMLStreamException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(output); - Metadata metadata = item.getMetadata(); - metadata.write(writer); - writer.close(); - return new XSLPipeline(new ByteArrayInputStream(output.toByteArray()), true); - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XitemRepository.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XitemRepository.java deleted file mode 100644 index b4c60a3171d..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XitemRepository.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xoai.dataprovider.exceptions.IdDoesNotExistException; -import com.lyncode.xoai.dataprovider.exceptions.OAIException; -import com.lyncode.xoai.dataprovider.filter.ScopedFilter; -import com.lyncode.xoai.dataprovider.handlers.results.ListItemIdentifiersResult; -import com.lyncode.xoai.dataprovider.handlers.results.ListItemsResults; -import com.lyncode.xoai.dataprovider.model.Item; -import com.lyncode.xoai.dataprovider.model.ItemIdentifier; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.dataprovider.repository.ItemRepository; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.harvest.server.OAIRecord; -import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean; -import edu.harvard.iq.dataverse.util.StringUtil; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.logging.Logger; - -/** - * - * @author Leonid Andreev - * Implements an XOAI "Item Repository". Retrieves Dataverse "OAIRecords" - * representing harvestable local datasets and translates them into - * XOAI "items". - */ - -public class XitemRepository implements ItemRepository { - private static Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository"); - - private OAIRecordServiceBean recordService; - private DatasetServiceBean datasetService; - - public XitemRepository (OAIRecordServiceBean recordService, DatasetServiceBean datasetService) { - super(); - this.recordService = recordService; - this.datasetService = datasetService; - } - - private List list = new ArrayList(); - - - @Override - public Item getItem(String identifier) throws IdDoesNotExistException, OAIException { - logger.fine("getItem; calling findOaiRecordsByGlobalId, identifier " + identifier); - List oaiRecords = recordService.findOaiRecordsByGlobalId(identifier); - if (oaiRecords != null && !oaiRecords.isEmpty()) { - Xitem xoaiItem = null; - for (OAIRecord record : oaiRecords) { - if (xoaiItem == null) { - Dataset dataset = datasetService.findByGlobalId(record.getGlobalId()); - if (dataset != null) { - xoaiItem = new Xitem(record).withDataset(dataset); - } - } else { - // Adding extra set specs to the XOAI Item, if this record - // is part of multiple sets: - if (!StringUtil.isEmpty(record.getSetName())) { - xoaiItem.getSets().add(new Set(record.getSetName())); - } - } - } - if (xoaiItem != null) { - return xoaiItem; - } - } - - throw new IdDoesNotExistException(); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length) throws OAIException { - return getItemIdentifiers(filters, offset, length, null, null, null); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, Date from) throws OAIException { - return getItemIdentifiers(filters, offset, length, null, from, null); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiersUntil(List filters, int offset, int length, Date until) throws OAIException { - return getItemIdentifiers(filters, offset, length, null, null, until); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, Date from, Date until) throws OAIException { - return getItemIdentifiers(filters, offset, length, null, from, until); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, String setSpec) throws OAIException { - return getItemIdentifiers(filters, offset, length, setSpec, null, null); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, String setSpec, Date from) throws OAIException { - return getItemIdentifiers(filters, offset, length, setSpec, from, null); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiersUntil(List filters, int offset, int length, String setSpec, Date until) throws OAIException { - return getItemIdentifiers(filters, offset, length, setSpec, null, until); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, String setSpec, Date from, Date until) throws OAIException { - logger.fine("calling getItemIdentifiers; offset=" + offset - + ", length=" + length - + ", setSpec=" + setSpec - + ", from=" + from - + ", until=" + until); - - List oaiRecords = recordService.findOaiRecordsBySetName(setSpec, from, until); - - logger.fine("total " + oaiRecords.size() + " returned"); - - List xoaiItems = new ArrayList<>(); - if (oaiRecords != null && !oaiRecords.isEmpty()) { - - for (int i = offset; i < offset + length && i < oaiRecords.size(); i++) { - OAIRecord record = oaiRecords.get(i); - xoaiItems.add(new Xitem(record)); - } - - // Run a second pass, looking for records in this set that occur - // in *other* sets. Then we'll add these multiple sets to the - // formatted output in the header: - addExtraSets(xoaiItems, setSpec, from, until); - - boolean hasMore = offset + length < oaiRecords.size(); - ListItemIdentifiersResult result = new ListItemIdentifiersResult(hasMore, xoaiItems); - logger.fine("returning result with " + xoaiItems.size() + " items."); - return result; - } - - return new ListItemIdentifiersResult(false, xoaiItems); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length) throws OAIException { - return getItems(filters, offset, length, null, null, null); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, Date from) throws OAIException { - return getItems(filters, offset, length, null, from, null); - } - - @Override - public ListItemsResults getItemsUntil(List filters, int offset, int length, Date until) throws OAIException { - return getItems(filters, offset, length, null, null, until); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, Date from, Date until) throws OAIException { - return getItems(filters, offset, length, null, from, until); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, String setSpec) throws OAIException { - return getItems(filters, offset, length, setSpec, null, null); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, String setSpec, Date from) throws OAIException { - return getItems(filters, offset, length, setSpec, from, null); - } - - @Override - public ListItemsResults getItemsUntil(List filters, int offset, int length, String setSpec, Date until) throws OAIException { - return getItems(filters, offset, length, setSpec, null, until); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, String setSpec, Date from, Date until) throws OAIException { - logger.fine("calling getItems; offset=" + offset - + ", length=" + length - + ", setSpec=" + setSpec - + ", from=" + from - + ", until=" + until); - - List oaiRecords = recordService.findOaiRecordsBySetName(setSpec, from, until); - - logger.fine("total " + oaiRecords.size() + " returned"); - - List xoaiItems = new ArrayList<>(); - if (oaiRecords != null && !oaiRecords.isEmpty()) { - - for (int i = offset; i < offset + length && i < oaiRecords.size(); i++) { - OAIRecord oaiRecord = oaiRecords.get(i); - Dataset dataset = datasetService.findByGlobalId(oaiRecord.getGlobalId()); - if (dataset != null) { - Xitem xItem = new Xitem(oaiRecord).withDataset(dataset); - xoaiItems.add(xItem); - } - } - - addExtraSets(xoaiItems, setSpec, from, until); - - boolean hasMore = offset + length < oaiRecords.size(); - ListItemsResults result = new ListItemsResults(hasMore, xoaiItems); - logger.fine("returning result with " + xoaiItems.size() + " items."); - return result; - } - - return new ListItemsResults(false, xoaiItems); - } - - private void addExtraSets(Object xoaiItemsList, String setSpec, Date from, Date until) { - - List xoaiItems = (List)xoaiItemsList; - - List oaiRecords = recordService.findOaiRecordsNotInThisSet(setSpec, from, until); - - if (oaiRecords == null || oaiRecords.isEmpty()) { - return; - } - - // Make a second pass through the list of xoaiItems already found for this set, - // and add any other sets in which this item occurs: - - int j = 0; - for (int i = 0; i < xoaiItems.size(); i++) { - // fast-forward the second list, until we find a record with this identifier, - // or until we are past this record (both lists are sorted alphabetically by - // the identifier: - Xitem xitem = xoaiItems.get(i); - - while (j < oaiRecords.size() && xitem.getIdentifier().compareTo(oaiRecords.get(j).getGlobalId()) > 0) { - j++; - } - - while (j < oaiRecords.size() && xitem.getIdentifier().equals(oaiRecords.get(j).getGlobalId())) { - xoaiItems.get(i).getSets().add(new Set(oaiRecords.get(j).getSetName())); - j++; - } - } - - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java deleted file mode 100644 index 15bd005cacf..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java +++ /dev/null @@ -1,86 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xml.exceptions.XmlWriteException; -import static com.lyncode.xoai.model.oaipmh.Granularity.Second; -import com.lyncode.xoai.model.oaipmh.ListRecords; -import com.lyncode.xoai.model.oaipmh.Record; -import com.lyncode.xoai.model.oaipmh.ResumptionToken; -import com.lyncode.xoai.xml.XmlWriter; -import static com.lyncode.xoai.xml.XmlWriter.defaultContext; -import java.io.ByteArrayOutputStream; - -import java.io.IOException; -import java.io.OutputStream; -import javax.xml.stream.XMLStreamException; - -/** - * - * @author Leonid Andreev - * - * This is the Dataverse extension of XOAI ListRecords, - * optimized to stream individual records using fast dumping - * of pre-exported metadata fragments (and by-passing expensive - * XML parsing and writing). - */ -public class XlistRecords extends ListRecords { - private static final String RECORD_FIELD = "record"; - private static final String RECORD_START_ELEMENT = "<"+RECORD_FIELD+">"; - private static final String RECORD_CLOSE_ELEMENT = ""; - private static final String RESUMPTION_TOKEN_FIELD = "resumptionToken"; - private static final String EXPIRATION_DATE_ATTRIBUTE = "expirationDate"; - private static final String COMPLETE_LIST_SIZE_ATTRIBUTE = "completeListSize"; - private static final String CURSOR_ATTRIBUTE = "cursor"; - - public void writeToStream(OutputStream outputStream) throws IOException { - if (!this.records.isEmpty()) { - for (Record record : this.records) { - outputStream.write(RECORD_START_ELEMENT.getBytes()); - outputStream.flush(); - - ((Xrecord)record).writeToStream(outputStream); - - outputStream.write(RECORD_CLOSE_ELEMENT.getBytes()); - outputStream.flush(); - } - } - - if (resumptionToken != null) { - - String resumptionTokenString = resumptionTokenToString(resumptionToken); - if (resumptionTokenString == null) { - throw new IOException("XlistRecords: failed to output resumption token"); - } - outputStream.write(resumptionTokenString.getBytes()); - outputStream.flush(); - } - } - - private String resumptionTokenToString(ResumptionToken token) { - try { - ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(byteOutputStream, defaultContext()); - - writer.writeStartElement(RESUMPTION_TOKEN_FIELD); - - if (token.getExpirationDate() != null) - writer.writeAttribute(EXPIRATION_DATE_ATTRIBUTE, token.getExpirationDate(), Second); - if (token.getCompleteListSize() != null) - writer.writeAttribute(COMPLETE_LIST_SIZE_ATTRIBUTE, "" + token.getCompleteListSize()); - if (token.getCursor() != null) - writer.writeAttribute(CURSOR_ATTRIBUTE, "" + token.getCursor()); - if (token.getValue() != null) - writer.write(token.getValue()); - - writer.writeEndElement(); // resumptionToken; - writer.flush(); - writer.close(); - - String ret = byteOutputStream.toString(); - - return ret; - } catch (XMLStreamException | XmlWriteException e) { - return null; - } - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecordsHandler.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecordsHandler.java deleted file mode 100644 index 8fe13bc4044..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecordsHandler.java +++ /dev/null @@ -1,168 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xml.exceptions.XmlWriteException; -import com.lyncode.xoai.dataprovider.exceptions.BadArgumentException; -import com.lyncode.xoai.dataprovider.exceptions.CannotDisseminateFormatException; -import com.lyncode.xoai.dataprovider.exceptions.CannotDisseminateRecordException; -import com.lyncode.xoai.dataprovider.exceptions.DoesNotSupportSetsException; -import com.lyncode.xoai.dataprovider.exceptions.HandlerException; -import com.lyncode.xoai.dataprovider.exceptions.NoMatchesException; -import com.lyncode.xoai.dataprovider.exceptions.NoMetadataFormatsException; -import com.lyncode.xoai.dataprovider.exceptions.OAIException; -import com.lyncode.xoai.dataprovider.handlers.VerbHandler; -import com.lyncode.xoai.dataprovider.handlers.results.ListItemsResults; -import com.lyncode.xoai.dataprovider.handlers.helpers.ItemHelper; -import com.lyncode.xoai.dataprovider.handlers.helpers.ItemRepositoryHelper; -import com.lyncode.xoai.dataprovider.handlers.helpers.SetRepositoryHelper; -import com.lyncode.xoai.dataprovider.model.Context; -import com.lyncode.xoai.dataprovider.model.Item; -import com.lyncode.xoai.dataprovider.model.MetadataFormat; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.dataprovider.parameters.OAICompiledRequest; -import com.lyncode.xoai.dataprovider.repository.Repository; -import com.lyncode.xoai.model.oaipmh.Header; -import com.lyncode.xoai.model.oaipmh.ListRecords; -import com.lyncode.xoai.model.oaipmh.Metadata; -import com.lyncode.xoai.model.oaipmh.Record; -import com.lyncode.xoai.model.oaipmh.ResumptionToken; -import com.lyncode.xoai.xml.XSLPipeline; -import com.lyncode.xoai.xml.XmlWriter; -import edu.harvard.iq.dataverse.Dataset; - -import javax.xml.stream.XMLStreamException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.List; - -/** - * - * @author Leonid Andreev - * - * This is Dataverse's own implementation of ListRecords Verb Handler - * (used instead of the ListRecordsHandler provided by XOAI). - * It is customized to support the optimizations that allows - * Dataverse to directly output pre-exported metadata records to the output - * stream, bypassing expensive XML parsing and writing. - */ -public class XlistRecordsHandler extends VerbHandler { - private static java.util.logging.Logger logger = java.util.logging.Logger.getLogger("XlistRecordsHandler"); - private final ItemRepositoryHelper itemRepositoryHelper; - private final SetRepositoryHelper setRepositoryHelper; - - public XlistRecordsHandler(Context context, Repository repository) { - super(context, repository); - this.itemRepositoryHelper = new ItemRepositoryHelper(getRepository().getItemRepository()); - this.setRepositoryHelper = new SetRepositoryHelper(getRepository().getSetRepository()); - } - - @Override - public ListRecords handle(OAICompiledRequest parameters) throws OAIException, HandlerException { - XlistRecords res = new XlistRecords(); - int length = getRepository().getConfiguration().getMaxListRecords(); - - if (parameters.hasSet() && !getRepository().getSetRepository().supportSets()) - throw new DoesNotSupportSetsException(); - - int offset = getOffset(parameters); - ListItemsResults result; - if (!parameters.hasSet()) { - if (parameters.hasFrom() && !parameters.hasUntil()) - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getFrom()); - else if (!parameters.hasFrom() && parameters.hasUntil()) - result = itemRepositoryHelper.getItemsUntil(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getUntil()); - else if (parameters.hasFrom() && parameters.hasUntil()) - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getFrom(), parameters.getUntil()); - else - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix()); - } else { - if (!setRepositoryHelper.exists(getContext(), parameters.getSet())) { - // throw new NoMatchesException(); - } - if (parameters.hasFrom() && !parameters.hasUntil()) - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getSet(), parameters.getFrom()); - else if (!parameters.hasFrom() && parameters.hasUntil()) - result = itemRepositoryHelper.getItemsUntil(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getSet(), parameters.getUntil()); - else if (parameters.hasFrom() && parameters.hasUntil()) - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getSet(), parameters.getFrom(), - parameters.getUntil()); - else - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getSet()); - } - - List results = result.getResults(); - if (results.isEmpty()) throw new NoMatchesException(); - for (Item i : results) - res.withRecord(this.createRecord(parameters, i)); - - - ResumptionToken.Value currentResumptionToken = new ResumptionToken.Value(); - if (parameters.hasResumptionToken()) { - currentResumptionToken = parameters.getResumptionToken(); - } else if (result.hasMore()) { - currentResumptionToken = parameters.extractResumptionToken(); - } - - XresumptionTokenHelper resumptionTokenHelper = new XresumptionTokenHelper(currentResumptionToken, - getRepository().getConfiguration().getMaxListRecords()); - res.withResumptionToken(resumptionTokenHelper.resolve(result.hasMore())); - - return res; - } - - - private int getOffset(OAICompiledRequest parameters) { - if (!parameters.hasResumptionToken()) - return 0; - if (parameters.getResumptionToken().getOffset() == null) - return 0; - return parameters.getResumptionToken().getOffset().intValue(); - } - - private Record createRecord(OAICompiledRequest parameters, Item item) - throws BadArgumentException, CannotDisseminateRecordException, - OAIException, NoMetadataFormatsException, CannotDisseminateFormatException { - MetadataFormat format = getContext().formatForPrefix(parameters.getMetadataPrefix()); - Header header = new Header(); - - Dataset dataset = ((Xitem)item).getDataset(); - Xrecord xrecord = new Xrecord().withFormatName(parameters.getMetadataPrefix()).withDataset(dataset); - header.withIdentifier(item.getIdentifier()); - - ItemHelper itemHelperWrap = new ItemHelper(item); - header.withDatestamp(item.getDatestamp()); - for (Set set : itemHelperWrap.getSets(getContext(), getRepository().getFilterResolver())) - header.withSetSpec(set.getSpec()); - if (item.isDeleted()) - header.withStatus(Header.Status.DELETED); - - xrecord.withHeader(header); - xrecord.withMetadata(item.getMetadata()); - - return xrecord; - } - - - private XSLPipeline toPipeline(Item item) throws XmlWriteException, XMLStreamException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(output); - Metadata metadata = item.getMetadata(); - metadata.write(writer); - writer.close(); - return new XSLPipeline(new ByteArrayInputStream(output.toByteArray()), true); - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xmetadata.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xmetadata.java deleted file mode 100644 index 225b9b13777..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xmetadata.java +++ /dev/null @@ -1,27 +0,0 @@ - -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xml.exceptions.XmlWriteException; -import com.lyncode.xoai.model.oaipmh.Metadata; -import com.lyncode.xoai.xml.XmlWriter; - -/** - * - * @author Leonid Andreev - */ -public class Xmetadata extends Metadata { - - - public Xmetadata(String value) { - super(value); - } - - - @Override - public void write(XmlWriter writer) throws XmlWriteException { - // Do nothing! - // - rather than writing Metadata as an XML writer stram, we will write - // the pre-exported *and pre-validated* content as a byte stream, directly. - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java deleted file mode 100644 index 7e115c78f06..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java +++ /dev/null @@ -1,184 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xoai.model.oaipmh.Header; -import com.lyncode.xoai.model.oaipmh.Record; -import com.lyncode.xoai.xml.XmlWriter; -import static com.lyncode.xoai.xml.XmlWriter.defaultContext; - -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.export.ExportException; -import edu.harvard.iq.dataverse.export.ExportService; -import static edu.harvard.iq.dataverse.util.SystemConfig.FQDN; -import static edu.harvard.iq.dataverse.util.SystemConfig.SITE_URL; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.UnknownHostException; -import org.apache.poi.util.ReplacingInputStream; - -/** - * - * @author Leonid Andreev - * - * This is the Dataverse extension of XOAI Record, - * optimized to directly output a pre-exported metadata record to the - * output stream, thus by-passing expensive parsing and writing by - * an XML writer, as in the original XOAI implementation. - */ - -public class Xrecord extends Record { - private static final String METADATA_FIELD = "metadata"; - private static final String METADATA_START_ELEMENT = "<"+METADATA_FIELD+">"; - private static final String METADATA_END_ELEMENT = ""; - private static final String HEADER_FIELD = "header"; - private static final String STATUS_ATTRIBUTE = "status"; - private static final String IDENTIFIER_FIELD = "identifier"; - private static final String DATESTAMP_FIELD = "datestamp"; - private static final String SETSPEC_FIELD = "setSpec"; - private static final String DATAVERSE_EXTENDED_METADATA_FORMAT = "dataverse_json"; - private static final String DATAVERSE_EXTENDED_METADATA_API = "/api/datasets/export"; - - protected Dataset dataset; - protected String formatName; - - - public Dataset getDataset() { - return dataset; - } - - public Xrecord withDataset(Dataset dataset) { - this.dataset = dataset; - return this; - } - - - public String getFormatName() { - return formatName; - } - - - public Xrecord withFormatName(String formatName) { - this.formatName = formatName; - return this; - } - - public void writeToStream(OutputStream outputStream) throws IOException { - outputStream.flush(); - - String headerString = itemHeaderToString(this.header); - - if (headerString == null) { - throw new IOException("Xrecord: failed to stream item header."); - } - - outputStream.write(headerString.getBytes()); - - // header.getStatus() is only non-null when it's indicating "deleted". - if (header.getStatus() == null) { // Deleted records should not show metadata - if (!isExtendedDataverseMetadataMode(formatName)) { - outputStream.write(METADATA_START_ELEMENT.getBytes()); - - outputStream.flush(); - - if (dataset != null && formatName != null) { - InputStream inputStream = null; - try { - inputStream = new ReplacingInputStream( - ExportService.getInstance().getExport(dataset, formatName), - "", - "" - ); - } catch (ExportException ex) { - inputStream = null; - } - - if (inputStream == null) { - throw new IOException("Xrecord: failed to open metadata stream."); - } - writeMetadataStream(inputStream, outputStream); - } - outputStream.write(METADATA_END_ELEMENT.getBytes()); - } else { - outputStream.write(customMetadataExtensionRef(this.dataset.getGlobalIdString()).getBytes()); - } - } - outputStream.flush(); - } - - private String itemHeaderToString(Header header) { - try { - ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(byteOutputStream, defaultContext()); - - writer.writeStartElement(HEADER_FIELD); - - if (header.getStatus() != null) { - writer.writeAttribute(STATUS_ATTRIBUTE, header.getStatus().value()); - } - writer.writeElement(IDENTIFIER_FIELD, header.getIdentifier()); - writer.writeElement(DATESTAMP_FIELD, header.getDatestamp()); - for (String setSpec : header.getSetSpecs()) { - writer.writeElement(SETSPEC_FIELD, setSpec); - } - writer.writeEndElement(); // header - writer.flush(); - writer.close(); - - String ret = byteOutputStream.toString(); - - return ret; - } catch (Exception ex) { - return null; - } - } - - private void writeMetadataStream(InputStream inputStream, OutputStream outputStream) throws IOException { - int bufsize; - byte[] buffer = new byte[4 * 8192]; - - while ((bufsize = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bufsize); - outputStream.flush(); - } - - inputStream.close(); - } - - private String customMetadataExtensionRef(String identifier) { - String ret = "<" + METADATA_FIELD - + " directApiCall=\"" - + getDataverseSiteUrl() - + DATAVERSE_EXTENDED_METADATA_API - + "?exporter=" - + DATAVERSE_EXTENDED_METADATA_FORMAT - + "&persistentId=" - + identifier - + "\"" - + "/>"; - - return ret; - } - - private boolean isExtendedDataverseMetadataMode(String formatName) { - return DATAVERSE_EXTENDED_METADATA_FORMAT.equals(formatName); - } - - private String getDataverseSiteUrl() { - String hostUrl = System.getProperty(SITE_URL); - if (hostUrl != null && !"".equals(hostUrl)) { - return hostUrl; - } - String hostName = System.getProperty(FQDN); - if (hostName == null) { - try { - hostName = InetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException e) { - return null; - } - } - hostUrl = "https://" + hostName; - return hostUrl; - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XresumptionTokenHelper.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XresumptionTokenHelper.java deleted file mode 100644 index 7f9eac2cbe8..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XresumptionTokenHelper.java +++ /dev/null @@ -1,61 +0,0 @@ - -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xoai.dataprovider.handlers.helpers.ResumptionTokenHelper; -import com.lyncode.xoai.model.oaipmh.ResumptionToken; -import static java.lang.Math.round; -import static com.google.common.base.Predicates.isNull; - -/** - * - * @author Leonid Andreev - * Dataverse's own version of the XOAI ResumptionTokenHelper - * Fixes the issue with the offset cursor: the OAI validation spec - * insists that it starts with 0, while the XOAI implementation uses 1 - * as the initial offset. - */ -public class XresumptionTokenHelper { - - private ResumptionToken.Value current; - private long maxPerPage; - private Long totalResults; - - public XresumptionTokenHelper(ResumptionToken.Value current, long maxPerPage) { - this.current = current; - this.maxPerPage = maxPerPage; - } - - public XresumptionTokenHelper withTotalResults(long totalResults) { - this.totalResults = totalResults; - return this; - } - - public ResumptionToken resolve (boolean hasMoreResults) { - if (isInitialOffset() && !hasMoreResults) return null; - else { - if (hasMoreResults) { - ResumptionToken.Value next = current.next(maxPerPage); - return populate(new ResumptionToken(next)); - } else { - ResumptionToken resumptionToken = new ResumptionToken(); - resumptionToken.withCursor(round((current.getOffset()) / maxPerPage)); - if (totalResults != null) - resumptionToken.withCompleteListSize(totalResults); - return resumptionToken; - } - } - } - - private boolean isInitialOffset() { - return isNull().apply(current.getOffset()) || current.getOffset() == 0; - } - - private ResumptionToken populate(ResumptionToken resumptionToken) { - if (totalResults != null) - resumptionToken.withCompleteListSize(totalResults); - resumptionToken.withCursor(round((resumptionToken.getValue().getOffset() - maxPerPage)/ maxPerPage)); - return resumptionToken; - } - - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/DateUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/DateUtil.java index d6f22471f68..669780b9436 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/DateUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/DateUtil.java @@ -16,7 +16,9 @@ */ public class DateUtil { + public static String YEAR_PATTERN = "yyyy"; public static String YEAR_DASH_MONTH_PATTERN = "yyyy-MM"; + public static String YEAR_DASH_MONTH_DASH_DAY_PATTERN = "yyyy-MM-dd"; public static String formatDate(Date dateToformat) { String formattedDate; @@ -63,4 +65,40 @@ public static String formatDate(Timestamp datetimeToformat) { } } + public static Date parseDate(String dateString) { + SimpleDateFormat sdf; + Date date; + + // YYYY-MM-DD + date = parseDate(dateString, YEAR_DASH_MONTH_DASH_DAY_PATTERN); + if (date != null) { + return date; + } + + // YYYY-MM + date = parseDate(dateString, YEAR_DASH_MONTH_PATTERN); + if (date != null) { + return date; + } + + // YYYT + date = parseDate(dateString, YEAR_PATTERN); + return date; + + } + + public static Date parseDate(String dateString, String format) { + + try { + SimpleDateFormat sdf = new SimpleDateFormat(format); + Date date = sdf.parse(dateString); + return date; + } catch (ParseException ex) { + // ignore + } catch (Exception ex) { + // ignore + } + return null; + } + } diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties index 16298d83118..be02bb1b090 100644 --- a/src/main/resources/META-INF/microprofile-config.properties +++ b/src/main/resources/META-INF/microprofile-config.properties @@ -8,3 +8,10 @@ dataverse.db.host=localhost dataverse.db.port=5432 dataverse.db.user=dataverse dataverse.db.name=dataverse +# OAI SERVER +dataverse.oai.server.maxidentifiers=100 +dataverse.oai.server.maxrecords=10 +dataverse.oai.server.maxsets=100 +# the OAI repository name, as shown by the Identify verb, +# can be customized via the setting below: +#dataverse.oai.server.repositoryname=