Merge pull request #1 from IQSS/develop

Sync repo from base

README.md

@@ -1,4 +1,4 @@
-Dataverse®
+Dataverse® 
 ===============
 
 Dataverse is an [open source][] web application for sharing, citing, analyzing, and preserving research data (developed by the [Data Science and Products team](http://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](http://iq.harvard.edu/) and the [Dataverse community][]).
@@ -17,6 +17,8 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi
 
 [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org)
 
+[![Waffle.io - Columns and their card count](https://badge.waffle.io/IQSS/dataverse.svg?columns=all)](https://waffle.io/IQSS/dataverse)
+
 [![Build Status](https://travis-ci.org/IQSS/dataverse.svg?branch=develop)](https://travis-ci.org/IQSS/dataverse) [![Coverage Status](https://coveralls.io/repos/IQSS/dataverse/badge.svg?branch=develop&service=github)](https://coveralls.io/github/IQSS/dataverse?branch=develop)
 
 [dataverse.org]: https://dataverse.org

conf/docker-aio/1prep.sh

@@ -9,6 +9,7 @@ mkdir -p testdata/doc/sphinx-guides/source/_static/util/
 cp ../solr/7.3.0/schema.xml testdata/
 cp ../solr/7.3.0/solrconfig.xml testdata/
 cp ../jhove/jhove.conf testdata/
+cp ../jhove/jhoveConfig.xsd testdata/
 cd ../../
 cp -r scripts conf/docker-aio/testdata/
 cp doc/sphinx-guides/source/_static/util/pg8-createsequence-prep.sql conf/docker-aio/testdata/doc/sphinx-guides/source/_static/util/
@@ -21,5 +22,6 @@ mvn package
 cd scripts/installer
 make clean
 make
+mkdir -p ../../conf/docker-aio/dv/install
 cp dvinstall.zip ../../conf/docker-aio/dv/install/
 

conf/docker-aio/c7.dockerfile

@@ -1,6 +1,8 @@
 FROM centos:7
 # OS dependencies
-RUN yum install -y java-1.8.0-openjdk-headless postgresql-server sudo epel-release unzip perl curl httpd
+RUN yum install -y https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-7-x86_64/pgdg-centos96-9.6-3.noarch.rpm
+#RUN yum install -y java-1.8.0-openjdk-headless postgresql-server sudo epel-release unzip perl curl httpd
+RUN yum install -y java-1.8.0-openjdk-headless postgresql96-server sudo epel-release unzip perl curl httpd
 RUN yum install -y jq
 
 # copy and unpack dependencies (solr, glassfish)
@@ -10,14 +12,15 @@ COPY testdata/solrconfig.xml /tmp/dv
 # IPv6 and localhost appears to be related to some of the intermittant connection issues
 COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
-COPY testdata/httpd.conf /etc/httpd/conf 
+COPY httpd.conf /etc/httpd/conf 
 RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.3.0dv.tgz 
 RUN cd /opt ; tar zxf /tmp/dv/deps/glassfish4dv.tgz
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
 COPY domain-restmonitor.xml /opt/glassfish4/glassfish/domains/domain1/config/domain.xml
 
-RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data
+#RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data
+RUN sudo -u postgres /usr/pgsql-9.6/bin/initdb -D /var/lib/pgsql/data
 
 # copy configuration related files
 RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/
@@ -39,6 +42,10 @@ EXPOSE 8080
 # apache port, http
 EXPOSE 80
 
+# debugger ports (jmx,jdb)
+EXPOSE 8686
+EXPOSE 9009
+
 RUN mkdir /opt/dv
 
 # yeah - still not happy if glassfish isn't in /usr/local :<

conf/docker-aio/entrypoint.bash

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
-sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data &
+#sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data &
+sudo -u postgres /usr/pgsql-9.6/bin/postgres -D /var/lib/pgsql/data &
 cd /opt/solr-7.3.0/
 # TODO: Run Solr as non-root and remove "-force".
 bin/solr start -force
@@ -11,6 +12,6 @@ apachectl -DFOREGROUND &
 
 # TODO: Run Glassfish as non-root.
 cd /opt/glassfish4
-bin/asadmin start-domain
+bin/asadmin start-domain --debug
 sleep infinity
 

conf/docker-aio/httpd.conf

@@ -0,0 +1,30 @@
+
+Include conf.d/*.conf
+Include conf.modules.d/*.conf
+ServerName localhost
+Listen 80 443
+PidFile run/httpd.pid
+DocumentRoot "/var/www/html"
+TypesConfig /etc/mime.types
+User apache
+Group apache
+
+<VirtualHost *:80>
+  ServerName localhost
+  LogLevel debug
+ ErrorLog logs/error_log
+ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+ CustomLog logs/access_log combined
+
+  # proxy config (aka - what to send to glassfish or not)
+  ProxyPassMatch ^/RApacheInfo$ !
+  ProxyPassMatch ^/custom !
+  ProxyPassMatch ^/dataexplore !
+  ProxyPassMatch ^/Shibboleth.sso !
+  ProxyPassMatch ^/shibboleth-ds !
+  # pass everything else to Glassfish
+  ProxyPass / ajp://localhost:8009/
+# glassfish can be slow sometimes
+  ProxyTimeout 300 
+
+</VirtualHost>

conf/docker-aio/install.bash

@@ -3,7 +3,7 @@ sudo -u postgres createuser --superuser dvnapp
 #./entrypoint.bash &
 unzip dvinstall.zip
 cd dvinstall/
-./install -admin_email=pameyer+dvinstall@crystal.harvard.edu -y -f > install.out 2> install.err
+./install -admin_email=dvAdmin@mailinator.com -y -f > install.out 2> install.err
 
 echo "installer complete"
 cat install.err

conf/docker-aio/prep_it.bash

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# run through all the steps to setup docker-aio to run integration tests
+
+# hard-codes several assumptions: image is named dv0, container is named dv, port is 8084
+
+# setupIT.bash retries
+n_retries=10
+
+# glassfish healthy/ready retries
+n_wait=5
+
+cd conf/docker-aio
+./0prep_deps.sh
+./1prep.sh
+docker build -t dv0 -f c7.dockerfile .
+i_retry=0
+setup_ok=0
+while [ $i_retry -lt $n_retries ]
+do
+	# cleanup from previous runs if necessary
+	docker rm -f dv
+	# start container
+	docker run -d -p 8084:80 -p 8083:8080 -p 9010:9009 --name dv dv0
+
+	# wait for glassfish to be healthy
+	i_wait=0
+	d_wait=10
+	while [ $i_wait -lt $n_wait ]
+	do
+		h=`docker inspect -f "{{.State.Health.Status}}" dv`
+		if [ "healthy" == "${h}" ]; then
+			break
+		else
+			sleep $d_wait
+		fi
+		i_wait=$(( $i_wait + 1 ))
+
+	done
+
+	# try setupIT.bash
+	docker exec -it dv /opt/dv/setupIT.bash
+	err=$?
+	if [ $err -eq 0 ]
+	then
+		setup_ok=1
+		echo "setupIT worked ($i_retry tries)"
+		break # success!
+	fi
+	echo "$i_retry try in loop"
+	i_retry=$(( $i_retry + 1 ))
+done
+if [ $setup_ok -ne 1 ]
+then
+	echo "failed setupIT after $i_retry tries ; bailing out"
+	exit 1
+fi
+
+# handle config for the private url test
+./seturl.bash
+err=$?
+if [ $err -ne 0 ]; then
+	echo "seturl fail; bailing out"
+	exit 1
+fi
+
+cd ../..
+echo "docker-aio ready to run integration tests ($i_retry)"
+curl http://localhost:8084/api/info/version
+echo $?
+

conf/docker-aio/readme.md

@@ -0,0 +1,59 @@
+# Docker All-In-One
+
+First pass docker all-in-one image, intended for running integration tests against.
+Also usable for normal development and system evaluation; not intended for production.
+
+### Requirements:
+ - java8 compiler, maven, make, wget, docker
+
+### Quickstart:
+ - in the root of the repository, run `./conf/docker-aio/prep_it.bash`
+ - if all goes well, you should see the results of the `api/info/version` endpoint, including the deployed build (eg `{"status":"OK","data":{"version":"4.8.6","build":"develop-c3e9f40"}}`). If not, you may need to read the non-quickstart instructions.
+ - run integration tests: `./conf/docker-aio/run-test-suite.sh`
+
+----
+
+## More in-depth documentation:
+
+
+### Initial setup (aka - do once):
+- `cd conf/docker-aio` and run `./0prep_deps.sh` to created Glassfish and Solr tarballs in `conf/docker-aio/dv/deps`.
+
+### Per-build:
+
+> Note: If you encounter any issues, see the Troubleshooting section at the end of this document.
+
+#### Setup
+
+- `cd conf/docker-aio`, and run `./1prep.sh` to copy files for integration test data into docker build context; `1prep.sh` will also build the war file and installation zip file
+- build the docker image: `docker build -t dv0 -f c7.dockerfile .`
+
+- Run image: `docker run -d -p 8083:8080 -p 8084:80 --name dv dv0` (aka - forward port 8083 locally to 8080 in the container for glassfish, and 8084 to 80 for apache); if you'd like to connect a java debugger to glassfish, use `docker run -d -p 8083:8080 -p 8084:80 -p 9010:9009 --name dv dv0`
+
+- Installation (integration test): `docker exec -it dv /opt/dv/setupIT.bash` 
+  (Note that it's possible to customize the installation by editing `conf/docker-aio/default.config` and running `docker exec -it dv /opt/dv/install.bash` but for the purposes of integration testing, the `setupIT.bash` script above works fine.)
+
+- update `dataverse.siteUrl` (appears only necessary for `DatasetsIT.testPrivateUrl`): `docker exec -it dv /usr/local/glassfish4/bin/asadmin create-jvm-options "-Ddataverse.siteUrl=http\://localhost\:8084"`
+
+#### Run integration tests: 
+
+First, cd back to the root of the repo where the `pom.xml` file is (`cd ../..` assuming you're still in the `conf/docker-aio` directory). Then run the test suite with script below:
+
+`conf/docker-aio/run-test-suite.sh`
+
+There isn't any strict requirement on the local port (8083, 8084 in this doc), the name of the image (dv0) or container (dv), these can be changed as desired as long as they are consistent.
+
+### Troubleshooting Notes:
+
+* If Dataverse' build fails due to an error about `Module` being ambiguous, you might be using a Java 9 compiler.
+
+* If you see an error like this: 
+ ```
+ docker: Error response from daemon: Conflict. The container name "/dv" is already in use by container "5f72a45b68c86c7b0f4305b83ce7d663020329ea4e30fa2a3ce9ddb05223533d"
+ You have to remove (or rename) that container to be able to reuse that name.
+ ``` 
+    run something like `docker ps -a | grep dv` to see the container left over from the last run and something like `docker rm 5f72a45b68c8` to remove it. Then try the `docker run` command above again.
+
+* `empty reply from server` or `Failed to connect to ::1: Cannot assign requested address` tend to indicate either that you haven't given glassfish enough time to start, or your docker setup is in an inconsistent state and should probably be restarted.
+
+* For manually fiddling around with the created dataverse, use user `dataverseAdmin` with password `admin`.

conf/docker-aio/testscripts/install

@@ -15,7 +15,8 @@ export SMTP_SERVER=localhost
 export MEM_HEAP_SIZE=2048
 export GLASSFISH_DOMAIN=domain1
 cd scripts/installer
-cp pgdriver/postgresql-8.4-703.jdbc4.jar $GLASSFISH_ROOT/glassfish/lib
+cp pgdriver/postgresql-42.2.2.jar $GLASSFISH_ROOT/glassfish/lib
 #cp ../../conf/jhove/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
 cp /opt/dv/testdata/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
+cp /opt/dv/testdata/jhoveConfig.xsd $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhoveConfig.xsd
 ./glassfish-setup.sh

conf/docker-dcm/.gitignore

@@ -0,0 +1,2 @@
+*.rpm
+upload*.bash

conf/docker-dcm/0prep.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+
+wget https://github.com/sbgrid/data-capture-module/releases/download/0.1/dcm-0.1-1.noarch.rpm

conf/docker-dcm/bashrc

@@ -0,0 +1,18 @@
+# .bashrc
+
+# User specific aliases and functions
+
+alias rm='rm -i'
+alias cp='cp -i'
+alias mv='mv -i'
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+# these are dummy values, obviously
+export UPLOADHOST=dcmsrv
+export DVAPIKEY=burrito
+export DVHOSTINT=dvsrv
+export DVHOST=dvsrv

conf/docker-dcm/c6client.dockerfile

@@ -0,0 +1,7 @@
+# build from repo root
+FROM centos:6
+RUN yum install -y epel-release
+RUN yum install -y rsync openssh-clients jq curl wget lynx
+RUN useradd depositor
+USER depositor
+WORKDIR /home/depositor

conf/docker-dcm/configure_dcm.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+echo "dcm configs on dv side to be done"
+
+# in homage to dataverse traditions, reset to insecure "burrito" admin API key
+sudo -u postgres psql -c "update apitoken set tokenstring='burrito' where id=1;" dvndb
+
+# dataverse configs for DCM
+curl -X PUT -d "SHA-1" "http://localhost:8080/api/admin/settings/:FileFixityChecksumAlgorithm"
+curl -X PUT "http://localhost:8080/api/admin/settings/:UploadMethods" -d "dcm/rsync+ssh"
+curl -X PUT "http://localhost:8080/api/admin/settings/:DataCaptureModuleUrl" -d "http://dcmsrv"
+
+# configure for RSAL downloads; but no workflows or RSAL yet
+curl -X PUT "http://localhost:8080/api/admin/settings/:DownloadMethods" -d "rsal/rsync"
+
+# publish root dataverse
+curl -X POST -H "X-Dataverse-key: burrito" "http://localhost:8080/api/dataverses/root/actions/:publish"
+
+# symlink `hold` volume 
+mkdir -p /usr/local/glassfish4/glassfish/domains/domain1/files/10.5072/
+ln -s /hold /usr/local/glassfish4/glassfish/domains/domain1/files/10.5072/FK2
+

conf/docker-dcm/create.bash

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+
+# user creates dataset
+k_d=burrito
+dv_d=root
+h=http://dvsrv
+
+fn=dataset.json
+#dset_id=`curl -s -H "X-Dataverse-key: $k_d" -X POST --upload-file $fn $h/api/dataverses/$dv_d/datasets | jq .data.id`
+r=`curl -s -H "X-Dataverse-key: $k_d" -X POST --upload-file $fn $h/api/dataverses/$dv_d/datasets`
+echo $r
+dset_id=`echo $r | jq .data.id`
+echo "dataset created with id: $dset_id"
+
+if [ "null" == "${dset_id}" ]; then
+	echo "error - no dataset id from create command"
+	exit 1
+fi
+echo "dataset created; internal/db id: ${dset_id}"
+
+