diff --git a/.github/workflows/check_property_files.yml b/.github/workflows/check_property_files.yml new file mode 100644 index 00000000000..505310aab35 --- /dev/null +++ b/.github/workflows/check_property_files.yml @@ -0,0 +1,32 @@ +name: "Properties Check" +on: + pull_request: + paths: + - "src/**/*.properties" + - "scripts/api/data/metadatablocks/*" +jobs: + duplicate_keys: + name: Duplicate Keys + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run duplicates detection script + shell: bash + run: tests/check_duplicate_properties.sh + + metadata_blocks_properties: + name: Metadata Blocks Properties + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup GraalVM + Native Image + uses: graalvm/setup-graalvm@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + java-version: '21' + distribution: 'graalvm-community' + - name: Setup JBang + uses: jbangdev/setup-jbang@main + - name: Run metadata block properties verification script + shell: bash + run: tests/verify_mdb_properties.sh diff --git a/doc/release-notes/make-data-count-.md b/doc/release-notes/make-data-count-.md new file mode 100644 index 00000000000..9022582dddb --- /dev/null +++ b/doc/release-notes/make-data-count-.md @@ -0,0 +1,3 @@ +### Counter Processor 1.05 Support + +This release includes support for counter-processor-1.05 for processing Make Data Count metrics. If you are running Make Data Counts support, you should reinstall/reconfigure counter-processor as described in the latest Guides. (For existing installations, note that counter-processor-1.05 requires a Python3, so you will need to follow the full counter-processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse.) diff --git a/doc/sphinx-guides/source/_static/util/counter_daily.sh b/doc/sphinx-guides/source/_static/util/counter_daily.sh index 674972b18f2..5095a83b7e2 100644 --- a/doc/sphinx-guides/source/_static/util/counter_daily.sh +++ b/doc/sphinx-guides/source/_static/util/counter_daily.sh @@ -1,6 +1,6 @@ #! /bin/bash -COUNTER_PROCESSOR_DIRECTORY="/usr/local/counter-processor-0.1.04" +COUNTER_PROCESSOR_DIRECTORY="/usr/local/counter-processor-1.05" MDC_LOG_DIRECTORY="/usr/local/payara6/glassfish/domains/domain1/logs/mdc" # counter_daily.sh diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index fe32af6649a..51bc2c4a9fe 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -16,7 +16,7 @@ Architecture Dataverse installations who would like support for Make Data Count must install `Counter Processor`_, a Python project created by California Digital Library (CDL) which is part of the Make Data Count project and which runs the software in production as part of their `DASH`_ data sharing platform. -.. _Counter Processor: https://github.com/CDLUC3/counter-processor +.. _Counter Processor: https://github.com/gdcc/counter-processor .. _DASH: https://cdluc3.github.io/dash/ The diagram below shows how Counter Processor interacts with your Dataverse installation and the DataCite hub, once configured. Dataverse installations using Handles rather than DOIs should note the limitations in the next section of this page. @@ -84,9 +84,9 @@ Configure Counter Processor * Change to the directory where you installed Counter Processor. - * ``cd /usr/local/counter-processor-0.1.04`` + * ``cd /usr/local/counter-processor-1.05`` -* Download :download:`counter-processor-config.yaml <../_static/admin/counter-processor-config.yaml>` to ``/usr/local/counter-processor-0.1.04``. +* Download :download:`counter-processor-config.yaml <../_static/admin/counter-processor-config.yaml>` to ``/usr/local/counter-processor-1.05``. * Edit the config file and pay particular attention to the FIXME lines. @@ -99,7 +99,7 @@ Soon we will be setting up a cron job to run nightly but we start with a single * Change to the directory where you installed Counter Processor. - * ``cd /usr/local/counter-processor-0.1.04`` + * ``cd /usr/local/counter-processor-1.05`` * If you are running Counter Processor for the first time in the middle of a month, you will need create blank log files for the previous days. e.g.: diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst index a28757165c5..6a1edcf7ebd 100644 --- a/doc/sphinx-guides/source/container/dev-usage.rst +++ b/doc/sphinx-guides/source/container/dev-usage.rst @@ -341,6 +341,11 @@ The steps below describe options to enable the later in different IDEs. **IMPORTANT**: This tool uses a Bash shell script and is thus limited to Mac and Linux OS. +Exploring the Database +---------------------- + +See :ref:`db-name-creds` in the Developer Guide. + Using a Debugger ---------------- diff --git a/doc/sphinx-guides/source/contributor/documentation.md b/doc/sphinx-guides/source/contributor/documentation.md index 12a4266c9ff..ebaf111aa4b 100644 --- a/doc/sphinx-guides/source/contributor/documentation.md +++ b/doc/sphinx-guides/source/contributor/documentation.md @@ -74,8 +74,10 @@ In some parts of the documentation, graphs are rendered as images using the Sphi Building the guides requires the ``dot`` executable from GraphViz. -This requires having `GraphViz `_ installed and either having ``dot`` on the path or -`adding options to the make call `_. +This requires having [GraphViz](https://graphviz.org) installed and either having ``dot`` on the path or +[adding options to the `make` call](https://groups.google.com/forum/#!topic/sphinx-users/yXgNey_0M3I). + +On a Mac we recommend installing GraphViz through [Homebrew](). Once you have Homebrew installed and configured to work with your shell, you can type `brew install graphviz`. ### Editing and Building the Guides diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 2837f901d5e..f0d1b116ea7 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -32,14 +32,20 @@ Install Java The Dataverse Software requires Java 17. -On Mac and Windows, we suggest downloading OpenJDK from https://adoptium.net (formerly `AdoptOpenJDK `_) or `SDKMAN `_. +On Mac and Windows, we suggest using `SDKMAN `_ to install Temurin (Eclipe's name for its OpenJDK distribution). Type ``sdk install java 17`` and then hit the "tab" key until you get to a version that ends with ``-tem`` and then hit enter. + +Alternatively you can download Temurin from https://adoptium.net (formerly `AdoptOpenJDK `_). On Linux, you are welcome to use the OpenJDK available from package managers. Install Maven ~~~~~~~~~~~~~ -Follow instructions at https://maven.apache.org +If you are using SKDMAN, run this command: + +``sdk install maven`` + +Otherwise, follow instructions at https://maven.apache.org. Install and Start Docker ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/developers/make-data-count.rst b/doc/sphinx-guides/source/developers/make-data-count.rst index edad580e451..f347e7b8ff9 100644 --- a/doc/sphinx-guides/source/developers/make-data-count.rst +++ b/doc/sphinx-guides/source/developers/make-data-count.rst @@ -1,7 +1,7 @@ Make Data Count =============== -Support for Make Data Count is a feature of the Dataverse Software that is described in the :doc:`/admin/make-data-count` section of the Admin Guide. In order for developers to work on the feature, they must install Counter Processor, a Python 3 application, as described below. Counter Processor can be found at https://github.com/CDLUC3/counter-processor +Support for Make Data Count is a feature of the Dataverse Software that is described in the :doc:`/admin/make-data-count` section of the Admin Guide. In order for developers to work on the feature, they must install Counter Processor, a Python 3 application, as described below. Counter Processor can be found at https://github.com/gdcc/counter-processor .. contents:: |toctitle| :local: @@ -49,7 +49,7 @@ Once you are done with your configuration, you can run Counter Processor like th ``su - counter`` -``cd /usr/local/counter-processor-0.1.04`` +``cd /usr/local/counter-processor-1.05`` ``CONFIG_FILE=counter-processor-config.yaml python39 main.py`` @@ -82,7 +82,7 @@ Second, if you are also sending your SUSHI report to Make Data Count, you will n ``curl -H "Authorization: Bearer $JSON_WEB_TOKEN" -X DELETE https://$MDC_SERVER/reports/$REPORT_ID`` -To get the ``REPORT_ID``, look at the logs generated in ``/usr/local/counter-processor-0.1.04/tmp/datacite_response_body.txt`` +To get the ``REPORT_ID``, look at the logs generated in ``/usr/local/counter-processor-1.05/tmp/datacite_response_body.txt`` To read more about the Make Data Count api, see https://github.com/datacite/sashimi diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index 839ae3aa19d..f5ffbac0c07 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -94,23 +94,63 @@ Then configure the JVM option mentioned in :ref:`install-imagemagick` to the pat Database Schema Exploration --------------------------- -With over 100 tables, the Dataverse Software PostgreSQL database ("dvndb") can be somewhat daunting for newcomers. Here are some tips for coming up to speed. (See also the :doc:`sql-upgrade-scripts` section.) +With over 100 tables, the Dataverse PostgreSQL database can be somewhat daunting for newcomers. Here are some tips for coming up to speed. (See also the :doc:`sql-upgrade-scripts` section.) + +.. _db-name-creds: + +Database Name and Credentials +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The default database name and credentials depends on how you set up your dev environment. + +.. list-table:: + :header-rows: 1 + :align: left + + * - MPCONFIG Key + - Docker + - Classic + * - dataverse.db.name + - ``dataverse`` + - ``dvndb`` + * - dataverse.db.user + - ``dataverse`` + - ``dvnapp`` + * - dataverse.db.password + - ``secret`` + - ``secret`` + +Here's an example of using these credentials from within the PostgreSQL container (see :doc:`/container/index`): + +.. code-block:: bash + + pdurbin@beamish dataverse % docker exec -it postgres-1 bash + root@postgres:/# export PGPASSWORD=secret + root@postgres:/# psql -h localhost -U dataverse dataverse + psql (16.3 (Debian 16.3-1.pgdg120+1)) + Type "help" for help. + + dataverse=# select id,alias from dataverse limit 1; + id | alias + ----+------- + 1 | root + (1 row) + +See also :ref:`database-persistence` in the Installation Guide. pgAdmin -~~~~~~~~ +~~~~~~~ -Back in the :doc:`classic-dev-env` section, we had you install pgAdmin, which can help you explore the tables and execute SQL commands. It's also listed in the :doc:`tools` section. +If you followed the :doc:`classic-dev-env` section, we had you install pgAdmin, which can help you explore the tables and execute SQL commands. It's also listed in the :doc:`tools` section. SchemaSpy ~~~~~~~~~ SchemaSpy is a tool that creates a website of entity-relationship diagrams based on your database. -As part of our build process for running integration tests against the latest code in the "develop" branch, we drop the database on the "phoenix" server, recreate the database by deploying the latest war file, and run SchemaSpy to create the following site: http://phoenix.dataverse.org/schemaspy/latest/relationships.html +We periodically run SchemaSpy and publish the output: https://guides.dataverse.org/en/6.2/schemaspy/index.html -To run this command on your laptop, download SchemaSpy and take a look at the syntax in ``scripts/deploy/phoenix.dataverse.org/post`` - -To read more about the phoenix server, see the :doc:`testing` section. +To run SchemaSpy locally, take a look at the syntax in ``scripts/deploy/phoenix.dataverse.org/post``. Deploying With ``asadmin`` -------------------------- diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 151d44e6841..f61321ef245 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -428,7 +428,7 @@ firewalled from your Dataverse installation host). Counter Processor ----------------- -Counter Processor is required to enable Make Data Count metrics in a Dataverse installation. See the :doc:`/admin/make-data-count` section of the Admin Guide for a description of this feature. Counter Processor is open source and we will be downloading it from https://github.com/CDLUC3/counter-processor +Counter Processor is required to enable Make Data Count metrics in a Dataverse installation. See the :doc:`/admin/make-data-count` section of the Admin Guide for a description of this feature. Counter Processor is open source and we will be downloading it from https://github.com/gdcc/counter-processor Installing Counter Processor ============================ @@ -438,9 +438,9 @@ A scripted installation using Ansible is mentioned in the :doc:`/developers/make As root, download and install Counter Processor:: cd /usr/local - wget https://github.com/CDLUC3/counter-processor/archive/v0.1.04.tar.gz - tar xvfz v0.1.04.tar.gz - cd /usr/local/counter-processor-0.1.04 + wget https://github.com/gdcc/counter-processor/archive/refs/tags/v1.05.tar.gz + tar xvfz v1.05.tar.gz + cd /usr/local/counter-processor-1.05 Installing GeoLite Country Database =================================== @@ -451,7 +451,7 @@ The process required to sign up, download the database, and to configure automat As root, change to the Counter Processor directory you just created, download the GeoLite2-Country tarball from MaxMind, untar it, and copy the geoip database into place:: - + tar xvfz GeoLite2-Country.tar.gz cp GeoLite2-Country_*/GeoLite2-Country.mmdb maxmind_geoip @@ -461,12 +461,12 @@ Creating a counter User As root, create a "counter" user and change ownership of Counter Processor directory to this new user:: useradd counter - chown -R counter:counter /usr/local/counter-processor-0.1.04 + chown -R counter:counter /usr/local/counter-processor-1.05 Installing Counter Processor Python Requirements ================================================ -Counter Processor version 0.1.04 requires Python 3.7 or higher. This version of Python is available in many operating systems, and is purportedly available for RHEL7 or CentOS 7 via Red Hat Software Collections. Alternately, one may compile it from source. +Counter Processor version 1.05 requires Python 3.7 or higher. This version of Python is available in many operating systems, and is purportedly available for RHEL7 or CentOS 7 via Red Hat Software Collections. Alternately, one may compile it from source. The following commands are intended to be run as root but we are aware that Pythonistas might prefer fancy virtualenv or similar setups. Pull requests are welcome to improve these steps! @@ -477,7 +477,7 @@ Install Python 3.9:: Install Counter Processor Python requirements:: python3.9 -m ensurepip - cd /usr/local/counter-processor-0.1.04 + cd /usr/local/counter-processor-1.05 pip3 install -r requirements.txt See the :doc:`/admin/make-data-count` section of the Admin Guide for how to configure and run Counter Processor. diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetrics.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetrics.java index ac3dff356eb..15ee35c6404 100644 --- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetrics.java +++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetrics.java @@ -117,10 +117,10 @@ public class DatasetMetrics implements Serializable { * For an example of sending various metric types (total-dataset-requests, * unique-dataset-investigations, etc) for a given month (2018-04) per * country (DK, US, etc.) see - * https://github.com/CDLUC3/counter-processor/blob/5ce045a09931fb680a32edcc561f88a407cccc8d/good_test.json#L893 + * https://github.com/gdcc/counter-processor/blob/5ce045a09931fb680a32edcc561f88a407cccc8d/good_test.json#L893 * * counter-processor uses GeoLite2 for IP lookups according to their - * https://github.com/CDLUC3/counter-processor#download-the-free-ip-to-geolocation-database + * https://github.com/gdcc/counter-processor#download-the-free-ip-to-geolocation-database */ @Column(nullable = true) private String countryCode; diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountUtil.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountUtil.java index 8f32750f090..30cbed18337 100644 --- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountUtil.java @@ -27,7 +27,7 @@ * How to Make Your Data Count July 10th, 2018). * * The recommended starting point to implement Make Data Count is - * https://github.com/CDLUC3/Make-Data-Count/blob/master/getting-started.md + * https://github.com/gdcc/Make-Data-Count/blob/master/getting-started.md * which specifically recommends reading the "COUNTER Code of Practice for * Research Data" mentioned in the user facing docs. * @@ -35,7 +35,7 @@ * https://dash.ucmerced.edu/stash/dataset/doi:10.6071/M3RP49 * * For processing logs we could try DASH's - * https://github.com/CDLUC3/counter-processor + * https://github.com/gdcc/counter-processor * * Next, DataOne implemented it, and you can see an example dataset here: * https://search.dataone.org/view/doi:10.5063/F1Z899CZ diff --git a/src/main/java/propertyFiles/codeMeta20.properties b/src/main/java/propertyFiles/codeMeta20.properties index c0e7eac6d4a..4f3eb087aa4 100644 --- a/src/main/java/propertyFiles/codeMeta20.properties +++ b/src/main/java/propertyFiles/codeMeta20.properties @@ -1,5 +1,6 @@ metadatablock.name=codeMeta20 -metadatablock.displayName=Software Metadata (CodeMeta 2.0) +metadatablock.displayName=Software Metadata (CodeMeta v2.0) +metadatablock.displayFacet=Software datasetfieldtype.codeVersion.title=Software Version datasetfieldtype.codeVersion.description=Version of the software instance, usually following some convention like SemVer etc. datasetfieldtype.codeVersion.watermark=e.g. 0.2.1 or 1.3 or 2021.1 etc diff --git a/tests/check_duplicate_properties.sh b/tests/check_duplicate_properties.sh new file mode 100755 index 00000000000..7d053bdba4b --- /dev/null +++ b/tests/check_duplicate_properties.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# This script will check Java *.properties files within the src dir for duplicates +# and print logs with file annotations about it. + +set -euo pipefail + +FAIL=0 + +while IFS= read -r -d '' FILE; do + + # Scan the whole file for duplicates + FILTER=$(grep -a -v -E "^(#.*|\s*$)" "$FILE" | cut -d"=" -f1 | sort | uniq -c | tr -s " " | { grep -vs "^ 1 " || true; }) + + # If there are any duplicates present, analyse further to point people to the source + if [ -n "$FILTER" ]; then + FAIL=1 + + echo "::group::$FILE" + for KEY in $(echo "$FILTER" | cut -d" " -f3); do + # Find duplicate lines' numbers by grepping for the KEY and cutting the number from the output + DUPLICATE_LINES=$(grep -n -E -e "^$KEY=" "$FILE" | cut -d":" -f1) + # Join the found line numbers for better error log + DUPLICATE_NUMBERS=$(echo "$DUPLICATE_LINES" | paste -sd ',') + + # This form will make Github annotate the lines in the PR that changes the properties file + for LINE_NUMBER in $DUPLICATE_LINES; do + echo "::error file=$FILE,line=$LINE_NUMBER::Found duplicate for key '$KEY' in lines $DUPLICATE_NUMBERS" + done + done + echo "::endgroup::" + fi +done < <( find "$(git rev-parse --show-toplevel)" -wholename "*/src/*.properties" -print0 ) + +if [ "$FAIL" -eq 1 ]; then + exit 1 +fi diff --git a/tests/verify_mdb_properties.sh b/tests/verify_mdb_properties.sh new file mode 100755 index 00000000000..bc62c0f503a --- /dev/null +++ b/tests/verify_mdb_properties.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# This script will check our metadata block files and scan if the properties files contain all the matching keys. + +set -euo pipefail + +if ! which jbang > /dev/null 2>&1; then + echo "Cannot find jbang on path. Did you install it?" >&2 + exit 1 +fi +if ! which native-image > /dev/null 2>&1; then + echo "Cannot find GraalVM native-image on path. Did you install it?" >&2 + exit 1 +fi + +FAIL=0 + +# We need a small Java app here, replacing spaces, converting to lower case but especially to replace UTF-8 chars with nearest ascii / strip accents because of +# https://github.com/IQSS/dataverse/blob/dddcf29188a5c35174f3c94ffc1c4cb1d7fc0552/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java#L139-L140 +# This cannot be replaced by another tool, as it behaves rather individually. +DIR=$(mktemp -d) +SOURCE="$DIR/stripaccents.java" +STRIP_BIN="$(dirname "$0")/stripaccents" +cat > "$SOURCE" << EOF +///usr/bin/env jbang "\$0" "\$@" ; exit \$? +//JAVA 11+ +//DEPS org.apache.commons:commons-lang3:3.12.0 +import org.apache.commons.lang3.StringUtils; +import java.nio.charset.StandardCharsets; +import java.io.IOException; +class stripaccents { + public static void main(String[] args) throws IOException { + String input = new String(System.in.readAllBytes(), StandardCharsets.UTF_8).toLowerCase().replace(" ", "_"); + System.out.println(StringUtils.stripAccents(input)); + } +} +EOF +jbang export native --force --fresh -O "$STRIP_BIN" "$SOURCE" + +while IFS= read -r -d '' MDB; do + + echo "::group::$MDB" + BLOCK_NAME=$(sed -n "2p" "$MDB" | cut -f2) + BLOCK_DISPLAY_NAME=$(sed -n "2p" "$MDB" | cut -f4) + PROPERTIES_FILE="$(git rev-parse --show-toplevel)/src/main/java/propertyFiles/$BLOCK_NAME.properties" + + # Check correct file exists + if [ ! -r "$PROPERTIES_FILE" ]; then + echo "::error::Missing properties file for metadata block '$BLOCK_NAME', expected at '$PROPERTIES_FILE'" + FAIL=1 + continue + fi + + # Check metadata block properties exist and are equal to TSV source + if ! grep -a -q -e "^metadatablock.name=$BLOCK_NAME$" "$PROPERTIES_FILE"; then + echo "::error::Missing 'metadatablock.name=$BLOCK_NAME' or different from TSV source in $PROPERTIES_FILE" + FAIL=1 + fi + if ! grep -a -q -e "^metadatablock.displayName=$BLOCK_DISPLAY_NAME$" "$PROPERTIES_FILE"; then + echo "::error::Missing 'metadatablock.displayName=$BLOCK_DISPLAY_NAME' or different from TSV source in $PROPERTIES_FILE" + FAIL=1 + fi + if ! grep -a -q -e "^metadatablock.displayFacet=" "$PROPERTIES_FILE"; then + echo "::error::Missing 'metadatablock.displayFacet=...' in $PROPERTIES_FILE" + FAIL=1 + fi + + # Check dataset fields + for FIELD in $(grep -a -A1000 "^#datasetField" "$MDB" | tail -n+2 | grep -a -B1000 "^#controlledVocabulary" | head -n-1 | cut -f2); do + for ENTRY in title description watermark; do + if ! grep -a -q -e "^datasetfieldtype.$FIELD.$ENTRY=" "$PROPERTIES_FILE"; then + echo "::error::Missing key 'datasetfieldtype.$FIELD.$ENTRY=...' in $PROPERTIES_FILE" + FAIL=1 + fi + done + done + + # Check CV entries + while read -r LINE; do + FIELD_NAME=$(echo "$LINE" | cut -f1) + # See https://github.com/IQSS/dataverse/blob/dddcf29188a5c35174f3c94ffc1c4cb1d7fc0552/src/main/java/edu/harvard/iq/dataverse/ControlledVocabularyValue.java#L139-L140 + # Square brackets are special in grep with expressions activated, so escape them if present! + FIELD_VALUE=$(echo "$LINE" | cut -f2 | "$STRIP_BIN" | sed -e 's/\([][]\)/\\\1/g' ) + + if ! grep -q -a -e "^controlledvocabulary.$FIELD_NAME.$FIELD_VALUE=" "$PROPERTIES_FILE"; then + echo "::error::Missing key 'controlledvocabulary.$FIELD_NAME.$FIELD_VALUE=...' in $PROPERTIES_FILE" + FAIL=1 + fi + done < <(grep -a -A1000 "^#controlledVocabulary" "$MDB" | tail -n+2) + + echo "::endgroup::" + +done < <( find "$(git rev-parse --show-toplevel)/scripts/api/data/metadatablocks" -name '*.tsv' -print0 ) + +rm "$SOURCE" "$STRIP_BIN" + +if [ "$FAIL" -eq 1 ]; then + exit 1 +fi