diff --git a/CHANGELOG.md b/CHANGELOG.md index c354f66e..057420b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +## [1.2.3] - 2023-04-13 +### Added +- Add CITATION.cff +- Allow cli arguments for start script. +- Enable configuration to organize internal storage of metadata documents (https://github.com/kit-data-manager/metastore2/issues/241) + +### Changed +- Update several badges +- Bump com.networknt:json-schema-validator from 1.0.78 to 1.0.79 +- Bump edu.kit.datamanager:repo-core from 1.1.1 to 1.1.2 +- Bump edu.kit.datamanager:service-base from 1.1.0 to 1.1.1 +- Bump gradle from 7.6 to 7.6.1. +- Bump io.freefair.maven-publish-java from 6.6.3 to 8.0.1 +- Bump io.freefair.lombok from 6.6.3 to 8.0.1 +- Bump javersVersion from 6.11.0 to 6.14.0 +- Bump org.mockito:mockito-core from 5.1.1 to 5.3.0 +- Bump org.owasp.dependencycheck from 8.1.0 to 8.2.1 +- Bump org.postgresql:postgresql from 42.5.4 to 42.6.0 +- Bump org.springframework.boot from 2.7.9 to 2.7.10 +- Bump org.springframework.cloud:spring-cloud-starter-config from 3.1.5 to 3.1.6. +- Bump org.springframework.data:spring-data-elasticsearch from 4.4.8 to 4.4.10. +- Bump org.springframework:spring-messaging from 5.3.25 to 5.3.26 +- Bump springDocVersion from 1.6.14 to 1.7.0 + +### Fixed +- Providing (invalid) version number while updating schema document may break old versions. (https://github.com/kit-data-manager/metastore2/issues/245) +- Trace log may slow down service. (https://github.com/kit-data-manager/metastore2/issues/233) +- Calling REST-Endpoint for UI fails if no page information is provided. (https://github.com/kit-data-manager/metastore2/issues/264) +- Problem running MetaStore standalone in a docker container. (https://github.com/kit-data-manager/metastore2/issues/270) + ## [1.2.2] - 2023-02-28 ### Fixed - Add configuration to use the service behind a proxy. (https://github.com/kit-data-manager/metastore2/issues/218) @@ -280,7 +310,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Registry for XSD files and support for XML metadata -[Unreleased]: https://github.com/kit-data-manager/metastore2/compare/v1.2.2...HEAD +[Unreleased]: https://github.com/kit-data-manager/metastore2/compare/v1.2.3...HEAD +[1.2.3]: https://github.com/kit-data-manager/metastore2/compare/v1.2.2...v1.2.3 [1.2.2]: https://github.com/kit-data-manager/metastore2/compare/v1.2.1...v1.2.2 [1.2.1]: https://github.com/kit-data-manager/metastore2/compare/v1.2.0...v1.2.1 [1.2.0]: https://github.com/kit-data-manager/metastore2/compare/v1.1.0...v1.2.0 diff --git a/build.gradle b/build.gradle index 7a58cb7b..2311d8c8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,9 +1,9 @@ plugins { - id 'org.springframework.boot' version '2.7.9' + id 'org.springframework.boot' version '2.7.10' id 'io.spring.dependency-management' version '1.1.0' - id 'io.freefair.lombok' version '6.6.3' - id 'io.freefair.maven-publish-java' version '6.6.3' - id 'org.owasp.dependencycheck' version '8.1.0' + id 'io.freefair.lombok' version '8.0.1' + id 'io.freefair.maven-publish-java' version '8.0.1' + id 'org.owasp.dependencycheck' version '8.2.1' id 'org.asciidoctor.jvm.convert' version '3.3.2' id 'net.researchgate.release' version '3.0.2' id 'com.gorylenko.gradle-git-properties' version '2.4.1' @@ -16,8 +16,8 @@ group = 'edu.kit.datamanager' ext { // versions of dependencies - springDocVersion = '1.6.14' - javersVersion = '6.11.0' + springDocVersion = '1.7.0' + javersVersion = '6.14.0' keycloakVersion = '19.0.0' // directory for generated code snippets during tests @@ -46,7 +46,7 @@ if (System.getProperty('profile') == 'minimal') { dependencies { // Spring - implementation 'org.springframework:spring-messaging:5.3.25' + implementation 'org.springframework:spring-messaging:5.3.26' implementation 'org.springframework.cloud:spring-cloud-gateway-mvc:3.1.6' // Spring Boot @@ -66,7 +66,7 @@ dependencies { // cloud support - implementation "org.springframework.cloud:spring-cloud-starter-config:3.1.5" + implementation "org.springframework.cloud:spring-cloud-starter-config:3.1.6" implementation "org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:3.1.5" // springdoc @@ -98,7 +98,7 @@ dependencies { implementation "org.javers:javers-core:${javersVersion}" // driver for postgres - implementation "org.postgresql:postgresql:42.5.4" + implementation "org.postgresql:postgresql:42.6.0" //driver for h2 implementation "com.h2database:h2:2.1.214" @@ -107,17 +107,17 @@ dependencies { implementation "org.apache.tika:tika-core:2.7.0" // JSON validator - implementation "com.networknt:json-schema-validator:1.0.77" + implementation "com.networknt:json-schema-validator:1.0.79" // XML validator // https://mvnrepository.com/artifact/xerces/xercesImpl implementation 'xerces:xercesImpl:2.12.2' // datamanager - implementation "edu.kit.datamanager:repo-core:1.1.1" - implementation "edu.kit.datamanager:service-base:1.1.0" + implementation "edu.kit.datamanager:repo-core:1.1.2" + implementation "edu.kit.datamanager:service-base:1.1.1" // elasticsearch (since service-base 1.1.0) - implementation "org.springframework.data:spring-data-elasticsearch:4.4.8" + implementation "org.springframework.data:spring-data-elasticsearch:4.4.10" // DOIP SDK implementation "net.dona.doip:doip-sdk:2.1.0" @@ -131,8 +131,9 @@ dependencies { testImplementation "org.springframework.security:spring-security-test" //Java 11 Support - testImplementation "org.mockito:mockito-core:5.1.1" + testImplementation "org.mockito:mockito-core:5.3.0" testImplementation "junit:junit:4.13.2" + testImplementation "com.github.stefanbirkner:system-lambda:1.2.1" } compileJava { diff --git a/build.sh b/build.sh index e4062030..d472a50e 100644 --- a/build.sh +++ b/build.sh @@ -87,7 +87,7 @@ testForCommands="chmod cp dirname find java javac mkdir git" for command in $testForCommands do - if ! type $command >> /dev/null; then + if ! type "$command" >> /dev/null; then echo "Error: command '$command' is not installed!" exit 1 fi @@ -178,7 +178,7 @@ jarFile=($(ls $REPO_NAME*.jar)) echo "################################################################################" echo "# Start micro service" echo "################################################################################" - echo "java -cp \".:\$jarFile\" -Dloader.path=\"file://\$ACTUAL_DIR/\$jarFile,./lib/,.\" -jar \$jarFile" + echo "java -cp \".:\$jarFile\" -Dloader.path=\"file://\$ACTUAL_DIR/\$jarFile,./lib/,.\" -jar \$jarFile \$*" } > run.sh # make script executable chmod 755 run.sh diff --git a/docker-compose.yml b/docker-compose.yml index af0060cd..ecc5cfcd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,10 +63,12 @@ services: networks: - dps metastore: - image: kitdm/metastore2:v1.2.2 + image: kitdm/metastore2:v1.2.3 container_name: metastore.docker environment: - HOSTNAMES=metastore.docker + - REPO_SEARCH_ENABLED=true + - REPO_MESSAGING_ENABLED=true depends_on: rabbitmq: condition: service_started diff --git a/docker/buildDocker.sh b/docker/buildDocker.sh index df823738..39c6e8c8 100644 --- a/docker/buildDocker.sh +++ b/docker/buildDocker.sh @@ -41,7 +41,7 @@ testForCommands="dirname date docker git" for command in $testForCommands do - if ! type $command >> /dev/null; then + if ! type "$command" >> /dev/null; then echo "Error: command '$command' is not installed!" exit 1 fi @@ -55,7 +55,7 @@ ACTUAL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ################################################################################ # Determine repo name ################################################################################ -REPO_NAME=$($ACTUAL_DIR/../gradlew -q printProjectName) +REPO_NAME=$("$ACTUAL_DIR"/../gradlew -q printProjectName) # Use only last line REPO_NAME=${REPO_NAME##*$'\n'} @@ -80,14 +80,14 @@ else TAG_NAME=$LAST_TAG-$(date -u +%Y-%m-%d) fi -cd $ACTUAL_DIR/.. || { echo "Failure changing to directory $ACTUAL_DIR/.."; exit 1; } +cd "$ACTUAL_DIR"/.. || { echo "Failure changing to directory $ACTUAL_DIR/.."; exit 1; } ################################################################################ # Build local docker ################################################################################ -printInfo Build docker container kitdm/$REPO_NAME:$TAG_NAME +printInfo Build docker container kitdm/"$REPO_NAME":"$TAG_NAME" -if ! docker build -t kitdm/$REPO_NAME:$TAG_NAME .; then +if ! docker build -t kitdm/"$REPO_NAME":"$TAG_NAME" .; then echo . printInfo "ERROR while building docker container!" usage diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 249e5832..943f0cbf 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 070cb702..50832291 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip +networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index a69d9cb6..65dcd68d 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,10 +80,10 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' @@ -143,12 +143,16 @@ fi if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac diff --git a/gradlew.bat b/gradlew.bat index 53a6b238..6689b85b 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -26,6 +26,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% diff --git a/metaStoreFramework.sh b/metaStoreFramework.sh deleted file mode 100644 index cf1ef4f6..00000000 --- a/metaStoreFramework.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/bin/bash -################################################################################ -# Management for metastore framework -# - Managing the following instances -# - elasticsearch -# - rabbitMQ -# - indexing-service -# - metastore2 -# Usage: -# bash metastoreFramework.sh [init|start|stop] -################################################################################ - -################################################################################ -# Define default values for variables -################################################################################ -# no defaults yet! - -################################################################################ -# START DECLARATION FUNCTIONS -################################################################################ - -################################################################################ -function usage { -################################################################################ - echo "Script for managing metastore service." - echo "USAGE:" - echo " $0 [init|start|stop]" - echo " " - echo " init - Initialize/Reset the whole framework" - echo " start - Start stopped framework" - echo " stop - Stop framework" - exit 1 -} - -################################################################################ -function checkParameters { -################################################################################ - # Check no of parameters. - if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters!" - usage - fi -} - -################################################################################ -function initFramework { -################################################################################ -printInfo "Setup Framework" - -echo "Setup configuration directories for metaStore and indexing-Service" -mkdir -p "$ACTUAL_DIR/settings/metastore" -mkdir -p "$ACTUAL_DIR/settings/indexing" - -echo "Setup network for docker..." -docker network create network4datamanager - -echo "Start RabbitMQ server..." -deleteDockerContainer rabbitmq4docker -docker run -d --hostname rabbitmq --net network4datamanager --name rabbitmq4docker -p 5672:5672 -p 15672:15672 rabbitmq:3-management - -echo "Start metaStore2..." -deleteDockerContainer metastore4docker -docker run -d -v "$ACTUAL_DIR/settings/metastore":/spring/metastore2/config --net network4datamanager --name metastore4docker -p8040:8040 kitdm/metastore2:latest - -echo "Start elasticsearch server..." -deleteDockerContainer elasticsearch4metastore -docker run -d --net network4datamanager --name elasticsearch4metastore -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.9.3 - -echo "Start Indexing-Service..." -deleteDockerContainer indexing4metastore -docker run -d -v "$ACTUAL_DIR/settings/metastore":/spring/indexing-service/config --net network4datamanager --name indexing4metastore -p 8050:8050 indexing-service:latest - -printInfo "Ready to use metastore" -} - -################################################################################ -function startFramework { -################################################################################ -printInfo "(Re)start metastore and all linked services..." - -echo "Start RabbitMQ server..." -docker start rabbitmq4docker - -echo "Start metastore2..." -docker start metastore4docker - -echo "Start elasticsearch server..." -docker start elasticsearch4metastore - -echo "Start Indexing-Service..." -docker start indexing4docker - -printInfo "Framework started!" -} - -################################################################################ -function stopFramework { -################################################################################ -printInfo "Shutdown metastore and all linked services..." - -echo "Stop Indexing-Service..." -docker stop indexing4docker - -echo "Stop elasticsearch server..." -docker stop elasticsearch4metastore - -echo "Stop metastore2..." -docker stop metastore4docker - -echo "Stop RabbitMQ server..." -docker stop rabbitmq4docker - -printInfo "Framework stopped!" -} - -################################################################################ -function deleteDockerContainer { -################################################################################ -printInfo "Delete docker image '$1'" - -if docker ps | grep -q "$1"; then - echo "Docker container '$1' still running -> Stop docker container" - docker stop $1 -fi - -if docker ps -a | grep -q "$1"; then - echo "Docker container '$1' exists -> Remove docker container" - docker rm $1 -fi -} - -################################################################################ -function printInfo { -################################################################################ -echo "---------------------------------------------------------------------------" -echo "$*" -echo "---------------------------------------------------------------------------" -} - -################################################################################ -# END DECLARATION FUNCTIONS / START OF SCRIPT -################################################################################ - -################################################################################ -# Test for commands used in this script -################################################################################ -testForCommands="type echo grep mkdir docker" - -for command in $testForCommands -do - if ! type $command >> /dev/null; then - echo "Error: command '$command' is not installed!" - exit 1 - fi -done - -################################################################################ -# Determine directory of script. -################################################################################ -ACTUAL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -################################################################################ -# Check parameters -################################################################################ -checkParameters "$*" - -################################################################################ -# Manage framework -################################################################################ - -case "$1" in - init) initFramework - ;; - start) startFramework - ;; - stop) stopFramework - ;; - *) usage - ;; -esac diff --git a/settings/application-default.properties b/settings/application-default.properties index f8f1ffc9..bebc5894 100644 --- a/settings/application-default.properties +++ b/settings/application-default.properties @@ -16,6 +16,26 @@ server.port: 8040 metastore.schema.schemaFolder:file://INSTALLATION_DIR/schema metastore.metadata.metadataFolder:file://INSTALLATION_DIR/metadata +############################################################################### +# Setup folder management for metadata documents +# Note: Creating more than 50.000 directories inside one directory might cause +# performance issues. +# Possible values: +# - simple (No hierarchie at all) +# - idBased (Split ID in small parts and create directory structure out of it) +# - dateBased (Create directory structure by date) (default) +############################################################################### +metastore.metadata.storagepattern:dateBased + +# Configuration for 'idBased' e.g. 4ca2cc62-df0b-49b1-b622-ce855554adfc -> 4ca2/cc62/df0b/49b1/b622/ce85/5554/4ca2cc62df0b49b1b622ce855554adfc +# default: charPerDirectory: 4, maxDepth: 7 +repo.plugin.storage.id.charPerDirectory:4 +repo.plugin.storage.id.maxDepth:7 + +# Configuration for 'dateBased' e.g.: @{year}/@{year}_@{month}_@{day}_@{hour}_@{minute} +# default: @{year}/@{month} +repo.plugin.storage.date.pathPattern:@{year}/@{month} + ############################################################################### # Setup schema registries. (Optional, no longer necessary) ############################################################################### @@ -71,7 +91,7 @@ repo.messaging.receiver.exchange: metastore_events repo.messaging.receiver.queue: metastoreEventQueue repo.messaging.receiver.routingKeys: metadata.# -################################################################################ +############################################################################### # Search - Elasticsearch # It's recommended to install elasticsearch behind a firewall with no direct # access from clients. @@ -79,7 +99,7 @@ repo.messaging.receiver.routingKeys: metadata.# repo.search.enabled = false repo.search.url = http://localhost:9200 -############################################################################## +############################################################################### # Database ############################################################################### spring.datasource.driver-class-name: org.h2.Driver diff --git a/settings/application-docker.properties b/settings/application-docker.properties index f7f7a999..ae403fc6 100644 --- a/settings/application-docker.properties +++ b/settings/application-docker.properties @@ -5,12 +5,37 @@ ############################################################################### server.port: 8040 +############################################################################### +# Proxy (enable this line if you want to use the service behind a proxy.) +############################################################################### +#server.forward-headers-strategy=framework + ############################################################################### # Setup paths for schema and metadata ############################################################################### metastore.schema.schemaFolder:file://INSTALLATION_DIR/schema metastore.metadata.metadataFolder:file://INSTALLATION_DIR/metadata +############################################################################### +# Setup folder management for metadata documents +# Note: Creating more than 50.000 directories inside one directory might cause +# performance issues. +# Possible values: +# - simple (No hierarchie at all) +# - idBased (Split ID in small parts and create directory structure out of it) +# - dateBased (Create directory structure by date) (default) +############################################################################### +metastore.metadata.storagepattern:dateBased + +# Configuration for 'idBased' e.g. 4ca2cc62-df0b-49b1-b622-ce855554adfc -> 4ca2/cc62/df0b/49b1/b622/ce85/5554/4ca2cc62df0b49b1b622ce855554adfc +# default: charPerDirectory: 4, maxDepth: 7 +repo.plugin.storage.id.charPerDirectory:4 +repo.plugin.storage.id.maxDepth:7 + +# Configuration for 'dateBased' e.g.: @{year}/@{year}_@{month}_@{day}_@{hour}_@{minute} +# default: @{year}/@{month} +repo.plugin.storage.date.pathPattern:@{year}/@{month} + ############################################################################### # Setup schema registries. (Optional, no longer necessary) ############################################################################### @@ -57,7 +82,7 @@ repo.auth.jwtSecret:vkfvoswsohwrxgjaxipuiyyjgubggzdaqrcuupbugxtnalhiegkppdgjgwxs # Messaging - RabbitMQ ############################################################################### repo.schedule.rate:1000 -repo.messaging.enabled: true +repo.messaging.enabled: false repo.messaging.hostname:rabbitmq.docker repo.messaging.port:5672 repo.messaging.sender.exchange: metastore_events @@ -65,7 +90,7 @@ repo.messaging.receiver.exchange: metastore_events repo.messaging.receiver.queue: metastoreEventQueue repo.messaging.receiver.routingKeys: metadata.# -################################################################################ +############################################################################### # Search - Elasticsearch # It's recommended to install elasticsearch behind a firewall with no direct # access from clients. diff --git a/settings/application-postgres.properties b/settings/application-postgres.properties index 9d7409ca..988fd1d3 100644 --- a/settings/application-postgres.properties +++ b/settings/application-postgres.properties @@ -16,6 +16,26 @@ server.port: 8040 metastore.schema.schemaFolder:file://INSTALLATION_DIR/schema metastore.metadata.metadataFolder:file://INSTALLATION_DIR/metadata +############################################################################### +# Setup folder management for metadata documents +# Note: Creating more than 50.000 directories inside one directory might cause +# performance issues. +# Possible values: +# - simple (No hierarchie at all) +# - idBased (Split ID in small parts and create directory structure out of it) +# - dateBased (Create directory structure by date) (default) +############################################################################### +metastore.metadata.storagepattern:dateBased + +# Configuration for 'idBased' e.g. 4ca2cc62-df0b-49b1-b622-ce855554adfc -> 4ca2/cc62/df0b/49b1/b622/ce85/5554/4ca2cc62df0b49b1b622ce855554adfc +# default: charPerDirectory: 4, maxDepth: 7 +repo.plugin.storage.id.charPerDirectory:4 +repo.plugin.storage.id.maxDepth:7 + +# Configuration for 'dateBased' e.g.: @{year}/@{year}_@{month}_@{day}_@{hour}_@{minute} +# default: @{year}/@{month} +repo.plugin.storage.date.pathPattern:@{year}/@{month} + ############################################################################### # Setup schema registries. (Optional, no longer necessary) ############################################################################### @@ -70,7 +90,7 @@ repo.messaging.receiver.exchange: metastore_events repo.messaging.receiver.queue: metastoreEventQueue repo.messaging.receiver.routingKeys: metadata.# -################################################################################ +############################################################################### # Search - Elasticsearch # It's recommended to install elasticsearch behind a firewall with no direct # access from clients. diff --git a/setupMetastoreFrameworkWithDocker.md b/setupMetastoreFrameworkWithDocker.md index c39cb948..1757613d 100644 --- a/setupMetastoreFrameworkWithDocker.md +++ b/setupMetastoreFrameworkWithDocker.md @@ -1,101 +1,24 @@ -# How to Setup a complete Framework with Docker +# How to Setup a complete Framework with Docker Compose -## Step 1 - Create network for docker: -``` -docker network create network4datamanager -``` - -## Step 2 - Start RabbitMQ server: -``` -docker run -d --hostname rabbitmq --net network4datamanager --name rabbitmq4docker rabbitmq:3-management -``` - -## Step 3 - Start metastore2: -``` -docker run -d -p8040:8040 --net network4datamanager --name metastore4docker kitdm/metastore2:latest -``` - -## Step 4 - Start elasticsearch server: -``` -docker run -d --net network4datamanager --name elasticsearch4metastore -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.9.3 -``` - -## Step 5 - Start Indexing-Service: -``` -docker run -d --net network4datamanager --name indexing4metastore -p 8050:8050 indexing-service:latest -``` - -## Step 6 - Register schema (for metastore2) -``` -``` - -## Step 7 - Register mapping (for elasticsearch): -``` -``` - -## Step 8 - You're ready to ingest your metadata to metastore: -``` -``` - -## Step 9 - All metadata is now available via elasticsearch. +# Create Docker Framework + Build and start framework: ``` +docker compose up ``` # Stop Docker Framework - - -## Step 1 - Stop Indexing-Service: -``` -docker stop indexing4docker -``` - -## Step 2 - Stop elasticsearch server: -``` -docker stop elasticsearch4metastore + Stop framework: ``` - -## Step 3 - Stop metastore2: -``` -docker stop metastore4docker -``` - -## Step 4 - Stop RabbitMQ server: -``` -docker stop rabbitmq4docker +docker compose stop ``` - -# Start Docker Framework -## Step 1 - Start RabbitMQ server: +# Restart Docker Framework + Restart framework: ``` -docker start rabbitmq4docker +docker compose start ``` +# Reset Framework -## Step 2 - Start metastore2: -``` -docker start metastore4docker -``` -## Step 3 - Start elasticsearch server: ``` -docker start elasticsearch4metastore +docker compose down ``` -## Step 4 - Start Indexing-Service: -``` -docker start indexing4docker -``` +For more information type: 'docker compose --help' \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/metastore2/Application.java b/src/main/java/edu/kit/datamanager/metastore2/Application.java index 5ef3dfd9..2ccdcc1c 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/Application.java +++ b/src/main/java/edu/kit/datamanager/metastore2/Application.java @@ -208,13 +208,6 @@ public IContentInformationService schemaInformationService() { return new ContentInformationService(); } - @Bean - public IdBasedStorageService idBasedStorageService() { - IdBasedStorageService ibss = new IdBasedStorageService(); - ibss.configure(storageServiceProperties()); - return ibss; - } - @Bean @ConfigurationProperties("repo") public ApplicationProperties applicationProperties() { @@ -318,11 +311,16 @@ public MetastoreConfiguration metadataConfig() { break; } } - LOG.trace("Looking for storageServices...."); + LOG.trace("Looking for storageService '{}'....", this.applicationProperties.getStoragePattern()); for (IRepoStorageService storageService : this.storageServices) { LOG.trace(".... '{}'", storageService.getServiceName()); if ("simple".equals(storageService.getServiceName())) { - rbc.setStorageService(storageService); + rbc.setStorageService(storageService); // Should be used as default + } + if (this.applicationProperties.getStoragePattern().equals(storageService.getServiceName())) { + LOG.trace("Configure '{}' with '{}'", storageService.getServiceName(), storageServiceProperties()); + storageService.configure(storageServiceProperties()); + rbc.setStorageService(storageService); break; } } diff --git a/src/main/java/edu/kit/datamanager/metastore2/configuration/ApplicationProperties.java b/src/main/java/edu/kit/datamanager/metastore2/configuration/ApplicationProperties.java index 73047615..2b90f7fe 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/configuration/ApplicationProperties.java +++ b/src/main/java/edu/kit/datamanager/metastore2/configuration/ApplicationProperties.java @@ -51,6 +51,9 @@ public class ApplicationProperties extends GenericApplicationProperties{ @Value("${metastore.metadata.metadataFolder}") private URL metadataFolder; + @Value("${metastore.metadata.storagepattern:dateBased}") + private String storagePattern; + @Value("${metastore.metadata.schemaRegistries: }") private String[] schemaRegistries; diff --git a/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/MetastoreOAIPMHRepository.java b/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/MetastoreOAIPMHRepository.java index 79fdecd9..62459dc0 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/MetastoreOAIPMHRepository.java +++ b/src/main/java/edu/kit/datamanager/metastore2/oaipmh/service/MetastoreOAIPMHRepository.java @@ -64,6 +64,9 @@ import org.purl.dc.elements._1.ElementContainer; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Example; +import org.springframework.data.domain.ExampleMatcher; +import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; import org.springframework.web.servlet.support.ServletUriComponentsBuilder; @@ -181,13 +184,15 @@ public boolean isPrefixSupported(String prefix) { boolean exists = DC_SCHEMA.getMetadataPrefix().equals(prefix) || DATACITE_SCHEMA.getMetadataPrefix().equals(prefix); LOGGER.trace(prefix + ": " + exists); if (!exists) { - List findAll = metadataFormatDao.findAll(); - for (MetadataFormat item : findAll) { - LOGGER.trace("." + item.getMetadataPrefix()); - if (prefix.equalsIgnoreCase(item.getMetadataPrefix())) { - exists = true; - break; - } + MetadataFormat metadataFormat = new MetadataFormat(); + metadataFormat.setMetadataPrefix(prefix); + ExampleMatcher caseInsensitive = ExampleMatcher.matchingAll().withIgnoreCase(); + Example example = Example.of(metadataFormat, caseInsensitive); + Optional findOne = metadataFormatDao.findOne(example); + if (findOne.isPresent()) { + MetadataFormat item = findOne.get(); + LOGGER.trace("Found at least one item with prefix: " + item.getMetadataPrefix()); + exists = true; } } return exists; @@ -213,16 +218,21 @@ public void listMetadataFormats(OAIPMHBuilder builder) { //@TODO extend by other formats builder.addMetadataFormat(DC_SCHEMA); builder.addMetadataFormat(DATACITE_SCHEMA); - List allXmlSchemas = metadataFormatDao.findAll(); + long noOfEntries = metadataFormatDao.count(); + long entriesPerPage = 50; + long page = 0; // add also the schema registered in the schema registry - for (MetadataFormat metadataFormat : allXmlSchemas) { - MetadataFormatType item = new MetadataFormatType(); - item.setMetadataNamespace(metadataFormat.getMetadataNamespace()); - item.setMetadataPrefix(metadataFormat.getMetadataPrefix()); - item.setSchema(metadataFormat.getSchema()); - builder.addMetadataFormat(item); - } - + do { + List allXmlSchemas = metadataFormatDao.findAll(PageRequest.of((int) page, (int) entriesPerPage)).getContent(); + for (MetadataFormat metadataFormat : allXmlSchemas) { + MetadataFormatType item = new MetadataFormatType(); + item.setMetadataNamespace(metadataFormat.getMetadataNamespace()); + item.setMetadataPrefix(metadataFormat.getMetadataPrefix()); + item.setSchema(metadataFormat.getSchema()); + builder.addMetadataFormat(item); + } + page++; + } while (page * entriesPerPage < noOfEntries); } @Override @@ -344,7 +354,7 @@ private Document getMetadataDocument(DataRecord object, String schemaId) { try { URL url = new URL(object.getMetadataDocumentUri()); byte[] readFileToByteArray = FileUtils.readFileToByteArray(Paths.get(url.toURI()).toFile()); - try (InputStream inputStream = new ByteArrayInputStream(readFileToByteArray)) { + try ( InputStream inputStream = new ByteArrayInputStream(readFileToByteArray)) { IOUtils.copy(inputStream, bout); wasError = false; } @@ -482,23 +492,23 @@ private List getEntities(OAIPMHBuilder builder) { LOGGER.trace("findBySchemaIdAndLastUpdateBetween({},{},{}, Page({},{}))", findMetadataPrefix, from, until, page, maxElementsPerList); overallCount = dataRecordDao.countBySchemaIdInAndLastUpdateBetween(findMetadataPrefix, from, until); results = dataRecordDao.findBySchemaIdInAndLastUpdateBetween(findMetadataPrefix, from, until, PageRequest.of(page, maxElementsPerList)); - LOGGER.trace("Found '" + results.size() + "' elements of '" + dataRecordDao.findAll().size() + "' elements in total!"); + LOGGER.trace("Found '" + results.size() + "' elements of '" + dataRecordDao.count() + "' elements in total!"); } else { LOGGER.trace("findBySchemaIdAndLastUpdateBetween({},{},{}, Page({},{}))", prefix, from, until, page, maxElementsPerList); overallCount = dataRecordDao.countBySchemaIdAndLastUpdateBetween(prefix, from, until); results = dataRecordDao.findBySchemaIdAndLastUpdateBetween(prefix, from, until, PageRequest.of(page, maxElementsPerList)); - LOGGER.trace("Found '" + results.size() + "' elements of '" + dataRecordDao.findAll().size() + "' elements in total!"); + LOGGER.trace("Found '" + results.size() + "' elements of '" + dataRecordDao.count() + "' elements in total!"); } if (LOGGER.isTraceEnabled()) { - LOGGER.trace("List all items:"); - List findAll = dataRecordDao.findAll(); + LOGGER.trace("List top 100 of all items:"); + List findAll = dataRecordDao.findAll(PageRequest.of(0, 100)).getContent(); for (DataRecord item : findAll) { LOGGER.trace("-> " + item); } } LOGGER.trace("Setting next resumption token."); int cursor = currentCursor + results.size(); - + if (cursor == overallCount) { LOGGER.debug("New cursor {} exceeds element count {}, no more elements available. Setting resumption token to 'null'.", (currentCursor + maxElementsPerList), overallCount); //lsit complete, add no resumptiontoken diff --git a/src/main/java/edu/kit/datamanager/metastore2/runner/ElasticIndexerRunner.java b/src/main/java/edu/kit/datamanager/metastore2/runner/ElasticIndexerRunner.java index 4051aa52..a4550cf2 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/runner/ElasticIndexerRunner.java +++ b/src/main/java/edu/kit/datamanager/metastore2/runner/ElasticIndexerRunner.java @@ -27,17 +27,20 @@ import edu.kit.datamanager.metastore2.domain.SchemaRecord; import edu.kit.datamanager.metastore2.domain.Url2Path; import edu.kit.datamanager.metastore2.web.impl.MetadataControllerImpl; +import edu.kit.datamanager.metastore2.web.impl.SchemaRegistryControllerImpl; import edu.kit.datamanager.service.IMessagingService; import edu.kit.datamanager.service.impl.LogfileMessagingService; import edu.kit.datamanager.util.ControllerUtils; -import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; +import org.springframework.data.domain.PageRequest; import org.springframework.hateoas.server.mvc.WebMvcLinkBuilder; import org.springframework.stereotype.Component; @@ -52,7 +55,7 @@ public class ElasticIndexerRunner implements CommandLineRunner { @Parameter(names = {"--reindex"}, description = "Elasticsearch index should be build from existing documents.") boolean updateIndex; @Parameter(names = {"--indices", "-i"}, description = "Only for given indices (comma separated) or all indices if not present.") - List indices; + Set indices; @Parameter(names = {"--updateDate", "-u"}, description = "Starting reindexing only for documents updated at earliest on update date.") Date updateDate; @@ -83,7 +86,7 @@ public void run(String... args) throws Exception { updateDate = new Date(0); } if (indices == null) { - indices = new ArrayList<>(); + indices = new HashSet<>(); } } catch (Exception ex) { argueParser.usage(); @@ -91,8 +94,9 @@ public void run(String... args) throws Exception { } if (updateIndex) { LOG.info("Start ElasticIndexer Runner for indices '{}' and update date '{}'", indices, updateDate); + LOG.info("No of schemas: '{}'", schemaRecordDao.count()); // Try to determine URL of repository - List findAllSchemas = schemaRecordDao.findAll(); + List findAllSchemas = schemaRecordDao.findAll(PageRequest.of(0, 3)).getContent(); if (!findAllSchemas.isEmpty()) { // There is at least one schema. // Try to fetch baseURL from this @@ -103,14 +107,26 @@ public void run(String... args) throws Exception { if (indices.isEmpty()) { LOG.info("Reindex all indices!"); - for (SchemaRecord item : findAllSchemas) { - indices.add(item.getSchemaId()); - } + long noOfEntries = url2PathDao.count(); + long entriesPerPage = 50; + long page = 0; + // add also the schema registered in the schema registry + do { + List allSchemas = schemaRecordDao.findAll(PageRequest.of((int) page, (int) entriesPerPage)).getContent(); + LOG.trace("Add '{}' schemas of '{}'", allSchemas.size(), noOfEntries); + for (SchemaRecord item : allSchemas) { + indices.add(item.getSchemaId()); + } + page++; + } while (page * entriesPerPage < noOfEntries); } for (String index : indices) { LOG.info("Reindex '{}'", index); List findBySchemaId = dataRecordDao.findBySchemaIdAndLastUpdateAfter(index, updateDate.toInstant()); + List findSchemaBySchemaId = schemaRecordDao.findBySchemaIdOrderByVersionDesc(index); + LOG.trace("Search for documents for schema '{}' and update date '{}'", index, updateDate); + LOG.trace("No of documents: '{}'", findBySchemaId.size()); for (DataRecord item : findBySchemaId) { MetadataRecord result = toMetadataRecord(item, baseUrl); LOG.trace("Sending CREATE event."); @@ -119,13 +135,15 @@ public void run(String... args) throws Exception { } LOG.trace("Search for alternative schemaId (given as URL)"); DataRecord templateRecord = new DataRecord(); - for (SchemaRecord debug : schemaRecordDao.findBySchemaIdOrderByVersionDesc(index)) { + for (SchemaRecord debug : findSchemaBySchemaId) { templateRecord.setSchemaId(debug.getSchemaId()); templateRecord.setSchemaVersion(debug.getVersion()); List findByPath1 = url2PathDao.findByPath(debug.getSchemaDocumentUri()); for (Url2Path path : findByPath1) { LOG.trace("SchemaRecord: '{}'", debug); List findBySchemaUrl = dataRecordDao.findBySchemaIdAndLastUpdateAfter(path.getUrl(), updateDate.toInstant()); + LOG.trace("Search for documents for schema '{}' and update date '{}'", path.getUrl(), updateDate); + LOG.trace("No of documents: '{}'", findBySchemaUrl.size()); for (DataRecord item : findBySchemaUrl) { templateRecord.setMetadataId(item.getMetadataId()); templateRecord.setVersion(item.getVersion()); @@ -144,13 +162,34 @@ public void run(String... args) throws Exception { } } + /** + * Transform DataRecord to MetadataRecord. + * @param record DataRecord holding all information about metadata document. + * @param baseUrl Base URL for accessing service. + * @return MetadataRecord of metadata document. + */ private MetadataRecord toMetadataRecord(DataRecord record, String baseUrl) { String metadataIdWithVersion = baseUrl + WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(MetadataControllerImpl.class).getMetadataDocumentById(record.getMetadataId(), record.getVersion(), null, null)).toUri().toString(); MetadataRecord returnValue = new MetadataRecord(); - returnValue.setId(metadataIdWithVersion); + returnValue.setId(record.getMetadataId()); + returnValue.setSchemaVersion(record.getSchemaVersion()); + returnValue.setRecordVersion(record.getVersion()); returnValue.setMetadataDocumentUri(metadataIdWithVersion); - returnValue.setSchema(ResourceIdentifier.factoryUrlResourceIdentifier(record.getSchemaId())); - LOG.trace("MetadataRecord: '{}'", returnValue); + returnValue.setSchema(ResourceIdentifier.factoryUrlResourceIdentifier(toSchemaUrl(record, baseUrl))); + return returnValue; } + + /** + * Transform schemaID to URL if it is an internal + * + * @param dataRecord DataRecord holding schemaID and schema version. + * @param baseUrl Base URL for accessing service. + * @return URL to Schema as String. + */ + private String toSchemaUrl(DataRecord dataRecord, String baseUrl) { + String schemaUrl; + schemaUrl = baseUrl + WebMvcLinkBuilder.linkTo(WebMvcLinkBuilder.methodOn(SchemaRegistryControllerImpl.class).getSchemaDocumentById(dataRecord.getSchemaId(), dataRecord.getVersion(), null, null)).toUri(); + return schemaUrl; + } } diff --git a/src/main/java/edu/kit/datamanager/metastore2/util/MetadataSchemaRecordUtil.java b/src/main/java/edu/kit/datamanager/metastore2/util/MetadataSchemaRecordUtil.java index 6fc03543..a55d4459 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/util/MetadataSchemaRecordUtil.java +++ b/src/main/java/edu/kit/datamanager/metastore2/util/MetadataSchemaRecordUtil.java @@ -236,6 +236,7 @@ record = Json.mapper().readValue(recordDocument.getInputStream(), MetadataSchema ControllerUtils.checkEtag(eTag, dataResource); SchemaRecord schemaRecord = schemaRecordDao.findFirstBySchemaIdOrderByVersionDesc(dataResource.getId()); if (record != null) { + record.setSchemaVersion(schemaRecord.getVersion()); MetadataSchemaRecord existingRecord = migrateToMetadataSchemaRecord(applicationProperties, dataResource, false); existingRecord = mergeRecords(existingRecord, record); mergeSchemaRecord(schemaRecord, existingRecord); @@ -707,7 +708,10 @@ public static void cleanUp(SchemaRecord schemaRecord) { if (findByUrl.isEmpty()) { if (LOG.isTraceEnabled()) { LOG.trace("-----------------------------------------"); - url2PathDao.findAll().forEach((item) -> { + Page page = url2PathDao.findAll(PageRequest.of(0, 100)); + LOG.trace("List '{}' of '{}'", page.getSize(), page.getTotalElements()); + LOG.trace("-----------------------------------------"); + page.getContent().forEach((item) -> { LOG.trace("- {}", item); }); LOG.trace("-----------------------------------------"); @@ -1119,7 +1123,10 @@ public static ResourceIdentifier getSchemaIdentifier(MetastoreConfiguration appl if (LOG.isTraceEnabled()) { LOG.trace("Looking for path '{}'", metadataSchemaRecord.getSchemaDocumentUri()); LOG.trace("-----------------------------------------"); - url2PathDao.findAll().forEach((item) -> { + Page page = url2PathDao.findAll(PageRequest.of(0, 100)); + LOG.trace("List '{}' of '{}'", page.getSize(), page.getTotalElements()); + LOG.trace("-----------------------------------------"); + page.getContent().forEach((item) -> { LOG.trace("- {}", item); }); LOG.trace("-----------------------------------------"); diff --git a/src/main/java/edu/kit/datamanager/metastore2/web/impl/FrontendControllerImpl.java b/src/main/java/edu/kit/datamanager/metastore2/web/impl/FrontendControllerImpl.java index 1bdea6bc..4c7f83f2 100644 --- a/src/main/java/edu/kit/datamanager/metastore2/web/impl/FrontendControllerImpl.java +++ b/src/main/java/edu/kit/datamanager/metastore2/web/impl/FrontendControllerImpl.java @@ -56,14 +56,12 @@ public ResponseEntity findAllSchemasForTabulator( final HttpServletResponse hsr, final UriComponentsBuilder ucb) { - LOG.trace("Performing findAllSchemasForTabulator()."); - List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); - if (authorizationIdentities != null) { - LOG.trace("Creating (READ) permission specification. '{}'", authorizationIdentities); - } else { - LOG.trace("No permission information provided. Skip creating permission specification."); + LOG.trace("Performing findAllSchemasForTabulator( pgbl='{}').",pgbl); + Pageable pageable = PageRequest.of(0, 10, Sort.by("id").ascending()); + if (pgbl != null) { + pageable = PageRequest.of(pgbl.getPageNumber() < 1 ? 0 : pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); } - Pageable pageable = PageRequest.of(pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); + LOG.trace("Pageable: '{}'", pageable); ResponseEntity> responseEntity4schemaRecords = schemaControllerImpl.getRecords(null, null, null, null, pageable, wr, hsr, ucb); List schemaRecords = responseEntity4schemaRecords.getBody(); @@ -83,16 +81,14 @@ public ResponseEntity findAllMetadataForTabulator( final WebRequest wr, final HttpServletResponse hsr, final UriComponentsBuilder ucb) { - LOG.trace("Performing findAllMetadataForTabulator()."); - List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); - if (authorizationIdentities != null) { - LOG.trace("Creating (READ) permission specification. '{}'", authorizationIdentities); - } else { - LOG.trace("No permission information provided. Skip creating permission specification."); + LOG.trace("Performing findAllMetadataForTabulator( id='{}', pgbl='{}').", id, pgbl); + Pageable pageable = PageRequest.of(0, 10, Sort.by("id").ascending()); + if (pgbl != null) { + pageable = PageRequest.of(pgbl.getPageNumber() < 1 ? 0 : pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); } - Pageable pageable = PageRequest.of(pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); - List schemaIds = id == null ? null : Arrays.asList(id); - ResponseEntity< List> responseEntity4metadataRecords = metadtaControllerImpl.getRecords(null, null, schemaIds, null, null, pageable, wr, hsr, ucb); + LOG.trace("Pageable: '{}'", pageable); + List metadataDocumentId = id == null ? null : Arrays.asList(id); + ResponseEntity< List> responseEntity4metadataRecords = metadtaControllerImpl.getRecords(null, null, metadataDocumentId, null, null, pageable, wr, hsr, ucb); List metadataRecords = responseEntity4metadataRecords.getBody(); String pageSize = responseEntity4metadataRecords.getHeaders().getFirst("Content-Range"); @@ -105,16 +101,17 @@ public ResponseEntity findAllMetadataForTabulator( } @Override - public ResponseEntity getSchemaRecordsForUi(Pageable pgbl, WebRequest wr, HttpServletResponse hsr, UriComponentsBuilder ucb) { - LOG.trace("Performing getSchemaRecordsForUi()."); - List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); - if (authorizationIdentities != null) { - LOG.trace("Creating (READ) permission specification. '{}'", authorizationIdentities); - } else { - LOG.trace("No permission information provided. Skip creating permission specification."); + public ResponseEntity getSchemaRecordsForUi( + @Parameter(hidden = true) final Pageable pgbl, + final WebRequest wr, + final HttpServletResponse hsr, + final UriComponentsBuilder ucb) { + LOG.trace("Performing getSchemaRecordsForUi( pgbl='{}').", pgbl); + Pageable pageable = PageRequest.of(0, 10, Sort.by("id").ascending()); + if (pgbl != null) { + pageable = PageRequest.of(pgbl.getPageNumber() < 1 ? 0 : pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); } - - Pageable pageable = PageRequest.of(pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); + LOG.trace("Pageable: '{}'", pageable); ResponseEntity> responseEntity4schemaRecords = schemaControllerImpl.getRecords(null, null, null, null, pageable, wr, hsr, ucb); List schemaRecords = responseEntity4schemaRecords.getBody(); @@ -131,16 +128,19 @@ public ResponseEntity getSchemaRecordsForUi(Pageable } @Override - public ResponseEntity getMetadataRecordsForUi(@RequestParam(value = "id", required = false) String id, Pageable pgbl, WebRequest wr, HttpServletResponse hsr, UriComponentsBuilder ucb) { - LOG.trace("Performing getMetadataRecordsForUi()."); - List authorizationIdentities = AuthenticationHelper.getAuthorizationIdentities(); - if (authorizationIdentities != null) { - LOG.trace("Creating (READ) permission specification. '{}'", authorizationIdentities); - } else { - LOG.trace("No permission information provided. Skip creating permission specification."); + public ResponseEntity getMetadataRecordsForUi( + @RequestParam(value = "id", required = false) String id, + @Parameter(hidden = true) final Pageable pgbl, + final WebRequest wr, + final HttpServletResponse hsr, + final UriComponentsBuilder ucb) { + LOG.trace("Performing getMetadataRecordsForUi( id='{}', pgbl='{}').", id, pgbl); + Pageable pageable = PageRequest.of(0, 10, Sort.by("id").ascending()); + if (pgbl != null) { + pageable = PageRequest.of(pgbl.getPageNumber() < 1 ? 0 : pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); } + LOG.trace("Pageable: '{}'", pageable); - Pageable pageable = PageRequest.of(pgbl.getPageNumber() - 1, pgbl.getPageSize(), Sort.by("id").ascending()); List schemaIds = id == null ? null : Arrays.asList(id); ResponseEntity< List> responseEntity4metadataRecords = metadtaControllerImpl.getRecords(null, null, schemaIds, null, null, pageable, wr, hsr, ucb); List metadataRecords = responseEntity4metadataRecords.getBody(); diff --git a/src/main/resources/static/editor/lib/js/metadataeditor.js b/src/main/resources/static/editor/lib/js/metadataeditor.js index 8083c05e..abcac8fb 100644 --- a/src/main/resources/static/editor/lib/js/metadataeditor.js +++ b/src/main/resources/static/editor/lib/js/metadataeditor.js @@ -89,7 +89,6 @@ var buttons = { function generateIcon(fontAwesome, tooltip) { return "\n\"; } -; /** * Default table layout @@ -144,7 +143,7 @@ function modalTemplate(modalInput) { '' + ''; } -; + /** * describes the internal representation of the Editor in case a form will be generated. * @returns {editorDefinitionForm} diff --git a/src/test/java/edu/kit/datamanager/metastore2/runner/ElasticIndexerRunnerTest.java b/src/test/java/edu/kit/datamanager/metastore2/runner/ElasticIndexerRunnerTest.java new file mode 100644 index 00000000..df49ba63 --- /dev/null +++ b/src/test/java/edu/kit/datamanager/metastore2/runner/ElasticIndexerRunnerTest.java @@ -0,0 +1,350 @@ +/* + * Copyright 2023 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.metastore2.runner; + +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.kit.datamanager.entities.PERMISSION; +import edu.kit.datamanager.metastore2.configuration.MetastoreConfiguration; +import edu.kit.datamanager.metastore2.dao.IDataRecordDao; +import edu.kit.datamanager.metastore2.dao.ILinkedMetadataRecordDao; +import edu.kit.datamanager.metastore2.dao.ISchemaRecordDao; +import edu.kit.datamanager.metastore2.dao.IUrl2PathDao; +import edu.kit.datamanager.metastore2.domain.MetadataRecord; +import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord; +import edu.kit.datamanager.metastore2.domain.ResourceIdentifier; +import edu.kit.datamanager.repo.dao.IAllIdentifiersDao; +import edu.kit.datamanager.repo.dao.IContentInformationDao; +import edu.kit.datamanager.repo.dao.IDataResourceDao; +import edu.kit.datamanager.repo.domain.acl.AclEntry; +import edu.kit.datamanager.util.AuthenticationHelper; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.Comparator; +import java.util.Date; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Stream; +import org.javers.core.Javers; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.restdocs.JUnitRestDocumentation; +import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.documentationConfiguration; +import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener; +import org.springframework.security.web.FilterChainProxy; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.support.DirtiesContextTestExecutionListener; +import org.springframework.test.context.transaction.TransactionalTestExecutionListener; +import org.springframework.test.context.web.ServletTestExecutionListener; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; +import static com.github.stefanbirkner.systemlambda.SystemLambda.*; +/** + * + * @author hartmann-v + */ +@RunWith(SpringJUnit4ClassRunner.class) +// Or create a test version of Application.class that stubs out services used by the CommandLineRunner +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) //RANDOM_PORT) +@EntityScan("edu.kit.datamanager") +@EnableJpaRepositories("edu.kit.datamanager") +@ComponentScan({"edu.kit.datamanager"}) +@AutoConfigureMockMvc +@TestExecutionListeners(listeners = {ServletTestExecutionListener.class, + DependencyInjectionTestExecutionListener.class, + DirtiesContextTestExecutionListener.class, + TransactionalTestExecutionListener.class, + WithSecurityContextTestExecutionListener.class}) +@ActiveProfiles("test") +@TestPropertySource(properties = {"server.port=41417"}) +@TestPropertySource(properties = {"spring.datasource.url=jdbc:h2:mem:db_md_accesswithaai4json;DB_CLOSE_DELAY=-1;MODE=LEGACY;NON_KEYWORDS=VALUE"}) +@TestPropertySource(properties = {"metastore.schema.schemaFolder=file:///tmp/metastore2/elasticRunner/schema"}) +@TestPropertySource(properties = {"metastore.metadata.metadataFolder=file:///tmp/metastore2/elasticRunner/metadata"}) +@TestPropertySource(properties = {"metastore.metadata.schemaRegistries="}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +public class ElasticIndexerRunnerTest { + + private static Boolean alreadyInitialized = Boolean.FALSE; + + private MockMvc mockMvc; + + @Autowired + private ElasticIndexerRunner eir; + @Autowired + private WebApplicationContext context; + @Autowired + private FilterChainProxy springSecurityFilterChain; + @Autowired + Javers javers = null; + @Autowired + private ILinkedMetadataRecordDao metadataRecordDao; + @Autowired + private IDataResourceDao dataResourceDao; + @Autowired + private IDataRecordDao dataRecordDao; + @Autowired + private ISchemaRecordDao schemaRecordDao; + @Autowired + private IContentInformationDao contentInformationDao; + @Autowired + private IAllIdentifiersDao allIdentifiersDao; + @Autowired + private IUrl2PathDao url2PathDao; + @Autowired + private MetastoreConfiguration metadataConfig; + @Rule + public JUnitRestDocumentation restDocumentation = new JUnitRestDocumentation(); + + + private final static String TEMP_DIR_4_ALL = "/tmp/metastore2/elasticRunner/"; + private final static String TEMP_DIR_4_SCHEMAS = TEMP_DIR_4_ALL + "schema/"; + private final static String TEMP_DIR_4_METADATA = TEMP_DIR_4_ALL + "metadata/"; + private final static String JSON_SCHEMA = "{\n" + + " \"$schema\": \"http://json-schema.org/draft/2019-09/schema#\",\n" + + " \"$id\": \"http://www.example.org/schema/json\",\n" + + " \"type\": \"object\",\n" + + " \"title\": \"Json schema for tests\",\n" + + " \"default\": {},\n" + + " \"required\": [\n" + + " \"title\",\n" + + " \"date\"\n" + + " ],\n" + + " \"properties\": {\n" + + " \"title\": {\n" + + " \"$id\": \"#/properties/string\",\n" + + " \"type\": \"string\",\n" + + " \"title\": \"Title\",\n" + + " \"description\": \"Title of object.\"\n" + + " },\n" + + " \"date\": {\n" + + " \"$id\": \"#/properties/string\",\n" + + " \"type\": \"string\",\n" + + " \"format\": \"date\",\n" + + " \"title\": \"Date\",\n" + + " \"description\": \"Date of object\"\n" + + " }\n" + + " },\n" + + " \"additionalProperties\": false\n" + + "}"; + private final static String JSON_DOCUMENT = "{\n" + + " \"title\": \"Json schema for tests\",\n" + + " \"date\": \"2022-07-29\"\n" + + "}"; + + public ElasticIndexerRunnerTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + // setup mockMvc + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.context) + .addFilters(springSecurityFilterChain) + .apply(documentationConfiguration(this.restDocumentation).uris() + .withPort(41417)) + .build(); + eir.indices = null; + eir.updateDate = null; + eir.updateIndex = false; + if (!isInitialized()) { + System.out.println("------ElasticIndexerRunnerTest--------"); + System.out.println("------" + this.metadataConfig); + System.out.println("------------------------------------------------------"); + + contentInformationDao.deleteAll(); + dataResourceDao.deleteAll(); + metadataRecordDao.deleteAll(); + schemaRecordDao.deleteAll(); + dataRecordDao.deleteAll(); + allIdentifiersDao.deleteAll(); + url2PathDao.deleteAll(); + + try { + // Create schema only once. + try (Stream walk = Files.walk(Paths.get(URI.create("file://" + TEMP_DIR_4_SCHEMAS)))) { + walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + Paths.get(TEMP_DIR_4_SCHEMAS).toFile().mkdir(); + try (Stream walk = Files.walk(Paths.get(URI.create("file://" + TEMP_DIR_4_METADATA)))) { + walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + Paths.get(TEMP_DIR_4_METADATA).toFile().mkdir(); + } catch (IOException ex) { + ex.printStackTrace(); + } + + } + } + + @After + public void tearDown() { + } + + + @Test + public void testElasticRunnerWithWrongParameter() throws Exception { + int statusCode = catchSystemExit(() -> { + eir.run("hallo"); + }); + Assert.assertEquals(0, statusCode); + } + + @Test + public void testElasticRunnerWithDate() throws Exception { + Date now = new Date(); + Date oneHourBack = new Date(now.getTime() - 3600000); + String date = new SimpleDateFormat("yyyy-MM-dd").format(oneHourBack); + System.out.println(oneHourBack.toString()); + System.out.println(date); + eir.run("-u", date); + Assert.assertTrue(true); + } + + @Test + public void testElasticRunnerWithAnyIndices() throws Exception { + eir.run("-i", "any"); + Assert.assertTrue(true); + } + + @Test + public void testElasticRunnerWithWrongIndices() throws Exception { + eir.run("--reindex", "-i", "any"); + Assert.assertTrue(true); + } + + @Test + public void testElasticRunnerWithAllIndices() throws Exception { + ingestSchemaRecord("elastic"); + eir.run("--reindex"); + Assert.assertTrue(true); + } + + + @Test + public void testElasticRunnerWithAllIndicesAndMetadataDocument() throws Exception { + String schemaLocation = ingestSchemaRecord("elastic2"); + ingestMetadataRecord("elastic2"); + ingestMetadataRecord(schemaLocation, true); + eir.run("--reindex"); + Assert.assertTrue(true); + } + + private String ingestSchemaRecord(String schemaId) throws Exception { + MetadataSchemaRecord schemaRecord = new MetadataSchemaRecord(); + schemaRecord.setSchemaId(schemaId); + schemaRecord.setSchemaVersion(1l); + schemaRecord.setType(MetadataSchemaRecord.SCHEMA_TYPE.JSON); + Set aclEntries = new HashSet<>(); + aclEntries.add(new AclEntry(AuthenticationHelper.ANONYMOUS_USER_PRINCIPAL, PERMISSION.READ)); + schemaRecord.setAcl(aclEntries); + + ObjectMapper mapper = new ObjectMapper(); + + MockMultipartFile recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(schemaRecord).getBytes()); + MockMultipartFile schemaFile = new MockMultipartFile("schema", "schema.json", "application/json", JSON_SCHEMA.getBytes()); + + String schemaLocation = this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/schemas"). + file(recordFile). + file(schemaFile)). + andDo(print()). + andExpect(status().isCreated()). + andReturn().getResponse().getHeader("Location"); + return schemaLocation; + } + + /** + * Ingest metadata with 'otheruser' set permissions for admin, user and guest. + * + * @param schemaId + * @throws Exception + */ + private void ingestMetadataRecord(String schemaId) throws Exception { + ingestMetadataRecord(schemaId, false); + } + + /** + * Ingest metadata with 'otheruser' set permissions for admin, user and guest. + * + * @param schemaId + * @param isUrl + * @throws Exception + */ + private void ingestMetadataRecord(String schemaId, boolean isUrl) throws Exception { + MetadataRecord record = new MetadataRecord(); + if (isUrl) { + record.setSchema(ResourceIdentifier.factoryUrlResourceIdentifier(schemaId)); + } else { + record.setSchema(ResourceIdentifier.factoryInternalResourceIdentifier(schemaId)); + } + record.setRelatedResource(ResourceIdentifier.factoryInternalResourceIdentifier("resource of " + schemaId)); + ObjectMapper mapper = new ObjectMapper(); + + MockMultipartFile recordFile = new MockMultipartFile("record", "record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + MockMultipartFile schemaFile = new MockMultipartFile("document", "metadata.json", "application/json", JSON_DOCUMENT.getBytes()); + + this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/metadata"). + file(recordFile). + file(schemaFile)). + andDo(print()).andExpect(status().isCreated()). + andReturn(); + } + + public static synchronized boolean isInitialized() { + boolean returnValue = alreadyInitialized; + alreadyInitialized = Boolean.TRUE; + + return returnValue; + } + +} diff --git a/src/test/java/edu/kit/datamanager/metastore2/test/FrontendControllerTest.java b/src/test/java/edu/kit/datamanager/metastore2/test/FrontendControllerTest.java new file mode 100644 index 00000000..1ebb603d --- /dev/null +++ b/src/test/java/edu/kit/datamanager/metastore2/test/FrontendControllerTest.java @@ -0,0 +1,269 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package edu.kit.datamanager.metastore2.test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.kit.datamanager.entities.PERMISSION; +import edu.kit.datamanager.metastore2.configuration.MetastoreConfiguration; +import edu.kit.datamanager.metastore2.dao.IDataRecordDao; +import edu.kit.datamanager.metastore2.dao.ILinkedMetadataRecordDao; +import edu.kit.datamanager.metastore2.dao.ISchemaRecordDao; +import edu.kit.datamanager.metastore2.domain.MetadataRecord; +import edu.kit.datamanager.metastore2.domain.ResourceIdentifier; +import edu.kit.datamanager.repo.dao.IAllIdentifiersDao; +import edu.kit.datamanager.repo.dao.IContentInformationDao; +import edu.kit.datamanager.repo.dao.IDataResourceDao; +import edu.kit.datamanager.repo.domain.acl.AclEntry; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Stream; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.restdocs.JUnitRestDocumentation; +import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.documentationConfiguration; +import org.springframework.security.test.context.support.WithSecurityContextTestExecutionListener; +import org.springframework.security.web.FilterChainProxy; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.support.DirtiesContextTestExecutionListener; +import org.springframework.test.context.transaction.TransactionalTestExecutionListener; +import org.springframework.test.context.web.ServletTestExecutionListener; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.redirectedUrlPattern; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; +import org.hamcrest.core.IsNot; + +/** + * + * @author Torridity + */ +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) //RANDOM_PORT) +@EntityScan("edu.kit.datamanager") +@EnableJpaRepositories("edu.kit.datamanager") +@ComponentScan({"edu.kit.datamanager"}) +@AutoConfigureMockMvc +@TestExecutionListeners(listeners = {ServletTestExecutionListener.class, + DependencyInjectionTestExecutionListener.class, + DirtiesContextTestExecutionListener.class, + TransactionalTestExecutionListener.class, + WithSecurityContextTestExecutionListener.class}) +@ActiveProfiles("test") +@TestPropertySource(properties = {"server.port=41418"}) +@TestPropertySource(properties = {"spring.datasource.url=jdbc:h2:mem:db_frontend;DB_CLOSE_DELAY=-1;MODE=LEGACY;NON_KEYWORDS=VALUE"}) +@TestPropertySource(properties = {"spring.jpa.database-platform=org.hibernate.dialect.H2Dialect"}) +@TestPropertySource(properties = {"spring.jpa.defer-datasource-initialization=true"}) +@TestPropertySource(properties = {"metastore.schema.schemaFolder=file:///tmp/metastore2/frontend/schema"}) +@TestPropertySource(properties = {"metastore.metadata.metadataFolder=file:///tmp/metastore2/frontend/metadata"}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +public class FrontendControllerTest { + + private final static String TEMP_DIR_4_ALL = "/tmp/metastore2/frontend/"; + private final static String TEMP_DIR_4_SCHEMAS = TEMP_DIR_4_ALL + "schema/"; + private final static String TEMP_DIR_4_METADATA = TEMP_DIR_4_ALL + "metadata/"; + private static final String SCHEMA_ID = "my_dc"; + private static final String INVALID_SCHEMA = "invalid_dc"; + private static final String RELATED_RESOURCE_STRING = "anyResourceId"; + private static final ResourceIdentifier RELATED_RESOURCE = ResourceIdentifier.factoryInternalResourceIdentifier(RELATED_RESOURCE_STRING); + + private final static String DC_DOCUMENT = CreateSchemaUtil.KIT_DOCUMENT; + + private MockMvc mockMvc; + @Autowired + private WebApplicationContext context; + @Autowired + private FilterChainProxy springSecurityFilterChain; + @Autowired + private ILinkedMetadataRecordDao metadataRecordDao; + @Autowired + private IDataResourceDao dataResourceDao; + @Autowired + private IDataRecordDao dataRecordDao; + @Autowired + private ISchemaRecordDao schemaRecordDao; + @Autowired + private IContentInformationDao contentInformationDao; + @Autowired + private IAllIdentifiersDao allIdentifiersDao; + @Autowired + private MetastoreConfiguration metadataConfig; + @Autowired + private MetastoreConfiguration schemaConfig; + @Rule + public JUnitRestDocumentation restDocumentation = new JUnitRestDocumentation(); + + @Before + public void setUp() throws Exception { + System.out.println("------MetadataControllerTest--------------------------"); + System.out.println("------" + this.metadataConfig); + System.out.println("------------------------------------------------------"); + + contentInformationDao.deleteAll(); + dataResourceDao.deleteAll(); + metadataRecordDao.deleteAll(); + schemaRecordDao.deleteAll(); + dataRecordDao.deleteAll(); + allIdentifiersDao.deleteAll(); + + try { + // setup mockMvc + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.context) + .addFilters(springSecurityFilterChain) + .apply(documentationConfiguration(this.restDocumentation).uris() + .withPort(41418)) + .build(); + // Create schema only once. + try ( Stream walk = Files.walk(Paths.get(URI.create("file://" + TEMP_DIR_4_SCHEMAS)))) { + walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + Paths.get(TEMP_DIR_4_SCHEMAS).toFile().mkdir(); + Paths.get(TEMP_DIR_4_SCHEMAS + INVALID_SCHEMA).toFile().createNewFile(); + CreateSchemaUtil.ingestKitSchemaRecord(mockMvc, SCHEMA_ID, schemaConfig.getJwtSecret()); + try ( Stream walk = Files.walk(Paths.get(URI.create("file://" + TEMP_DIR_4_METADATA)))) { + walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + Paths.get(TEMP_DIR_4_METADATA).toFile().mkdir(); + } catch (IOException ex) { + ex.printStackTrace(); + } + } + + @Test + public void testGetMetadataRecords4Tabulator1() throws Exception { + MvcResult res = this.mockMvc.perform(get("/api/v1/ui/metadata/")). + andDo(print()). + andExpect(status().isOk()). + andExpect(MockMvcResultMatchers.jsonPath("$", IsNot.not(Matchers.hasKey("data")))). + andExpect(MockMvcResultMatchers.jsonPath("$", Matchers.hasKey("last_page"))). + andReturn(); + //String metadataRecordId = + createDCMetadataRecord(); + + res = this.mockMvc.perform(get("/api/v1/ui/metadata/?page=1")). + andDo(print()). + andExpect(status().isOk()). + andExpect(MockMvcResultMatchers.jsonPath("$.data", Matchers.hasSize(1))). + andExpect(MockMvcResultMatchers.jsonPath("$", Matchers.hasKey("last_page"))). + andReturn(); + } + + @Test + public void testGetMetadataRecords4Tabulator2() throws Exception { + MvcResult res = this.mockMvc.perform(get("/api/v1/ui/metadata/"). + accept("application/tabulator+json")). + andDo(print()). + andExpect(status().isOk()). + andExpect(MockMvcResultMatchers.jsonPath("$", IsNot.not(Matchers.hasKey("data")))). + andExpect(MockMvcResultMatchers.jsonPath("$", Matchers.hasKey("last_page"))). + andReturn(); + //String metadataRecordId = + createDCMetadataRecord(); + + res = this.mockMvc.perform(get("/api/v1/ui/metadata/?page=1"). + accept("application/tabulator+json")). + andDo(print()). + andExpect(status().isOk()). + andExpect(MockMvcResultMatchers.jsonPath("$.data", Matchers.hasSize(1))). + andExpect(MockMvcResultMatchers.jsonPath("$", Matchers.hasKey("last_page"))). + andReturn(); + } + + @Test + public void testGetSchemaRecords4Tabulator1() throws Exception { + MvcResult res = this.mockMvc.perform(get("/api/v1/ui/schemas/")). + andDo(print()). + andExpect(status().isOk()). + andExpect(MockMvcResultMatchers.jsonPath("$.data", Matchers.hasSize(1))). + andExpect(MockMvcResultMatchers.jsonPath("$", Matchers.hasKey("last_page"))). + andReturn(); + } + + @Test + public void testGetSchemaRecords4Tabulator2() throws Exception { + MvcResult res = this.mockMvc.perform(get("/api/v1/ui/schemas/"). + accept("application/tabulator+json")). + andDo(print()). + andExpect(status().isOk()). + andExpect(MockMvcResultMatchers.jsonPath("$.data", Matchers.hasSize(1))). + andExpect(MockMvcResultMatchers.jsonPath("$", Matchers.hasKey("last_page"))). + andReturn(); + } + + @Test + public void testGetSchemaRecords4Tabulator3() throws Exception { + MvcResult res = this.mockMvc.perform(get("/api/v1/ui/schemas/?page=1")). + andDo(print()). + andExpect(status().isOk()). + andExpect(MockMvcResultMatchers.jsonPath("$.data", Matchers.hasSize(1))). + andExpect(MockMvcResultMatchers.jsonPath("$", Matchers.hasKey("last_page"))). + andReturn(); + } + + @Test + public void testGetSchemaRecords4Tabulator4() throws Exception { + MvcResult res = this.mockMvc.perform(get("/api/v1/ui/schemas/?page=1"). + accept("application/tabulator+json")). + andDo(print()). + andExpect(status().isOk()). + andExpect(MockMvcResultMatchers.jsonPath("$.data", Matchers.hasSize(1))). + andExpect(MockMvcResultMatchers.jsonPath("$", Matchers.hasKey("last_page"))). + andReturn(); + } + + private String createDCMetadataRecord() throws Exception { + MetadataRecord record = new MetadataRecord(); +// record.setId("my_id"); + record.setSchema(ResourceIdentifier.factoryInternalResourceIdentifier(SCHEMA_ID)); + record.setRelatedResource(RELATED_RESOURCE); + Set aclEntries = new HashSet<>(); + aclEntries.add(new AclEntry("SELF", PERMISSION.READ)); + aclEntries.add(new AclEntry("test2", PERMISSION.ADMINISTRATE)); + record.setAcl(aclEntries); + ObjectMapper mapper = new ObjectMapper(); + + MockMultipartFile recordFile = new MockMultipartFile("record", "metadata-record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + MockMultipartFile metadataFile = new MockMultipartFile("document", "metadata.xml", "application/xml", DC_DOCUMENT.getBytes()); + + MvcResult andReturn = this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/metadata"). + file(recordFile). + file(metadataFile)).andDo(print()).andExpect(status().isCreated()).andExpect(redirectedUrlPattern("http://*:*/**/*?version=1")).andReturn(); + MetadataRecord result = mapper.readValue(andReturn.getResponse().getContentAsString(), MetadataRecord.class); + + return result.getId(); + } +} diff --git a/src/test/java/edu/kit/datamanager/metastore2/test/JsonSchemaRegistryControllerTest.java b/src/test/java/edu/kit/datamanager/metastore2/test/JsonSchemaRegistryControllerTest.java index ac63db87..f4dd9091 100644 --- a/src/test/java/edu/kit/datamanager/metastore2/test/JsonSchemaRegistryControllerTest.java +++ b/src/test/java/edu/kit/datamanager/metastore2/test/JsonSchemaRegistryControllerTest.java @@ -43,6 +43,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.domain.PageRequest; import org.springframework.http.MediaType; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockMultipartFile; @@ -635,7 +636,7 @@ public void testGetSchemaDocument() throws Exception { @Test public void testGetSchemaDocumentWithMissingSchemaFile() throws Exception { ingestSchemaRecord(); - String contentUri = contentInformationDao.findAll().get(0).getContentUri(); + String contentUri = contentInformationDao.findAll(PageRequest.of(0,2)).getContent().get(0).getContentUri(); //delete schema file URI uri = new URI(contentUri); @@ -696,7 +697,7 @@ public void testValidateWithoutValidator() throws Exception { public void testValidateWithMissingSchemaFile() throws Exception { ingestSchemaRecord(); // Get location of schema file. - String contentUri = contentInformationDao.findAll().get(0).getContentUri(); + String contentUri = contentInformationDao.findAll(PageRequest.of(0,2)).getContent().get(0).getContentUri(); //delete schema file URI uri = new URI(contentUri); Files.delete(Paths.get(uri)); diff --git a/src/test/java/edu/kit/datamanager/metastore2/test/MetadataControllerTest.java b/src/test/java/edu/kit/datamanager/metastore2/test/MetadataControllerTest.java index 9a4fcc54..2b9d33be 100644 --- a/src/test/java/edu/kit/datamanager/metastore2/test/MetadataControllerTest.java +++ b/src/test/java/edu/kit/datamanager/metastore2/test/MetadataControllerTest.java @@ -940,6 +940,54 @@ public void testUpdateRecord() throws Exception { Assert.assertEquals(record.getMetadataDocumentUri().replace("version=1", "version=2"), record2.getMetadataDocumentUri()); } + @Test + public void testUpdateRecordWithWrongVersion() throws Exception { + String metadataRecordId = createDCMetadataRecord(); + MvcResult result = this.mockMvc.perform(get("/api/v1/metadata/" + metadataRecordId).header("Accept", MetadataRecord.METADATA_RECORD_MEDIA_TYPE)).andDo(print()).andExpect(status().isOk()).andReturn(); + String etag = result.getResponse().getHeader("ETag"); + String body = result.getResponse().getContentAsString(); + + ObjectMapper mapper = new ObjectMapper(); + MetadataRecord record = mapper.readValue(body, MetadataRecord.class); + record.setRecordVersion(0l); + MockMultipartFile recordFile = new MockMultipartFile("record", "metadata-record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + MockMultipartFile metadataFile = new MockMultipartFile("document", "metadata.xml", "application/xml", DC_DOCUMENT_VERSION_2.getBytes()); + + result = this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/metadata/" + record.getId()). + file(recordFile). + file(metadataFile).header("If-Match", etag).with(putMultipart())).andDo(print()).andExpect(status().isOk()).andReturn(); + +// result = this.mockMvc.perform(put("/api/v1/metadata/dc").header("If-Match", etag).contentType(MetadataRecord.METADATA_RECORD_MEDIA_TYPE).content(mapper.writeValueAsString(record))).andDo(print()).andExpect(status().isOk()).andReturn(); + body = result.getResponse().getContentAsString(); + + MetadataRecord record2 = mapper.readValue(body, MetadataRecord.class); + Assert.assertNotEquals(record.getDocumentHash(), record2.getDocumentHash()); + Assert.assertEquals(record.getCreatedAt(), record2.getCreatedAt()); + Assert.assertEquals(record.getSchema().getIdentifier(), record2.getSchema().getIdentifier()); + Assert.assertEquals(Long.valueOf(2l), record2.getRecordVersion());// version should be 2 + if (record.getAcl() != null) { + Assert.assertTrue(record.getAcl().containsAll(record2.getAcl())); + } + Assert.assertTrue(record.getLastUpdate().isBefore(record2.getLastUpdate())); + // Check for new metadata document. + result = this.mockMvc.perform(get("/api/v1/metadata/" + metadataRecordId)).andDo(print()).andExpect(status().isOk()).andReturn(); + String content = result.getResponse().getContentAsString(); + + String dcMetadata = DC_DOCUMENT_VERSION_2; + + Assert.assertEquals(dcMetadata, content); + + Assert.assertEquals(record.getMetadataDocumentUri().replace("version=1", "version=2"), record2.getMetadataDocumentUri()); + // Check for old metadata document. + result = this.mockMvc.perform(get("/api/v1/metadata/" + metadataRecordId + "?version=1")).andDo(print()).andExpect(status().isOk()).andReturn(); + content = result.getResponse().getContentAsString(); + + dcMetadata = DC_DOCUMENT; + + Assert.assertEquals(dcMetadata, content); + + Assert.assertEquals(record.getMetadataDocumentUri().replace("version=1", "version=2"), record2.getMetadataDocumentUri()); + } @Test public void testUpdateRecordIgnoreACL() throws Exception { diff --git a/src/test/java/edu/kit/datamanager/metastore2/test/MetadataControllerTestAccessWithAuthenticationEnabled4Json.java b/src/test/java/edu/kit/datamanager/metastore2/test/MetadataControllerTestAccessWithAuthenticationEnabled4Json.java index 191ae6ab..07be484b 100644 --- a/src/test/java/edu/kit/datamanager/metastore2/test/MetadataControllerTestAccessWithAuthenticationEnabled4Json.java +++ b/src/test/java/edu/kit/datamanager/metastore2/test/MetadataControllerTestAccessWithAuthenticationEnabled4Json.java @@ -18,7 +18,6 @@ import edu.kit.datamanager.metastore2.domain.MetadataRecord; import edu.kit.datamanager.metastore2.domain.MetadataSchemaRecord; import edu.kit.datamanager.metastore2.domain.ResourceIdentifier; -import edu.kit.datamanager.metastore2.domain.SchemaRecord; import edu.kit.datamanager.repo.dao.IAllIdentifiersDao; import edu.kit.datamanager.repo.dao.IContentInformationDao; import edu.kit.datamanager.repo.dao.IDataResourceDao; diff --git a/src/test/java/edu/kit/datamanager/metastore2/test/SchemaRegistryControllerTest.java b/src/test/java/edu/kit/datamanager/metastore2/test/SchemaRegistryControllerTest.java index 21f74fdc..a338a188 100644 --- a/src/test/java/edu/kit/datamanager/metastore2/test/SchemaRegistryControllerTest.java +++ b/src/test/java/edu/kit/datamanager/metastore2/test/SchemaRegistryControllerTest.java @@ -53,6 +53,7 @@ import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; @@ -617,7 +618,7 @@ public void testGetSchemaDocument() throws Exception { @Test public void testGetSchemaDocumentWithMissingSchemaFile() throws Exception { ingestSchemaRecord(); - String contentUri = contentInformationDao.findAll().get(0).getContentUri(); + String contentUri = contentInformationDao.findAll(PageRequest.of(0,2)).getContent().get(0).getContentUri(); //delete schema file URI uri = new URI(contentUri); Files.delete(Paths.get(uri)); @@ -678,7 +679,7 @@ public void testValidateWithoutValidator() throws Exception { public void testValidateWithMissingSchemaFile() throws Exception { ingestSchemaRecord(); // Get location of schema file. - String contentUri = contentInformationDao.findAll().get(0).getContentUri(); + String contentUri = contentInformationDao.findAll(PageRequest.of(0,2)).getContent().get(0).getContentUri(); //delete schema file URI uri = new URI(contentUri); Files.delete(Paths.get(uri)); @@ -887,6 +888,49 @@ public void testUpdateRecordAndDocument() throws Exception { Assert.assertEquals(KIT_SCHEMA_V2, content); } + @Test + public void testUpdateRecordAndDocumentWithWrongVersion() throws Exception { + String schemaId = "updateRecordAndDocumentWithWrongVersion"; + ingestSchemaRecord(schemaId); + MvcResult result = this.mockMvc.perform(get("/api/v1/schemas/" + schemaId).header("Accept", MetadataSchemaRecord.METADATA_SCHEMA_RECORD_MEDIA_TYPE)).andDo(print()).andExpect(status().isOk()).andReturn(); + String etag = result.getResponse().getHeader("ETag"); + String body = result.getResponse().getContentAsString(); + + ObjectMapper mapper = new ObjectMapper(); + MetadataSchemaRecord record = mapper.readValue(body, MetadataSchemaRecord.class); + record.setSchemaVersion(0l); + String mimeTypeBefore = record.getMimeType(); + record.setMimeType(MediaType.APPLICATION_JSON.toString()); + MockMultipartFile recordFile = new MockMultipartFile("record", "metadata-record.json", "application/json", mapper.writeValueAsString(record).getBytes()); + MockMultipartFile schemaFile = new MockMultipartFile("schema", "schema.xsd", "application/xml", KIT_SCHEMA_V2.getBytes()); + + result = this.mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/schemas/" + schemaId). + file(recordFile).file(schemaFile).header("If-Match", etag).with(putMultipart())).andDo(print()).andExpect(status().isOk()).andExpect(redirectedUrlPattern("http://*:*/**/" + record.getSchemaId() + "?version=*")).andReturn(); + body = result.getResponse().getContentAsString(); + + MetadataSchemaRecord record2 = mapper.readValue(body, MetadataSchemaRecord.class); + Assert.assertNotEquals(mimeTypeBefore, record2.getMimeType());//mime type was changed by update + Assert.assertEquals(record.getCreatedAt(), record2.getCreatedAt()); + testForNextVersion(record.getSchemaDocumentUri(), record2.getSchemaDocumentUri()); +// Assert.assertEquals(record.getSchemaDocumentUri().replace("version=1", "version=2"), record2.getSchemaDocumentUri()); + Assert.assertNotEquals(record.getSchemaHash(), record2.getSchemaHash()); + Assert.assertEquals(record.getSchemaId(), record2.getSchemaId()); + Assert.assertEquals(2l, (long) record2.getSchemaVersion());//version is not changing for metadata update + if (record.getAcl() != null) { + Assert.assertTrue(record.getAcl().containsAll(record2.getAcl())); + } + Assert.assertTrue(record.getLastUpdate().isBefore(record2.getLastUpdate())); + // Test also document for update + result = this.mockMvc.perform(get("/api/v1/schemas/" + schemaId)).andDo(print()).andExpect(status().isOk()).andReturn(); + String content = result.getResponse().getContentAsString(); + + Assert.assertEquals(KIT_SCHEMA_V2, content); + // Test also old document + result = this.mockMvc.perform(get("/api/v1/schemas/" + schemaId+ "?version=1")).andDo(print()).andExpect(status().isOk()).andReturn(); + content = result.getResponse().getContentAsString(); + Assert.assertEquals(KIT_SCHEMA, content); + } + @Test public void testUpdateOnlyDocument() throws Exception { String schemaId = "updateRecordDocumentOnly";