Add option on desktop to remove the collections from solr #105 [skip ci]

keeps · Oct 14, 2019 · 61bdf79 · 61bdf79
1 parent 60e291b
commit 61bdf79
Show file tree

Hide file tree

Showing 25 changed files with 448 additions and 131 deletions.
diff --git a/pom.xml b/pom.xml
@@ -25,7 +25,7 @@
 
         <!-- Convenience properties to set versions -->
         <version.gwt>2.8.2</version.gwt>
-        <version.dbptk>2.6.0-SNAPSHOT</version.dbptk>
+        <version.dbptk>2.6.0-SNAPSHOT-2</version.dbptk>
         <version.roda>3.0.2</version.roda>
 
         <version.jersey>2.27</version.jersey>
@@ -103,23 +103,6 @@
                     </execution>
                 </executions>
             </plugin>
-            <!--<plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-source-plugin</artifactId>
-                <version>3.0.1</version>
-                <configuration>
-                    <excludeResources>true</excludeResources>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>attach-sources</id>
-                        <phase>install</phase>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>-->
             <plugin>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
@@ -160,11 +143,6 @@
                         <format>json</format>
                     </configuration>
                 </plugin>
-                <plugin>
-                    <groupId>com.spotify</groupId>
-                    <artifactId>docker-maven-plugin</artifactId>
-                    <version>1.1.1</version>
-                </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-clean-plugin</artifactId>
@@ -372,14 +350,14 @@
             <artifactId>gwt-user</artifactId>
         </dependency>
         <dependency>
-          <groupId>com.google.gwt</groupId>
-          <artifactId>gwt-dev</artifactId>
-          <exclusions>
-              <exclusion>
-                  <groupId>org.mortbay.jasper</groupId>
-                  <artifactId>apache-jsp</artifactId>
-              </exclusion>
-          </exclusions>
+            <groupId>com.google.gwt</groupId>
+            <artifactId>gwt-dev</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.mortbay.jasper</groupId>
+                    <artifactId>apache-jsp</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <!-- viewer -->
         <dependency>

diff --git a/src/main/java/com/databasepreservation/DBVTK.java b/src/main/java/com/databasepreservation/DBVTK.java
@@ -1,33 +1,38 @@
 package com.databasepreservation;
+
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+
 import javax.servlet.http.HttpServlet;
-import com.databasepreservation.common.shared.client.ClientLogger;
+
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.web.servlet.ServletRegistrationBean;
 import org.springframework.boot.web.servlet.context.ServletWebServerInitializedEvent;
 import org.springframework.context.ApplicationListener;
 import org.springframework.context.annotation.Bean;
-import com.databasepreservation.common.server.BrowserServiceImpl;
-import com.databasepreservation.common.server.ClientLoggerImpl;
-import com.databasepreservation.common.server.ViewerConfiguration;
-import com.databasepreservation.common.shared.ViewerConstants;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.Ordered;
 import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
 
+import com.databasepreservation.common.server.BrowserServiceImpl;
+import com.databasepreservation.common.server.ClientLoggerImpl;
+import com.databasepreservation.common.server.ViewerConfiguration;
+import com.databasepreservation.common.shared.ViewerConstants;
+
 @SpringBootApplication
 public class DBVTK {
   public static void main(String[] args) {
     ViewerConfiguration.getInstance();
     SpringApplication.run(DBVTK.class, args);
   }
+
   @Configuration
   public static class DefaultView implements WebMvcConfigurer {
+
     @Override
     public void addViewControllers(ViewControllerRegistry registry) {
       if (ViewerConstants.DESKTOP.equals(System.getProperty("env", "server"))) {
@@ -36,6 +41,7 @@ public void addViewControllers(ViewControllerRegistry registry) {
       }
     }
   }
+
   @Bean
   public ServletRegistrationBean<HttpServlet> browserService() {
     ServletRegistrationBean<HttpServlet> bean;

diff --git a/src/main/java/com/databasepreservation/common/client/BrowserService.java b/src/main/java/com/databasepreservation/common/client/BrowserService.java
@@ -96,6 +96,12 @@ void editSearch(String databaseUUID, String savedSearchUUID, String name, String
   void deleteSearch(String databaseUUID, String savedSearchUUID)
     throws AuthorizationDeniedException, GenericException, RequestNotValidException, NotFoundException;
 
+  Boolean deleteDatabaseCollection(String databaseUUID);
+
+  Boolean deleteRowsCollection(String databaseUUID);
+
+  Boolean deleteAllCollections(String databaseUUID);
+
   Boolean isAuthenticationEnabled() throws RODAException;
 
   User getAuthenticatedUser() throws RODAException;
@@ -139,10 +145,10 @@ boolean createSIARD(String UUID, ConnectionParameters connectionParameters,
     ExportOptionsParameters exportOptionsParameters, MetadataExportOptionsParameters metadataExportOptionsParameters)
     throws GenericException;
 
-  boolean migrateToDBMS(String databaseUUID, String siard, ConnectionParameters connectionParameters)
+  boolean migrateToDBMS(String databaseUUID, String siardVersion, String siardPath, ConnectionParameters connectionParameters)
     throws GenericException;
 
-  boolean migrateToSIARD(String databaseUUID, String siardPath, TableAndColumnsParameters tableAndColumnsParameters,
+  boolean migrateToSIARD(String databaseUUID, String siardVersion, String siardPath, TableAndColumnsParameters tableAndColumnsParameters,
     ExportOptionsParameters exportOptionsParameters, MetadataExportOptionsParameters metadataExportOptions)
     throws GenericException;
 

diff --git a/src/main/java/com/databasepreservation/common/client/BrowserServiceAsync.java b/src/main/java/com/databasepreservation/common/client/BrowserServiceAsync.java
@@ -86,6 +86,12 @@ void editSearch(String databaseUUID, String savedSearchUUID, String name, String
 
   void deleteSearch(String databaseUUID, String savedSearchUUID, AsyncCallback<Void> async);
 
+  void deleteDatabaseCollection(String databaseUUID, AsyncCallback<Boolean> async);
+
+  void deleteRowsCollection(String databaseUUID, AsyncCallback<Boolean> async);
+
+  void deleteAllCollections(String databaseUUId, AsyncCallback<Boolean> async);
+
   void isAuthenticationEnabled(AsyncCallback<Boolean> async);
 
   /**
@@ -142,10 +148,10 @@ void createSIARD(String UUID, ConnectionParameters connectionParameters,
     ExportOptionsParameters exportOptionsParameters, MetadataExportOptionsParameters metadataExportOptionsParameters,
     AsyncCallback<Boolean> async);
 
-  void migrateToDBMS(String databaseUUID, String siard, ConnectionParameters connectionParameters,
+  void migrateToDBMS(String databaseUUID, String siardVersion, String siardPath, ConnectionParameters connectionParameters,
     AsyncCallback<Boolean> async);
 
-  void migrateToSIARD(String databaseUUID, String siardPath, TableAndColumnsParameters tableAndColumnsParameters,
+  void migrateToSIARD(String databaseUUID, String siardVersion, String siardPath, TableAndColumnsParameters tableAndColumnsParameters,
     ExportOptionsParameters exportOptionsParameters, MetadataExportOptionsParameters metadataExportOptionsParameters,
     AsyncCallback<Boolean> async);
 

diff --git a/src/main/java/com/databasepreservation/common/server/BrowserServiceImpl.java b/src/main/java/com/databasepreservation/common/server/BrowserServiceImpl.java
@@ -1,5 +1,8 @@
 package com.databasepreservation.common.server;
 
+import static com.databasepreservation.common.shared.ViewerConstants.SOLR_INDEX_ROW_COLLECTION_NAME_PREFIX;
+import static com.databasepreservation.common.shared.ViewerConstants.SOLR_SEARCHES_DATABASE_UUID;
+
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -8,6 +11,7 @@
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletException;
 
+import com.databasepreservation.common.server.index.DatabaseRowsSolrManager;
 import org.apache.commons.lang3.StringUtils;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -23,6 +27,7 @@
 import org.roda.core.data.v2.index.IndexResult;
 import org.roda.core.data.v2.index.facet.Facets;
 import org.roda.core.data.v2.index.filter.Filter;
+import org.roda.core.data.v2.index.filter.SimpleFilterParameter;
 import org.roda.core.data.v2.index.sort.Sorter;
 import org.roda.core.data.v2.index.sublist.Sublist;
 import org.roda.core.data.v2.user.User;
@@ -33,6 +38,7 @@
 import com.databasepreservation.common.server.controller.SIARDController;
 import com.databasepreservation.common.server.controller.UserLoginController;
 import com.databasepreservation.common.server.index.factory.SolrClientFactory;
+import com.databasepreservation.common.server.index.schema.SolrDefaultCollectionRegistry;
 import com.databasepreservation.common.server.index.utils.SolrUtils;
 import com.databasepreservation.common.shared.ProgressData;
 import com.databasepreservation.common.shared.ValidationProgressData;
@@ -212,6 +218,40 @@ public void deleteSearch(String databaseUUID, String savedSearchUUID)
     ViewerFactory.getSolrManager().deleteSavedSearch(savedSearchUUID);
   }
 
+  @Override
+  public Boolean deleteDatabaseCollection(String databaseUUID) {
+    ViewerFactory.getSolrManager().deleteDatabasesCollection(databaseUUID);
+
+    return true;
+  }
+
+  @Override
+  public Boolean deleteRowsCollection(String databaseUUID) {
+    try {
+      final String collectionName = SOLR_INDEX_ROW_COLLECTION_NAME_PREFIX + databaseUUID;
+      if (SolrClientFactory.get().deleteCollection(collectionName)) {
+        Filter savedSearchFilter = new Filter(new SimpleFilterParameter(SOLR_SEARCHES_DATABASE_UUID, databaseUUID));
+        SolrUtils.delete(ViewerFactory.getSolrClient(), SolrDefaultCollectionRegistry.get(SavedSearch.class),
+            savedSearchFilter);
+
+        ViewerFactory.getSolrManager().markDatabaseCollection(databaseUUID, ViewerDatabase.Status.METADATA_ONLY);
+        return true;
+      }
+    } catch (GenericException | RequestNotValidException e) {
+      LOGGER.error("Error trying to remove the collection from Solr", e);
+      return false;
+    }
+    return false;
+  }
+
+  @Override
+  public Boolean deleteAllCollections(String databaseUUID) {
+    if (deleteRowsCollection(databaseUUID)) {
+      return deleteDatabaseCollection(databaseUUID);
+    }
+    return false;
+  }
+
   @Override
   public Boolean isAuthenticationEnabled() throws RODAException {
     return ViewerConfiguration.getInstance().getIsAuthenticationEnabled();
@@ -341,16 +381,18 @@ public boolean createSIARD(String UUID, ConnectionParameters connectionParameter
   }
 
   @Override
-  public boolean migrateToDBMS(String databaseUUID, String siard, ConnectionParameters connectionParameters)
+  public boolean migrateToDBMS(String databaseUUID, String siardVersion, String siardPath,
+    ConnectionParameters connectionParameters)
     throws GenericException {
-    return SIARDController.migrateToDBMS(databaseUUID, siard, connectionParameters);
+    return SIARDController.migrateToDBMS(databaseUUID, siardVersion, siardPath, connectionParameters);
   }
 
   @Override
-  public boolean migrateToSIARD(String databaseUUID, String siardPath,
+  public boolean migrateToSIARD(String databaseUUID, String siardVersion, String siardPath,
     TableAndColumnsParameters tableAndColumnsParameters, ExportOptionsParameters exportOptionsParameters,
     MetadataExportOptionsParameters metadataExportOptions) throws GenericException {
-    return SIARDController.migrateToSIARD(databaseUUID, siardPath, tableAndColumnsParameters, exportOptionsParameters,
+    return SIARDController.migrateToSIARD(databaseUUID, siardVersion, siardPath, tableAndColumnsParameters,
+      exportOptionsParameters,
       metadataExportOptions);
   }
 

diff --git a/src/main/java/com/databasepreservation/common/server/controller/SIARDController.java b/src/main/java/com/databasepreservation/common/server/controller/SIARDController.java
@@ -144,20 +144,20 @@ public static boolean testConnection(String databaseUUID, String parametersJson)
     return false;
   }
 
-  public static boolean migrateToSIARD(String databaseUUID, String siard,
+  public static boolean migrateToSIARD(String databaseUUID, String siardVersion, String siardPath,
     TableAndColumnsParameters tableAndColumnsParameters, ExportOptionsParameters exportOptionsParameters,
     MetadataExportOptionsParameters metadataExportOptionsParameters) throws GenericException {
     Reporter reporter = getReporter(databaseUUID);
-    File f = new File(siard);
+    File f = new File(siardPath);
     if (f.exists() && !f.isDirectory()) {
       LOGGER.info("starting to convert database");
       final DatabaseMigration databaseMigration = initializeDatabaseMigration(reporter);
 
       databaseMigration.filterFactories(new ArrayList<>());
 
       // BUILD Import Module
-      databaseMigration.importModule(new SIARD2ModuleFactory());
-      databaseMigration.importModuleParameter(SIARD2ModuleFactory.PARAMETER_FILE, siard);
+      databaseMigration.importModule(getSIARDImportModuleFactory(siardVersion));
+      databaseMigration.importModuleParameter(SIARD2ModuleFactory.PARAMETER_FILE, siardPath);
 
       // BUILD Export Module
       setupSIARDExportModule(databaseMigration, tableAndColumnsParameters, exportOptionsParameters,
@@ -180,19 +180,19 @@ public static boolean migrateToSIARD(String databaseUUID, String siard,
     }
   }
 
-  public static boolean migrateToDBMS(String databaseUUID, String siard, ConnectionParameters connectionParameters)
+  public static boolean migrateToDBMS(String databaseUUID, String siardVersion, String siardPath, ConnectionParameters connectionParameters)
     throws GenericException {
     Reporter reporter = getReporter(databaseUUID);
-    File f = new File(siard);
+    File f = new File(siardPath);
     if (f.exists() && !f.isDirectory()) {
       LOGGER.info("starting to convert database");
       final DatabaseMigration databaseMigration = initializeDatabaseMigration(reporter);
 
       databaseMigration.filterFactories(new ArrayList<>());
 
       // BUILD Import Module
-      databaseMigration.importModule(new SIARD2ModuleFactory());
-      databaseMigration.importModuleParameter(SIARD2ModuleFactory.PARAMETER_FILE, siard);
+      databaseMigration.importModule(getSIARDImportModuleFactory(siardVersion));
+      databaseMigration.importModuleParameter(SIARD2ModuleFactory.PARAMETER_FILE, siardPath);
 
       // BUILD Export Module
       final DatabaseModuleFactory exportModuleFactory = getDatabaseExportModuleFactory(
@@ -386,7 +386,7 @@ public static String getDatabaseExportModules() throws GenericException {
     Set<DatabaseModuleFactory> databaseModuleFactories = ReflectionUtils.collectDatabaseModuleFactories();
     for (DatabaseModuleFactory factory : databaseModuleFactories) {
       if (!factory.getModuleName().equals("list-tables")) {
-        if (!factory.getModuleName().toLowerCase().contains("siard")) {
+        if (!factory.getModuleName().toLowerCase().contains("siard") && !factory.getModuleName().toLowerCase().equals("internal-dbvtk-export")) {
           if (factory.isEnabled() && factory.producesExportModules()) {
             getDatabaseModulesParameters(factory, dbptkModule);
           }
@@ -715,6 +715,29 @@ private static DatabaseModuleFactory getDatabaseImportModuleFactory(String modul
     return factory;
   }
 
+  private static DatabaseModuleFactory getSIARDImportModuleFactory(String version) {
+    Set<DatabaseModuleFactory> databaseModuleFactories = ReflectionUtils.collectDatabaseModuleFactories();
+    final String moduleName;
+    if (version.equals("2.0") || version.equals("2.1")) {
+      moduleName = "siard-2";
+    } else if (version.equals("1.0")) {
+      moduleName = "siard-1";
+    } else {
+      moduleName = "";
+    }
+
+    DatabaseModuleFactory factory = null;
+
+    for (DatabaseModuleFactory dbFactory : databaseModuleFactories) {
+      if (dbFactory.isEnabled() && dbFactory.producesImportModules()) {
+        if (dbFactory.getModuleName().equals(moduleName))
+          factory = dbFactory;
+      }
+    }
+
+    return factory;
+  }
+
   private static DatabaseModuleFactory getDatabaseExportModuleFactory(String moduleName) {
     Set<DatabaseModuleFactory> databaseModuleFactories = ReflectionUtils.collectDatabaseModuleFactories();