Iqss/7140 google cloud archiver

doc/sphinx-guides/source/installation/config.rst

@@ -779,7 +779,7 @@ Dataverse may be configured to submit a copy of published Datasets, packaged as
 
 Dataverse offers an internal archive workflow which may be configured as a PostPublication workflow via an admin API call to manually submit previously published Datasets and prior versions to a configured archive such as Chronopolis. The workflow creates a `JSON-LD <http://www.openarchives.org/ore/0.9/jsonld>`_ serialized `OAI-ORE <https://www.openarchives.org/ore/>`_ map file, which is also available as a metadata export format in the Dataverse web interface.
 
-At present, the DPNSubmitToArchiveCommand and LocalSubmitToArchiveCommand are the only implementations extending the AbstractSubmitToArchiveCommand and using the configurable mechanisms discussed below.
+At present, the DPNSubmitToArchiveCommand, LocalSubmitToArchiveCommand, and GoogleCloudSubmitToArchive are the only implementations extending the AbstractSubmitToArchiveCommand and using the configurable mechanisms discussed below.
 
 .. _Duracloud Configuration:
 
@@ -827,10 +827,36 @@ ArchiverClassName - the fully qualified class to be used for archiving. For exam
 
 \:ArchiverSettings - the archiver class can access required settings including existing Dataverse settings and dynamically defined ones specific to the class. This setting is a comma-separated list of those settings. For example\:
 
-``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":BagItLocalPath”``
+``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":BagItLocalPath"``
 
 :BagItLocalPath is the file path that you've set in :ArchiverSettings.
 
+.. _Google Cloud Configuration:
+
+Google Cloud Configuration
+++++++++++++++++++++++++++
+
+The Google Cloud Archiver can send Dataverse Bags to a bucket in Google's cloud, including those in the 'Coldline' storage class (cheaper, with slower access) 
+
+``curl http://localhost:8080/api/admin/settings/:ArchiverClassName -X PUT -d "edu.harvard.iq.dataverse.engine.command.impl.GoogleCloudSubmitToArchiveCommand"``
+
+``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":":GoogleCloudBucket, :GoogleCloudProject"``
+
+The Google Cloud Archiver defines two custom settings, both are required:
+
+\:GoogleCloudBucket - the name of the bucket to use. For example:
+
+``curl http://localhost:8080/api/admin/settings/:GoogleCloudBucket -X PUT -d "qdr-archive"``
+
+\:GoogleCloudProject - the name of the project managing the bucket. For example:
+
+``curl http://localhost:8080/api/admin/settings/:GoogleCloudProject -X PUT -d "qdr-project"``
+
+In addition, the Google Cloud Archiver requires that the googlecloudkey.json file for the project be placed in the 'dataverse.files.directory' directory. This file can be created in the Google Cloud Console.
+
+.. _Local Path Configuration:
+
+
 API Call
 ++++++++
 

pom.xml

@@ -57,7 +57,7 @@
             </releases>
         </pluginRepository>
     </pluginRepositories>
-    <!--Maven checks for dependendies from these repos in the order shown in the pom.xml
+    <!--Maven checks for dependencies from these repos in the order shown in the pom.xml
         This isn't well documented and seems to change between maven versions -MAD 4.9.4 -->
     <repositories>
         <repository>
@@ -127,6 +127,13 @@
                 <artifactId>httpclient</artifactId>
                 <version>${httpcomponents.client.version}</version>
             </dependency>
+            <dependency>
+              <groupId>com.google.cloud</groupId>
+              <artifactId>google-cloud-bom</artifactId>
+              <version>0.115.0-alpha</version>
+              <type>pom</type>
+              <scope>import</scope>
+            </dependency>
             <dependency>
                 <groupId>org.testcontainers</groupId>
                 <artifactId>testcontainers-bom</artifactId>
@@ -137,7 +144,7 @@
         </dependencies>
     </dependencyManagement>
     <!-- Declare any DIRECT dependencies here.
-         In case the depency is both transitive and direct (e. g. some common lib for logging),
+         In case the dependency is both transitive and direct (e. g. some common lib for logging),
          manage the version above and add the direct dependency here WITHOUT version tag, too.
     -->
     <!-- TODO: Housekeeping is utterly needed. -->
@@ -581,6 +588,11 @@
             <artifactId>opennlp-tools</artifactId>
             <version>1.9.1</version>
         </dependency>
+        <dependency>
+          <groupId>com.google.cloud</groupId>
+          <artifactId>google-cloud-storage</artifactId>
+          <version>1.97.0</version>
+        </dependency>
 
 
         <!-- TESTING DEPENDENCIES -->

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java

@@ -99,7 +99,12 @@ public void run() {
                                 }
                             }
                         }).start();
-
+                        //Have seen Pipe Closed errors for other archivers when used as a workflow without this delay loop
+                        int i=0;
+                        while(digestInputStream.available()<=0 && i<100) {
+                            Thread.sleep(10);
+                            i++;
+                        }
                         String checksum = store.addContent(spaceName, "datacite.xml", digestInputStream, -1l, null, null,
                                 null);
                         logger.fine("Content: datacite.xml added with checksum: " + checksum);
@@ -133,7 +138,11 @@ public void run() {
                                     }
                                 }
                             }).start();
-
+                            i=0;
+                            while(digestInputStream.available()<=0 && i<100) {
+                                Thread.sleep(10);
+                                i++;
+                            }
                             checksum = store.addContent(spaceName, fileName, digestInputStream2, -1l, null, null,
                                     null);
                             logger.fine("Content: " + fileName + " added with checksum: " + checksum);
@@ -174,6 +183,9 @@ public void run() {
                         logger.severe(rte.getMessage());
                         return new Failure("Error in generating datacite.xml file",
                                 "DuraCloud Submission Failure: metadata file not created");
+                    } catch (InterruptedException e) {
+                        logger.warning(e.getLocalizedMessage());
+                        e.printStackTrace();
                     }
                 } catch (ContentStoreException e) {
                     logger.warning(e.getMessage());