added retries to vocabulary calls

gbif · Sep 18, 2024 · 3157b82 · 3157b82
1 parent 4211281
commit 3157b82
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 24 deletions.
diff --git a/pom.xml b/pom.xml
@@ -135,6 +135,7 @@
     <avro.version>1.8.2</avro.version>
     <jaxb-api.version>2.3.1</jaxb-api.version>
     <swagger-annotations.version>2.2.3</swagger-annotations.version>
+    <resilience4j.version>1.7.0</resilience4j.version>
 
     <!-- Hadoop -->
     <hadoop.version>2.6.0-cdh5.16.2</hadoop.version>
@@ -663,6 +664,13 @@
         <version>${cache2k.version}</version>
       </dependency>
 
+      <!-- Fault tolerance - Used for API service calls-->
+      <dependency>
+        <groupId>io.github.resilience4j</groupId>
+        <artifactId>resilience4j-retry</artifactId>
+        <version>${resilience4j.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>com.zaxxer</groupId>
         <artifactId>HikariCP</artifactId>

diff --git a/registry-service/pom.xml b/registry-service/pom.xml
@@ -87,6 +87,11 @@
             <artifactId>cache2k-api</artifactId>
         </dependency>
 
+      <dependency>
+        <groupId>io.github.resilience4j</groupId>
+        <artifactId>resilience4j-retry</artifactId>
+      </dependency>
+
         <!-- Test dependencies -->
         <dependency>
             <groupId>org.junit.jupiter</groupId>

diff --git a/...rvice/src/main/java/org/gbif/registry/service/collections/utils/LatimerCoreConverter.java b/...rvice/src/main/java/org/gbif/registry/service/collections/utils/LatimerCoreConverter.java
@@ -1,5 +1,16 @@
 package org.gbif.registry.service.collections.utils;
 
+import java.math.BigDecimal;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.elasticsearch.common.Strings;
 import org.gbif.api.model.collections.AlternativeCode;
 import org.gbif.api.model.collections.Collection;
 import org.gbif.api.model.collections.CollectionEntity;
@@ -25,20 +36,6 @@
 import org.gbif.vocabulary.api.ConceptView;
 import org.gbif.vocabulary.client.ConceptClient;
 
-import java.math.BigDecimal;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-import org.elasticsearch.common.Strings;
-
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
 @Slf4j
 @NoArgsConstructor(access = AccessLevel.PRIVATE)
 public class LatimerCoreConverter {
@@ -353,8 +350,8 @@ public static ObjectGroup toObjectGroup(
         .forEach(
             ct -> {
               ConceptView conceptView =
-                  conceptClient.getFromLatestRelease(
-                      Vocabularies.COLLECTION_CONTENT_TYPE, ct, false, false);
+                  Vocabularies.getConceptLatestRelease(
+                      Vocabularies.COLLECTION_CONTENT_TYPE, ct, conceptClient);
               if (conceptView != null
                   && conceptView.getConcept().getTags().stream()
                       .anyMatch(t -> t.getName().equals("ltc:discipline"))) {

diff --git a/registry-service/src/main/java/org/gbif/registry/service/collections/utils/Vocabularies.java b/registry-service/src/main/java/org/gbif/registry/service/collections/utils/Vocabularies.java
@@ -1,6 +1,14 @@
 package org.gbif.registry.service.collections.utils;
 
 import com.google.common.base.Strings;
+import io.github.resilience4j.core.IntervalFunction;
+import io.github.resilience4j.retry.Retry;
+import io.github.resilience4j.retry.RetryConfig;
+import java.time.Duration;
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.Function;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.NoArgsConstructor;
@@ -17,14 +25,17 @@
 import org.gbif.vocabulary.api.ConceptView;
 import org.gbif.vocabulary.client.ConceptClient;
 
-import java.util.*;
-import java.util.function.BiConsumer;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-
 @NoArgsConstructor(access = AccessLevel.PRIVATE)
 public class Vocabularies {
 
+  private static final Retry RETRY =
+      Retry.of(
+          "vocabularyCall",
+          RetryConfig.custom()
+              .maxAttempts(7)
+              .intervalFunction(IntervalFunction.ofExponentialBackoff(Duration.ofSeconds(1)))
+              .build());
+
   static final Map<String, Function<Institution, java.util.Collection<String>>>
       INSTITUTION_VOCAB_FIELDS = new HashMap<>();
   static final Map<String, Function<Collection, java.util.Collection<String>>>
@@ -122,7 +133,10 @@ public static <T extends CollectionEntity> void checkVocabsValues(
   private static void checkConcept(
       ConceptClient conceptClient, String vocabName, String conceptValue, StringJoiner errors) {
     ConceptView conceptFound =
-        conceptClient.getFromLatestRelease(vocabName, conceptValue, false, false);
+        Retry.decorateSupplier(
+                RETRY,
+                () -> conceptClient.getFromLatestRelease(vocabName, conceptValue, false, false))
+            .get();
 
     if (conceptFound == null) {
       errors.add(conceptValue + " is not a concept of the " + vocabName + " vocabulary");
@@ -174,8 +188,16 @@ public static <T extends SearchRequest> void addChildrenConcepts(
   private static Set<String> findChildren(
       ConceptClient conceptClient, String vocabName, String conceptName, Set<String> allChildren) {
     PagingResponse<ConceptView> result =
-        conceptClient.listConceptsLatestRelease(
-            vocabName, ConceptListParams.builder().name(conceptName).includeChildren(true).build());
+        Retry.decorateSupplier(
+                RETRY,
+                () ->
+                    conceptClient.listConceptsLatestRelease(
+                        vocabName,
+                        ConceptListParams.builder()
+                            .name(conceptName)
+                            .includeChildren(true)
+                            .build()))
+            .get();
 
     if (result.getResults() == null
         || result.getResults().isEmpty()
@@ -197,6 +219,13 @@ private static Set<String> findChildren(
     return allChildren;
   }
 
+  public static ConceptView getConceptLatestRelease(
+      String vocabulary, String conceptName, ConceptClient conceptClient) {
+    return Retry.decorateSupplier(
+            RETRY, () -> conceptClient.getFromLatestRelease(vocabulary, conceptName, false, false))
+        .get();
+  }
+
   @AllArgsConstructor(staticName = "of")
   @NoArgsConstructor
   private static class SearchRequestField<T extends SearchRequest> {