diff --git a/hystrix-contrib/hystrix-clj/src/test/clojure/com/netflix/hystrix/core_test.clj b/hystrix-contrib/hystrix-clj/src/test/clojure/com/netflix/hystrix/core_test.clj index 132664fa7..4f70fb021 100644 --- a/hystrix-contrib/hystrix-clj/src/test/clojure/com/netflix/hystrix/core_test.clj +++ b/hystrix-contrib/hystrix-clj/src/test/clojure/com/netflix/hystrix/core_test.clj @@ -215,7 +215,7 @@ (wait-for-observable o))))) (testing "observes command with a Scheduler" (let [o (observe-later-on (normalize base-def) - (rx.concurrency.Schedulers/newThread) + (rx.schedulers.Schedulers/newThread) 75 19 23)] (is (instance? rx.Observable o)) (is (= (+ 75 19 23) @@ -317,7 +317,7 @@ (is (= 103 (wait-for-observable (observe #'my-fn-command 90 13)))) (is (= 105 (wait-for-observable (observe-later #'my-fn-command 91 14)))) (is (= 107 (wait-for-observable (observe-later-on #'my-fn-command - (rx.concurrency.Schedulers/newThread) + (rx.schedulers.Schedulers/newThread) 92 15))))))) (defcollapser my-collapser diff --git a/hystrix-core/build.gradle b/hystrix-core/build.gradle index f9ce881c5..a7ab85165 100644 --- a/hystrix-core/build.gradle +++ b/hystrix-core/build.gradle @@ -4,10 +4,10 @@ apply plugin: 'idea' dependencies { compile 'com.netflix.archaius:archaius-core:0.4.1' - compile 'com.netflix.rxjava:rxjava-core:0.16.1' + compile 'com.netflix.rxjava:rxjava-core:0.17.0' compile 'org.slf4j:slf4j-api:1.7.0' compile 'com.google.code.findbugs:jsr305:2.0.0' - provided 'junit:junit-dep:4.10' + testCompile 'junit:junit-dep:4.10' } javadoc { diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/Hystrix.java b/hystrix-core/src/main/java/com/netflix/hystrix/Hystrix.java index c43616d63..c55aa5081 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/Hystrix.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/Hystrix.java @@ -1,17 +1,13 @@ package com.netflix.hystrix; -import static org.junit.Assert.*; - import java.util.LinkedList; import java.util.NoSuchElementException; import java.util.concurrent.TimeUnit; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.netflix.hystrix.HystrixCommand.Setter; -import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; +import rx.functions.Action0; /** * Lifecycle management of Hystrix. @@ -85,18 +81,35 @@ public static HystrixCommandKey getCurrentThreadExecutingCommand() { return currentCommand.get().peek(); } - /* package */static void startCurrentThreadExecutingCommand(HystrixCommandKey key) { + /** + * + * @return Action0 to perform the same work as `endCurrentThreadExecutingCommand()` but can be done from any thread + */ + /* package */static Action0 startCurrentThreadExecutingCommand(HystrixCommandKey key) { + final LinkedList list = currentCommand.get(); try { - currentCommand.get().push(key); + list.push(key); } catch (Exception e) { logger.warn("Unable to record command starting", e); } + return new Action0() { + + @Override + public void call() { + endCurrentThreadExecutingCommand(list); + } + + }; } /* package */static void endCurrentThreadExecutingCommand() { + endCurrentThreadExecutingCommand(currentCommand.get()); + } + + private static void endCurrentThreadExecutingCommand(LinkedList list) { try { - if (!currentCommand.get().isEmpty()) { - currentCommand.get().pop(); + if (!list.isEmpty()) { + list.pop(); } } catch (NoSuchElementException e) { // this shouldn't be possible since we check for empty above and this is thread-isolated @@ -106,163 +119,4 @@ public static HystrixCommandKey getCurrentThreadExecutingCommand() { } } - public static class UnitTest { - @Test - public void testNotInThread() { - assertNull(getCurrentThreadExecutingCommand()); - } - - @Test - public void testInsideHystrixThread() { - - assertNull(getCurrentThreadExecutingCommand()); - - HystrixCommand command = new HystrixCommand(Setter - .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) - .andCommandKey(HystrixCommandKey.Factory.asKey("CommandName"))) { - - @Override - protected Boolean run() { - assertEquals("CommandName", getCurrentThreadExecutingCommand().name()); - - return getCurrentThreadExecutingCommand() != null; - } - - }; - - assertTrue(command.execute()); - assertNull(getCurrentThreadExecutingCommand()); - } - - @Test - public void testInsideNestedHystrixThread() { - - HystrixCommand command = new HystrixCommand(Setter - .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) - .andCommandKey(HystrixCommandKey.Factory.asKey("OuterCommand"))) { - - @Override - protected Boolean run() { - - assertEquals("OuterCommand", getCurrentThreadExecutingCommand().name()); - - if (getCurrentThreadExecutingCommand() == null) { - throw new RuntimeException("BEFORE expected it to run inside a thread"); - } - - HystrixCommand command2 = new HystrixCommand(Setter - .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) - .andCommandKey(HystrixCommandKey.Factory.asKey("InnerCommand"))) { - - @Override - protected Boolean run() { - assertEquals("InnerCommand", getCurrentThreadExecutingCommand().name()); - - return getCurrentThreadExecutingCommand() != null; - } - - }; - - if (getCurrentThreadExecutingCommand() == null) { - throw new RuntimeException("AFTER expected it to run inside a thread"); - } - - return command2.execute(); - } - - }; - - assertTrue(command.execute()); - - assertNull(getCurrentThreadExecutingCommand()); - } - - @Test - public void testInsideHystrixSemaphoreExecute() { - - HystrixCommand command = new HystrixCommand(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) - .andCommandKey(HystrixCommandKey.Factory.asKey("SemaphoreIsolatedCommandName")) - .andCommandPropertiesDefaults(HystrixCommandProperties.Setter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { - - @Override - protected Boolean run() { - assertEquals("SemaphoreIsolatedCommandName", getCurrentThreadExecutingCommand().name()); - - return getCurrentThreadExecutingCommand() != null; - } - - }; - - // it should be true for semaphore isolation as well - assertTrue(command.execute()); - // and then be null again once done - assertNull(getCurrentThreadExecutingCommand()); - } - - @Test - public void testInsideHystrixSemaphoreQueue() throws Exception { - - HystrixCommand command = new HystrixCommand(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) - .andCommandKey(HystrixCommandKey.Factory.asKey("SemaphoreIsolatedCommandName")) - .andCommandPropertiesDefaults(HystrixCommandProperties.Setter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { - - @Override - protected Boolean run() { - assertEquals("SemaphoreIsolatedCommandName", getCurrentThreadExecutingCommand().name()); - - return getCurrentThreadExecutingCommand() != null; - } - - }; - - // it should be true for semaphore isolation as well - assertTrue(command.queue().get()); - // and then be null again once done - assertNull(getCurrentThreadExecutingCommand()); - } - - @Test - public void testThreadNestedInsideHystrixSemaphore() { - - HystrixCommand command = new HystrixCommand(Setter - .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) - .andCommandKey(HystrixCommandKey.Factory.asKey("OuterSemaphoreCommand")) - .andCommandPropertiesDefaults(HystrixCommandProperties.Setter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { - - @Override - protected Boolean run() { - - assertEquals("OuterSemaphoreCommand", getCurrentThreadExecutingCommand().name()); - - if (getCurrentThreadExecutingCommand() == null) { - throw new RuntimeException("BEFORE expected it to run inside a semaphore"); - } - - HystrixCommand command2 = new HystrixCommand(Setter - .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) - .andCommandKey(HystrixCommandKey.Factory.asKey("InnerCommand"))) { - - @Override - protected Boolean run() { - assertEquals("InnerCommand", getCurrentThreadExecutingCommand().name()); - - return getCurrentThreadExecutingCommand() != null; - } - - }; - - if (getCurrentThreadExecutingCommand() == null) { - throw new RuntimeException("AFTER expected it to run inside a semaphore"); - } - - return command2.execute(); - } - - }; - - assertTrue(command.execute()); - - assertNull(getCurrentThreadExecutingCommand()); - } - } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCircuitBreaker.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCircuitBreaker.java index 7bb208ccb..9200de48b 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCircuitBreaker.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCircuitBreaker.java @@ -15,19 +15,13 @@ */ package com.netflix.hystrix; -import static org.junit.Assert.*; - import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.concurrent.ThreadSafe; -import org.junit.Test; - import com.netflix.hystrix.HystrixCommandMetrics.HealthCounts; -import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifierDefault; -import com.netflix.hystrix.util.HystrixRollingNumberEvent; /** * Circuit-breaker logic that is hooked into {@link HystrixCommand} execution and will stop allowing executions if failures have gone past the defined threshold. @@ -117,7 +111,7 @@ public static HystrixCircuitBreaker getInstance(HystrixCommandKey key) { /** * Clears all circuit breakers. If new requests come in instances will be recreated. */ - /* package */ static void reset() { + /* package */static void reset() { circuitBreakersByCommand.clear(); } } @@ -239,495 +233,4 @@ public void markSuccess() { } - /** - * A simple circuit breaker intended for unit testing of the {@link HystrixCommand} object, NOT production use. - *

- * This uses simple logic to 'trip' the circuit after 3 subsequent failures and doesn't recover. - * - * @ExcludeFromJavadoc - */ - /* package */class TestCircuitBreaker implements HystrixCircuitBreaker { - - final HystrixCommandMetrics metrics; - private boolean forceShortCircuit = false; - - public TestCircuitBreaker() { - this.metrics = UnitTest.getMetrics(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter()); - forceShortCircuit = false; - } - - public TestCircuitBreaker setForceShortCircuit(boolean value) { - this.forceShortCircuit = value; - return this; - } - - @Override - public boolean isOpen() { - if (forceShortCircuit) { - return true; - } else { - return metrics.getCumulativeCount(HystrixRollingNumberEvent.FAILURE) >= 3; - } - } - - @Override - public void markSuccess() { - // we don't need to do anything since we're going to permanently trip the circuit - } - - @Override - public boolean allowRequest() { - return !isOpen(); - } - - } - - /* - * Why unit tests as inner classes? => http://benjchristensen.com/2011/10/23/junit-tests-as-inner-classes/ - */ - public static class UnitTest { - - private HystrixCommandKey key = CommandKeyForUnitTest.KEY_ONE; - - /** - * Test that if all 'marks' are successes during the test window that it does NOT trip the circuit. - * Test that if all 'marks' are failures during the test window that it trips the circuit. - */ - @Test - public void testTripCircuit() { - try { - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter(); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - metrics.markSuccess(1000); - metrics.markSuccess(1000); - metrics.markSuccess(1000); - metrics.markSuccess(1000); - - // this should still allow requests as everything has been successful - assertTrue(cb.allowRequest()); - assertFalse(cb.isOpen()); - - // fail - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - - // everything has failed in the test window so we should return false now - assertFalse(cb.allowRequest()); - assertTrue(cb.isOpen()); - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * Test that if the % of failures is higher than the threshold that the circuit trips. - */ - @Test - public void testTripCircuitOnFailuresAboveThreshold() { - try { - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter(); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - // this should start as allowing requests - assertTrue(cb.allowRequest()); - assertFalse(cb.isOpen()); - - // success with high latency - metrics.markSuccess(400); - metrics.markSuccess(400); - metrics.markFailure(10); - metrics.markSuccess(400); - metrics.markFailure(10); - metrics.markFailure(10); - metrics.markSuccess(400); - metrics.markFailure(10); - metrics.markFailure(10); - - // this should trip the circuit as the error percentage is above the threshold - assertFalse(cb.allowRequest()); - assertTrue(cb.isOpen()); - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * Test that if the % of failures is higher than the threshold that the circuit trips. - */ - @Test - public void testCircuitDoesNotTripOnFailuresBelowThreshold() { - try { - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter(); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - // this should start as allowing requests - assertTrue(cb.allowRequest()); - assertFalse(cb.isOpen()); - - // success with high latency - metrics.markSuccess(400); - metrics.markSuccess(400); - metrics.markFailure(10); - metrics.markSuccess(400); - metrics.markSuccess(40); - metrics.markSuccess(400); - metrics.markFailure(10); - metrics.markFailure(10); - - // this should remain open as the failure threshold is below the percentage limit - assertTrue(cb.allowRequest()); - assertFalse(cb.isOpen()); - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * Test that if all 'marks' are timeouts that it will trip the circuit. - */ - @Test - public void testTripCircuitOnTimeouts() { - try { - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter(); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - // this should start as allowing requests - assertTrue(cb.allowRequest()); - assertFalse(cb.isOpen()); - - // timeouts - metrics.markTimeout(2000); - metrics.markTimeout(2000); - metrics.markTimeout(2000); - metrics.markTimeout(2000); - - // everything has been a timeout so we should not allow any requests - assertFalse(cb.allowRequest()); - assertTrue(cb.isOpen()); - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * Test that if the % of timeouts is higher than the threshold that the circuit trips. - */ - @Test - public void testTripCircuitOnTimeoutsAboveThreshold() { - try { - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter(); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - // this should start as allowing requests - assertTrue(cb.allowRequest()); - assertFalse(cb.isOpen()); - - // success with high latency - metrics.markSuccess(400); - metrics.markSuccess(400); - metrics.markTimeout(10); - metrics.markSuccess(400); - metrics.markTimeout(10); - metrics.markTimeout(10); - metrics.markSuccess(400); - metrics.markTimeout(10); - metrics.markTimeout(10); - - // this should trip the circuit as the error percentage is above the threshold - assertFalse(cb.allowRequest()); - assertTrue(cb.isOpen()); - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * Test that on an open circuit that a single attempt will be allowed after a window of time to see if issues are resolved. - */ - @Test - public void testSingleTestOnOpenCircuitAfterTimeWindow() { - try { - int sleepWindow = 200; - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - // fail - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - - // everything has failed in the test window so we should return false now - assertFalse(cb.allowRequest()); - assertTrue(cb.isOpen()); - - // wait for sleepWindow to pass - Thread.sleep(sleepWindow + 50); - - // we should now allow 1 request - assertTrue(cb.allowRequest()); - // but the circuit should still be open - assertTrue(cb.isOpen()); - // and further requests are still blocked - assertFalse(cb.allowRequest()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * Test that an open circuit is closed after 1 success. - */ - @Test - public void testCircuitClosedAfterSuccess() { - try { - int sleepWindow = 200; - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - // fail - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markTimeout(1000); - - // everything has failed in the test window so we should return false now - assertFalse(cb.allowRequest()); - assertTrue(cb.isOpen()); - - // wait for sleepWindow to pass - Thread.sleep(sleepWindow + 50); - - // we should now allow 1 request - assertTrue(cb.allowRequest()); - // but the circuit should still be open - assertTrue(cb.isOpen()); - // and further requests are still blocked - assertFalse(cb.allowRequest()); - - // the 'singleTest' succeeds so should cause the circuit to be closed - metrics.markSuccess(500); - cb.markSuccess(); - - // all requests should be open again - assertTrue(cb.allowRequest()); - assertTrue(cb.allowRequest()); - assertTrue(cb.allowRequest()); - // and the circuit should be closed again - assertFalse(cb.isOpen()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * Test that an open circuit is closed after 1 success... when the sleepWindow is smaller than the statisticalWindow and 'failure' stats are still sticking around. - *

- * This means that the statistical window needs to be cleared otherwise it will still calculate the failure percentage below the threshold and immediately open the circuit again. - */ - @Test - public void testCircuitClosedAfterSuccessAndClearsStatisticalWindow() { - try { - int statisticalWindow = 200; - int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow).withMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - // fail - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - - // everything has failed in the test window so we should return false now - assertFalse(cb.allowRequest()); - assertTrue(cb.isOpen()); - - // wait for sleepWindow to pass - Thread.sleep(sleepWindow + 50); - - // we should now allow 1 request - assertTrue(cb.allowRequest()); - // but the circuit should still be open - assertTrue(cb.isOpen()); - // and further requests are still blocked - assertFalse(cb.allowRequest()); - - // the 'singleTest' succeeds so should cause the circuit to be closed - metrics.markSuccess(500); - cb.markSuccess(); - - // all requests should be open again - assertTrue(cb.allowRequest()); - assertTrue(cb.allowRequest()); - assertTrue(cb.allowRequest()); - // and the circuit should be closed again - assertFalse(cb.isOpen()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * Over a period of several 'windows' a single attempt will be made and fail and then finally succeed and close the circuit. - *

- * Ensure the circuit is kept open through the entire testing period and that only the single attempt in each window is made. - */ - @Test - public void testMultipleTimeWindowRetriesBeforeClosingCircuit() { - try { - int sleepWindow = 200; - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - // fail - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - - // everything has failed in the test window so we should return false now - assertFalse(cb.allowRequest()); - assertTrue(cb.isOpen()); - - // wait for sleepWindow to pass - Thread.sleep(sleepWindow + 50); - - // we should now allow 1 request - assertTrue(cb.allowRequest()); - // but the circuit should still be open - assertTrue(cb.isOpen()); - // and further requests are still blocked - assertFalse(cb.allowRequest()); - - // the 'singleTest' fails so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep - metrics.markFailure(1000); - - assertFalse(cb.allowRequest()); - assertFalse(cb.allowRequest()); - assertFalse(cb.allowRequest()); - - // wait for sleepWindow to pass - Thread.sleep(sleepWindow + 50); - - // we should now allow 1 request - assertTrue(cb.allowRequest()); - // but the circuit should still be open - assertTrue(cb.isOpen()); - // and further requests are still blocked - assertFalse(cb.allowRequest()); - - // the 'singleTest' fails again so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep - metrics.markFailure(1000); - - assertFalse(cb.allowRequest()); - assertFalse(cb.allowRequest()); - assertFalse(cb.allowRequest()); - - // wait for sleepWindow to pass - Thread.sleep(sleepWindow + 50); - - // we should now allow 1 request - assertTrue(cb.allowRequest()); - // but the circuit should still be open - assertTrue(cb.isOpen()); - // and further requests are still blocked - assertFalse(cb.allowRequest()); - - // now it finally succeeds - metrics.markSuccess(200); - cb.markSuccess(); - - // all requests should be open again - assertTrue(cb.allowRequest()); - assertTrue(cb.allowRequest()); - assertTrue(cb.allowRequest()); - // and the circuit should be closed again - assertFalse(cb.isOpen()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * When volume of reporting during a statistical window is lower than a defined threshold the circuit - * will not trip regardless of whatever statistics are calculated. - */ - @Test - public void testLowVolumeDoesNotTripCircuit() { - try { - int sleepWindow = 200; - int lowVolume = 5; - - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow).withCircuitBreakerRequestVolumeThreshold(lowVolume); - HystrixCommandMetrics metrics = getMetrics(properties); - HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); - - // fail - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - metrics.markFailure(1000); - - // even though it has all failed we won't trip the circuit because the volume is low - assertTrue(cb.allowRequest()); - assertFalse(cb.isOpen()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - } - - /** - * Utility method for creating {@link HystrixCommandMetrics} for unit tests. - */ - private static HystrixCommandMetrics getMetrics(HystrixCommandProperties.Setter properties) { - return new HystrixCommandMetrics(CommandKeyForUnitTest.KEY_ONE, CommandOwnerForUnitTest.OWNER_ONE, HystrixCommandProperties.Setter.asMock(properties), HystrixEventNotifierDefault.getInstance()); - } - - /** - * Utility method for creating {@link HystrixCircuitBreaker} for unit tests. - */ - private static HystrixCircuitBreaker getCircuitBreaker(HystrixCommandKey key, HystrixCommandGroupKey commandGroup, HystrixCommandMetrics metrics, HystrixCommandProperties.Setter properties) { - return new HystrixCircuitBreakerImpl(key, commandGroup, HystrixCommandProperties.Setter.asMock(properties), metrics); - } - - private static enum CommandOwnerForUnitTest implements HystrixCommandGroupKey { - OWNER_ONE, OWNER_TWO; - } - - private static enum ThreadPoolKeyForUnitTest implements HystrixThreadPoolKey { - THREAD_POOL_ONE, THREAD_POOL_TWO; - } - - private static enum CommandKeyForUnitTest implements HystrixCommandKey { - KEY_ONE, KEY_TWO; - } - } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java index 28ffdcda3..6f71d0556 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapser.java @@ -15,34 +15,21 @@ */ package com.netflix.hystrix; -import static org.junit.Assert.*; - -import java.lang.ref.Reference; -import java.lang.ref.SoftReference; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.concurrent.NotThreadSafe; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import rx.Observable; import rx.Scheduler; -import rx.concurrency.Schedulers; +import rx.schedulers.Schedulers; import rx.subjects.ReplaySubject; -import com.netflix.hystrix.HystrixCommand.UnitTest.TestHystrixCommand; import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; import com.netflix.hystrix.collapser.CollapserTimer; import com.netflix.hystrix.collapser.HystrixCollapserBridge; @@ -51,11 +38,8 @@ import com.netflix.hystrix.collapser.RequestCollapserFactory; import com.netflix.hystrix.exception.HystrixRuntimeException; import com.netflix.hystrix.strategy.HystrixPlugins; -import com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable; import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; -import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableHolder; import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; -import com.netflix.hystrix.util.HystrixTimer.TimerListener; /** * Collapse multiple requests into a single {@link HystrixCommand} execution based on a time window and optionally a max batch size. @@ -128,7 +112,7 @@ protected HystrixCollapser(Setter setter) { this(setter.collapserKey, setter.scope, new RealCollapserTimer(), setter.propertiesSetter); } - private HystrixCollapser(HystrixCollapserKey collapserKey, Scope scope, CollapserTimer timer, HystrixCollapserProperties.Setter propertiesBuilder) { + /* package for tests */ HystrixCollapser(HystrixCollapserKey collapserKey, Scope scope, CollapserTimer timer, HystrixCollapserProperties.Setter propertiesBuilder) { if (collapserKey == null || collapserKey.name().trim().equals("")) { String defaultKeyName = getDefaultNameFromClass(getClass()); collapserKey = HystrixCollapserKey.Factory.asKey(defaultKeyName); @@ -343,7 +327,7 @@ public Observable observe() { public Observable toObservable() { // when we callback with the data we want to do the work // on a separate thread than the one giving us the callback - return toObservable(Schedulers.threadPoolForComputation()); + return toObservable(Schedulers.computation()); } /** @@ -577,1260 +561,4 @@ public Setter andCollapserPropertiesDefaults(HystrixCollapserProperties.Setter p @SuppressWarnings("rawtypes") private static ConcurrentHashMap, String> defaultNameCache = new ConcurrentHashMap, String>(); - public static class UnitTests { - - static AtomicInteger counter = new AtomicInteger(); - - @Before - public void init() { - counter.set(0); - // since we're going to modify properties of the same class between tests, wipe the cache each time - reset(); - /* we must call this to simulate a new request lifecycle running and clearing caches */ - HystrixRequestContext.initializeContext(); - } - - @After - public void cleanup() { - // instead of storing the reference from initialize we'll just get the current state and shutdown - if (HystrixRequestContext.getContextForCurrentThread() != null) { - // it may be null if a test shuts the context down manually - HystrixRequestContext.getContextForCurrentThread().shutdown(); - } - } - - @Test - public void testTwoRequests() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); - Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); - timer.incrementTime(10); // let time pass that equals the default delay/period - - assertEquals("1", response1.get()); - assertEquals("2", response2.get()); - - assertEquals(1, counter.get()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testMultipleBatches() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); - Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); - timer.incrementTime(10); // let time pass that equals the default delay/period - - assertEquals("1", response1.get()); - assertEquals("2", response2.get()); - - assertEquals(1, counter.get()); - - // now request more - Future response3 = new TestRequestCollapser(timer, counter, 3).queue(); - timer.incrementTime(10); // let time pass that equals the default delay/period - - assertEquals("3", response3.get()); - - // we should have had it execute twice now - assertEquals(2, counter.get()); - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testMaxRequestsInBatch() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapser(timer, counter, 1, 2, 10).queue(); - Future response2 = new TestRequestCollapser(timer, counter, 2, 2, 10).queue(); - Future response3 = new TestRequestCollapser(timer, counter, 3, 2, 10).queue(); - timer.incrementTime(10); // let time pass that equals the default delay/period - - assertEquals("1", response1.get()); - assertEquals("2", response2.get()); - assertEquals("3", response3.get()); - - // we should have had it execute twice because the batch size was 2 - assertEquals(2, counter.get()); - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testRequestsOverTime() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); - timer.incrementTime(5); - Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); - timer.incrementTime(8); - // should execute here - Future response3 = new TestRequestCollapser(timer, counter, 3).queue(); - timer.incrementTime(6); - Future response4 = new TestRequestCollapser(timer, counter, 4).queue(); - timer.incrementTime(8); - // should execute here - Future response5 = new TestRequestCollapser(timer, counter, 5).queue(); - timer.incrementTime(10); - // should execute here - - // wait for all tasks to complete - assertEquals("1", response1.get()); - assertEquals("2", response2.get()); - assertEquals("3", response3.get()); - assertEquals("4", response4.get()); - assertEquals("5", response5.get()); - - System.out.println("number of executions: " + counter.get()); - assertEquals(3, counter.get()); - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testUnsubscribeOnOneDoesntKillBatch() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); - Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); - - // kill the first - response1.cancel(true); - - timer.incrementTime(10); // let time pass that equals the default delay/period - - // the first is cancelled so should return null - assertEquals(null, response1.get()); - // we should still get a response on the second - assertEquals("2", response2.get()); - - assertEquals(1, counter.get()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testShardedRequests() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestShardedRequestCollapser(timer, counter, "1a").queue(); - Future response2 = new TestShardedRequestCollapser(timer, counter, "2b").queue(); - Future response3 = new TestShardedRequestCollapser(timer, counter, "3b").queue(); - Future response4 = new TestShardedRequestCollapser(timer, counter, "4a").queue(); - timer.incrementTime(10); // let time pass that equals the default delay/period - - assertEquals("1a", response1.get()); - assertEquals("2b", response2.get()); - assertEquals("3b", response3.get()); - assertEquals("4a", response4.get()); - - /* we should get 2 batches since it gets sharded */ - assertEquals(2, counter.get()); - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testRequestScope() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapser(timer, counter, "1").queue(); - Future response2 = new TestRequestCollapser(timer, counter, "2").queue(); - - // simulate a new request - RequestCollapserFactory.resetRequest(); - - Future response3 = new TestRequestCollapser(timer, counter, "3").queue(); - Future response4 = new TestRequestCollapser(timer, counter, "4").queue(); - - timer.incrementTime(10); // let time pass that equals the default delay/period - - assertEquals("1", response1.get()); - assertEquals("2", response2.get()); - assertEquals("3", response3.get()); - assertEquals("4", response4.get()); - - // 2 different batches should execute, 1 per request - assertEquals(2, counter.get()); - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testGlobalScope() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestGloballyScopedRequestCollapser(timer, counter, "1").queue(); - Future response2 = new TestGloballyScopedRequestCollapser(timer, counter, "2").queue(); - - // simulate a new request - RequestCollapserFactory.resetRequest(); - - Future response3 = new TestGloballyScopedRequestCollapser(timer, counter, "3").queue(); - Future response4 = new TestGloballyScopedRequestCollapser(timer, counter, "4").queue(); - - timer.incrementTime(10); // let time pass that equals the default delay/period - - assertEquals("1", response1.get()); - assertEquals("2", response2.get()); - assertEquals("3", response3.get()); - assertEquals("4", response4.get()); - - // despite having cleared the cache in between we should have a single execution because this is on the global not request cache - assertEquals(1, counter.get()); - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testErrorHandlingViaFutureException() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapserWithFaultyCreateCommand(timer, counter, "1").queue(); - Future response2 = new TestRequestCollapserWithFaultyCreateCommand(timer, counter, "2").queue(); - timer.incrementTime(10); // let time pass that equals the default delay/period - - try { - response1.get(); - fail("we should have received an exception"); - } catch (ExecutionException e) { - // what we expect - } - try { - response2.get(); - fail("we should have received an exception"); - } catch (ExecutionException e) { - // what we expect - } - - assertEquals(0, counter.get()); - assertEquals(0, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testErrorHandlingWhenMapToResponseFails() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapserWithFaultyMapToResponse(timer, counter, "1").queue(); - Future response2 = new TestRequestCollapserWithFaultyMapToResponse(timer, counter, "2").queue(); - timer.incrementTime(10); // let time pass that equals the default delay/period - - try { - response1.get(); - fail("we should have received an exception"); - } catch (ExecutionException e) { - // what we expect - } - try { - response2.get(); - fail("we should have received an exception"); - } catch (ExecutionException e) { - // what we expect - } - - // the batch failed so no executions - assertEquals(0, counter.get()); - // but it still executed the command once - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testRequestVariableLifecycle1() throws Exception { - // simulate request lifecycle - HystrixRequestContext requestContext = HystrixRequestContext.initializeContext(); - - // do actual work - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); - timer.incrementTime(5); - Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); - timer.incrementTime(8); - // should execute here - Future response3 = new TestRequestCollapser(timer, counter, 3).queue(); - timer.incrementTime(6); - Future response4 = new TestRequestCollapser(timer, counter, 4).queue(); - timer.incrementTime(8); - // should execute here - Future response5 = new TestRequestCollapser(timer, counter, 5).queue(); - timer.incrementTime(10); - // should execute here - - // wait for all tasks to complete - assertEquals("1", response1.get()); - assertEquals("2", response2.get()); - assertEquals("3", response3.get()); - assertEquals("4", response4.get()); - assertEquals("5", response5.get()); - - // each task should have been executed 3 times - for (TestCollapserTimer.ATask t : timer.tasks) { - assertEquals(3, t.task.count.get()); - } - - System.out.println("timer.tasks.size() A: " + timer.tasks.size()); - System.out.println("tasks in test: " + timer.tasks); - - // simulate request lifecycle - requestContext.shutdown(); - - System.out.println("timer.tasks.size() B: " + timer.tasks.size()); - - HystrixRequestVariableHolder> rv = RequestCollapserFactory.getRequestVariable(new TestRequestCollapser(timer, counter, 1).getCollapserKey().name()); - - assertNotNull(rv); - // they should have all been removed as part of ThreadContext.remove() - assertEquals(0, timer.tasks.size()); - } - - @Test - public void testRequestVariableLifecycle2() throws Exception { - // simulate request lifecycle - HystrixRequestContext requestContext = HystrixRequestContext.initializeContext(); - - final TestCollapserTimer timer = new TestCollapserTimer(); - final ConcurrentLinkedQueue> responses = new ConcurrentLinkedQueue>(); - ConcurrentLinkedQueue threads = new ConcurrentLinkedQueue(); - - // kick off work (simulating a single request with multiple threads) - for (int t = 0; t < 5; t++) { - Thread th = new Thread(new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { - - @Override - public void run() { - for (int i = 0; i < 100; i++) { - responses.add(new TestRequestCollapser(timer, counter, 1).queue()); - } - } - })); - - threads.add(th); - th.start(); - } - - for (Thread th : threads) { - // wait for each thread to finish - th.join(); - } - - // we expect 5 threads * 100 responses each - assertEquals(500, responses.size()); - - for (Future f : responses) { - // they should not be done yet because the counter hasn't incremented - assertFalse(f.isDone()); - } - - timer.incrementTime(5); - Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); - timer.incrementTime(8); - // should execute here - Future response3 = new TestRequestCollapser(timer, counter, 3).queue(); - timer.incrementTime(6); - Future response4 = new TestRequestCollapser(timer, counter, 4).queue(); - timer.incrementTime(8); - // should execute here - Future response5 = new TestRequestCollapser(timer, counter, 5).queue(); - timer.incrementTime(10); - // should execute here - - // wait for all tasks to complete - for (Future f : responses) { - assertEquals("1", f.get()); - } - assertEquals("2", response2.get()); - assertEquals("3", response3.get()); - assertEquals("4", response4.get()); - assertEquals("5", response5.get()); - - // each task should have been executed 3 times - for (TestCollapserTimer.ATask t : timer.tasks) { - assertEquals(3, t.task.count.get()); - } - - // simulate request lifecycle - requestContext.shutdown(); - - HystrixRequestVariableHolder> rv = RequestCollapserFactory.getRequestVariable(new TestRequestCollapser(timer, counter, 1).getCollapserKey().name()); - - assertNotNull(rv); - // they should have all been removed as part of ThreadContext.remove() - assertEquals(0, timer.tasks.size()); - } - - /** - * Test Request scoped caching of commands so that a 2nd duplicate call doesn't execute but returns the previous Future - */ - @Test - public void testRequestCache1() { - // simulate request lifecycle - HystrixRequestContext.initializeContext(); - - final TestCollapserTimer timer = new TestCollapserTimer(); - SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", true); - SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", true); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f1.get()); - assertEquals("A", f2.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // we should have executed a command once - assertEquals(1, counter.get()); - - Future f3 = command1.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f3.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // we should still have executed only one command - assertEquals(1, counter.get()); - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - - HystrixCommand command = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[1])[0]; - assertEquals(2, command.getExecutionEvents().size()); - assertTrue(command.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); - } - - /** - * Test Request scoped caching doesn't prevent different ones from executing - */ - @Test - public void testRequestCache2() { - // simulate request lifecycle - HystrixRequestContext.initializeContext(); - - final TestCollapserTimer timer = new TestCollapserTimer(); - SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", true); - SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", true); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f1.get()); - assertEquals("B", f2.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // we should have executed a command once - assertEquals(1, counter.get()); - - Future f3 = command1.queue(); - Future f4 = command2.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f3.get()); - assertEquals("B", f4.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // we should still have executed only one command - assertEquals(1, counter.get()); - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - - HystrixCommand command = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[1])[0]; - assertEquals(2, command.getExecutionEvents().size()); - assertTrue(command.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); - } - - /** - * Test Request scoped caching with a mixture of commands - */ - @Test - public void testRequestCache3() { - // simulate request lifecycle - HystrixRequestContext.initializeContext(); - - final TestCollapserTimer timer = new TestCollapserTimer(); - SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", true); - SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", true); - SuccessfulCacheableCollapsedCommand command3 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", true); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - Future f3 = command3.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f1.get()); - assertEquals("B", f2.get()); - assertEquals("B", f3.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // we should have executed a command once - assertEquals(1, counter.get()); - - Future f4 = command1.queue(); - Future f5 = command2.queue(); - Future f6 = command3.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f4.get()); - assertEquals("B", f5.get()); - assertEquals("B", f6.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // we should still have executed only one command - assertEquals(1, counter.get()); - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - - HystrixCommand command = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[1])[0]; - assertEquals(2, command.getExecutionEvents().size()); - assertTrue(command.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); - } - - /** - * Test Request scoped caching with a mixture of commands - */ - @Test - public void testNoRequestCache3() { - // simulate request lifecycle - HystrixRequestContext.initializeContext(); - - final TestCollapserTimer timer = new TestCollapserTimer(); - SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", false); - SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", false); - SuccessfulCacheableCollapsedCommand command3 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", false); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - Future f3 = command3.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f1.get()); - assertEquals("B", f2.get()); - assertEquals("B", f3.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // we should have executed a command once - assertEquals(1, counter.get()); - - Future f4 = command1.queue(); - Future f5 = command2.queue(); - Future f6 = command3.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f4.get()); - assertEquals("B", f5.get()); - assertEquals("B", f6.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // request caching is turned off on this so we expect 2 command executions - assertEquals(2, counter.get()); - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - - // we expect to see it with SUCCESS and COLLAPSED and both - HystrixCommand commandA = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[2])[0]; - assertEquals(2, commandA.getExecutionEvents().size()); - assertTrue(commandA.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(commandA.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); - - // we expect to see it with SUCCESS and COLLAPSED and both - HystrixCommand commandB = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[2])[1]; - assertEquals(2, commandB.getExecutionEvents().size()); - assertTrue(commandB.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(commandB.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); - } - - /** - * Test that a command that throws an Exception when cached will re-throw the exception. - */ - @Test - public void testRequestCacheWithException() { - // simulate request lifecycle - HystrixRequestContext.initializeContext(); - - ConcurrentLinkedQueue>> commands = new ConcurrentLinkedQueue>>(); - - final TestCollapserTimer timer = new TestCollapserTimer(); - // pass in 'null' which will cause an NPE to be thrown - SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, null, true, commands); - SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, null, true, commands); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f1.get()); - assertEquals("A", f2.get()); - fail("exception should have been thrown"); - } catch (Exception e) { - // expected - } - - // this should be 0 because we never complete execution - assertEquals(0, counter.get()); - - // it should have executed 1 command - assertEquals(1, commands.size()); - assertTrue(commands.peek().getExecutionEvents().contains(HystrixEventType.FAILURE)); - assertTrue(commands.peek().getExecutionEvents().contains(HystrixEventType.COLLAPSED)); - - SuccessfulCacheableCollapsedCommand command3 = new SuccessfulCacheableCollapsedCommand(timer, counter, null, true, commands); - Future f3 = command3.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f3.get()); - fail("exception should have been thrown"); - } catch (Exception e) { - // expected - } - - // this should be 0 because we never complete execution - assertEquals(0, counter.get()); - - // it should still be 1 ... no new executions - assertEquals(1, commands.size()); - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - - HystrixCommand command = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[1])[0]; - assertEquals(2, command.getExecutionEvents().size()); - assertTrue(command.getExecutionEvents().contains(HystrixEventType.FAILURE)); - assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); - } - - /** - * Test that a command that times out will still be cached and when retrieved will re-throw the exception. - */ - @Test - public void testRequestCacheWithTimeout() { - // simulate request lifecycle - HystrixRequestContext.initializeContext(); - - ConcurrentLinkedQueue>> commands = new ConcurrentLinkedQueue>>(); - - final TestCollapserTimer timer = new TestCollapserTimer(); - // pass in 'null' which will cause an NPE to be thrown - SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "TIMEOUT", true, commands); - SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "TIMEOUT", true, commands); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f1.get()); - assertEquals("A", f2.get()); - fail("exception should have been thrown"); - } catch (Exception e) { - // expected - } - - // this should be 0 because we never complete execution - assertEquals(0, counter.get()); - - // it should have executed 1 command - assertEquals(1, commands.size()); - assertTrue(commands.peek().getExecutionEvents().contains(HystrixEventType.TIMEOUT)); - assertTrue(commands.peek().getExecutionEvents().contains(HystrixEventType.COLLAPSED)); - - Future f3 = command1.queue(); - - // increment past batch time so it executes - timer.incrementTime(15); - - try { - assertEquals("A", f3.get()); - fail("exception should have been thrown"); - } catch (Exception e) { - // expected - } - - // this should be 0 because we never complete execution - assertEquals(0, counter.get()); - - // it should still be 1 ... no new executions - assertEquals(1, commands.size()); - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test how the collapser behaves when the circuit is short-circuited - */ - @Test - public void testRequestWithCommandShortCircuited() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestRequestCollapserWithShortCircuitedCommand(timer, counter, "1").queue(); - Future response2 = new TestRequestCollapserWithShortCircuitedCommand(timer, counter, "2").queue(); - timer.incrementTime(10); // let time pass that equals the default delay/period - - try { - response1.get(); - fail("we should have received an exception"); - } catch (ExecutionException e) { - // e.printStackTrace(); - // what we expect - } - try { - response2.get(); - fail("we should have received an exception"); - } catch (ExecutionException e) { - // e.printStackTrace(); - // what we expect - } - - assertEquals(0, counter.get()); - // it will execute once (short-circuited) - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a Void response type - null being set as response. - * - * @throws Exception - */ - @Test - public void testVoidResponseTypeFireAndForgetCollapsing1() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestCollapserWithVoidResponseType(timer, counter, 1).queue(); - Future response2 = new TestCollapserWithVoidResponseType(timer, counter, 2).queue(); - timer.incrementTime(100); // let time pass that equals the default delay/period - - // normally someone wouldn't wait on these, but we need to make sure they do in fact return - // and not block indefinitely in case someone does call get() - assertEquals(null, response1.get()); - assertEquals(null, response2.get()); - - assertEquals(1, counter.get()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a Void response type - response never being set in mapResponseToRequest - * - * @throws Exception - */ - @Test - public void testVoidResponseTypeFireAndForgetCollapsing2() throws Exception { - TestCollapserTimer timer = new TestCollapserTimer(); - Future response1 = new TestCollapserWithVoidResponseTypeAndMissingMapResponseToRequests(timer, counter, 1).queue(); - Future response2 = new TestCollapserWithVoidResponseTypeAndMissingMapResponseToRequests(timer, counter, 2).queue(); - timer.incrementTime(100); // let time pass that equals the default delay/period - - // we will fetch one of these just so we wait for completion ... but expect an error - try { - assertEquals(null, response1.get()); - fail("expected an error as mapResponseToRequests did not set responses"); - } catch (ExecutionException e) { - assertTrue(e.getCause() instanceof IllegalStateException); - assertTrue(e.getCause().getMessage().startsWith("No response set by")); - } - - assertEquals(1, counter.get()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a Void response type with execute - response being set in mapResponseToRequest to null - * - * @throws Exception - */ - @Test - public void testVoidResponseTypeFireAndForgetCollapsing3() throws Exception { - CollapserTimer timer = new RealCollapserTimer(); - assertNull(new TestCollapserWithVoidResponseType(timer, counter, 1).execute()); - - assertEquals(1, counter.get()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - private static class TestRequestCollapser extends HystrixCollapser, String, String> { - - private final AtomicInteger count; - private final String value; - private ConcurrentLinkedQueue>> commandsExecuted; - - public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, int value) { - this(timer, counter, String.valueOf(value)); - } - - public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value) { - this(timer, counter, value, 10000, 10); - } - - public TestRequestCollapser(Scope scope, TestCollapserTimer timer, AtomicInteger counter, String value) { - this(scope, timer, counter, value, 10000, 10); - } - - public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value, ConcurrentLinkedQueue>> executionLog) { - this(timer, counter, value, 10000, 10, executionLog); - } - - public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, int value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds) { - this(timer, counter, String.valueOf(value), defaultMaxRequestsInBatch, defaultTimerDelayInMilliseconds); - } - - public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds) { - this(timer, counter, value, defaultMaxRequestsInBatch, defaultTimerDelayInMilliseconds, null); - } - - public TestRequestCollapser(Scope scope, TestCollapserTimer timer, AtomicInteger counter, String value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds) { - this(scope, timer, counter, value, defaultMaxRequestsInBatch, defaultTimerDelayInMilliseconds, null); - } - - public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds, ConcurrentLinkedQueue>> executionLog) { - this(Scope.REQUEST, timer, counter, value, defaultMaxRequestsInBatch, defaultTimerDelayInMilliseconds, executionLog); - } - - public TestRequestCollapser(Scope scope, TestCollapserTimer timer, AtomicInteger counter, String value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds, ConcurrentLinkedQueue>> executionLog) { - // use a CollapserKey based on the CollapserTimer object reference so it's unique for each timer as we don't want caching - // of properties to occur and we're using the default HystrixProperty which typically does caching - super(collapserKeyFromString(timer), scope, timer, HystrixCollapserProperties.Setter().withMaxRequestsInBatch(defaultMaxRequestsInBatch).withTimerDelayInMilliseconds(defaultTimerDelayInMilliseconds)); - this.count = counter; - this.value = value; - this.commandsExecuted = executionLog; - } - - @Override - public String getRequestArgument() { - return value; - } - - @Override - public HystrixCommand> createCommand(final Collection> requests) { - /* return a mocked command */ - HystrixCommand> command = new TestCollapserCommand(requests); - if (commandsExecuted != null) { - commandsExecuted.add(command); - } - return command; - } - - @Override - public void mapResponseToRequests(List batchResponse, Collection> requests) { - // count how many times a batch is executed (this method is executed once per batch) - System.out.println("increment count: " + count.incrementAndGet()); - - // for simplicity I'll assume it's a 1:1 mapping between lists ... in real implementations they often need to index to maps - // to allow random access as the response size does not match the request size - if (batchResponse.size() != requests.size()) { - throw new RuntimeException("lists don't match in size => " + batchResponse.size() + " : " + requests.size()); - } - int i = 0; - for (CollapsedRequest request : requests) { - request.setResponse(batchResponse.get(i++)); - } - - } - - } - - /** - * Shard on the artificially provided 'type' variable. - */ - private static class TestShardedRequestCollapser extends TestRequestCollapser { - - public TestShardedRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value) { - super(timer, counter, value); - } - - @Override - protected Collection>> shardRequests(Collection> requests) { - Collection> typeA = new ArrayList>(); - Collection> typeB = new ArrayList>(); - - for (CollapsedRequest request : requests) { - if (request.getArgument().endsWith("a")) { - typeA.add(request); - } else if (request.getArgument().endsWith("b")) { - typeB.add(request); - } - } - - ArrayList>> shards = new ArrayList>>(); - shards.add(typeA); - shards.add(typeB); - return shards; - } - - } - - /** - * Test the global scope - */ - private static class TestGloballyScopedRequestCollapser extends TestRequestCollapser { - - public TestGloballyScopedRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value) { - super(Scope.GLOBAL, timer, counter, value); - } - - } - - /** - * Throw an exception when creating a command. - */ - private static class TestRequestCollapserWithFaultyCreateCommand extends TestRequestCollapser { - - public TestRequestCollapserWithFaultyCreateCommand(TestCollapserTimer timer, AtomicInteger counter, String value) { - super(timer, counter, value); - } - - @Override - public HystrixCommand> createCommand(Collection> requests) { - throw new RuntimeException("some failure"); - } - - } - - /** - * Throw an exception when creating a command. - */ - private static class TestRequestCollapserWithShortCircuitedCommand extends TestRequestCollapser { - - public TestRequestCollapserWithShortCircuitedCommand(TestCollapserTimer timer, AtomicInteger counter, String value) { - super(timer, counter, value); - } - - @Override - public HystrixCommand> createCommand(Collection> requests) { - // args don't matter as it's short-circuited - return new ShortCircuitedCommand(); - } - - } - - /** - * Throw an exception when mapToResponse is invoked - */ - private static class TestRequestCollapserWithFaultyMapToResponse extends TestRequestCollapser { - - public TestRequestCollapserWithFaultyMapToResponse(TestCollapserTimer timer, AtomicInteger counter, String value) { - super(timer, counter, value); - } - - @Override - public void mapResponseToRequests(List batchResponse, Collection> requests) { - // pretend we blow up with an NPE - throw new NullPointerException("batchResponse was null and we blew up"); - } - - } - - private static class TestCollapserCommand extends TestHystrixCommand> { - - private final Collection> requests; - - TestCollapserCommand(Collection> requests) { - super(testPropsBuilder().setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(50))); - this.requests = requests; - } - - @Override - protected List run() { - System.out.println(">>> TestCollapserCommand run() ... batch size: " + requests.size()); - // simulate a batch request - ArrayList response = new ArrayList(); - for (CollapsedRequest request : requests) { - if (request.getArgument() == null) { - throw new NullPointerException("Simulated Error"); - } - if (request.getArgument() == "TIMEOUT") { - try { - Thread.sleep(200); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - response.add(request.getArgument()); - } - return response; - } - - } - - /** - * A Command implementation that supports caching. - */ - private static class SuccessfulCacheableCollapsedCommand extends TestRequestCollapser { - - private final boolean cacheEnabled; - - public SuccessfulCacheableCollapsedCommand(TestCollapserTimer timer, AtomicInteger counter, String value, boolean cacheEnabled) { - super(timer, counter, value); - this.cacheEnabled = cacheEnabled; - } - - public SuccessfulCacheableCollapsedCommand(TestCollapserTimer timer, AtomicInteger counter, String value, boolean cacheEnabled, ConcurrentLinkedQueue>> executionLog) { - super(timer, counter, value, executionLog); - this.cacheEnabled = cacheEnabled; - } - - @Override - public String getCacheKey() { - if (cacheEnabled) - return "aCacheKey_" + super.value; - else - return null; - } - } - - private static class ShortCircuitedCommand extends HystrixCommand> { - - protected ShortCircuitedCommand() { - super(HystrixCommand.Setter.withGroupKey( - HystrixCommandGroupKey.Factory.asKey("shortCircuitedCommand")) - .andCommandPropertiesDefaults(HystrixCommandProperties.Setter - .getUnitTestPropertiesSetter() - .withCircuitBreakerForceOpen(true))); - } - - @Override - protected List run() throws Exception { - System.out.println("*** execution (this shouldn't happen)"); - // this won't ever get called as we're forcing short-circuiting - ArrayList values = new ArrayList(); - values.add("hello"); - return values; - } - - } - - private static class FireAndForgetCommand extends HystrixCommand { - - protected FireAndForgetCommand(List values) { - super(HystrixCommand.Setter.withGroupKey( - HystrixCommandGroupKey.Factory.asKey("fireAndForgetCommand")) - .andCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter())); - } - - @Override - protected Void run() throws Exception { - System.out.println("*** FireAndForgetCommand execution: " + Thread.currentThread()); - return null; - } - - } - - private static class TestCollapserTimer implements CollapserTimer { - - private final ConcurrentLinkedQueue tasks = new ConcurrentLinkedQueue(); - - @Override - public Reference addListener(final TimerListener collapseTask) { - System.out.println("add listener: " + collapseTask); - tasks.add(new ATask(new TestTimerListener(collapseTask))); - - /** - * This is a hack that overrides 'clear' of a WeakReference to match the required API - * but then removes the strong-reference we have inside 'tasks'. - *

- * We do this so our unit tests know if the WeakReference is cleared correctly, and if so then the ATack is removed from 'tasks' - */ - return new SoftReference(collapseTask) { - @Override - public void clear() { - System.out.println("tasks: " + tasks); - System.out.println("**** clear TimerListener: tasks.size => " + tasks.size()); - // super.clear(); - for (ATask t : tasks) { - if (t.task.actualListener.equals(collapseTask)) { - tasks.remove(t); - } - } - } - - }; - } - - /** - * Increment time by X. Note that incrementing by multiples of delay or period time will NOT execute multiple times. - *

- * You must call incrementTime multiple times each increment being larger than 'period' on subsequent calls to cause multiple executions. - *

- * This is because executing multiple times in a tight-loop would not achieve the correct behavior, such as batching, since it will all execute "now" not after intervals of time. - * - * @param timeInMilliseconds - */ - public synchronized void incrementTime(int timeInMilliseconds) { - for (ATask t : tasks) { - t.incrementTime(timeInMilliseconds); - } - } - - private static class ATask { - final TestTimerListener task; - final int delay = 10; - - // our relative time that we'll use - volatile int time = 0; - volatile int executionCount = 0; - - private ATask(TestTimerListener task) { - this.task = task; - } - - public synchronized void incrementTime(int timeInMilliseconds) { - time += timeInMilliseconds; - if (task != null) { - if (executionCount == 0) { - System.out.println("ExecutionCount 0 => Time: " + time + " Delay: " + delay); - if (time >= delay) { - // first execution, we're past the delay time - executeTask(); - } - } else { - System.out.println("ExecutionCount 1+ => Time: " + time + " Delay: " + delay); - if (time >= delay) { - // subsequent executions, we're past the interval time - executeTask(); - } - } - } - } - - private synchronized void executeTask() { - System.out.println("Executing task ..."); - task.tick(); - this.time = 0; // we reset time after each execution - this.executionCount++; - System.out.println("executionCount: " + executionCount); - } - } - - } - - private static class TestTimerListener implements TimerListener { - - private final TimerListener actualListener; - private final AtomicInteger count = new AtomicInteger(); - - public TestTimerListener(TimerListener actual) { - this.actualListener = actual; - } - - @Override - public void tick() { - count.incrementAndGet(); - actualListener.tick(); - } - - @Override - public int getIntervalTimeInMilliseconds() { - return 10; - } - - } - - private static HystrixCollapserKey collapserKeyFromString(final Object o) { - return new HystrixCollapserKey() { - - @Override - public String name() { - return String.valueOf(o); - } - - }; - } - - private static class TestCollapserWithVoidResponseType extends HystrixCollapser { - - private final AtomicInteger count; - private final Integer value; - - public TestCollapserWithVoidResponseType(CollapserTimer timer, AtomicInteger counter, int value) { - super(collapserKeyFromString(timer), Scope.REQUEST, timer, HystrixCollapserProperties.Setter().withMaxRequestsInBatch(1000).withTimerDelayInMilliseconds(50)); - this.count = counter; - this.value = value; - } - - @Override - public Integer getRequestArgument() { - return value; - } - - @Override - protected HystrixCommand createCommand(Collection> requests) { - - ArrayList args = new ArrayList(); - for (CollapsedRequest request : requests) { - args.add(request.getArgument()); - } - return new FireAndForgetCommand(args); - } - - @Override - protected void mapResponseToRequests(Void batchResponse, Collection> requests) { - count.incrementAndGet(); - for (CollapsedRequest r : requests) { - r.setResponse(null); - } - } - - } - - private static class TestCollapserWithVoidResponseTypeAndMissingMapResponseToRequests extends HystrixCollapser { - - private final AtomicInteger count; - private final Integer value; - - public TestCollapserWithVoidResponseTypeAndMissingMapResponseToRequests(CollapserTimer timer, AtomicInteger counter, int value) { - super(collapserKeyFromString(timer), Scope.REQUEST, timer, HystrixCollapserProperties.Setter().withMaxRequestsInBatch(1000).withTimerDelayInMilliseconds(50)); - this.count = counter; - this.value = value; - } - - @Override - public Integer getRequestArgument() { - return value; - } - - @Override - protected HystrixCommand createCommand(Collection> requests) { - - ArrayList args = new ArrayList(); - for (CollapsedRequest request : requests) { - args.add(request.getArgument()); - } - return new FireAndForgetCommand(args); - } - - @Override - protected void mapResponseToRequests(Void batchResponse, Collection> requests) { - count.incrementAndGet(); - } - - } - } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java index 81814dedc..cf50aa866 100755 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommand.java @@ -15,137 +15,38 @@ */ package com.netflix.hystrix; -import static org.junit.Assert.*; - -import java.io.IOException; -import java.lang.ref.Reference; -import java.lang.ref.SoftReference; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; import javax.annotation.concurrent.NotThreadSafe; import javax.annotation.concurrent.ThreadSafe; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import rx.Observable; -import rx.Observable.OnSubscribeFunc; -import rx.Observer; +import rx.Observable.OnSubscribe; import rx.Scheduler; -import rx.Subscription; -import rx.concurrency.Schedulers; -import rx.operators.SafeObservableSubscription; -import rx.subjects.ReplaySubject; -import rx.subscriptions.Subscriptions; -import rx.util.functions.Action0; -import rx.util.functions.Action1; -import rx.util.functions.Func1; -import rx.util.functions.Func2; +import rx.Subscriber; +import rx.schedulers.Schedulers; -import com.netflix.config.ConfigurationManager; -import com.netflix.hystrix.HystrixCircuitBreaker.NoOpCircuitBreaker; -import com.netflix.hystrix.HystrixCircuitBreaker.TestCircuitBreaker; import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; +import com.netflix.hystrix.HystrixExecutableBase.ObservableCommand; +import com.netflix.hystrix.HystrixExecutableBase.TryableSemaphore; import com.netflix.hystrix.exception.HystrixBadRequestException; import com.netflix.hystrix.exception.HystrixRuntimeException; -import com.netflix.hystrix.exception.HystrixRuntimeException.FailureType; -import com.netflix.hystrix.strategy.HystrixPlugins; -import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; -import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategyDefault; -import com.netflix.hystrix.strategy.concurrency.HystrixContextCallable; -import com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable; -import com.netflix.hystrix.strategy.concurrency.HystrixContextScheduler; -import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; -import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifier; import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook; -import com.netflix.hystrix.strategy.metrics.HystrixMetricsPublisherFactory; -import com.netflix.hystrix.strategy.properties.HystrixPropertiesFactory; import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; -import com.netflix.hystrix.strategy.properties.HystrixProperty; -import com.netflix.hystrix.util.ExceptionThreadingUtility; -import com.netflix.hystrix.util.HystrixRollingNumberEvent; -import com.netflix.hystrix.util.HystrixTimer; -import com.netflix.hystrix.util.HystrixTimer.TimerListener; /** * Used to wrap code that will execute potentially risky functionality (typically meaning a service call over the network) * with fault and latency tolerance, statistics and performance metrics capture, circuit breaker and bulkhead functionality. + * This command is essentially a blocking command but provides an Observable facade if used with observe() * * @param * the return type */ @ThreadSafe -public abstract class HystrixCommand implements HystrixExecutable { - - private static final Logger logger = LoggerFactory.getLogger(HystrixCommand.class); - - private final HystrixCircuitBreaker circuitBreaker; - private final HystrixThreadPool threadPool; - private final HystrixThreadPoolKey threadPoolKey; - private final HystrixCommandProperties properties; - private final HystrixCommandMetrics metrics; - - /* result of execution (if this command instance actually gets executed, which may not occur due to request caching) */ - private volatile ExecutionResult executionResult = ExecutionResult.EMPTY; - - /* If this command executed and timed-out */ - private final AtomicReference isCommandTimedOut = new AtomicReference(TimedOutStatus.NOT_EXECUTED); - private final AtomicBoolean isExecutionComplete = new AtomicBoolean(false); - private final AtomicBoolean isExecutedInThread = new AtomicBoolean(false); - - private static enum TimedOutStatus {NOT_EXECUTED, COMPLETED, TIMED_OUT}; - - private final HystrixCommandKey commandKey; - private final HystrixCommandGroupKey commandGroup; - - /* FALLBACK Semaphore */ - private final TryableSemaphore fallbackSemaphoreOverride; - /* each circuit has a semaphore to restrict concurrent fallback execution */ - private static final ConcurrentHashMap fallbackSemaphorePerCircuit = new ConcurrentHashMap(); - /* END FALLBACK Semaphore */ +public abstract class HystrixCommand implements HystrixExecutable, HystrixExecutableInfo { - /* EXECUTION Semaphore */ - private final TryableSemaphore executionSemaphoreOverride; - /* each circuit has a semaphore to restrict concurrent fallback execution */ - private static final ConcurrentHashMap executionSemaphorePerCircuit = new ConcurrentHashMap(); - /* END EXECUTION Semaphore */ - - private final AtomicReference> timeoutTimer = new AtomicReference>(); - - private AtomicBoolean started = new AtomicBoolean(); - private volatile long invocationStartTime = -1; - - /** - * Instance of RequestCache logic - */ - private final HystrixRequestCache requestCache; - - /** - * Plugin implementations - */ - private final HystrixEventNotifier eventNotifier; - private final HystrixConcurrencyStrategy concurrencyStrategy; - private final HystrixCommandExecutionHook executionHook; + private final HystrixCommandFromObservableCommand observableCommand; /** * Construct a {@link HystrixCommand} with defined {@link HystrixCommandGroupKey}. @@ -188,143 +89,141 @@ protected HystrixCommand(Setter setter) { *

* Most of the args will revert to a valid default if 'null' is passed in. */ - private HystrixCommand(HystrixCommandGroupKey group, HystrixCommandKey key, HystrixThreadPoolKey threadPoolKey, HystrixCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, + /* package for testing */HystrixCommand(HystrixCommandGroupKey group, HystrixCommandKey key, HystrixThreadPoolKey threadPoolKey, HystrixCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, HystrixCommandProperties.Setter commandPropertiesDefaults, HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults, HystrixCommandMetrics metrics, TryableSemaphore fallbackSemaphore, TryableSemaphore executionSemaphore, HystrixPropertiesStrategy propertiesStrategy, HystrixCommandExecutionHook executionHook) { - /* - * CommandGroup initialization - */ - if (group == null) { - throw new IllegalStateException("HystrixCommandGroup can not be NULL"); - } else { - this.commandGroup = group; - } /* * CommandKey initialization */ + HystrixCommandKey commandKey = null; if (key == null || key.name().trim().equals("")) { - final String keyName = getDefaultNameFromClass(getClass()); - this.commandKey = HystrixCommandKey.Factory.asKey(keyName); + // use the HystrixCommand class rather than this ObservableCommand class if we have it + final String keyName = HystrixExecutableBase.getDefaultNameFromClass(getClass()); + commandKey = HystrixCommandKey.Factory.asKey(keyName); } else { - this.commandKey = key; + commandKey = key; } - /* - * Properties initialization + this.observableCommand = new HystrixCommandFromObservableCommand(this, group, commandKey, threadPoolKey, circuitBreaker, threadPool, commandPropertiesDefaults, threadPoolPropertiesDefaults, metrics, fallbackSemaphore, executionSemaphore, propertiesStrategy, executionHook); + } + + /** + * Fluent interface for arguments to the {@link HystrixObservableCommand} constructor. + *

+ * The required arguments are set via the 'with' factory method and optional arguments via the 'and' chained methods. + *

+ * Example: + * 

 {@code
+     *  Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("GroupName"))
+                .andCommandKey(HystrixCommandKey.Factory.asKey("CommandName"))
+                .andEventNotifier(notifier);
+     * }
+ */ + @NotThreadSafe + public static class Setter { + + protected final HystrixCommandGroupKey groupKey; + protected HystrixCommandKey commandKey; + protected HystrixThreadPoolKey threadPoolKey; + protected HystrixCommandProperties.Setter commandPropertiesDefaults; + protected HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults; + + /** + * Setter factory method containing required values. + *

+ * All optional arguments can be set via the chained methods. + * + * @param groupKey + * {@link HystrixCommandGroupKey} used to group together multiple {@link HystrixObservableCommand} objects. + *

+ * The {@link HystrixCommandGroupKey} is used to represent a common relationship between commands. For example, a library or team name, the system all related commands interace + * with, + * common business purpose etc. */ - if (propertiesStrategy == null) { - this.properties = HystrixPropertiesFactory.getCommandProperties(this.commandKey, commandPropertiesDefaults); - } else { - // used for unit testing - this.properties = propertiesStrategy.getCommandProperties(this.commandKey, commandPropertiesDefaults); + protected Setter(HystrixCommandGroupKey groupKey) { + this.groupKey = groupKey; } - /* - * ThreadPoolKey - * - * This defines which thread-pool this command should run on. - * - * It uses the HystrixThreadPoolKey if provided, then defaults to use HystrixCommandGroup. + /** + * Setter factory method with required values. + *

+ * All optional arguments can be set via the chained methods. * - * It can then be overridden by a property if defined so it can be changed at runtime. + * @param groupKey + * {@link HystrixCommandGroupKey} used to group together multiple {@link HystrixObservableCommand} objects. + *

+ * The {@link HystrixCommandGroupKey} is used to represent a common relationship between commands. For example, a library or team name, the system all related commands interace + * with, + * common business purpose etc. */ - if (this.properties.executionIsolationThreadPoolKeyOverride().get() == null) { - // we don't have a property overriding the value so use either HystrixThreadPoolKey or HystrixCommandGroup - if (threadPoolKey == null) { - /* use HystrixCommandGroup if HystrixThreadPoolKey is null */ - this.threadPoolKey = HystrixThreadPoolKey.Factory.asKey(commandGroup.name()); - } else { - this.threadPoolKey = threadPoolKey; - } - } else { - // we have a property defining the thread-pool so use it instead - this.threadPoolKey = HystrixThreadPoolKey.Factory.asKey(properties.executionIsolationThreadPoolKeyOverride().get()); + public static Setter withGroupKey(HystrixCommandGroupKey groupKey) { + return new Setter(groupKey); } - /* strategy: HystrixEventNotifier */ - this.eventNotifier = HystrixPlugins.getInstance().getEventNotifier(); - - /* strategy: HystrixConcurrentStrategy */ - this.concurrencyStrategy = HystrixPlugins.getInstance().getConcurrencyStrategy(); - - /* - * Metrics initialization + /** + * @param commandKey + * {@link HystrixCommandKey} used to identify a {@link HystrixObservableCommand} instance for statistics, circuit-breaker, properties, etc. + *

+ * By default this will be derived from the instance class name. + *

+ * NOTE: Every unique {@link HystrixCommandKey} will result in new instances of {@link HystrixCircuitBreaker}, {@link HystrixCommandMetrics} and {@link HystrixCommandProperties}. + * Thus, + * the number of variants should be kept to a finite and reasonable number to avoid high-memory usage or memory leacks. + *

+ * Hundreds of keys is fine, tens of thousands is probably not. + * @return Setter for fluent interface via method chaining */ - if (metrics == null) { - this.metrics = HystrixCommandMetrics.getInstance(this.commandKey, this.commandGroup, this.properties); - } else { - this.metrics = metrics; + public Setter andCommandKey(HystrixCommandKey commandKey) { + this.commandKey = commandKey; + return this; } - /* - * CircuitBreaker initialization + /** + * @param threadPoolKey + * {@link HystrixThreadPoolKey} used to define which thread-pool this command should run in (when configured to run on separate threads via + * {@link HystrixCommandProperties#executionIsolationStrategy()}). + *

+ * By default this is derived from the {@link HystrixCommandGroupKey} but if injected this allows multiple commands to have the same {@link HystrixCommandGroupKey} but different + * thread-pools. + * @return Setter for fluent interface via method chaining */ - if (this.properties.circuitBreakerEnabled().get()) { - if (circuitBreaker == null) { - // get the default implementation of HystrixCircuitBreaker - this.circuitBreaker = HystrixCircuitBreaker.Factory.getInstance(this.commandKey, this.commandGroup, this.properties, this.metrics); - } else { - this.circuitBreaker = circuitBreaker; - } - } else { - this.circuitBreaker = new NoOpCircuitBreaker(); + public Setter andThreadPoolKey(HystrixThreadPoolKey threadPoolKey) { + this.threadPoolKey = threadPoolKey; + return this; } - /* strategy: HystrixMetricsPublisherCommand */ - HystrixMetricsPublisherFactory.createOrRetrievePublisherForCommand(this.commandKey, this.commandGroup, this.metrics, this.circuitBreaker, this.properties); - - /* strategy: HystrixCommandExecutionHook */ - if (executionHook == null) { - this.executionHook = HystrixPlugins.getInstance().getCommandExecutionHook(); - } else { - // used for unit testing - this.executionHook = executionHook; + /** + * Optional + * + * @param commandPropertiesDefaults + * {@link HystrixCommandProperties.Setter} with property overrides for this specific instance of {@link HystrixObservableCommand}. + *

+ * See the {@link HystrixPropertiesStrategy} JavaDocs for more information on properties and order of precedence. + * @return Setter for fluent interface via method chaining + */ + public Setter andCommandPropertiesDefaults(HystrixCommandProperties.Setter commandPropertiesDefaults) { + this.commandPropertiesDefaults = commandPropertiesDefaults; + return this; } - /* - * ThreadPool initialization + /** + * Optional + * + * @param threadPoolPropertiesDefaults + * {@link HystrixThreadPoolProperties.Setter} with property overrides for the {@link HystrixThreadPool} used by this specific instance of {@link HystrixObservableCommand}. + *

+ * See the {@link HystrixPropertiesStrategy} JavaDocs for more information on properties and order of precedence. + * @return Setter for fluent interface via method chaining */ - if (threadPool == null) { - // get the default implementation of HystrixThreadPool - this.threadPool = HystrixThreadPool.Factory.getInstance(this.threadPoolKey, threadPoolPropertiesDefaults); - } else { - this.threadPool = threadPool; + public Setter andThreadPoolPropertiesDefaults(HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults) { + this.threadPoolPropertiesDefaults = threadPoolPropertiesDefaults; + return this; } - /* fallback semaphore override if applicable */ - this.fallbackSemaphoreOverride = fallbackSemaphore; - - /* execution semaphore override if applicable */ - this.executionSemaphoreOverride = executionSemaphore; - - /* setup the request cache for this instance */ - this.requestCache = HystrixRequestCache.getInstance(this.commandKey, this.concurrencyStrategy); - } - - private static String getDefaultNameFromClass(@SuppressWarnings("rawtypes") Class cls) { - String fromCache = defaultNameCache.get(cls); - if (fromCache != null) { - return fromCache; - } - // generate the default - // default HystrixCommandKey to use if the method is not overridden - String name = cls.getSimpleName(); - if (name.equals("")) { - // we don't have a SimpleName (anonymous inner class) so use the full class name - name = cls.getName(); - name = name.substring(name.lastIndexOf('.') + 1, name.length()); - } - defaultNameCache.put(cls, name); - return name; } - // this is a micro-optimization but saves about 1-2microseconds (on 2011 MacBook Pro) - // on the repetitive string processing that will occur on the same classes over and over again - @SuppressWarnings("rawtypes") - private static ConcurrentHashMap, String> defaultNameCache = new ConcurrentHashMap, String>(); - /** * Implement this method with code to be executed when {@link #execute()} or {@link #queue()} are invoked. * @@ -353,365 +252,18 @@ protected R getFallback() { } /** - * @return {@link HystrixCommandGroupKey} used to group together multiple {@link HystrixCommand} objects. - *

- * The {@link HystrixCommandGroupKey} is used to represent a common relationship between commands. For example, a library or team name, the system all related commands interace with, - * common business purpose etc. - */ - public HystrixCommandGroupKey getCommandGroup() { - return commandGroup; - } - - /** - * @return {@link HystrixCommandKey} identifying this command instance for statistics, circuit-breaker, properties, etc. - */ - public HystrixCommandKey getCommandKey() { - return commandKey; - } - - /** - * @return {@link HystrixThreadPoolKey} identifying which thread-pool this command uses (when configured to run on separate threads via - * {@link HystrixCommandProperties#executionIsolationStrategy()}). - */ - public HystrixThreadPoolKey getThreadPoolKey() { - return threadPoolKey; - } - - /* package */HystrixCircuitBreaker getCircuitBreaker() { - return circuitBreaker; - } - - /** - * The {@link HystrixCommandMetrics} associated with this {@link HystrixCommand} instance. - * - * @return HystrixCommandMetrics - */ - public HystrixCommandMetrics getMetrics() { - return metrics; - } - - /** - * The {@link HystrixCommandProperties} associated with this {@link HystrixCommand} instance. - * - * @return HystrixCommandProperties - */ - public HystrixCommandProperties getProperties() { - return properties; - } - - /** - * Allow the Collapser to mark this command instance as being used for a collapsed request and how many requests were collapsed. - * - * @param sizeOfBatch - */ - /* package */void markAsCollapsedCommand(int sizeOfBatch) { - getMetrics().markCollapsed(sizeOfBatch); - executionResult = executionResult.addEvents(HystrixEventType.COLLAPSED); - } - - /** - * Used for synchronous execution of command. - * - * @return R - * Result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. - * @throws HystrixRuntimeException - * if a failure occurs and a fallback cannot be retrieved - * @throws HystrixBadRequestException - * if invalid arguments or state were used representing a user failure, not a system failure - * @throws IllegalStateException - * if invoked more than once - */ - public R execute() { - try { - return queue().get(); - } catch (Exception e) { - throw decomposeException(e); - } - } - - /** - * Used for asynchronous execution of command. - *

- * This will queue up the command on the thread pool and return an {@link Future} to get the result once it completes. - *

- * NOTE: If configured to not run in a separate thread, this will have the same effect as {@link #execute()} and will block. - *

- * We don't throw an exception but just flip to synchronous execution so code doesn't need to change in order to switch a command from running on a separate thread to the calling thread. - * - * @return {@code Future} Result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. - * @throws HystrixRuntimeException - * if a fallback does not exist - *

- *

    - *
  • via {@code Future.get()} in {@link ExecutionException#getCause()} if a failure occurs
    • - *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • - *
- * @throws HystrixBadRequestException - * via {@code Future.get()} in {@link ExecutionException#getCause()} if invalid arguments or state were used representing a user failure, not a system failure - * @throws IllegalStateException - * if invoked more than once - */ - public Future queue() { - /* - * --- Schedulers.immediate() - * - * We use the 'immediate' schedule since Future.get() is blocking so we don't want to bother doing the callback to the Future on a separate thread - * as we don't need to separate the Hystrix thread from user threads since they are already providing it via the Future.get() call. - * - * --- performAsyncTimeout: false - * - * We pass 'false' to tell the Observable we will block on it so it doesn't schedule an async timeout. - * - * This optimizes for using the calling thread to do the timeout rather than scheduling another thread. - * - * In a tight-loop of executing commands this optimization saves a few microseconds per execution. - * It also just makes no sense to use a separate thread to timeout the command when the calling thread - * is going to sit waiting on it. - */ - final ObservableCommand o = toObservable(Schedulers.immediate(), false); - final Future f = o.toBlockingObservable().toFuture(); - - /* special handling of error states that throw immediately */ - if (f.isDone()) { - try { - f.get(); - return f; - } catch (Exception e) { - RuntimeException re = decomposeException(e); - if (re instanceof HystrixBadRequestException) { - return f; - } else if (re instanceof HystrixRuntimeException) { - HystrixRuntimeException hre = (HystrixRuntimeException) re; - if (hre.getFailureType() == FailureType.COMMAND_EXCEPTION || hre.getFailureType() == FailureType.TIMEOUT) { - // we don't throw these types from queue() only from queue().get() as they are execution errors - return f; - } else { - // these are errors we throw from queue() as they as rejection type errors - throw hre; - } - } else { - throw re; - } - } - } - - return new Future() { - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return f.cancel(mayInterruptIfRunning); - } - - @Override - public boolean isCancelled() { - return f.isCancelled(); - } - - @Override - public boolean isDone() { - return f.isDone(); - } - - @Override - public R get() throws InterruptedException, ExecutionException { - return performBlockingGetWithTimeout(o, f); - } - - /** - * --- Non-Blocking Timeout (performAsyncTimeout:true) --- - * - * When 'toObservable' is done with non-blocking timeout then timeout functionality is provided - * by a separate HystrixTimer thread that will "tick" and cancel the underlying async Future inside the Observable. - * - * This method allows stealing that responsibility and letting the thread that's going to block anyways - * do the work to reduce pressure on the HystrixTimer. - * - * Blocking via queue().get() on a non-blocking timeout will work it's just less efficient - * as it involves an extra thread and cancels the scheduled action that does the timeout. - * - * --- Blocking Timeout (performAsyncTimeout:false) --- - * - * When blocking timeout is assumed (default behavior for execute/queue flows) then the async - * timeout will not have been scheduled and this will wait in a blocking manner and if a timeout occurs - * trigger the timeout logic that comes from inside the Observable/Observer. - * - * - * --- Examples - * - * Stack for timeout with performAsyncTimeout=false (note the calling thread via get): - * - * at com.netflix.hystrix.HystrixCommand$TimeoutObservable$1$1.tick(HystrixCommand.java:788) - * at com.netflix.hystrix.HystrixCommand$1.performBlockingGetWithTimeout(HystrixCommand.java:536) - * at com.netflix.hystrix.HystrixCommand$1.get(HystrixCommand.java:484) - * at com.netflix.hystrix.HystrixCommand.execute(HystrixCommand.java:413) - * - * - * Stack for timeout with performAsyncTimeout=true (note the HystrixTimer involved): - * - * at com.netflix.hystrix.HystrixCommand$TimeoutObservable$1$1.tick(HystrixCommand.java:799) - * at com.netflix.hystrix.util.HystrixTimer$1.run(HystrixTimer.java:101) - * - * - * - * @param o - * @param f - * @throws InterruptedException - * @throws ExecutionException - */ - protected R performBlockingGetWithTimeout(final ObservableCommand o, final Future f) throws InterruptedException, ExecutionException { - // shortcut if already done - if (f.isDone()) { - return f.get(); - } - - // it's still working so proceed with blocking/timeout logic - HystrixCommand originalCommand = o.getCommand(); - /** - * One thread will get the timeoutTimer if it's set and clear it then do blocking timeout. - *

- * If non-blocking timeout was scheduled this will unschedule it. If it wasn't scheduled it is basically - * a no-op but fits the same interface so blocking and non-blocking flows both work the same. - *

- * This "originalCommand" concept exists because of request caching. We only do the work and timeout logic - * on the original, not the cached responses. However, whichever the first thread is that comes in to block - * will be the one who performs the timeout logic. - *

- * If request caching is disabled then it will always go into here. - */ - if (originalCommand != null) { - Reference timer = originalCommand.timeoutTimer.getAndSet(null); - if (timer != null) { - /** - * If an async timeout was scheduled then: - * - * - We are going to clear the Reference so the scheduler threads stop managing the timeout - * and we'll take over instead since we're going to be blocking on it anyways. - * - * - Other threads (since we won the race) will just wait on the normal Future which will release - * once the Observable is marked as completed (which may come via timeout) - * - * If an async timeout was not scheduled: - * - * - We go through the same flow as we receive the same interfaces just the "timer.clear()" will do nothing. - */ - // get the timer we'll use to perform the timeout - TimerListener l = timer.get(); - // remove the timer from the scheduler - timer.clear(); - - // determine how long we should wait for, taking into account time since work started - // and when this thread came in to block. If invocationTime hasn't been set then assume time remaining is entire timeout value - // as this maybe a case of multiple threads trying to run this command in which one thread wins but even before the winning thread is able to set - // the starttime another thread going via the Cached command route gets here first. - long timeout = originalCommand.properties.executionIsolationThreadTimeoutInMilliseconds().get(); - long timeRemaining = timeout; - long currTime = System.currentTimeMillis(); - if (originalCommand.invocationStartTime != -1) { - timeRemaining = (originalCommand.invocationStartTime - + originalCommand.properties.executionIsolationThreadTimeoutInMilliseconds().get()) - - currTime; - - } - if (timeRemaining > 0) { - // we need to block with the calculated timeout - try { - return f.get(timeRemaining, TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - if (l != null) { - // this perform the timeout logic on the Observable/Observer - l.tick(); - } - } - } else { - // this means it should have already timed out so do so if it is not completed - if (!f.isDone()) { - if (l != null) { - l.tick(); - } - } - } - } - } - // other threads will block until the "l.tick" occurs and releases the underlying Future. - return f.get(); - } - - @Override - public R get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - return get(); - } - - }; - - } - - /** - * Take an Exception and determine whether to throw it, its cause or a new HystrixRuntimeException. - *

- * This will only throw an HystrixRuntimeException, HystrixBadRequestException or IllegalStateException - * - * @param e - * @return HystrixRuntimeException, HystrixBadRequestException or IllegalStateException - */ - protected RuntimeException decomposeException(Exception e) { - if (e instanceof IllegalStateException) { - return (IllegalStateException) e; - } - if (e instanceof HystrixBadRequestException) { - return (HystrixBadRequestException) e; - } - if (e.getCause() instanceof HystrixBadRequestException) { - return (HystrixBadRequestException) e.getCause(); - } - if (e instanceof HystrixRuntimeException) { - return (HystrixRuntimeException) e; - } - // if we have an exception we know about we'll throw it directly without the wrapper exception - if (e.getCause() instanceof HystrixRuntimeException) { - return (HystrixRuntimeException) e.getCause(); - } - // we don't know what kind of exception this is so create a generic message and throw a new HystrixRuntimeException - String message = getLogMessagePrefix() + " failed while executing."; - logger.debug(message, e); // debug only since we're throwing the exception and someone higher will do something with it - return new HystrixRuntimeException(FailureType.COMMAND_EXCEPTION, this.getClass(), message, e, null); - } - - /** - * Used for asynchronous execution of command with a callback by subscribing to the {@link Observable}. - *

- * This eagerly starts execution of the command the same as {@link #queue()} and {@link #execute()}. - * A lazy {@link Observable} can be obtained from {@link #toObservable()}. + * Key to be used for request caching. *

- * Callback Scheduling + * By default this returns null which means "do not cache". *

- *

    - *
  • When using {@link ExecutionIsolationStrategy#THREAD} this defaults to using {@link Schedulers#threadPoolForComputation()} for callbacks.
    • - *
  • When using {@link ExecutionIsolationStrategy#SEMAPHORE} this defaults to using {@link Schedulers#immediate()} for callbacks.
    • - *
- * Use {@link #toObservable(rx.Scheduler)} to schedule the callback differently. + * To enable caching override this method and return a string key uniquely representing the state of a command instance. *

- * See https://github.com/Netflix/RxJava/wiki for more information. + * If multiple command instances in the same request scope match keys then only the first will be executed and all others returned from cache. * - * @return {@code Observable} that executes and calls back with the result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. - * @throws HystrixRuntimeException - * if a fallback does not exist - *

- *

    - *
  • via {@code Observer#onError} if a failure occurs
    • - *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • - *
- * @throws HystrixBadRequestException - * via {@code Observer#onError} if invalid arguments or state were used representing a user failure, not a system failure - * @throws IllegalStateException - * if invoked more than once + * @return cacheKey */ - public Observable observe() { - // us a ReplaySubject to buffer the eagerly subscribed-to Observable - ReplaySubject subject = ReplaySubject.create(); - // eagerly kick off subscription - toObservable().subscribe(subject); - // return the subject that can be subscribed to later while the execution has already started - return subject; + protected String getCacheKey() { + return null; } /** @@ -741,8 +293,8 @@ public Observable observe() { * if invoked more than once */ public Observable toObservable() { - if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { - return toObservable(Schedulers.threadPoolForComputation()); + if (observableCommand.properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { + return toObservable(Schedulers.computation()); } else { // semaphore isolation is all blocking, no new threads involved // so we'll use the calling thread @@ -774,6418 +326,177 @@ public Observable toObservable(Scheduler observeOn) { return toObservable(observeOn, true); } - private ObservableCommand toObservable(Scheduler observeOn, boolean performAsyncTimeout) { - /* this is a stateful object so can only be used once */ - if (!started.compareAndSet(false, true)) { - throw new IllegalStateException("This instance can only be executed once. Please instantiate a new instance."); - } - - /* try from cache first */ - if (isRequestCachingEnabled()) { - Observable fromCache = requestCache.get(getCacheKey()); - if (fromCache != null) { - /* mark that we received this response from cache */ - metrics.markResponseFromCache(); - return new CachedObservableResponse((CachedObservableOriginal) fromCache, this); - } - } - - final HystrixCommand _this = this; - - // create an Observable that will lazily execute when subscribed to - Observable o = Observable.create(new OnSubscribeFunc() { - - @Override - public Subscription onSubscribe(Observer observer) { - try { - /* used to track userThreadExecutionTime */ - invocationStartTime = System.currentTimeMillis(); - - // mark that we're starting execution on the ExecutionHook - executionHook.onStart(_this); - - /* determine if we're allowed to execute */ - if (!circuitBreaker.allowRequest()) { - // record that we are returning a short-circuited fallback - metrics.markShortCircuited(); - // short-circuit and go directly to fallback (or throw an exception if no fallback implemented) - try { - observer.onNext(getFallbackOrThrowException(HystrixEventType.SHORT_CIRCUITED, FailureType.SHORTCIRCUIT, "short-circuited")); - observer.onCompleted(); - } catch (Exception e) { - observer.onError(e); - } - return Subscriptions.empty(); - } else { - /* not short-circuited so proceed with queuing the execution */ - try { - if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { - return subscribeWithThreadIsolation(observer); - } else { - return subscribeWithSemaphoreIsolation(observer); - } - } catch (RuntimeException e) { - observer.onError(e); - return Subscriptions.empty(); - } - } - } finally { - recordExecutedCommand(); - } - } - }); - - if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { - // wrap for timeout support - o = new TimeoutObservable(o, _this, performAsyncTimeout); - } - - // error handling - o = o.onErrorResumeNext(new Func1>() { - - @Override - public Observable call(Throwable e) { - // count that we are throwing an exception and re-throw it - metrics.markExceptionThrown(); - return Observable.error(e); - } - }); - - // we want to hand off work to a different scheduler so we don't tie up the Hystrix thread - if (!Schedulers.immediate().equals(observeOn)) { - // don't waste overhead if it's the 'immediate' scheduler - // otherwise we'll 'observeOn' and wrap with the HystrixContextScheduler - // to copy state across threads (if threads are involved) - o = o.observeOn(new HystrixContextScheduler(concurrencyStrategy, observeOn)); - } - - o = o.finallyDo(new Action0() { - - @Override - public void call() { - Reference tl = timeoutTimer.get(); - if (tl != null) { - tl.clear(); - } - } - - }); - - // put in cache - if (isRequestCachingEnabled()) { - // wrap it for caching - o = new CachedObservableOriginal(o.cache(), this); - Observable fromCache = requestCache.putIfAbsent(getCacheKey(), o); - if (fromCache != null) { - // another thread beat us so we'll use the cached value instead - o = new CachedObservableResponse((CachedObservableOriginal) fromCache, this); - } - // we just created an ObservableCommand so we cast and return it - return (ObservableCommand) o; - } else { - // no request caching so a simple wrapper just to pass 'this' along with the Observable - return new ObservableCommand(o, this); - } - } - - /** - * Wraps a source Observable and remembers the original HystrixCommand. - *

- * Used for request caching so multiple commands can respond from a single Observable but also get access to the originating HystrixCommand. - * - * @param - */ - private static class CachedObservableOriginal extends ObservableCommand { - - final HystrixCommand originalCommand; - - CachedObservableOriginal(final Observable actual, HystrixCommand command) { - super(new OnSubscribeFunc() { - - @Override - public Subscription onSubscribe(final Observer observer) { - return actual.subscribe(observer); - } - }, command); - this.originalCommand = command; - } - } - - private static class ObservableCommand extends Observable { - private final HystrixCommand command; - - ObservableCommand(OnSubscribeFunc func, final HystrixCommand command) { - super(func); - this.command = command; - } - - public HystrixCommand getCommand() { - return command; - } - - ObservableCommand(final Observable originalObservable, final HystrixCommand command) { - super(new OnSubscribeFunc() { - - @Override - public Subscription onSubscribe(Observer observer) { - return originalObservable.subscribe(observer); - } - }); - this.command = command; - } - + private ObservableCommand toObservable(final Scheduler observeOn, boolean performAsyncTimeout) { + return observableCommand.toObservable(observeOn, performAsyncTimeout); } - /** - * Wraps a CachedObservableOriginal as it is being returned from cache. - *

- * As the Observable completes it copies state used for ExecutionResults - * and metrics that differentiate between the original and the de-duped "response from cache" command execution. - * - * @param - */ - private static class CachedObservableResponse extends ObservableCommand { - final CachedObservableOriginal originalObservable; - - CachedObservableResponse(final CachedObservableOriginal originalObservable, final HystrixCommand commandOfDuplicateCall) { - super(new OnSubscribeFunc() { - - @Override - public Subscription onSubscribe(final Observer observer) { - return originalObservable.subscribe(new Observer() { - - @Override - public void onCompleted() { - completeCommand(); - observer.onCompleted(); - } + /* package */static class HystrixCommandFromObservableCommand extends HystrixObservableCommand { - @Override - public void onError(Throwable e) { - completeCommand(); - observer.onError(e); - } + private final HystrixCommand original; - @Override - public void onNext(R v) { - observer.onNext(v); - } - - private void completeCommand() { - // when the observable completes we then update the execution results of the duplicate command - // set this instance to the result that is from cache - commandOfDuplicateCall.executionResult = originalObservable.originalCommand.executionResult; - // add that this came from cache - commandOfDuplicateCall.executionResult = commandOfDuplicateCall.executionResult.addEvents(HystrixEventType.RESPONSE_FROM_CACHE); - // set the execution time to 0 since we retrieved from cache - commandOfDuplicateCall.executionResult = commandOfDuplicateCall.executionResult.setExecutionTime(-1); - // record that this command executed - commandOfDuplicateCall.recordExecutedCommand(); - } - }); - } - }, commandOfDuplicateCall); - this.originalObservable = originalObservable; + protected HystrixCommandFromObservableCommand(HystrixCommand o, HystrixCommandGroupKey group, HystrixCommandKey key, HystrixThreadPoolKey threadPoolKey, HystrixCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, + HystrixCommandProperties.Setter commandPropertiesDefaults, HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults, + HystrixCommandMetrics metrics, TryableSemaphore fallbackSemaphore, TryableSemaphore executionSemaphore, + HystrixPropertiesStrategy propertiesStrategy, HystrixCommandExecutionHook executionHook) { + super(group, key, threadPoolKey, circuitBreaker, threadPool, commandPropertiesDefaults, threadPoolPropertiesDefaults, metrics, fallbackSemaphore, executionSemaphore, propertiesStrategy, executionHook); + this.original = o; } - /* - * This is a cached response so we want the command of the observable we're wrapping. - */ - public HystrixCommand getCommand() { - return originalObservable.originalCommand; + /* package */HystrixCommand getOriginal() { + return original; } - } - - private static class TimeoutObservable extends Observable { - public TimeoutObservable(final Observable o, final HystrixCommand originalCommand, final boolean isNonBlocking) { - super(new OnSubscribeFunc() { + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { @Override - public Subscription onSubscribe(final Observer observer) { - // TODO this is using a private API of Rx so either move off of it or get Rx to make it public - // TODO better yet, get TimeoutObservable part of Rx - final SafeObservableSubscription s = new SafeObservableSubscription(); - - /* - * Define the action to perform on timeout outside of the TimerListener to it can capture the HystrixRequestContext - * of the calling thread which doesn't exist on the Timer thread. - */ - final HystrixContextRunnable timeoutRunnable = new HystrixContextRunnable(originalCommand.concurrencyStrategy, new Runnable() { - - @Override - public void run() { - try { - R v = originalCommand.getFallbackOrThrowException(HystrixEventType.TIMEOUT, FailureType.TIMEOUT, "timed-out", new TimeoutException()); - observer.onNext(v); - observer.onCompleted(); - } catch (HystrixRuntimeException re) { - observer.onError(re); - } - } - }); - - TimerListener listener = new TimerListener() { - - @Override - public void tick() { - // if we can go from NOT_EXECUTED to TIMED_OUT then we do the timeout codepath - // otherwise it means we lost a race and the run() execution completed - if (originalCommand.isCommandTimedOut.compareAndSet(TimedOutStatus.NOT_EXECUTED, TimedOutStatus.TIMED_OUT)) { - // do fallback logic - - // report timeout failure - originalCommand.metrics.markTimeout(System.currentTimeMillis() - originalCommand.invocationStartTime); - - // we record execution time because we are returning before - originalCommand.recordTotalExecutionTime(originalCommand.invocationStartTime); - - timeoutRunnable.run(); - } - - s.unsubscribe(); - } - - @Override - public int getIntervalTimeInMilliseconds() { - return originalCommand.properties.executionIsolationThreadTimeoutInMilliseconds().get(); - } - }; - - Reference _tl = null; - if (isNonBlocking) { - /* - * Scheduling a separate timer to do timeouts is more expensive - * so we'll only do it if we're being used in a non-blocking manner. - */ - _tl = HystrixTimer.getInstance().addTimerListener(listener); - } else { - /* - * Otherwise we just set the hook that queue().get() can trigger if a timeout occurs. - * - * This allows the blocking and non-blocking approaches to be coded basically the same way - * though it is admittedly awkward if we were just blocking (the use of Reference annoys me for example) - */ - _tl = new SoftReference(listener); + public void call(Subscriber s) { + try { + s.onNext(original.run()); + s.onCompleted(); + } catch (Throwable e) { + s.onError(e); } - final Reference tl = _tl; - - // set externally so execute/queue can see this - originalCommand.timeoutTimer.set(tl); - - return s.wrap(o.subscribe(new Observer() { - - @Override - public void onCompleted() { - tl.clear(); - observer.onCompleted(); - } - - @Override - public void onError(Throwable e) { - tl.clear(); - observer.onError(e); - } - - @Override - public void onNext(R v) { - observer.onNext(v); - } - - })); - } - }); - } - } - - private Subscription subscribeWithSemaphoreIsolation(final Observer observer) { - TryableSemaphore executionSemaphore = getExecutionSemaphore(); - // acquire a permit - if (executionSemaphore.tryAcquire()) { - try { - try { - // store the command that is being run - Hystrix.startCurrentThreadExecutingCommand(getCommandKey()); - - // execute outside of future so that fireAndForget will still work (ie. someone calls queue() but not get()) and so that multiple requests can be deduped through request caching - R r = executeCommand(); - r = executionHook.onComplete(this, r); - observer.onNext(r); - /* execution time (must occur before terminal state otherwise a race condition can occur if requested by client) */ - recordTotalExecutionTime(invocationStartTime); - /* now complete which releases the consumer */ - observer.onCompleted(); - // empty subscription since we executed synchronously - return Subscriptions.empty(); - } catch (Exception e) { - /* execution time (must occur before terminal state otherwise a race condition can occur if requested by client) */ - recordTotalExecutionTime(invocationStartTime); - observer.onError(e); - // empty subscription since we executed synchronously - return Subscriptions.empty(); - } finally { - // pop the command that is being run - Hystrix.endCurrentThreadExecutingCommand(); } - } finally { - // release the semaphore - executionSemaphore.release(); - } - } else { - metrics.markSemaphoreRejection(); - logger.debug("HystrixCommand Execution Rejection by Semaphore."); // debug only since we're throwing the exception and someone higher will do something with it - // retrieve a fallback or throw an exception if no fallback available - observer.onNext(getFallbackOrThrowException(HystrixEventType.SEMAPHORE_REJECTED, FailureType.REJECTED_SEMAPHORE_EXECUTION, "could not acquire a semaphore for execution")); - observer.onCompleted(); - // empty subscription since we executed synchronously - return Subscriptions.empty(); + }); } - } - - private Subscription subscribeWithThreadIsolation(final Observer observer) { - // mark that we are executing in a thread (even if we end up being rejected we still were a THREAD execution and not SEMAPHORE) - isExecutedInThread.set(true); - // final reference to the current calling thread so the child thread can access it if needed - final Thread callingThread = Thread.currentThread(); - - final HystrixCommand _this = this; - - try { - if (!threadPool.isQueueSpaceAvailable()) { - // we are at the property defined max so want to throw a RejectedExecutionException to simulate reaching the real max - throw new RejectedExecutionException("Rejected command because thread-pool queueSize is at rejection threshold."); - } - - // wrap the synchronous execute() method in a Callable and execute in the threadpool - final Future f = threadPool.getExecutor().submit(new HystrixContextCallable(concurrencyStrategy, new Callable() { + @Override + protected Observable getFallback() { + return Observable.create(new OnSubscribe() { @Override - public R call() throws Exception { - boolean recordDuration = true; - try { - // assign 'callingThread' to our NFExceptionThreadingUtility ThreadLocal variable so that if we blow up - // anywhere along the way the exception knows who the calling thread is and can include it in the stacktrace - ExceptionThreadingUtility.assignCallingThread(callingThread); - - // execution hook - executionHook.onThreadStart(_this); - - // count the active thread - threadPool.markThreadExecution(); - - try { - // store the command that is being run - Hystrix.startCurrentThreadExecutingCommand(getCommandKey()); - // execute the command - R r = executeCommand(); - // if we can go from NOT_EXECUTED to COMPLETED then we did not timeout - if (isCommandTimedOut.compareAndSet(TimedOutStatus.NOT_EXECUTED, TimedOutStatus.COMPLETED)) { - // give the hook an opportunity to modify it - r = executionHook.onComplete(_this, r); - // pass to the observer - observer.onNext(r); - // state changes before termination - preTerminationWork(recordDuration); - /* now complete which releases the consumer */ - observer.onCompleted(); - return r; - } else { - // this means we lost the race and the timeout logic has or is being executed - // state changes before termination - // do not recordDuration as this is a timeout and the tick would have set the duration already. - recordDuration = false; - preTerminationWork(recordDuration); - return null; - } - } finally { - // pop this off the thread now that it's done - Hystrix.endCurrentThreadExecutingCommand(); - } - } catch (Exception e) { - // state changes before termination - preTerminationWork(recordDuration); - // if we can go from NOT_EXECUTED to COMPLETED then we did not timeout - if (isCommandTimedOut.compareAndSet(TimedOutStatus.NOT_EXECUTED, TimedOutStatus.COMPLETED)) { - observer.onError(e); - } - throw e; - } - } - - private void preTerminationWork(boolean recordDuration) { - if(recordDuration) { - /* execution time (must occur before terminal state otherwise a race condition can occur if requested by client) */ - recordTotalExecutionTime(invocationStartTime); - } - threadPool.markThreadCompletion(); - + public void call(Subscriber s) { try { - executionHook.onThreadComplete(_this); - } catch (Exception e) { - logger.warn("ExecutionHook.onThreadComplete threw an exception that will be ignored.", e); + s.onNext(original.getFallback()); + s.onCompleted(); + } catch (Throwable e) { + s.onError(e); } } - })); - - return new Subscription() { - - @Override - public void unsubscribe() { - f.cancel(properties.executionIsolationThreadInterruptOnTimeout().get()); - } - - }; - - } catch (RejectedExecutionException e) { - // mark on counter - metrics.markThreadPoolRejection(); - // use a fallback instead (or throw exception if not implemented) - observer.onNext(getFallbackOrThrowException(HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "could not be queued for execution", e)); - observer.onCompleted(); - return Subscriptions.empty(); - } catch (Exception e) { - // unknown exception - logger.error(getLogMessagePrefix() + ": Unexpected exception while submitting to queue.", e); - observer.onNext(getFallbackOrThrowException(HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "had unexpected exception while attempting to queue for execution.", e)); - observer.onCompleted(); - return Subscriptions.empty(); + }); } - } - - /** - * Executes the command and marks success/failure on the circuit-breaker and calls getFallback if a failure occurs. - *

- * This does NOT use the circuit-breaker to determine if the command should be executed, use execute() for that. This method will ALWAYS attempt to execute the method. - * - * @return R - */ - private R executeCommand() { - /** - * NOTE: Be very careful about what goes in this method. It gets invoked within another thread in most circumstances. - * - * The modifications of booleans 'isResponseFromFallback' etc are going across thread-boundaries thus those - * variables MUST be volatile otherwise they are not guaranteed to be seen by the user thread when the executing thread modifies them. - */ - - /* capture start time for logging */ - long startTime = System.currentTimeMillis(); - // allow tracking how many concurrent threads are executing - metrics.incrementConcurrentExecutionCount(); - try { - executionHook.onRunStart(this); - R response = executionHook.onRunSuccess(this, run()); - long duration = System.currentTimeMillis() - startTime; - metrics.addCommandExecutionTime(duration); - if (isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT) { - // the command timed out in the wrapping thread so we will return immediately - // and not increment any of the counters below or other such logic - return null; - } else { - // report success - executionResult = executionResult.addEvents(HystrixEventType.SUCCESS); - metrics.markSuccess(duration); - circuitBreaker.markSuccess(); - eventNotifier.markCommandExecution(getCommandKey(), properties.executionIsolationStrategy().get(), (int) duration, executionResult.events); - return response; - } - } catch (HystrixBadRequestException e) { - try { - Exception decorated = executionHook.onRunError(this, e); - if (decorated instanceof HystrixBadRequestException) { - e = (HystrixBadRequestException) decorated; - } else { - logger.warn("ExecutionHook.onRunError returned an exception that was not an instance of HystrixBadRequestException so will be ignored.", decorated); - } - } catch (Exception hookException) { - logger.warn("Error calling ExecutionHook.onRunError", hookException); - } - - /* - * HystrixBadRequestException is treated differently and allowed to propagate without any stats tracking or fallback logic - */ - throw e; - } catch (Throwable t) { - Exception e = null; - if (t instanceof Exception) { - e = (Exception) t; - } else { - // Hystrix 1.x uses Exception, not Throwable so to prevent a breaking change Throwable will be wrapped in Exception - e = new Exception("Throwable caught while executing.", t); - } - try { - e = executionHook.onRunError(this, e); - } catch (Exception hookException) { - logger.warn("Error calling ExecutionHook.endRunFailure", hookException); - } - - if (isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT) { - // http://jira/browse/API-4905 HystrixCommand: Error/Timeout Double-count if both occur - // this means we have already timed out then we don't count this error stat and we just return - // as this means the user-thread has already returned, we've already done fallback logic - // and we've already counted the timeout stat - logger.debug("Error executing HystrixCommand.run() [TimedOut]. Proceeding to fallback logic ...", e); - return null; - } else { - logger.debug("Error executing HystrixCommand.run(). Proceeding to fallback logic ...", e); - } - // report failure - metrics.markFailure(System.currentTimeMillis() - startTime); - // record the exception - executionResult = executionResult.setException(e); - return getFallbackOrThrowException(HystrixEventType.FAILURE, FailureType.COMMAND_EXCEPTION, "failed", e); - } finally { - metrics.decrementConcurrentExecutionCount(); - // record that we're completed - isExecutionComplete.set(true); + @Override + protected String getCacheKey() { + return original.getCacheKey(); } } - /** - * Execute getFallback() within protection of a semaphore that limits number of concurrent executions. - *

- * Fallback implementations shouldn't perform anything that can be blocking, but we protect against it anyways in case someone doesn't abide by the contract. - *

- * If something in the getFallback() implementation is latent (such as a network call) then the semaphore will cause us to start rejecting requests rather than allowing potentially - * all threads to pile up and block. - * - * @return K - * @throws UnsupportedOperationException - * if getFallback() not implemented - * @throws HystrixException - * if getFallback() fails (throws an Exception) or is rejected by the semaphore - */ - private R getFallbackWithProtection() { - TryableSemaphore fallbackSemaphore = getFallbackSemaphore(); - // acquire a permit - if (fallbackSemaphore.tryAcquire()) { - try { - executionHook.onFallbackStart(this); - return executionHook.onFallbackSuccess(this, getFallback()); - } catch (RuntimeException e) { - Exception decorated = executionHook.onFallbackError(this, e); - if (decorated instanceof RuntimeException) { - e = (RuntimeException) decorated; - } else { - logger.warn("ExecutionHook.onFallbackError returned an exception that was not an instance of RuntimeException so will be ignored.", decorated); - } + @Override + public HystrixCommandGroupKey getCommandGroup() { + return observableCommand.getCommandGroup(); + } - // re-throw to calling method - throw e; - } finally { - fallbackSemaphore.release(); - } - } else { - metrics.markFallbackRejection(); - logger.debug("HystrixCommand Fallback Rejection."); // debug only since we're throwing the exception and someone higher will do something with it - // if we couldn't acquire a permit, we "fail fast" by throwing an exception - throw new HystrixRuntimeException(FailureType.REJECTED_SEMAPHORE_FALLBACK, this.getClass(), getLogMessagePrefix() + " fallback execution rejected.", null, null); - } + @Override + public HystrixCommandKey getCommandKey() { + return observableCommand.getCommandKey(); } - /** - * Record the duration of execution as response or exception is being returned to the caller. - */ - private void recordTotalExecutionTime(long startTime) { - long duration = System.currentTimeMillis() - startTime; - // the total execution time for the user thread including queuing, thread scheduling, run() execution - metrics.addUserThreadExecutionTime(duration); + @Override + public HystrixThreadPoolKey getThreadPoolKey() { + return observableCommand.getThreadPoolKey(); + } - /* - * We record the executionTime for command execution. - * - * If the command is never executed (rejected, short-circuited, etc) then it will be left unset. - * - * For this metric we include failures and successes as we use it for per-request profiling and debugging - * whereas 'metrics.addCommandExecutionTime(duration)' is used by stats across many requests. - */ - executionResult = executionResult.setExecutionTime((int) duration); + @Override + public HystrixCommandMetrics getMetrics() { + return observableCommand.getMetrics(); } - /** - * Record that this command was executed in the HystrixRequestLog. - *

- * This can be treated as an async operation as it just adds a references to "this" in the log even if the current command is still executing. - */ - private void recordExecutedCommand() { - if (properties.requestLogEnabled().get()) { - // log this command execution regardless of what happened - if (concurrencyStrategy instanceof HystrixConcurrencyStrategyDefault) { - // if we're using the default we support only optionally using a request context - if (HystrixRequestContext.isCurrentThreadInitialized()) { - HystrixRequestLog.getCurrentRequest(concurrencyStrategy).addExecutedCommand(this); - } - } else { - // if it's a custom strategy it must ensure the context is initialized - if (HystrixRequestLog.getCurrentRequest(concurrencyStrategy) != null) { - HystrixRequestLog.getCurrentRequest(concurrencyStrategy).addExecutedCommand(this); - } - } - } + @Override + public HystrixCommandProperties getProperties() { + return observableCommand.getProperties(); } - /** - * Whether the 'circuit-breaker' is open meaning that execute() will immediately return - * the getFallback() response and not attempt a HystrixCommand execution. - * - * @return boolean - */ + @Override public boolean isCircuitBreakerOpen() { - return circuitBreaker.isOpen(); + return observableCommand.isCircuitBreakerOpen(); } - /** - * If this command has completed execution either successfully, via fallback or failure. - * - * @return boolean - */ + @Override public boolean isExecutionComplete() { - return isExecutionComplete.get(); + return observableCommand.isExecutionComplete(); } - /** - * Whether the execution occurred in a separate thread. - *

- * This should be called only once execute()/queue()/fireOrForget() are called otherwise it will always return false. - *

- * This specifies if a thread execution actually occurred, not just if it is configured to be executed in a thread. - * - * @return boolean - */ + @Override public boolean isExecutedInThread() { - return isExecutedInThread.get(); + return observableCommand.isExecutedInThread(); } - /** - * Whether the response was returned successfully either by executing run() or from cache. - * - * @return boolean - */ + @Override public boolean isSuccessfulExecution() { - return executionResult.events.contains(HystrixEventType.SUCCESS); + return observableCommand.isSuccessfulExecution(); } - /** - * Whether the run() resulted in a failure (exception). - * - * @return boolean - */ + @Override public boolean isFailedExecution() { - return executionResult.events.contains(HystrixEventType.FAILURE); + return observableCommand.isFailedExecution(); } - /** - * Get the Throwable/Exception thrown that caused the failure. - *

- * If isFailedExecution() == true then this would represent the Exception thrown by the run() method. - *

- * If isFailedExecution() == false then this would return null. - * - * @return Throwable or null - */ + @Override public Throwable getFailedExecutionException() { - return executionResult.exception; + return observableCommand.getFailedExecutionException(); } - /** - * Whether the response received from was the result of some type of failure - * and getFallback() being called. - * - * @return boolean - */ + @Override public boolean isResponseFromFallback() { - return executionResult.events.contains(HystrixEventType.FALLBACK_SUCCESS); + return observableCommand.isResponseFromFallback(); } - /** - * Whether the response received was the result of a timeout - * and getFallback() being called. - * - * @return boolean - */ + @Override public boolean isResponseTimedOut() { - return executionResult.events.contains(HystrixEventType.TIMEOUT); + return observableCommand.isResponseTimedOut(); } - /** - * Whether the response received was a fallback as result of being - * short-circuited (meaning isCircuitBreakerOpen() == true) and getFallback() being called. - * - * @return boolean - */ + @Override public boolean isResponseShortCircuited() { - return executionResult.events.contains(HystrixEventType.SHORT_CIRCUITED); + return observableCommand.isResponseShortCircuited(); } - /** - * Whether the response is from cache and run() was not invoked. - * - * @return boolean - */ + @Override public boolean isResponseFromCache() { - return executionResult.events.contains(HystrixEventType.RESPONSE_FROM_CACHE); + return observableCommand.isResponseFromCache(); } - /** - * Whether the response received was a fallback as result of being - * rejected (from thread-pool or semaphore) and getFallback() being called. - * - * @return boolean - */ + @Override public boolean isResponseRejected() { - return executionResult.events.contains(HystrixEventType.THREAD_POOL_REJECTED) || executionResult.events.contains(HystrixEventType.SEMAPHORE_REJECTED); + return observableCommand.isResponseRejected(); } - /** - * List of HystrixCommandEventType enums representing events that occurred during execution. - *

- * Examples of events are SUCCESS, FAILURE, TIMEOUT, and SHORT_CIRCUITED - * - * @return {@code List} - */ + @Override public List getExecutionEvents() { - return executionResult.events; + return observableCommand.getExecutionEvents(); } - /** - * The execution time of this command instance in milliseconds, or -1 if not executed. - * - * @return int - */ + @Override public int getExecutionTimeInMilliseconds() { - return executionResult.executionTime; - } - - /** - * Get the TryableSemaphore this HystrixCommand should use if a fallback occurs. - * - * @param circuitBreaker - * @param fallbackSemaphore - * @return TryableSemaphore - */ - private TryableSemaphore getFallbackSemaphore() { - if (fallbackSemaphoreOverride == null) { - TryableSemaphore _s = fallbackSemaphorePerCircuit.get(commandKey.name()); - if (_s == null) { - // we didn't find one cache so setup - fallbackSemaphorePerCircuit.putIfAbsent(commandKey.name(), new TryableSemaphore(properties.fallbackIsolationSemaphoreMaxConcurrentRequests())); - // assign whatever got set (this or another thread) - return fallbackSemaphorePerCircuit.get(commandKey.name()); - } else { - return _s; - } - } else { - return fallbackSemaphoreOverride; - } + return observableCommand.getExecutionTimeInMilliseconds(); } - /** - * Get the TryableSemaphore this HystrixCommand should use for execution if not running in a separate thread. - * - * @param circuitBreaker - * @param fallbackSemaphore - * @return TryableSemaphore - */ - private TryableSemaphore getExecutionSemaphore() { - if (executionSemaphoreOverride == null) { - TryableSemaphore _s = executionSemaphorePerCircuit.get(commandKey.name()); - if (_s == null) { - // we didn't find one cache so setup - executionSemaphorePerCircuit.putIfAbsent(commandKey.name(), new TryableSemaphore(properties.executionIsolationSemaphoreMaxConcurrentRequests())); - // assign whatever got set (this or another thread) - return executionSemaphorePerCircuit.get(commandKey.name()); - } else { - return _s; - } - } else { - return executionSemaphoreOverride; - } + @Override + public Future queue() { + return observableCommand.queue(); } - /** - * @throws HystrixRuntimeException - */ - private R getFallbackOrThrowException(HystrixEventType eventType, FailureType failureType, String message) { - return getFallbackOrThrowException(eventType, failureType, message, null); + @Override + public R execute() { + return observableCommand.execute(); } - /** - * @throws HystrixRuntimeException - */ - private R getFallbackOrThrowException(HystrixEventType eventType, FailureType failureType, String message, Exception e) { - try { - if (properties.fallbackEnabled().get()) { - /* fallback behavior is permitted so attempt */ - try { - // record the executionResult - // do this before executing fallback so it can be queried from within getFallback (see See https://github.com/Netflix/Hystrix/pull/144) - executionResult = executionResult.addEvents(eventType); - - // retrieve the fallback - R fallback = getFallbackWithProtection(); - // mark fallback on counter - metrics.markFallbackSuccess(); - // record the executionResult - executionResult = executionResult.addEvents(HystrixEventType.FALLBACK_SUCCESS); - return executionHook.onComplete(this, fallback); - } catch (UnsupportedOperationException fe) { - logger.debug("No fallback for HystrixCommand. ", fe); // debug only since we're throwing the exception and someone higher will do something with it - - /* executionHook for all errors */ - try { - e = executionHook.onError(this, failureType, e); - } catch (Exception hookException) { - logger.warn("Error calling ExecutionHook.onError", hookException); - } - - throw new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and no fallback available.", e, fe); - } catch (Exception fe) { - logger.debug("HystrixCommand execution " + failureType.name() + " and fallback retrieval failed.", fe); - metrics.markFallbackFailure(); - // record the executionResult - executionResult = executionResult.addEvents(HystrixEventType.FALLBACK_FAILURE); - - /* executionHook for all errors */ - try { - e = executionHook.onError(this, failureType, e); - } catch (Exception hookException) { - logger.warn("Error calling ExecutionHook.onError", hookException); - } - - throw new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and failed retrieving fallback.", e, fe); - } - } else { - /* fallback is disabled so throw HystrixRuntimeException */ - - logger.debug("Fallback disabled for HystrixCommand so will throw HystrixRuntimeException. ", e); // debug only since we're throwing the exception and someone higher will do something with it - // record the executionResult - executionResult = executionResult.addEvents(eventType); - - /* executionHook for all errors */ - try { - e = executionHook.onError(this, failureType, e); - } catch (Exception hookException) { - logger.warn("Error calling ExecutionHook.onError", hookException); - } - throw new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and fallback disabled.", e, null); - } - } finally { - // record that we're completed (to handle non-successful events we do it here as well as at the end of executeCommand - isExecutionComplete.set(true); - } + @Override + public Observable observe() { + return observableCommand.observe(); } - /* ******************************************************************************** */ - /* ******************************************************************************** */ - /* Result Status */ - /* ******************************************************************************** */ - /* ******************************************************************************** */ - - /** - * Immutable holder class for the status of command execution. - *

- * Contained within a class to simplify the sharing of it across Futures/threads that result from request caching. - *

- * This object can be referenced and "modified" by parent and child threads as well as by different instances of HystrixCommand since - * 1 instance could create an ExecutionResult, cache a Future that refers to it, a 2nd instance execution then retrieves a Future - * from cache and wants to append RESPONSE_FROM_CACHE to whatever the ExecutionResult was from the first command execution. - *

- * This being immutable forces and ensure thread-safety instead of using AtomicInteger/ConcurrentLinkedQueue and determining - * when it's safe to mutate the object directly versus needing to deep-copy clone to a new instance. - */ - private static class ExecutionResult { - private final List events; - private final int executionTime; - private final Exception exception; - - private ExecutionResult(HystrixEventType... events) { - this(Arrays.asList(events), -1, null); - } - - public ExecutionResult setExecutionTime(int executionTime) { - return new ExecutionResult(events, executionTime, exception); - } - - public ExecutionResult setException(Exception e) { - return new ExecutionResult(events, executionTime, e); - } - - private ExecutionResult(List events, int executionTime, Exception e) { - // we are safe assigning the List reference instead of deep-copying - // because we control the original list in 'newEvent' - this.events = events; - this.executionTime = executionTime; - this.exception = e; - } - - // we can return a static version since it's immutable - private static ExecutionResult EMPTY = new ExecutionResult(new HystrixEventType[0]); - - /** - * Creates a new ExecutionResult by adding the defined 'events' to the ones on the current instance. - * - * @param events - * @return - */ - public ExecutionResult addEvents(HystrixEventType... events) { - ArrayList newEvents = new ArrayList(); - newEvents.addAll(this.events); - for (HystrixEventType e : events) { - newEvents.add(e); - } - return new ExecutionResult(Collections.unmodifiableList(newEvents), executionTime, exception); - } + /* package */void markAsCollapsedCommand(int sizeOfBatch) { + observableCommand.markAsCollapsedCommand(sizeOfBatch); } - /* ******************************************************************************** */ - /* ******************************************************************************** */ - /* RequestCache */ - /* ******************************************************************************** */ - /* ******************************************************************************** */ - - /** - * Key to be used for request caching. - *

- * By default this returns null which means "do not cache". - *

- * To enable caching override this method and return a string key uniquely representing the state of a command instance. - *

- * If multiple command instances in the same request scope match keys then only the first will be executed and all others returned from cache. - * - * @return cacheKey - */ - protected String getCacheKey() { - return null; - } - - private boolean isRequestCachingEnabled() { - return properties.requestCacheEnabled().get(); - } - - /* ******************************************************************************** */ - /* ******************************************************************************** */ - /* TryableSemaphore */ - /* ******************************************************************************** */ - /* ******************************************************************************** */ - - /** - * Semaphore that only supports tryAcquire and never blocks and that supports a dynamic permit count. - *

- * Using AtomicInteger increment/decrement instead of java.util.concurrent.Semaphore since we don't need blocking and need a custom implementation to get the dynamic permit count and since - * AtomicInteger achieves the same behavior and performance without the more complex implementation of the actual Semaphore class using AbstractQueueSynchronizer. - */ - private static class TryableSemaphore { - private final HystrixProperty numberOfPermits; - private final AtomicInteger count = new AtomicInteger(0); - - public TryableSemaphore(HystrixProperty numberOfPermits) { - this.numberOfPermits = numberOfPermits; - } - - /** - * Use like this: - *

- * - * 

-         * if (s.tryAcquire()) {
-         * try {
-         * // do work that is protected by 's'
-         * } finally {
-         * s.release();
-         * }
-         * }
-         *
- * - * @return boolean - */ - public boolean tryAcquire() { - int currentCount = count.incrementAndGet(); - if (currentCount > numberOfPermits.get()) { - count.decrementAndGet(); - return false; - } else { - return true; - } - } - - /** - * ONLY call release if tryAcquire returned true. - *

- * - * 

-         * if (s.tryAcquire()) {
-         * try {
-         * // do work that is protected by 's'
-         * } finally {
-         * s.release();
-         * }
-         * }
-         *
- */ - public void release() { - count.decrementAndGet(); - } - - public int getNumberOfPermitsUsed() { - return count.get(); - } - - } - - private String getLogMessagePrefix() { - return getCommandKey().name(); - } - - /** - * Fluent interface for arguments to the {@link HystrixCommand} constructor. - *

- * The required arguments are set via the 'with' factory method and optional arguments via the 'and' chained methods. - *

- * Example: - * 

 {@code
-     *  Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("GroupName"))
-                .andCommandKey(HystrixCommandKey.Factory.asKey("CommandName"))
-                .andEventNotifier(notifier);
-     * }
- */ - @NotThreadSafe - public static class Setter { - - private final HystrixCommandGroupKey groupKey; - private HystrixCommandKey commandKey; - private HystrixThreadPoolKey threadPoolKey; - private HystrixCommandProperties.Setter commandPropertiesDefaults; - private HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults; - - /** - * Setter factory method containing required values. - *

- * All optional arguments can be set via the chained methods. - * - * @param groupKey - * {@link HystrixCommandGroupKey} used to group together multiple {@link HystrixCommand} objects. - *

- * The {@link HystrixCommandGroupKey} is used to represent a common relationship between commands. For example, a library or team name, the system all related commands interace - * with, - * common business purpose etc. - */ - private Setter(HystrixCommandGroupKey groupKey) { - this.groupKey = groupKey; - } - - /** - * Setter factory method with required values. - *

- * All optional arguments can be set via the chained methods. - * - * @param groupKey - * {@link HystrixCommandGroupKey} used to group together multiple {@link HystrixCommand} objects. - *

- * The {@link HystrixCommandGroupKey} is used to represent a common relationship between commands. For example, a library or team name, the system all related commands interace - * with, - * common business purpose etc. - */ - public static Setter withGroupKey(HystrixCommandGroupKey groupKey) { - return new Setter(groupKey); - } - - /** - * @param commandKey - * {@link HystrixCommandKey} used to identify a {@link HystrixCommand} instance for statistics, circuit-breaker, properties, etc. - *

- * By default this will be derived from the instance class name. - *

- * NOTE: Every unique {@link HystrixCommandKey} will result in new instances of {@link HystrixCircuitBreaker}, {@link HystrixCommandMetrics} and {@link HystrixCommandProperties}. - * Thus, - * the number of variants should be kept to a finite and reasonable number to avoid high-memory usage or memory leacks. - *

- * Hundreds of keys is fine, tens of thousands is probably not. - * @return Setter for fluent interface via method chaining - */ - public Setter andCommandKey(HystrixCommandKey commandKey) { - this.commandKey = commandKey; - return this; - } - - /** - * @param threadPoolKey - * {@link HystrixThreadPoolKey} used to define which thread-pool this command should run in (when configured to run on separate threads via - * {@link HystrixCommandProperties#executionIsolationStrategy()}). - *

- * By default this is derived from the {@link HystrixCommandGroupKey} but if injected this allows multiple commands to have the same {@link HystrixCommandGroupKey} but different - * thread-pools. - * @return Setter for fluent interface via method chaining - */ - public Setter andThreadPoolKey(HystrixThreadPoolKey threadPoolKey) { - this.threadPoolKey = threadPoolKey; - return this; - } - - /** - * Optional - * - * @param commandPropertiesDefaults - * {@link HystrixCommandProperties.Setter} with property overrides for this specific instance of {@link HystrixCommand}. - *

- * See the {@link HystrixPropertiesStrategy} JavaDocs for more information on properties and order of precedence. - * @return Setter for fluent interface via method chaining - */ - public Setter andCommandPropertiesDefaults(HystrixCommandProperties.Setter commandPropertiesDefaults) { - this.commandPropertiesDefaults = commandPropertiesDefaults; - return this; - } - - /** - * Optional - * - * @param threadPoolPropertiesDefaults - * {@link HystrixThreadPoolProperties.Setter} with property overrides for the {@link HystrixThreadPool} used by this specific instance of {@link HystrixCommand}. - *

- * See the {@link HystrixPropertiesStrategy} JavaDocs for more information on properties and order of precedence. - * @return Setter for fluent interface via method chaining - */ - public Setter andThreadPoolPropertiesDefaults(HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults) { - this.threadPoolPropertiesDefaults = threadPoolPropertiesDefaults; - return this; - } - - } - - public static class UnitTest { - - @Before - public void prepareForTest() { - /* we must call this to simulate a new request lifecycle running and clearing caches */ - HystrixRequestContext.initializeContext(); - } - - @After - public void cleanup() { - // instead of storing the reference from initialize we'll just get the current state and shutdown - if (HystrixRequestContext.getContextForCurrentThread() != null) { - // it could have been set NULL by the test - HystrixRequestContext.getContextForCurrentThread().shutdown(); - } - - // force properties to be clean as well - ConfigurationManager.getConfigInstance().clear(); - } - - /** - * Test a successful command execution. - */ - @Test - public void testExecutionSuccess() { - try { - TestHystrixCommand command = new SuccessfulTestCommand(); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(true, command.execute()); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - - assertEquals(null, command.getFailedExecutionException()); - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isSuccessfulExecution()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - - } catch (Exception e) { - e.printStackTrace(); - fail("We received an exception."); - } - } - - /** - * Test that a command can not be executed multiple times. - */ - @Test - public void testExecutionMultipleTimes() { - SuccessfulTestCommand command = new SuccessfulTestCommand(); - assertFalse(command.isExecutionComplete()); - // first should succeed - assertEquals(true, command.execute()); - assertTrue(command.isExecutionComplete()); - assertTrue(command.isExecutedInThread()); - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isSuccessfulExecution()); - try { - // second should fail - command.execute(); - fail("we should not allow this ... it breaks the state of request logs"); - } catch (IllegalStateException e) { - e.printStackTrace(); - // we want to get here - } - - try { - // queue should also fail - command.queue(); - fail("we should not allow this ... it breaks the state of request logs"); - } catch (IllegalStateException e) { - e.printStackTrace(); - // we want to get here - } - } - - /** - * Test a command execution that throws an HystrixException and didn't implement getFallback. - */ - @Test - public void testExecutionKnownFailureWithNoFallback() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - TestHystrixCommand command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); - try { - command.execute(); - fail("we shouldn't get here"); - } catch (HystrixRuntimeException e) { - e.printStackTrace(); - assertNotNull(e.getFallbackException()); - assertNotNull(e.getImplementingClass()); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - } catch (Exception e) { - e.printStackTrace(); - fail("We should always get an HystrixRuntimeException when an error occurs."); - } - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution that throws an unknown exception (not HystrixException) and didn't implement getFallback. - */ - @Test - public void testExecutionUnknownFailureWithNoFallback() { - TestHystrixCommand command = new UnknownFailureTestCommandWithoutFallback(); - try { - command.execute(); - fail("we shouldn't get here"); - } catch (HystrixRuntimeException e) { - e.printStackTrace(); - assertNotNull(e.getFallbackException()); - assertNotNull(e.getImplementingClass()); - - } catch (Exception e) { - e.printStackTrace(); - fail("We should always get an HystrixRuntimeException when an error occurs."); - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution that fails but has a fallback. - */ - @Test - public void testExecutionFailureWithFallback() { - TestHystrixCommand command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); - try { - assertEquals(false, command.execute()); - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - assertEquals("we failed with a simulated issue", command.getFailedExecutionException().getMessage()); - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution that fails, has getFallback implemented but that fails as well. - */ - @Test - public void testExecutionFailureWithFallbackFailure() { - TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); - try { - command.execute(); - fail("we shouldn't get here"); - } catch (HystrixRuntimeException e) { - System.out.println("------------------------------------------------"); - e.printStackTrace(); - System.out.println("------------------------------------------------"); - assertNotNull(e.getFallbackException()); - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a successful command execution (asynchronously). - */ - @Test - public void testQueueSuccess() { - TestHystrixCommand command = new SuccessfulTestCommand(); - try { - Future future = command.queue(); - assertEquals(true, future.get()); - } catch (Exception e) { - e.printStackTrace(); - fail("We received an exception."); - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isSuccessfulExecution()); - - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution (asynchronously) that throws an HystrixException and didn't implement getFallback. - */ - @Test - public void testQueueKnownFailureWithNoFallback() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - TestHystrixCommand command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); - try { - command.queue().get(); - fail("we shouldn't get here"); - } catch (Exception e) { - e.printStackTrace(); - if (e.getCause() instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); - - assertNotNull(de.getFallbackException()); - assertNotNull(de.getImplementingClass()); - } else { - fail("the cause should be HystrixRuntimeException"); - } - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution (asynchronously) that throws an unknown exception (not HystrixException) and didn't implement getFallback. - */ - @Test - public void testQueueUnknownFailureWithNoFallback() { - TestHystrixCommand command = new UnknownFailureTestCommandWithoutFallback(); - try { - command.queue().get(); - fail("we shouldn't get here"); - } catch (Exception e) { - e.printStackTrace(); - if (e.getCause() instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); - assertNotNull(de.getFallbackException()); - assertNotNull(de.getImplementingClass()); - } else { - fail("the cause should be HystrixRuntimeException"); - } - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution (asynchronously) that fails but has a fallback. - */ - @Test - public void testQueueFailureWithFallback() { - TestHystrixCommand command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); - try { - Future future = command.queue(); - assertEquals(false, future.get()); - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution (asynchronously) that fails, has getFallback implemented but that fails as well. - */ - @Test - public void testQueueFailureWithFallbackFailure() { - TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); - try { - command.queue().get(); - fail("we shouldn't get here"); - } catch (Exception e) { - if (e.getCause() instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); - e.printStackTrace(); - assertNotNull(de.getFallbackException()); - } else { - fail("the cause should be HystrixRuntimeException"); - } - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a successful command execution. - */ - @Test - public void testObserveSuccess() { - try { - TestHystrixCommand command = new SuccessfulTestCommand(); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(true, command.observe().toBlockingObservable().single()); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - - assertEquals(null, command.getFailedExecutionException()); - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isSuccessfulExecution()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - - } catch (Exception e) { - e.printStackTrace(); - fail("We received an exception."); - } - } - - /** - * Test a successful command execution. - */ - @Test - public void testObserveOnScheduler() throws Exception { - - final AtomicReference commandThread = new AtomicReference(); - final AtomicReference subscribeThread = new AtomicReference(); - - TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder()) { - - @Override - protected Boolean run() { - commandThread.set(Thread.currentThread()); - return true; - } - }; - - final CountDownLatch latch = new CountDownLatch(1); - - Scheduler customScheduler = new Scheduler() { - - private final Scheduler self = this; - - @Override - public Subscription schedule(T state, Func2 action) { - return schedule(state, action, 0, TimeUnit.MILLISECONDS); - } - - @Override - public Subscription schedule(final T state, final Func2 action, long delayTime, TimeUnit unit) { - new Thread("RxScheduledThread") { - @Override - public void run() { - action.call(self, state); - } - }.start(); - - // not testing unsubscribe behavior - return Subscriptions.empty(); - } - - }; - - command.toObservable(customScheduler).subscribe(new Observer() { - - @Override - public void onCompleted() { - latch.countDown(); - - } - - @Override - public void onError(Throwable e) { - latch.countDown(); - e.printStackTrace(); - - } - - @Override - public void onNext(Boolean args) { - subscribeThread.set(Thread.currentThread()); - } - }); - - if (!latch.await(2000, TimeUnit.MILLISECONDS)) { - fail("timed out"); - } - - assertNotNull(commandThread.get()); - assertNotNull(subscribeThread.get()); - - System.out.println("Command Thread: " + commandThread.get()); - System.out.println("Subscribe Thread: " + subscribeThread.get()); - - assertTrue(commandThread.get().getName().startsWith("hystrix-")); - assertTrue(subscribeThread.get().getName().equals("RxScheduledThread")); - } - - /** - * Test a successful command execution. - */ - @Test - public void testObserveOnComputationSchedulerByDefaultForThreadIsolation() throws Exception { - - final AtomicReference commandThread = new AtomicReference(); - final AtomicReference subscribeThread = new AtomicReference(); - - TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder()) { - - @Override - protected Boolean run() { - commandThread.set(Thread.currentThread()); - return true; - } - }; - - final CountDownLatch latch = new CountDownLatch(1); - - command.toObservable().subscribe(new Observer() { - - @Override - public void onCompleted() { - latch.countDown(); - - } - - @Override - public void onError(Throwable e) { - latch.countDown(); - e.printStackTrace(); - - } - - @Override - public void onNext(Boolean args) { - subscribeThread.set(Thread.currentThread()); - } - }); - - if (!latch.await(2000, TimeUnit.MILLISECONDS)) { - fail("timed out"); - } - - assertNotNull(commandThread.get()); - assertNotNull(subscribeThread.get()); - - System.out.println("Command Thread: " + commandThread.get()); - System.out.println("Subscribe Thread: " + subscribeThread.get()); - - assertTrue(commandThread.get().getName().startsWith("hystrix-")); - assertTrue(subscribeThread.get().getName().startsWith("RxComputationThreadPool")); - } - - /** - * Test a successful command execution. - */ - @Test - public void testObserveOnImmediateSchedulerByDefaultForSemaphoreIsolation() throws Exception { - - final AtomicReference commandThread = new AtomicReference(); - final AtomicReference subscribeThread = new AtomicReference(); - - TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { - - @Override - protected Boolean run() { - commandThread.set(Thread.currentThread()); - return true; - } - }; - - final CountDownLatch latch = new CountDownLatch(1); - - command.toObservable().subscribe(new Observer() { - - @Override - public void onCompleted() { - latch.countDown(); - - } - - @Override - public void onError(Throwable e) { - latch.countDown(); - e.printStackTrace(); - - } - - @Override - public void onNext(Boolean args) { - subscribeThread.set(Thread.currentThread()); - } - }); - - if (!latch.await(2000, TimeUnit.MILLISECONDS)) { - fail("timed out"); - } - - assertNotNull(commandThread.get()); - assertNotNull(subscribeThread.get()); - - System.out.println("Command Thread: " + commandThread.get()); - System.out.println("Subscribe Thread: " + subscribeThread.get()); - - String mainThreadName = Thread.currentThread().getName(); - - // semaphore should be on the calling thread - assertTrue(commandThread.get().getName().equals(mainThreadName)); - assertTrue(subscribeThread.get().getName().equals(mainThreadName)); - } - - /** - * Test that the circuit-breaker will 'trip' and prevent command execution on subsequent calls. - */ - @Test - public void testCircuitBreakerTripsAfterFailures() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - /* fail 3 times and then it should trip the circuit and stop executing */ - // failure 1 - KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt1.execute(); - assertTrue(attempt1.isResponseFromFallback()); - assertFalse(attempt1.isCircuitBreakerOpen()); - assertFalse(attempt1.isResponseShortCircuited()); - - // failure 2 - KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt2.execute(); - assertTrue(attempt2.isResponseFromFallback()); - assertFalse(attempt2.isCircuitBreakerOpen()); - assertFalse(attempt2.isResponseShortCircuited()); - - // failure 3 - KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt3.execute(); - assertTrue(attempt3.isResponseFromFallback()); - assertFalse(attempt3.isResponseShortCircuited()); - // it should now be 'open' and prevent further executions - assertTrue(attempt3.isCircuitBreakerOpen()); - - // attempt 4 - KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt4.execute(); - assertTrue(attempt4.isResponseFromFallback()); - // this should now be true as the response will be short-circuited - assertTrue(attempt4.isResponseShortCircuited()); - // this should remain open - assertTrue(attempt4.isCircuitBreakerOpen()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test that the circuit-breaker will 'trip' and prevent command execution on subsequent calls. - */ - @Test - public void testCircuitBreakerTripsAfterFailuresViaQueue() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - try { - /* fail 3 times and then it should trip the circuit and stop executing */ - // failure 1 - KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt1.queue().get(); - assertTrue(attempt1.isResponseFromFallback()); - assertFalse(attempt1.isCircuitBreakerOpen()); - assertFalse(attempt1.isResponseShortCircuited()); - - // failure 2 - KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt2.queue().get(); - assertTrue(attempt2.isResponseFromFallback()); - assertFalse(attempt2.isCircuitBreakerOpen()); - assertFalse(attempt2.isResponseShortCircuited()); - - // failure 3 - KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt3.queue().get(); - assertTrue(attempt3.isResponseFromFallback()); - assertFalse(attempt3.isResponseShortCircuited()); - // it should now be 'open' and prevent further executions - assertTrue(attempt3.isCircuitBreakerOpen()); - - // attempt 4 - KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt4.queue().get(); - assertTrue(attempt4.isResponseFromFallback()); - // this should now be true as the response will be short-circuited - assertTrue(attempt4.isResponseShortCircuited()); - // this should remain open - assertTrue(attempt4.isCircuitBreakerOpen()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received fallbacks."); - } - } - - /** - * Test that the circuit-breaker is shared across HystrixCommand objects with the same CommandKey. - *

- * This will test HystrixCommand objects with a single circuit-breaker (as if each injected with same CommandKey) - *

- * Multiple HystrixCommand objects with the same dependency use the same circuit-breaker. - */ - @Test - public void testCircuitBreakerAcrossMultipleCommandsButSameCircuitBreaker() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - /* fail 3 times and then it should trip the circuit and stop executing */ - // failure 1 - KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt1.execute(); - assertTrue(attempt1.isResponseFromFallback()); - assertFalse(attempt1.isCircuitBreakerOpen()); - assertFalse(attempt1.isResponseShortCircuited()); - - // failure 2 with a different command, same circuit breaker - KnownFailureTestCommandWithoutFallback attempt2 = new KnownFailureTestCommandWithoutFallback(circuitBreaker); - try { - attempt2.execute(); - } catch (Exception e) { - // ignore ... this doesn't have a fallback so will throw an exception - } - assertTrue(attempt2.isFailedExecution()); - assertFalse(attempt2.isResponseFromFallback()); // false because no fallback - assertFalse(attempt2.isCircuitBreakerOpen()); - assertFalse(attempt2.isResponseShortCircuited()); - - // failure 3 of the Hystrix, 2nd for this particular HystrixCommand - KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt3.execute(); - assertTrue(attempt2.isFailedExecution()); - assertTrue(attempt3.isResponseFromFallback()); - assertFalse(attempt3.isResponseShortCircuited()); - - // it should now be 'open' and prevent further executions - // after having 3 failures on the Hystrix that these 2 different HystrixCommand objects are for - assertTrue(attempt3.isCircuitBreakerOpen()); - - // attempt 4 - KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker); - attempt4.execute(); - assertTrue(attempt4.isResponseFromFallback()); - // this should now be true as the response will be short-circuited - assertTrue(attempt4.isResponseShortCircuited()); - // this should remain open - assertTrue(attempt4.isCircuitBreakerOpen()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test that the circuit-breaker is different between HystrixCommand objects with a different Hystrix. - */ - @Test - public void testCircuitBreakerAcrossMultipleCommandsAndDifferentDependency() { - TestCircuitBreaker circuitBreaker_one = new TestCircuitBreaker(); - TestCircuitBreaker circuitBreaker_two = new TestCircuitBreaker(); - /* fail 3 times, twice on one Hystrix, once on a different Hystrix ... circuit-breaker should NOT open */ - - // failure 1 - KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker_one); - attempt1.execute(); - assertTrue(attempt1.isResponseFromFallback()); - assertFalse(attempt1.isCircuitBreakerOpen()); - assertFalse(attempt1.isResponseShortCircuited()); - - // failure 2 with a different HystrixCommand implementation and different Hystrix - KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker_two); - attempt2.execute(); - assertTrue(attempt2.isResponseFromFallback()); - assertFalse(attempt2.isCircuitBreakerOpen()); - assertFalse(attempt2.isResponseShortCircuited()); - - // failure 3 but only 2nd of the Hystrix.ONE - KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker_one); - attempt3.execute(); - assertTrue(attempt3.isResponseFromFallback()); - assertFalse(attempt3.isResponseShortCircuited()); - - // it should remain 'closed' since we have only had 2 failures on Hystrix.ONE - assertFalse(attempt3.isCircuitBreakerOpen()); - - // this one should also remain closed as it only had 1 failure for Hystrix.TWO - assertFalse(attempt2.isCircuitBreakerOpen()); - - // attempt 4 (3rd attempt for Hystrix.ONE) - KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker_one); - attempt4.execute(); - // this should NOW flip to true as this is the 3rd failure for Hystrix.ONE - assertTrue(attempt3.isCircuitBreakerOpen()); - assertTrue(attempt3.isResponseFromFallback()); - assertFalse(attempt3.isResponseShortCircuited()); - - // Hystrix.TWO should still remain closed - assertFalse(attempt2.isCircuitBreakerOpen()); - - assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(3, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(3, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker_one.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker_two.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test that the circuit-breaker being disabled doesn't wreak havoc. - */ - @Test - public void testExecutionSuccessWithCircuitBreakerDisabled() { - TestHystrixCommand command = new TestCommandWithoutCircuitBreaker(); - try { - assertEquals(true, command.execute()); - } catch (Exception e) { - e.printStackTrace(); - fail("We received an exception."); - } - - // we'll still get metrics ... just not the circuit breaker opening/closing - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution timeout where the command didn't implement getFallback. - */ - @Test - public void testExecutionTimeoutWithNoFallback() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); - try { - command.execute(); - fail("we shouldn't get here"); - } catch (Exception e) { - // e.printStackTrace(); - if (e instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e; - assertNotNull(de.getFallbackException()); - assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof TimeoutException); - } else { - fail("the exception should be HystrixRuntimeException"); - } - } - // the time should be 50+ since we timeout at 50ms - assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); - - assertTrue(command.isResponseTimedOut()); - assertFalse(command.isResponseFromFallback()); - assertFalse(command.isResponseRejected()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution timeout where the command implemented getFallback. - */ - @Test - public void testExecutionTimeoutWithFallback() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); - try { - assertEquals(false, command.execute()); - // the time should be 50+ since we timeout at 50ms - assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); - assertTrue(command.isResponseTimedOut()); - assertTrue(command.isResponseFromFallback()); - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a command execution timeout where the command implemented getFallback but it fails. - */ - @Test - public void testExecutionTimeoutFallbackFailure() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE); - try { - command.execute(); - fail("we shouldn't get here"); - } catch (Exception e) { - if (e instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e; - assertNotNull(de.getFallbackException()); - assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof TimeoutException); - } else { - fail("the exception should be HystrixRuntimeException"); - } - } - // the time should be 50+ since we timeout at 50ms - assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test that the circuit-breaker counts a command execution timeout as a 'timeout' and not just failure. - */ - @Test - public void testCircuitBreakerOnExecutionTimeout() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); - try { - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - - command.execute(); - - assertTrue(command.isResponseFromFallback()); - assertFalse(command.isCircuitBreakerOpen()); - assertFalse(command.isResponseShortCircuited()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isResponseTimedOut()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test that the command finishing AFTER a timeout (because thread continues in background) does not register a SUCCESS - */ - @Test - public void testCountersOnExecutionTimeout() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); - try { - command.execute(); - - /* wait long enough for the command to have finished */ - Thread.sleep(200); - - /* response should still be the same as 'testCircuitBreakerOnExecutionTimeout' */ - assertTrue(command.isResponseFromFallback()); - assertFalse(command.isCircuitBreakerOpen()); - assertFalse(command.isResponseShortCircuited()); - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isResponseTimedOut()); - assertFalse(command.isSuccessfulExecution()); - - /* failure and timeout count should be the same as 'testCircuitBreakerOnExecutionTimeout' */ - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - - /* we should NOT have a 'success' counter */ - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a queued command execution timeout where the command didn't implement getFallback. - *

- * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking - * indefinitely by skipping the timeout protection of the execute() command. - */ - @Test - public void testQueuedExecutionTimeoutWithNoFallback() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); - try { - command.queue().get(); - fail("we shouldn't get here"); - } catch (Exception e) { - e.printStackTrace(); - if (e instanceof ExecutionException && e.getCause() instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); - assertNotNull(de.getFallbackException()); - assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof TimeoutException); - } else { - fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); - } - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isResponseTimedOut()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a queued command execution timeout where the command implemented getFallback. - *

- * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking - * indefinitely by skipping the timeout protection of the execute() command. - */ - @Test - public void testQueuedExecutionTimeoutWithFallback() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); - try { - assertEquals(false, command.queue().get()); - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a queued command execution timeout where the command implemented getFallback but it fails. - *

- * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking - * indefinitely by skipping the timeout protection of the execute() command. - */ - @Test - public void testQueuedExecutionTimeoutFallbackFailure() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE); - try { - command.queue().get(); - fail("we shouldn't get here"); - } catch (Exception e) { - if (e instanceof ExecutionException && e.getCause() instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); - assertNotNull(de.getFallbackException()); - assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof TimeoutException); - } else { - fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); - } - } - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a queued command execution timeout where the command didn't implement getFallback. - *

- * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking - * indefinitely by skipping the timeout protection of the execute() command. - */ - @Test - public void testObservedExecutionTimeoutWithNoFallback() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); - try { - command.observe().toBlockingObservable().single(); - fail("we shouldn't get here"); - } catch (Exception e) { - e.printStackTrace(); - if (e instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e; - assertNotNull(de.getFallbackException()); - assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof TimeoutException); - } else { - fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); - } - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isResponseTimedOut()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a queued command execution timeout where the command implemented getFallback. - *

- * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking - * indefinitely by skipping the timeout protection of the execute() command. - */ - @Test - public void testObservedExecutionTimeoutWithFallback() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); - try { - assertEquals(false, command.observe().toBlockingObservable().single()); - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test a queued command execution timeout where the command implemented getFallback but it fails. - *

- * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking - * indefinitely by skipping the timeout protection of the execute() command. - */ - @Test - public void testObservedExecutionTimeoutFallbackFailure() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE); - try { - command.observe().toBlockingObservable().single(); - fail("we shouldn't get here"); - } catch (Exception e) { - if (e instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e; - assertNotNull(de.getFallbackException()); - assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof TimeoutException); - } else { - fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); - } - } - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test that the circuit-breaker counts a command execution timeout as a 'timeout' and not just failure. - */ - @Test - public void testShortCircuitFallbackCounter() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().setForceShortCircuit(true); - try { - new KnownFailureTestCommandWithFallback(circuitBreaker).execute(); - - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - - KnownFailureTestCommandWithFallback command = new KnownFailureTestCommandWithFallback(circuitBreaker); - command.execute(); - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - - // will be -1 because it never attempted execution - assertTrue(command.getExecutionTimeInMilliseconds() == -1); - assertTrue(command.isResponseShortCircuited()); - assertFalse(command.isResponseTimedOut()); - - // because it was short-circuited to a fallback we don't count an error - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test when a command fails to get queued up in the threadpool where the command didn't implement getFallback. - *

- * We specifically want to protect against developers getting random thread exceptions and instead just correctly receiving HystrixRuntimeException when no fallback exists. - */ - @Test - public void testRejectedThreadWithNoFallback() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SingleThreadedPool pool = new SingleThreadedPool(1); - // fill up the queue - pool.queue.add(new Runnable() { - - @Override - public void run() { - System.out.println("**** queue filler1 ****"); - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - }); - - Future f = null; - TestCommandRejection command = null; - try { - f = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_NOT_IMPLEMENTED).queue(); - command = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_NOT_IMPLEMENTED); - command.queue(); - fail("we shouldn't get here"); - } catch (Exception e) { - e.printStackTrace(); - - // will be -1 because it never attempted execution - assertTrue(command.getExecutionTimeInMilliseconds() == -1); - assertTrue(command.isResponseRejected()); - assertFalse(command.isResponseShortCircuited()); - assertFalse(command.isResponseTimedOut()); - - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - if (e instanceof HystrixRuntimeException && e.getCause() instanceof RejectedExecutionException) { - HystrixRuntimeException de = (HystrixRuntimeException) e; - assertNotNull(de.getFallbackException()); - assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof RejectedExecutionException); - } else { - fail("the exception should be HystrixRuntimeException with cause as RejectedExecutionException"); - } - } - - try { - f.get(); - } catch (Exception e) { - e.printStackTrace(); - fail("The first one should succeed."); - } - - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(50, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test when a command fails to get queued up in the threadpool where the command implemented getFallback. - *

- * We specifically want to protect against developers getting random thread exceptions and instead just correctly receives a fallback. - */ - @Test - public void testRejectedThreadWithFallback() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SingleThreadedPool pool = new SingleThreadedPool(1); - // fill up the queue - pool.queue.add(new Runnable() { - - @Override - public void run() { - System.out.println("**** queue filler1 ****"); - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - }); - - try { - TestCommandRejection command1 = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS); - command1.queue(); - TestCommandRejection command2 = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS); - assertEquals(false, command2.queue().get()); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertFalse(command1.isResponseRejected()); - assertFalse(command1.isResponseFromFallback()); - assertTrue(command2.isResponseRejected()); - assertTrue(command2.isResponseFromFallback()); - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test when a command fails to get queued up in the threadpool where the command implemented getFallback but it fails. - *

- * We specifically want to protect against developers getting random thread exceptions and instead just correctly receives an HystrixRuntimeException. - */ - @Test - public void testRejectedThreadWithFallbackFailure() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SingleThreadedPool pool = new SingleThreadedPool(1); - // fill up the queue - pool.queue.add(new Runnable() { - - @Override - public void run() { - System.out.println("**** queue filler1 ****"); - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - }); - - try { - new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_FAILURE).queue(); - assertEquals(false, new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_FAILURE).queue().get()); - fail("we shouldn't get here"); - } catch (Exception e) { - e.printStackTrace(); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - if (e instanceof HystrixRuntimeException && e.getCause() instanceof RejectedExecutionException) { - HystrixRuntimeException de = (HystrixRuntimeException) e; - assertNotNull(de.getFallbackException()); - assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof RejectedExecutionException); - } else { - fail("the exception should be HystrixRuntimeException with cause as RejectedExecutionException"); - } - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test that we can reject a thread using isQueueSpaceAvailable() instead of just when the pool rejects. - *

- * For example, we have queue size set to 100 but want to reject when we hit 10. - *

- * This allows us to use FastProperties to control our rejection point whereas we can't resize a queue after it's created. - */ - @Test - public void testRejectedThreadUsingQueueSize() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SingleThreadedPool pool = new SingleThreadedPool(10, 1); - // put 1 item in the queue - // the thread pool won't pick it up because we're bypassing the pool and adding to the queue directly so this will keep the queue full - pool.queue.add(new Runnable() { - - @Override - public void run() { - System.out.println("**** queue filler1 ****"); - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - }); - - TestCommandRejection command = null; - try { - // this should fail as we already have 1 in the queue - command = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_NOT_IMPLEMENTED); - command.queue(); - fail("we shouldn't get here"); - } catch (Exception e) { - e.printStackTrace(); - - // will be -1 because it never attempted execution - assertTrue(command.getExecutionTimeInMilliseconds() == -1); - assertTrue(command.isResponseRejected()); - assertFalse(command.isResponseShortCircuited()); - assertFalse(command.isResponseTimedOut()); - - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - if (e instanceof HystrixRuntimeException && e.getCause() instanceof RejectedExecutionException) { - HystrixRuntimeException de = (HystrixRuntimeException) e; - assertNotNull(de.getFallbackException()); - assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof RejectedExecutionException); - } else { - fail("the exception should be HystrixRuntimeException with cause as RejectedExecutionException"); - } - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testTimedOutCommandDoesNotExecute() { - SingleThreadedPool pool = new SingleThreadedPool(5); - - TestCircuitBreaker s1 = new TestCircuitBreaker(); - TestCircuitBreaker s2 = new TestCircuitBreaker(); - - // execution will take 100ms, thread pool has a 600ms timeout - CommandWithCustomThreadPool c1 = new CommandWithCustomThreadPool(s1, pool, 100, HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(600)); - // execution will take 200ms, thread pool has a 20ms timeout - CommandWithCustomThreadPool c2 = new CommandWithCustomThreadPool(s2, pool, 200, HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(20)); - // queue up c1 first - Future c1f = c1.queue(); - // now queue up c2 and wait on it - boolean receivedException = false; - try { - c2.queue().get(); - } catch (Exception e) { - // we expect to get an exception here - receivedException = true; - } - - if (!receivedException) { - fail("We expect to receive an exception for c2 as it's supposed to timeout."); - } - - // c1 will complete after 100ms - try { - c1f.get(); - } catch (Exception e1) { - e1.printStackTrace(); - fail("we should not have failed while getting c1"); - } - assertTrue("c1 is expected to executed but didn't", c1.didExecute); - - // c2 will timeout after 20 ms ... we'll wait longer than the 200ms time to make sure - // the thread doesn't keep running in the background and execute - try { - Thread.sleep(400); - } catch (Exception e) { - throw new RuntimeException("Failed to sleep"); - } - assertFalse("c2 is not expected to execute, but did", c2.didExecute); - - assertEquals(1, s1.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, s1.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, s2.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, s2.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, s2.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testFallbackSemaphore() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - // single thread should work - try { - boolean result = new TestSemaphoreCommandWithSlowFallback(circuitBreaker, 1, 200).queue().get(); - assertTrue(result); - } catch (Exception e) { - // we shouldn't fail on this one - throw new RuntimeException(e); - } - - // 2 threads, the second should be rejected by the fallback semaphore - boolean exceptionReceived = false; - Future result = null; - try { - result = new TestSemaphoreCommandWithSlowFallback(circuitBreaker, 1, 400).queue(); - // make sure that thread gets a chance to run before queuing the next one - Thread.sleep(50); - Future result2 = new TestSemaphoreCommandWithSlowFallback(circuitBreaker, 1, 200).queue(); - result2.get(); - } catch (Exception e) { - e.printStackTrace(); - exceptionReceived = true; - } - - try { - assertTrue(result.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - if (!exceptionReceived) { - fail("We expected an exception on the 2nd get"); - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - // TestSemaphoreCommandWithSlowFallback always fails so all 3 should show failure - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - // the 1st thread executes single-threaded and gets a fallback, the next 2 are concurrent so only 1 of them is permitted by the fallback semaphore so 1 is rejected - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - // whenever a fallback_rejection occurs it is also a fallback_failure - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - // we should not have rejected any via the "execution semaphore" but instead via the "fallback semaphore" - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - // the rest should not be involved in this test - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testExecutionSemaphoreWithQueue() { - final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - // single thread should work - try { - boolean result = new TestSemaphoreCommand(circuitBreaker, 1, 200).queue().get(); - assertTrue(result); - } catch (Exception e) { - // we shouldn't fail on this one - throw new RuntimeException(e); - } - - final AtomicBoolean exceptionReceived = new AtomicBoolean(); - - final TryableSemaphore semaphore = - new TryableSemaphore(HystrixProperty.Factory.asProperty(1)); - - Runnable r = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { - - @Override - public void run() { - try { - new TestSemaphoreCommand(circuitBreaker, semaphore, 200).queue().get(); - } catch (Exception e) { - e.printStackTrace(); - exceptionReceived.set(true); - } - } - - }); - // 2 threads, the second should be rejected by the semaphore - Thread t1 = new Thread(r); - Thread t2 = new Thread(r); - - t1.start(); - t2.start(); - try { - t1.join(); - t2.join(); - } catch (Exception e) { - e.printStackTrace(); - fail("failed waiting on threads"); - } - - if (!exceptionReceived.get()) { - fail("We expected an exception on the 2nd get"); - } - - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - // we don't have a fallback so threw an exception when rejected - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - // not a failure as the command never executed so can't fail - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - // no fallback failure as there isn't a fallback implemented - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - // we should have rejected via semaphore - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - // the rest should not be involved in this test - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testExecutionSemaphoreWithExecution() { - final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - // single thread should work - try { - TestSemaphoreCommand command = new TestSemaphoreCommand(circuitBreaker, 1, 200); - boolean result = command.execute(); - assertFalse(command.isExecutedInThread()); - assertTrue(result); - } catch (Exception e) { - // we shouldn't fail on this one - throw new RuntimeException(e); - } - - final ArrayBlockingQueue results = new ArrayBlockingQueue(2); - - final AtomicBoolean exceptionReceived = new AtomicBoolean(); - - final TryableSemaphore semaphore = - new TryableSemaphore(HystrixProperty.Factory.asProperty(1)); - - Runnable r = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { - - @Override - public void run() { - try { - results.add(new TestSemaphoreCommand(circuitBreaker, semaphore, 200).execute()); - } catch (Exception e) { - e.printStackTrace(); - exceptionReceived.set(true); - } - } - - }); - // 2 threads, the second should be rejected by the semaphore - Thread t1 = new Thread(r); - Thread t2 = new Thread(r); - - t1.start(); - t2.start(); - try { - t1.join(); - t2.join(); - } catch (Exception e) { - e.printStackTrace(); - fail("failed waiting on threads"); - } - - if (!exceptionReceived.get()) { - fail("We expected an exception on the 2nd get"); - } - - // only 1 value is expected as the other should have thrown an exception - assertEquals(1, results.size()); - // should contain only a true result - assertTrue(results.contains(Boolean.TRUE)); - assertFalse(results.contains(Boolean.FALSE)); - - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - // no failure ... we throw an exception because of rejection but the command does not fail execution - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - // there is no fallback implemented so no failure can occur on it - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - // we rejected via semaphore - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - // the rest should not be involved in this test - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testRejectedExecutionSemaphoreWithFallback() { - final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - final ArrayBlockingQueue results = new ArrayBlockingQueue(2); - - final AtomicBoolean exceptionReceived = new AtomicBoolean(); - - Runnable r = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { - - @Override - public void run() { - try { - results.add(new TestSemaphoreCommandWithFallback(circuitBreaker, 1, 200, false).execute()); - } catch (Exception e) { - e.printStackTrace(); - exceptionReceived.set(true); - } - } - - }); - - // 2 threads, the second should be rejected by the semaphore and return fallback - Thread t1 = new Thread(r); - Thread t2 = new Thread(r); - - t1.start(); - t2.start(); - try { - t1.join(); - t2.join(); - } catch (Exception e) { - e.printStackTrace(); - fail("failed waiting on threads"); - } - - if (exceptionReceived.get()) { - fail("We should have received a fallback response"); - } - - // both threads should have returned values - assertEquals(2, results.size()); - // should contain both a true and false result - assertTrue(results.contains(Boolean.TRUE)); - assertTrue(results.contains(Boolean.FALSE)); - - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - // the rest should not be involved in this test - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - System.out.println("**** DONE"); - - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Tests that semaphores are counted separately for commands with unique keys - */ - @Test - public void testSemaphorePermitsInUse() { - final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - - // this semaphore will be shared across multiple command instances - final TryableSemaphore sharedSemaphore = - new TryableSemaphore(HystrixProperty.Factory.asProperty(3)); - - // used to wait until all commands have started - final CountDownLatch startLatch = new CountDownLatch(sharedSemaphore.numberOfPermits.get() + 1); - - // used to signal that all command can finish - final CountDownLatch sharedLatch = new CountDownLatch(1); - - final Runnable sharedSemaphoreRunnable = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { - public void run() { - try { - new LatchedSemaphoreCommand(circuitBreaker, sharedSemaphore, startLatch, sharedLatch).execute(); - } catch (Exception e) { - e.printStackTrace(); - } - } - }); - - // creates group of threads each using command sharing a single semaphore - - // I create extra threads and commands so that I can verify that some of them fail to obtain a semaphore - final int sharedThreadCount = sharedSemaphore.numberOfPermits.get() * 2; - final Thread[] sharedSemaphoreThreads = new Thread[sharedThreadCount]; - for (int i = 0; i < sharedThreadCount; i++) { - sharedSemaphoreThreads[i] = new Thread(sharedSemaphoreRunnable); - } - - // creates thread using isolated semaphore - final TryableSemaphore isolatedSemaphore = - new TryableSemaphore(HystrixProperty.Factory.asProperty(1)); - - final CountDownLatch isolatedLatch = new CountDownLatch(1); - - // tracks failures to obtain semaphores - final AtomicInteger failureCount = new AtomicInteger(); - - final Thread isolatedThread = new Thread(new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { - public void run() { - try { - new LatchedSemaphoreCommand(circuitBreaker, isolatedSemaphore, startLatch, isolatedLatch).execute(); - } catch (Exception e) { - e.printStackTrace(); - failureCount.incrementAndGet(); - } - } - })); - - // verifies no permits in use before starting threads - assertEquals("wrong number of permits for shared semaphore", 0, sharedSemaphore.getNumberOfPermitsUsed()); - assertEquals("wrong number of permits for isolated semaphore", 0, isolatedSemaphore.getNumberOfPermitsUsed()); - - for (int i = 0; i < sharedThreadCount; i++) { - sharedSemaphoreThreads[i].start(); - } - isolatedThread.start(); - - // waits until all commands have started - try { - startLatch.await(1000, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - // verifies that all semaphores are in use - assertEquals("wrong number of permits for shared semaphore", - sharedSemaphore.numberOfPermits.get().longValue(), sharedSemaphore.getNumberOfPermitsUsed()); - assertEquals("wrong number of permits for isolated semaphore", - isolatedSemaphore.numberOfPermits.get().longValue(), isolatedSemaphore.getNumberOfPermitsUsed()); - - // signals commands to finish - sharedLatch.countDown(); - isolatedLatch.countDown(); - - try { - for (int i = 0; i < sharedThreadCount; i++) { - sharedSemaphoreThreads[i].join(); - } - isolatedThread.join(); - } catch (Exception e) { - e.printStackTrace(); - fail("failed waiting on threads"); - } - - // verifies no permits in use after finishing threads - assertEquals("wrong number of permits for shared semaphore", 0, sharedSemaphore.getNumberOfPermitsUsed()); - assertEquals("wrong number of permits for isolated semaphore", 0, isolatedSemaphore.getNumberOfPermitsUsed()); - - // verifies that some executions failed - final int expectedFailures = sharedSemaphore.getNumberOfPermitsUsed(); - assertEquals("failures expected but did not happen", expectedFailures, failureCount.get()); - } - - /** - * Test that HystrixOwner can be passed in dynamically. - */ - @Test - public void testDynamicOwner() { - try { - TestHystrixCommand command = new DynamicOwnerTestCommand(CommandGroupForUnitTest.OWNER_ONE); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(true, command.execute()); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - } catch (Exception e) { - e.printStackTrace(); - fail("We received an exception."); - } - } - - /** - * Test a successful command execution. - */ - @Test - public void testDynamicOwnerFails() { - try { - TestHystrixCommand command = new DynamicOwnerTestCommand(null); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(true, command.execute()); - fail("we should have thrown an exception as we need an owner"); - } catch (Exception e) { - // success if we get here - } - } - - /** - * Test that HystrixCommandKey can be passed in dynamically. - */ - @Test - public void testDynamicKey() { - try { - DynamicOwnerAndKeyTestCommand command1 = new DynamicOwnerAndKeyTestCommand(CommandGroupForUnitTest.OWNER_ONE, CommandKeyForUnitTest.KEY_ONE); - assertEquals(true, command1.execute()); - DynamicOwnerAndKeyTestCommand command2 = new DynamicOwnerAndKeyTestCommand(CommandGroupForUnitTest.OWNER_ONE, CommandKeyForUnitTest.KEY_TWO); - assertEquals(true, command2.execute()); - - // 2 different circuit breakers should be created - assertNotSame(command1.getCircuitBreaker(), command2.getCircuitBreaker()); - } catch (Exception e) { - e.printStackTrace(); - fail("We received an exception."); - } - } - - /** - * Test Request scoped caching of commands so that a 2nd duplicate call doesn't execute but returns the previous Future - */ - @Test - public void testRequestCache1() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); - SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); - - assertTrue(command1.isCommandRunningInThread()); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - - try { - assertEquals("A", f1.get()); - assertEquals("A", f2.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - assertTrue(command1.executed); - // the second one should not have executed as it should have received the cached value instead - assertFalse(command2.executed); - - // the execution log for command1 should show a SUCCESS - assertEquals(1, command1.getExecutionEvents().size()); - assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(command1.getExecutionTimeInMilliseconds() > -1); - assertFalse(command1.isResponseFromCache()); - - // the execution log for command2 should show it came from cache - assertEquals(2, command2.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); - assertTrue(command2.getExecutionTimeInMilliseconds() == -1); - assertTrue(command2.isResponseFromCache()); - - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test Request scoped caching doesn't prevent different ones from executing - */ - @Test - public void testRequestCache2() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); - SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "B"); - - assertTrue(command1.isCommandRunningInThread()); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - - try { - assertEquals("A", f1.get()); - assertEquals("B", f2.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - assertTrue(command1.executed); - // both should execute as they are different - assertTrue(command2.executed); - - // the execution log for command1 should show a SUCCESS - assertEquals(1, command1.getExecutionEvents().size()); - assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command2 should show a SUCCESS - assertEquals(1, command2.getExecutionEvents().size()); - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(command2.getExecutionTimeInMilliseconds() > -1); - assertFalse(command2.isResponseFromCache()); - - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test Request scoped caching with a mixture of commands - */ - @Test - public void testRequestCache3() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); - SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "B"); - SuccessfulCacheableCommand command3 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); - - assertTrue(command1.isCommandRunningInThread()); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - Future f3 = command3.queue(); - - try { - assertEquals("A", f1.get()); - assertEquals("B", f2.get()); - assertEquals("A", f3.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - assertTrue(command1.executed); - // both should execute as they are different - assertTrue(command2.executed); - // but the 3rd should come from cache - assertFalse(command3.executed); - - // the execution log for command1 should show a SUCCESS - assertEquals(1, command1.getExecutionEvents().size()); - assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command2 should show a SUCCESS - assertEquals(1, command2.getExecutionEvents().size()); - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command3 should show it came from cache - assertEquals(2, command3.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE - assertTrue(command3.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); - assertTrue(command3.getExecutionTimeInMilliseconds() == -1); - assertTrue(command3.isResponseFromCache()); - - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test Request scoped caching of commands so that a 2nd duplicate call doesn't execute but returns the previous Future - */ - @Test - public void testRequestCacheWithSlowExecution() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SlowCacheableCommand command1 = new SlowCacheableCommand(circuitBreaker, "A", 200); - SlowCacheableCommand command2 = new SlowCacheableCommand(circuitBreaker, "A", 100); - SlowCacheableCommand command3 = new SlowCacheableCommand(circuitBreaker, "A", 100); - SlowCacheableCommand command4 = new SlowCacheableCommand(circuitBreaker, "A", 100); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - Future f3 = command3.queue(); - Future f4 = command4.queue(); - - try { - assertEquals("A", f2.get()); - assertEquals("A", f3.get()); - assertEquals("A", f4.get()); - - assertEquals("A", f1.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - assertTrue(command1.executed); - // the second one should not have executed as it should have received the cached value instead - assertFalse(command2.executed); - assertFalse(command3.executed); - assertFalse(command4.executed); - - // the execution log for command1 should show a SUCCESS - assertEquals(1, command1.getExecutionEvents().size()); - assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(command1.getExecutionTimeInMilliseconds() > -1); - assertFalse(command1.isResponseFromCache()); - - // the execution log for command2 should show it came from cache - assertEquals(2, command2.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); - assertTrue(command2.getExecutionTimeInMilliseconds() == -1); - assertTrue(command2.isResponseFromCache()); - - assertTrue(command3.isResponseFromCache()); - assertTrue(command3.getExecutionTimeInMilliseconds() == -1); - assertTrue(command4.isResponseFromCache()); - assertTrue(command4.getExecutionTimeInMilliseconds() == -1); - - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - - System.out.println("HystrixRequestLog: " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); - } - - /** - * Test Request scoped caching with a mixture of commands - */ - @Test - public void testNoRequestCache3() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, false, "A"); - SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, false, "B"); - SuccessfulCacheableCommand command3 = new SuccessfulCacheableCommand(circuitBreaker, false, "A"); - - assertTrue(command1.isCommandRunningInThread()); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - Future f3 = command3.queue(); - - try { - assertEquals("A", f1.get()); - assertEquals("B", f2.get()); - assertEquals("A", f3.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - assertTrue(command1.executed); - // both should execute as they are different - assertTrue(command2.executed); - // this should also execute since we disabled the cache - assertTrue(command3.executed); - - // the execution log for command1 should show a SUCCESS - assertEquals(1, command1.getExecutionEvents().size()); - assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command2 should show a SUCCESS - assertEquals(1, command2.getExecutionEvents().size()); - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command3 should show a SUCCESS - assertEquals(1, command3.getExecutionEvents().size()); - assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test Request scoped caching with a mixture of commands - */ - @Test - public void testRequestCacheViaQueueSemaphore1() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); - SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "B"); - SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); - - assertFalse(command1.isCommandRunningInThread()); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - Future f3 = command3.queue(); - - try { - assertEquals("A", f1.get()); - assertEquals("B", f2.get()); - assertEquals("A", f3.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - assertTrue(command1.executed); - // both should execute as they are different - assertTrue(command2.executed); - // but the 3rd should come from cache - assertFalse(command3.executed); - - // the execution log for command1 should show a SUCCESS - assertEquals(1, command1.getExecutionEvents().size()); - assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command2 should show a SUCCESS - assertEquals(1, command2.getExecutionEvents().size()); - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command3 should show it comes from cache - assertEquals(2, command3.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE - assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - assertTrue(command3.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); - - assertTrue(command3.isResponseFromCache()); - assertTrue(command3.getExecutionTimeInMilliseconds() == -1); - - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test Request scoped caching with a mixture of commands - */ - @Test - public void testNoRequestCacheViaQueueSemaphore1() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); - SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "B"); - SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); - - assertFalse(command1.isCommandRunningInThread()); - - Future f1 = command1.queue(); - Future f2 = command2.queue(); - Future f3 = command3.queue(); - - try { - assertEquals("A", f1.get()); - assertEquals("B", f2.get()); - assertEquals("A", f3.get()); - } catch (Exception e) { - throw new RuntimeException(e); - } - - assertTrue(command1.executed); - // both should execute as they are different - assertTrue(command2.executed); - // this should also execute because caching is disabled - assertTrue(command3.executed); - - // the execution log for command1 should show a SUCCESS - assertEquals(1, command1.getExecutionEvents().size()); - assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command2 should show a SUCCESS - assertEquals(1, command2.getExecutionEvents().size()); - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command3 should show a SUCCESS - assertEquals(1, command3.getExecutionEvents().size()); - assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test Request scoped caching with a mixture of commands - */ - @Test - public void testRequestCacheViaExecuteSemaphore1() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); - SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "B"); - SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); - - assertFalse(command1.isCommandRunningInThread()); - - String f1 = command1.execute(); - String f2 = command2.execute(); - String f3 = command3.execute(); - - assertEquals("A", f1); - assertEquals("B", f2); - assertEquals("A", f3); - - assertTrue(command1.executed); - // both should execute as they are different - assertTrue(command2.executed); - // but the 3rd should come from cache - assertFalse(command3.executed); - - // the execution log for command1 should show a SUCCESS - assertEquals(1, command1.getExecutionEvents().size()); - assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command2 should show a SUCCESS - assertEquals(1, command2.getExecutionEvents().size()); - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command3 should show it comes from cache - assertEquals(2, command3.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE - assertTrue(command3.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); - - assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test Request scoped caching with a mixture of commands - */ - @Test - public void testNoRequestCacheViaExecuteSemaphore1() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); - SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "B"); - SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); - - assertFalse(command1.isCommandRunningInThread()); - - String f1 = command1.execute(); - String f2 = command2.execute(); - String f3 = command3.execute(); - - assertEquals("A", f1); - assertEquals("B", f2); - assertEquals("A", f3); - - assertTrue(command1.executed); - // both should execute as they are different - assertTrue(command2.executed); - // this should also execute because caching is disabled - assertTrue(command3.executed); - - // the execution log for command1 should show a SUCCESS - assertEquals(1, command1.getExecutionEvents().size()); - assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command2 should show a SUCCESS - assertEquals(1, command2.getExecutionEvents().size()); - assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - // the execution log for command3 should show a SUCCESS - assertEquals(1, command3.getExecutionEvents().size()); - assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); - - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testNoRequestCacheOnTimeoutThrowsException() throws Exception { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - NoRequestCacheTimeoutWithoutFallback r1 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); - try { - System.out.println("r1 value: " + r1.execute()); - // we should have thrown an exception - fail("expected a timeout"); - } catch (HystrixRuntimeException e) { - assertTrue(r1.isResponseTimedOut()); - // what we want - } - - NoRequestCacheTimeoutWithoutFallback r2 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); - try { - r2.execute(); - // we should have thrown an exception - fail("expected a timeout"); - } catch (HystrixRuntimeException e) { - assertTrue(r2.isResponseTimedOut()); - // what we want - } - - NoRequestCacheTimeoutWithoutFallback r3 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); - Future f3 = r3.queue(); - try { - f3.get(); - // we should have thrown an exception - fail("expected a timeout"); - } catch (ExecutionException e) { - e.printStackTrace(); - assertTrue(r3.isResponseTimedOut()); - // what we want - } - - Thread.sleep(500); // timeout on command is set to 200ms - - NoRequestCacheTimeoutWithoutFallback r4 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); - try { - r4.execute(); - // we should have thrown an exception - fail("expected a timeout"); - } catch (HystrixRuntimeException e) { - assertTrue(r4.isResponseTimedOut()); - assertFalse(r4.isResponseFromFallback()); - // what we want - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testRequestCacheOnTimeoutCausesNullPointerException() throws Exception { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - // Expect it to time out - all results should be false - assertFalse(new RequestCacheNullPointerExceptionCase(circuitBreaker).execute()); - assertFalse(new RequestCacheNullPointerExceptionCase(circuitBreaker).execute()); // return from cache #1 - assertFalse(new RequestCacheNullPointerExceptionCase(circuitBreaker).execute()); // return from cache #2 - Thread.sleep(500); // timeout on command is set to 200ms - Boolean value = new RequestCacheNullPointerExceptionCase(circuitBreaker).execute(); // return from cache #3 - assertFalse(value); - RequestCacheNullPointerExceptionCase c = new RequestCacheNullPointerExceptionCase(circuitBreaker); - Future f = c.queue(); // return from cache #4 - // the bug is that we're getting a null Future back, rather than a Future that returns false - assertNotNull(f); - assertFalse(f.get()); - - assertTrue(c.isResponseFromFallback()); - assertTrue(c.isResponseTimedOut()); - assertFalse(c.isFailedExecution()); - assertFalse(c.isResponseShortCircuited()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(5, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - - HystrixCommand[] executeCommands = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[] {}); - - System.out.println(":executeCommands[0].getExecutionEvents()" + executeCommands[0].getExecutionEvents()); - assertEquals(2, executeCommands[0].getExecutionEvents().size()); - assertTrue(executeCommands[0].getExecutionEvents().contains(HystrixEventType.FALLBACK_SUCCESS)); - assertTrue(executeCommands[0].getExecutionEvents().contains(HystrixEventType.TIMEOUT)); - assertTrue(executeCommands[0].getExecutionTimeInMilliseconds() > -1); - assertTrue(executeCommands[0].isResponseTimedOut()); - assertTrue(executeCommands[0].isResponseFromFallback()); - assertFalse(executeCommands[0].isResponseFromCache()); - - assertEquals(3, executeCommands[1].getExecutionEvents().size()); // it will include FALLBACK_SUCCESS/TIMEOUT + RESPONSE_FROM_CACHE - assertTrue(executeCommands[1].getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); - assertTrue(executeCommands[1].getExecutionTimeInMilliseconds() == -1); - assertTrue(executeCommands[1].isResponseFromCache()); - assertTrue(executeCommands[1].isResponseTimedOut()); - assertTrue(executeCommands[1].isResponseFromFallback()); - } - - @Test - public void testRequestCacheOnTimeoutThrowsException() throws Exception { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - RequestCacheTimeoutWithoutFallback r1 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); - try { - System.out.println("r1 value: " + r1.execute()); - // we should have thrown an exception - fail("expected a timeout"); - } catch (HystrixRuntimeException e) { - assertTrue(r1.isResponseTimedOut()); - // what we want - } - - RequestCacheTimeoutWithoutFallback r2 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); - try { - r2.execute(); - // we should have thrown an exception - fail("expected a timeout"); - } catch (HystrixRuntimeException e) { - assertTrue(r2.isResponseTimedOut()); - // what we want - } - - RequestCacheTimeoutWithoutFallback r3 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); - Future f3 = r3.queue(); - try { - f3.get(); - // we should have thrown an exception - fail("expected a timeout"); - } catch (ExecutionException e) { - e.printStackTrace(); - assertTrue(r3.isResponseTimedOut()); - // what we want - } - - Thread.sleep(500); // timeout on command is set to 200ms - - RequestCacheTimeoutWithoutFallback r4 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); - try { - r4.execute(); - // we should have thrown an exception - fail("expected a timeout"); - } catch (HystrixRuntimeException e) { - assertTrue(r4.isResponseTimedOut()); - assertFalse(r4.isResponseFromFallback()); - // what we want - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testRequestCacheOnThreadRejectionThrowsException() throws Exception { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - CountDownLatch completionLatch = new CountDownLatch(1); - RequestCacheThreadRejectionWithoutFallback r1 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); - try { - System.out.println("r1: " + r1.execute()); - // we should have thrown an exception - fail("expected a rejection"); - } catch (HystrixRuntimeException e) { - assertTrue(r1.isResponseRejected()); - // what we want - } - - RequestCacheThreadRejectionWithoutFallback r2 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); - try { - System.out.println("r2: " + r2.execute()); - // we should have thrown an exception - fail("expected a rejection"); - } catch (HystrixRuntimeException e) { - // e.printStackTrace(); - assertTrue(r2.isResponseRejected()); - // what we want - } - - RequestCacheThreadRejectionWithoutFallback r3 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); - try { - System.out.println("f3: " + r3.queue().get()); - // we should have thrown an exception - fail("expected a rejection"); - } catch (HystrixRuntimeException e) { - // e.printStackTrace(); - assertTrue(r3.isResponseRejected()); - // what we want - } - - // let the command finish (only 1 should actually be blocked on this due to the response cache) - completionLatch.countDown(); - - // then another after the command has completed - RequestCacheThreadRejectionWithoutFallback r4 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); - try { - System.out.println("r4: " + r4.execute()); - // we should have thrown an exception - fail("expected a rejection"); - } catch (HystrixRuntimeException e) { - // e.printStackTrace(); - assertTrue(r4.isResponseRejected()); - assertFalse(r4.isResponseFromFallback()); - // what we want - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /** - * Test that we can do basic execution without a RequestVariable being initialized. - */ - @Test - public void testBasicExecutionWorksWithoutRequestVariable() { - try { - /* force the RequestVariable to not be initialized */ - HystrixRequestContext.setContextOnCurrentThread(null); - - TestHystrixCommand command = new SuccessfulTestCommand(); - assertEquals(true, command.execute()); - - TestHystrixCommand command2 = new SuccessfulTestCommand(); - assertEquals(true, command2.queue().get()); - - // we should be able to execute without a RequestVariable if ... - // 1) We don't have a cacheKey - // 2) We don't ask for the RequestLog - // 3) We don't do collapsing - - } catch (Exception e) { - e.printStackTrace(); - fail("We received an exception => " + e.getMessage()); - } - } - - /** - * Test that if we try and execute a command with a cacheKey without initializing RequestVariable that it gives an error. - */ - @Test - public void testCacheKeyExecutionRequiresRequestVariable() { - try { - /* force the RequestVariable to not be initialized */ - HystrixRequestContext.setContextOnCurrentThread(null); - - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - - SuccessfulCacheableCommand command = new SuccessfulCacheableCommand(circuitBreaker, true, "one"); - assertEquals(true, command.execute()); - - SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "two"); - assertEquals(true, command2.queue().get()); - - fail("We expect an exception because cacheKey requires RequestVariable."); - - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. - */ - @Test - public void testBadRequestExceptionViaExecuteInThread() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - try { - new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).execute(); - fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); - } catch (HystrixBadRequestException e) { - // success - e.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - } - - /** - * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. - */ - @Test - public void testBadRequestExceptionViaQueueInThread() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - try { - new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).queue().get(); - fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); - } catch (ExecutionException e) { - e.printStackTrace(); - if (e.getCause() instanceof HystrixBadRequestException) { - // success - } else { - fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - } - - /** - * Test that BadRequestException behavior works the same on a cached response. - */ - @Test - public void testBadRequestExceptionViaQueueInThreadOnResponseFromCache() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - - // execute once to cache the value - try { - new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).execute(); - } catch (Throwable e) { - // ignore - } - - try { - new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).queue().get(); - fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); - } catch (ExecutionException e) { - e.printStackTrace(); - if (e.getCause() instanceof HystrixBadRequestException) { - // success - } else { - fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - } - - /** - * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. - */ - @Test - public void testBadRequestExceptionViaExecuteInSemaphore() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - try { - new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.SEMAPHORE).execute(); - fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); - } catch (HystrixBadRequestException e) { - // success - e.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - } - - /** - * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. - */ - @Test - public void testBadRequestExceptionViaQueueInSemaphore() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - try { - new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.SEMAPHORE).queue().get(); - fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); - } catch (ExecutionException e) { - e.printStackTrace(); - if (e.getCause() instanceof HystrixBadRequestException) { - // success - } else { - fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); - } - } catch (Exception e) { - e.printStackTrace(); - fail(); - } - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - } - - /** - * Test a checked Exception being thrown - */ - @Test - public void testCheckedExceptionViaExecute() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - CommandWithCheckedException command = new CommandWithCheckedException(circuitBreaker); - try { - command.execute(); - fail("we expect to receive a " + Exception.class.getSimpleName()); - } catch (Exception e) { - assertEquals("simulated checked exception message", e.getCause().getMessage()); - } - - assertEquals("simulated checked exception message", command.getFailedExecutionException().getMessage()); - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - } - - /** - * Test a java.lang.Error being thrown - * - * @throws InterruptedException - */ - @Test - public void testCheckedExceptionViaObserve() throws InterruptedException { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - CommandWithCheckedException command = new CommandWithCheckedException(circuitBreaker); - final AtomicReference t = new AtomicReference(); - final CountDownLatch latch = new CountDownLatch(1); - try { - command.observe().subscribe(new Observer() { - - @Override - public void onCompleted() { - latch.countDown(); - } - - @Override - public void onError(Throwable e) { - t.set(e); - latch.countDown(); - } - - @Override - public void onNext(Boolean args) { - - } - - }); - } catch (Exception e) { - e.printStackTrace(); - fail("we should not get anything thrown, it should be emitted via the Observer#onError method"); - } - - latch.await(1, TimeUnit.SECONDS); - assertNotNull(t.get()); - t.get().printStackTrace(); - - assertTrue(t.get() instanceof HystrixRuntimeException); - assertEquals("simulated checked exception message", t.get().getCause().getMessage()); - assertEquals("simulated checked exception message", command.getFailedExecutionException().getMessage()); - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - } - - /** - * Test a java.lang.Error being thrown - */ - @Test - public void testErrorThrownViaExecute() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - CommandWithErrorThrown command = new CommandWithErrorThrown(circuitBreaker); - try { - command.execute(); - fail("we expect to receive a " + Error.class.getSimpleName()); - } catch (Exception e) { - // the actual error is an extra cause level deep because Hystrix needs to wrap Throwable/Error as it's public - // methods only support Exception and it's not a strong enough reason to break backwards compatibility and jump to version 2.x - // so HystrixRuntimeException -> wrapper Exception -> actual Error - assertEquals("simulated java.lang.Error message", e.getCause().getCause().getMessage()); - } - - assertEquals("simulated java.lang.Error message", command.getFailedExecutionException().getCause().getMessage()); - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - } - - /** - * Test a java.lang.Error being thrown - */ - @Test - public void testErrorThrownViaQueue() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - CommandWithErrorThrown command = new CommandWithErrorThrown(circuitBreaker); - try { - command.queue().get(); - fail("we expect to receive an Exception"); - } catch (Exception e) { - // one cause down from ExecutionException to HystrixRuntime - // then the actual error is an extra cause level deep because Hystrix needs to wrap Throwable/Error as it's public - // methods only support Exception and it's not a strong enough reason to break backwards compatibility and jump to version 2.x - // so ExecutionException -> HystrixRuntimeException -> wrapper Exception -> actual Error - assertEquals("simulated java.lang.Error message", e.getCause().getCause().getCause().getMessage()); - } - - assertEquals("simulated java.lang.Error message", command.getFailedExecutionException().getCause().getMessage()); - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - } - - /** - * Test a java.lang.Error being thrown - * - * @throws InterruptedException - */ - @Test - public void testErrorThrownViaObserve() throws InterruptedException { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - CommandWithErrorThrown command = new CommandWithErrorThrown(circuitBreaker); - final AtomicReference t = new AtomicReference(); - final CountDownLatch latch = new CountDownLatch(1); - try { - command.observe().subscribe(new Observer() { - - @Override - public void onCompleted() { - latch.countDown(); - } - - @Override - public void onError(Throwable e) { - t.set(e); - latch.countDown(); - } - - @Override - public void onNext(Boolean args) { - - } - - }); - } catch (Exception e) { - e.printStackTrace(); - fail("we should not get anything thrown, it should be emitted via the Observer#onError method"); - } - - latch.await(1, TimeUnit.SECONDS); - assertNotNull(t.get()); - t.get().printStackTrace(); - - assertTrue(t.get() instanceof HystrixRuntimeException); - // the actual error is an extra cause level deep because Hystrix needs to wrap Throwable/Error as it's public - // methods only support Exception and it's not a strong enough reason to break backwards compatibility and jump to version 2.x - assertEquals("simulated java.lang.Error message", t.get().getCause().getCause().getMessage()); - assertEquals("simulated java.lang.Error message", command.getFailedExecutionException().getCause().getMessage()); - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isFailedExecution()); - - assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - } - - /** - * Execution hook on successful execution - */ - @Test - public void testExecutionHookSuccessfulCommand() { - /* test with execute() */ - TestHystrixCommand command = new SuccessfulTestCommand(); - command.execute(); - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we expect a successful response from run() - assertNotNull(command.builder.executionHook.runSuccessResponse); - // we do not expect an exception - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should not be run as we were successful - assertEquals(0, command.builder.executionHook.startFallback.get()); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackFailureException); - - // the execute() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response from execute() since run() succeeded - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception since run() succeeded - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - - /* test with queue() */ - command = new SuccessfulTestCommand(); - try { - command.queue().get(); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we expect a successful response from run() - assertNotNull(command.builder.executionHook.runSuccessResponse); - // we do not expect an exception - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should not be run as we were successful - assertEquals(0, command.builder.executionHook.startFallback.get()); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackFailureException); - - // the queue() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response from queue() since run() succeeded - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception since run() succeeded - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on successful execution with "fire and forget" approach - */ - @Test - public void testExecutionHookSuccessfulCommandViaFireAndForget() { - TestHystrixCommand command = new SuccessfulTestCommand(); - try { - // do not block on "get()" ... fire this asynchronously - command.queue(); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // wait for command to execute without calling get on the future - while (!command.isExecutionComplete()) { - try { - Thread.sleep(10); - } catch (InterruptedException e) { - throw new RuntimeException("interrupted"); - } - } - - /* - * All the hooks should still work even though we didn't call get() on the future - */ - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we expect a successful response from run() - assertNotNull(command.builder.executionHook.runSuccessResponse); - // we do not expect an exception - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should not be run as we were successful - assertEquals(0, command.builder.executionHook.startFallback.get()); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackFailureException); - - // the queue() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response from queue() since run() succeeded - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception since run() succeeded - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on successful execution with multiple get() calls to Future - */ - @Test - public void testExecutionHookSuccessfulCommandWithMultipleGetsOnFuture() { - TestHystrixCommand command = new SuccessfulTestCommand(); - try { - Future f = command.queue(); - f.get(); - f.get(); - f.get(); - f.get(); - } catch (Exception e) { - throw new RuntimeException(e); - } - - /* - * Despite multiple calls to get() we should only have 1 call to the hooks. - */ - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we expect a successful response from run() - assertNotNull(command.builder.executionHook.runSuccessResponse); - // we do not expect an exception - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should not be run as we were successful - assertEquals(0, command.builder.executionHook.startFallback.get()); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackFailureException); - - // the queue() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response from queue() since run() succeeded - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception since run() succeeded - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on failed execution without a fallback - */ - @Test - public void testExecutionHookRunFailureWithoutFallback() { - /* test with execute() */ - TestHystrixCommand command = new UnknownFailureTestCommandWithoutFallback(); - try { - command.execute(); - fail("Expecting exception"); - } catch (Exception e) { - // ignore - } - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we should not have a response - assertNull(command.builder.executionHook.runSuccessResponse); - // we should have an exception - assertNotNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run since run() failed - assertEquals(1, command.builder.executionHook.startFallback.get()); - // no response since fallback is not implemented - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // not null since it's not implemented and throws an exception - assertNotNull(command.builder.executionHook.fallbackFailureException); - - // the execute() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should not have a response from execute() since we do not have a fallback and run() failed - assertNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should have an exception since run() failed - assertNotNull(command.builder.executionHook.endExecuteFailureException); - // run() failure - assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - - /* test with queue() */ - command = new UnknownFailureTestCommandWithoutFallback(); - try { - command.queue().get(); - fail("Expecting exception"); - } catch (Exception e) { - // ignore - } - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we should not have a response - assertNull(command.builder.executionHook.runSuccessResponse); - // we should have an exception - assertNotNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run since run() failed - assertEquals(1, command.builder.executionHook.startFallback.get()); - // no response since fallback is not implemented - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // not null since it's not implemented and throws an exception - assertNotNull(command.builder.executionHook.fallbackFailureException); - - // the queue() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should not have a response from queue() since we do not have a fallback and run() failed - assertNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should have an exception since run() failed - assertNotNull(command.builder.executionHook.endExecuteFailureException); - // run() failure - assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - - } - - /** - * Execution hook on failed execution with a fallback - */ - @Test - public void testExecutionHookRunFailureWithFallback() { - /* test with execute() */ - TestHystrixCommand command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); - command.execute(); - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we should not have a response from run since run() failed - assertNull(command.builder.executionHook.runSuccessResponse); - // we should have an exception since run() failed - assertNotNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run since run() failed - assertEquals(1, command.builder.executionHook.startFallback.get()); - // a response since fallback is implemented - assertNotNull(command.builder.executionHook.fallbackSuccessResponse); - // null since it's implemented and succeeds - assertNull(command.builder.executionHook.fallbackFailureException); - - // the execute() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response from execute() since we expect a fallback despite failure of run() - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception because we expect a fallback - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - - /* test with queue() */ - command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); - try { - command.queue().get(); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we should not have a response from run since run() failed - assertNull(command.builder.executionHook.runSuccessResponse); - // we should have an exception since run() failed - assertNotNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run since run() failed - assertEquals(1, command.builder.executionHook.startFallback.get()); - // a response since fallback is implemented - assertNotNull(command.builder.executionHook.fallbackSuccessResponse); - // null since it's implemented and succeeds - assertNull(command.builder.executionHook.fallbackFailureException); - - // the queue() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response from queue() since we expect a fallback despite failure of run() - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception because we expect a fallback - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on failed execution with a fallback failure - */ - @Test - public void testExecutionHookRunFailureWithFallbackFailure() { - /* test with execute() */ - TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); - try { - command.execute(); - fail("Expecting exception"); - } catch (Exception e) { - // ignore - } - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we should not have a response because run() and fallback fail - assertNull(command.builder.executionHook.runSuccessResponse); - // we should have an exception because run() and fallback fail - assertNotNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run since run() failed - assertEquals(1, command.builder.executionHook.startFallback.get()); - // no response since fallback fails - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // not null since it's implemented but fails - assertNotNull(command.builder.executionHook.fallbackFailureException); - - // the execute() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should not have a response because run() and fallback fail - assertNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should have an exception because run() and fallback fail - assertNotNull(command.builder.executionHook.endExecuteFailureException); - // run() failure - assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - - /* test with queue() */ - command = new KnownFailureTestCommandWithFallbackFailure(); - try { - command.queue().get(); - fail("Expecting exception"); - } catch (Exception e) { - // ignore - } - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we should not have a response because run() and fallback fail - assertNull(command.builder.executionHook.runSuccessResponse); - // we should have an exception because run() and fallback fail - assertNotNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run since run() failed - assertEquals(1, command.builder.executionHook.startFallback.get()); - // no response since fallback fails - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // not null since it's implemented but fails - assertNotNull(command.builder.executionHook.fallbackFailureException); - - // the queue() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should not have a response because run() and fallback fail - assertNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should have an exception because run() and fallback fail - assertNotNull(command.builder.executionHook.endExecuteFailureException); - // run() failure - assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - assertEquals(1, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on timeout without a fallback - */ - @Test - public void testExecutionHookTimeoutWithoutFallback() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); - try { - command.queue().get(); - fail("Expecting exception"); - } catch (Exception e) { - // ignore - } - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we should not have a response because of timeout and no fallback - assertNull(command.builder.executionHook.runSuccessResponse); - // we should not have an exception because run() didn't fail, it timed out - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run due to timeout - assertEquals(1, command.builder.executionHook.startFallback.get()); - // no response since no fallback - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // not null since no fallback implementation - assertNotNull(command.builder.executionHook.fallbackFailureException); - - // execution occurred - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should not have a response because of timeout and no fallback - assertNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should have an exception because of timeout and no fallback - assertNotNull(command.builder.executionHook.endExecuteFailureException); - // timeout failure - assertEquals(FailureType.TIMEOUT, command.builder.executionHook.endExecuteFailureType); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - - // we need to wait for the thread to complete before the onThreadComplete hook will be called - try { - Thread.sleep(400); - } catch (InterruptedException e) { - // ignore - } - assertEquals(1, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on timeout with a fallback - */ - @Test - public void testExecutionHookTimeoutWithFallback() { - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); - try { - command.queue().get(); - } catch (Exception e) { - throw new RuntimeException("not expecting", e); - } - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we should not have a response because of timeout - assertNull(command.builder.executionHook.runSuccessResponse); - // we should not have an exception because run() didn't fail, it timed out - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run due to timeout - assertEquals(1, command.builder.executionHook.startFallback.get()); - // response since we have a fallback - assertNotNull(command.builder.executionHook.fallbackSuccessResponse); - // null since fallback succeeds - assertNull(command.builder.executionHook.fallbackFailureException); - - // execution occurred - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response because of fallback - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception because of fallback - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(1, command.builder.executionHook.threadStart.get()); - - // we need to wait for the thread to complete before the onThreadComplete hook will be called - try { - Thread.sleep(400); - } catch (InterruptedException e) { - // ignore - } - assertEquals(1, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on rejected with a fallback - */ - @Test - public void testExecutionHookRejectedWithFallback() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); - SingleThreadedPool pool = new SingleThreadedPool(1); - - try { - // fill the queue - new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS).queue(); - new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS).queue(); - } catch (Exception e) { - // ignore - } - - TestCommandRejection command = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS); - try { - // now execute one that will be rejected - command.queue().get(); - } catch (Exception e) { - throw new RuntimeException("not expecting", e); - } - - assertTrue(command.isResponseRejected()); - - // the run() method should not run as we're rejected - assertEquals(0, command.builder.executionHook.startRun.get()); - // we should not have a response because of rejection - assertNull(command.builder.executionHook.runSuccessResponse); - // we should not have an exception because we didn't run - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run due to rejection - assertEquals(1, command.builder.executionHook.startFallback.get()); - // response since we have a fallback - assertNotNull(command.builder.executionHook.fallbackSuccessResponse); - // null since fallback succeeds - assertNull(command.builder.executionHook.fallbackFailureException); - - // execution occurred - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response because of fallback - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception because of fallback - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(0, command.builder.executionHook.threadStart.get()); - assertEquals(0, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on short-circuit with a fallback - */ - @Test - public void testExecutionHookShortCircuitedWithFallbackViaQueue() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().setForceShortCircuit(true); - KnownFailureTestCommandWithoutFallback command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); - try { - // now execute one that will be short-circuited - command.queue().get(); - fail("we expect an error as there is no fallback"); - } catch (Exception e) { - // expecting - } - - assertTrue(command.isResponseShortCircuited()); - - // the run() method should not run as we're rejected - assertEquals(0, command.builder.executionHook.startRun.get()); - // we should not have a response because of rejection - assertNull(command.builder.executionHook.runSuccessResponse); - // we should not have an exception because we didn't run - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run due to rejection - assertEquals(1, command.builder.executionHook.startFallback.get()); - // no response since we don't have a fallback - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // not null since fallback fails and throws an exception - assertNotNull(command.builder.executionHook.fallbackFailureException); - - // execution occurred - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should not have a response because fallback fails - assertNull(command.builder.executionHook.endExecuteSuccessResponse); - // we won't have an exception because short-circuit doesn't have one - assertNull(command.builder.executionHook.endExecuteFailureException); - // but we do expect to receive a onError call with FailureType.SHORTCIRCUIT - assertEquals(FailureType.SHORTCIRCUIT, command.builder.executionHook.endExecuteFailureType); - - // thread execution - assertEquals(0, command.builder.executionHook.threadStart.get()); - assertEquals(0, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on short-circuit with a fallback - */ - @Test - public void testExecutionHookShortCircuitedWithFallbackViaExecute() { - TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().setForceShortCircuit(true); - KnownFailureTestCommandWithoutFallback command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); - try { - // now execute one that will be short-circuited - command.execute(); - fail("we expect an error as there is no fallback"); - } catch (Exception e) { - // expecting - } - - assertTrue(command.isResponseShortCircuited()); - - // the run() method should not run as we're rejected - assertEquals(0, command.builder.executionHook.startRun.get()); - // we should not have a response because of rejection - assertNull(command.builder.executionHook.runSuccessResponse); - // we should not have an exception because we didn't run - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should be run due to rejection - assertEquals(1, command.builder.executionHook.startFallback.get()); - // no response since we don't have a fallback - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // not null since fallback fails and throws an exception - assertNotNull(command.builder.executionHook.fallbackFailureException); - - // execution occurred - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should not have a response because fallback fails - assertNull(command.builder.executionHook.endExecuteSuccessResponse); - // we won't have an exception because short-circuit doesn't have one - assertNull(command.builder.executionHook.endExecuteFailureException); - // but we do expect to receive a onError call with FailureType.SHORTCIRCUIT - assertEquals(FailureType.SHORTCIRCUIT, command.builder.executionHook.endExecuteFailureType); - - // thread execution - assertEquals(0, command.builder.executionHook.threadStart.get()); - assertEquals(0, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on successful execution with semaphore isolation - */ - @Test - public void testExecutionHookSuccessfulCommandWithSemaphoreIsolation() { - /* test with execute() */ - TestSemaphoreCommand command = new TestSemaphoreCommand(new TestCircuitBreaker(), 1, 10); - command.execute(); - - assertFalse(command.isExecutedInThread()); - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we expect a successful response from run() - assertNotNull(command.builder.executionHook.runSuccessResponse); - // we do not expect an exception - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should not be run as we were successful - assertEquals(0, command.builder.executionHook.startFallback.get()); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackFailureException); - - // the execute() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response from execute() since run() succeeded - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception since run() succeeded - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(0, command.builder.executionHook.threadStart.get()); - assertEquals(0, command.builder.executionHook.threadComplete.get()); - - /* test with queue() */ - command = new TestSemaphoreCommand(new TestCircuitBreaker(), 1, 10); - try { - command.queue().get(); - } catch (Exception e) { - throw new RuntimeException(e); - } - - assertFalse(command.isExecutedInThread()); - - // the run() method should run as we're not short-circuited or rejected - assertEquals(1, command.builder.executionHook.startRun.get()); - // we expect a successful response from run() - assertNotNull(command.builder.executionHook.runSuccessResponse); - // we do not expect an exception - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should not be run as we were successful - assertEquals(0, command.builder.executionHook.startFallback.get()); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // null since it didn't run - assertNull(command.builder.executionHook.fallbackFailureException); - - // the queue() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should have a response from queue() since run() succeeded - assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); - // we should not have an exception since run() succeeded - assertNull(command.builder.executionHook.endExecuteFailureException); - - // thread execution - assertEquals(0, command.builder.executionHook.threadStart.get()); - assertEquals(0, command.builder.executionHook.threadComplete.get()); - } - - /** - * Execution hook on successful execution with semaphore isolation - */ - @Test - public void testExecutionHookFailureWithSemaphoreIsolation() { - /* test with execute() */ - final TryableSemaphore semaphore = - new TryableSemaphore(HystrixProperty.Factory.asProperty(0)); - - TestSemaphoreCommand command = new TestSemaphoreCommand(new TestCircuitBreaker(), semaphore, 200); - try { - command.execute(); - fail("we expect a failure"); - } catch (Exception e) { - // expected - } - - assertFalse(command.isExecutedInThread()); - assertTrue(command.isResponseRejected()); - - // the run() method should not run as we are rejected - assertEquals(0, command.builder.executionHook.startRun.get()); - // null as run() does not get invoked - assertNull(command.builder.executionHook.runSuccessResponse); - // null as run() does not get invoked - assertNull(command.builder.executionHook.runFailureException); - - // the fallback() method should run because of rejection - assertEquals(1, command.builder.executionHook.startFallback.get()); - // null since there is no fallback - assertNull(command.builder.executionHook.fallbackSuccessResponse); - // not null since the fallback is not implemented - assertNotNull(command.builder.executionHook.fallbackFailureException); - - // the execute() method was used - assertEquals(1, command.builder.executionHook.startExecute.get()); - // we should not have a response since fallback has nothing - assertNull(command.builder.executionHook.endExecuteSuccessResponse); - // we won't have an exception because rejection doesn't have one - assertNull(command.builder.executionHook.endExecuteFailureException); - // but we do expect to receive a onError call with FailureType.SHORTCIRCUIT - assertEquals(FailureType.REJECTED_SEMAPHORE_EXECUTION, command.builder.executionHook.endExecuteFailureType); - - // thread execution - assertEquals(0, command.builder.executionHook.threadStart.get()); - assertEquals(0, command.builder.executionHook.threadComplete.get()); - } - - /** - * Test a command execution that fails but has a fallback. - */ - @Test - public void testExecutionFailureWithFallbackImplementedButDisabled() { - TestHystrixCommand commandEnabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), true); - try { - assertEquals(false, commandEnabled.execute()); - } catch (Exception e) { - e.printStackTrace(); - fail("We should have received a response from the fallback."); - } - - TestHystrixCommand commandDisabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), false); - try { - assertEquals(false, commandDisabled.execute()); - fail("expect exception thrown"); - } catch (Exception e) { - // expected - } - - assertEquals("we failed with a simulated issue", commandDisabled.getFailedExecutionException().getMessage()); - - assertTrue(commandDisabled.isFailedExecution()); - - assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(1, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, commandDisabled.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - @Test - public void testExecutionTimeoutValue() { - HystrixCommand.Setter properties = HystrixCommand.Setter - .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestKey")) - .andCommandPropertiesDefaults(HystrixCommandProperties.Setter() - .withExecutionIsolationThreadTimeoutInMilliseconds(50)); - - HystrixCommand command = new HystrixCommand(properties) { - @Override - protected String run() throws Exception { - Thread.sleep(3000); - // should never reach here - return "hello"; - } - - @Override - protected String getFallback() { - if (isResponseTimedOut()) { - return "timed-out"; - } else { - return "abc"; - } - } - }; - - String value = command.execute(); - assertTrue(command.isResponseTimedOut()); - assertEquals("expected fallback value", "timed-out", value); - - } - - /** - * See https://github.com/Netflix/Hystrix/issues/212 - */ - @Test - public void testObservableTimeoutNoFallbackThreadContext() { - final AtomicReference onErrorThread = new AtomicReference(); - final AtomicBoolean isRequestContextInitialized = new AtomicBoolean(); - TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); - try { - command.toObservable().doOnError(new Action1() { - - @Override - public void call(Throwable t1) { - System.out.println("onError: " + t1); - System.out.println("onError Thread: " + Thread.currentThread()); - System.out.println("ThreadContext in onError: " + HystrixRequestContext.isCurrentThreadInitialized()); - onErrorThread.set(Thread.currentThread()); - isRequestContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); - } - - }).toBlockingObservable().single(); - throw new RuntimeException("expected error to be thrown"); - } catch (Throwable e) { - assertTrue(isRequestContextInitialized.get()); - assertTrue(onErrorThread.get().getName().startsWith("RxComputationThreadPool")); - - if (e instanceof HystrixRuntimeException) { - HystrixRuntimeException de = (HystrixRuntimeException) e; - assertNotNull(de.getFallbackException()); - assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); - assertNotNull(de.getImplementingClass()); - assertNotNull(de.getCause()); - assertTrue(de.getCause() instanceof TimeoutException); - } else { - fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); - } - } - - assertTrue(command.getExecutionTimeInMilliseconds() > -1); - assertTrue(command.isResponseTimedOut()); - - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); - assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); - - assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); - - assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); - } - - /* ******************************************************************************** */ - /* ******************************************************************************** */ - /* private HystrixCommand class implementations for unit testing */ - /* ******************************************************************************** */ - /* ******************************************************************************** */ - - /** - * Used by UnitTest command implementations to provide base defaults for constructor and a builder pattern for the arguments being passed in. - */ - /* package */static abstract class TestHystrixCommand extends HystrixCommand { - - final TestCommandBuilder builder; - - TestHystrixCommand(TestCommandBuilder builder) { - super(builder.owner, builder.dependencyKey, builder.threadPoolKey, builder.circuitBreaker, builder.threadPool, - builder.commandPropertiesDefaults, builder.threadPoolPropertiesDefaults, builder.metrics, - builder.fallbackSemaphore, builder.executionSemaphore, TEST_PROPERTIES_FACTORY, builder.executionHook); - this.builder = builder; - } - - static TestCommandBuilder testPropsBuilder() { - return new TestCommandBuilder(); - } - - static class TestCommandBuilder { - TestCircuitBreaker _cb = new TestCircuitBreaker(); - HystrixCommandGroupKey owner = CommandGroupForUnitTest.OWNER_ONE; - HystrixCommandKey dependencyKey = null; - HystrixThreadPoolKey threadPoolKey = null; - HystrixCircuitBreaker circuitBreaker = _cb; - HystrixThreadPool threadPool = null; - HystrixCommandProperties.Setter commandPropertiesDefaults = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter(); - HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults = HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder(); - HystrixCommandMetrics metrics = _cb.metrics; - TryableSemaphore fallbackSemaphore = null; - TryableSemaphore executionSemaphore = null; - TestExecutionHook executionHook = new TestExecutionHook(); - - TestCommandBuilder setOwner(HystrixCommandGroupKey owner) { - this.owner = owner; - return this; - } - - TestCommandBuilder setCommandKey(HystrixCommandKey dependencyKey) { - this.dependencyKey = dependencyKey; - return this; - } - - TestCommandBuilder setThreadPoolKey(HystrixThreadPoolKey threadPoolKey) { - this.threadPoolKey = threadPoolKey; - return this; - } - - TestCommandBuilder setCircuitBreaker(HystrixCircuitBreaker circuitBreaker) { - this.circuitBreaker = circuitBreaker; - return this; - } - - TestCommandBuilder setThreadPool(HystrixThreadPool threadPool) { - this.threadPool = threadPool; - return this; - } - - TestCommandBuilder setCommandPropertiesDefaults(HystrixCommandProperties.Setter commandPropertiesDefaults) { - this.commandPropertiesDefaults = commandPropertiesDefaults; - return this; - } - - TestCommandBuilder setThreadPoolPropertiesDefaults(HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults) { - this.threadPoolPropertiesDefaults = threadPoolPropertiesDefaults; - return this; - } - - TestCommandBuilder setMetrics(HystrixCommandMetrics metrics) { - this.metrics = metrics; - return this; - } - - TestCommandBuilder setFallbackSemaphore(TryableSemaphore fallbackSemaphore) { - this.fallbackSemaphore = fallbackSemaphore; - return this; - } - - TestCommandBuilder setExecutionSemaphore(TryableSemaphore executionSemaphore) { - this.executionSemaphore = executionSemaphore; - return this; - } - - } - - } - - /** - * Successful execution - no fallback implementation. - */ - private static class SuccessfulTestCommand extends TestHystrixCommand { - - public SuccessfulTestCommand() { - this(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter()); - } - - public SuccessfulTestCommand(HystrixCommandProperties.Setter properties) { - super(testPropsBuilder().setCommandPropertiesDefaults(properties)); - } - - @Override - protected Boolean run() { - return true; - } - - } - - /** - * Successful execution - no fallback implementation. - */ - private static class DynamicOwnerTestCommand extends TestHystrixCommand { - - public DynamicOwnerTestCommand(HystrixCommandGroupKey owner) { - super(testPropsBuilder().setOwner(owner)); - } - - @Override - protected Boolean run() { - System.out.println("successfully executed"); - return true; - } - - } - - /** - * Successful execution - no fallback implementation. - */ - private static class DynamicOwnerAndKeyTestCommand extends TestHystrixCommand { - - public DynamicOwnerAndKeyTestCommand(HystrixCommandGroupKey owner, HystrixCommandKey key) { - super(testPropsBuilder().setOwner(owner).setCommandKey(key).setCircuitBreaker(null).setMetrics(null)); - // we specifically are NOT passing in a circuit breaker here so we test that it creates a new one correctly based on the dynamic key - } - - @Override - protected Boolean run() { - System.out.println("successfully executed"); - return true; - } - - } - - /** - * Failed execution with unknown exception (not HystrixException) - no fallback implementation. - */ - private static class UnknownFailureTestCommandWithoutFallback extends TestHystrixCommand { - - private UnknownFailureTestCommandWithoutFallback() { - super(testPropsBuilder()); - } - - @Override - protected Boolean run() { - System.out.println("*** simulated failed execution ***"); - throw new RuntimeException("we failed with an unknown issue"); - } - - } - - /** - * Failed execution with known exception (HystrixException) - no fallback implementation. - */ - private static class KnownFailureTestCommandWithoutFallback extends TestHystrixCommand { - - private KnownFailureTestCommandWithoutFallback(TestCircuitBreaker circuitBreaker) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); - } - - @Override - protected Boolean run() { - System.out.println("*** simulated failed execution ***"); - throw new RuntimeException("we failed with a simulated issue"); - } - - } - - /** - * Failed execution - fallback implementation successfully returns value. - */ - private static class KnownFailureTestCommandWithFallback extends TestHystrixCommand { - - public KnownFailureTestCommandWithFallback(TestCircuitBreaker circuitBreaker) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); - } - - public KnownFailureTestCommandWithFallback(TestCircuitBreaker circuitBreaker, boolean fallbackEnabled) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withFallbackEnabled(fallbackEnabled))); - } - - @Override - protected Boolean run() { - System.out.println("*** simulated failed execution ***"); - throw new RuntimeException("we failed with a simulated issue"); - } - - @Override - protected Boolean getFallback() { - return false; - } - } - - /** - * Failed execution - fallback implementation throws exception. - */ - private static class KnownFailureTestCommandWithFallbackFailure extends TestHystrixCommand { - - private KnownFailureTestCommandWithFallbackFailure() { - super(testPropsBuilder()); - } - - @Override - protected Boolean run() { - System.out.println("*** simulated failed execution ***"); - throw new RuntimeException("we failed with a simulated issue"); - } - - @Override - protected Boolean getFallback() { - throw new RuntimeException("failed while getting fallback"); - } - } - - /** - * A Command implementation that supports caching. - */ - private static class SuccessfulCacheableCommand extends TestHystrixCommand { - - private final boolean cacheEnabled; - private volatile boolean executed = false; - private final String value; - - public SuccessfulCacheableCommand(TestCircuitBreaker circuitBreaker, boolean cacheEnabled, String value) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); - this.value = value; - this.cacheEnabled = cacheEnabled; - } - - @Override - protected String run() { - executed = true; - System.out.println("successfully executed"); - return value; - } - - public boolean isCommandRunningInThread() { - return super.getProperties().executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD); - } - - @Override - public String getCacheKey() { - if (cacheEnabled) - return value; - else - return null; - } - } - - /** - * A Command implementation that supports caching. - */ - private static class SuccessfulCacheableCommandViaSemaphore extends TestHystrixCommand { - - private final boolean cacheEnabled; - private volatile boolean executed = false; - private final String value; - - public SuccessfulCacheableCommandViaSemaphore(TestCircuitBreaker circuitBreaker, boolean cacheEnabled, String value) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))); - this.value = value; - this.cacheEnabled = cacheEnabled; - } - - @Override - protected String run() { - executed = true; - System.out.println("successfully executed"); - return value; - } - - public boolean isCommandRunningInThread() { - return super.getProperties().executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD); - } - - @Override - public String getCacheKey() { - if (cacheEnabled) - return value; - else - return null; - } - } - - /** - * A Command implementation that supports caching and execution takes a while. - *

- * Used to test scenario where Futures are returned with a backing call still executing. - */ - private static class SlowCacheableCommand extends TestHystrixCommand { - - private final String value; - private final int duration; - private volatile boolean executed = false; - - public SlowCacheableCommand(TestCircuitBreaker circuitBreaker, String value, int duration) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); - this.value = value; - this.duration = duration; - } - - @Override - protected String run() { - executed = true; - try { - Thread.sleep(duration); - } catch (Exception e) { - e.printStackTrace(); - } - System.out.println("successfully executed"); - return value; - } - - @Override - public String getCacheKey() { - return value; - } - } - - /** - * Successful execution - no fallback implementation, circuit-breaker disabled. - */ - private static class TestCommandWithoutCircuitBreaker extends TestHystrixCommand { - - private TestCommandWithoutCircuitBreaker() { - super(testPropsBuilder().setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withCircuitBreakerEnabled(false))); - } - - @Override - protected Boolean run() { - System.out.println("successfully executed"); - return true; - } - - } - - /** - * This should timeout. - */ - private static class TestCommandWithTimeout extends TestHystrixCommand { - - private final long timeout; - - private final static int FALLBACK_NOT_IMPLEMENTED = 1; - private final static int FALLBACK_SUCCESS = 2; - private final static int FALLBACK_FAILURE = 3; - - private final int fallbackBehavior; - - private TestCommandWithTimeout(long timeout, int fallbackBehavior) { - super(testPropsBuilder().setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds((int) timeout))); - this.timeout = timeout; - this.fallbackBehavior = fallbackBehavior; - } - - @Override - protected Boolean run() { - System.out.println("***** running"); - try { - Thread.sleep(timeout * 10); - } catch (InterruptedException e) { - e.printStackTrace(); - // ignore and sleep some more to simulate a dependency that doesn't obey interrupts - try { - Thread.sleep(timeout * 2); - } catch (Exception e2) { - // ignore - } - System.out.println("after interruption with extra sleep"); - } - return true; - } - - @Override - protected Boolean getFallback() { - if (fallbackBehavior == FALLBACK_SUCCESS) { - return false; - } else if (fallbackBehavior == FALLBACK_FAILURE) { - throw new RuntimeException("failed on fallback"); - } else { - // FALLBACK_NOT_IMPLEMENTED - return super.getFallback(); - } - } - } - - /** - * Threadpool with 1 thread, queue of size 1 - */ - private static class SingleThreadedPool implements HystrixThreadPool { - - final LinkedBlockingQueue queue; - final ThreadPoolExecutor pool; - private final int rejectionQueueSizeThreshold; - - public SingleThreadedPool(int queueSize) { - this(queueSize, 100); - } - - public SingleThreadedPool(int queueSize, int rejectionQueueSizeThreshold) { - queue = new LinkedBlockingQueue(queueSize); - pool = new ThreadPoolExecutor(1, 1, 1, TimeUnit.MINUTES, queue); - this.rejectionQueueSizeThreshold = rejectionQueueSizeThreshold; - } - - @Override - public ThreadPoolExecutor getExecutor() { - return pool; - } - - @Override - public void markThreadExecution() { - // not used for this test - } - - @Override - public void markThreadCompletion() { - // not used for this test - } - - @Override - public boolean isQueueSpaceAvailable() { - return queue.size() < rejectionQueueSizeThreshold; - } - - } - - /** - * This has a ThreadPool that has a single thread and queueSize of 1. - */ - private static class TestCommandRejection extends TestHystrixCommand { - - private final static int FALLBACK_NOT_IMPLEMENTED = 1; - private final static int FALLBACK_SUCCESS = 2; - private final static int FALLBACK_FAILURE = 3; - - private final int fallbackBehavior; - - private final int sleepTime; - - private TestCommandRejection(TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int sleepTime, int timeout, int fallbackBehavior) { - super(testPropsBuilder().setThreadPool(threadPool).setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(timeout))); - this.fallbackBehavior = fallbackBehavior; - this.sleepTime = sleepTime; - } - - @Override - protected Boolean run() { - System.out.println(">>> TestCommandRejection running"); - try { - Thread.sleep(sleepTime); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return true; - } - - @Override - protected Boolean getFallback() { - if (fallbackBehavior == FALLBACK_SUCCESS) { - return false; - } else if (fallbackBehavior == FALLBACK_FAILURE) { - throw new RuntimeException("failed on fallback"); - } else { - // FALLBACK_NOT_IMPLEMENTED - return super.getFallback(); - } - } - } - - /** - * Command that receives a custom thread-pool, sleepTime, timeout - */ - private static class CommandWithCustomThreadPool extends TestHystrixCommand { - - public boolean didExecute = false; - - private final int sleepTime; - - private CommandWithCustomThreadPool(TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int sleepTime, HystrixCommandProperties.Setter properties) { - super(testPropsBuilder().setThreadPool(threadPool).setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics).setCommandPropertiesDefaults(properties)); - this.sleepTime = sleepTime; - } - - @Override - protected Boolean run() { - System.out.println("**** Executing CommandWithCustomThreadPool. Execution => " + sleepTime); - didExecute = true; - try { - Thread.sleep(sleepTime); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return true; - } - } - - /** - * The run() will fail and getFallback() take a long time. - */ - private static class TestSemaphoreCommandWithSlowFallback extends TestHystrixCommand { - - private final long fallbackSleep; - - private TestSemaphoreCommandWithSlowFallback(TestCircuitBreaker circuitBreaker, int fallbackSemaphoreExecutionCount, long fallbackSleep) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withFallbackIsolationSemaphoreMaxConcurrentRequests(fallbackSemaphoreExecutionCount))); - this.fallbackSleep = fallbackSleep; - } - - @Override - protected Boolean run() { - throw new RuntimeException("run fails"); - } - - @Override - protected Boolean getFallback() { - try { - Thread.sleep(fallbackSleep); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return true; - } - } - - private static class NoRequestCacheTimeoutWithoutFallback extends TestHystrixCommand { - public NoRequestCacheTimeoutWithoutFallback(TestCircuitBreaker circuitBreaker) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(200))); - - // we want it to timeout - } - - @Override - protected Boolean run() { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - System.out.println(">>>> Sleep Interrupted: " + e.getMessage()); - // e.printStackTrace(); - } - return true; - } - - @Override - public String getCacheKey() { - return null; - } - } - - /** - * The run() will take time. No fallback implementation. - */ - private static class TestSemaphoreCommand extends TestHystrixCommand { - - private final long executionSleep; - - private TestSemaphoreCommand(TestCircuitBreaker circuitBreaker, int executionSemaphoreCount, long executionSleep) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter() - .withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE) - .withExecutionIsolationSemaphoreMaxConcurrentRequests(executionSemaphoreCount))); - this.executionSleep = executionSleep; - } - - private TestSemaphoreCommand(TestCircuitBreaker circuitBreaker, TryableSemaphore semaphore, long executionSleep) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter() - .withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE)) - .setExecutionSemaphore(semaphore)); - this.executionSleep = executionSleep; - } - - @Override - protected Boolean run() { - try { - Thread.sleep(executionSleep); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return true; - } - } - - /** - * Semaphore based command that allows caller to use latches to know when it has started and signal when it - * would like the command to finish - */ - private static class LatchedSemaphoreCommand extends TestHystrixCommand { - - private final CountDownLatch startLatch, waitLatch; - - /** - * - * @param circuitBreaker - * @param semaphore - * @param startLatch - * this command calls {@link java.util.concurrent.CountDownLatch#countDown()} immediately - * upon running - * @param waitLatch - * this command calls {@link java.util.concurrent.CountDownLatch#await()} once it starts - * to run. The caller can use the latch to signal the command to finish - */ - private LatchedSemaphoreCommand(TestCircuitBreaker circuitBreaker, TryableSemaphore semaphore, - CountDownLatch startLatch, CountDownLatch waitLatch) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE)) - .setExecutionSemaphore(semaphore)); - this.startLatch = startLatch; - this.waitLatch = waitLatch; - } - - @Override - protected Boolean run() { - // signals caller that run has started - this.startLatch.countDown(); - - try { - // waits for caller to countDown latch - this.waitLatch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - return false; - } - return true; - } - } - - /** - * The run() will take time. Contains fallback. - */ - private static class TestSemaphoreCommandWithFallback extends TestHystrixCommand { - - private final long executionSleep; - private final Boolean fallback; - - private TestSemaphoreCommandWithFallback(TestCircuitBreaker circuitBreaker, int executionSemaphoreCount, long executionSleep, Boolean fallback) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE).withExecutionIsolationSemaphoreMaxConcurrentRequests(executionSemaphoreCount))); - this.executionSleep = executionSleep; - this.fallback = fallback; - } - - @Override - protected Boolean run() { - try { - Thread.sleep(executionSleep); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return true; - } - - @Override - protected Boolean getFallback() { - return fallback; - } - - } - - private static class RequestCacheNullPointerExceptionCase extends TestHystrixCommand { - public RequestCacheNullPointerExceptionCase(TestCircuitBreaker circuitBreaker) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(200))); - // we want it to timeout - } - - @Override - protected Boolean run() { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return true; - } - - @Override - protected Boolean getFallback() { - return false; - } - - @Override - public String getCacheKey() { - return "A"; - } - } - - private static class RequestCacheTimeoutWithoutFallback extends TestHystrixCommand { - public RequestCacheTimeoutWithoutFallback(TestCircuitBreaker circuitBreaker) { - super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(200))); - // we want it to timeout - } - - @Override - protected Boolean run() { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - System.out.println(">>>> Sleep Interrupted: " + e.getMessage()); - // e.printStackTrace(); - } - return true; - } - - @Override - public String getCacheKey() { - return "A"; - } - } - - private static class RequestCacheThreadRejectionWithoutFallback extends TestHystrixCommand { - - final CountDownLatch completionLatch; - - public RequestCacheThreadRejectionWithoutFallback(TestCircuitBreaker circuitBreaker, CountDownLatch completionLatch) { - super(testPropsBuilder() - .setCircuitBreaker(circuitBreaker) - .setMetrics(circuitBreaker.metrics) - .setThreadPool(new HystrixThreadPool() { - - @Override - public ThreadPoolExecutor getExecutor() { - return null; - } - - @Override - public void markThreadExecution() { - - } - - @Override - public void markThreadCompletion() { - - } - - @Override - public boolean isQueueSpaceAvailable() { - // always return false so we reject everything - return false; - } - - })); - this.completionLatch = completionLatch; - } - - @Override - protected Boolean run() { - try { - if (completionLatch.await(1000, TimeUnit.MILLISECONDS)) { - throw new RuntimeException("timed out waiting on completionLatch"); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - return true; - } - - @Override - public String getCacheKey() { - return "A"; - } - } - - private static class BadRequestCommand extends TestHystrixCommand { - - public BadRequestCommand(TestCircuitBreaker circuitBreaker, ExecutionIsolationStrategy isolationType) { - super(testPropsBuilder() - .setCircuitBreaker(circuitBreaker) - .setCommandPropertiesDefaults(HystrixCommandProperties.Setter.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(isolationType))); - } - - @Override - protected Boolean run() { - throw new HystrixBadRequestException("Message to developer that they passed in bad data or something like that."); - } - - @Override - protected Boolean getFallback() { - return false; - } - - @Override - protected String getCacheKey() { - return "one"; - } - - } - - private static class CommandWithErrorThrown extends TestHystrixCommand { - - public CommandWithErrorThrown(TestCircuitBreaker circuitBreaker) { - super(testPropsBuilder() - .setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); - } - - @Override - protected Boolean run() throws Exception { - throw new Error("simulated java.lang.Error message"); - } - - } - - private static class CommandWithCheckedException extends TestHystrixCommand { - - public CommandWithCheckedException(TestCircuitBreaker circuitBreaker) { - super(testPropsBuilder() - .setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); - } - - @Override - protected Boolean run() throws Exception { - throw new IOException("simulated checked exception message"); - } - - } - - enum CommandKeyForUnitTest implements HystrixCommandKey { - KEY_ONE, KEY_TWO; - } - - enum CommandGroupForUnitTest implements HystrixCommandGroupKey { - OWNER_ONE, OWNER_TWO; - } - - enum ThreadPoolKeyForUnitTest implements HystrixThreadPoolKey { - THREAD_POOL_ONE, THREAD_POOL_TWO; - } - - private static HystrixPropertiesStrategy TEST_PROPERTIES_FACTORY = new TestPropertiesFactory(); - - private static class TestPropertiesFactory extends HystrixPropertiesStrategy { - - @Override - public HystrixCommandProperties getCommandProperties(HystrixCommandKey commandKey, HystrixCommandProperties.Setter builder) { - if (builder == null) { - builder = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter(); - } - return HystrixCommandProperties.Setter.asMock(builder); - } - - @Override - public HystrixThreadPoolProperties getThreadPoolProperties(HystrixThreadPoolKey threadPoolKey, HystrixThreadPoolProperties.Setter builder) { - if (builder == null) { - builder = HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder(); - } - return HystrixThreadPoolProperties.Setter.asMock(builder); - } - - @Override - public HystrixCollapserProperties getCollapserProperties(HystrixCollapserKey collapserKey, HystrixCollapserProperties.Setter builder) { - throw new IllegalStateException("not expecting collapser properties"); - } - - @Override - public String getCommandPropertiesCacheKey(HystrixCommandKey commandKey, HystrixCommandProperties.Setter builder) { - return null; - } - - @Override - public String getThreadPoolPropertiesCacheKey(HystrixThreadPoolKey threadPoolKey, com.netflix.hystrix.HystrixThreadPoolProperties.Setter builder) { - return null; - } - - @Override - public String getCollapserPropertiesCacheKey(HystrixCollapserKey collapserKey, com.netflix.hystrix.HystrixCollapserProperties.Setter builder) { - return null; - } - - } - - private static class TestExecutionHook extends HystrixCommandExecutionHook { - - AtomicInteger startExecute = new AtomicInteger(); - - @Override - public void onStart(HystrixCommand commandInstance) { - super.onStart(commandInstance); - startExecute.incrementAndGet(); - } - - Object endExecuteSuccessResponse = null; - - @Override - public T onComplete(HystrixCommand commandInstance, T response) { - endExecuteSuccessResponse = response; - return super.onComplete(commandInstance, response); - } - - Exception endExecuteFailureException = null; - FailureType endExecuteFailureType = null; - - @Override - public Exception onError(HystrixCommand commandInstance, FailureType failureType, Exception e) { - endExecuteFailureException = e; - endExecuteFailureType = failureType; - return super.onError(commandInstance, failureType, e); - } - - AtomicInteger startRun = new AtomicInteger(); - - @Override - public void onRunStart(HystrixCommand commandInstance) { - super.onRunStart(commandInstance); - startRun.incrementAndGet(); - } - - Object runSuccessResponse = null; - - @Override - public T onRunSuccess(HystrixCommand commandInstance, T response) { - runSuccessResponse = response; - return super.onRunSuccess(commandInstance, response); - } - - Exception runFailureException = null; - - @Override - public Exception onRunError(HystrixCommand commandInstance, Exception e) { - runFailureException = e; - return super.onRunError(commandInstance, e); - } - - AtomicInteger startFallback = new AtomicInteger(); - - @Override - public void onFallbackStart(HystrixCommand commandInstance) { - super.onFallbackStart(commandInstance); - startFallback.incrementAndGet(); - } - - Object fallbackSuccessResponse = null; - - @Override - public T onFallbackSuccess(HystrixCommand commandInstance, T response) { - fallbackSuccessResponse = response; - return super.onFallbackSuccess(commandInstance, response); - } - - Exception fallbackFailureException = null; - - @Override - public Exception onFallbackError(HystrixCommand commandInstance, Exception e) { - fallbackFailureException = e; - return super.onFallbackError(commandInstance, e); - } - - AtomicInteger threadStart = new AtomicInteger(); - - @Override - public void onThreadStart(HystrixCommand commandInstance) { - super.onThreadStart(commandInstance); - threadStart.incrementAndGet(); - } - - AtomicInteger threadComplete = new AtomicInteger(); - - @Override - public void onThreadComplete(HystrixCommand commandInstance) { - super.onThreadComplete(commandInstance); - threadComplete.incrementAndGet(); - } - - } + /* package */HystrixCircuitBreaker getCircuitBreaker() { + return observableCommand.getCircuitBreaker(); } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java index 506be2291..535367639 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java @@ -15,8 +15,6 @@ */ package com.netflix.hystrix; -import static org.junit.Assert.*; - import java.util.Collection; import java.util.Collections; import java.util.concurrent.ConcurrentHashMap; @@ -24,15 +22,11 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.netflix.hystrix.HystrixCommand.UnitTest.CommandGroupForUnitTest; -import com.netflix.hystrix.HystrixCommand.UnitTest.CommandKeyForUnitTest; import com.netflix.hystrix.strategy.HystrixPlugins; import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifier; -import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifierDefault; import com.netflix.hystrix.util.HystrixRollingNumber; import com.netflix.hystrix.util.HystrixRollingNumberEvent; import com.netflix.hystrix.util.HystrixRollingPercentile; @@ -459,58 +453,4 @@ public int getErrorPercentage() { } } - public static class UnitTest { - - /** - * Testing the ErrorPercentage because this method could be easy to miss when making changes elsewhere. - */ - @Test - public void testGetErrorPercentage() { - - try { - HystrixCommandProperties.Setter properties = HystrixCommandProperties.Setter.getUnitTestPropertiesSetter(); - HystrixCommandMetrics metrics = getMetrics(properties); - - metrics.markSuccess(100); - assertEquals(0, metrics.getHealthCounts().getErrorPercentage()); - - metrics.markFailure(1000); - assertEquals(50, metrics.getHealthCounts().getErrorPercentage()); - - metrics.markSuccess(100); - metrics.markSuccess(100); - assertEquals(25, metrics.getHealthCounts().getErrorPercentage()); - - metrics.markTimeout(5000); - metrics.markTimeout(5000); - assertEquals(50, metrics.getHealthCounts().getErrorPercentage()); - - metrics.markSuccess(100); - metrics.markSuccess(100); - metrics.markSuccess(100); - - // latent - metrics.markSuccess(5000); - - // 6 success + 1 latent success + 1 failure + 2 timeout = 10 total - // latent success not considered error - // error percentage = 1 failure + 2 timeout / 10 - assertEquals(30, metrics.getHealthCounts().getErrorPercentage()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Error occurred: " + e.getMessage()); - } - - } - - /** - * Utility method for creating {@link HystrixCommandMetrics} for unit tests. - */ - private static HystrixCommandMetrics getMetrics(HystrixCommandProperties.Setter properties) { - return new HystrixCommandMetrics(CommandKeyForUnitTest.KEY_ONE, CommandGroupForUnitTest.OWNER_ONE, HystrixCommandProperties.Setter.asMock(properties), HystrixEventNotifierDefault.getInstance()); - } - - } - } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandProperties.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandProperties.java index 15936117f..523d2ce6c 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandProperties.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandProperties.java @@ -16,18 +16,14 @@ package com.netflix.hystrix; import static com.netflix.hystrix.strategy.properties.HystrixProperty.Factory.*; -import static org.junit.Assert.*; import java.util.concurrent.Future; import javax.annotation.concurrent.NotThreadSafe; -import org.junit.After; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.netflix.config.ConfigurationManager; import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty; import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty.DynamicStringProperty; import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; @@ -44,13 +40,13 @@ public abstract class HystrixCommandProperties { private static final Logger logger = LoggerFactory.getLogger(HystrixCommandProperties.class); /* defaults */ - private static final Integer default_metricsRollingStatisticalWindow = 10000;// default => statisticalWindow: 10000 = 10 seconds (and default of 10 buckets so each bucket is 1 second) + /* package */ static final Integer default_metricsRollingStatisticalWindow = 10000;// default => statisticalWindow: 10000 = 10 seconds (and default of 10 buckets so each bucket is 1 second) private static final Integer default_metricsRollingStatisticalWindowBuckets = 10;// default => statisticalWindowBuckets: 10 = 10 buckets in a 10 second window so each bucket is 1 second private static final Integer default_circuitBreakerRequestVolumeThreshold = 20;// default => statisticalWindowVolumeThreshold: 20 requests in 10 seconds must occur before statistics matter private static final Integer default_circuitBreakerSleepWindowInMilliseconds = 5000;// default => sleepWindow: 5000 = 5 seconds that we will sleep before trying again after tripping the circuit private static final Integer default_circuitBreakerErrorThresholdPercentage = 50;// default => errorThresholdPercentage = 50 = if 50%+ of requests in 10 seconds are failures or latent when we will trip the circuit private static final Boolean default_circuitBreakerForceOpen = false;// default => forceCircuitOpen = false (we want to allow traffic) - private static final Boolean default_circuitBreakerForceClosed = false;// default => ignoreErrors = false + /* package */ static final Boolean default_circuitBreakerForceClosed = false;// default => ignoreErrors = false private static final Integer default_executionIsolationThreadTimeoutInMilliseconds = 1000; // default => executionTimeoutInMilliseconds: 1000 = 1 second private static final ExecutionIsolationStrategy default_executionIsolationStrategy = ExecutionIsolationStrategy.THREAD; private static final Boolean default_executionIsolationThreadInterruptOnTimeout = true; @@ -500,7 +496,7 @@ public static class Setter { private Boolean requestCacheEnabled = null; private Boolean requestLogEnabled = null; - private Setter() { + /* package */ Setter() { } public Boolean getCircuitBreakerEnabled() { @@ -692,324 +688,7 @@ public Setter withRequestLogEnabled(boolean value) { return this; } - /** - * Utility method for creating baseline properties for unit tests. - */ - /* package */static HystrixCommandProperties.Setter getUnitTestPropertiesSetter() { - return new HystrixCommandProperties.Setter() - .withExecutionIsolationThreadTimeoutInMilliseconds(1000)// when an execution will be timed out - .withExecutionIsolationStrategy(ExecutionIsolationStrategy.THREAD) // we want thread execution by default in tests - .withExecutionIsolationThreadInterruptOnTimeout(true) - .withCircuitBreakerForceOpen(false) // we don't want short-circuiting by default - .withCircuitBreakerErrorThresholdPercentage(40) // % of 'marks' that must be failed to trip the circuit - .withMetricsRollingStatisticalWindowInMilliseconds(5000)// milliseconds back that will be tracked - .withMetricsRollingStatisticalWindowBuckets(5) // buckets - .withCircuitBreakerRequestVolumeThreshold(0) // in testing we will not have a threshold unless we're specifically testing that feature - .withCircuitBreakerSleepWindowInMilliseconds(5000000) // milliseconds after tripping circuit before allowing retry (by default set VERY long as we want it to effectively never allow a singleTest for most unit tests) - .withCircuitBreakerEnabled(true) - .withRequestLogEnabled(true) - .withExecutionIsolationSemaphoreMaxConcurrentRequests(20) - .withFallbackIsolationSemaphoreMaxConcurrentRequests(10) - .withFallbackEnabled(true) - .withCircuitBreakerForceClosed(false) - .withMetricsRollingPercentileEnabled(true) - .withRequestCacheEnabled(true) - .withMetricsRollingPercentileWindowInMilliseconds(60000) - .withMetricsRollingPercentileWindowBuckets(12) - .withMetricsRollingPercentileBucketSize(1000) - .withMetricsHealthSnapshotIntervalInMilliseconds(0); - } - - /** - * Return a static representation of the properties with values from the Builder so that UnitTests can create properties that are not affected by the actual implementations which pick up their - * values dynamically. - * - * @param builder - * @return HystrixCommandProperties - */ - /* package */static HystrixCommandProperties asMock(final Setter builder) { - return new HystrixCommandProperties(UnitTest.TestKey.TEST) { - - @Override - public HystrixProperty circuitBreakerEnabled() { - return HystrixProperty.Factory.asProperty(builder.circuitBreakerEnabled); - } - - @Override - public HystrixProperty circuitBreakerErrorThresholdPercentage() { - return HystrixProperty.Factory.asProperty(builder.circuitBreakerErrorThresholdPercentage); - } - - @Override - public HystrixProperty circuitBreakerForceClosed() { - return HystrixProperty.Factory.asProperty(builder.circuitBreakerForceClosed); - } - - @Override - public HystrixProperty circuitBreakerForceOpen() { - return HystrixProperty.Factory.asProperty(builder.circuitBreakerForceOpen); - } - - @Override - public HystrixProperty circuitBreakerRequestVolumeThreshold() { - return HystrixProperty.Factory.asProperty(builder.circuitBreakerRequestVolumeThreshold); - } - - @Override - public HystrixProperty circuitBreakerSleepWindowInMilliseconds() { - return HystrixProperty.Factory.asProperty(builder.circuitBreakerSleepWindowInMilliseconds); - } - - @Override - public HystrixProperty executionIsolationSemaphoreMaxConcurrentRequests() { - return HystrixProperty.Factory.asProperty(builder.executionIsolationSemaphoreMaxConcurrentRequests); - } - - @Override - public HystrixProperty executionIsolationStrategy() { - return HystrixProperty.Factory.asProperty(builder.executionIsolationStrategy); - } - - @Override - public HystrixProperty executionIsolationThreadInterruptOnTimeout() { - return HystrixProperty.Factory.asProperty(builder.executionIsolationThreadInterruptOnTimeout); - } - - @Override - public HystrixProperty executionIsolationThreadPoolKeyOverride() { - return HystrixProperty.Factory.nullProperty(); - } - - @Override - public HystrixProperty executionIsolationThreadTimeoutInMilliseconds() { - return HystrixProperty.Factory.asProperty(builder.executionIsolationThreadTimeoutInMilliseconds); - } - - @Override - public HystrixProperty fallbackIsolationSemaphoreMaxConcurrentRequests() { - return HystrixProperty.Factory.asProperty(builder.fallbackIsolationSemaphoreMaxConcurrentRequests); - } - - @Override - public HystrixProperty fallbackEnabled() { - return HystrixProperty.Factory.asProperty(builder.fallbackEnabled); - } - - @Override - public HystrixProperty metricsHealthSnapshotIntervalInMilliseconds() { - return HystrixProperty.Factory.asProperty(builder.metricsHealthSnapshotIntervalInMilliseconds); - } - - @Override - public HystrixProperty metricsRollingPercentileBucketSize() { - return HystrixProperty.Factory.asProperty(builder.metricsRollingPercentileBucketSize); - } - - @Override - public HystrixProperty metricsRollingPercentileEnabled() { - return HystrixProperty.Factory.asProperty(builder.metricsRollingPercentileEnabled); - } - - @Override - public HystrixProperty metricsRollingPercentileWindow() { - return HystrixProperty.Factory.asProperty(builder.metricsRollingPercentileWindowInMilliseconds); - } - - @Override - public HystrixProperty metricsRollingPercentileWindowBuckets() { - return HystrixProperty.Factory.asProperty(builder.metricsRollingPercentileWindowBuckets); - } - - @Override - public HystrixProperty metricsRollingStatisticalWindowInMilliseconds() { - return HystrixProperty.Factory.asProperty(builder.metricsRollingStatisticalWindowInMilliseconds); - } - - @Override - public HystrixProperty metricsRollingStatisticalWindowBuckets() { - return HystrixProperty.Factory.asProperty(builder.metricsRollingStatisticalWindowBuckets); - } - - @Override - public HystrixProperty requestCacheEnabled() { - return HystrixProperty.Factory.asProperty(builder.requestCacheEnabled); - } - - @Override - public HystrixProperty requestLogEnabled() { - return HystrixProperty.Factory.asProperty(builder.requestLogEnabled); - } - - }; - } } - public static class UnitTest { - // NOTE: We use "unitTestPrefix" as a prefix so we can't end up pulling in external properties that change unit test behavior - - public enum TestKey implements HystrixCommandKey { - TEST; - } - - private static class TestPropertiesCommand extends HystrixCommandProperties { - - protected TestPropertiesCommand(HystrixCommandKey key, Setter builder, String propertyPrefix) { - super(key, builder, propertyPrefix); - } - - } - - @After - public void cleanup() { - ConfigurationManager.getConfigInstance().clear(); - } - - @Test - public void testBooleanBuilderOverride1() { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, - new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(true), "unitTestPrefix"); - - // the builder override should take precedence over the default - assertEquals(true, properties.circuitBreakerForceClosed().get()); - } - - @Test - public void testBooleanBuilderOverride2() { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, - new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(false), "unitTestPrefix"); - - // the builder override should take precedence over the default - assertEquals(false, properties.circuitBreakerForceClosed().get()); - } - - @Test - public void testBooleanCodeDefault() { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); - assertEquals(default_circuitBreakerForceClosed, properties.circuitBreakerForceClosed().get()); - } - - @Test - public void testBooleanGlobalDynamicOverrideOfCodeDefault() throws Exception { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed", true); - - // the global dynamic property should take precedence over the default - assertEquals(true, properties.circuitBreakerForceClosed().get()); - - // cleanup - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed"); - } - - @Test - public void testBooleanInstanceBuilderOverrideOfGlobalDynamicOverride1() throws Exception { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, - new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(true), "unitTestPrefix"); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed", false); - - // the builder injected should take precedence over the global dynamic property - assertEquals(true, properties.circuitBreakerForceClosed().get()); - - // cleanup - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed"); - } - - @Test - public void testBooleanInstanceBuilderOverrideOfGlobalDynamicOverride2() throws Exception { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, - new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(false), "unitTestPrefix"); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed", true); - - // the builder injected should take precedence over the global dynamic property - assertEquals(false, properties.circuitBreakerForceClosed().get()); - - // cleanup - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed"); - } - - @Test - public void testBooleanInstanceDynamicOverrideOfEverything() throws Exception { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, - new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(false), "unitTestPrefix"); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed", false); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.TEST.circuitBreaker.forceClosed", true); - - // the instance specific dynamic property should take precedence over everything - assertEquals(true, properties.circuitBreakerForceClosed().get()); - - // cleanup - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed"); - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.TEST.circuitBreaker.forceClosed"); - } - - @Test - public void testIntegerBuilderOverride() { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, - new HystrixCommandProperties.Setter().withMetricsRollingStatisticalWindowInMilliseconds(5000), "unitTestPrefix"); - - // the builder override should take precedence over the default - assertEquals(5000, properties.metricsRollingStatisticalWindowInMilliseconds().get().intValue()); - } - - @Test - public void testIntegerCodeDefault() { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); - assertEquals(default_metricsRollingStatisticalWindow, properties.metricsRollingStatisticalWindowInMilliseconds().get()); - } - - @Test - public void testIntegerGlobalDynamicOverrideOfCodeDefault() throws Exception { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.metrics.rollingStats.timeInMilliseconds", 1234); - - // the global dynamic property should take precedence over the default - assertEquals(1234, properties.metricsRollingStatisticalWindowInMilliseconds().get().intValue()); - - // cleanup - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.metrics.rollingStats.timeInMilliseconds"); - } - - @Test - public void testIntegerInstanceBuilderOverrideOfGlobalDynamicOverride() throws Exception { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, - new HystrixCommandProperties.Setter().withMetricsRollingStatisticalWindowInMilliseconds(5000), "unitTestPrefix"); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.rollingStats.timeInMilliseconds", 3456); - - // the builder injected should take precedence over the global dynamic property - assertEquals(5000, properties.metricsRollingStatisticalWindowInMilliseconds().get().intValue()); - - // cleanup - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.rollingStats.timeInMilliseconds"); - } - - @Test - public void testIntegerInstanceDynamicOverrideOfEverything() throws Exception { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, - new HystrixCommandProperties.Setter().withMetricsRollingStatisticalWindowInMilliseconds(5000), "unitTestPrefix"); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.metrics.rollingStats.timeInMilliseconds", 1234); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.TEST.metrics.rollingStats.timeInMilliseconds", 3456); - - // the instance specific dynamic property should take precedence over everything - assertEquals(3456, properties.metricsRollingStatisticalWindowInMilliseconds().get().intValue()); - - // cleanup - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.metrics.rollingStats.timeInMilliseconds"); - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.TEST.metrics.rollingStats.timeInMilliseconds"); - } - - @Test - public void testThreadPoolOnlyHasInstanceOverride() throws Exception { - HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.threadPoolKeyOverride", 1234); - // it should be null - assertEquals(null, properties.executionIsolationThreadPoolKeyOverride().get()); - ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.TEST.threadPoolKeyOverride", "testPool"); - // now it should have a value - assertEquals("testPool", properties.executionIsolationThreadPoolKeyOverride().get()); - - // cleanup - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.threadPoolKeyOverride"); - ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.TEST.threadPoolKeyOverride"); - } - } } \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutable.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutable.java index b4fb1637d..77785e7c7 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutable.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutable.java @@ -18,7 +18,7 @@ import java.util.concurrent.Future; import rx.Observable; -import rx.concurrency.Schedulers; +import rx.schedulers.Schedulers; import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; import com.netflix.hystrix.exception.HystrixBadRequestException; diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutableBase.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutableBase.java new file mode 100644 index 000000000..0263875be --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutableBase.java @@ -0,0 +1,1344 @@ +/** + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.hystrix; + +import java.lang.ref.Reference; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Observable; +import rx.Scheduler; +import rx.Subscriber; +import rx.schedulers.Schedulers; +import rx.subjects.ReplaySubject; + +import com.netflix.hystrix.HystrixCircuitBreaker.NoOpCircuitBreaker; +import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; +import com.netflix.hystrix.exception.HystrixBadRequestException; +import com.netflix.hystrix.exception.HystrixRuntimeException; +import com.netflix.hystrix.exception.HystrixRuntimeException.FailureType; +import com.netflix.hystrix.strategy.HystrixPlugins; +import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; +import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategyDefault; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; +import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifier; +import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook; +import com.netflix.hystrix.strategy.metrics.HystrixMetricsPublisherFactory; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesFactory; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; +import com.netflix.hystrix.strategy.properties.HystrixProperty; +import com.netflix.hystrix.util.HystrixTimer.TimerListener; + +/* package */abstract class HystrixExecutableBase implements HystrixExecutable, HystrixExecutableInfo { + // TODO make this package private + + private static final Logger logger = LoggerFactory.getLogger(HystrixExecutableBase.class); + protected final HystrixCircuitBreaker circuitBreaker; + protected final HystrixThreadPool threadPool; + protected final HystrixThreadPoolKey threadPoolKey; + protected final HystrixCommandProperties properties; + + protected static enum TimedOutStatus { + NOT_EXECUTED, COMPLETED, TIMED_OUT + }; + + protected final HystrixCommandMetrics metrics; + + protected final HystrixCommandKey commandKey; + protected final HystrixCommandGroupKey commandGroup; + + /** + * Plugin implementations + */ + protected final HystrixEventNotifier eventNotifier; + protected final HystrixConcurrencyStrategy concurrencyStrategy; + protected final HystrixCommandExecutionHook executionHook; + + /* FALLBACK Semaphore */ + protected final TryableSemaphore fallbackSemaphoreOverride; + /* each circuit has a semaphore to restrict concurrent fallback execution */ + protected static final ConcurrentHashMap fallbackSemaphorePerCircuit = new ConcurrentHashMap(); + /* END FALLBACK Semaphore */ + + /* EXECUTION Semaphore */ + protected final TryableSemaphore executionSemaphoreOverride; + /* each circuit has a semaphore to restrict concurrent fallback execution */ + protected static final ConcurrentHashMap executionSemaphorePerCircuit = new ConcurrentHashMap(); + /* END EXECUTION Semaphore */ + + protected final AtomicReference> timeoutTimer = new AtomicReference>(); + + protected AtomicBoolean started = new AtomicBoolean(); + protected volatile long invocationStartTime = -1; + + /* result of execution (if this command instance actually gets executed, which may not occur due to request caching) */ + protected volatile ExecutionResult executionResult = ExecutionResult.EMPTY; + + /* If this command executed and timed-out */ + protected final AtomicReference isCommandTimedOut = new AtomicReference(TimedOutStatus.NOT_EXECUTED); + protected final AtomicBoolean isExecutionComplete = new AtomicBoolean(false); + protected final AtomicBoolean isExecutedInThread = new AtomicBoolean(false); + + /** + * Instance of RequestCache logic + */ + protected final HystrixRequestCache requestCache; + protected final HystrixRequestLog currentRequestLog; + + // this is a micro-optimization but saves about 1-2microseconds (on 2011 MacBook Pro) + // on the repetitive string processing that will occur on the same classes over and over again + private static ConcurrentHashMap, String> defaultNameCache = new ConcurrentHashMap, String>(); + + /* package */static String getDefaultNameFromClass(Class cls) { + String fromCache = defaultNameCache.get(cls); + if (fromCache != null) { + return fromCache; + } + // generate the default + // default HystrixCommandKey to use if the method is not overridden + String name = cls.getSimpleName(); + if (name.equals("")) { + // we don't have a SimpleName (anonymous inner class) so use the full class name + name = cls.getName(); + name = name.substring(name.lastIndexOf('.') + 1, name.length()); + } + defaultNameCache.put(cls, name); + return name; + } + + protected HystrixExecutableBase(HystrixCommandGroupKey group, HystrixCommandKey key, HystrixThreadPoolKey threadPoolKey, HystrixCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, + HystrixCommandProperties.Setter commandPropertiesDefaults, HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults, + HystrixCommandMetrics metrics, TryableSemaphore fallbackSemaphore, TryableSemaphore executionSemaphore, + HystrixPropertiesStrategy propertiesStrategy, HystrixCommandExecutionHook executionHook) { + + /* + * CommandGroup initialization + */ + if (group == null) { + throw new IllegalStateException("HystrixCommandGroup can not be NULL"); + } else { + this.commandGroup = group; + } + + /* + * CommandKey initialization + */ + if (key == null || key.name().trim().equals("")) { + final String keyName = getDefaultNameFromClass(getClass()); + this.commandKey = HystrixCommandKey.Factory.asKey(keyName); + } else { + this.commandKey = key; + } + + /* + * Properties initialization + */ + if (propertiesStrategy == null) { + this.properties = HystrixPropertiesFactory.getCommandProperties(this.commandKey, commandPropertiesDefaults); + } else { + // used for unit testing + this.properties = propertiesStrategy.getCommandProperties(this.commandKey, commandPropertiesDefaults); + } + + /* + * ThreadPoolKey + * + * This defines which thread-pool this command should run on. + * + * It uses the HystrixThreadPoolKey if provided, then defaults to use HystrixCommandGroup. + * + * It can then be overridden by a property if defined so it can be changed at runtime. + */ + if (this.properties.executionIsolationThreadPoolKeyOverride().get() == null) { + // we don't have a property overriding the value so use either HystrixThreadPoolKey or HystrixCommandGroup + if (threadPoolKey == null) { + /* use HystrixCommandGroup if HystrixThreadPoolKey is null */ + this.threadPoolKey = HystrixThreadPoolKey.Factory.asKey(commandGroup.name()); + } else { + this.threadPoolKey = threadPoolKey; + } + } else { + // we have a property defining the thread-pool so use it instead + this.threadPoolKey = HystrixThreadPoolKey.Factory.asKey(properties.executionIsolationThreadPoolKeyOverride().get()); + } + + /* strategy: HystrixEventNotifier */ + this.eventNotifier = HystrixPlugins.getInstance().getEventNotifier(); + + /* strategy: HystrixConcurrentStrategy */ + this.concurrencyStrategy = HystrixPlugins.getInstance().getConcurrencyStrategy(); + + /* + * Metrics initialization + */ + if (metrics == null) { + this.metrics = HystrixCommandMetrics.getInstance(this.commandKey, this.commandGroup, this.properties); + } else { + this.metrics = metrics; + } + + /* + * CircuitBreaker initialization + */ + if (this.properties.circuitBreakerEnabled().get()) { + if (circuitBreaker == null) { + // get the default implementation of HystrixCircuitBreaker + this.circuitBreaker = HystrixCircuitBreaker.Factory.getInstance(this.commandKey, this.commandGroup, this.properties, this.metrics); + } else { + this.circuitBreaker = circuitBreaker; + } + } else { + this.circuitBreaker = new NoOpCircuitBreaker(); + } + + /* strategy: HystrixMetricsPublisherCommand */ + HystrixMetricsPublisherFactory.createOrRetrievePublisherForCommand(this.commandKey, this.commandGroup, this.metrics, this.circuitBreaker, this.properties); + + /* strategy: HystrixCommandExecutionHook */ + if (executionHook == null) { + this.executionHook = new ExecutionHookDeprecationWrapper(HystrixPlugins.getInstance().getCommandExecutionHook()); + } else { + // used for unit testing + if (executionHook instanceof ExecutionHookDeprecationWrapper) { + this.executionHook = executionHook; + } else { + this.executionHook = new ExecutionHookDeprecationWrapper(executionHook); + } + } + + /* + * ThreadPool initialization + */ + if (threadPool == null) { + // get the default implementation of HystrixThreadPool + this.threadPool = HystrixThreadPool.Factory.getInstance(this.threadPoolKey, threadPoolPropertiesDefaults); + } else { + this.threadPool = threadPool; + } + + /* fallback semaphore override if applicable */ + this.fallbackSemaphoreOverride = fallbackSemaphore; + + /* execution semaphore override if applicable */ + this.executionSemaphoreOverride = executionSemaphore; + + /* setup the request cache for this instance */ + this.requestCache = HystrixRequestCache.getInstance(this.commandKey, this.concurrencyStrategy); + + /* store reference to request log regardless of which thread later hits it */ + if (concurrencyStrategy instanceof HystrixConcurrencyStrategyDefault) { + // if we're using the default we support only optionally using a request context + if (HystrixRequestContext.isCurrentThreadInitialized()) { + currentRequestLog = HystrixRequestLog.getCurrentRequest(concurrencyStrategy); + } else { + currentRequestLog = null; + } + } else { + // if it's a custom strategy it must ensure the context is initialized + if (HystrixRequestLog.getCurrentRequest(concurrencyStrategy) != null) { + currentRequestLog = HystrixRequestLog.getCurrentRequest(concurrencyStrategy); + } else { + currentRequestLog = null; + } + } + } + + /** + * Allow the Collapser to mark this command instance as being used for a collapsed request and how many requests were collapsed. + * + * @param sizeOfBatch + */ + /* package */void markAsCollapsedCommand(int sizeOfBatch) { + getMetrics().markCollapsed(sizeOfBatch); + executionResult = executionResult.addEvents(HystrixEventType.COLLAPSED); + } + + /** + * Used for synchronous execution of command. + * + * @return R + * Result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * @throws HystrixRuntimeException + * if a failure occurs and a fallback cannot be retrieved + * @throws HystrixBadRequestException + * if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once + */ + public R execute() { + try { + return queue().get(); + } catch (Exception e) { + throw decomposeException(e); + } + } + + /** + * Used for asynchronous execution of command. + *

+ * This will queue up the command on the thread pool and return an {@link Future} to get the result once it completes. + *

+ * NOTE: If configured to not run in a separate thread, this will have the same effect as {@link #execute()} and will block. + *

+ * We don't throw an exception but just flip to synchronous execution so code doesn't need to change in order to switch a command from running on a separate thread to the calling thread. + * + * @return {@code Future} Result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * @throws HystrixRuntimeException + * if a fallback does not exist + *

+ *

    + *
  • via {@code Future.get()} in {@link ExecutionException#getCause()} if a failure occurs
    • + *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • + *
+ * @throws HystrixBadRequestException + * via {@code Future.get()} in {@link ExecutionException#getCause()} if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once + */ + public Future queue() { + /* + * --- Schedulers.immediate() + * + * We use the 'immediate' schedule since Future.get() is blocking so we don't want to bother doing the callback to the Future on a separate thread + * as we don't need to separate the Hystrix thread from user threads since they are already providing it via the Future.get() call. + * + * --- performAsyncTimeout: false + * + * We pass 'false' to tell the Observable we will block on it so it doesn't schedule an async timeout. + * + * This optimizes for using the calling thread to do the timeout rather than scheduling another thread. + * + * In a tight-loop of executing commands this optimization saves a few microseconds per execution. + * It also just makes no sense to use a separate thread to timeout the command when the calling thread + * is going to sit waiting on it. + */ + final ObservableCommand o = toObservable(Schedulers.immediate(), false); + final Future f = o.toBlockingObservable().toFuture(); + + /* special handling of error states that throw immediately */ + if (f.isDone()) { + try { + f.get(); + return f; + } catch (Exception e) { + RuntimeException re = decomposeException(e); + if (re instanceof HystrixBadRequestException) { + return f; + } else if (re instanceof HystrixRuntimeException) { + HystrixRuntimeException hre = (HystrixRuntimeException) re; + if (hre.getFailureType() == FailureType.COMMAND_EXCEPTION || hre.getFailureType() == FailureType.TIMEOUT) { + // we don't throw these types from queue() only from queue().get() as they are execution errors + return f; + } else { + // these are errors we throw from queue() as they as rejection type errors + throw hre; + } + } else { + throw re; + } + } + } + + return new Future() { + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return f.cancel(mayInterruptIfRunning); + } + + @Override + public boolean isCancelled() { + return f.isCancelled(); + } + + @Override + public boolean isDone() { + return f.isDone(); + } + + @Override + public R get() throws InterruptedException, ExecutionException { + return performBlockingGetWithTimeout(o, f); + } + + /** + * --- Non-Blocking Timeout (performAsyncTimeout:true) --- + * + * When 'toObservable' is done with non-blocking timeout then timeout functionality is provided + * by a separate HystrixTimer thread that will "tick" and cancel the underlying async Future inside the Observable. + * + * This method allows stealing that responsibility and letting the thread that's going to block anyways + * do the work to reduce pressure on the HystrixTimer. + * + * Blocking via queue().get() on a non-blocking timeout will work it's just less efficient + * as it involves an extra thread and cancels the scheduled action that does the timeout. + * + * --- Blocking Timeout (performAsyncTimeout:false) --- + * + * When blocking timeout is assumed (default behavior for execute/queue flows) then the async + * timeout will not have been scheduled and this will wait in a blocking manner and if a timeout occurs + * trigger the timeout logic that comes from inside the Observable/Observer. + * + * + * --- Examples + * + * Stack for timeout with performAsyncTimeout=false (note the calling thread via get): + * + * at com.netflix.hystrix.HystrixCommand$TimeoutObservable$1$1.tick(HystrixCommand.java:788) + * at com.netflix.hystrix.HystrixCommand$1.performBlockingGetWithTimeout(HystrixCommand.java:536) + * at com.netflix.hystrix.HystrixCommand$1.get(HystrixCommand.java:484) + * at com.netflix.hystrix.HystrixCommand.execute(HystrixCommand.java:413) + * + * + * Stack for timeout with performAsyncTimeout=true (note the HystrixTimer involved): + * + * at com.netflix.hystrix.HystrixCommand$TimeoutObservable$1$1.tick(HystrixCommand.java:799) + * at com.netflix.hystrix.util.HystrixTimer$1.run(HystrixTimer.java:101) + * + * + * + * @param o + * @param f + * @throws InterruptedException + * @throws ExecutionException + */ + protected R performBlockingGetWithTimeout(final ObservableCommand o, final Future f) throws InterruptedException, ExecutionException { + // shortcut if already done + if (f.isDone()) { + return f.get(); + } + + // it's still working so proceed with blocking/timeout logic + HystrixExecutableBase originalCommand = o.getCommand(); + /** + * One thread will get the timeoutTimer if it's set and clear it then do blocking timeout. + *

+ * If non-blocking timeout was scheduled this will unschedule it. If it wasn't scheduled it is basically + * a no-op but fits the same interface so blocking and non-blocking flows both work the same. + *

+ * This "originalCommand" concept exists because of request caching. We only do the work and timeout logic + * on the original, not the cached responses. However, whichever the first thread is that comes in to block + * will be the one who performs the timeout logic. + *

+ * If request caching is disabled then it will always go into here. + */ + if (originalCommand != null) { + Reference timer = originalCommand.timeoutTimer.getAndSet(null); + if (timer != null) { + /** + * If an async timeout was scheduled then: + * + * - We are going to clear the Reference so the scheduler threads stop managing the timeout + * and we'll take over instead since we're going to be blocking on it anyways. + * + * - Other threads (since we won the race) will just wait on the normal Future which will release + * once the Observable is marked as completed (which may come via timeout) + * + * If an async timeout was not scheduled: + * + * - We go through the same flow as we receive the same interfaces just the "timer.clear()" will do nothing. + */ + // get the timer we'll use to perform the timeout + TimerListener l = timer.get(); + // remove the timer from the scheduler + timer.clear(); + + // determine how long we should wait for, taking into account time since work started + // and when this thread came in to block. If invocationTime hasn't been set then assume time remaining is entire timeout value + // as this maybe a case of multiple threads trying to run this command in which one thread wins but even before the winning thread is able to set + // the starttime another thread going via the Cached command route gets here first. + long timeout = originalCommand.properties.executionIsolationThreadTimeoutInMilliseconds().get(); + long timeRemaining = timeout; + long currTime = System.currentTimeMillis(); + if (originalCommand.invocationStartTime != -1) { + timeRemaining = (originalCommand.invocationStartTime + + originalCommand.properties.executionIsolationThreadTimeoutInMilliseconds().get()) + - currTime; + + } + if (timeRemaining > 0) { + // we need to block with the calculated timeout + try { + return f.get(timeRemaining, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + if (l != null) { + // this perform the timeout logic on the Observable/Observer + l.tick(); + } + } + } else { + // this means it should have already timed out so do so if it is not completed + if (!f.isDone()) { + if (l != null) { + l.tick(); + } + } + } + } + } + // other threads will block until the "l.tick" occurs and releases the underlying Future. + return f.get(); + } + + @Override + public R get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + return get(); + } + + }; + + } + + /** + * Used for asynchronous execution of command with a callback by subscribing to the {@link Observable}. + *

+ * This eagerly starts execution of the command the same as {@link #queue()} and {@link #execute()}. + * A lazy {@link Observable} can be obtained from {@link #toObservable()}. + *

+ * Callback Scheduling + *

+ *

    + *
  • When using {@link ExecutionIsolationStrategy#THREAD} this defaults to using {@link Schedulers#threadPoolForComputation()} for callbacks.
    • + *
  • When using {@link ExecutionIsolationStrategy#SEMAPHORE} this defaults to using {@link Schedulers#immediate()} for callbacks.
    • + *
+ * Use {@link #toObservable(rx.Scheduler)} to schedule the callback differently. + *

+ * See https://github.com/Netflix/RxJava/wiki for more information. + * + * @return {@code Observable} that executes and calls back with the result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * @throws HystrixRuntimeException + * if a fallback does not exist + *

+ *

    + *
  • via {@code Observer#onError} if a failure occurs
    • + *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • + *
+ * @throws HystrixBadRequestException + * via {@code Observer#onError} if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once + */ + public Observable observe() { + // us a ReplaySubject to buffer the eagerly subscribed-to Observable + ReplaySubject subject = ReplaySubject.create(); + // eagerly kick off subscription + toObservable().subscribe(subject); + // return the subject that can be subscribed to later while the execution has already started + return subject; + } + + /** + * A lazy {@link Observable} that will execute the command when subscribed to. + *

+ * See https://github.com/Netflix/RxJava/wiki for more information. + * + * @param observeOn + * The {@link Scheduler} to execute callbacks on. + * @return {@code Observable} that lazily executes and calls back with the result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * @throws HystrixRuntimeException + * if a fallback does not exist + *

+ *

    + *
  • via {@code Observer#onError} if a failure occurs
    • + *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • + *
+ * @throws HystrixBadRequestException + * via {@code Observer#onError} if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once + */ + public Observable toObservable(Scheduler observeOn) { + return toObservable(observeOn, true); + } + + public abstract Observable toObservable(); + + protected abstract ObservableCommand toObservable(final Scheduler observeOn, boolean performAsyncTimeout); + + /** + * Get the TryableSemaphore this HystrixCommand should use if a fallback occurs. + * + * @param circuitBreaker + * @param fallbackSemaphore + * @return TryableSemaphore + */ + protected TryableSemaphore getFallbackSemaphore() { + if (fallbackSemaphoreOverride == null) { + TryableSemaphore _s = fallbackSemaphorePerCircuit.get(commandKey.name()); + if (_s == null) { + // we didn't find one cache so setup + fallbackSemaphorePerCircuit.putIfAbsent(commandKey.name(), new TryableSemaphore(properties.fallbackIsolationSemaphoreMaxConcurrentRequests())); + // assign whatever got set (this or another thread) + return fallbackSemaphorePerCircuit.get(commandKey.name()); + } else { + return _s; + } + } else { + return fallbackSemaphoreOverride; + } + } + + /** + * Get the TryableSemaphore this HystrixCommand should use for execution if not running in a separate thread. + * + * @param circuitBreaker + * @param fallbackSemaphore + * @return TryableSemaphore + */ + protected TryableSemaphore getExecutionSemaphore() { + if (executionSemaphoreOverride == null) { + TryableSemaphore _s = executionSemaphorePerCircuit.get(commandKey.name()); + if (_s == null) { + // we didn't find one cache so setup + executionSemaphorePerCircuit.putIfAbsent(commandKey.name(), new TryableSemaphore(properties.executionIsolationSemaphoreMaxConcurrentRequests())); + // assign whatever got set (this or another thread) + return executionSemaphorePerCircuit.get(commandKey.name()); + } else { + return _s; + } + } else { + return executionSemaphoreOverride; + } + } + + protected static class ObservableCommand extends Observable { + private final HystrixExecutableBase command; + + ObservableCommand(OnSubscribe func, final HystrixExecutableBase command) { + super(func); + this.command = command; + } + + public HystrixExecutableBase getCommand() { + return command; + } + + ObservableCommand(final Observable originalObservable, final HystrixExecutableBase command) { + super(new OnSubscribe() { + + @Override + public void call(Subscriber observer) { + originalObservable.subscribe(observer); + } + }); + this.command = command; + } + + } + + /** + * Wraps a source Observable and remembers the original HystrixCommand. + *

+ * Used for request caching so multiple commands can respond from a single Observable but also get access to the originating HystrixCommand. + * + * @param + */ + protected static class CachedObservableOriginal extends ObservableCommand { + + final HystrixExecutableBase originalCommand; + + CachedObservableOriginal(final Observable actual, HystrixExecutableBase command) { + super(new OnSubscribe() { + + @Override + public void call(final Subscriber observer) { + actual.subscribe(observer); + } + }, command); + this.originalCommand = command; + } + } + + /** + * Wraps a CachedObservableOriginal as it is being returned from cache. + *

+ * As the Observable completes it copies state used for ExecutionResults + * and metrics that differentiate between the original and the de-duped "response from cache" command execution. + * + * @param + */ + protected static class CachedObservableResponse extends ObservableCommand { + final CachedObservableOriginal originalObservable; + + CachedObservableResponse(final CachedObservableOriginal originalObservable, final HystrixExecutableBase commandOfDuplicateCall) { + super(new OnSubscribe() { + + @Override + public void call(final Subscriber observer) { + originalObservable.subscribe(new Subscriber() { + + @Override + public void onCompleted() { + completeCommand(); + observer.onCompleted(); + } + + @Override + public void onError(Throwable e) { + completeCommand(); + observer.onError(e); + } + + @Override + public void onNext(R v) { + observer.onNext(v); + } + + private void completeCommand() { + // when the observable completes we then update the execution results of the duplicate command + // set this instance to the result that is from cache + commandOfDuplicateCall.executionResult = originalObservable.originalCommand.executionResult; + // add that this came from cache + commandOfDuplicateCall.executionResult = commandOfDuplicateCall.executionResult.addEvents(HystrixEventType.RESPONSE_FROM_CACHE); + // set the execution time to 0 since we retrieved from cache + commandOfDuplicateCall.executionResult = commandOfDuplicateCall.executionResult.setExecutionTime(-1); + // record that this command executed + commandOfDuplicateCall.recordExecutedCommand(); + } + }); + } + }, commandOfDuplicateCall); + this.originalObservable = originalObservable; + } + + /* + * This is a cached response so we want the command of the observable we're wrapping. + */ + public HystrixExecutableBase getCommand() { + return originalObservable.originalCommand; + } + } + + /** + * @return {@link HystrixCommandGroupKey} used to group together multiple {@link HystrixObservableCommand} objects. + *

+ * The {@link HystrixCommandGroupKey} is used to represent a common relationship between commands. For example, a library or team name, the system all related commands interace with, + * common business purpose etc. + */ + public HystrixCommandGroupKey getCommandGroup() { + return commandGroup; + } + + /** + * @return {@link HystrixCommandKey} identifying this command instance for statistics, circuit-breaker, properties, etc. + */ + public HystrixCommandKey getCommandKey() { + return commandKey; + } + + /** + * @return {@link HystrixThreadPoolKey} identifying which thread-pool this command uses (when configured to run on separate threads via + * {@link HystrixCommandProperties#executionIsolationStrategy()}). + */ + public HystrixThreadPoolKey getThreadPoolKey() { + return threadPoolKey; + } + + /* package */HystrixCircuitBreaker getCircuitBreaker() { + return circuitBreaker; + } + + /** + * The {@link HystrixCommandMetrics} associated with this {@link HystrixObservableCommand} instance. + * + * @return HystrixCommandMetrics + */ + public HystrixCommandMetrics getMetrics() { + return metrics; + } + + /** + * The {@link HystrixCommandProperties} associated with this {@link HystrixObservableCommand} instance. + * + * @return HystrixCommandProperties + */ + public HystrixCommandProperties getProperties() { + return properties; + } + + /** + * Record the duration of execution as response or exception is being returned to the caller. + */ + protected void recordTotalExecutionTime(long startTime) { + long duration = System.currentTimeMillis() - startTime; + // the total execution time for the user thread including queuing, thread scheduling, run() execution + metrics.addUserThreadExecutionTime(duration); + + /* + * We record the executionTime for command execution. + * + * If the command is never executed (rejected, short-circuited, etc) then it will be left unset. + * + * For this metric we include failures and successes as we use it for per-request profiling and debugging + * whereas 'metrics.addCommandExecutionTime(duration)' is used by stats across many requests. + */ + executionResult = executionResult.setExecutionTime((int) duration); + } + + /** + * Record that this command was executed in the HystrixRequestLog. + *

+ * This can be treated as an async operation as it just adds a references to "this" in the log even if the current command is still executing. + */ + protected void recordExecutedCommand() { + if (properties.requestLogEnabled().get()) { + // log this command execution regardless of what happened + if (currentRequestLog != null) { + currentRequestLog.addExecutedCommand(this); + } + } + } + + /** + * Take an Exception and determine whether to throw it, its cause or a new HystrixRuntimeException. + *

+ * This will only throw an HystrixRuntimeException, HystrixBadRequestException or IllegalStateException + * + * @param e + * @return HystrixRuntimeException, HystrixBadRequestException or IllegalStateException + */ + protected RuntimeException decomposeException(Exception e) { + if (e instanceof IllegalStateException) { + return (IllegalStateException) e; + } + if (e instanceof HystrixBadRequestException) { + return (HystrixBadRequestException) e; + } + if (e.getCause() instanceof HystrixBadRequestException) { + return (HystrixBadRequestException) e.getCause(); + } + if (e instanceof HystrixRuntimeException) { + return (HystrixRuntimeException) e; + } + // if we have an exception we know about we'll throw it directly without the wrapper exception + if (e.getCause() instanceof HystrixRuntimeException) { + return (HystrixRuntimeException) e.getCause(); + } + // we don't know what kind of exception this is so create a generic message and throw a new HystrixRuntimeException + String message = getLogMessagePrefix() + " failed while executing."; + logger.debug(message, e); // debug only since we're throwing the exception and someone higher will do something with it + return new HystrixRuntimeException(FailureType.COMMAND_EXCEPTION, this.getClass(), message, e, null); + + } + + /* ******************************************************************************** */ + /* ******************************************************************************** */ + /* TryableSemaphore */ + /* ******************************************************************************** */ + /* ******************************************************************************** */ + + /** + * Semaphore that only supports tryAcquire and never blocks and that supports a dynamic permit count. + *

+ * Using AtomicInteger increment/decrement instead of java.util.concurrent.Semaphore since we don't need blocking and need a custom implementation to get the dynamic permit count and since + * AtomicInteger achieves the same behavior and performance without the more complex implementation of the actual Semaphore class using AbstractQueueSynchronizer. + */ + /* package */static class TryableSemaphore { + protected final HystrixProperty numberOfPermits; + private final AtomicInteger count = new AtomicInteger(0); + + public TryableSemaphore(HystrixProperty numberOfPermits) { + this.numberOfPermits = numberOfPermits; + } + + /** + * Use like this: + *

+ * + * 

+         * if (s.tryAcquire()) {
+         * try {
+         * // do work that is protected by 's'
+         * } finally {
+         * s.release();
+         * }
+         * }
+         *
+ * + * @return boolean + */ + public boolean tryAcquire() { + int currentCount = count.incrementAndGet(); + if (currentCount > numberOfPermits.get()) { + count.decrementAndGet(); + return false; + } else { + return true; + } + } + + /** + * ONLY call release if tryAcquire returned true. + *

+ * + * 

+         * if (s.tryAcquire()) {
+         * try {
+         * // do work that is protected by 's'
+         * } finally {
+         * s.release();
+         * }
+         * }
+         *
+ */ + public void release() { + count.decrementAndGet(); + } + + public int getNumberOfPermitsUsed() { + return count.get(); + } + + } + + /* ******************************************************************************** */ + /* ******************************************************************************** */ + /* Result Status */ + /* ******************************************************************************** */ + /* ******************************************************************************** */ + + /** + * Immutable holder class for the status of command execution. + *

+ * Contained within a class to simplify the sharing of it across Futures/threads that result from request caching. + *

+ * This object can be referenced and "modified" by parent and child threads as well as by different instances of HystrixCommand since + * 1 instance could create an ExecutionResult, cache a Future that refers to it, a 2nd instance execution then retrieves a Future + * from cache and wants to append RESPONSE_FROM_CACHE to whatever the ExecutionResult was from the first command execution. + *

+ * This being immutable forces and ensure thread-safety instead of using AtomicInteger/ConcurrentLinkedQueue and determining + * when it's safe to mutate the object directly versus needing to deep-copy clone to a new instance. + */ + protected static class ExecutionResult { + protected final List events; + private final int executionTime; + private final Exception exception; + + private ExecutionResult(HystrixEventType... events) { + this(Arrays.asList(events), -1, null); + } + + public ExecutionResult setExecutionTime(int executionTime) { + return new ExecutionResult(events, executionTime, exception); + } + + public ExecutionResult setException(Exception e) { + return new ExecutionResult(events, executionTime, e); + } + + private ExecutionResult(List events, int executionTime, Exception e) { + // we are safe assigning the List reference instead of deep-copying + // because we control the original list in 'newEvent' + this.events = events; + this.executionTime = executionTime; + this.exception = e; + } + + // we can return a static version since it's immutable + private static ExecutionResult EMPTY = new ExecutionResult(new HystrixEventType[0]); + + /** + * Creates a new ExecutionResult by adding the defined 'events' to the ones on the current instance. + * + * @param events + * @return + */ + public ExecutionResult addEvents(HystrixEventType... events) { + ArrayList newEvents = new ArrayList(); + newEvents.addAll(this.events); + for (HystrixEventType e : events) { + newEvents.add(e); + } + return new ExecutionResult(Collections.unmodifiableList(newEvents), executionTime, exception); + } + + public int getExecutionTime() { + return executionTime; + } + + public Exception getException() { + return exception; + } + } + + /* ******************************************************************************** */ + /* ******************************************************************************** */ + /* RequestCache */ + /* ******************************************************************************** */ + /* ******************************************************************************** */ + + /** + * Key to be used for request caching. + *

+ * By default this returns null which means "do not cache". + *

+ * To enable caching override this method and return a string key uniquely representing the state of a command instance. + *

+ * If multiple command instances in the same request scope match keys then only the first will be executed and all others returned from cache. + * + * @return cacheKey + */ + protected String getCacheKey() { + return null; + } + + protected boolean isRequestCachingEnabled() { + return properties.requestCacheEnabled().get() && getCacheKey() != null; + } + + protected String getLogMessagePrefix() { + return getCommandKey().name(); + } + + /** + * Whether the 'circuit-breaker' is open meaning that execute() will immediately return + * the getFallback() response and not attempt a HystrixCommand execution. + * + * @return boolean + */ + public boolean isCircuitBreakerOpen() { + return circuitBreaker.isOpen(); + } + + /** + * If this command has completed execution either successfully, via fallback or failure. + * + * @return boolean + */ + public boolean isExecutionComplete() { + return isExecutionComplete.get(); + } + + /** + * Whether the execution occurred in a separate thread. + *

+ * This should be called only once execute()/queue()/fireOrForget() are called otherwise it will always return false. + *

+ * This specifies if a thread execution actually occurred, not just if it is configured to be executed in a thread. + * + * @return boolean + */ + public boolean isExecutedInThread() { + return isExecutedInThread.get(); + } + + /** + * Whether the response was returned successfully either by executing run() or from cache. + * + * @return boolean + */ + public boolean isSuccessfulExecution() { + return executionResult.events.contains(HystrixEventType.SUCCESS); + } + + /** + * Whether the run() resulted in a failure (exception). + * + * @return boolean + */ + public boolean isFailedExecution() { + return executionResult.events.contains(HystrixEventType.FAILURE); + } + + /** + * Get the Throwable/Exception thrown that caused the failure. + *

+ * If isFailedExecution() == true then this would represent the Exception thrown by the run() method. + *

+ * If isFailedExecution() == false then this would return null. + * + * @return Throwable or null + */ + public Throwable getFailedExecutionException() { + return executionResult.getException(); + } + + /** + * Whether the response received from was the result of some type of failure + * and getFallback() being called. + * + * @return boolean + */ + public boolean isResponseFromFallback() { + return executionResult.events.contains(HystrixEventType.FALLBACK_SUCCESS); + } + + /** + * Whether the response received was the result of a timeout + * and getFallback() being called. + * + * @return boolean + */ + public boolean isResponseTimedOut() { + return executionResult.events.contains(HystrixEventType.TIMEOUT); + } + + /** + * Whether the response received was a fallback as result of being + * short-circuited (meaning isCircuitBreakerOpen() == true) and getFallback() being called. + * + * @return boolean + */ + public boolean isResponseShortCircuited() { + return executionResult.events.contains(HystrixEventType.SHORT_CIRCUITED); + } + + /** + * Whether the response is from cache and run() was not invoked. + * + * @return boolean + */ + public boolean isResponseFromCache() { + return executionResult.events.contains(HystrixEventType.RESPONSE_FROM_CACHE); + } + + /** + * Whether the response received was a fallback as result of being + * rejected (from thread-pool or semaphore) and getFallback() being called. + * + * @return boolean + */ + public boolean isResponseRejected() { + return executionResult.events.contains(HystrixEventType.THREAD_POOL_REJECTED) || executionResult.events.contains(HystrixEventType.SEMAPHORE_REJECTED); + } + + /** + * List of HystrixCommandEventType enums representing events that occurred during execution. + *

+ * Examples of events are SUCCESS, FAILURE, TIMEOUT, and SHORT_CIRCUITED + * + * @return {@code List} + */ + public List getExecutionEvents() { + return executionResult.events; + } + + /** + * The execution time of this command instance in milliseconds, or -1 if not executed. + * + * @return int + */ + public int getExecutionTimeInMilliseconds() { + return executionResult.getExecutionTime(); + } + + protected Exception getExceptionFromThrowable(Throwable t) { + Exception e = null; + if (t instanceof Exception) { + e = (Exception) t; + } else { + // Hystrix 1.x uses Exception, not Throwable so to prevent a breaking change Throwable will be wrapped in Exception + e = new Exception("Throwable caught while executing.", t); + } + return e; + } + + private static class ExecutionHookDeprecationWrapper extends HystrixCommandExecutionHook { + + private final HystrixCommandExecutionHook actual; + + ExecutionHookDeprecationWrapper(HystrixCommandExecutionHook actual) { + this.actual = actual; + } + + @Override + @Deprecated + public void onRunStart(HystrixCommand commandInstance) { + actual.onRunStart(commandInstance); + } + + @Override + public void onRunStart(HystrixExecutable commandInstance) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + onRunStart(c); + } + actual.onRunStart(commandInstance); + } + + @Override + @Deprecated + public T onRunSuccess(HystrixCommand commandInstance, T response) { + return actual.onRunSuccess(commandInstance, response); + } + + @Override + public T onRunSuccess(HystrixExecutable commandInstance, T response) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + response = onRunSuccess(c, response); + } + return actual.onRunSuccess(commandInstance, response); + } + + @Override + @Deprecated + public Exception onRunError(HystrixCommand commandInstance, Exception e) { + return actual.onRunError(commandInstance, e); + } + + @Override + public Exception onRunError(HystrixExecutable commandInstance, Exception e) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + e = onRunError(c, e); + } + return actual.onRunError(commandInstance, e); + } + + @Override + @Deprecated + public void onFallbackStart(HystrixCommand commandInstance) { + actual.onFallbackStart(commandInstance); + } + + @Override + public void onFallbackStart(HystrixExecutable commandInstance) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + onFallbackStart(c); + } + actual.onFallbackStart(commandInstance); + } + + @Override + @Deprecated + public T onFallbackSuccess(HystrixCommand commandInstance, T fallbackResponse) { + return actual.onFallbackSuccess(commandInstance, fallbackResponse); + } + + @Override + public T onFallbackSuccess(HystrixExecutable commandInstance, T fallbackResponse) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + fallbackResponse = onFallbackSuccess(c, fallbackResponse); + } + return actual.onFallbackSuccess(commandInstance, fallbackResponse); + } + + @Override + @Deprecated + public Exception onFallbackError(HystrixCommand commandInstance, Exception e) { + return actual.onFallbackError(commandInstance, e); + } + + @Override + public Exception onFallbackError(HystrixExecutable commandInstance, Exception e) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + e = onFallbackError(c, e); + } + return actual.onFallbackError(commandInstance, e); + } + + @Override + @Deprecated + public void onStart(HystrixCommand commandInstance) { + actual.onStart(commandInstance); + } + + @Override + public void onStart(HystrixExecutable commandInstance) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + onStart(c); + } + actual.onStart(commandInstance); + } + + @Override + @Deprecated + public T onComplete(HystrixCommand commandInstance, T response) { + return actual.onComplete(commandInstance, response); + } + + @Override + public T onComplete(HystrixExecutable commandInstance, T response) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + response = onComplete(c, response); + } + return actual.onComplete(commandInstance, response); + } + + @Override + @Deprecated + public Exception onError(HystrixCommand commandInstance, FailureType failureType, Exception e) { + return actual.onError(commandInstance, failureType, e); + } + + @Override + public Exception onError(HystrixExecutable commandInstance, FailureType failureType, Exception e) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + e = onError(c, failureType, e); + } + return actual.onError(commandInstance, failureType, e); + } + + @Override + @Deprecated + public void onThreadStart(HystrixCommand commandInstance) { + actual.onThreadStart(commandInstance); + } + + @Override + public void onThreadStart(HystrixExecutable commandInstance) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + onThreadStart(c); + } + actual.onThreadStart(commandInstance); + } + + @Override + @Deprecated + public void onThreadComplete(HystrixCommand commandInstance) { + actual.onThreadComplete(commandInstance); + } + + @Override + public void onThreadComplete(HystrixExecutable commandInstance) { + HystrixCommand c = getHystrixCommandFromAbstractIfApplicable(commandInstance); + if (c != null) { + onThreadComplete(c); + } + actual.onThreadComplete(commandInstance); + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private HystrixCommand getHystrixCommandFromAbstractIfApplicable(HystrixExecutable commandInstance) { + if (commandInstance instanceof HystrixCommand.HystrixCommandFromObservableCommand) { + return ((HystrixCommand.HystrixCommandFromObservableCommand) commandInstance).getOriginal(); + } else { + return null; + } + } + } +} diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutableInfo.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutableInfo.java new file mode 100644 index 000000000..eb387c29d --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixExecutableInfo.java @@ -0,0 +1,43 @@ +package com.netflix.hystrix; + +import java.util.List; + +public interface HystrixExecutableInfo { + + public HystrixCommandGroupKey getCommandGroup(); + + public HystrixCommandKey getCommandKey(); + + public HystrixThreadPoolKey getThreadPoolKey(); + + public HystrixCommandMetrics getMetrics(); + + public HystrixCommandProperties getProperties(); + + public boolean isCircuitBreakerOpen(); + + public boolean isExecutionComplete(); + + public boolean isExecutedInThread(); + + public boolean isSuccessfulExecution(); + + public boolean isFailedExecution(); + + public Throwable getFailedExecutionException(); + + public boolean isResponseFromFallback(); + + public boolean isResponseTimedOut(); + + public boolean isResponseShortCircuited(); + + public boolean isResponseFromCache(); + + public boolean isResponseRejected(); + + public List getExecutionEvents(); + + public int getExecutionTimeInMilliseconds(); + +} \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixObservableCommand.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixObservableCommand.java new file mode 100644 index 000000000..b5c358d38 --- /dev/null +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixObservableCommand.java @@ -0,0 +1,898 @@ +/** + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.hystrix; + +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.concurrent.NotThreadSafe; +import javax.annotation.concurrent.ThreadSafe; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Notification; +import rx.Observable; +import rx.Observable.OnSubscribe; +import rx.Observable.Operator; +import rx.Scheduler; +import rx.Subscriber; +import rx.functions.Action0; +import rx.functions.Action1; +import rx.functions.Func1; +import rx.schedulers.Schedulers; +import rx.subscriptions.CompositeSubscription; + +import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; +import com.netflix.hystrix.exception.HystrixBadRequestException; +import com.netflix.hystrix.exception.HystrixRuntimeException; +import com.netflix.hystrix.exception.HystrixRuntimeException.FailureType; +import com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable; +import com.netflix.hystrix.strategy.concurrency.HystrixContextScheduler; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; +import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; +import com.netflix.hystrix.util.HystrixTimer; +import com.netflix.hystrix.util.HystrixTimer.TimerListener; + +/** + * Used to wrap code that will execute potentially risky functionality (typically meaning a service call over the network) + * with fault and latency tolerance, statistics and performance metrics capture, circuit breaker and bulkhead functionality. + * This command should be used for a purely non-blocking call pattern. The caller of this command will be subscribed to the Observable returned by the run() method. + * + * @param + * the return type + */ +@ThreadSafe +public abstract class HystrixObservableCommand extends HystrixExecutableBase implements HystrixExecutable, HystrixExecutableInfo { + + private static final Logger logger = LoggerFactory.getLogger(HystrixObservableCommand.class); + + /** + * Construct a {@link HystrixObservableCommand} with defined {@link HystrixCommandGroupKey}. + *

+ * The {@link HystrixCommandKey} will be derived from the implementing class name. + * + * @param group + * {@link HystrixCommandGroupKey} used to group together multiple {@link HystrixObservableCommand} objects. + *

+ * The {@link HystrixCommandGroupKey} is used to represent a common relationship between commands. For example, a library or team name, the system all related commands interace with, + * common business purpose etc. + */ + protected HystrixObservableCommand(HystrixCommandGroupKey group) { + // use 'null' to specify use the default + this(new Setter(group)); + } + + /** + * Construct a {@link HystrixObservableCommand} with defined {@link Setter} that allows injecting property and strategy overrides and other optional arguments. + *

+ * NOTE: The {@link HystrixCommandKey} is used to associate a {@link HystrixObservableCommand} with {@link HystrixCircuitBreaker}, {@link HystrixCommandMetrics} and other objects. + *

+ * Do not create multiple {@link HystrixObservableCommand} implementations with the same {@link HystrixCommandKey} but different injected default properties as the first instantiated will win. + *

+ * Properties passed in via {@link Setter#andCommandPropertiesDefaults} or {@link Setter#andThreadPoolPropertiesDefaults} are cached for the given {@link HystrixCommandKey} for the life of the JVM + * or until {@link Hystrix#reset()} is called. Dynamic properties allow runtime changes. Read more on the Hystrix Wiki. + * + * @param setter + * Fluent interface for constructor arguments + */ + protected HystrixObservableCommand(Setter setter) { + // use 'null' to specify use the default + this(setter.groupKey, setter.commandKey, setter.threadPoolKey, null, null, setter.commandPropertiesDefaults, setter.threadPoolPropertiesDefaults, null, null, null, null, null); + } + + /** + * Allow constructing a {@link HystrixObservableCommand} with injection of most aspects of its functionality. + *

+ * Some of these never have a legitimate reason for injection except in unit testing. + *

+ * Most of the args will revert to a valid default if 'null' is passed in. + */ + HystrixObservableCommand(HystrixCommandGroupKey group, HystrixCommandKey key, HystrixThreadPoolKey threadPoolKey, HystrixCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, + HystrixCommandProperties.Setter commandPropertiesDefaults, HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults, + HystrixCommandMetrics metrics, TryableSemaphore fallbackSemaphore, TryableSemaphore executionSemaphore, + HystrixPropertiesStrategy propertiesStrategy, HystrixCommandExecutionHook executionHook) { + super(group, key, threadPoolKey, circuitBreaker, threadPool, commandPropertiesDefaults, threadPoolPropertiesDefaults, metrics, fallbackSemaphore, executionSemaphore, propertiesStrategy, executionHook); + } + + /** + * Fluent interface for arguments to the {@link HystrixObservableCommand} constructor. + *

+ * The required arguments are set via the 'with' factory method and optional arguments via the 'and' chained methods. + *

+ * Example: + * 

 {@code
+     *  Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("GroupName"))
+                .andCommandKey(HystrixCommandKey.Factory.asKey("CommandName"))
+                .andEventNotifier(notifier);
+     * }
+ */ + @NotThreadSafe + public static class Setter { + + protected final HystrixCommandGroupKey groupKey; + protected HystrixCommandKey commandKey; + protected HystrixThreadPoolKey threadPoolKey; + protected HystrixCommandProperties.Setter commandPropertiesDefaults; + protected HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults; + + /** + * Setter factory method containing required values. + *

+ * All optional arguments can be set via the chained methods. + * + * @param groupKey + * {@link HystrixCommandGroupKey} used to group together multiple {@link HystrixObservableCommand} objects. + *

+ * The {@link HystrixCommandGroupKey} is used to represent a common relationship between commands. For example, a library or team name, the system all related commands interace + * with, + * common business purpose etc. + */ + protected Setter(HystrixCommandGroupKey groupKey) { + this.groupKey = groupKey; + + // default to using SEMAPHORE for ObservableCommand + commandPropertiesDefaults = setDefaults(HystrixCommandProperties.Setter()); + } + + /** + * Setter factory method with required values. + *

+ * All optional arguments can be set via the chained methods. + * + * @param groupKey + * {@link HystrixCommandGroupKey} used to group together multiple {@link HystrixObservableCommand} objects. + *

+ * The {@link HystrixCommandGroupKey} is used to represent a common relationship between commands. For example, a library or team name, the system all related commands interace + * with, + * common business purpose etc. + */ + public static Setter withGroupKey(HystrixCommandGroupKey groupKey) { + return new Setter(groupKey); + } + + /** + * @param commandKey + * {@link HystrixCommandKey} used to identify a {@link HystrixObservableCommand} instance for statistics, circuit-breaker, properties, etc. + *

+ * By default this will be derived from the instance class name. + *

+ * NOTE: Every unique {@link HystrixCommandKey} will result in new instances of {@link HystrixCircuitBreaker}, {@link HystrixCommandMetrics} and {@link HystrixCommandProperties}. + * Thus, + * the number of variants should be kept to a finite and reasonable number to avoid high-memory usage or memory leacks. + *

+ * Hundreds of keys is fine, tens of thousands is probably not. + * @return Setter for fluent interface via method chaining + */ + public Setter andCommandKey(HystrixCommandKey commandKey) { + this.commandKey = commandKey; + return this; + } + + /** + * Optional + * + * @param commandPropertiesDefaults + * {@link HystrixCommandProperties.Setter} with property overrides for this specific instance of {@link HystrixObservableCommand}. + *

+ * See the {@link HystrixPropertiesStrategy} JavaDocs for more information on properties and order of precedence. + * @return Setter for fluent interface via method chaining + */ + public Setter andCommandPropertiesDefaults(HystrixCommandProperties.Setter commandPropertiesDefaults) { + this.commandPropertiesDefaults = setDefaults(commandPropertiesDefaults); + return this; + } + + private HystrixCommandProperties.Setter setDefaults(HystrixCommandProperties.Setter commandPropertiesDefaults) { + if (commandPropertiesDefaults.getExecutionIsolationStrategy() == null) { + // default to using SEMAPHORE for ObservableCommand if the user didn't set it + commandPropertiesDefaults.withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE); + } + return commandPropertiesDefaults; + } + + } + + /** + * Implement this method with code to be executed when {@link #execute()} or {@link #queue()} are invoked. + * + * @return R response type + */ + protected abstract Observable run(); + + /** + * If {@link #execute()} or {@link #queue()} fails in any way then this method will be invoked to provide an opportunity to return a fallback response. + *

+ * This should do work that does not require network transport to produce. + *

+ * In other words, this should be a static or cached result that can immediately be returned upon failure. + *

+ * If network traffic is wanted for fallback (such as going to MemCache) then the fallback implementation should invoke another {@link HystrixObservableCommand} instance that protects against + * that network + * access and possibly has another level of fallback that does not involve network access. + *

+ * DEFAULT BEHAVIOR: It throws UnsupportedOperationException. + * + * @return R or UnsupportedOperationException if not implemented + */ + protected Observable getFallback() { + return Observable.error(new UnsupportedOperationException("No fallback available.")); + } + + /** + * A lazy {@link Observable} that will execute the command when subscribed to. + *

+ * Callback Scheduling + *

+ *

    + *
  • When using {@link ExecutionIsolationStrategy#THREAD} this defaults to using {@link Schedulers#threadPoolForComputation()} for callbacks.
    • + *
  • When using {@link ExecutionIsolationStrategy#SEMAPHORE} this defaults to using {@link Schedulers#immediate()} for callbacks.
    • + *
+ *

+ * See https://github.com/Netflix/RxJava/wiki for more information. + * + * @return {@code Observable} that lazily executes and calls back with the result of {@link #run()} execution or a fallback from {@link #getFallback()} if the command fails for any reason. + * + * @throws HystrixRuntimeException + * if a fallback does not exist + *

+ *

    + *
  • via {@code Observer#onError} if a failure occurs
    • + *
  • or immediately if the command can not be queued (such as short-circuited, thread-pool/semaphore rejected)
    • + *
+ * @throws HystrixBadRequestException + * via {@code Observer#onError} if invalid arguments or state were used representing a user failure, not a system failure + * @throws IllegalStateException + * if invoked more than once + */ + public Observable toObservable() { + return toObservable(Schedulers.computation()); + } + + protected ObservableCommand toObservable(final Scheduler observeOn, boolean performAsyncTimeout) { + /* this is a stateful object so can only be used once */ + if (!started.compareAndSet(false, true)) { + throw new IllegalStateException("This instance can only be executed once. Please instantiate a new instance."); + } + + /* try from cache first */ + if (isRequestCachingEnabled()) { + Observable fromCache = requestCache.get(getCacheKey()); + if (fromCache != null) { + /* mark that we received this response from cache */ + metrics.markResponseFromCache(); + return new CachedObservableResponse((CachedObservableOriginal) fromCache, this); + } + } + + final HystrixObservableCommand _this = this; + final AtomicReference endCurrentThreadExecutingCommand = new AtomicReference(); // don't like how this is being done + + // create an Observable that will lazily execute when subscribed to + Observable o = Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber observer) { + // async record keeping + recordExecutedCommand(); + + // mark that we're starting execution on the ExecutionHook + executionHook.onStart(_this); + + /* determine if we're allowed to execute */ + if (circuitBreaker.allowRequest()) { + final TryableSemaphore executionSemaphore = getExecutionSemaphore(); + // acquire a permit + if (executionSemaphore.tryAcquire()) { + try { + /* used to track userThreadExecutionTime */ + invocationStartTime = System.currentTimeMillis(); + + // store the command that is being run + endCurrentThreadExecutingCommand.set(Hystrix.startCurrentThreadExecutingCommand(getCommandKey())); + + getRunObservableDecoratedForMetricsAndErrorHandling(observeOn) + .doOnTerminate(new Action0() { + + @Override + public void call() { + // release the semaphore + // this is done here instead of below so that the acquire/release happens where it is guaranteed + // and not affected by the conditional circuit-breaker checks, timeouts, etc + executionSemaphore.release(); + + } + }).subscribe(observer); + } catch (RuntimeException e) { + observer.onError(e); + } + } else { + metrics.markSemaphoreRejection(); + logger.debug("HystrixCommand Execution Rejection by Semaphore."); // debug only since we're throwing the exception and someone higher will do something with it + // retrieve a fallback or throw an exception if no fallback available + getFallbackOrThrowException(HystrixEventType.SEMAPHORE_REJECTED, FailureType.REJECTED_SEMAPHORE_EXECUTION, "could not acquire a semaphore for execution").subscribe(observer); + } + } else { + // record that we are returning a short-circuited fallback + metrics.markShortCircuited(); + // short-circuit and go directly to fallback (or throw an exception if no fallback implemented) + try { + getFallbackOrThrowException(HystrixEventType.SHORT_CIRCUITED, FailureType.SHORTCIRCUIT, "short-circuited").subscribe(observer); + } catch (Exception e) { + observer.onError(e); + } + } + } + }); + + // wrap for timeout support + o = o.lift(new HystrixObservableTimeoutOperator(_this, performAsyncTimeout)); + + // error handling at very end (this means fallback didn't exist or failed) + o = o.onErrorResumeNext(new Func1>() { + + @Override + public Observable call(Throwable t) { + // count that we are throwing an exception and re-throw it + metrics.markExceptionThrown(); + return Observable.error(t); + } + + }); + + // any final cleanup needed + o = o.doOnTerminate(new Action0() { + + @Override + public void call() { + Reference tl = timeoutTimer.get(); + if (tl != null) { + tl.clear(); + } + + try { + // if we executed we will record the execution time + if (invocationStartTime > 0 && !isResponseRejected()) { + /* execution time (must occur before terminal state otherwise a race condition can occur if requested by client) */ + recordTotalExecutionTime(invocationStartTime); + } + + // pop the command that is being run + if (endCurrentThreadExecutingCommand.get() != null) { + endCurrentThreadExecutingCommand.get().call(); + } + } finally { + metrics.decrementConcurrentExecutionCount(); + // record that we're completed + isExecutionComplete.set(true); + } + } + + }); + + // put in cache + if (isRequestCachingEnabled()) { + // wrap it for caching + o = new CachedObservableOriginal(o.cache(), this); + Observable fromCache = requestCache.putIfAbsent(getCacheKey(), o); + if (fromCache != null) { + // another thread beat us so we'll use the cached value instead + o = new CachedObservableResponse((CachedObservableOriginal) fromCache, this); + } + // we just created an ObservableCommand so we cast and return it + return (ObservableCommand) o; + } else { + // no request caching so a simple wrapper just to pass 'this' along with the Observable + return new ObservableCommand(o, this); + } + } + + /** + * This decorate "Hystrix" functionality around the run() Observable. + * + * @return R + */ + private Observable getRunObservableDecoratedForMetricsAndErrorHandling(final Scheduler observeOn) { + final HystrixObservableCommand _cmd = this; + + final HystrixObservableCommand _self = this; + // allow tracking how many concurrent threads are executing + metrics.incrementConcurrentExecutionCount(); + + final HystrixRequestContext currentRequestContext = HystrixRequestContext.getContextForCurrentThread(); + + Observable run = null; + if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { + // mark that we are executing in a thread (even if we end up being rejected we still were a THREAD execution and not SEMAPHORE) + isExecutedInThread.set(true); + + run = Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + executionHook.onRunStart(_self); + executionHook.onThreadStart(_self); + if (isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT) { + // the command timed out in the wrapping thread so we will return immediately + // and not increment any of the counters below or other such logic + s.onError(new RuntimeException("timed out before executing run()")); + } else { + // not timed out so execute + try { + final Action0 endCurrentThread = Hystrix.startCurrentThreadExecutingCommand(getCommandKey()); + run().doOnTerminate(new Action0() { + + @Override + public void call() { + // TODO is this actually the end of the thread? + executionHook.onThreadComplete(_self); + endCurrentThread.call(); + } + }).subscribe(s); + } catch (Throwable t) { + // the run() method is a user provided implementation so can throw instead of using Observable.onError + // so we catch it here and turn it into Observable.error + Observable. error(t).subscribe(s); + } + } + } + }).subscribeOn(threadPool.getScheduler()); + } else { + // semaphore isolated + executionHook.onRunStart(_self); + try { + run = run(); + } catch (Throwable t) { + // the run() method is a user provided implementation so can throw instead of using Observable.onError + // so we catch it here and turn it into Observable.error + run = Observable.error(t); + } + } + + run = run.doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + setRequestContextIfNeeded(currentRequestContext); + } + + }).map(new Func1() { + + @Override + public R call(R t1) { + return executionHook.onRunSuccess(_cmd, t1); + } + + }).doOnCompleted(new Action0() { + // this must come before onErrorResumeNext as we only want successful onCompletes processed here + @Override + public void call() { + if (isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT) { + // the command timed out in the wrapping thread so we will return immediately + // and not increment any of the counters below or other such logic + } else { + long duration = System.currentTimeMillis() - invocationStartTime; + metrics.addCommandExecutionTime(duration); + // report success + executionResult = executionResult.addEvents(HystrixEventType.SUCCESS); + metrics.markSuccess(duration); + circuitBreaker.markSuccess(); + eventNotifier.markCommandExecution(getCommandKey(), properties.executionIsolationStrategy().get(), (int) duration, executionResult.events); + } + } + + }).onErrorResumeNext(new Func1>() { + + @Override + public Observable call(Throwable t) { + Exception e = getExceptionFromThrowable(t); + if (e instanceof RejectedExecutionException) { + // mark on counter + metrics.markThreadPoolRejection(); + // use a fallback instead (or throw exception if not implemented) + return getFallbackOrThrowException(HystrixEventType.THREAD_POOL_REJECTED, FailureType.REJECTED_THREAD_EXECUTION, "could not be queued for execution", e); + } else if (t instanceof HystrixBadRequestException) { + try { + Exception decorated = executionHook.onRunError(_self, (Exception) t); + + if (decorated instanceof HystrixBadRequestException) { + t = (HystrixBadRequestException) decorated; + } else { + logger.warn("ExecutionHook.onRunError returned an exception that was not an instance of HystrixBadRequestException so will be ignored.", decorated); + } + } catch (Exception hookException) { + logger.warn("Error calling ExecutionHook.onRunError", hookException); + } + + /* + * HystrixBadRequestException is treated differently and allowed to propagate without any stats tracking or fallback logic + */ + return Observable.error(t); + } else { + try { + e = executionHook.onRunError(_self, e); + } catch (Exception hookException) { + logger.warn("Error calling ExecutionHook.endRunFailure", hookException); + } + + if (isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT) { + // http://jira/browse/API-4905 HystrixCommand: Error/Timeout Double-count if both occur + // this means we have already timed out then we don't count this error stat and we just return + // as this means the user-thread has already returned, we've already done fallback logic + // and we've already counted the timeout stat + logger.debug("Error executing HystrixCommand.run() [TimedOut]. Proceeding to fallback logic ...", e); + return Observable.empty(); + } else { + logger.debug("Error executing HystrixCommand.run(). Proceeding to fallback logic ...", e); + } + + // report failure + metrics.markFailure(System.currentTimeMillis() - invocationStartTime); + // record the exception + executionResult = executionResult.setException(e); + return getFallbackOrThrowException(HystrixEventType.FAILURE, FailureType.COMMAND_EXCEPTION, "failed", e); + } + } + }).doOnEach(new Action1>() { + // setting again as the fallback could have lost the context + @Override + public void call(Notification n) { + setRequestContextIfNeeded(currentRequestContext); + } + + }).map(new Func1() { + + @Override + public R call(R t1) { + // allow transforming the results via the executionHook whether it came from success or fallback + return executionHook.onComplete(_cmd, t1); + } + + }); + + if (properties.executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD)) { + // we want to hand off work to a different scheduler so we don't tie up the Hystrix thread + if (!Schedulers.immediate().equals(observeOn)) { + // don't waste overhead if it's the 'immediate' scheduler + // otherwise we'll 'observeOn' and wrap with the HystrixContextScheduler + // to copy state across threads (if threads are involved) + run = run.observeOn(new HystrixContextScheduler(concurrencyStrategy, observeOn)); + } + } + + return run; + } + + /** + * Execute getFallback() within protection of a semaphore that limits number of concurrent executions. + *

+ * Fallback implementations shouldn't perform anything that can be blocking, but we protect against it anyways in case someone doesn't abide by the contract. + *

+ * If something in the getFallback() implementation is latent (such as a network call) then the semaphore will cause us to start rejecting requests rather than allowing potentially + * all threads to pile up and block. + * + * @return K + * @throws UnsupportedOperationException + * if getFallback() not implemented + * @throws HystrixException + * if getFallback() fails (throws an Exception) or is rejected by the semaphore + */ + private Observable getFallbackWithProtection() { + final TryableSemaphore fallbackSemaphore = getFallbackSemaphore(); + + // acquire a permit + if (fallbackSemaphore.tryAcquire()) { + executionHook.onFallbackStart(this); + final HystrixObservableCommand _cmd = this; + + Observable fallback = null; + try { + fallback = getFallback(); + } catch (Throwable t) { + // getFallback() is user provided and can throw so we catch it and turn it into Observable.error + fallback = Observable.error(t); + } + + return fallback.map(new Func1() { + + @Override + public R call(R t1) { + // allow transforming the value + return executionHook.onFallbackSuccess(_cmd, t1); + } + + }).onErrorResumeNext(new Func1>() { + + @Override + public Observable call(Throwable t) { + Exception e = getExceptionFromThrowable(t); + Exception decorated = executionHook.onFallbackError(_cmd, e); + + if (decorated instanceof RuntimeException) { + e = (RuntimeException) decorated; + } else { + logger.warn("ExecutionHook.onFallbackError returned an exception that was not an instance of RuntimeException so will be ignored.", decorated); + } + return Observable.error(e); + } + + }).doOnTerminate(new Action0() { + + @Override + public void call() { + fallbackSemaphore.release(); + } + + }); + } else { + metrics.markFallbackRejection(); + + logger.debug("HystrixCommand Fallback Rejection."); // debug only since we're throwing the exception and someone higher will do something with it + // if we couldn't acquire a permit, we "fail fast" by throwing an exception + return Observable.error(new HystrixRuntimeException(FailureType.REJECTED_SEMAPHORE_FALLBACK, this.getClass(), getLogMessagePrefix() + " fallback execution rejected.", null, null)); + } + } + + /** + * @throws HystrixRuntimeException + */ + private Observable getFallbackOrThrowException(HystrixEventType eventType, FailureType failureType, String message) { + return getFallbackOrThrowException(eventType, failureType, message, null); + } + + /** + * @throws HystrixRuntimeException + */ + private Observable getFallbackOrThrowException(final HystrixEventType eventType, final FailureType failureType, final String message, final Exception originalException) { + final HystrixRequestContext currentRequestContext = HystrixRequestContext.getContextForCurrentThread(); + + if (properties.fallbackEnabled().get()) { + /* fallback behavior is permitted so attempt */ + // record the executionResult + // do this before executing fallback so it can be queried from within getFallback (see See https://github.com/Netflix/Hystrix/pull/144) + executionResult = executionResult.addEvents(eventType); + final HystrixObservableCommand _cmd = this; + + return getFallbackWithProtection().map(new Func1() { + + @Override + public R call(R t1) { + return executionHook.onComplete(_cmd, t1); + } + + }).doOnCompleted(new Action0() { + + @Override + public void call() { + // mark fallback on counter + metrics.markFallbackSuccess(); + // record the executionResult + executionResult = executionResult.addEvents(HystrixEventType.FALLBACK_SUCCESS); + } + + }).onErrorResumeNext(new Func1>() { + + @Override + public Observable call(Throwable t) { + Exception e = originalException; + Exception fe = getExceptionFromThrowable(t); + + if (fe instanceof UnsupportedOperationException) { + logger.debug("No fallback for HystrixCommand. ", fe); // debug only since we're throwing the exception and someone higher will do something with it + + /* executionHook for all errors */ + try { + e = executionHook.onError(_cmd, failureType, e); + } catch (Exception hookException) { + logger.warn("Error calling ExecutionHook.onError", hookException); + } + + return Observable.error(new HystrixRuntimeException(failureType, _cmd.getClass(), getLogMessagePrefix() + " " + message + " and no fallback available.", e, fe)); + } else { + logger.debug("HystrixCommand execution " + failureType.name() + " and fallback retrieval failed.", fe); + metrics.markFallbackFailure(); + // record the executionResult + executionResult = executionResult.addEvents(HystrixEventType.FALLBACK_FAILURE); + + /* executionHook for all errors */ + try { + e = executionHook.onError(_cmd, failureType, e); + } catch (Exception hookException) { + logger.warn("Error calling ExecutionHook.onError", hookException); + } + + return Observable.error(new HystrixRuntimeException(failureType, _cmd.getClass(), getLogMessagePrefix() + " " + message + " and failed retrieving fallback.", e, fe)); + } + } + + }).doOnTerminate(new Action0() { + + @Override + public void call() { + // record that we're completed (to handle non-successful events we do it here as well as at the end of executeCommand + isExecutionComplete.set(true); + } + + }).doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + setRequestContextIfNeeded(currentRequestContext); + } + + }); + } else { + /* fallback is disabled so throw HystrixRuntimeException */ + Exception e = originalException; + + logger.debug("Fallback disabled for HystrixCommand so will throw HystrixRuntimeException. ", e); // debug only since we're throwing the exception and someone higher will do something with it + // record the executionResult + executionResult = executionResult.addEvents(eventType); + + /* executionHook for all errors */ + try { + e = executionHook.onError(this, failureType, e); + } catch (Exception hookException) { + logger.warn("Error calling ExecutionHook.onError", hookException); + } + return Observable. error(new HystrixRuntimeException(failureType, this.getClass(), getLogMessagePrefix() + " " + message + " and fallback disabled.", e, null)).doOnTerminate(new Action0() { + + @Override + public void call() { + // record that we're completed (to handle non-successful events we do it here as well as at the end of executeCommand + isExecutionComplete.set(true); + } + + }).doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + setRequestContextIfNeeded(currentRequestContext); + } + + }); + } + } + + private static class HystrixObservableTimeoutOperator implements Operator { + + final HystrixObservableCommand originalCommand; + final boolean isNonBlocking; + + public HystrixObservableTimeoutOperator(final HystrixObservableCommand originalCommand, final boolean isNonBlocking) { + this.originalCommand = originalCommand; + this.isNonBlocking = isNonBlocking; + } + + @Override + public Subscriber call(final Subscriber child) { + final CompositeSubscription s = new CompositeSubscription(); + // if the child unsubscribes we unsubscribe our parent as well + child.add(s); + + /* + * Define the action to perform on timeout outside of the TimerListener to it can capture the HystrixRequestContext + * of the calling thread which doesn't exist on the Timer thread. + */ + final HystrixContextRunnable timeoutRunnable = new HystrixContextRunnable(originalCommand.concurrencyStrategy, new Runnable() { + + @Override + public void run() { + try { + Observable v = originalCommand.getFallbackOrThrowException(HystrixEventType.TIMEOUT, FailureType.TIMEOUT, "timed-out", new TimeoutException()); + /* + * we subscribeOn the computation scheduler as we don't want to use the Timer thread, nor can we use the + * THREAD isolation pool as it may be saturated and that's the reason we're in fallback. The fallback logic + * should not perform IO and thus we run on the computation event loops. + */ + v.subscribeOn(new HystrixContextScheduler(originalCommand.concurrencyStrategy, Schedulers.computation())).subscribe(child); + } catch (HystrixRuntimeException re) { + child.onError(re); + } + } + }); + + TimerListener listener = new TimerListener() { + + @Override + public void tick() { + // if we can go from NOT_EXECUTED to TIMED_OUT then we do the timeout codepath + // otherwise it means we lost a race and the run() execution completed + if (originalCommand.isCommandTimedOut.compareAndSet(TimedOutStatus.NOT_EXECUTED, TimedOutStatus.TIMED_OUT)) { + // do fallback logic + // report timeout failure + originalCommand.metrics.markTimeout(System.currentTimeMillis() - originalCommand.invocationStartTime); + + // we record execution time because we are returning before + originalCommand.recordTotalExecutionTime(originalCommand.invocationStartTime); + + timeoutRunnable.run(); + } + + // shut down the original request + s.unsubscribe(); + } + + @Override + public int getIntervalTimeInMilliseconds() { + return originalCommand.properties.executionIsolationThreadTimeoutInMilliseconds().get(); + } + }; + + Reference _tl = null; + if (isNonBlocking) { + /* + * Scheduling a separate timer to do timeouts is more expensive + * so we'll only do it if we're being used in a non-blocking manner. + */ + _tl = HystrixTimer.getInstance().addTimerListener(listener); + } else { + /* + * Otherwise we just set the hook that queue().get() can trigger if a timeout occurs. + * + * This allows the blocking and non-blocking approaches to be coded basically the same way + * though it is admittedly awkward if we were just blocking (the use of Reference annoys me for example) + */ + _tl = new SoftReference(listener); + } + final Reference tl = _tl; + + // set externally so execute/queue can see this + originalCommand.timeoutTimer.set(tl); + + return new Subscriber(s) { + + @Override + public void onCompleted() { + if (originalCommand.isCommandTimedOut.compareAndSet(TimedOutStatus.NOT_EXECUTED, TimedOutStatus.COMPLETED)) { + tl.clear(); + child.onCompleted(); + } + } + + @Override + public void onError(Throwable e) { + if (originalCommand.isCommandTimedOut.compareAndSet(TimedOutStatus.NOT_EXECUTED, TimedOutStatus.COMPLETED)) { + tl.clear(); + child.onError(e); + } + } + + @Override + public void onNext(R v) { + // TODO does this need to compareAndSet(TimedOutStatus.NOT_EXECUTED, TimedOutStatus.NOT_EXECUTED) + // to be thread-safe, and does that even work? + if (originalCommand.isCommandTimedOut.get().equals(TimedOutStatus.NOT_EXECUTED)) { + child.onNext(v); + } + } + + }; + } + + } + + private static void setRequestContextIfNeeded(final HystrixRequestContext currentRequestContext) { + if (!HystrixRequestContext.isCurrentThreadInitialized()) { + // even if the user Observable doesn't have context we want it set for chained operators + HystrixRequestContext.setContextOnCurrentThread(currentRequestContext); + } + } +} diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java index b2a707f4d..9e170e185 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestCache.java @@ -15,24 +15,14 @@ */ package com.netflix.hystrix; -import static org.junit.Assert.*; - import java.util.concurrent.ConcurrentHashMap; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import rx.Observable; -import rx.Observable.OnSubscribeFunc; -import rx.Observer; -import rx.Subscription; -import rx.subscriptions.Subscriptions; -import rx.util.functions.Func1; import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; -import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategyDefault; -import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableDefault; import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableHolder; import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle; @@ -277,77 +267,4 @@ public boolean equals(Object obj) { } - public static class UnitTest { - - @Test - public void testCache() { - HystrixConcurrencyStrategy strategy = HystrixConcurrencyStrategyDefault.getInstance(); - HystrixRequestContext context = HystrixRequestContext.initializeContext(); - try { - HystrixRequestCache cache1 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command1"), strategy); - cache1.putIfAbsent("valueA", new TestObservable("a1")); - cache1.putIfAbsent("valueA", new TestObservable("a2")); - cache1.putIfAbsent("valueB", new TestObservable("b1")); - - HystrixRequestCache cache2 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command2"), strategy); - cache2.putIfAbsent("valueA", new TestObservable("a3")); - - assertEquals("a1", cache1.get("valueA").toBlockingObservable().last()); - assertEquals("b1", cache1.get("valueB").toBlockingObservable().last()); - - assertEquals("a3", cache2.get("valueA").toBlockingObservable().last()); - assertNull(cache2.get("valueB")); - } catch (Exception e) { - fail("Exception: " + e.getMessage()); - e.printStackTrace(); - } finally { - context.shutdown(); - } - - context = HystrixRequestContext.initializeContext(); - try { - // with a new context the instance should have nothing in it - HystrixRequestCache cache = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command1"), strategy); - assertNull(cache.get("valueA")); - assertNull(cache.get("valueB")); - } finally { - context.shutdown(); - } - } - - @Test - public void testClearCache() { - HystrixConcurrencyStrategy strategy = HystrixConcurrencyStrategyDefault.getInstance(); - HystrixRequestContext context = HystrixRequestContext.initializeContext(); - try { - HystrixRequestCache cache1 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command1"), strategy); - cache1.putIfAbsent("valueA", new TestObservable("a1")); - assertEquals("a1", cache1.get("valueA").toBlockingObservable().last()); - cache1.clear("valueA"); - assertNull(cache1.get("valueA")); - } catch (Exception e) { - fail("Exception: " + e.getMessage()); - e.printStackTrace(); - } finally { - context.shutdown(); - } - } - - private static class TestObservable extends Observable { - public TestObservable(final String value) { - super(new OnSubscribeFunc() { - - @Override - public Subscription onSubscribe(Observer observer) { - observer.onNext(value); - observer.onCompleted(); - return Subscriptions.empty(); - } - - }); - } - } - - } - } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestLog.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestLog.java index ec89add8f..3111074a4 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestLog.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixRequestLog.java @@ -15,8 +15,6 @@ */ package com.netflix.hystrix; -import static org.junit.Assert.*; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -27,7 +25,6 @@ import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,7 +49,7 @@ public class HystrixRequestLog { * Intended to help prevent memory leaks when someone isn't aware of the * HystrixRequestContext lifecycle or enabling/disabling RequestLog. */ - private static final int MAX_STORAGE = 1000; + /* package */static final int MAX_STORAGE = 1000; private static final HystrixRequestVariableHolder currentRequestLog = new HystrixRequestVariableHolder(new HystrixRequestVariableLifecycle() { @Override @@ -71,6 +68,11 @@ public void shutdown(HystrixRequestLog value) { */ private LinkedBlockingQueue> executedCommands = new LinkedBlockingQueue>(MAX_STORAGE); + /** + * History of {@link HystrixExecutableInfo} executed in this request. + */ + private LinkedBlockingQueue> allExecutedCommands = new LinkedBlockingQueue>(MAX_STORAGE); + // prevent public instantiation private HystrixRequestLog() { } @@ -101,21 +103,41 @@ public static HystrixRequestLog getCurrentRequest() { * * @return {@code Collection>} */ + @Deprecated public Collection> getExecutedCommands() { return Collections.unmodifiableCollection(executedCommands); } + /** + * Retrieve {@link HystrixCommand} instances that were executed during this {@link HystrixRequestContext}. + * + * @return {@code Collection>} + */ + public Collection> getAllExecutedCommands() { + return Collections.unmodifiableCollection(allExecutedCommands); + } + /** * Add {@link HystrixCommand} instance to the request log. * * @param command * {@code HystrixCommand} */ - /* package */void addExecutedCommand(HystrixCommand command) { - if (!executedCommands.offer(command)) { + /* package */void addExecutedCommand(HystrixExecutableInfo command) { + if (!allExecutedCommands.offer(command)) { // see RequestLog: Reduce Chance of Memory Leak https://github.com/Netflix/Hystrix/issues/53 logger.warn("RequestLog ignoring command after reaching limit of " + MAX_STORAGE + ". See https://github.com/Netflix/Hystrix/issues/53 for more information."); } + + // TODO remove this when deprecation completed + if (command instanceof HystrixCommand.HystrixCommandFromObservableCommand) { + @SuppressWarnings("rawtypes") + HystrixCommand _c = ((HystrixCommand.HystrixCommandFromObservableCommand) command).getOriginal(); + if (!executedCommands.offer(_c)) { + // see RequestLog: Reduce Chance of Memory Leak https://github.com/Netflix/Hystrix/issues/53 + logger.warn("RequestLog ignoring command after reaching limit of " + MAX_STORAGE + ". See https://github.com/Netflix/Hystrix/issues/53 for more information."); + } + } } /** @@ -146,7 +168,7 @@ public String getExecutedCommandsAsString() { LinkedHashMap aggregatedCommandsExecuted = new LinkedHashMap(); Map aggregatedCommandExecutionTime = new HashMap(); - for (HystrixCommand command : executedCommands) { + for (HystrixExecutableInfo command : allExecutedCommands) { StringBuilder displayString = new StringBuilder(); displayString.append(command.getCommandKey().name()); @@ -205,185 +227,4 @@ public String getExecutedCommandsAsString() { } } - public static class UnitTest { - - private static final String DIGITS_REGEX = "\\[\\d+"; - - @Test - public void testSuccess() { - HystrixRequestContext context = HystrixRequestContext.initializeContext(); - try { - new TestCommand("A", false, true).execute(); - String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); - // strip the actual count so we can compare reliably - log = log.replaceAll(DIGITS_REGEX, "["); - assertEquals("TestCommand[SUCCESS][ms]", log); - } finally { - context.shutdown(); - } - } - - @Test - public void testSuccessFromCache() { - HystrixRequestContext context = HystrixRequestContext.initializeContext(); - try { - // 1 success - new TestCommand("A", false, true).execute(); - // 4 success from cache - new TestCommand("A", false, true).execute(); - new TestCommand("A", false, true).execute(); - new TestCommand("A", false, true).execute(); - new TestCommand("A", false, true).execute(); - String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); - // strip the actual count so we can compare reliably - log = log.replaceAll(DIGITS_REGEX, "["); - assertEquals("TestCommand[SUCCESS][ms], TestCommand[SUCCESS, RESPONSE_FROM_CACHE][ms]x4", log); - } finally { - context.shutdown(); - } - } - - @Test - public void testFailWithFallbackSuccess() { - HystrixRequestContext context = HystrixRequestContext.initializeContext(); - try { - // 1 failure - new TestCommand("A", true, false).execute(); - // 4 failures from cache - new TestCommand("A", true, false).execute(); - new TestCommand("A", true, false).execute(); - new TestCommand("A", true, false).execute(); - new TestCommand("A", true, false).execute(); - String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); - // strip the actual count so we can compare reliably - log = log.replaceAll(DIGITS_REGEX, "["); - assertEquals("TestCommand[FAILURE, FALLBACK_SUCCESS][ms], TestCommand[FAILURE, FALLBACK_SUCCESS, RESPONSE_FROM_CACHE][ms]x4", log); - } finally { - context.shutdown(); - } - } - - @Test - public void testFailWithFallbackFailure() { - HystrixRequestContext context = HystrixRequestContext.initializeContext(); - try { - // 1 failure - try { - new TestCommand("A", true, true).execute(); - } catch (Exception e) { - } - // 1 failure from cache - try { - new TestCommand("A", true, true).execute(); - } catch (Exception e) { - } - String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); - // strip the actual count so we can compare reliably - log = log.replaceAll(DIGITS_REGEX, "["); - assertEquals("TestCommand[FAILURE, FALLBACK_FAILURE][ms], TestCommand[FAILURE, FALLBACK_FAILURE, RESPONSE_FROM_CACHE][ms]", log); - } finally { - context.shutdown(); - } - } - - @Test - public void testMultipleCommands() { - - HystrixRequestContext context = HystrixRequestContext.initializeContext(); - try { - - // 1 success - new TestCommand("GetData", "A", false, false).execute(); - - // 1 success - new TestCommand("PutData", "B", false, false).execute(); - - // 1 success - new TestCommand("GetValues", "C", false, false).execute(); - - // 1 success from cache - new TestCommand("GetValues", "C", false, false).execute(); - - // 1 failure - try { - new TestCommand("A", true, true).execute(); - } catch (Exception e) { - } - // 1 failure from cache - try { - new TestCommand("A", true, true).execute(); - } catch (Exception e) { - } - String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); - // strip the actual count so we can compare reliably - log = log.replaceAll(DIGITS_REGEX, "["); - assertEquals("GetData[SUCCESS][ms], PutData[SUCCESS][ms], GetValues[SUCCESS][ms], GetValues[SUCCESS, RESPONSE_FROM_CACHE][ms], TestCommand[FAILURE, FALLBACK_FAILURE][ms], TestCommand[FAILURE, FALLBACK_FAILURE, RESPONSE_FROM_CACHE][ms]", log); - } finally { - context.shutdown(); - } - - } - - @Test - public void testMaxLimit() { - HystrixRequestContext context = HystrixRequestContext.initializeContext(); - try { - for (int i = 0; i < MAX_STORAGE; i++) { - new TestCommand("A", false, true).execute(); - } - // then execute again some more - for (int i = 0; i < 10; i++) { - new TestCommand("A", false, true).execute(); - } - - assertEquals(MAX_STORAGE, HystrixRequestLog.getCurrentRequest().executedCommands.size()); - } finally { - context.shutdown(); - } - } - } - - private static class TestCommand extends HystrixCommand { - - private final String value; - private final boolean fail; - private final boolean failOnFallback; - - public TestCommand(String commandName, String value, boolean fail, boolean failOnFallback) { - super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("RequestLogTestCommand")).andCommandKey(HystrixCommandKey.Factory.asKey(commandName))); - this.value = value; - this.fail = fail; - this.failOnFallback = failOnFallback; - } - - public TestCommand(String value, boolean fail, boolean failOnFallback) { - super(HystrixCommandGroupKey.Factory.asKey("RequestLogTestCommand")); - this.value = value; - this.fail = fail; - this.failOnFallback = failOnFallback; - } - - @Override - protected String run() { - if (fail) { - throw new RuntimeException("forced failure"); - } else { - return value; - } - } - - @Override - protected String getFallback() { - if (failOnFallback) { - throw new RuntimeException("forced fallback failure"); - } else { - return value + "-fallback"; - } - } - - @Override - protected String getCacheKey() { - return value; - } - } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixThreadPool.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixThreadPool.java index d040e87a6..62643b259 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixThreadPool.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixThreadPool.java @@ -22,13 +22,13 @@ import javax.annotation.concurrent.ThreadSafe; +import rx.Scheduler; + import com.netflix.hystrix.strategy.HystrixPlugins; import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; +import com.netflix.hystrix.strategy.concurrency.HystrixContextScheduler; import com.netflix.hystrix.strategy.metrics.HystrixMetricsPublisherFactory; import com.netflix.hystrix.strategy.properties.HystrixPropertiesFactory; -import org.junit.Test; - -import static org.junit.Assert.*; /** * ThreadPool used to executed {@link HystrixCommand#run()} on separate threads when configured to do so with {@link HystrixCommandProperties#executionIsolationStrategy()}. @@ -52,6 +52,8 @@ public interface HystrixThreadPool { */ public ThreadPoolExecutor getExecutor(); + public Scheduler getScheduler(); + /** * Mark when a thread begins executing a command. */ @@ -80,7 +82,7 @@ public interface HystrixThreadPool { * Use the String from HystrixThreadPoolKey.name() instead of the HystrixThreadPoolKey instance as it's just an interface and we can't ensure the object * we receive implements hashcode/equals correctly and do not want the default hashcode/equals which would create a new threadpool for every object we get even if the name is the same */ - private static ConcurrentHashMap threadPools = new ConcurrentHashMap(); + /* package */final static ConcurrentHashMap threadPools = new ConcurrentHashMap(); /** * Get the {@link HystrixThreadPool} instance for a given {@link HystrixThreadPoolKey}. @@ -124,7 +126,7 @@ public interface HystrixThreadPool { * and causing thread-pools to initialize while also trying to shutdown. *

*/ - /* package */ static synchronized void shutdown() { + /* package */static synchronized void shutdown() { for (HystrixThreadPool pool : threadPools.values()) { pool.getExecutor().shutdown(); } @@ -138,7 +140,7 @@ public interface HystrixThreadPool { * and causing thread-pools to initialize while also trying to shutdown. *

*/ - /* package */ static synchronized void shutdown(long timeout, TimeUnit unit) { + /* package */static synchronized void shutdown(long timeout, TimeUnit unit) { for (HystrixThreadPool pool : threadPools.values()) { pool.getExecutor().shutdown(); } @@ -157,11 +159,12 @@ public interface HystrixThreadPool { * @ExcludeFromJavadoc */ @ThreadSafe - /* package */ static class HystrixThreadPoolDefault implements HystrixThreadPool { + /* package */static class HystrixThreadPoolDefault implements HystrixThreadPool { private final HystrixThreadPoolProperties properties; private final BlockingQueue queue; private final ThreadPoolExecutor threadPool; private final HystrixThreadPoolMetrics metrics; + private final Scheduler scheduler; public HystrixThreadPoolDefault(HystrixThreadPoolKey threadPoolKey, HystrixThreadPoolProperties.Setter propertiesDefaults) { this.properties = HystrixPropertiesFactory.getThreadPoolProperties(threadPoolKey, propertiesDefaults); @@ -169,6 +172,7 @@ public HystrixThreadPoolDefault(HystrixThreadPoolKey threadPoolKey, HystrixThrea this.queue = concurrencyStrategy.getBlockingQueue(properties.maxQueueSize().get()); this.threadPool = concurrencyStrategy.getThreadPool(threadPoolKey, properties.coreSize(), properties.coreSize(), properties.keepAliveTimeMinutes(), TimeUnit.MINUTES, queue); this.metrics = HystrixThreadPoolMetrics.getInstance(threadPoolKey, threadPool, properties); + this.scheduler = new HystrixContextScheduler(concurrencyStrategy, this); /* strategy: HystrixMetricsPublisherThreadPool */ HystrixMetricsPublisherFactory.createOrRetrievePublisherForThreadPool(threadPoolKey, this.metrics, this.properties); @@ -176,12 +180,21 @@ public HystrixThreadPoolDefault(HystrixThreadPoolKey threadPoolKey, HystrixThrea @Override public ThreadPoolExecutor getExecutor() { - // allow us to change things via fast-properties by setting it each time + touchConfig(); + return threadPool; + } + + @Override + public Scheduler getScheduler() { + touchConfig(); + return scheduler; + } + + // allow us to change things via fast-properties by setting it each time + private void touchConfig() { threadPool.setCorePoolSize(properties.coreSize().get()); threadPool.setMaximumPoolSize(properties.coreSize().get()); // we always want maxSize the same as coreSize, we are not using a dynamically resizing pool threadPool.setKeepAliveTime(properties.keepAliveTimeMinutes().get(), TimeUnit.MINUTES); // this doesn't really matter since we're not resizing - - return threadPool; } @Override @@ -212,43 +225,4 @@ public boolean isQueueSpaceAvailable() { } - public static class UnitTest { - - @Test - public void testShutdown() { - // other unit tests will probably have run before this so get the count - int count = Factory.threadPools.size(); - - HystrixThreadPool pool = Factory.getInstance(HystrixThreadPoolKey.Factory.asKey("threadPoolFactoryTest"), - HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder()); - - assertEquals(count + 1, Factory.threadPools.size()); - assertFalse(pool.getExecutor().isShutdown()); - - Factory.shutdown(); - - // ensure all pools were removed from the cache - assertEquals(0, Factory.threadPools.size()); - assertTrue(pool.getExecutor().isShutdown()); - } - - @Test - public void testShutdownWithWait() { - // other unit tests will probably have run before this so get the count - int count = Factory.threadPools.size(); - - HystrixThreadPool pool = Factory.getInstance(HystrixThreadPoolKey.Factory.asKey("threadPoolFactoryTest"), - HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder()); - - assertEquals(count + 1, Factory.threadPools.size()); - assertFalse(pool.getExecutor().isShutdown()); - - Factory.shutdown(1, TimeUnit.SECONDS); - - // ensure all pools were removed from the cache - assertEquals(0, Factory.threadPools.size()); - assertTrue(pool.getExecutor().isShutdown()); - } - } - } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunction.java b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunction.java index 91f9633f9..b5c5a8bfe 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunction.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunction.java @@ -1,19 +1,11 @@ package com.netflix.hystrix.collapser; -import static org.junit.Assert.*; - -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicReference; -import org.junit.Test; - -import rx.Observable; -import rx.Observable.OnSubscribeFunc; +import rx.Observable.OnSubscribe; import rx.Observer; -import rx.Subscription; +import rx.Subscriber; import rx.subscriptions.BooleanSubscription; -import rx.util.functions.Func1; import com.netflix.hystrix.HystrixCollapser.CollapsedRequest; @@ -28,7 +20,7 @@ * * @param */ -/* package */class CollapsedRequestObservableFunction implements CollapsedRequest, OnSubscribeFunc { +/* package */class CollapsedRequestObservableFunction implements CollapsedRequest, OnSubscribe { private final R argument; private final AtomicReference> rh = new AtomicReference>(new CollapsedRequestObservableFunction.ResponseHolder()); private final BooleanSubscription subscription = new BooleanSubscription(); @@ -57,9 +49,6 @@ public R getArgument() { @Override public void setResponse(T response) { while (true) { - if (subscription.isUnsubscribed()) { - return; - } ResponseHolder r = rh.get(); if (r.isResponseSet()) { throw new IllegalStateException("setResponse can only be called once"); @@ -68,6 +57,9 @@ public void setResponse(T response) { throw new IllegalStateException("Exception is already set so response can not be => Response: " + response + " subscription: " + subscription.isUnsubscribed() + " observer: " + r.getObserver() + " Exception: " + r.getException().getMessage(), r.getException()); } + if (subscription.isUnsubscribed()) { + return; + } ResponseHolder nr = r.setResponse(response); if (rh.compareAndSet(r, nr)) { // success @@ -117,9 +109,6 @@ public void setExceptionIfResponseNotReceived(Exception e) { @Override public void setException(Exception e) { while (true) { - if (subscription.isUnsubscribed()) { - return; - } CollapsedRequestObservableFunction.ResponseHolder r = rh.get(); if (r.getException() != null) { throw new IllegalStateException("setException can only be called once"); @@ -128,6 +117,9 @@ public void setException(Exception e) { throw new IllegalStateException("Response is already set so exception can not be => Response: " + r.getResponse() + " Exception: " + e.getMessage(), e); } + if (subscription.isUnsubscribed()) { + return; + } ResponseHolder nr = r.setException(e); if (rh.compareAndSet(r, nr)) { // success @@ -140,7 +132,8 @@ public void setException(Exception e) { } @Override - public Subscription onSubscribe(Observer observer) { + public void call(Subscriber observer) { + observer.add(subscription); while (true) { CollapsedRequestObservableFunction.ResponseHolder r = rh.get(); if (r.getObserver() != null) { @@ -155,7 +148,6 @@ public Subscription onSubscribe(Observer observer) { // we'll retry } } - return subscription; } private static void sendResponseIfRequired(BooleanSubscription subscription, CollapsedRequestObservableFunction.ResponseHolder r) { @@ -230,161 +222,4 @@ public Exception getException() { } - public static class UnitTest { - - @Test - public void testSetResponseSuccess() throws InterruptedException, ExecutionException { - CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); - Observable o = Observable.create(cr); - Future v = o.toBlockingObservable().toFuture(); - - cr.setResponse("theResponse"); - - // fetch value - assertEquals("theResponse", v.get()); - } - - @Test - public void testSetNullResponseSuccess() throws InterruptedException, ExecutionException { - CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); - Observable o = Observable.create(cr); - Future v = o.toBlockingObservable().toFuture(); - - cr.setResponse(null); - - // fetch value - assertEquals(null, v.get()); - } - - @Test - public void testSetException() throws InterruptedException, ExecutionException { - CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); - Observable o = Observable.create(cr); - Future v = o.toBlockingObservable().toFuture(); - - cr.setException(new RuntimeException("anException")); - - // fetch value - try { - v.get(); - fail("expected exception"); - } catch (ExecutionException e) { - assertEquals("anException", e.getCause().getMessage()); - } - } - - @Test - public void testSetExceptionAfterResponse() throws InterruptedException, ExecutionException { - CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); - Observable o = Observable.create(cr); - Future v = o.toBlockingObservable().toFuture(); - - cr.setResponse("theResponse"); - - try { - cr.setException(new RuntimeException("anException")); - fail("expected IllegalState"); - } catch (IllegalStateException e) { - - } - - assertEquals("theResponse", v.get()); - } - - @Test - public void testSetResponseAfterException() throws InterruptedException, ExecutionException { - CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); - Observable o = Observable.create(cr); - Future v = o.toBlockingObservable().toFuture(); - - cr.setException(new RuntimeException("anException")); - - try { - cr.setResponse("theResponse"); - fail("expected IllegalState"); - } catch (IllegalStateException e) { - - } - - try { - v.get(); - fail("expected exception"); - } catch (ExecutionException e) { - assertEquals("anException", e.getCause().getMessage()); - } - } - - @Test - public void testSetResponseDuplicate() throws InterruptedException, ExecutionException { - CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); - Observable o = Observable.create(cr); - Future v = o.toBlockingObservable().toFuture(); - - cr.setResponse("theResponse"); - - try { - cr.setResponse("theResponse2"); - fail("expected IllegalState"); - } catch (IllegalStateException e) { - - } - - assertEquals("theResponse", v.get()); - } - - @Test - public void testSetResponseAfterUnsubscribe() throws InterruptedException, ExecutionException { - CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); - Observable o = Observable.create(cr); - Future f = o.toBlockingObservable().toFuture(); - - // cancel/unsubscribe - f.cancel(true); - - try { - cr.setResponse("theResponse"); - } catch (IllegalStateException e) { - fail("this should have done nothing as it was unsubscribed already"); - } - - // if you fetch after canceling it should be null - assertEquals(null, f.get()); - } - - @Test - public void testSetExceptionAfterUnsubscribe() throws InterruptedException, ExecutionException { - CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); - Observable o = Observable.create(cr); - Future f = o.toBlockingObservable().toFuture(); - - // cancel/unsubscribe - f.cancel(true); - - try { - cr.setException(new RuntimeException("anException")); - } catch (IllegalStateException e) { - fail("this should have done nothing as it was unsubscribed already"); - } - - // if you fetch after canceling it should be null - assertEquals(null, f.get()); - } - - @Test - public void testUnsubscribeAfterSetResponse() throws InterruptedException, ExecutionException { - CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); - Observable o = Observable.create(cr); - Future v = o.toBlockingObservable().toFuture(); - - cr.setResponse("theResponse"); - - // unsubscribe after the value is sent - v.cancel(true); - - // still get value as it was set before canceling - assertEquals("theResponse", v.get()); - } - - } - } \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestBatch.java b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestBatch.java index bab9c68f1..84918bdd6 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestBatch.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/collapser/RequestBatch.java @@ -105,7 +105,8 @@ public void executeBatchIfNotAlreadyStarted() { try { // create a new command to handle this batch of requests Observable o = commandCollapser.createObservableCommand(shardRequests); - o.subscribe(new RequestBatch.BatchRequestObserver(commandCollapser, shardRequests)); + // if more than one item is emitted we fail + o.single().subscribe(new RequestBatch.BatchRequestObserver(commandCollapser, shardRequests)); } catch (Exception e) { logger.error("Exception while creating and queueing command with batch.", e); // if a failure occurs we want to pass that exception to all of the Futures that we've returned @@ -179,34 +180,17 @@ private BatchRequestObserver(HystrixCollapserBridge request : requests) { - try { - request.setException(e); - } catch (IllegalStateException e2) { - logger.debug("Failed trying to setException on CollapsedRequest", e2); - } - } - } - - @Override - public void onNext(BatchReturnType response) { + public void onCompleted() { try { + // use a boolean since null can be a real response + if (!receivedResponse) { + onError(new IllegalStateException("Did not receive batch response.")); + return; + } commandCollapser.mapResponseToRequests(response, requests); } catch (Throwable e) { // handle Throwable in case anything is thrown so we don't block Observers waiting for onError/onCompleted @@ -240,6 +224,34 @@ public void onNext(BatchReturnType response) { } } } + + @Override + public void onError(Throwable t) { + Exception e = null; + if (t instanceof Exception) { + e = (Exception) t; + } else { + // Hystrix 1.x uses Exception, not Throwable so to prevent a breaking change Throwable will be wrapped in Exception + e = new Exception("Throwable caught while executing command with batch.", t); + } + logger.error("Exception while executing command with batch.", e); + // if a failure occurs we want to pass that exception to all of the Futures that we've returned + for (CollapsedRequest request : requests) { + try { + request.setException(e); + } catch (IllegalStateException e2) { + logger.debug("Failed trying to setException on CollapsedRequest", e2); + } + } + } + + @Override + public void onNext(BatchReturnType response) { + receivedResponse = true; + this.response = response; + // we want to wait until onComplete to do the processing + // so we don't release the callers before metrics/logs/etc are available + } } } \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/exception/HystrixRuntimeException.java b/hystrix-core/src/main/java/com/netflix/hystrix/exception/HystrixRuntimeException.java index 64caaade6..0792dbd45 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/exception/HystrixRuntimeException.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/exception/HystrixRuntimeException.java @@ -16,6 +16,7 @@ package com.netflix.hystrix.exception; import com.netflix.hystrix.HystrixCommand; +import com.netflix.hystrix.HystrixExecutable; import com.netflix.hystrix.util.ExceptionThreadingUtility; /** @@ -26,7 +27,7 @@ public class HystrixRuntimeException extends RuntimeException { private static final long serialVersionUID = 5219160375476046229L; - private final Class commandClass; + private final Class commandClass; private final Throwable fallbackException; private final FailureType failureCause; @@ -34,7 +35,7 @@ public static enum FailureType { COMMAND_EXCEPTION, TIMEOUT, SHORTCIRCUIT, REJECTED_THREAD_EXECUTION, REJECTED_SEMAPHORE_EXECUTION, REJECTED_SEMAPHORE_FALLBACK } - public HystrixRuntimeException(FailureType failureCause, Class commandClass, String message, Exception cause, Throwable fallbackException) { + public HystrixRuntimeException(FailureType failureCause, Class commandClass, String message, Exception cause, Throwable fallbackException) { super(message, cause); this.failureCause = failureCause; this.commandClass = commandClass; @@ -42,7 +43,7 @@ public HystrixRuntimeException(FailureType failureCause, Class commandClass, String message, Throwable cause, Throwable fallbackException) { + public HystrixRuntimeException(FailureType failureCause, Class commandClass, String message, Throwable cause, Throwable fallbackException) { super(message, cause); this.failureCause = failureCause; this.commandClass = commandClass; @@ -64,7 +65,7 @@ public FailureType getFailureType() { * * @return {@code Class } */ - public Class getImplementingClass() { + public Class getImplementingClass() { return commandClass; } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/HystrixPlugins.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/HystrixPlugins.java index 4e9241f4c..4dd583db3 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/HystrixPlugins.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/HystrixPlugins.java @@ -15,22 +15,10 @@ */ package com.netflix.hystrix.strategy; -import static org.junit.Assert.*; - -import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicReference; -import org.junit.After; -import org.junit.Test; - -import rx.util.functions.Action1; - -import com.netflix.hystrix.Hystrix; -import com.netflix.hystrix.HystrixCommand; -import com.netflix.hystrix.HystrixCommandGroupKey; import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategyDefault; -import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifier; import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifierDefault; import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook; @@ -53,11 +41,11 @@ public class HystrixPlugins { private final static HystrixPlugins INSTANCE = new HystrixPlugins(); - private final AtomicReference notifier = new AtomicReference(); - private final AtomicReference concurrencyStrategy = new AtomicReference(); - private final AtomicReference metricsPublisher = new AtomicReference(); - private final AtomicReference propertiesFactory = new AtomicReference(); - private final AtomicReference commandExecutionHook = new AtomicReference(); + /* package */ final AtomicReference notifier = new AtomicReference(); + /* package */ final AtomicReference concurrencyStrategy = new AtomicReference(); + /* package */ final AtomicReference metricsPublisher = new AtomicReference(); + /* package */ final AtomicReference propertiesFactory = new AtomicReference(); + /* package */ final AtomicReference commandExecutionHook = new AtomicReference(); private HystrixPlugins() { @@ -291,210 +279,4 @@ private static Object getPluginImplementationViaProperty(Class pluginClass) { } } - public static class UnitTest { - - @After - public void reset() { - // use private access to reset so we can test different initializations via the public static flow - HystrixPlugins.getInstance().concurrencyStrategy.set(null); - HystrixPlugins.getInstance().metricsPublisher.set(null); - HystrixPlugins.getInstance().notifier.set(null); - HystrixPlugins.getInstance().propertiesFactory.set(null); - } - - @Test - public void testEventNotifierDefaultImpl() { - HystrixEventNotifier impl = HystrixPlugins.getInstance().getEventNotifier(); - assertTrue(impl instanceof HystrixEventNotifierDefault); - } - - @Test - public void testEventNotifierViaRegisterMethod() { - HystrixPlugins.getInstance().registerEventNotifier(new HystrixEventNotifierTestImpl()); - HystrixEventNotifier impl = HystrixPlugins.getInstance().getEventNotifier(); - assertTrue(impl instanceof HystrixEventNotifierTestImpl); - } - - @Test - public void testEventNotifierViaProperty() { - try { - String fullClass = getFullClassNameForTestClass(HystrixEventNotifierTestImpl.class); - System.setProperty("hystrix.plugin.HystrixEventNotifier.implementation", fullClass); - HystrixEventNotifier impl = HystrixPlugins.getInstance().getEventNotifier(); - assertTrue(impl instanceof HystrixEventNotifierTestImpl); - } finally { - System.clearProperty("hystrix.plugin.HystrixEventNotifier.implementation"); - } - } - - // inside UnitTest so it is stripped from Javadocs - public static class HystrixEventNotifierTestImpl extends HystrixEventNotifier { - // just use defaults - } - - @Test - public void testConcurrencyStrategyDefaultImpl() { - HystrixConcurrencyStrategy impl = HystrixPlugins.getInstance().getConcurrencyStrategy(); - assertTrue(impl instanceof HystrixConcurrencyStrategyDefault); - } - - @Test - public void testConcurrencyStrategyViaRegisterMethod() { - HystrixPlugins.getInstance().registerConcurrencyStrategy(new HystrixConcurrencyStrategyTestImpl()); - HystrixConcurrencyStrategy impl = HystrixPlugins.getInstance().getConcurrencyStrategy(); - assertTrue(impl instanceof HystrixConcurrencyStrategyTestImpl); - } - - @Test - public void testConcurrencyStrategyViaProperty() { - try { - String fullClass = getFullClassNameForTestClass(HystrixConcurrencyStrategyTestImpl.class); - System.setProperty("hystrix.plugin.HystrixConcurrencyStrategy.implementation", fullClass); - HystrixConcurrencyStrategy impl = HystrixPlugins.getInstance().getConcurrencyStrategy(); - assertTrue(impl instanceof HystrixConcurrencyStrategyTestImpl); - } finally { - System.clearProperty("hystrix.plugin.HystrixConcurrencyStrategy.implementation"); - } - } - - // inside UnitTest so it is stripped from Javadocs - public static class HystrixConcurrencyStrategyTestImpl extends HystrixConcurrencyStrategy { - // just use defaults - } - - @Test - public void testMetricsPublisherDefaultImpl() { - HystrixMetricsPublisher impl = HystrixPlugins.getInstance().getMetricsPublisher(); - assertTrue(impl instanceof HystrixMetricsPublisherDefault); - } - - @Test - public void testMetricsPublisherViaRegisterMethod() { - HystrixPlugins.getInstance().registerMetricsPublisher(new HystrixMetricsPublisherTestImpl()); - HystrixMetricsPublisher impl = HystrixPlugins.getInstance().getMetricsPublisher(); - assertTrue(impl instanceof HystrixMetricsPublisherTestImpl); - } - - @Test - public void testMetricsPublisherViaProperty() { - try { - String fullClass = getFullClassNameForTestClass(HystrixMetricsPublisherTestImpl.class); - System.setProperty("hystrix.plugin.HystrixMetricsPublisher.implementation", fullClass); - HystrixMetricsPublisher impl = HystrixPlugins.getInstance().getMetricsPublisher(); - assertTrue(impl instanceof HystrixMetricsPublisherTestImpl); - } finally { - System.clearProperty("hystrix.plugin.HystrixMetricsPublisher.implementation"); - } - } - - // inside UnitTest so it is stripped from Javadocs - public static class HystrixMetricsPublisherTestImpl extends HystrixMetricsPublisher { - // just use defaults - } - - @Test - public void testPropertiesStrategyDefaultImpl() { - HystrixPropertiesStrategy impl = HystrixPlugins.getInstance().getPropertiesStrategy(); - assertTrue(impl instanceof HystrixPropertiesStrategyDefault); - } - - @Test - public void testPropertiesStrategyViaRegisterMethod() { - HystrixPlugins.getInstance().registerPropertiesStrategy(new HystrixPropertiesStrategyTestImpl()); - HystrixPropertiesStrategy impl = HystrixPlugins.getInstance().getPropertiesStrategy(); - assertTrue(impl instanceof HystrixPropertiesStrategyTestImpl); - } - - @Test - public void testPropertiesStrategyViaProperty() { - try { - String fullClass = getFullClassNameForTestClass(HystrixPropertiesStrategyTestImpl.class); - System.setProperty("hystrix.plugin.HystrixPropertiesStrategy.implementation", fullClass); - HystrixPropertiesStrategy impl = HystrixPlugins.getInstance().getPropertiesStrategy(); - assertTrue(impl instanceof HystrixPropertiesStrategyTestImpl); - } finally { - System.clearProperty("hystrix.plugin.HystrixPropertiesStrategy.implementation"); - } - } - - // inside UnitTest so it is stripped from Javadocs - public static class HystrixPropertiesStrategyTestImpl extends HystrixPropertiesStrategy { - // just use defaults - } - - private static String getFullClassNameForTestClass(Class cls) { - return HystrixPlugins.class.getPackage().getName() + "." + HystrixPlugins.class.getSimpleName() + "$UnitTest$" + cls.getSimpleName(); - } - - - private static final ThreadLocal testRequestIdThreadLocal = new ThreadLocal(); - - public static class DummyCommand extends HystrixCommand { - - public DummyCommand() { - super(HystrixCommandGroupKey.Factory.asKey("Dummy")); - } - - @Override - protected Void run() throws Exception { - System.out.println("requestId (run) = " + testRequestIdThreadLocal.get()); - Thread.sleep(2000); - return null; - } - } - - @Test - public void testRequestContextViaPluginInTimeout() { - HystrixPlugins.getInstance().registerConcurrencyStrategy(new HystrixConcurrencyStrategy() { - @Override - public Callable wrapCallable(final Callable callable) { - return new RequestIdCallable(callable); - } - }); - - HystrixRequestContext context = HystrixRequestContext.initializeContext(); - - testRequestIdThreadLocal.set("foobar"); - final AtomicReference valueInTimeout = new AtomicReference(); - - new DummyCommand().toObservable() - .doOnError(new Action1() { - @Override - public void call(Throwable throwable) { - System.out.println("initialized = " + HystrixRequestContext.isCurrentThreadInitialized()); - System.out.println("requestId (timeout) = " + testRequestIdThreadLocal.get()); - valueInTimeout.set(testRequestIdThreadLocal.get()); - } - }) - .materialize() - .toBlockingObservable().single(); - - context.shutdown(); - Hystrix.reset(); - - assertEquals("foobar", valueInTimeout.get()); - } - - private static class RequestIdCallable implements Callable { - private final Callable callable; - private final String requestId; - - public RequestIdCallable(Callable callable) { - this.callable = callable; - this.requestId = testRequestIdThreadLocal.get(); - } - - @Override - public T call() throws Exception { - String original = testRequestIdThreadLocal.get(); - testRequestIdThreadLocal.set(requestId); - try { - return callable.call(); - } finally { - testRequestIdThreadLocal.set(original); - } - } - } - } - } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixConcurrencyStrategy.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixConcurrencyStrategy.java index ee8b3d175..322c17a52 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixConcurrencyStrategy.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixConcurrencyStrategy.java @@ -24,18 +24,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import rx.util.functions.Action1; -import rx.util.functions.Func1; - -import com.netflix.config.ConfigurationManager; -import com.netflix.hystrix.Hystrix; import com.netflix.hystrix.HystrixCommand; -import com.netflix.hystrix.HystrixCommandGroupKey; -import com.netflix.hystrix.HystrixRequestLog; import com.netflix.hystrix.HystrixThreadPool; import com.netflix.hystrix.HystrixThreadPoolKey; import com.netflix.hystrix.strategy.HystrixPlugins; @@ -162,64 +151,4 @@ public void shutdown(T value) { }; } - - public static class UnitTest { - - @Before - public void prepareForTest() { - /* we must call this to simulate a new request lifecycle running and clearing caches */ - HystrixRequestContext.initializeContext(); - } - - @After - public void cleanup() { - // instead of storing the reference from initialize we'll just get the current state and shutdown - if (HystrixRequestContext.getContextForCurrentThread() != null) { - // it could have been set NULL by the test - HystrixRequestContext.getContextForCurrentThread().shutdown(); - } - - // force properties to be clean as well - ConfigurationManager.getConfigInstance().clear(); - } - - /** - * If the RequestContext does not get transferred across threads correctly this blows up. - * No specific assertions are necessary. - */ - @Test - public void testRequestContextPropagatesAcrossObserveOnPool() { - new SimpleCommand().execute(); - new SimpleCommand().observe().map(new Func1() { - - @Override - public String call(String s) { - System.out.println("Map => Commands: " + HystrixRequestLog.getCurrentRequest().getExecutedCommands()); - return s; - } - }).toBlockingObservable().forEach(new Action1() { - - @Override - public void call(String s) { - System.out.println("Result [" + s + "] => Commands: " + HystrixRequestLog.getCurrentRequest().getExecutedCommands()); - } - }); - } - - private static class SimpleCommand extends HystrixCommand { - - public SimpleCommand() { - super(HystrixCommandGroupKey.Factory.asKey("SimpleCommand")); - } - - @Override - protected String run() throws Exception { - System.out.println("Executing => Commands: " + HystrixRequestLog.getCurrentRequest().getExecutedCommands()); - return "Hello"; - } - - } - - } - } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextFunc2.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContexSchedulerAction.java similarity index 53% rename from hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextFunc2.java rename to hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContexSchedulerAction.java index a3ebc67e3..298304dc6 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextFunc2.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContexSchedulerAction.java @@ -18,11 +18,11 @@ import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicReference; -import rx.Scheduler; -import rx.Subscription; -import rx.util.functions.Func2; +import rx.Scheduler.Inner; +import rx.functions.Action1; +import rx.functions.Func2; -import com.netflix.hystrix.strategy.HystrixPlugins; +import com.netflix.hystrix.strategy.concurrency.HystrixContextScheduler.HystrixContextInnerScheduler; /** * Wrapper around {@link Func2} that manages the {@link HystrixRequestContext} initialization and cleanup for the execution of the {@link Func2} @@ -32,45 +32,40 @@ * * @ExcludeFromJavadoc */ -public class HystrixContextFunc2 implements Func2 { +public class HystrixContexSchedulerAction implements Action1 { - private final Func2 actual; + private final Action1 actual; private final HystrixRequestContext parentThreadState; - private final Callable c; + private final Callable c; /* - * This is a workaround to needing to use Callable but - * needing to pass `Scheduler t1, T t2` into it after construction. + * This is a workaround to needing to use Callable but + * needing to pass `Inner t1` into it after construction. * - * Think of it like sticking t1 and t2 on the stack and then calling the function + * Think of it like sticking t1 on the stack and then calling the function * that uses them. * * This should all be thread-safe without issues despite multi-step execution - * because this Func2 is only ever executed once by Hystrix and construction will always - * precede `call` being invoked once. - * + * because this Action0 is only ever executed once by Hystrix and construction will always + * precede `call` being invoked once. */ - private final AtomicReference t1Holder = new AtomicReference(); - private final AtomicReference t2Holder = new AtomicReference(); + private final AtomicReference t1Holder = new AtomicReference(); - public HystrixContextFunc2(Func2 action) { - this(HystrixPlugins.getInstance().getConcurrencyStrategy(), action); - } - - public HystrixContextFunc2(final HystrixConcurrencyStrategy concurrencyStrategy, Func2 action) { + public HystrixContexSchedulerAction(final HystrixConcurrencyStrategy concurrencyStrategy, Action1 action) { this.actual = action; this.parentThreadState = HystrixRequestContext.getContextForCurrentThread(); - this.c = concurrencyStrategy.wrapCallable(new Callable() { + this.c = concurrencyStrategy.wrapCallable(new Callable() { @Override - public Subscription call() throws Exception { + public Void call() throws Exception { HystrixRequestContext existingState = HystrixRequestContext.getContextForCurrentThread(); try { // set the state of this thread to that of its parent HystrixRequestContext.setContextOnCurrentThread(parentThreadState); - // execute actual Func2 with the state of the parent - return actual.call(new HystrixContextScheduler(concurrencyStrategy, t1Holder.get()), t2Holder.get()); + // execute actual Action1 with the state of the parent + actual.call(new HystrixContextInnerScheduler(concurrencyStrategy, t1Holder.get())); + return null; } finally { // restore this thread back to its original state HystrixRequestContext.setContextOnCurrentThread(existingState); @@ -80,11 +75,10 @@ public Subscription call() throws Exception { } @Override - public Subscription call(Scheduler t1, T t2) { + public void call(Inner inner) { try { - this.t1Holder.set(t1); - this.t2Holder.set(t2); - return c.call(); + this.t1Holder.set(inner); + c.call(); } catch (Exception e) { throw new RuntimeException("Failed executing wrapped Func2", e); } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextCallable.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextCallable.java index 7e907b325..eefb31df0 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextCallable.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextCallable.java @@ -17,8 +17,6 @@ import java.util.concurrent.Callable; -import com.netflix.hystrix.strategy.HystrixPlugins; - /** * Wrapper around {@link Callable} that manages the {@link HystrixRequestContext} initialization and cleanup for the execution of the {@link Callable} * @@ -32,10 +30,6 @@ public class HystrixContextCallable implements Callable { private final Callable actual; private final HystrixRequestContext parentThreadState; - public HystrixContextCallable(Callable actual) { - this(HystrixPlugins.getInstance().getConcurrencyStrategy(), actual); - } - public HystrixContextCallable(HystrixConcurrencyStrategy concurrencyStrategy, Callable actual) { this.actual = concurrencyStrategy.wrapCallable(actual); this.parentThreadState = HystrixRequestContext.getContextForCurrentThread(); diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextRunnable.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextRunnable.java index 3c36addd4..5f9cde7d3 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextRunnable.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextRunnable.java @@ -17,8 +17,6 @@ import java.util.concurrent.Callable; -import com.netflix.hystrix.strategy.HystrixPlugins; - /** * Wrapper around {@link Runnable} that manages the {@link HystrixRequestContext} initialization and cleanup for the execution of the {@link Runnable} * @@ -29,10 +27,6 @@ public class HystrixContextRunnable implements Runnable { private final Callable actual; private final HystrixRequestContext parentThreadState; - public HystrixContextRunnable(Runnable actual) { - this(HystrixPlugins.getInstance().getConcurrencyStrategy(), actual); - } - public HystrixContextRunnable(HystrixConcurrencyStrategy concurrencyStrategy, final Runnable actual) { this.actual = concurrencyStrategy.wrapCallable(new Callable() { diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java index e66a42e88..7f4c028bf 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/concurrency/HystrixContextScheduler.java @@ -15,40 +15,125 @@ */ package com.netflix.hystrix.strategy.concurrency; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import rx.Scheduler; import rx.Subscription; -import rx.util.functions.Func2; +import rx.functions.Action1; +import rx.schedulers.Schedulers; +import rx.subscriptions.BooleanSubscription; +import com.netflix.hystrix.HystrixThreadPool; import com.netflix.hystrix.strategy.HystrixPlugins; /** - * Wrap a {@link Scheduler} so that scheduled actions are wrapped with {@link HystrixContextFunc2} so that + * Wrap a {@link Scheduler} so that scheduled actions are wrapped with {@link HystrixContexSchedulerAction} so that * the {@link HystrixRequestContext} is properly copied across threads (if they are used by the {@link Scheduler}). */ public class HystrixContextScheduler extends Scheduler { private final HystrixConcurrencyStrategy concurrencyStrategy; private final Scheduler actualScheduler; + private final HystrixThreadPool threadPool; public HystrixContextScheduler(Scheduler scheduler) { - this(HystrixPlugins.getInstance().getConcurrencyStrategy(), scheduler); + this.actualScheduler = scheduler; + this.concurrencyStrategy = HystrixPlugins.getInstance().getConcurrencyStrategy(); + this.threadPool = null; } public HystrixContextScheduler(HystrixConcurrencyStrategy concurrencyStrategy, Scheduler scheduler) { this.actualScheduler = scheduler; this.concurrencyStrategy = concurrencyStrategy; + this.threadPool = null; + } + + public HystrixContextScheduler(HystrixConcurrencyStrategy concurrencyStrategy, HystrixThreadPool threadPool) { + this.concurrencyStrategy = concurrencyStrategy; + this.threadPool = threadPool; + this.actualScheduler = Schedulers.executor(threadPool.getExecutor()); } @Override - public Subscription schedule(final T state, final Func2 action) { - return actualScheduler.schedule(state, new HystrixContextFunc2(concurrencyStrategy, action)); + public Subscription schedule(Action1 action) { + InnerHystrixContextScheduler inner = new InnerHystrixContextScheduler(); + inner.schedule(action); + return inner; } @Override - public Subscription schedule(T state, Func2 action, long delayTime, TimeUnit unit) { - return actualScheduler.schedule(state, new HystrixContextFunc2(concurrencyStrategy, action), delayTime, unit); + public Subscription schedule(Action1 action, long delayTime, TimeUnit unit) { + InnerHystrixContextScheduler inner = new InnerHystrixContextScheduler(); + inner.schedule(action, delayTime, unit); + return inner; + } + + private class InnerHystrixContextScheduler extends Inner { + + private BooleanSubscription s = new BooleanSubscription(); + + @Override + public void unsubscribe() { + s.unsubscribe(); + } + + @Override + public boolean isUnsubscribed() { + return s.isUnsubscribed(); + } + + @Override + public void schedule(Action1 action, long delayTime, TimeUnit unit) { + if (threadPool != null) { + if (!threadPool.isQueueSpaceAvailable()) { + throw new RejectedExecutionException("Rejected command because thread-pool queueSize is at rejection threshold."); + } + } + actualScheduler.schedule(new HystrixContexSchedulerAction(concurrencyStrategy, action), delayTime, unit); + } + + @Override + public void schedule(Action1 action) { + if (threadPool != null) { + if (!threadPool.isQueueSpaceAvailable()) { + throw new RejectedExecutionException("Rejected command because thread-pool queueSize is at rejection threshold."); + } + } + actualScheduler.schedule(new HystrixContexSchedulerAction(concurrencyStrategy, action)); + } + } + public static class HystrixContextInnerScheduler extends Inner { + + private final HystrixConcurrencyStrategy concurrencyStrategy; + private final Inner actual; + + HystrixContextInnerScheduler(HystrixConcurrencyStrategy concurrencyStrategy, Inner actual) { + this.concurrencyStrategy = concurrencyStrategy; + this.actual = actual; + } + + @Override + public void unsubscribe() { + actual.unsubscribe(); + } + + @Override + public boolean isUnsubscribed() { + return actual.isUnsubscribed(); + } + + @Override + public void schedule(Action1 action, long delayTime, TimeUnit unit) { + actual.schedule(new HystrixContexSchedulerAction(concurrencyStrategy, action), delayTime, unit); + } + + @Override + public void schedule(Action1 action) { + actual.schedule(new HystrixContexSchedulerAction(concurrencyStrategy, action)); + } + + } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/executionhook/HystrixCommandExecutionHook.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/executionhook/HystrixCommandExecutionHook.java index 33b29a0b1..1ce80d9b4 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/executionhook/HystrixCommandExecutionHook.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/executionhook/HystrixCommandExecutionHook.java @@ -17,6 +17,7 @@ import com.netflix.hystrix.HystrixCommand; import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; +import com.netflix.hystrix.HystrixExecutable; import com.netflix.hystrix.exception.HystrixRuntimeException; import com.netflix.hystrix.exception.HystrixRuntimeException.FailureType; import com.netflix.hystrix.strategy.HystrixPlugins; @@ -46,10 +47,15 @@ public abstract class HystrixCommandExecutionHook { * * @since 1.2 */ + @Deprecated public void onRunStart(HystrixCommand commandInstance) { // do nothing by default } + public void onRunStart(HystrixExecutable commandInstance) { + // do nothing by default + } + /** * Invoked after successful execution of {@link HystrixCommand#run()} with response value. * @@ -61,11 +67,17 @@ public void onRunStart(HystrixCommand commandInstance) { * * @since 1.2 */ + @Deprecated public T onRunSuccess(HystrixCommand commandInstance, T response) { // pass-thru by default return response; } + public T onRunSuccess(HystrixExecutable commandInstance, T response) { + // pass-thru by default + return response; + } + /** * Invoked after failed execution of {@link HystrixCommand#run()} with thrown Exception. * @@ -77,11 +89,17 @@ public T onRunSuccess(HystrixCommand commandInstance, T response) { * * @since 1.2 */ + @Deprecated public Exception onRunError(HystrixCommand commandInstance, Exception e) { // pass-thru by default return e; } + public Exception onRunError(HystrixExecutable commandInstance, Exception e) { + // pass-thru by default + return e; + } + /** * Invoked before {@link HystrixCommand#getFallback()} is about to be executed. * @@ -90,10 +108,15 @@ public Exception onRunError(HystrixCommand commandInstance, Exception e) * * @since 1.2 */ + @Deprecated public void onFallbackStart(HystrixCommand commandInstance) { // do nothing by default } + public void onFallbackStart(HystrixExecutable commandInstance) { + // do nothing by default + } + /** * Invoked after successful execution of {@link HystrixCommand#getFallback()} with response value. * @@ -105,11 +128,17 @@ public void onFallbackStart(HystrixCommand commandInstance) { * * @since 1.2 */ + @Deprecated public T onFallbackSuccess(HystrixCommand commandInstance, T fallbackResponse) { // pass-thru by default return fallbackResponse; } + public T onFallbackSuccess(HystrixExecutable commandInstance, T fallbackResponse) { + // pass-thru by default + return fallbackResponse; + } + /** * Invoked after failed execution of {@link HystrixCommand#getFallback()} with thrown exception. * @@ -121,11 +150,17 @@ public T onFallbackSuccess(HystrixCommand commandInstance, T fallbackResp * * @since 1.2 */ + @Deprecated public Exception onFallbackError(HystrixCommand commandInstance, Exception e) { // pass-thru by default return e; } + public Exception onFallbackError(HystrixExecutable commandInstance, Exception e) { + // pass-thru by default + return e; + } + /** * Invoked before {@link HystrixCommand} executes. * @@ -134,10 +169,15 @@ public Exception onFallbackError(HystrixCommand commandInstance, Exceptio * * @since 1.2 */ + @Deprecated public void onStart(HystrixCommand commandInstance) { // do nothing by default } + public void onStart(HystrixExecutable commandInstance) { + // do nothing by default + } + /** * Invoked after completion of {@link HystrixCommand} execution that results in a response. *

@@ -151,11 +191,17 @@ public void onStart(HystrixCommand commandInstance) { * * @since 1.2 */ + @Deprecated public T onComplete(HystrixCommand commandInstance, T response) { // pass-thru by default return response; } + public T onComplete(HystrixExecutable commandInstance, T response) { + // pass-thru by default + return response; + } + /** * Invoked after failed completion of {@link HystrixCommand} execution. * @@ -171,11 +217,17 @@ public T onComplete(HystrixCommand commandInstance, T response) { * * @since 1.2 */ + @Deprecated public Exception onError(HystrixCommand commandInstance, FailureType failureType, Exception e) { // pass-thru by default return e; } + public Exception onError(HystrixExecutable commandInstance, FailureType failureType, Exception e) { + // pass-thru by default + return e; + } + /** * Invoked at start of thread execution when {@link HystrixCommand} is executed using {@link ExecutionIsolationStrategy#THREAD}. * @@ -184,10 +236,15 @@ public Exception onError(HystrixCommand commandInstance, FailureType fail * * @since 1.2 */ + @Deprecated public void onThreadStart(HystrixCommand commandInstance) { // do nothing by default } + public void onThreadStart(HystrixExecutable commandInstance) { + // do nothing by default + } + /** * Invoked at completion of thread execution when {@link HystrixCommand} is executed using {@link ExecutionIsolationStrategy#THREAD}. * @@ -196,8 +253,13 @@ public void onThreadStart(HystrixCommand commandInstance) { * * @since 1.2 */ + @Deprecated public void onThreadComplete(HystrixCommand commandInstance) { // do nothing by default } + public void onThreadComplete(HystrixExecutable commandInstance) { + // do nothing by default + } + } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/metrics/HystrixMetricsPublisherFactory.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/metrics/HystrixMetricsPublisherFactory.java index 3b1216f02..9eee5a947 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/metrics/HystrixMetricsPublisherFactory.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/metrics/HystrixMetricsPublisherFactory.java @@ -15,13 +15,7 @@ */ package com.netflix.hystrix.strategy.metrics; -import static org.junit.Assert.*; - -import java.util.ArrayList; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -import org.junit.Test; import com.netflix.hystrix.HystrixCircuitBreaker; import com.netflix.hystrix.HystrixCommand; @@ -95,14 +89,14 @@ private HystrixMetricsPublisherFactory() { this(HystrixPlugins.getInstance().getMetricsPublisher()); } - private HystrixMetricsPublisherFactory(HystrixMetricsPublisher strategy) { + /* package */ HystrixMetricsPublisherFactory(HystrixMetricsPublisher strategy) { this.strategy = strategy; } // String is CommandKey.name() (we can't use CommandKey directly as we can't guarantee it implements hashcode/equals correctly) private final ConcurrentHashMap commandPublishers = new ConcurrentHashMap(); - private HystrixMetricsPublisherCommand getPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, HystrixCircuitBreaker circuitBreaker, HystrixCommandProperties properties) { + /* package */ HystrixMetricsPublisherCommand getPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, HystrixCircuitBreaker circuitBreaker, HystrixCommandProperties properties) { // attempt to retrieve from cache first HystrixMetricsPublisherCommand publisher = commandPublishers.get(commandKey.name()); if (publisher != null) { @@ -127,7 +121,7 @@ private HystrixMetricsPublisherCommand getPublisherForCommand(HystrixCommandKey // String is ThreadPoolKey.name() (we can't use ThreadPoolKey directly as we can't guarantee it implements hashcode/equals correctly) private final ConcurrentHashMap threadPoolPublishers = new ConcurrentHashMap(); - private HystrixMetricsPublisherThreadPool getPublisherForThreadPool(HystrixThreadPoolKey threadPoolKey, HystrixThreadPoolMetrics metrics, HystrixThreadPoolProperties properties) { + /* package */ HystrixMetricsPublisherThreadPool getPublisherForThreadPool(HystrixThreadPoolKey threadPoolKey, HystrixThreadPoolMetrics metrics, HystrixThreadPoolProperties properties) { // attempt to retrieve from cache first HystrixMetricsPublisherThreadPool publisher = threadPoolPublishers.get(threadPoolKey.name()); if (publisher != null) { @@ -149,81 +143,4 @@ private HystrixMetricsPublisherThreadPool getPublisherForThreadPool(HystrixThrea } } - public static class UnitTest { - - /** - * Assert that we only call a publisher once for a given Command or ThreadPool key. - */ - @Test - public void testSingleInitializePerKey() { - final TestHystrixMetricsPublisher publisher = new TestHystrixMetricsPublisher(); - final HystrixMetricsPublisherFactory factory = new HystrixMetricsPublisherFactory(publisher); - ArrayList threads = new ArrayList(); - for (int i = 0; i < 20; i++) { - threads.add(new Thread(new Runnable() { - - @Override - public void run() { - factory.getPublisherForCommand(TestCommandKey.TEST_A, null, null, null, null); - factory.getPublisherForCommand(TestCommandKey.TEST_B, null, null, null, null); - factory.getPublisherForThreadPool(TestThreadPoolKey.TEST_A, null, null); - } - - })); - } - - // start them - for (Thread t : threads) { - t.start(); - } - - // wait for them to finish - for (Thread t : threads) { - try { - t.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - // we should see 2 commands and 1 threadPool publisher created - assertEquals(2, publisher.commandCounter.get()); - assertEquals(1, publisher.threadCounter.get()); - } - } - - private static class TestHystrixMetricsPublisher extends HystrixMetricsPublisher { - - AtomicInteger commandCounter = new AtomicInteger(); - AtomicInteger threadCounter = new AtomicInteger(); - - @Override - public HystrixMetricsPublisherCommand getMetricsPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, HystrixCircuitBreaker circuitBreaker, HystrixCommandProperties properties) { - return new HystrixMetricsPublisherCommand() { - @Override - public void initialize() { - commandCounter.incrementAndGet(); - } - }; - } - - @Override - public HystrixMetricsPublisherThreadPool getMetricsPublisherForThreadPool(HystrixThreadPoolKey threadPoolKey, HystrixThreadPoolMetrics metrics, HystrixThreadPoolProperties properties) { - return new HystrixMetricsPublisherThreadPool() { - @Override - public void initialize() { - threadCounter.incrementAndGet(); - } - }; - } - - } - - private static enum TestCommandKey implements HystrixCommandKey { - TEST_A, TEST_B; - } - - private static enum TestThreadPoolKey implements HystrixThreadPoolKey { - TEST_A, TEST_B; - } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/properties/HystrixPropertiesChainedArchaiusProperty.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/properties/HystrixPropertiesChainedArchaiusProperty.java index 9f8033a34..8981299a5 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/properties/HystrixPropertiesChainedArchaiusProperty.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/properties/HystrixPropertiesChainedArchaiusProperty.java @@ -15,19 +15,13 @@ */ package com.netflix.hystrix.strategy.properties; -import static org.junit.Assert.*; - import java.util.ArrayList; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import org.junit.After; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.netflix.config.ConfigurationManager; import com.netflix.config.PropertyWrapper; /** @@ -376,200 +370,4 @@ public String getValue() { } } - public static class UnitTest { - - @After - public void cleanUp() { - // Tests which use ConfigurationManager.getConfigInstance() will leave the singleton in an initialize state, - // this will leave the singleton in a reasonable state between tests. - ConfigurationManager.getConfigInstance().clear(); - } - - @Test - public void testString() throws Exception { - - DynamicStringProperty pString = new DynamicStringProperty("defaultString", "default-default"); - HystrixPropertiesChainedArchaiusProperty.StringProperty fString = new HystrixPropertiesChainedArchaiusProperty.StringProperty("overrideString", pString); - - assertTrue("default-default".equals(fString.get())); - - ConfigurationManager.getConfigInstance().setProperty("defaultString", "default"); - assertTrue("default".equals(fString.get())); - - ConfigurationManager.getConfigInstance().setProperty("overrideString", "override"); - assertTrue("override".equals(fString.get())); - - ConfigurationManager.getConfigInstance().clearProperty("overrideString"); - assertTrue("default".equals(fString.get())); - - ConfigurationManager.getConfigInstance().clearProperty("defaultString"); - assertTrue("default-default".equals(fString.get())); - } - - @Test - public void testInteger() throws Exception { - - DynamicIntegerProperty pInt = new DynamicIntegerProperty("defaultInt", -1); - HystrixPropertiesChainedArchaiusProperty.IntegerProperty fInt = new HystrixPropertiesChainedArchaiusProperty.IntegerProperty("overrideInt", pInt); - - assertTrue(-1 == fInt.get()); - - ConfigurationManager.getConfigInstance().setProperty("defaultInt", 10); - assertTrue(10 == fInt.get()); - - ConfigurationManager.getConfigInstance().setProperty("overrideInt", 11); - assertTrue(11 == fInt.get()); - - ConfigurationManager.getConfigInstance().clearProperty("overrideInt"); - assertTrue(10 == fInt.get()); - - ConfigurationManager.getConfigInstance().clearProperty("defaultInt"); - assertTrue(-1 == fInt.get()); - } - - @Test - public void testBoolean() throws Exception { - - DynamicBooleanProperty pBoolean = new DynamicBooleanProperty("defaultBoolean", true); - HystrixPropertiesChainedArchaiusProperty.BooleanProperty fBoolean = new HystrixPropertiesChainedArchaiusProperty.BooleanProperty("overrideBoolean", pBoolean); - - System.out.println("pBoolean: " + pBoolean.get()); - System.out.println("fBoolean: " + fBoolean.get()); - - assertTrue(fBoolean.get()); - - ConfigurationManager.getConfigInstance().setProperty("defaultBoolean", Boolean.FALSE); - - System.out.println("pBoolean: " + pBoolean.get()); - System.out.println("fBoolean: " + fBoolean.get()); - - assertFalse(fBoolean.get()); - - ConfigurationManager.getConfigInstance().setProperty("overrideBoolean", Boolean.TRUE); - assertTrue(fBoolean.get()); - - ConfigurationManager.getConfigInstance().clearProperty("overrideBoolean"); - assertFalse(fBoolean.get()); - - ConfigurationManager.getConfigInstance().clearProperty("defaultBoolean"); - assertTrue(fBoolean.get()); - } - - @Test - public void testChainingString() throws Exception { - - DynamicStringProperty node1 = new DynamicStringProperty("node1", "v1"); - StringProperty node2 = new HystrixPropertiesChainedArchaiusProperty.StringProperty("node2", node1); - - HystrixPropertiesChainedArchaiusProperty.StringProperty node3 = new HystrixPropertiesChainedArchaiusProperty.StringProperty("node3", node2); - - assertTrue("" + node3.get(), "v1".equals(node3.get())); - - ConfigurationManager.getConfigInstance().setProperty("node1", "v11"); - assertTrue("v11".equals(node3.get())); - - ConfigurationManager.getConfigInstance().setProperty("node2", "v22"); - assertTrue("v22".equals(node3.get())); - - ConfigurationManager.getConfigInstance().clearProperty("node1"); - assertTrue("v22".equals(node3.get())); - - ConfigurationManager.getConfigInstance().setProperty("node3", "v33"); - assertTrue("v33".equals(node3.get())); - - ConfigurationManager.getConfigInstance().clearProperty("node2"); - assertTrue("v33".equals(node3.get())); - - ConfigurationManager.getConfigInstance().setProperty("node2", "v222"); - assertTrue("v33".equals(node3.get())); - - ConfigurationManager.getConfigInstance().clearProperty("node3"); - assertTrue("v222".equals(node3.get())); - - ConfigurationManager.getConfigInstance().clearProperty("node2"); - assertTrue("v1".equals(node3.get())); - - ConfigurationManager.getConfigInstance().setProperty("node2", "v2222"); - assertTrue("v2222".equals(node3.get())); - } - - @Test - public void testChainingInteger() throws Exception { - - DynamicIntegerProperty node1 = new DynamicIntegerProperty("node1", 1); - IntegerProperty node2 = new HystrixPropertiesChainedArchaiusProperty.IntegerProperty("node2", node1); - - HystrixPropertiesChainedArchaiusProperty.IntegerProperty node3 = new HystrixPropertiesChainedArchaiusProperty.IntegerProperty("node3", node2); - - assertTrue("" + node3.get(), 1 == node3.get()); - - ConfigurationManager.getConfigInstance().setProperty("node1", 11); - assertTrue(11 == node3.get()); - - ConfigurationManager.getConfigInstance().setProperty("node2", 22); - assertTrue(22 == node3.get()); - - ConfigurationManager.getConfigInstance().clearProperty("node1"); - assertTrue(22 == node3.get()); - - ConfigurationManager.getConfigInstance().setProperty("node3", 33); - assertTrue(33 == node3.get()); - - ConfigurationManager.getConfigInstance().clearProperty("node2"); - assertTrue(33 == node3.get()); - - ConfigurationManager.getConfigInstance().setProperty("node2", 222); - assertTrue(33 == node3.get()); - - ConfigurationManager.getConfigInstance().clearProperty("node3"); - assertTrue(222 == node3.get()); - - ConfigurationManager.getConfigInstance().clearProperty("node2"); - assertTrue(1 == node3.get()); - - ConfigurationManager.getConfigInstance().setProperty("node2", 2222); - assertTrue(2222 == node3.get()); - } - - @Test - public void testAddCallback() throws Exception { - - final DynamicStringProperty node1 = new DynamicStringProperty("n1", "n1"); - final HystrixPropertiesChainedArchaiusProperty.StringProperty node2 = new HystrixPropertiesChainedArchaiusProperty.StringProperty("n2", node1); - - final AtomicInteger callbackCount = new AtomicInteger(0); - - node2.addCallback(new Runnable() { - @Override - public void run() { - callbackCount.incrementAndGet(); - } - }); - - assertTrue(0 == callbackCount.get()); - - assertTrue("n1".equals(node2.get())); - assertTrue(0 == callbackCount.get()); - - ConfigurationManager.getConfigInstance().setProperty("n1", "n11"); - assertTrue("n11".equals(node2.get())); - assertTrue(0 == callbackCount.get()); - - ConfigurationManager.getConfigInstance().setProperty("n2", "n22"); - assertTrue("n22".equals(node2.get())); - assertTrue(1 == callbackCount.get()); - - ConfigurationManager.getConfigInstance().clearProperty("n1"); - assertTrue("n22".equals(node2.get())); - assertTrue(1 == callbackCount.get()); - - ConfigurationManager.getConfigInstance().setProperty("n2", "n222"); - assertTrue("n222".equals(node2.get())); - assertTrue(2 == callbackCount.get()); - - ConfigurationManager.getConfigInstance().clearProperty("n2"); - assertTrue("n1".equals(node2.get())); - assertTrue(3 == callbackCount.get()); - } - } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/properties/HystrixProperty.java b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/properties/HystrixProperty.java index ed9a55d91..3ccf82658 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/strategy/properties/HystrixProperty.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/strategy/properties/HystrixProperty.java @@ -15,10 +15,6 @@ */ package com.netflix.hystrix.strategy.properties; -import static org.junit.Assert.*; - -import org.junit.Test; - import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty.DynamicBooleanProperty; import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty.DynamicIntegerProperty; import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty.DynamicLongProperty; @@ -181,72 +177,4 @@ public T get() { } - public static class UnitTest { - - @Test - public void testNested1() { - HystrixProperty a = Factory.asProperty("a"); - assertEquals("a", a.get()); - - HystrixProperty aWithDefault = Factory.asProperty(a, "b"); - assertEquals("a", aWithDefault.get()); - } - - @Test - public void testNested2() { - HystrixProperty nullValue = Factory.nullProperty(); - - HystrixProperty withDefault = Factory.asProperty(nullValue, "b"); - assertEquals("b", withDefault.get()); - } - - @Test - public void testNested3() { - HystrixProperty nullValue = Factory.nullProperty(); - HystrixProperty a = Factory.asProperty(nullValue, "a"); - - HystrixProperty withDefault = Factory.asProperty(a, "b"); - assertEquals("a", withDefault.get()); - } - - @Test - public void testNested4() { - HystrixProperty nullValue = Factory.nullProperty(); - HystrixProperty a = Factory.asProperty(nullValue, null); - - HystrixProperty withDefault = Factory.asProperty(a, "b"); - assertEquals("b", withDefault.get()); - } - - @Test - public void testNested5() { - HystrixProperty nullValue = Factory.nullProperty(); - HystrixProperty a = Factory.asProperty(nullValue, null); - - @SuppressWarnings("unchecked") - HystrixProperty withDefault = Factory.asProperty(a, Factory.asProperty("b")); - assertEquals("b", withDefault.get()); - } - - @Test - public void testSeries1() { - HystrixProperty nullValue = Factory.nullProperty(); - HystrixProperty a = Factory.asProperty(nullValue, null); - - @SuppressWarnings("unchecked") - HystrixProperty withDefault = Factory.asProperty(a, nullValue, nullValue, Factory.asProperty("b")); - assertEquals("b", withDefault.get()); - } - - @Test - public void testSeries2() { - HystrixProperty nullValue = Factory.nullProperty(); - HystrixProperty a = Factory.asProperty(nullValue, null); - - @SuppressWarnings("unchecked") - HystrixProperty withDefault = Factory.asProperty(a, nullValue, Factory.asProperty("b"), nullValue, Factory.asProperty("c")); - assertEquals("b", withDefault.get()); - } - - } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/util/ExceptionThreadingUtility.java b/hystrix-core/src/main/java/com/netflix/hystrix/util/ExceptionThreadingUtility.java index 47fd569ef..5fda4b242 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/util/ExceptionThreadingUtility.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/util/ExceptionThreadingUtility.java @@ -15,14 +15,10 @@ */ package com.netflix.hystrix.util; -import org.junit.Test; - import java.util.Arrays; import java.util.HashSet; import java.util.Set; -import static org.junit.Assert.assertEquals; - /** * Used to capture a stacktrace from one thread and append it to the stacktrace of another *

@@ -32,7 +28,7 @@ public class ExceptionThreadingUtility { private final static String messageForCause = "Calling Thread included as the last 'caused by' on the chain."; - private static void attachCallingThreadStack(Throwable e, StackTraceElement[] stack) { + /* package */ static void attachCallingThreadStack(Throwable e, StackTraceElement[] stack) { Set seenCauses = new HashSet(); Throwable callingThrowable = new Throwable(messageForCause); @@ -121,49 +117,4 @@ public static void assignCallingThread(Thread callingThread) { callingThreadCache.set(callingThread); } - public static class UnitTest { - private final Throwable ex1 = new Throwable("Ex1"); - private final Throwable ex2 = new Throwable("Ex2", ex1); - - public UnitTest() { - ex1.initCause(ex2); - } - - @Test - public void testAttachCallingThreadStackParentThenChild() { - ExceptionThreadingUtility.attachCallingThreadStack(ex1, ex2.getStackTrace()); - assertEquals("Ex1", ex1.getMessage()); - assertEquals("Ex2", ex1.getCause().getMessage()); - assertEquals("Ex2", ex2.getMessage()); - assertEquals("Ex1", ex2.getCause().getMessage()); - } - - @Test - public void testAttachCallingThreadStackChildThenParent() { - ExceptionThreadingUtility.attachCallingThreadStack(ex2, ex1.getStackTrace()); - assertEquals("Ex1", ex1.getMessage()); - assertEquals("Ex2", ex1.getCause().getMessage()); - assertEquals("Ex2", ex2.getMessage()); - assertEquals("Ex1", ex2.getCause().getMessage()); - } - - @Test - public void testAttachCallingThreadStackAddExceptionsToEachOther() { - ExceptionThreadingUtility.attachCallingThreadStack(ex1, ex2.getStackTrace()); - ExceptionThreadingUtility.attachCallingThreadStack(ex2, ex1.getStackTrace()); - assertEquals("Ex1", ex1.getMessage()); - assertEquals("Ex2", ex2.getMessage()); - assertEquals("Ex2", ex1.getCause().getMessage()); - assertEquals("Ex1", ex2.getCause().getMessage()); - } - - @Test - public void testAttachCallingThreadStackAddExceptionToItself() { - ExceptionThreadingUtility.attachCallingThreadStack(ex2, ex2.getStackTrace()); - assertEquals("Ex1", ex1.getMessage()); - assertEquals("Ex2", ex1.getCause().getMessage()); - assertEquals("Ex2", ex2.getMessage()); - assertEquals("Ex1", ex2.getCause().getMessage()); - } - } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingNumber.java b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingNumber.java index 47cf47674..f7e451929 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingNumber.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingNumber.java @@ -1,12 +1,12 @@ /** * Copyright 2012 Netflix, Inc. - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,20 +15,16 @@ */ package com.netflix.hystrix.util; -import static org.junit.Assert.*; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.concurrent.locks.ReentrantLock; import javax.annotation.concurrent.ThreadSafe; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +44,7 @@ * For example, on each read to getSum/getCount it will iterate buckets to sum the data so that on writes we don't need to maintain the overall sum and pay the synchronization cost at each write to * ensure the sum is up-to-date when the read can easily iterate each bucket to get the sum when it needs it. *

- * See inner-class UnitTest for usage and expected behavior examples. + * See UnitTest for usage and expected behavior examples. */ @ThreadSafe public class HystrixRollingNumber { @@ -57,10 +53,10 @@ public class HystrixRollingNumber { private static final Time ACTUAL_TIME = new ActualTime(); private final Time time; - private final HystrixProperty timeInMilliseconds; - private final HystrixProperty numberOfBuckets; + final HystrixProperty timeInMilliseconds; + final HystrixProperty numberOfBuckets; - private final BucketCircularArray buckets; + final BucketCircularArray buckets; private final CumulativeSum cumulativeSum = new CumulativeSum(); public HystrixRollingNumber(HystrixProperty timeInMilliseconds, HystrixProperty numberOfBuckets) { @@ -68,7 +64,7 @@ public HystrixRollingNumber(HystrixProperty timeInMilliseconds, Hystrix } /* used for unit testing */ - private HystrixRollingNumber(Time time, int timeInMilliseconds, int numberOfBuckets) { + /* package for testing */HystrixRollingNumber(Time time, int timeInMilliseconds, int numberOfBuckets) { this(time, HystrixProperty.Factory.asProperty(timeInMilliseconds), HystrixProperty.Factory.asProperty(numberOfBuckets)); } @@ -84,7 +80,7 @@ private HystrixRollingNumber(Time time, HystrixProperty timeInMilliseco buckets = new BucketCircularArray(numberOfBuckets.get()); } - private int getBucketSizeInMilliseconds() { + /* package for testing */int getBucketSizeInMilliseconds() { return timeInMilliseconds.get() / numberOfBuckets.get(); } @@ -252,7 +248,7 @@ public long getRollingMaxValue(HystrixRollingNumberEvent type) { private ReentrantLock newBucketLock = new ReentrantLock(); - private Bucket getCurrentBucket() { + /* package for testing */Bucket getCurrentBucket() { long currentTime = time.getCurrentTimeInMillis(); /* a shortcut to try and get the most common result of immediately finding the current bucket */ @@ -348,7 +344,7 @@ private Bucket getCurrentBucket() { } } - private static interface Time { + /* package */static interface Time { public long getCurrentTimeInMillis(); } @@ -364,7 +360,7 @@ public long getCurrentTimeInMillis() { /** * Counters for a given 'bucket' of time. */ - private static class Bucket { + /* package */static class Bucket { final long windowStart; final LongAdder[] adderForCounterType; final LongMaxUpdater[] updaterForCounterType; @@ -426,7 +422,7 @@ LongMaxUpdater getMaxUpdater(HystrixRollingNumberEvent type) { /** * Cumulative counters (from start of JVM) from each Type */ - private static class CumulativeSum { + /* package */static class CumulativeSum { final LongAdder[] adderForCounterType; final LongMaxUpdater[] updaterForCounterType; @@ -505,7 +501,7 @@ LongMaxUpdater getMaxUpdater(HystrixRollingNumberEvent type) { *

* benjchristensen => This implementation was chosen based on performance testing I did and documented at: http://benjchristensen.com/2011/10/08/atomiccirculararray/ */ - private class BucketCircularArray implements Iterable { + /* package */class BucketCircularArray implements Iterable { private final AtomicReference state; private final int dataLength; // we don't resize, we always stay the same, so remember this private final int numBuckets; @@ -674,675 +670,4 @@ private Bucket[] getArray() { } - /* - * Why unit tests as inner classes? => http://benjchristensen.com/2011/10/23/junit-tests-as-inner-classes/ - */ - public static class UnitTest { - - @Test - public void testCreatesBuckets() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - // confirm the initial settings - assertEquals(200, counter.timeInMilliseconds.get().intValue()); - assertEquals(10, counter.numberOfBuckets.get().intValue()); - assertEquals(20, counter.getBucketSizeInMilliseconds()); - - // we start out with 0 buckets in the queue - assertEquals(0, counter.buckets.size()); - - // add a success in each interval which should result in all 10 buckets being created with 1 success in each - for (int i = 0; i < counter.numberOfBuckets.get(); i++) { - counter.increment(HystrixRollingNumberEvent.SUCCESS); - time.increment(counter.getBucketSizeInMilliseconds()); - } - - // confirm we have all 10 buckets - assertEquals(10, counter.buckets.size()); - - // add 1 more and we should still only have 10 buckets since that's the max - counter.increment(HystrixRollingNumberEvent.SUCCESS); - assertEquals(10, counter.buckets.size()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testResetBuckets() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // we start out with 0 buckets in the queue - assertEquals(0, counter.buckets.size()); - - // add 1 - counter.increment(HystrixRollingNumberEvent.SUCCESS); - - // confirm we have 1 bucket - assertEquals(1, counter.buckets.size()); - - // confirm we still have 1 bucket - assertEquals(1, counter.buckets.size()); - - // add 1 - counter.increment(HystrixRollingNumberEvent.SUCCESS); - - // we should now have a single bucket with no values in it instead of 2 or more buckets - assertEquals(1, counter.buckets.size()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testEmptyBucketsFillIn() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // add 1 - counter.increment(HystrixRollingNumberEvent.SUCCESS); - - // we should have 1 bucket - assertEquals(1, counter.buckets.size()); - - // wait past 3 bucket time periods (the 1st bucket then 2 empty ones) - time.increment(counter.getBucketSizeInMilliseconds() * 3); - - // add another - counter.increment(HystrixRollingNumberEvent.SUCCESS); - - // we should have 4 (1 + 2 empty + 1 new one) buckets - assertEquals(4, counter.buckets.size()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testIncrementInSingleBucket() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // increment - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.FAILURE); - counter.increment(HystrixRollingNumberEvent.FAILURE); - counter.increment(HystrixRollingNumberEvent.TIMEOUT); - - // we should have 1 bucket - assertEquals(1, counter.buckets.size()); - - // the count should be 4 - assertEquals(4, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SUCCESS).sum()); - assertEquals(2, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.FAILURE).sum()); - assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testTimeout() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // increment - counter.increment(HystrixRollingNumberEvent.TIMEOUT); - - // we should have 1 bucket - assertEquals(1, counter.buckets.size()); - - // the count should be 1 - assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); - assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds() * 3); - - // incremenet again in latest bucket - counter.increment(HystrixRollingNumberEvent.TIMEOUT); - - // we should have 4 buckets - assertEquals(4, counter.buckets.size()); - - // the counts of the last bucket - assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); - - // the total counts - assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testShortCircuited() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // increment - counter.increment(HystrixRollingNumberEvent.SHORT_CIRCUITED); - - // we should have 1 bucket - assertEquals(1, counter.buckets.size()); - - // the count should be 1 - assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SHORT_CIRCUITED).sum()); - assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds() * 3); - - // incremenet again in latest bucket - counter.increment(HystrixRollingNumberEvent.SHORT_CIRCUITED); - - // we should have 4 buckets - assertEquals(4, counter.buckets.size()); - - // the counts of the last bucket - assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SHORT_CIRCUITED).sum()); - - // the total counts - assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testThreadPoolRejection() { - testCounterType(HystrixRollingNumberEvent.THREAD_POOL_REJECTED); - } - - @Test - public void testFallbackSuccess() { - testCounterType(HystrixRollingNumberEvent.FALLBACK_SUCCESS); - } - - @Test - public void testFallbackFailure() { - testCounterType(HystrixRollingNumberEvent.FALLBACK_FAILURE); - } - - @Test - public void testExceptionThrow() { - testCounterType(HystrixRollingNumberEvent.EXCEPTION_THROWN); - } - - private void testCounterType(HystrixRollingNumberEvent type) { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // increment - counter.increment(type); - - // we should have 1 bucket - assertEquals(1, counter.buckets.size()); - - // the count should be 1 - assertEquals(1, counter.buckets.getLast().getAdder(type).sum()); - assertEquals(1, counter.getRollingSum(type)); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds() * 3); - - // increment again in latest bucket - counter.increment(type); - - // we should have 4 buckets - assertEquals(4, counter.buckets.size()); - - // the counts of the last bucket - assertEquals(1, counter.buckets.getLast().getAdder(type).sum()); - - // the total counts - assertEquals(2, counter.getRollingSum(type)); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testIncrementInMultipleBuckets() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // increment - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.FAILURE); - counter.increment(HystrixRollingNumberEvent.FAILURE); - counter.increment(HystrixRollingNumberEvent.TIMEOUT); - counter.increment(HystrixRollingNumberEvent.TIMEOUT); - counter.increment(HystrixRollingNumberEvent.SHORT_CIRCUITED); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds() * 3); - - // increment - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.FAILURE); - counter.increment(HystrixRollingNumberEvent.FAILURE); - counter.increment(HystrixRollingNumberEvent.FAILURE); - counter.increment(HystrixRollingNumberEvent.TIMEOUT); - counter.increment(HystrixRollingNumberEvent.SHORT_CIRCUITED); - - // we should have 4 buckets - assertEquals(4, counter.buckets.size()); - - // the counts of the last bucket - assertEquals(2, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SUCCESS).sum()); - assertEquals(3, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.FAILURE).sum()); - assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); - assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SHORT_CIRCUITED).sum()); - - // the total counts - assertEquals(6, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(5, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); - assertEquals(3, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.SHORT_CIRCUITED)); - - // wait until window passes - time.increment(counter.timeInMilliseconds.get()); - - // increment - counter.increment(HystrixRollingNumberEvent.SUCCESS); - - // the total counts should now include only the last bucket after a reset since the window passed - assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); - assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testCounterRetrievalRefreshesBuckets() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // increment - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.SUCCESS); - counter.increment(HystrixRollingNumberEvent.FAILURE); - counter.increment(HystrixRollingNumberEvent.FAILURE); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds() * 3); - - // we should have 1 bucket since nothing has triggered the update of buckets in the elapsed time - assertEquals(1, counter.buckets.size()); - - // the total counts - assertEquals(4, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); - - // we should have 4 buckets as the counter 'gets' should have triggered the buckets being created to fill in time - assertEquals(4, counter.buckets.size()); - - // wait until window passes - time.increment(counter.timeInMilliseconds.get()); - - // the total counts should all be 0 (and the buckets cleared by the get, not only increment) - assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); - - // increment - counter.increment(HystrixRollingNumberEvent.SUCCESS); - - // the total counts should now include only the last bucket after a reset since the window passed - assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testUpdateMax1() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // increment - counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 10); - - // we should have 1 bucket - assertEquals(1, counter.buckets.size()); - - // the count should be 10 - assertEquals(10, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); - assertEquals(10, counter.getRollingMaxValue(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds() * 3); - - // increment again in latest bucket - counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 20); - - // we should have 4 buckets - assertEquals(4, counter.buckets.size()); - - // the max - assertEquals(20, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); - - // counts per bucket - long values[] = counter.getValues(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE); - assertEquals(10, values[0]); // oldest bucket - assertEquals(0, values[1]); - assertEquals(0, values[2]); - assertEquals(20, values[3]); // latest bucket - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testUpdateMax2() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - // increment - counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 10); - counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 30); - counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 20); - - // we should have 1 bucket - assertEquals(1, counter.buckets.size()); - - // the count should be 30 - assertEquals(30, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); - assertEquals(30, counter.getRollingMaxValue(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds() * 3); - - counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 30); - counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 30); - counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 50); - - // we should have 4 buckets - assertEquals(4, counter.buckets.size()); - - // the count - assertEquals(50, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); - assertEquals(50, counter.getValueOfLatestBucket(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); - - // values per bucket - long values[] = counter.getValues(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE); - assertEquals(30, values[0]); // oldest bucket - assertEquals(0, values[1]); - assertEquals(0, values[2]); - assertEquals(50, values[3]); // latest bucket - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testMaxValue() { - MockedTime time = new MockedTime(); - try { - HystrixRollingNumberEvent type = HystrixRollingNumberEvent.THREAD_MAX_ACTIVE; - - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - - counter.updateRollingMax(type, 10); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds()); - - counter.updateRollingMax(type, 30); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds()); - - counter.updateRollingMax(type, 40); - - // sleep to get to a new bucket - time.increment(counter.getBucketSizeInMilliseconds()); - - counter.updateRollingMax(type, 15); - - assertEquals(40, counter.getRollingMaxValue(type)); - - } catch (Exception e) { - e.printStackTrace(); - fail("Exception: " + e.getMessage()); - } - } - - @Test - public void testEmptySum() { - MockedTime time = new MockedTime(); - HystrixRollingNumberEvent type = HystrixRollingNumberEvent.COLLAPSED; - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - assertEquals(0, counter.getRollingSum(type)); - } - - @Test - public void testEmptyMax() { - MockedTime time = new MockedTime(); - HystrixRollingNumberEvent type = HystrixRollingNumberEvent.THREAD_MAX_ACTIVE; - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - assertEquals(0, counter.getRollingMaxValue(type)); - } - - @Test - public void testEmptyLatestValue() { - MockedTime time = new MockedTime(); - HystrixRollingNumberEvent type = HystrixRollingNumberEvent.THREAD_MAX_ACTIVE; - HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); - assertEquals(0, counter.getValueOfLatestBucket(type)); - } - - @Test - public void testRolling() { - MockedTime time = new MockedTime(); - HystrixRollingNumberEvent type = HystrixRollingNumberEvent.THREAD_MAX_ACTIVE; - HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); - // iterate over 20 buckets on a queue sized for 2 - for (int i = 0; i < 20; i++) { - // first bucket - counter.getCurrentBucket(); - try { - time.increment(counter.getBucketSizeInMilliseconds()); - } catch (Exception e) { - // ignore - } - - assertEquals(2, counter.getValues(type).length); - - counter.getValueOfLatestBucket(type); - - // System.out.println("Head: " + counter.buckets.state.get().head); - // System.out.println("Tail: " + counter.buckets.state.get().tail); - } - } - - @Test - public void testCumulativeCounterAfterRolling() { - MockedTime time = new MockedTime(); - HystrixRollingNumberEvent type = HystrixRollingNumberEvent.SUCCESS; - HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); - - assertEquals(0, counter.getCumulativeSum(type)); - - // iterate over 20 buckets on a queue sized for 2 - for (int i = 0; i < 20; i++) { - // first bucket - counter.increment(type); - try { - time.increment(counter.getBucketSizeInMilliseconds()); - } catch (Exception e) { - // ignore - } - - assertEquals(2, counter.getValues(type).length); - - counter.getValueOfLatestBucket(type); - - } - - // cumulative count should be 20 (for the number of loops above) regardless of buckets rolling - assertEquals(20, counter.getCumulativeSum(type)); - } - - @Test - public void testCumulativeCounterAfterRollingAndReset() { - MockedTime time = new MockedTime(); - HystrixRollingNumberEvent type = HystrixRollingNumberEvent.SUCCESS; - HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); - - assertEquals(0, counter.getCumulativeSum(type)); - - // iterate over 20 buckets on a queue sized for 2 - for (int i = 0; i < 20; i++) { - // first bucket - counter.increment(type); - try { - time.increment(counter.getBucketSizeInMilliseconds()); - } catch (Exception e) { - // ignore - } - - assertEquals(2, counter.getValues(type).length); - - counter.getValueOfLatestBucket(type); - - if (i == 5 || i == 15) { - // simulate a reset occurring every once in a while - // so we ensure the absolute sum is handling it okay - counter.reset(); - } - } - - // cumulative count should be 20 (for the number of loops above) regardless of buckets rolling - assertEquals(20, counter.getCumulativeSum(type)); - } - - @Test - public void testCumulativeCounterAfterRollingAndReset2() { - MockedTime time = new MockedTime(); - HystrixRollingNumberEvent type = HystrixRollingNumberEvent.SUCCESS; - HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); - - assertEquals(0, counter.getCumulativeSum(type)); - - counter.increment(type); - counter.increment(type); - counter.increment(type); - - // iterate over 20 buckets on a queue sized for 2 - for (int i = 0; i < 20; i++) { - try { - time.increment(counter.getBucketSizeInMilliseconds()); - } catch (Exception e) { - // ignore - } - - if (i == 5 || i == 15) { - // simulate a reset occurring every once in a while - // so we ensure the absolute sum is handling it okay - counter.reset(); - } - } - - // no increments during the loop, just some before and after - counter.increment(type); - counter.increment(type); - - // cumulative count should be 5 regardless of buckets rolling - assertEquals(5, counter.getCumulativeSum(type)); - } - - @Test - public void testCumulativeCounterAfterRollingAndReset3() { - MockedTime time = new MockedTime(); - HystrixRollingNumberEvent type = HystrixRollingNumberEvent.SUCCESS; - HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); - - assertEquals(0, counter.getCumulativeSum(type)); - - counter.increment(type); - counter.increment(type); - counter.increment(type); - - // iterate over 20 buckets on a queue sized for 2 - for (int i = 0; i < 20; i++) { - try { - time.increment(counter.getBucketSizeInMilliseconds()); - } catch (Exception e) { - // ignore - } - } - - // since we are rolling over the buckets it should reset naturally - - // no increments during the loop, just some before and after - counter.increment(type); - counter.increment(type); - - // cumulative count should be 5 regardless of buckets rolling - assertEquals(5, counter.getCumulativeSum(type)); - } - - private static class MockedTime implements Time { - - private AtomicInteger time = new AtomicInteger(0); - - @Override - public long getCurrentTimeInMillis() { - return time.get(); - } - - public void increment(int millis) { - time.addAndGet(millis); - } - - } - } - } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingPercentile.java b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingPercentile.java index cb7f4584c..c17d8ccfb 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingPercentile.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingPercentile.java @@ -15,8 +15,6 @@ */ package com.netflix.hystrix.util; -import static org.junit.Assert.*; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -29,8 +27,6 @@ import javax.annotation.concurrent.NotThreadSafe; -import org.junit.Assert; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +50,7 @@ public class HystrixRollingPercentile { private static final Time ACTUAL_TIME = new ActualTime(); private final Time time; - private final BucketCircularArray buckets; + /* package for testing */ final BucketCircularArray buckets; private final HystrixProperty timeInMilliseconds; private final HystrixProperty numberOfBuckets; private final HystrixProperty bucketDataLength; @@ -63,7 +59,7 @@ public class HystrixRollingPercentile { /* * This will get flipped each time a new bucket is created. */ - private volatile PercentileSnapshot currentPercentileSnapshot = new PercentileSnapshot(0); + /* package for testing */ volatile PercentileSnapshot currentPercentileSnapshot = new PercentileSnapshot(0); /** * @@ -89,7 +85,7 @@ public HystrixRollingPercentile(HystrixProperty timeInMilliseconds, Hys } - private HystrixRollingPercentile(Time time, HystrixProperty timeInMilliseconds, HystrixProperty numberOfBuckets, HystrixProperty bucketDataLength, HystrixProperty enabled) { + /* package for testing */ HystrixRollingPercentile(Time time, HystrixProperty timeInMilliseconds, HystrixProperty numberOfBuckets, HystrixProperty bucketDataLength, HystrixProperty enabled) { this.time = time; this.timeInMilliseconds = timeInMilliseconds; this.numberOfBuckets = numberOfBuckets; @@ -318,12 +314,12 @@ public int length() { } @NotThreadSafe - private static class PercentileSnapshot { + /* package for testing */ static class PercentileSnapshot { private final int[] data; private final int length; private int mean; - private PercentileSnapshot(Bucket[] buckets) { + /* package for testing */ PercentileSnapshot(Bucket[] buckets) { int lengthFromBuckets = 0; // we need to calculate it dynamically as it could have been changed by properties (rare, but possible) // also this way we capture the actual index size rather than the max so size the int[] to only what we need @@ -352,7 +348,7 @@ private PercentileSnapshot(Bucket[] buckets) { Arrays.sort(this.data, 0, length); } - private PercentileSnapshot(int... data) { + /* package for testing */ PercentileSnapshot(int... data) { this.data = data; this.length = data.length; @@ -365,7 +361,7 @@ private PercentileSnapshot(int... data) { Arrays.sort(this.data, 0, length); } - private int getMean() { + /* package for testing */ int getMean() { return mean; } @@ -428,7 +424,7 @@ private int computePercentile(double percent) { *

* benjchristensen => This implementation was chosen based on performance testing I did and documented at: http://benjchristensen.com/2011/10/08/atomiccirculararray/ */ - private class BucketCircularArray implements Iterable { + /* package for testing */ class BucketCircularArray implements Iterable { private final AtomicReference state; private final int dataLength; // we don't resize, we always stay the same, so remember this private final int numBuckets; @@ -596,7 +592,7 @@ private Bucket[] getArray() { /** * Counters for a given 'bucket' of time. */ - private static class Bucket { + /* package for testing */ static class Bucket { final long windowStart; final PercentileBucketData data; @@ -607,7 +603,7 @@ private static class Bucket { } - private static interface Time { + /* package for testing */ static interface Time { public long getCurrentTimeInMillis(); } @@ -620,654 +616,4 @@ public long getCurrentTimeInMillis() { } - public static class UnitTest { - - private static final HystrixProperty timeInMilliseconds = HystrixProperty.Factory.asProperty(60000); - private static final HystrixProperty numberOfBuckets = HystrixProperty.Factory.asProperty(12); // 12 buckets at 5000ms each - private static final HystrixProperty bucketDataLength = HystrixProperty.Factory.asProperty(1000); - private static final HystrixProperty enabled = HystrixProperty.Factory.asProperty(true); - - @Test - public void testRolling() { - MockedTime time = new MockedTime(); - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); - p.addValue(1000); - p.addValue(1000); - p.addValue(1000); - p.addValue(2000); - - assertEquals(1, p.buckets.size()); - - // no bucket turnover yet so percentile not yet generated - assertEquals(0, p.getPercentile(50)); - - time.increment(6000); - - // still only 1 bucket until we touch it again - assertEquals(1, p.buckets.size()); - - // a bucket has been created so we have a new percentile - assertEquals(1000, p.getPercentile(50)); - - // now 2 buckets since getting a percentile causes bucket retrieval - assertEquals(2, p.buckets.size()); - - p.addValue(1000); - p.addValue(500); - - // should still be 2 buckets - assertEquals(2, p.buckets.size()); - - p.addValue(200); - p.addValue(200); - p.addValue(1600); - p.addValue(200); - p.addValue(1600); - p.addValue(1600); - - // we haven't progressed to a new bucket so the percentile should be the same and ignore the most recent bucket - assertEquals(1000, p.getPercentile(50)); - - // increment to another bucket so we include all of the above in the PercentileSnapshot - time.increment(6000); - - // the rolling version should have the same data as creating a snapshot like this - PercentileSnapshot ps = new PercentileSnapshot(1000, 1000, 1000, 2000, 1000, 500, 200, 200, 1600, 200, 1600, 1600); - - assertEquals(ps.getPercentile(0.15), p.getPercentile(0.15)); - assertEquals(ps.getPercentile(0.50), p.getPercentile(0.50)); - assertEquals(ps.getPercentile(0.90), p.getPercentile(0.90)); - assertEquals(ps.getPercentile(0.995), p.getPercentile(0.995)); - - System.out.println("100th: " + ps.getPercentile(100) + " " + p.getPercentile(100)); - System.out.println("99.5th: " + ps.getPercentile(99.5) + " " + p.getPercentile(99.5)); - System.out.println("99th: " + ps.getPercentile(99) + " " + p.getPercentile(99)); - System.out.println("90th: " + ps.getPercentile(90) + " " + p.getPercentile(90)); - System.out.println("50th: " + ps.getPercentile(50) + " " + p.getPercentile(50)); - System.out.println("10th: " + ps.getPercentile(10) + " " + p.getPercentile(10)); - - // mean = 1000+1000+1000+2000+1000+500+200+200+1600+200+1600+1600/12 - assertEquals(991, ps.getMean()); - } - - @Test - public void testValueIsZeroAfterRollingWindowPassesAndNoTraffic() { - MockedTime time = new MockedTime(); - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); - p.addValue(1000); - p.addValue(1000); - p.addValue(1000); - p.addValue(2000); - p.addValue(4000); - - assertEquals(1, p.buckets.size()); - - // no bucket turnover yet so percentile not yet generated - assertEquals(0, p.getPercentile(50)); - - time.increment(6000); - - // still only 1 bucket until we touch it again - assertEquals(1, p.buckets.size()); - - // a bucket has been created so we have a new percentile - assertEquals(1500, p.getPercentile(50)); - - // let 1 minute pass - time.increment(60000); - - // no data in a minute should mean all buckets are empty (or reset) so we should not have any percentiles - assertEquals(0, p.getPercentile(50)); - } - - @Test - public void testSampleDataOverTime1() { - System.out.println("\n\n***************************** testSampleDataOverTime1 \n"); - - MockedTime time = new MockedTime(); - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); - int previousTime = 0; - for (int i = 0; i < SampleDataHolder1.data.length; i++) { - int timeInMillisecondsSinceStart = SampleDataHolder1.data[i][0]; - int latency = SampleDataHolder1.data[i][1]; - time.increment(timeInMillisecondsSinceStart - previousTime); - previousTime = timeInMillisecondsSinceStart; - p.addValue(latency); - } - - System.out.println("0.01: " + p.getPercentile(0.01)); - System.out.println("Median: " + p.getPercentile(50)); - System.out.println("90th: " + p.getPercentile(90)); - System.out.println("99th: " + p.getPercentile(99)); - System.out.println("99.5th: " + p.getPercentile(99.5)); - System.out.println("99.99: " + p.getPercentile(99.99)); - - System.out.println("Median: " + p.getPercentile(50)); - System.out.println("Median: " + p.getPercentile(50)); - System.out.println("Median: " + p.getPercentile(50)); - - /* - * In a loop as a use case was found where very different values were calculated in subsequent requests. - */ - for (int i = 0; i < 10; i++) { - if (p.getPercentile(50) > 5) { - fail("We expect around 2 but got: " + p.getPercentile(50)); - } - - if (p.getPercentile(99.5) < 20) { - fail("We expect to see some high values over 20 but got: " + p.getPercentile(99.5)); - } - } - } - - @Test - public void testSampleDataOverTime2() { - System.out.println("\n\n***************************** testSampleDataOverTime2 \n"); - MockedTime time = new MockedTime(); - int previousTime = 0; - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); - for (int i = 0; i < SampleDataHolder2.data.length; i++) { - int timeInMillisecondsSinceStart = SampleDataHolder2.data[i][0]; - int latency = SampleDataHolder2.data[i][1]; - time.increment(timeInMillisecondsSinceStart - previousTime); - previousTime = timeInMillisecondsSinceStart; - p.addValue(latency); - } - - System.out.println("0.01: " + p.getPercentile(0.01)); - System.out.println("Median: " + p.getPercentile(50)); - System.out.println("90th: " + p.getPercentile(90)); - System.out.println("99th: " + p.getPercentile(99)); - System.out.println("99.5th: " + p.getPercentile(99.5)); - System.out.println("99.99: " + p.getPercentile(99.99)); - - if (p.getPercentile(50) > 90 || p.getPercentile(50) < 50) { - fail("We expect around 60-70 but got: " + p.getPercentile(50)); - } - - if (p.getPercentile(99) < 400) { - fail("We expect to see some high values over 400 but got: " + p.getPercentile(99)); - } - } - - public PercentileSnapshot getPercentileForValues(int... values) { - PercentileSnapshot p = new PercentileSnapshot(values); - return p; - } - - @Test - public void testPercentileAlgorithm_Median1() { - PercentileSnapshot list = new PercentileSnapshot(100, 100, 100, 100, 200, 200, 200, 300, 300, 300, 300); - Assert.assertEquals(200, list.getPercentile(50)); - } - - @Test - public void testPercentileAlgorithm_Median2() { - PercentileSnapshot list = new PercentileSnapshot(100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 500); - Assert.assertEquals(100, list.getPercentile(50)); - } - - @Test - public void testPercentileAlgorithm_Median3() { - PercentileSnapshot list = new PercentileSnapshot(50, 75, 100, 125, 160, 170, 180, 200, 210, 300, 500); - // list.addValue(50); // 1 - // list.addValue(75); // 2 - // list.addValue(100); // 3 - // list.addValue(125); // 4 - // list.addValue(160); // 5 - // list.addValue(170); // 6 - // list.addValue(180); // 7 - // list.addValue(200); // 8 - // list.addValue(210); // 9 - // list.addValue(300); // 10 - // list.addValue(500); // 11 - - Assert.assertEquals(175, list.getPercentile(50)); - } - - @Test - public void testPercentileAlgorithm_Median4() { - PercentileSnapshot list = new PercentileSnapshot(300, 75, 125, 500, 100, 160, 180, 200, 210, 50, 170); - // unsorted so it is expected to sort it for us - // list.addValue(300); // 10 - // list.addValue(75); // 2 - // list.addValue(125); // 4 - // list.addValue(500); // 11 - // list.addValue(100); // 3 - // list.addValue(160); // 5 - // list.addValue(180); // 7 - // list.addValue(200); // 8 - // list.addValue(210); // 9 - // list.addValue(50); // 1 - // list.addValue(170); // 6 - - Assert.assertEquals(175, list.getPercentile(50)); - } - - @Test - public void testPercentileAlgorithm_Extremes() { - PercentileSnapshot p = new PercentileSnapshot(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 800, 768, 657, 700, 867); - - System.out.println("0.01: " + p.getPercentile(0.01)); - System.out.println("10th: " + p.getPercentile(10)); - System.out.println("Median: " + p.getPercentile(50)); - System.out.println("75th: " + p.getPercentile(75)); - System.out.println("90th: " + p.getPercentile(90)); - System.out.println("99th: " + p.getPercentile(99)); - System.out.println("99.5th: " + p.getPercentile(99.5)); - System.out.println("99.99: " + p.getPercentile(99.99)); - Assert.assertEquals(2, p.getPercentile(50)); - Assert.assertEquals(2, p.getPercentile(10)); - Assert.assertEquals(2, p.getPercentile(75)); - if (p.getPercentile(95) < 600) { - fail("We expect the 90th to be over 600 to show the extremes but got: " + p.getPercentile(90)); - } - if (p.getPercentile(99) < 600) { - fail("We expect the 99th to be over 600 to show the extremes but got: " + p.getPercentile(99)); - } - } - - @Test - public void testPercentileAlgorithm_HighPercentile() { - PercentileSnapshot p = getPercentileForValues(1, 2, 3); - Assert.assertEquals(2, p.getPercentile(50)); - Assert.assertEquals(3, p.getPercentile(75)); - } - - @Test - public void testPercentileAlgorithm_LowPercentile() { - PercentileSnapshot p = getPercentileForValues(1, 2); - Assert.assertEquals(1, p.getPercentile(25)); - Assert.assertEquals(2, p.getPercentile(75)); - } - - @Test - public void testPercentileAlgorithm_Percentiles() { - PercentileSnapshot p = getPercentileForValues(10, 30, 20, 40); - Assert.assertEquals(22, p.getPercentile(30), 1.0e-5); - Assert.assertEquals(20, p.getPercentile(25), 1.0e-5); - Assert.assertEquals(40, p.getPercentile(75), 1.0e-5); - Assert.assertEquals(30, p.getPercentile(50), 1.0e-5); - - // invalid percentiles - Assert.assertEquals(10, p.getPercentile(-1)); - Assert.assertEquals(40, p.getPercentile(101)); - } - - @Test - public void testPercentileAlgorithm_NISTExample() { - PercentileSnapshot p = getPercentileForValues(951772, 951567, 951937, 951959, 951442, 950610, 951591, 951195, 951772, 950925, 951990, 951682); - Assert.assertEquals(951983, p.getPercentile(90)); - Assert.assertEquals(951990, p.getPercentile(100)); - } - - /** - * This code should work without throwing exceptions but the data returned will all be -1 since the rolling percentile is disabled. - */ - @Test - public void testDoesNothingWhenDisabled() { - MockedTime time = new MockedTime(); - int previousTime = 0; - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, HystrixProperty.Factory.asProperty(false)); - for (int i = 0; i < SampleDataHolder2.data.length; i++) { - int timeInMillisecondsSinceStart = SampleDataHolder2.data[i][0]; - int latency = SampleDataHolder2.data[i][1]; - time.increment(timeInMillisecondsSinceStart - previousTime); - previousTime = timeInMillisecondsSinceStart; - p.addValue(latency); - } - - assertEquals(-1, p.getPercentile(50)); - assertEquals(-1, p.getPercentile(75)); - assertEquals(-1, p.getMean()); - } - - private static class MockedTime implements Time { - - private AtomicInteger time = new AtomicInteger(0); - - @Override - public long getCurrentTimeInMillis() { - return time.get(); - } - - public void increment(int millis) { - time.addAndGet(millis); - } - - } - - }; - - /* sub-class to avoid 65k limit of a single class */ - private static class SampleDataHolder1 { - /* - * Array of [milliseconds, latency] - */ - private static int[][] data = new int[][] { - { 0, 3 }, { 43, 33 }, { 45, 11 }, { 45, 1 }, { 68, 13 }, { 88, 10 }, { 158, 2 }, { 158, 4 }, { 169, 12 }, { 267, 2 }, { 342, 2 }, { 438, 2 }, { 464, 7 }, { 504, 2 }, { 541, 6 }, { 541, 2 }, { 562, 2 }, { 581, 3 }, { 636, 2 }, { 778, 2 }, { 825, 1 }, { 859, 2 }, { 948, 1 }, { 1043, 2 }, { 1145, 2 }, { 1152, 1 }, { 1218, 5 }, - { 1229, 2 }, { 1259, 2 }, { 1333, 2 }, { 1349, 2 }, { 1392, 2 }, { 1468, 1 }, { 1551, 2 }, { 1586, 2 }, { 1685, 2 }, { 1696, 1 }, { 1807, 2 }, { 1817, 3 }, { 1817, 6 }, { 1847, 2 }, { 1870, 2 }, { 1939, 2 }, { 2050, 2 }, { 2129, 3 }, { 2141, 2 }, { 2265, 2 }, { 2414, 1 }, { 2693, 2 }, { 2703, 2 }, { 2791, 2 }, { 2838, 2 }, - { 2906, 2 }, { 2981, 2 }, { 3008, 2 }, { 3026, 4 }, { 3077, 2 }, { 3273, 2 }, { 3282, 2 }, { 3286, 2 }, { 3318, 3 }, { 3335, 5 }, { 3710, 2 }, { 3711, 1 }, { 3745, 2 }, { 3748, 4 }, { 3767, 3 }, { 3809, 3 }, { 3835, 35 }, { 4083, 1 }, { 4116, 2 }, { 4117, 1 }, { 4157, 1 }, { 4279, 2 }, { 4344, 2 }, { 4452, 2 }, { 4530, 2 }, - { 4583, 2 }, { 4647, 3 }, { 4758, 2 }, { 4776, 2 }, { 4793, 2 }, { 4901, 2 }, { 4909, 2 }, { 4962, 2 }, { 4984, 2 }, { 5022, 2 }, { 5139, 2 }, { 5166, 1 }, { 5174, 2 }, { 5187, 2 }, { 5225, 2 }, { 5234, 2 }, { 5263, 1 }, { 5325, 2 }, { 5355, 4 }, { 5407, 1 }, { 5414, 2 }, { 5589, 2 }, { 5595, 2 }, { 5747, 2 }, { 5780, 2 }, - { 5788, 2 }, { 5796, 2 }, { 5818, 2 }, { 5975, 1 }, { 6018, 1 }, { 6270, 2 }, { 6272, 2 }, { 6348, 2 }, { 6372, 2 }, { 6379, 2 }, { 6439, 2 }, { 6442, 2 }, { 6460, 2 }, { 6460, 2 }, { 6509, 2 }, { 6511, 1 }, { 6514, 4 }, { 6530, 8 }, { 6719, 2 }, { 6760, 2 }, { 6784, 2 }, { 6838, 1 }, { 6861, 2 }, { 6947, 2 }, { 7013, 2 }, - { 7075, 2 }, { 7122, 5 }, { 7130, 2 }, { 7209, 3 }, { 7259, 2 }, { 7309, 1 }, { 7315, 3 }, { 7322, 2 }, { 7348, 2 }, { 7420, 2 }, { 7461, 2 }, { 7545, 2 }, { 7554, 3 }, { 7630, 2 }, { 7666, 2 }, { 7815, 1 }, { 7972, 1 }, { 7972, 2 }, { 7988, 2 }, { 8049, 8 }, { 8254, 2 }, { 8269, 2 }, { 8352, 1 }, { 8378, 2 }, { 8526, 2 }, - { 8531, 2 }, { 8583, 2 }, { 8615, 2 }, { 8619, 3 }, { 8623, 2 }, { 8692, 1 }, { 8698, 2 }, { 8773, 2 }, { 8777, 3 }, { 8822, 2 }, { 8929, 2 }, { 8935, 2 }, { 9025, 2 }, { 9054, 2 }, { 9056, 1 }, { 9086, 2 }, { 9147, 3 }, { 9219, 2 }, { 9230, 3 }, { 9248, 2 }, { 9283, 2 }, { 9314, 2 }, { 9418, 1 }, { 9426, 2 }, { 9456, 1 }, - { 9594, 2 }, { 9628, 2 }, { 9642, 2 }, { 9646, 2 }, { 9686, 1 }, { 9709, 2 }, { 9771, 3 }, { 9782, 2 }, { 9884, 2 }, { 9914, 5 }, { 10004, 4 }, { 10033, 6 }, { 10052, 2 }, { 10086, 2 }, { 10168, 2 }, { 10176, 1 }, { 10228, 2 }, { 10312, 2 }, { 10372, 2 }, { 10622, 2 }, { 10685, 2 }, { 10687, 1 }, { 10787, 2 }, { 11010, 2 }, - { 11024, 2 }, { 11044, 2 }, { 11086, 2 }, { 11149, 1 }, { 11198, 2 }, { 11265, 2 }, { 11302, 2 }, { 11326, 2 }, { 11354, 2 }, { 11404, 1 }, { 11473, 2 }, { 11506, 2 }, { 11548, 4 }, { 11575, 2 }, { 11621, 4 }, { 11625, 3 }, { 11625, 1 }, { 11642, 4 }, { 11859, 5 }, { 11870, 2 }, { 11872, 3 }, { 11880, 7 }, { 11886, 3 }, - { 11905, 6 }, { 11880, 3 }, { 11912, 6 }, { 11916, 4 }, { 11916, 3 }, { 11965, 4 }, { 12068, 13 }, { 12106, 2 }, { 12120, 2 }, { 12221, 2 }, { 12257, 2 }, { 12361, 2 }, { 12411, 2 }, { 12473, 3 }, { 12554, 2 }, { 12583, 2 }, { 12654, 2 }, { 12665, 2 }, { 12744, 1 }, { 12775, 2 }, { 12858, 2 }, { 12993, 2 }, { 13007, 3 }, - { 13025, 4 }, { 13038, 2 }, { 13092, 4 }, { 13094, 5 }, { 13095, 1 }, { 13110, 2 }, { 13116, 1 }, { 13140, 2 }, { 13169, 1 }, { 13186, 2 }, { 13202, 2 }, { 13202, 1 }, { 13256, 2 }, { 13344, 2 }, { 13373, 2 }, { 13396, 3 }, { 13446, 2 }, { 13451, 3 }, { 13475, 2 }, { 13521, 1 }, { 13587, 2 }, { 13592, 2 }, { 13708, 3 }, - { 13711, 1 }, { 13741, 1 }, { 13757, 1 }, { 13847, 2 }, { 13881, 3 }, { 13915, 1 }, { 14005, 2 }, { 14028, 2 }, { 14037, 2 }, { 14074, 2 }, { 14135, 2 }, { 14176, 2 }, { 14227, 2 }, { 14228, 2 }, { 14271, 3 }, { 14279, 3 }, { 14493, 2 }, { 14535, 3 }, { 14535, 1 }, { 14680, 2 }, { 14717, 2 }, { 14725, 1 }, { 14790, 2 }, - { 14801, 1 }, { 14959, 2 }, { 15052, 2 }, { 15055, 1 }, { 15055, 1 }, { 15075, 2 }, { 15103, 8 }, { 15153, 16 }, { 15191, 2 }, { 15240, 2 }, { 15313, 2 }, { 15323, 2 }, { 15341, 1 }, { 15383, 2 }, { 15387, 2 }, { 15491, 2 }, { 15534, 2 }, { 15539, 2 }, { 15549, 2 }, { 15554, 1 }, { 15664, 1 }, { 15726, 2 }, { 15807, 2 }, - { 15842, 2 }, { 15897, 2 }, { 15913, 3 }, { 15925, 2 }, { 15935, 2 }, { 16131, 1 }, { 16211, 3 }, { 16249, 2 }, { 16268, 2 }, { 16307, 2 }, { 16398, 2 }, { 16498, 2 }, { 16518, 1 }, { 16552, 1 }, { 16571, 2 }, { 16592, 2 }, { 16601, 3 }, { 16638, 2 }, { 16698, 2 }, { 16712, 1 }, { 16767, 2 }, { 16789, 2 }, { 16992, 2 }, - { 17015, 2 }, { 17035, 2 }, { 17074, 3 }, { 17086, 3 }, { 17086, 1 }, { 17092, 1 }, { 17110, 4 }, { 17116, 3 }, { 17236, 2 }, { 17291, 2 }, { 17291, 2 }, { 17340, 2 }, { 17342, 1 }, { 17360, 3 }, { 17436, 3 }, { 17457, 2 }, { 17508, 1 }, { 17556, 2 }, { 17601, 2 }, { 17639, 2 }, { 17671, 2 }, { 17743, 2 }, { 17857, 2 }, - { 17915, 2 }, { 17992, 2 }, { 18077, 1 }, { 18088, 2 }, { 18158, 1 }, { 18239, 16 }, { 18242, 2 }, { 18252, 3 }, { 18299, 1 }, { 18405, 2 }, { 18433, 2 }, { 18444, 2 }, { 18490, 2 }, { 18497, 2 }, { 18516, 2 }, { 18540, 2 }, { 18598, 2 }, { 18649, 2 }, { 18658, 2 }, { 18683, 2 }, { 18728, 2 }, { 18767, 1 }, { 18821, 2 }, - { 18868, 2 }, { 18876, 2 }, { 18914, 14 }, { 19212, 1 }, { 19215, 1 }, { 19293, 2 }, { 19303, 2 }, { 19336, 2 }, { 19376, 2 }, { 19419, 2 }, { 19558, 2 }, { 19559, 1 }, { 19609, 2 }, { 19688, 2 }, { 19724, 2 }, { 19820, 1 }, { 19851, 2 }, { 19881, 2 }, { 19966, 2 }, { 19983, 3 }, { 19988, 4 }, { 20047, 1 }, { 20062, 2 }, - { 20091, 1 }, { 20152, 1 }, { 20183, 1 }, { 20208, 2 }, { 20346, 2 }, { 20386, 1 }, { 20459, 2 }, { 20505, 2 }, { 20520, 1 }, { 20560, 3 }, { 20566, 3 }, { 20566, 1 }, { 20610, 2 }, { 20652, 2 }, { 20694, 2 }, { 20740, 2 }, { 20756, 2 }, { 20825, 3 }, { 20895, 2 }, { 20959, 1 }, { 20995, 2 }, { 21017, 3 }, { 21039, 2 }, - { 21086, 1 }, { 21109, 3 }, { 21139, 3 }, { 21206, 2 }, { 21230, 2 }, { 21251, 3 }, { 21352, 2 }, { 21353, 2 }, { 21370, 3 }, { 21389, 1 }, { 21445, 3 }, { 21475, 2 }, { 21528, 2 }, { 21559, 3 }, { 21604, 2 }, { 21606, 1 }, { 21815, 2 }, { 21858, 3 }, { 21860, 3 }, { 22015, 2 }, { 22065, 2 }, { 22098, 5 }, { 22105, 2 }, - { 22158, 3 }, { 22197, 2 }, { 22254, 1 }, { 22353, 2 }, { 22404, 4 }, { 22422, 2 }, { 22569, 2 }, { 22634, 2 }, { 22639, 2 }, { 22861, 2 }, { 22868, 2 }, { 22876, 1 }, { 22902, 2 }, { 22925, 2 }, { 23080, 2 }, { 23085, 3 }, { 23089, 5 }, { 23329, 1 }, { 23349, 2 }, { 23559, 5 }, { 23567, 3 }, { 23574, 2 }, { 23584, 3 }, - { 23615, 3 }, { 23633, 2 }, { 23674, 2 }, { 23678, 1 }, { 23853, 2 }, { 23875, 2 }, { 24010, 4 }, { 24076, 2 }, { 24128, 6 }, { 24248, 2 }, { 24253, 2 }, { 24259, 1 }, { 24319, 2 }, { 24319, 1 }, { 24502, 3 }, { 24666, 2 }, { 24781, 3 }, { 24792, 2 }, { 24909, 2 }, { 24993, 2 }, { 25039, 1 }, { 25090, 3 }, { 25137, 1 }, - { 25138, 3 }, { 25140, 3 }, { 25155, 5 }, { 25411, 2 }, { 25460, 2 }, { 25564, 3 }, { 25586, 3 }, { 25630, 2 }, { 25765, 2 }, { 25789, 3 }, { 25803, 2 }, { 25851, 2 }, { 25872, 2 }, { 25887, 2 }, { 25981, 1 }, { 26016, 2 }, { 26019, 1 }, { 26029, 1 }, { 26104, 7 }, { 26144, 2 }, { 26275, 1 }, { 26295, 2 }, { 26298, 1 }, - { 26322, 2 }, { 26380, 2 }, { 26408, 4 }, { 26446, 1 }, { 26553, 1 }, { 26576, 1 }, { 26635, 1 }, { 26668, 2 }, { 26675, 2 }, { 26698, 4 }, { 26748, 9 }, { 26788, 2 }, { 26932, 2 }, { 26962, 2 }, { 27042, 2 }, { 27060, 2 }, { 27163, 3 }, { 27202, 2 }, { 27290, 2 }, { 27337, 3 }, { 27376, 2 }, { 27439, 2 }, { 27458, 4 }, - { 27515, 2 }, { 27518, 1 }, { 27541, 2 }, { 27585, 3 }, { 27633, 2 }, { 27695, 2 }, { 27702, 2 }, { 27861, 2 }, { 27924, 1 }, { 28025, 14 }, { 28058, 2 }, { 28143, 2 }, { 28215, 2 }, { 28240, 2 }, { 28241, 2 }, { 28285, 2 }, { 28324, 3 }, { 28378, 2 }, { 28514, 2 }, { 28529, 2 }, { 28538, 2 }, { 28565, 3 }, { 28697, 2 }, - { 28735, 2 }, { 28769, 2 }, { 28770, 4 }, { 28788, 4 }, { 28807, 3 }, { 28807, 4 }, { 28829, 1 }, { 28853, 2 }, { 28856, 7 }, { 28864, 2 }, { 28865, 3 }, { 28915, 2 }, { 28928, 2 }, { 28964, 2 }, { 28988, 1 }, { 29031, 2 }, { 29095, 2 }, { 29189, 2 }, { 29205, 1 }, { 29230, 1 }, { 29332, 2 }, { 29339, 2 }, { 29349, 5 }, - { 29449, 2 }, { 29471, 2 }, { 29578, 2 }, { 29859, 2 }, { 29878, 2 }, { 29947, 10 }, { 30083, 2 }, { 30121, 2 }, { 30128, 2 }, { 30155, 4 }, { 30157, 1 }, { 30272, 2 }, { 30281, 2 }, { 30286, 2 }, { 30305, 2 }, { 30408, 2 }, { 30444, 22 }, { 30612, 2 }, { 30628, 2 }, { 30747, 2 }, { 30783, 2 }, { 30808, 5 }, { 30868, 3 }, - { 30875, 2 }, { 30997, 2 }, { 31000, 2 }, { 31022, 3 }, { 31111, 1 }, { 31144, 2 }, { 31146, 3 }, { 31187, 2 }, { 31324, 2 }, { 31343, 2 }, { 31416, 2 }, { 31485, 2 }, { 31539, 2 }, { 31638, 2 }, { 31648, 2 }, { 31750, 2 }, { 31754, 2 }, { 31785, 10 }, { 31786, 5 }, { 31800, 2 }, { 31801, 4 }, { 31807, 7 }, { 31807, 3 }, - { 31807, 10 }, { 31808, 3 }, { 31808, 4 }, { 31818, 6 }, { 31825, 7 }, { 31838, 2 }, { 31911, 1 }, { 31974, 2 }, { 32010, 3 }, { 32031, 2 }, { 32040, 2 }, { 32063, 1 }, { 32078, 2 }, { 32156, 2 }, { 32198, 31 }, { 32257, 2 }, { 32257, 2 }, { 32265, 2 }, { 32330, 2 }, { 32369, 8 }, { 32404, 3 }, { 32425, 2 }, { 32432, 2 }, - { 32505, 2 }, { 32531, 2 }, { 32536, 2 }, { 32549, 2 }, { 32582, 3 }, { 32590, 4 }, { 32624, 2 }, { 32644, 2 }, { 32692, 2 }, { 32695, 4 }, { 32699, 3 }, { 32726, 4 }, { 32784, 2 }, { 32832, 2 }, { 32883, 6 }, { 32965, 4 }, { 33044, 2 }, { 33104, 2 }, { 33184, 2 }, { 33264, 1 }, { 33292, 2 }, { 33312, 1 }, { 33468, 2 }, - { 33471, 1 }, { 33565, 2 }, { 33627, 2 }, { 33659, 2 }, { 33709, 2 }, { 33766, 5 }, { 33836, 2 }, { 33875, 2 }, { 33954, 2 }, { 33959, 2 }, { 34050, 2 }, { 34090, 2 }, { 34168, 2 }, { 34233, 2 }, { 34461, 2 }, { 34462, 1 }, { 34463, 2 }, { 34472, 4 }, { 34500, 2 }, { 34520, 2 }, { 34544, 2 }, { 34614, 2 }, { 34662, 1 }, - { 34676, 2 }, { 34729, 4 }, { 34803, 2 }, { 34845, 2 }, { 34913, 2 }, { 34963, 6 }, { 35019, 2 }, { 35022, 2 }, { 35070, 2 }, { 35120, 2 }, { 35132, 2 }, { 35144, 2 }, { 35205, 2 }, { 35230, 3 }, { 35244, 2 }, { 35271, 4 }, { 35276, 2 }, { 35282, 2 }, { 35324, 3 }, { 35366, 3 }, { 35659, 2 }, { 35680, 2 }, { 35744, 2 }, - { 35758, 3 }, { 35796, 2 }, { 35830, 2 }, { 35841, 7 }, { 35843, 2 }, { 35856, 2 }, { 35914, 4 }, { 35929, 13 }, { 35993, 2 }, { 35997, 1 }, { 36046, 4 }, { 36046, 1 }, { 36051, 1 }, { 36111, 2 }, { 36208, 1 }, { 36208, 1 }, { 36306, 2 }, { 36325, 2 }, { 36386, 2 }, { 36405, 2 }, { 36443, 1 }, { 36455, 1 }, { 36538, 2 }, - { 36562, 2 }, { 36566, 2 }, { 36628, 2 }, { 36693, 2 }, { 36713, 2 }, { 36730, 2 }, { 36747, 2 }, { 36786, 2 }, { 36810, 1 }, { 36848, 2 }, { 36914, 1 }, { 36920, 2 }, { 36952, 2 }, { 37071, 2 }, { 37086, 1 }, { 37094, 3 }, { 37158, 3 }, { 37231, 2 }, { 37241, 2 }, { 37285, 2 }, { 37349, 2 }, { 37404, 2 }, { 37410, 1 }, - { 37433, 4 }, { 37615, 2 }, { 37659, 2 }, { 37742, 2 }, { 37773, 2 }, { 37867, 1 }, { 37890, 2 }, { 37960, 2 }, { 38042, 3 }, { 38241, 2 }, { 38400, 2 }, { 38461, 1 }, { 38551, 2 }, { 38611, 1 }, { 38657, 2 }, { 38729, 2 }, { 38748, 2 }, { 38815, 2 }, { 38852, 2 }, { 38890, 1 }, { 38954, 2 }, { 39119, 2 }, { 39162, 2 }, - { 39175, 3 }, { 39176, 2 }, { 39231, 2 }, { 39261, 2 }, { 39467, 2 }, { 39500, 2 }, { 39507, 2 }, { 39566, 2 }, { 39608, 2 }, { 39686, 6 }, { 39730, 2 }, { 39842, 1 }, { 39853, 1 }, { 39905, 2 }, { 39931, 2 }, { 39989, 2 }, { 40030, 2 }, { 40227, 2 }, { 40268, 2 }, { 40372, 2 }, { 40415, 1 }, { 40488, 3 }, { 40536, 2 }, - { 40676, 3 }, { 40677, 2 }, { 40755, 2 }, { 40842, 2 }, { 40849, 1 }, { 40870, 3 }, { 40873, 3 }, { 40972, 2 }, { 41033, 2 }, { 41190, 2 }, { 41273, 5 }, { 41273, 1 }, { 41293, 2 }, { 41367, 32 }, { 41376, 2 }, { 41420, 2 }, { 41473, 2 }, { 41473, 2 }, { 41493, 4 }, { 41521, 2 }, { 41533, 2 }, { 41554, 2 }, { 41568, 2 }, - { 41583, 3 }, { 41728, 2 }, { 41786, 2 }, { 41836, 1 }, { 41875, 2 }, { 41933, 2 }, { 42044, 2 }, { 42075, 2 }, { 42076, 2 }, { 42133, 2 }, { 42259, 29 }, { 42269, 3 }, { 42294, 2 }, { 42420, 2 }, { 42524, 2 }, { 42524, 1 }, { 42546, 1 }, { 42631, 2 }, { 42693, 2 }, { 42740, 2 }, { 42744, 4 }, { 42755, 1 }, { 42870, 2 }, - { 42894, 2 }, { 42939, 2 }, { 42973, 2 }, { 43016, 2 }, { 43070, 2 }, { 43105, 2 }, { 43115, 2 }, { 43375, 3 }, { 43387, 1 }, { 43424, 3 }, { 43448, 2 }, { 43480, 2 }, { 43498, 2 }, { 43651, 2 }, { 43727, 2 }, { 43879, 2 }, { 43910, 1 }, { 43977, 2 }, { 44003, 2 }, { 44080, 2 }, { 44082, 1 }, { 44136, 2 }, { 44169, 29 }, - { 44186, 2 }, { 44339, 2 }, { 44350, 1 }, { 44356, 1 }, { 44430, 2 }, { 44440, 1 }, { 44530, 1 }, { 44538, 2 }, { 44572, 2 }, { 44585, 2 }, { 44709, 2 }, { 44748, 2 }, { 44748, 2 }, { 44769, 2 }, { 44813, 2 }, { 44890, 2 }, { 45015, 2 }, { 45046, 4 }, { 45052, 2 }, { 45062, 2 }, { 45094, 6 }, { 45184, 2 }, { 45191, 2 }, - { 45201, 3 }, { 45216, 3 }, { 45227, 2 }, { 45269, 1 }, { 45294, 2 }, { 45314, 2 }, { 45345, 8 }, { 45352, 2 }, { 45365, 3 }, { 45378, 1 }, { 45392, 4 }, { 45405, 3 }, { 45410, 2 }, { 45448, 14 }, { 45450, 2 }, { 45457, 2 }, { 45466, 3 }, { 45481, 4 }, { 45486, 7 }, { 45533, 5 }, { 45576, 2 }, { 45649, 2 }, { 45917, 2 }, - { 45919, 6 }, { 45919, 1 }, { 45930, 15 }, { 45930, 2 }, { 46001, 5 }, { 46036, 2 }, { 46054, 2 }, { 46075, 2 }, { 46153, 2 }, { 46155, 2 }, { 46228, 2 }, { 46234, 2 }, { 46273, 2 }, { 46387, 2 }, { 46398, 2 }, { 46517, 2 }, { 46559, 2 }, { 46565, 1 }, { 46598, 2 }, { 46686, 2 }, { 46744, 2 }, { 46816, 3 }, { 46835, 2 }, - { 46921, 2 }, { 46938, 2 }, { 46991, 2 }, { 47038, 2 }, { 47098, 3 }, { 47107, 2 }, { 47201, 3 }, { 47327, 1 }, { 47327, 1 }, { 47338, 2 }, { 47395, 1 }, { 47499, 2 }, { 47504, 2 }, { 47515, 1 }, { 47516, 1 }, { 47600, 1 }, { 47604, 1 }, { 47707, 1 }, { 47728, 1 }, { 47748, 2 }, { 47763, 2 }, { 47807, 4 }, { 47814, 2 }, - { 47822, 2 }, { 47834, 2 }, { 47843, 3 }, { 47886, 2 }, { 47893, 2 }, { 48066, 2 }, { 48126, 2 }, { 48133, 1 }, { 48166, 2 }, { 48299, 1 }, { 48455, 2 }, { 48468, 2 }, { 48568, 2 }, { 48606, 2 }, { 48642, 2 }, { 48698, 2 }, { 48714, 2 }, { 48754, 2 }, { 48765, 3 }, { 48773, 5 }, { 48819, 2 }, { 48833, 2 }, { 48904, 2 }, - { 49000, 1 }, { 49113, 12 }, { 49140, 2 }, { 49276, 2 }, { 49353, 2 }, { 49411, 3 }, { 49418, 2 }, { 49540, 2 }, { 49544, 2 }, { 49584, 2 }, { 49602, 2 }, { 49784, 5 }, { 49822, 4 }, { 49822, 5 }, { 49828, 2 }, { 49866, 2 }, { 49922, 3 }, { 49959, 2 }, { 50045, 2 }, { 50134, 3 }, { 50140, 2 }, { 50237, 2 }, { 50247, 2 }, - { 50266, 13 }, { 50290, 2 }, { 50312, 4 }, { 50314, 1 }, { 50527, 2 }, { 50605, 1 }, { 50730, 2 }, { 50751, 2 }, { 50770, 2 }, { 50858, 2 }, { 50859, 2 }, { 50909, 2 }, { 50948, 3 }, { 51043, 2 }, { 51048, 2 }, { 51089, 2 }, { 51090, 2 }, { 51141, 2 }, { 51163, 2 }, { 51250, 2 }, { 51347, 2 }, { 51475, 2 }, { 51536, 2 }, - { 51544, 2 }, { 51595, 2 }, { 51602, 19 }, { 51643, 5 }, { 51702, 2 }, { 51702, 10 }, { 51764, 2 }, { 51793, 5 }, { 51812, 2 }, { 51839, 1 }, { 51938, 3 }, { 51941, 1 }, { 51967, 4 }, { 52049, 3 }, { 52074, 3 }, { 52098, 2 }, { 52118, 2 }, { 52119, 3 }, { 52227, 11 }, { 52246, 3 }, { 52282, 2 }, { 52451, 2 }, { 52583, 2 }, - { 52601, 1 }, { 52605, 2 }, { 52615, 2 }, { 52668, 2 }, { 52824, 2 }, { 53076, 1 }, { 53120, 1 }, { 53179, 2 }, { 53189, 2 }, { 53193, 1 }, { 53195, 2 }, { 53246, 2 }, { 53249, 2 }, { 53268, 1 }, { 53295, 2 }, { 53312, 2 }, { 53410, 2 }, { 53451, 2 }, { 53570, 2 }, { 53593, 2 }, { 53635, 2 }, { 53657, 2 }, { 53682, 3 }, - { 53728, 5 }, { 53733, 2 }, { 53753, 2 }, { 53787, 4 }, { 53807, 1 }, { 54008, 2 }, { 54059, 2 }, { 54060, 1 }, { 54080, 2 }, { 54090, 1 }, { 54138, 2 }, { 54149, 2 }, { 54168, 1 }, { 54171, 2 }, { 54216, 22 }, { 54233, 6 }, { 54434, 2 }, { 54534, 2 }, { 54562, 2 }, { 54763, 2 }, { 54791, 2 }, { 54816, 2 }, { 54909, 2 }, - { 54916, 3 }, { 54963, 2 }, { 54985, 2 }, { 54991, 3 }, { 55016, 3 }, { 55025, 3 }, { 55032, 2 }, { 55099, 2 }, { 55260, 2 }, { 55261, 2 }, { 55270, 3 }, { 55384, 2 }, { 55455, 2 }, { 55456, 2 }, { 55504, 3 }, { 55510, 2 }, { 55558, 2 }, { 55568, 2 }, { 55585, 2 }, { 55677, 2 }, { 55703, 2 }, { 55749, 2 }, { 55779, 2 }, - { 55789, 3 }, { 55792, 2 }, { 55830, 4 }, { 55835, 2 }, { 55879, 2 }, { 56076, 2 }, { 56118, 2 }, { 56314, 2 }, { 56392, 1 }, { 56411, 2 }, { 56459, 2 }, { 56553, 34 }, { 56575, 2 }, { 56733, 2 }, { 56762, 2 }, { 56793, 3 }, { 56877, 3 }, { 56927, 2 }, { 56981, 2 }, { 57014, 1 }, { 57149, 2 }, { 57162, 2 }, { 57186, 2 }, - { 57254, 2 }, { 57267, 1 }, { 57324, 2 }, { 57327, 2 }, { 57365, 4 }, { 57371, 2 }, { 57445, 2 }, { 57477, 2 }, { 57497, 2 }, { 57536, 2 }, { 57609, 2 }, { 57626, 2 }, { 57666, 2 }, { 57694, 2 }, { 57694, 2 }, { 57749, 2 }, { 57781, 7 }, { 57878, 2 }, { 57953, 2 }, { 58051, 2 }, { 58088, 2 }, { 58097, 2 }, { 58142, 3 }, - { 58142, 1 }, { 58197, 1 }, { 58221, 2 }, { 58222, 2 }, { 58244, 2 }, { 58290, 1 }, { 58296, 1 }, { 58325, 2 }, { 58378, 1 }, { 58389, 3 }, { 58430, 2 }, { 58454, 2 }, { 58551, 29 }, { 58563, 6 }, { 58681, 2 }, { 58751, 8 }, { 58752, 43 }, { 58790, 5 }, { 58846, 2 }, { 58879, 6 }, { 58953, 2 }, { 58998, 2 }, { 59010, 1 }, - { 59038, 5 }, { 59135, 2 }, { 59166, 2 }, { 59180, 2 }, { 59222, 2 }, { 59227, 2 }, { 59307, 2 }, { 59398, 3 }, { 59411, 2 }, { 59436, 3 }, { 59464, 2 }, { 59569, 2 }, { 59587, 2 }, { 59624, 3 }, { 59786, 2 }, { 59834, 2 }, { 59841, 2 }, { 59841, 1 }, { 59984, 2 }, { 59985, 2 }, { 60003, 3 }, { 60045, 2 }, { 60097, 2 }, - { 60148, 2 }, { 60172, 2 }, { 60203, 5 }, { 60565, 2 }, { 60625, 2 }, { 60743, 2 }, { 60781, 2 }, { 60892, 2 }, { 60977, 2 }, { 60979, 2 }, { 61021, 5 }, { 61021, 4 }, { 61026, 2 }, { 61139, 2 }, { 61165, 3 }, { 61204, 2 }, { 61207, 1 }, { 61248, 3 }, { 61257, 2 }, { 61264, 6 }, { 61272, 3 }, { 61410, 2 }, { 61410, 3 }, - { 61416, 2 }, { 61423, 1 }, { 61503, 2 }, { 61503, 2 }, { 61533, 2 }, { 61567, 2 }, { 61575, 2 }, { 61835, 1 }, { 61842, 1 }, { 61924, 2 }, { 61951, 6 }, { 61975, 2 }, { 61986, 3 }, { 62024, 1 }, { 62110, 2 }, { 62135, 2 }, { 62192, 2 }, { 62208, 2 }, { 62399, 2 }, { 62400, 1 }, { 62414, 2 }, { 62423, 3 }, { 62456, 3 }, - { 62459, 3 }, { 62478, 3 }, { 62484, 2 }, { 62510, 6 }, { 62511, 3 }, { 62565, 3 }, { 62610, 2 }, { 62875, 4 }, { 62896, 5 }, { 62898, 2 }, { 62904, 2 }, { 62938, 3 }, { 62943, 2 }, { 62977, 2 }, { 62989, 3 }, { 62998, 5 }, { 63069, 1 }, { 63093, 5 }, { 63107, 2 }, { 63113, 1 }, { 63231, 4 }, { 63253, 2 }, { 63286, 4 }, - { 63289, 2 }, { 63334, 1 }, { 63334, 4 }, { 63413, 2 }, { 63425, 2 }, { 63512, 10 }, { 63537, 1 }, { 63694, 1 }, { 63721, 4 }, { 63749, 2 }, { 63783, 17 }, { 63791, 3 }, { 63792, 2 }, { 63882, 25 }, { 63896, 1 }, { 63936, 2 }, { 63969, 3 }, { 63986, 2 }, { 63988, 2 }, { 64009, 10 }, { 64018, 2 }, { 64032, 6 }, { 64125, 2 }, - { 64195, 1 }, { 64221, 7 }, { 64390, 2 }, { 64459, 2 }, { 64568, 2 }, { 64784, 1 }, { 64789, 2 }, { 64829, 2 }, { 64848, 1 }, { 64914, 2 }, { 64928, 1 }, { 64939, 2 }, { 65026, 2 }, { 65057, 2 }, { 65070, 2 }, { 65193, 4 }, { 65235, 3 }, { 65242, 2 }, { 65281, 2 }, { 65320, 2 }, { 65365, 1 }, { 65414, 2 }, { 65445, 2 }, - { 65581, 2 }, { 65624, 1 }, { 65719, 2 }, { 65766, 2 }, { 65927, 2 }, { 66004, 1 }, { 66031, 2 }, { 66085, 1 }, { 66085, 2 }, { 66133, 2 }, { 66134, 2 }, { 66188, 1 }, { 66240, 2 }, { 66249, 2 }, { 66250, 2 }, { 66295, 2 }, { 66342, 1 }, { 66352, 3 }, { 66388, 3 }, { 66432, 2 }, { 66437, 47 }, { 66497, 2 }, { 66517, 2 }, - { 66526, 2 }, { 66546, 9 }, { 66605, 2 }, { 66753, 2 }, { 66792, 2 }, { 66796, 2 }, { 66828, 2 }, { 66899, 3 }, { 66970, 6 }, { 66981, 2 }, { 66983, 1 }, { 67009, 2 }, { 67017, 4 }, { 67115, 2 }, { 67117, 1 }, { 67130, 6 }, { 67132, 7 }, { 67162, 2 }, { 67179, 6 }, { 67236, 2 }, { 67263, 3 }, { 67274, 2 }, { 67274, 2 }, - { 67349, 3 }, { 67486, 2 }, { 67503, 3 }, { 67517, 1 }, { 67559, 1 }, { 67660, 2 }, { 67727, 2 }, { 67901, 2 }, { 67943, 4 }, { 67950, 2 }, { 67965, 3 }, { 68029, 2 }, { 68048, 2 }, { 68169, 2 }, { 68172, 1 }, { 68258, 2 }, { 68288, 1 }, { 68359, 2 }, { 68441, 2 }, { 68484, 2 }, { 68488, 2 }, { 68525, 2 }, { 68535, 2 }, - { 68575, 7 }, { 68575, 5 }, { 68583, 2 }, { 68588, 4 }, { 68593, 1 }, { 68597, 2 }, { 68636, 2 }, { 68636, 2 }, { 68667, 2 }, { 68785, 1 }, { 68914, 4 }, { 68915, 5 }, { 68940, 3 }, { 69010, 2 }, { 69063, 2 }, { 69076, 2 }, { 69235, 2 }, { 69270, 2 }, { 69298, 1 }, { 69350, 5 }, { 69432, 2 }, { 69514, 2 }, { 69562, 3 }, - { 69562, 4 }, { 69638, 1 }, { 69656, 2 }, { 69709, 2 }, { 69775, 2 }, { 69788, 2 }, { 70193, 2 }, { 70233, 2 }, { 70252, 2 }, { 70259, 2 }, { 70293, 3 }, { 70405, 3 }, { 70462, 2 }, { 70515, 3 }, { 70518, 2 }, { 70535, 6 }, { 70547, 6 }, { 70577, 6 }, { 70631, 17 }, { 70667, 2 }, { 70680, 1 }, { 70694, 1 }, { 70898, 2 }, - { 70916, 1 }, { 70936, 3 }, { 71033, 2 }, { 71126, 2 }, { 71158, 2 }, { 71162, 2 }, { 71421, 1 }, { 71441, 2 }, { 71557, 2 }, { 71789, 1 }, { 71816, 2 }, { 71850, 1 }, { 71869, 1 }, { 71961, 2 }, { 71973, 4 }, { 72064, 2 }, { 72110, 2 }, { 72117, 3 }, { 72164, 2 }, { 72266, 2 }, { 72325, 2 }, { 72326, 1 }, { 72420, 2 }, - { 72693, 2 }, { 72705, 1 }, { 72730, 2 }, { 72793, 2 }, { 72795, 1 }, { 72939, 1 }, { 72945, 3 }, { 72945, 2 }, { 73120, 1 }, { 73121, 5 }, { 73122, 4 }, { 73126, 1 }, { 73126, 1 }, { 73196, 3 }, { 73219, 2 }, { 73241, 6 }, { 73272, 3 }, { 73354, 1 }, { 73368, 2 }, { 73467, 1 }, { 73517, 2 }, { 73554, 2 }, { 73678, 2 }, - { 73838, 1 }, { 73881, 2 }, { 73958, 2 }, { 73985, 15 }, { 74092, 2 }, { 74205, 2 }, { 74245, 2 }, { 74277, 2 }, { 74286, 2 }, { 74353, 2 }, { 74403, 2 }, { 74428, 1 }, { 74468, 2 }, { 74481, 3 }, { 74511, 2 }, { 74537, 2 }, { 74596, 2 }, { 74750, 2 }, { 74754, 2 }, { 74861, 2 }, { 74933, 4 }, { 74970, 1 }, { 75003, 3 }, - { 75077, 1 }, { 75159, 2 }, { 75170, 2 }, { 75234, 2 }, { 75300, 3 }, { 75337, 2 }, { 75345, 2 }, { 75419, 1 }, { 75429, 2 }, { 75477, 1 }, { 75513, 2 }, { 75536, 2 }, { 75536, 2 }, { 75539, 1 }, { 75551, 2 }, { 75561, 2 }, { 75565, 2 }, { 75590, 2 }, { 75623, 5 }, { 75773, 6 }, { 75777, 6 }, { 75785, 2 }, { 75791, 2 }, - { 75804, 2 }, { 75862, 2 }, { 75924, 3 }, { 75927, 2 }, { 75996, 11 }, { 76000, 1 }, { 76006, 2 }, { 76020, 3 }, { 76110, 2 }, { 76126, 3 }, { 76131, 2 }, { 76136, 2 }, { 76144, 2 }, { 76203, 2 }, { 76229, 3 }, { 76244, 15 }, { 76246, 2 }, { 76300, 1 }, { 76403, 3 }, { 76545, 2 }, { 76569, 2 }, { 76813, 2 }, { 76821, 2 }, - { 76837, 2 }, { 76863, 2 }, { 77027, 2 }, { 77037, 2 }, { 77074, 3 }, { 77170, 2 }, { 77191, 2 }, { 77220, 2 }, { 77230, 2 }, { 77261, 2 }, { 77277, 2 }, { 77309, 2 }, { 77314, 2 }, { 77412, 2 }, { 77419, 2 }, { 77457, 2 }, { 77633, 3 }, { 77714, 2 }, { 77855, 2 }, { 77857, 1 }, { 77876, 2 }, { 77895, 2 }, { 77916, 5 }, - { 77947, 2 }, { 77948, 1 }, { 77966, 1 }, { 77996, 2 }, { 78025, 1 }, { 78064, 2 }, { 78100, 2 }, { 78113, 1 }, { 78114, 3 }, { 78167, 2 }, { 78175, 2 }, { 78260, 2 }, { 78261, 1 }, { 78265, 2 }, { 78286, 1 }, { 78300, 2 }, { 78327, 3 }, { 78363, 1 }, { 78384, 2 }, { 78459, 2 }, { 78516, 2 }, { 78612, 2 }, { 78643, 2 }, - { 78655, 2 }, { 78698, 1 }, { 78720, 3 }, { 78789, 3 }, { 78838, 5 }, { 78893, 1 }, { 78954, 7 }, { 79007, 2 }, { 79132, 3 }, { 79193, 2 }, { 79193, 2 }, { 79226, 2 }, { 79411, 2 }, { 79422, 1 }, { 79502, 2 }, { 79593, 2 }, { 79622, 2 }, { 79657, 3 }, { 79771, 2 }, { 79866, 2 }, { 79909, 2 }, { 80005, 2 }, { 80032, 2 }, - { 80060, 1 }, { 80132, 2 }, { 80149, 3 }, { 80251, 2 }, { 80363, 2 }, { 80379, 1 }, { 80464, 2 }, { 80498, 2 }, { 80553, 2 }, { 80556, 3 }, { 80559, 1 }, { 80571, 2 }, { 80652, 1 }, { 80703, 2 }, { 80754, 2 }, { 80754, 2 }, { 80860, 2 }, { 81055, 2 }, { 81087, 4 }, { 81210, 2 }, { 81211, 1 }, { 81216, 1 }, { 81223, 1 }, - { 81231, 1 }, { 81288, 2 }, { 81317, 2 }, { 81327, 3 }, { 81332, 2 }, { 81376, 2 }, { 81469, 2 }, { 81579, 2 }, { 81617, 1 }, { 81630, 2 }, { 81666, 2 }, { 81800, 2 }, { 81832, 2 }, { 81848, 2 }, { 81869, 2 }, { 81941, 3 }, { 82177, 3 }, { 82179, 2 }, { 82180, 2 }, { 82182, 4 }, { 82185, 2 }, { 82195, 2 }, { 82238, 4 }, - { 82265, 3 }, { 82295, 10 }, { 82299, 9 }, { 82367, 3 }, { 82379, 3 }, { 82380, 1 }, { 82505, 2 }, { 82568, 2 }, { 82620, 1 }, { 82637, 5 }, { 82821, 2 }, { 82841, 2 }, { 82945, 1 }, { 83020, 12 }, { 83072, 2 }, { 83181, 2 }, { 83240, 2 }, { 83253, 3 }, { 83261, 2 }, { 83288, 2 }, { 83291, 4 }, { 83295, 3 }, { 83365, 2 }, - { 83368, 2 }, { 83408, 2 }, { 83458, 2 }, { 83470, 2 }, { 83471, 1 }, { 83637, 3 }, { 83693, 2 }, { 83703, 2 }, { 83732, 2 }, { 83745, 1 }, { 83800, 4 }, { 83801, 3 }, { 83856, 3 }, { 83863, 5 }, { 83867, 2 }, { 83868, 3 }, { 83898, 7 }, { 83900, 4 }, { 83901, 5 }, { 83989, 2 }, { 84049, 35 }, { 84086, 2 }, { 84089, 2 }, - { 84115, 3 }, { 84130, 3 }, { 84132, 2 }, { 84143, 3 }, { 84173, 2 }, { 84185, 5 }, { 84297, 2 }, { 84390, 2 }, { 84497, 4 }, { 84657, 2 }, { 84657, 2 }, { 84724, 2 }, { 84775, 2 }, { 84870, 2 }, { 84892, 2 }, { 84910, 3 }, { 84935, 3 }, { 85002, 2 }, { 85051, 2 }, { 85052, 2 }, { 85135, 25 }, { 85135, 2 }, { 85144, 2 }, - { 85165, 3 }, { 85205, 2 }, { 85232, 2 }, { 85281, 5 }, { 85423, 6 }, { 85539, 2 }, { 85582, 4 }, { 85609, 2 }, { 85701, 36 }, { 85705, 2 }, { 85824, 2 }, { 85824, 2 }, { 85858, 30 }, { 85858, 28 }, { 85904, 35 }, { 85910, 2 }, { 85913, 2 }, { 85926, 3 }, { 85942, 4 }, { 85969, 4 }, { 85996, 1 }, { 86013, 3 }, { 86034, 13 }, - { 86068, 8 }, { 86069, 8 }, { 86089, 8 }, { 86193, 13 }, { 86217, 7 }, { 86219, 2 }, { 86250, 2 }, { 86304, 16 }, { 86317, 2 }, { 86322, 4 }, { 86325, 2 }, { 86333, 2 }, { 86394, 2 }, { 86433, 2 }, { 86469, 3 }, { 86512, 4 }, { 86537, 2 }, { 86627, 2 }, { 86658, 2 }, { 86810, 2 }, { 86813, 2 }, { 86884, 2 }, { 86947, 2 }, - { 87003, 2 }, { 87010, 5 }, { 87019, 2 }, { 87027, 2 }, { 87105, 2 }, { 87107, 2 }, { 87183, 2 }, { 87273, 2 }, { 87358, 3 }, { 87388, 3 }, { 87503, 4 }, { 87639, 2 }, { 87649, 4 }, { 87722, 2 }, { 87829, 2 }, { 87829, 1 }, { 87863, 2 }, { 87894, 2 }, { 87988, 32 }, { 88035, 27 }, { 88059, 3 }, { 88094, 5 }, { 88111, 21 }, - { 88129, 2 }, { 88175, 5 }, { 88256, 2 }, { 88329, 2 }, { 88415, 3 }, { 88482, 2 }, { 88502, 1 }, { 88529, 2 }, { 88551, 3 }, { 88552, 1 }, { 88713, 2 }, { 88797, 2 }, { 88844, 27 }, { 88925, 5 }, { 88935, 2 }, { 88944, 1 }, { 89073, 2 }, { 89095, 3 }, { 89283, 2 }, { 89294, 3 }, { 89299, 2 }, { 89324, 2 }, { 89368, 2 }, - { 89387, 2 }, { 89464, 2 }, { 89607, 2 }, { 89737, 2 }, { 89791, 2 }, { 89794, 3 }, { 89840, 2 }, { 89849, 3 }, { 89859, 2 }, { 89905, 2 }, { 89952, 38 }, { 90030, 7 }, { 90030, 6 }, { 90031, 1 }, { 90072, 2 }, { 90090, 2 }, { 90146, 3 }, { 90202, 23 }, { 90302, 3 }, { 90328, 14 }, { 90335, 14 }, { 90338, 8 }, { 90380, 2 }, - { 90434, 1 }, { 90482, 2 }, { 90527, 9 }, { 90537, 3 }, { 90545, 2 }, { 90639, 5 }, { 90642, 2 }, { 90709, 2 }, { 90775, 1 }, { 90806, 2 }, { 90845, 19 }, { 90872, 4 }, { 90884, 2 }, { 90910, 2 }, { 90994, 5 }, { 91046, 8 }, { 91059, 8 }, { 91096, 39 }, { 91147, 2 }, { 91168, 1 }, { 91493, 2 }, { 91513, 3 }, { 91618, 3 }, - { 91653, 2 }, { 91817, 2 }, { 91831, 3 }, { 91833, 3 }, { 91885, 2 }, { 91919, 2 }, { 91934, 2 }, { 92245, 1 }, { 92284, 2 }, { 92292, 4 }, { 92369, 3 }, { 92388, 2 }, { 92426, 7 }, { 92720, 14 }, { 92720, 6 }, { 92729, 9 }, { 92733, 13 }, { 92735, 6 }, { 92786, 2 }, { 92853, 31 }, { 92906, 2 }, { 93031, 7 }, { 93077, 2 }, - { 93102, 2 }, { 93109, 2 }, { 93122, 3 }, { 93214, 2 }, { 93330, 2 }, { 93395, 2 }, { 93506, 2 }, { 93564, 9 }, { 93713, 9 }, { 93722, 4 }, { 93840, 2 }, { 93877, 4 }, { 93891, 3 }, { 93948, 2 }, { 93981, 2 }, { 94012, 3 }, { 94033, 2 }, { 94121, 2 }, { 94165, 32 }, { 94181, 3 }, { 94210, 2 }, { 94216, 2 }, { 94230, 2 }, - { 94333, 31 }, { 94433, 3 }, { 94497, 3 }, { 94609, 2 }, { 94623, 2 }, { 94763, 2 }, { 94780, 2 }, { 95287, 2 }, { 95348, 2 }, { 95433, 5 }, { 95446, 2 }, { 95493, 7 }, { 95517, 3 }, { 95580, 2 }, { 95610, 5 }, { 95620, 2 }, { 95678, 3 }, { 95683, 2 }, { 95689, 2 }, { 95760, 2 }, { 95792, 2 }, { 95850, 2 }, { 95908, 2 }, - { 95908, 2 }, { 95967, 2 }, { 96022, 3 }, { 96088, 2 }, { 96460, 2 }, { 96554, 2 }, { 96597, 2 }, { 96763, 2 }, { 96808, 2 }, { 96854, 1 }, { 96963, 1 }, { 97007, 3 }, { 97125, 1 }, { 97128, 2 }, { 97133, 3 }, { 97142, 3 }, { 97156, 2 }, { 97223, 2 }, { 97244, 2 }, { 97303, 2 }, { 97355, 2 }, { 97356, 3 }, { 97393, 3 }, - { 97409, 1 }, { 97451, 2 }, { 97539, 2 }, { 97546, 2 }, { 97553, 2 }, { 97627, 2 }, { 97640, 2 }, { 97650, 6 }, { 97675, 2 }, { 97685, 3 }, { 97773, 2 }, { 97802, 4 }, { 97826, 19 }, { 97860, 2 }, { 97956, 2 }, { 97958, 2 }, { 97973, 3 }, { 97982, 2 }, { 98039, 2 }, { 98051, 2 }, { 98059, 2 }, { 98088, 2 }, { 98092, 4 }, - { 98147, 2 }, { 98147, 2 }, { 98169, 2 }, { 98207, 2 }, { 98277, 1 }, { 98277, 22 }, { 98285, 2 }, { 98324, 3 }, { 98324, 3 }, { 98381, 31 }, { 98390, 2 }, { 98404, 2 }, { 98415, 4 }, { 98460, 2 }, { 98462, 1 }, { 98475, 3 }, { 98485, 2 }, { 98640, 1 }, { 98798, 2 }, { 98800, 4 }, { 98821, 2 }, { 98895, 2 }, { 98936, 2 }, - { 98950, 2 }, { 98980, 2 }, { 99033, 2 }, { 99045, 2 }, { 99135, 2 }, { 99315, 30 }, { 99324, 2 }, { 99346, 2 }, { 99418, 2 }, { 99505, 2 }, { 99557, 2 }, { 99559, 2 }, { 99586, 2 }, { 99622, 2 }, { 99770, 1 }, { 99790, 2 }, { 99810, 2 }, { 99871, 1 }, { 99926, 2 }, { 99927, 2 }, { 99978, 2 }, { 99980, 2 }, { 100022, 3 }, - { 100024, 1 }, { 100069, 2 }, { 100150, 2 }, { 100225, 2 }, { 100246, 1 }, { 100310, 2 }, { 100361, 2 }, { 100428, 1 }, { 100434, 2 }, { 100450, 4 }, { 100546, 2 }, { 100551, 2 }, { 100551, 2 }, { 100554, 1 }, { 100597, 2 }, { 100676, 2 }, { 100693, 2 }, { 100827, 2 }, { 100928, 2 }, { 100928, 1 }, { 100935, 2 }, { 100937, 3 }, - { 101034, 2 }, { 101041, 2 }, { 101154, 2 }, { 101200, 4 }, { 101250, 2 }, { 101352, 2 }, { 101403, 2 }, { 101430, 1 }, { 101508, 3 }, { 101509, 3 }, { 101523, 10 }, { 101604, 2 }, { 101637, 2 }, { 101681, 4 }, { 101759, 1 }, { 101773, 1 }, { 101836, 1 }, { 101882, 4 }, { 101895, 2 }, { 101897, 2 }, { 101939, 2 }, { 101951, 6 }, - { 101956, 5 }, { 102055, 1 }, { 102085, 2 }, { 102093, 2 }, { 102209, 2 }, { 102258, 6 }, { 102271, 2 }, { 102284, 2 }, { 102332, 2 }, { 102354, 2 }, { 102366, 2 }, { 102424, 3 }, { 102456, 2 }, { 102496, 1 }, { 102497, 3 }, { 102519, 3 }, { 102554, 1 }, { 102610, 5 }, { 102657, 2 }, { 102661, 4 }, { 102695, 4 }, { 102707, 12 }, - { 102910, 2 }, { 102930, 5 }, { 102937, 9 }, { 102938, 7 }, { 102965, 6 }, { 102969, 7 }, { 103031, 2 }, { 103062, 2 }, { 103096, 2 }, { 103146, 2 }, { 103159, 2 }, { 103223, 2 }, { 103267, 2 }, { 103296, 2 }, { 103303, 2 }, { 103487, 2 }, { 103491, 2 }, { 103599, 2 }, { 103677, 2 }, { 103903, 1 }, { 104040, 2 }, { 104047, 1 }, - { 104052, 2 }, { 104057, 4 }, { 104057, 2 }, { 104062, 4 }, { 104091, 2 }, { 104189, 3 }, { 104283, 8 }, { 104288, 4 }, { 104305, 3 }, { 104445, 2 }, { 104472, 2 }, { 104475, 1 }, { 104497, 4 }, { 104548, 2 }, { 104582, 2 }, { 104626, 1 }, { 104716, 2 }, { 104826, 2 }, { 104849, 2 }, { 104872, 1 }, { 104945, 1 }, { 104948, 2 }, - { 105066, 2 }, { 105071, 1 }, { 105198, 4 }, { 105198, 4 }, { 105203, 2 }, { 105256, 6 }, { 105263, 2 }, { 105329, 2 }, { 105515, 2 }, { 105566, 2 }, { 105566, 2 }, { 105585, 2 }, { 105678, 2 }, { 105852, 2 }, { 105877, 2 }, { 105911, 2 }, { 106022, 1 }, { 106033, 2 }, { 106080, 2 }, { 106192, 2 }, { 106220, 3 }, { 106243, 2 }, - { 106323, 11 }, { 106371, 2 }, { 106608, 2 }, { 106624, 2 }, { 106680, 3 }, { 106688, 1 }, { 106800, 1 }, { 106800, 1 }, { 106821, 4 }, { 106853, 1 }, { 106930, 3 }, { 106937, 2 }, { 106955, 2 }, { 106996, 2 }, { 106996, 1 }, { 107148, 4 }, { 107213, 16 }, { 107213, 2 }, { 107243, 2 }, { 107360, 2 }, { 107408, 2 }, { 107509, 4 }, - { 107572, 2 }, { 107592, 2 }, { 107644, 5 }, { 107679, 2 }, { 107705, 3 }, { 107761, 4 }, { 107780, 2 }, { 107825, 2 }, { 108007, 2 }, { 108041, 4 }, { 108058, 2 }, { 108071, 1 }, { 108132, 2 }, { 108164, 2 }, { 108189, 2 }, { 108210, 2 }, { 108330, 2 }, { 108430, 2 }, { 108450, 2 }, { 108469, 2 }, { 108484, 2 }, { 108533, 2 }, - { 108588, 2 }, { 108594, 2 }, { 108690, 2 }, { 108785, 1 }, { 108814, 2 }, { 108818, 1 }, { 108820, 2 }, { 108889, 2 }, { 108951, 2 }, { 108959, 2 }, { 108963, 2 }, { 109034, 2 }, { 109172, 1 }, { 109176, 2 }, { 109195, 3 }, { 109229, 2 }, { 109256, 2 }, { 109290, 2 }, { 109304, 2 }, { 109333, 2 }, { 109343, 4 }, { 109347, 7 }, - { 109387, 2 }, { 109421, 1 }, { 109497, 2 }, { 109501, 3 }, { 109513, 2 }, { 109525, 3 }, { 109625, 4 }, { 109710, 2 }, { 109740, 2 }, { 109751, 2 }, { 109761, 2 }, { 109890, 8 }, { 109891, 4 }, { 109909, 2 }, { 109923, 1 }, { 110017, 2 }, { 110046, 2 }, { 110111, 2 }, { 110258, 2 }, { 110340, 2 }, { 110352, 2 }, { 110398, 2 }, - { 110583, 2 }, { 110600, 13 }, { 110626, 3 }, { 110709, 2 }, { 110772, 4 }, { 110773, 2 }, { 110813, 1 }, { 110890, 2 }, { 110898, 2 }, { 110954, 2 }, { 111120, 2 }, { 111132, 3 }, { 111163, 8 }, { 111224, 2 }, { 111340, 2 }, { 111398, 2 }, { 111555, 2 }, { 111597, 3 }, { 111607, 2 }, { 111655, 2 }, { 111691, 3 }, { 111835, 2 }, - { 111854, 2 }, { 111876, 16 }, { 111884, 1 }, { 111884, 2 }, { 111929, 2 }, { 111941, 2 }, { 111969, 2 }, { 112003, 2 }, { 112165, 2 }, { 112365, 2 }, { 112450, 1 }, { 112521, 2 }, { 112649, 4 }, { 112665, 2 }, { 112881, 1 }, { 112882, 2 }, { 112906, 2 }, { 112951, 2 }, { 112994, 2 }, { 112997, 2 }, { 113002, 2 }, { 113056, 1 }, - { 113077, 2 }, { 113208, 1 }, { 113320, 2 }, { 113326, 3 }, { 113375, 2 }, { 113530, 30 }, { 113530, 30 }, { 113537, 1 }, { 113563, 14 }, { 113592, 2 }, { 113637, 2 }, { 113768, 2 }, { 113850, 5 }, { 113892, 2 }, { 113916, 2 }, { 113965, 2 }, { 113976, 2 }, { 114037, 2 }, { 114149, 1 }, { 114158, 9 }, { 114201, 2 }, { 114262, 2 }, - { 114268, 4 }, { 114353, 2 }, { 114388, 2 }, { 114404, 2 }, { 114428, 5 }, { 114438, 2 }, { 114541, 2 }, { 114550, 2 }, { 114561, 2 }, { 114625, 3 }, { 114730, 2 }, { 114770, 1 }, { 114815, 4 }, { 114998, 2 }, { 115077, 2 }, { 115093, 2 }, { 115120, 2 }, { 115194, 2 }, { 115216, 3 }, { 115299, 2 }, { 115391, 3 }, { 115410, 2 }, - { 115542, 33 }, { 115581, 2 }, { 115618, 2 }, { 115645, 5 }, { 115647, 2 }, { 115697, 2 }, { 115725, 2 }, { 115740, 2 }, { 115757, 2 }, { 115763, 2 }, { 115770, 2 }, { 115787, 2 }, { 115916, 2 }, { 115928, 2 }, { 115962, 2 }, { 116020, 2 }, { 116022, 1 }, { 116089, 2 }, { 116159, 1 }, { 116196, 2 }, { 116247, 2 }, { 116254, 7 }, - { 116336, 2 }, { 116409, 2 }, { 116459, 2 }, { 116569, 2 }, { 116619, 2 }, { 116688, 2 }, { 116733, 2 }, { 116807, 3 }, { 116843, 2 }, { 116886, 1 }, { 116902, 2 }, { 116931, 2 }, { 116952, 2 }, { 116952, 2 }, { 117177, 2 }, { 117189, 2 }, { 117206, 2 }, { 117260, 29 }, { 117271, 6 }, { 117276, 3 }, { 117276, 5 }, { 117278, 3 }, - { 117278, 2 }, { 117359, 4 }, { 117380, 2 }, { 117414, 1 }, { 117503, 2 }, { 117517, 2 }, { 117530, 2 }, { 117574, 4 }, { 117575, 5 }, { 117577, 2 }, { 117606, 2 }, { 117645, 2 }, { 117655, 2 }, { 117692, 2 }, { 117705, 1 }, { 117731, 1 }, { 117762, 4 }, { 117780, 2 }, { 117974, 1 }, { 118057, 1 }, { 118099, 2 }, { 118107, 2 }, - { 118113, 2 }, { 118175, 2 }, { 118198, 2 }, { 118232, 2 }, { 118326, 1 }, { 118438, 31 }, { 118469, 2 }, { 118521, 31 }, { 118565, 2 }, { 118593, 2 }, { 118602, 2 }, { 118652, 2 }, { 118668, 2 }, { 118689, 3 }, { 118703, 14 }, { 118705, 2 }, { 118813, 2 }, { 118825, 2 }, { 118894, 3 }, { 118915, 2 }, { 118962, 2 }, { 118986, 2 }, - { 119045, 2 }, { 119054, 1 }, { 119054, 1 }, { 119119, 2 }, { 119149, 2 }, { 119206, 1 }, { 119316, 2 }, { 119387, 2 }, { 119404, 3 }, { 119516, 2 }, { 119520, 2 }, { 119571, 3 }, { 119573, 2 }, { 119610, 5 }, { 119621, 2 }, { 119623, 4 }, { 119672, 2 }, { 119692, 3 }, { 119734, 2 }, { 119742, 1 }, { 119754, 1 }, { 119785, 2 }, - { 120001, 2 }, { 120115, 4 }, { 120260, 2 }, { 120314, 2 }, { 120416, 2 }, { 120435, 1 }, { 120450, 3 }, { 120530, 2 }, { 120550, 5 }, { 120730, 2 }, { 120731, 2 }, { 120751, 3 }, { 120755, 2 }, { 120869, 2 }, { 120988, 2 }, { 121061, 2 }, { 121177, 2 }, { 121212, 2 }, { 121214, 1 }, { 121286, 2 }, { 121331, 1 }, { 121344, 2 }, - { 121407, 2 }, { 121424, 1 }, { 121491, 2 }, { 121568, 1 }, { 121588, 6 }, { 121651, 2 }, { 121676, 2 }, { 121785, 4 }, { 121830, 3 }, { 121919, 1 }, { 121951, 2 }, { 121991, 1 }, { 122056, 2 }, { 122062, 2 }, { 122144, 2 }, { 122183, 1 }, { 122331, 2 }, { 122466, 2 }, { 122558, 2 }, { 122570, 2 }, { 122676, 2 }, { 122733, 2 }, - { 122774, 6 }, { 122783, 2 }, { 122825, 2 }, { 122865, 2 }, { 122884, 2 }, { 122892, 2 }, { 122911, 2 }, { 122929, 2 }, { 122936, 2 }, { 123190, 2 }, { 123271, 2 }, { 123271, 2 }, { 123302, 7 }, { 123391, 2 }, { 123394, 2 }, { 123416, 1 }, { 123708, 2 }, { 123752, 2 }, { 123761, 2 }, { 123783, 2 }, { 123794, 2 }, { 123817, 2 }, - { 123820, 1 }, { 123823, 1 }, { 123857, 3 }, { 123886, 2 }, { 124023, 1 }, { 124029, 2 }, { 124042, 2 }, { 124056, 3 }, { 124071, 6 }, { 124105, 5 }, { 124143, 2 }, { 124191, 2 }, { 124207, 1 }, { 124257, 2 }, { 124306, 3 }, { 124338, 2 }, { 124388, 8 }, { 124400, 2 }, { 124418, 2 }, { 124502, 2 }, { 124521, 1 }, { 124533, 2 }, - { 124645, 2 }, { 124685, 1 }, { 124694, 2 }, { 124700, 1 }, { 124736, 2 }, { 124891, 7 }, { 124920, 2 }, { 124983, 2 }, { 125014, 2 }, { 125038, 2 }, { 125084, 2 }, { 125162, 2 }, { 125193, 2 }, { 125285, 2 }, { 125368, 2 }, { 125409, 2 }, { 125570, 2 }, { 125601, 2 }, { 125641, 1 }, { 125721, 2 }, { 125731, 2 }, { 125803, 2 }, - { 125904, 2 }, { 125973, 2 }, { 126018, 1 }, { 126034, 5 }, { 126094, 1 }, { 126144, 1 }, { 126195, 2 }, { 126297, 2 }, { 126389, 2 }, { 126429, 2 }, { 126439, 2 }, { 126499, 2 }, { 126501, 1 }, { 126587, 2 }, { 126663, 2 }, { 126681, 2 }, { 126687, 1 }, { 126781, 2 }, { 126783, 2 }, { 126840, 8 }, { 126843, 2 }, { 126959, 2 }, - { 127015, 2 }, { 127101, 2 }, { 127149, 2 }, { 127197, 3 }, { 127268, 2 }, { 127372, 2 }, { 127385, 2 }, { 127473, 4 }, { 127539, 2 }, { 127598, 2 }, { 127613, 14 }, { 127683, 3 }, { 127684, 2 }, { 127697, 2 }, { 127698, 3 }, { 127773, 2 }, { 127781, 1 }, { 127839, 2 }, { 127905, 2 }, { 127949, 2 }, { 128035, 2 }, { 128046, 1 }, - { 128167, 2 }, { 128271, 2 }, { 128307, 1 }, { 128320, 2 }, { 128330, 2 }, { 128375, 2 }, { 128381, 4 }, { 128447, 2 }, { 128462, 2 }, { 128466, 3 }, { 128466, 2 }, { 128496, 2 }, { 128589, 2 }, { 128616, 3 }, { 128679, 1 }, { 128770, 1 }, { 128793, 2 }, { 128802, 2 }, { 128813, 2 }, { 128900, 2 }, { 128949, 2 }, { 129269, 2 }, - { 129271, 3 }, { 129278, 2 }, { 129343, 1 }, { 129408, 2 }, { 129408, 1 }, { 129421, 6 }, { 129461, 2 }, { 129469, 3 }, { 129482, 2 }, { 129502, 2 }, { 129512, 2 }, { 129551, 2 }, { 129629, 2 }, { 129632, 2 }, { 129679, 1 }, { 129725, 2 }, { 130007, 2 }, { 130018, 16 }, { 130057, 2 }, { 130071, 2 }, { 130087, 2 }, { 130188, 1 }, - { 130202, 2 }, { 130316, 2 }, { 130328, 1 }, { 130466, 2 }, { 130549, 2 }, { 130649, 2 }, { 130705, 3 }, { 130800, 2 }, { 130907, 2 }, { 130989, 2 }, { 131103, 2 }, { 131127, 2 }, { 131200, 5 }, { 131241, 6 }, { 131351, 2 }, { 131413, 2 }, { 131448, 2 }, { 131599, 2 }, { 131634, 1 }, { 131687, 2 }, { 131739, 2 }, { 131758, 2 }, - { 131765, 2 }, { 131787, 3 }, { 131819, 3 }, { 131868, 2 }, { 131886, 2 }, { 131901, 4 }, { 131977, 2 }, { 131990, 2 }, { 132035, 2 }, { 132035, 2 }, { 132043, 2 }, { 132173, 2 }, { 132181, 4 }, { 132181, 6 }, { 132194, 5 }, { 132252, 2 }, { 132262, 6 }, { 132271, 1 }, { 132285, 2 }, { 132328, 2 }, { 132335, 1 }, { 132337, 1 }, - { 132389, 5 }, { 132430, 2 }, { 132451, 2 }, { 132499, 4 }, { 132503, 1 }, { 132520, 4 }, { 132541, 4 }, { 132860, 2 }, { 132862, 4 }, { 132874, 12 }, { 132874, 13 }, { 132875, 12 }, { 132911, 2 }, { 132973, 2 }, { 133051, 2 }, { 133062, 2 }, { 133067, 2 }, { 133138, 2 }, { 133184, 2 }, { 133231, 2 }, { 133297, 3 }, { 133344, 2 }, - { 133385, 4 }, { 133408, 2 }, { 133464, 2 }, { 133522, 2 }, { 133631, 2 }, { 133631, 2 }, { 133702, 2 }, { 133705, 1 }, { 133721, 2 }, { 133746, 2 }, { 133773, 3 }, { 133819, 2 }, { 133843, 2 }, { 133929, 2 }, { 133946, 2 }, { 134113, 4 }, { 134151, 2 }, { 134289, 1 }, { 134385, 2 }, { 134429, 2 }, { 134506, 2 }, { 134511, 2 }, - { 134521, 2 }, { 134558, 1 }, { 134710, 2 }, { 134738, 2 }, { 134751, 3 }, { 134818, 2 }, { 134820, 4 }, { 134879, 2 }, { 134919, 2 }, { 134947, 2 }, { 134948, 3 }, { 135040, 3 }, { 135125, 10 }, { 135155, 2 }, { 135228, 2 }, { 135255, 2 }, { 135296, 3 }, { 135322, 2 }, { 135349, 2 }, { 135428, 3 }, { 135476, 1 }, { 135503, 2 }, - { 135524, 2 }, { 135550, 4 }, { 135594, 2 }, { 135597, 2 }, { 135624, 3 }, { 135741, 2 }, { 135753, 2 }, { 135842, 2 }, { 135853, 2 }, { 135896, 3 }, { 136004, 1 }, { 136061, 1 }, { 136068, 1 }, { 136106, 2 }, { 136145, 2 }, { 136145, 2 }, { 136173, 2 }, { 136186, 2 }, { 136196, 2 }, { 136201, 2 }, { 136211, 2 }, { 136268, 2 }, - { 136298, 2 }, { 136377, 2 }, { 136420, 2 }, { 136475, 2 }, { 136486, 1 }, { 136554, 2 }, { 136641, 2 }, { 136770, 1 }, { 136873, 2 }, { 136877, 1 }, { 136906, 2 }, { 137092, 2 }, { 137143, 2 }, { 137200, 3 }, { 137232, 2 }, { 137239, 2 }, { 137248, 2 }, { 137281, 1 }, { 137301, 2 }, { 137314, 3 }, { 137352, 1 }, { 137365, 2 }, - { 137375, 2 }, { 137411, 2 }, { 137424, 2 }, { 137516, 2 }, { 137532, 2 }, { 137593, 2 }, { 137600, 2 }, { 137658, 2 }, { 137703, 2 }, { 137766, 2 }, { 137791, 2 }, { 137801, 2 }, { 137864, 2 }, { 137870, 3 }, { 137931, 2 }, { 138009, 3 }, { 138013, 1 }, { 138013, 1 }, { 138066, 2 }, { 138073, 2 }, { 138114, 2 }, { 138150, 2 }, - { 138236, 2 }, { 138276, 2 }, { 138286, 2 }, { 138298, 3 }, { 138309, 1 }, { 138373, 3 }, { 138524, 2 }, { 138535, 1 }, { 138593, 4 }, { 138611, 1 }, { 138725, 2 }, { 138807, 2 }, { 138819, 3 }, { 138849, 5 }, { 138867, 2 }, { 138907, 2 }, { 138930, 3 }, { 139026, 2 }, { 139102, 2 }, { 139108, 3 }, { 139184, 1 }, { 139209, 3 }, - { 139282, 2 }, { 139289, 4 }, { 139382, 1 }, { 139421, 1 }, { 139436, 2 }, { 139450, 1 }, { 139523, 3 }, { 139533, 2 }, { 139590, 2 }, { 139590, 2 }, { 139722, 2 }, { 139785, 2 }, { 139785, 1 }, { 139798, 2 }, { 139813, 2 }, { 139868, 2 }, { 139935, 3 }, { 139990, 3 }, { 140050, 2 }, { 140177, 2 }, { 140177, 4 }, { 140408, 2 }, - { 140420, 3 }, { 140461, 2 }, { 140578, 15 }, { 140605, 1 }, { 140662, 1 }, { 140755, 2 }, { 140786, 2 }, { 140846, 2 }, { 140874, 2 }, { 140959, 1 }, { 140973, 2 }, { 141128, 2 }, { 141132, 2 }, { 141257, 2 }, { 141290, 1 }, { 141360, 2 }, { 141472, 2 }, { 141545, 2 }, { 141545, 2 }, { 141575, 1 }, { 141606, 5 }, { 141655, 2 }, - { 141735, 2 }, { 141767, 5 }, { 141796, 2 }, { 141841, 2 }, { 141915, 2 }, { 141923, 1 }, { 141932, 2 }, { 141994, 2 }, { 142018, 2 }, { 142029, 3 }, { 142072, 2 }, { 142128, 2 }, { 142133, 1 }, { 142261, 2 }, { 142304, 1 }, { 142400, 2 }, { 142401, 2 }, { 142409, 2 }, { 142479, 2 }, { 142522, 1 }, { 142552, 1 }, { 142589, 2 }, - { 142596, 2 }, { 142753, 1 }, { 142766, 2 }, { 142796, 2 }, { 142836, 2 }, { 142871, 2 }, { 143058, 3 }, { 143059, 6 }, { 143063, 3 }, { 143065, 2 }, { 143141, 4 }, { 143173, 2 }, { 143374, 2 }, { 143399, 2 }, { 143406, 2 }, { 143429, 3 }, { 143430, 2 }, { 143462, 1 }, { 143579, 2 }, { 143663, 2 }, { 143844, 3 }, { 143851, 2 }, - { 143926, 2 }, { 143931, 2 }, { 144051, 6 }, { 144085, 10 }, { 144147, 2 }, { 144188, 4 }, { 144238, 4 }, { 144353, 2 }, { 144465, 2 }, { 144474, 2 }, { 144637, 2 }, { 144638, 1 }, { 144648, 1 }, { 144661, 3 }, { 144812, 2 }, { 144847, 2 }, { 144901, 8 }, { 145058, 2 }, { 145122, 8 }, { 145134, 2 }, { 145150, 2 }, { 145299, 1 }, - { 145313, 2 }, { 145314, 3 }, { 145374, 2 }, { 145412, 2 }, { 145432, 2 }, { 145446, 2 }, { 145534, 3 }, { 145592, 2 }, { 145614, 2 }, { 145648, 2 }, { 145721, 2 }, { 145858, 1 }, { 145970, 3 }, { 145984, 3 }, { 146004, 2 }, { 146016, 3 }, { 146048, 2 }, { 146097, 3 }, { 146103, 2 }, { 146136, 2 }, { 146194, 3 }, { 146230, 1 }, - { 146254, 2 }, { 146261, 4 }, { 146269, 4 }, { 146393, 2 }, { 146411, 3 }, { 146501, 2 }, { 146547, 2 }, { 146547, 2 }, { 146573, 2 }, { 146616, 2 }, { 146622, 3 }, { 146728, 3 }, { 146781, 5 }, { 146805, 4 }, { 146921, 2 }, { 147002, 3 }, { 147072, 2 }, { 147159, 2 }, { 147170, 2 }, { 147203, 1 }, { 147245, 2 }, { 147278, 2 }, - { 147422, 2 }, { 147471, 2 }, { 147491, 2 }, { 147607, 4 }, { 147693, 2 }, { 147763, 2 }, { 147775, 6 }, { 147776, 4 }, { 147824, 2 }, { 147922, 2 }, { 147922, 2 }, { 147937, 2 }, { 147957, 2 }, { 147980, 2 }, { 148008, 2 }, { 148018, 2 }, { 148046, 3 }, { 148071, 4 }, { 148106, 3 }, { 148122, 2 }, { 148139, 2 }, { 148175, 2 }, - { 148318, 2 }, { 148514, 2 }, { 148528, 2 }, { 148539, 2 }, { 148545, 2 }, { 148564, 2 }, { 148569, 2 }, { 148607, 3 }, { 148712, 2 }, { 148751, 2 }, { 148792, 4 }, { 148807, 2 }, { 148818, 2 }, { 148846, 9 }, { 148848, 2 }, { 148851, 2 }, { 148861, 3 }, { 148924, 32 }, { 148934, 2 }, { 149037, 1 }, { 149127, 3 }, { 149132, 2 }, - { 149181, 1 }, { 149181, 2 }, { 149206, 2 }, { 149216, 7 }, { 149240, 4 }, { 149240, 1 }, { 149279, 1 }, { 149280, 3 }, { 149292, 2 }, { 149314, 2 }, { 149344, 2 }, { 149364, 4 }, { 149388, 2 }, { 149438, 2 }, { 149520, 2 }, { 149566, 2 }, { 149630, 2 }, { 149682, 2 }, { 149691, 1 }, { 149703, 2 }, { 149775, 2 }, { 149796, 1 }, - { 149863, 1 }, { 149884, 2 }, { 149888, 1 }, { 149983, 2 }, { 150078, 3 }, { 150083, 6 }, { 150175, 1 }, { 150235, 2 }, { 150238, 2 }, { 150298, 3 }, { 150321, 2 }, { 150382, 2 }, { 150510, 4 }, { 150574, 2 }, { 150619, 5 }, { 150645, 2 }, { 150694, 2 }, { 150732, 8 }, { 150764, 2 }, { 150813, 2 }, { 150871, 2 }, { 150879, 2 }, - { 150888, 1 }, { 150920, 2 }, { 151009, 2 }, { 151013, 2 }, { 151019, 2 }, { 151063, 2 }, { 151067, 2 }, { 151125, 1 }, { 151151, 5 }, { 151172, 2 }, { 151197, 4 }, { 151228, 70 }, { 151292, 2 }, { 151354, 2 }, { 151392, 2 }, { 151396, 1 }, { 151412, 2 }, { 151514, 2 }, { 151529, 2 }, { 151567, 2 }, { 151589, 2 }, { 151641, 2 }, - { 151672, 2 }, { 151730, 2 }, { 151748, 2 }, { 151770, 1 }, { 151788, 2 }, { 151795, 2 }, { 151805, 2 }, { 152046, 5 }, { 152048, 3 }, { 152054, 2 }, { 152057, 3 }, { 152058, 2 }, { 152075, 2 }, { 152090, 9 }, { 152205, 2 }, { 152440, 2 }, { 152480, 2 }, { 152484, 2 }, { 152601, 2 }, { 152714, 2 }, { 152801, 2 }, { 152819, 10 }, - { 152825, 2 }, { 152896, 10 }, { 152937, 2 }, { 152938, 2 }, { 152939, 3 }, { 153042, 2 }, { 153069, 2 }, { 153099, 2 }, { 153164, 2 }, { 153234, 2 }, { 153266, 2 }, { 153345, 2 }, { 153420, 2 }, { 153479, 2 }, { 153488, 4 }, { 153502, 2 }, { 153663, 3 }, { 153740, 1 }, { 153780, 2 }, { 153824, 6 }, { 153938, 2 }, { 153985, 2 }, - { 154022, 2 }, { 154022, 1 }, { 154072, 4 }, { 154109, 2 }, { 154189, 2 }, { 154222, 2 }, { 154228, 2 }, { 154265, 15 }, { 154324, 2 }, { 154350, 2 }, { 154375, 2 }, { 154396, 2 }, { 154431, 2 }, { 154463, 2 }, { 154475, 3 }, { 154510, 1 }, { 154518, 1 }, { 154529, 2 }, { 154710, 2 }, { 154742, 2 }, { 154792, 2 }, { 154871, 4 }, - { 154960, 2 }, { 154961, 2 }, { 154964, 2 }, { 154989, 3 }, { 155002, 3 }, { 155079, 2 }, { 155105, 2 }, { 155107, 2 }, { 155258, 1 }, { 155328, 2 }, { 155328, 2 }, { 155347, 2 }, { 155369, 2 }, { 155447, 2 }, { 155482, 2 }, { 155508, 2 }, { 155531, 2 }, { 155553, 1 }, { 155647, 3 }, { 155659, 9 }, { 155859, 2 }, { 155960, 2 }, - { 156009, 2 }, { 156062, 2 }, { 156143, 3 }, { 156217, 3 }, { 156252, 7 }, { 156260, 2 }, { 156274, 2 }, { 156339, 2 }, { 156354, 3 }, { 156406, 2 }, { 156550, 2 }, { 156694, 2 }, { 156730, 12 }, { 156795, 4 }, { 156806, 3 }, { 156818, 5 }, { 156835, 3 }, { 156850, 3 }, { 156861, 4 }, { 156877, 2 }, { 156915, 6 }, { 156967, 2 }, - { 157046, 2 }, { 157208, 3 }, { 157260, 2 }, { 157364, 2 }, { 157365, 1 }, { 157371, 2 }, { 157440, 2 }, { 157453, 2 }, { 157482, 1 }, { 157505, 2 }, { 157511, 4 }, { 157522, 2 }, { 157562, 2 }, { 157562, 2 }, { 157702, 2 }, { 157734, 3 }, { 157807, 2 }, { 157851, 2 }, { 157882, 2 }, { 157957, 2 }, { 158227, 2 }, { 158284, 2 }, - { 158292, 2 }, { 158310, 3 }, { 158310, 3 }, { 158330, 2 }, { 158358, 1 }, { 158493, 2 }, { 158596, 2 }, { 158736, 2 }, { 158812, 1 }, { 158830, 2 }, { 158846, 1 }, { 158884, 1 }, { 158918, 2 }, { 159003, 2 }, { 159056, 2 }, { 159189, 2 }, { 159372, 2 }, { 159373, 2 }, { 159410, 3 }, { 159421, 4 }, { 159429, 2 }, { 159505, 2 }, - { 159559, 1 }, { 159574, 2 }, { 159587, 4 }, { 159683, 2 }, { 159745, 1 }, { 159748, 3 }, { 159858, 2 }, { 159945, 1 }, { 159971, 4 }, { 159982, 2 }, { 160079, 13 }, { 160084, 2 }, { 160085, 4 }, { 160104, 9 }, { 160197, 2 }, { 160295, 2 }, { 160365, 2 }, { 160372, 1 }, { 160392, 3 }, { 160408, 1 }, { 160446, 1 }, { 160540, 2 }, - { 160599, 1 }, { 160604, 2 }, { 160745, 3 }, { 160752, 2 }, { 160794, 2 }, { 160826, 2 }, { 160846, 2 }, { 160871, 2 }, { 160957, 2 }, { 160986, 2 }, { 161053, 2 }, { 161133, 3 }, { 161133, 1 }, { 161162, 3 }, { 161247, 2 }, { 161270, 2 }, { 161292, 6 }, { 161370, 2 }, { 161420, 2 }, { 161446, 2 }, { 161487, 2 }, { 161511, 3 }, - { 161512, 1 }, { 161580, 2 }, { 161782, 12 }, { 161784, 8 }, { 161786, 2 }, { 161786, 8 }, { 161787, 7 }, { 161795, 2 }, { 161825, 7 }, { 161833, 4 }, { 161892, 2 }, { 161930, 2 }, { 161992, 2 }, { 162054, 2 }, { 162176, 2 }, { 162183, 2 }, { 162219, 2 }, { 162245, 2 }, { 162288, 2 }, { 162361, 3 }, { 162370, 2 }, { 162388, 2 }, - { 162434, 2 }, { 162447, 2 }, { 162524, 2 }, { 162542, 3 }, { 162562, 2 }, { 162594, 2 }, { 162646, 1 }, { 162662, 2 }, { 162761, 2 }, { 162780, 2 }, { 162802, 2 }, { 162820, 3 }, { 162824, 2 }, { 162908, 2 }, { 162968, 2 }, { 163171, 2 }, { 163190, 2 }, { 163288, 1 }, { 163288, 1 }, { 163397, 2 }, { 163405, 6 }, { 163414, 1 }, - { 163506, 1 }, { 163558, 2 }, { 163565, 1 }, { 163568, 3 }, { 163619, 2 }, { 163633, 2 }, { 163678, 2 }, { 163745, 3 }, { 163765, 4 }, { 163773, 2 }, { 163793, 2 }, { 163878, 7 }, { 163949, 2 }, { 163975, 2 }, { 163991, 2 }, { 164016, 2 }, { 164020, 3 }, { 164068, 1 }, { 164076, 4 }, { 164082, 3 }, { 164176, 2 }, { 164236, 2 }, - { 164238, 1 }, { 164315, 2 }, { 164449, 2 }, { 164529, 2 }, { 164574, 4 }, { 164591, 2 }, { 164595, 2 }, { 164611, 2 }, { 164623, 4 }, { 164632, 10 }, { 164691, 2 }, { 164706, 2 }, { 164755, 2 }, { 164761, 2 }, { 164973, 2 }, { 165030, 2 }, { 165090, 2 }, { 165099, 1 }, { 165126, 2 }, { 165188, 2 }, { 165205, 2 }, { 165275, 1 }, - { 165347, 2 }, { 165381, 2 }, { 165562, 2 }, { 165563, 1 }, { 165594, 2 }, { 165641, 2 }, { 165663, 6 }, { 165759, 2 }, { 165811, 2 }, { 165822, 1 }, { 165830, 1 }, { 165903, 1 }, { 165921, 2 }, { 165953, 1 }, { 166022, 1 }, { 166294, 2 }, { 166333, 2 }, { 166420, 2 }, { 166433, 2 }, { 166442, 1 }, { 166536, 2 }, { 166543, 2 }, - { 166556, 2 }, { 166571, 2 }, { 166575, 1 }, { 166588, 2 }, { 166601, 2 }, { 166663, 3 }, { 166692, 1 }, { 166710, 2 }, { 166759, 2 }, { 166785, 3 }, { 166842, 2 }, { 166843, 2 }, { 166864, 2 }, { 166902, 2 }, { 166996, 2 }, { 166999, 2 }, { 167038, 2 }, { 167112, 4 }, { 167112, 2 }, { 167177, 2 }, { 167180, 2 }, { 167229, 1 }, - { 167298, 2 }, { 167306, 4 }, { 167309, 3 }, { 167402, 2 }, { 167405, 2 }, { 167433, 2 }, { 167435, 1 }, { 167461, 3 }, { 167553, 3 }, { 167688, 5 }, { 167689, 2 }, { 167709, 2 }, { 167744, 2 }, { 167821, 2 }, { 167825, 2 }, { 167925, 10 }, { 167969, 2 }, { 168024, 2 }, { 168089, 2 }, { 168104, 2 }, { 168194, 2 }, { 168305, 2 }, - { 168316, 2 }, { 168366, 2 }, { 168423, 2 }, { 168568, 3 }, { 168582, 2 }, { 168615, 3 }, { 168618, 2 }, { 168638, 2 }, { 168671, 2 }, { 168736, 2 }, { 168747, 2 }, { 168750, 4 }, { 168808, 3 }, { 168814, 4 }, { 168820, 2 }, { 168914, 2 }, { 168968, 2 }, { 168979, 2 }, { 169006, 2 }, { 169069, 2 }, { 169106, 3 }, { 169158, 2 }, - { 169158, 2 }, { 169189, 2 }, { 169253, 2 }, { 169259, 1 }, { 169279, 1 }, { 169325, 8 }, { 169349, 2 }, { 169353, 2 }, { 169378, 2 }, { 169432, 2 }, { 169476, 1 }, { 169476, 1 }, { 169525, 2 }, { 169538, 7 }, { 169555, 2 }, { 169571, 2 }, { 169594, 4 }, { 169687, 2 }, { 169799, 2 }, { 169831, 2 }, { 170042, 2 }, { 170061, 2 }, - { 170065, 1 }, { 170128, 6 }, { 170148, 20 }, { 170215, 70 }, { 170256, 60 }, { 170266, 69 }, { 170275, 7 }, { 170277, 6 }, { 170500, 3 }, { 170516, 3 }, { 170601, 2 }, { 170666, 2 }, { 170668, 4 }, { 170668, 1 }, { 170716, 3 }, { 170728, 3 }, { 170735, 5 }, { 170847, 3 }, { 170852, 9 }, { 170858, 2 }, { 170859, 3 }, { 170956, 2 }, - { 170956, 1 }, { 170967, 2 }, { 171005, 2 }, { 171113, 2 }, { 171279, 2 }, { 171400, 2 }, { 171405, 2 }, { 171448, 1 }, { 171490, 2 }, { 171567, 32 }, { 171590, 2 }, { 171723, 2 }, { 171737, 3 }, { 171958, 2 }, { 171967, 2 } - }; - } - - /* sub-class to avoid 65k limit of a single class */ - private static class SampleDataHolder2 { - /* - * Array of [milliseconds, latency] - */ - private static int[][] data = new int[][] { { 0, 3 }, { 43, 33 }, { 45, 11 }, { 45, 1 }, { 68, 13 }, { 88, 10 }, { 158, 68 }, { 158, 4 }, { 169, 168 }, { 267, 68 }, { 342, 68 }, { 438, 68 }, { 464, 7 }, { 504, 68 }, { 541, 6 }, { 541, 68 }, { 562, 68 }, { 581, 3 }, { 636, 68 }, { 778, 68 }, { 825, 1 }, { 859, 68 }, - { 948, 1 }, { 1043, 68 }, { 1145, 68 }, { 1152, 1 }, { 1218, 5 }, { 1229, 68 }, { 1259, 68 }, { 1333, 68 }, { 1349, 68 }, { 1392, 68 }, { 1468, 1 }, { 1551, 68 }, { 1586, 68 }, { 1685, 68 }, { 1696, 1 }, { 1807, 68 }, { 1817, 3 }, { 1817, 6 }, { 1847, 68 }, { 1870, 68 }, { 1939, 68 }, { 2050, 68 }, { 2129, 3 }, { 2141, 68 }, - { 2265, 68 }, { 2414, 1 }, { 2693, 68 }, { 2703, 68 }, { 2791, 68 }, { 2838, 68 }, { 2906, 68 }, { 2981, 68 }, { 3008, 68 }, { 3026, 4 }, { 3077, 68 }, { 3273, 68 }, { 3282, 68 }, { 3286, 68 }, { 3318, 3 }, { 3335, 5 }, { 3710, 68 }, { 3711, 1 }, { 3745, 68 }, { 3748, 4 }, { 3767, 3 }, { 3809, 3 }, { 3835, 35 }, { 4083, 1 }, - { 4116, 68 }, { 4117, 1 }, { 4157, 1 }, { 4279, 68 }, { 4344, 68 }, { 4452, 68 }, { 4530, 68 }, { 4583, 68 }, { 4647, 3 }, { 4758, 68 }, { 4776, 68 }, { 4793, 68 }, { 4901, 68 }, { 4909, 68 }, { 4962, 68 }, { 4984, 68 }, { 5022, 68 }, { 5139, 68 }, { 5166, 1 }, { 5174, 68 }, { 5187, 68 }, { 5225, 68 }, { 5234, 68 }, { 5263, 1 }, - { 5325, 68 }, { 5355, 4 }, { 5407, 1 }, { 5414, 68 }, { 5589, 68 }, { 5595, 68 }, { 5747, 68 }, { 5780, 68 }, { 5788, 68 }, { 5796, 68 }, { 5818, 68 }, { 5975, 1 }, { 6018, 1 }, { 6270, 68 }, { 6272, 68 }, { 6348, 68 }, { 6372, 68 }, { 6379, 68 }, { 6439, 68 }, { 6442, 68 }, { 6460, 68 }, { 6460, 68 }, { 6509, 68 }, { 6511, 1 }, - { 6514, 4 }, { 6530, 8 }, { 6719, 68 }, { 6760, 68 }, { 6784, 68 }, { 6838, 1 }, { 6861, 68 }, { 6947, 68 }, { 7013, 68 }, { 7075, 68 }, { 7122, 5 }, { 7130, 68 }, { 7209, 3 }, { 7259, 68 }, { 7309, 1 }, { 7315, 3 }, { 7322, 68 }, { 7348, 68 }, { 7420, 68 }, { 7461, 68 }, { 7545, 68 }, { 7554, 3 }, { 7630, 68 }, { 7666, 68 }, - { 7815, 1 }, { 7972, 1 }, { 7972, 68 }, { 7988, 68 }, { 8049, 8 }, { 8254, 68 }, { 8269, 68 }, { 8352, 1 }, { 8378, 68 }, { 8526, 68 }, { 8531, 68 }, { 8583, 68 }, { 8615, 68 }, { 8619, 3 }, { 8623, 68 }, { 8692, 1 }, { 8698, 68 }, { 8773, 68 }, { 8777, 3 }, { 8822, 68 }, { 8929, 68 }, { 8935, 68 }, { 9025, 68 }, { 9054, 68 }, - { 9056, 1 }, { 9086, 68 }, { 9147, 3 }, { 9219, 68 }, { 9230, 3 }, { 9248, 68 }, { 9283, 68 }, { 9314, 68 }, { 9418, 1 }, { 9426, 68 }, { 9456, 1 }, { 9594, 68 }, { 9628, 68 }, { 9642, 68 }, { 9646, 68 }, { 9686, 1 }, { 9709, 68 }, { 9771, 3 }, { 9782, 68 }, { 9884, 68 }, { 9914, 5 }, { 10004, 4 }, { 10033, 6 }, { 10052, 68 }, - { 10086, 68 }, { 10168, 68 }, { 10176, 1 }, { 10228, 68 }, { 10312, 68 }, { 10372, 68 }, { 10622, 68 }, { 10685, 68 }, { 10687, 1 }, { 10787, 68 }, { 11010, 68 }, { 11024, 68 }, { 11044, 68 }, { 11086, 68 }, { 11149, 1 }, { 11198, 68 }, { 11265, 68 }, { 11302, 68 }, { 11326, 68 }, { 11354, 68 }, { 11404, 1 }, { 11473, 68 }, - { 11506, 68 }, { 11548, 4 }, { 11575, 68 }, { 11621, 4 }, { 11625, 3 }, { 11625, 1 }, { 11642, 4 }, { 11859, 5 }, { 11870, 68 }, { 11872, 3 }, { 11880, 7 }, { 11886, 3 }, { 11905, 6 }, { 11880, 3 }, { 11912, 6 }, { 11916, 4 }, { 11916, 3 }, { 11965, 4 }, { 12068, 13 }, { 12106, 68 }, { 12120, 68 }, { 12221, 68 }, { 12257, 68 }, - { 12361, 68 }, { 12411, 68 }, { 12473, 3 }, { 12554, 68 }, { 12583, 68 }, { 12654, 68 }, { 12665, 68 }, { 12744, 1 }, { 12775, 68 }, { 12858, 68 }, { 12993, 68 }, { 13007, 3 }, { 13025, 4 }, { 13038, 68 }, { 13092, 4 }, { 13094, 5 }, { 13095, 1 }, { 13110, 68 }, { 13116, 1 }, { 13140, 68 }, { 13169, 1 }, { 13186, 68 }, - { 13202, 68 }, { 13202, 1 }, { 13256, 68 }, { 13344, 68 }, { 13373, 68 }, { 13396, 3 }, { 13446, 68 }, { 13451, 3 }, { 13475, 68 }, { 13521, 1 }, { 13587, 68 }, { 13592, 68 }, { 13708, 3 }, { 13711, 1 }, { 13741, 1 }, { 13757, 1 }, { 13847, 68 }, { 13881, 3 }, { 13915, 1 }, { 14005, 68 }, { 14028, 68 }, { 14037, 68 }, - { 14074, 68 }, { 14135, 68 }, { 14176, 68 }, { 14227, 68 }, { 14228, 68 }, { 14271, 3 }, { 14279, 3 }, { 14493, 68 }, { 14535, 3 }, { 14535, 1 }, { 14680, 68 }, { 14717, 68 }, { 14725, 1 }, { 14790, 68 }, { 14801, 1 }, { 14959, 68 }, { 15052, 68 }, { 15055, 1 }, { 15055, 1 }, { 15075, 68 }, { 15103, 8 }, { 15153, 16 }, - { 15191, 68 }, { 15240, 68 }, { 15313, 68 }, { 15323, 68 }, { 15341, 1 }, { 15383, 68 }, { 15387, 68 }, { 15491, 68 }, { 15534, 68 }, { 15539, 68 }, { 15549, 68 }, { 15554, 1 }, { 15664, 1 }, { 15726, 68 }, { 15807, 68 }, { 15842, 68 }, { 15897, 68 }, { 15913, 3 }, { 15925, 68 }, { 15935, 68 }, { 16131, 1 }, { 16211, 3 }, - { 16249, 68 }, { 16268, 68 }, { 16307, 68 }, { 16398, 68 }, { 16498, 68 }, { 16518, 1 }, { 16552, 1 }, { 16571, 68 }, { 16592, 68 }, { 16601, 3 }, { 16638, 68 }, { 16698, 68 }, { 16712, 1 }, { 16767, 68 }, { 16789, 68 }, { 16992, 68 }, { 17015, 68 }, { 17035, 68 }, { 17074, 3 }, { 17086, 3 }, { 17086, 1 }, { 17092, 1 }, - { 17110, 4 }, { 17116, 3 }, { 17236, 68 }, { 17291, 68 }, { 17291, 68 }, { 17340, 68 }, { 17342, 1 }, { 17360, 3 }, { 17436, 3 }, { 17457, 68 }, { 17508, 1 }, { 17556, 68 }, { 17601, 68 }, { 17639, 68 }, { 17671, 68 }, { 17743, 68 }, { 17857, 68 }, { 17915, 68 }, { 17992, 68 }, { 18077, 1 }, { 18088, 68 }, { 18158, 1 }, - { 18239, 16 }, { 18242, 68 }, { 18252, 3 }, { 18299, 1 }, { 18405, 68 }, { 18433, 68 }, { 18444, 68 }, { 18490, 68 }, { 18497, 68 }, { 18516, 68 }, { 18540, 68 }, { 18598, 68 }, { 18649, 68 }, { 18658, 68 }, { 18683, 68 }, { 18728, 68 }, { 18767, 1 }, { 18821, 68 }, { 18868, 68 }, { 18876, 68 }, { 18914, 14 }, { 19212, 1 }, - { 19215, 1 }, { 19293, 68 }, { 19303, 68 }, { 19336, 68 }, { 19376, 68 }, { 19419, 68 }, { 19558, 68 }, { 19559, 1 }, { 19609, 68 }, { 19688, 68 }, { 19724, 68 }, { 19820, 1 }, { 19851, 68 }, { 19881, 68 }, { 19966, 68 }, { 19983, 3 }, { 19988, 4 }, { 20047, 1 }, { 20062, 68 }, { 20091, 1 }, { 20152, 1 }, { 20183, 1 }, - { 20208, 68 }, { 20346, 68 }, { 20386, 1 }, { 20459, 68 }, { 20505, 68 }, { 20520, 1 }, { 20560, 3 }, { 20566, 3 }, { 20566, 1 }, { 20610, 68 }, { 20652, 68 }, { 20694, 68 }, { 20740, 68 }, { 20756, 68 }, { 20825, 3 }, { 20895, 68 }, { 20959, 1 }, { 20995, 68 }, { 21017, 3 }, { 21039, 68 }, { 21086, 1 }, { 21109, 3 }, { 21139, 3 }, - { 21206, 68 }, { 21230, 68 }, { 21251, 3 }, { 21352, 68 }, { 21353, 68 }, { 21370, 3 }, { 21389, 1 }, { 21445, 3 }, { 21475, 68 }, { 21528, 68 }, { 21559, 3 }, { 21604, 68 }, { 21606, 1 }, { 21815, 68 }, { 21858, 3 }, { 21860, 3 }, { 22015, 68 }, { 22065, 68 }, { 22098, 5 }, { 22105, 68 }, { 22158, 3 }, { 22197, 68 }, { 22254, 1 }, - { 22353, 68 }, { 22404, 4 }, { 22422, 68 }, { 22569, 68 }, { 22634, 68 }, { 22639, 68 }, { 22861, 68 }, { 22868, 68 }, { 22876, 1 }, { 22902, 68 }, { 22925, 68 }, { 23080, 68 }, { 23085, 3 }, { 23089, 5 }, { 23329, 1 }, { 23349, 68 }, { 23559, 5 }, { 23567, 3 }, { 23574, 68 }, { 23584, 3 }, { 23615, 3 }, { 23633, 68 }, - { 23674, 68 }, { 23678, 1 }, { 23853, 68 }, { 23875, 68 }, { 24010, 4 }, { 24076, 68 }, { 24128, 6 }, { 24248, 68 }, { 24253, 68 }, { 24259, 1 }, { 24319, 68 }, { 24319, 1 }, { 24502, 3 }, { 24666, 68 }, { 24781, 3 }, { 24792, 68 }, { 24909, 68 }, { 24993, 68 }, { 25039, 1 }, { 25090, 3 }, { 25137, 1 }, { 25138, 3 }, { 25140, 3 }, - { 25155, 5 }, { 25411, 68 }, { 25460, 68 }, { 25564, 3 }, { 25586, 3 }, { 25630, 68 }, { 25765, 68 }, { 25789, 3 }, { 25803, 68 }, { 25851, 68 }, { 25872, 68 }, { 25887, 68 }, { 25981, 1 }, { 26016, 68 }, { 26019, 1 }, { 26029, 1 }, { 26104, 7 }, { 26144, 68 }, { 26275, 1 }, { 26295, 68 }, { 26298, 1 }, { 26322, 68 }, - { 26380, 68 }, { 26408, 4 }, { 26446, 1 }, { 26553, 1 }, { 26576, 1 }, { 26635, 1 }, { 26668, 68 }, { 26675, 68 }, { 26698, 4 }, { 26748, 9 }, { 26788, 68 }, { 26932, 68 }, { 26962, 68 }, { 27042, 68 }, { 27060, 68 }, { 27163, 3 }, { 27202, 68 }, { 27290, 68 }, { 27337, 3 }, { 27376, 68 }, { 27439, 68 }, { 27458, 4 }, - { 27515, 68 }, { 27518, 1 }, { 27541, 68 }, { 27585, 3 }, { 27633, 68 }, { 27695, 68 }, { 27702, 68 }, { 27861, 68 }, { 27924, 1 }, { 28025, 14 }, { 28058, 68 }, { 28143, 68 }, { 28215, 68 }, { 28240, 68 }, { 28241, 68 }, { 28285, 68 }, { 28324, 3 }, { 28378, 68 }, { 28514, 68 }, { 28529, 68 }, { 28538, 68 }, { 28565, 3 }, - { 28697, 68 }, { 28735, 68 }, { 28769, 68 }, { 28770, 4 }, { 28788, 4 }, { 28807, 3 }, { 28807, 4 }, { 28829, 1 }, { 28853, 68 }, { 28856, 7 }, { 28864, 68 }, { 28865, 3 }, { 28915, 68 }, { 28928, 68 }, { 28964, 68 }, { 28988, 1 }, { 29031, 68 }, { 29095, 68 }, { 29189, 68 }, { 29205, 1 }, { 29230, 1 }, { 29332, 68 }, - { 29339, 68 }, { 29349, 5 }, { 29449, 68 }, { 29471, 68 }, { 29578, 68 }, { 29859, 68 }, { 29878, 68 }, { 29947, 10 }, { 30083, 68 }, { 30121, 68 }, { 30128, 68 }, { 30155, 4 }, { 30157, 1 }, { 30272, 68 }, { 30281, 68 }, { 30286, 68 }, { 30305, 68 }, { 30408, 68 }, { 30444, 268 }, { 30612, 68 }, { 30628, 68 }, { 30747, 68 }, - { 30783, 68 }, { 30808, 5 }, { 30868, 3 }, { 30875, 68 }, { 30997, 68 }, { 31000, 68 }, { 31022, 3 }, { 31111, 1 }, { 31144, 68 }, { 31146, 3 }, { 31187, 68 }, { 31324, 68 }, { 31343, 68 }, { 31416, 68 }, { 31485, 68 }, { 31539, 68 }, { 31638, 68 }, { 31648, 68 }, { 31750, 68 }, { 31754, 68 }, { 31785, 10 }, { 31786, 5 }, - { 31800, 68 }, { 31801, 4 }, { 31807, 7 }, { 31807, 3 }, { 31807, 10 }, { 31808, 3 }, { 31808, 4 }, { 31818, 6 }, { 31825, 7 }, { 31838, 68 }, { 31911, 1 }, { 31974, 68 }, { 32010, 3 }, { 32031, 68 }, { 32040, 68 }, { 32063, 1 }, { 32078, 68 }, { 32156, 68 }, { 32198, 31 }, { 32257, 68 }, { 32257, 68 }, { 32265, 68 }, - { 32330, 68 }, { 32369, 8 }, { 32404, 3 }, { 32425, 68 }, { 32432, 68 }, { 32505, 68 }, { 32531, 68 }, { 32536, 68 }, { 32549, 68 }, { 32582, 3 }, { 32590, 4 }, { 32624, 68 }, { 32644, 68 }, { 32692, 68 }, { 32695, 4 }, { 32699, 3 }, { 32726, 4 }, { 32784, 68 }, { 32832, 68 }, { 32883, 6 }, { 32965, 4 }, { 33044, 68 }, - { 33104, 68 }, { 33184, 68 }, { 33264, 1 }, { 33292, 68 }, { 33312, 1 }, { 33468, 68 }, { 33471, 1 }, { 33565, 68 }, { 33627, 68 }, { 33659, 68 }, { 33709, 68 }, { 33766, 5 }, { 33836, 68 }, { 33875, 68 }, { 33954, 68 }, { 33959, 68 }, { 34050, 68 }, { 34090, 68 }, { 34168, 68 }, { 34233, 68 }, { 34461, 68 }, { 34462, 1 }, - { 34463, 68 }, { 34472, 4 }, { 34500, 68 }, { 34520, 68 }, { 34544, 68 }, { 34614, 68 }, { 34662, 1 }, { 34676, 68 }, { 34729, 4 }, { 34803, 68 }, { 34845, 68 }, { 34913, 68 }, { 34963, 6 }, { 35019, 68 }, { 35022, 68 }, { 35070, 68 }, { 35120, 68 }, { 35132, 68 }, { 35144, 68 }, { 35205, 68 }, { 35230, 3 }, { 35244, 68 }, - { 35271, 4 }, { 35276, 68 }, { 35282, 68 }, { 35324, 3 }, { 35366, 3 }, { 35659, 68 }, { 35680, 68 }, { 35744, 68 }, { 35758, 3 }, { 35796, 68 }, { 35830, 68 }, { 35841, 7 }, { 35843, 68 }, { 35856, 68 }, { 35914, 4 }, { 35929, 13 }, { 35993, 68 }, { 35997, 1 }, { 36046, 4 }, { 36046, 1 }, { 36051, 1 }, { 36111, 68 }, { 36208, 1 }, - { 36208, 1 }, { 36306, 68 }, { 36325, 68 }, { 36386, 68 }, { 36405, 68 }, { 36443, 1 }, { 36455, 1 }, { 36538, 68 }, { 36562, 68 }, { 36566, 68 }, { 36628, 68 }, { 36693, 68 }, { 36713, 68 }, { 36730, 68 }, { 36747, 68 }, { 36786, 68 }, { 36810, 1 }, { 36848, 68 }, { 36914, 1 }, { 36920, 68 }, { 36952, 68 }, { 37071, 68 }, - { 37086, 1 }, { 37094, 3 }, { 37158, 3 }, { 37231, 68 }, { 37241, 68 }, { 37285, 68 }, { 37349, 68 }, { 37404, 68 }, { 37410, 1 }, { 37433, 4 }, { 37615, 68 }, { 37659, 68 }, { 37742, 68 }, { 37773, 68 }, { 37867, 1 }, { 37890, 68 }, { 37960, 68 }, { 38042, 3 }, { 38241, 68 }, { 38400, 68 }, { 38461, 1 }, { 38551, 68 }, - { 38611, 1 }, { 38657, 68 }, { 38729, 68 }, { 38748, 68 }, { 38815, 68 }, { 38852, 68 }, { 38890, 1 }, { 38954, 68 }, { 39119, 68 }, { 39162, 68 }, { 39175, 3 }, { 39176, 68 }, { 39231, 68 }, { 39261, 68 }, { 39467, 68 }, { 39500, 68 }, { 39507, 68 }, { 39566, 68 }, { 39608, 68 }, { 39686, 6 }, { 39730, 68 }, { 39842, 1 }, - { 39853, 1 }, { 39905, 68 }, { 39931, 68 }, { 39989, 68 }, { 40030, 68 }, { 40227, 68 }, { 40268, 68 }, { 40372, 68 }, { 40415, 1 }, { 40488, 3 }, { 40536, 68 }, { 40676, 3 }, { 40677, 68 }, { 40755, 68 }, { 40842, 68 }, { 40849, 1 }, { 40870, 3 }, { 40873, 3 }, { 40972, 68 }, { 41033, 68 }, { 41190, 68 }, { 41273, 5 }, - { 41273, 1 }, { 41293, 68 }, { 41367, 368 }, { 41376, 68 }, { 41420, 68 }, { 41473, 68 }, { 41473, 68 }, { 41493, 4 }, { 41521, 68 }, { 41533, 68 }, { 41554, 68 }, { 41568, 68 }, { 41583, 3 }, { 41728, 68 }, { 41786, 68 }, { 41836, 1 }, { 41875, 68 }, { 41933, 68 }, { 42044, 68 }, { 42075, 68 }, { 42076, 68 }, { 42133, 68 }, - { 42259, 29 }, { 42269, 3 }, { 42294, 68 }, { 42420, 68 }, { 42524, 68 }, { 42524, 1 }, { 42546, 1 }, { 42631, 68 }, { 42693, 68 }, { 42740, 68 }, { 42744, 4 }, { 42755, 1 }, { 42870, 68 }, { 42894, 68 }, { 42939, 68 }, { 42973, 68 }, { 43016, 68 }, { 43070, 68 }, { 43105, 68 }, { 43115, 68 }, { 43375, 3 }, { 43387, 1 }, - { 43424, 3 }, { 43448, 68 }, { 43480, 68 }, { 43498, 68 }, { 43651, 68 }, { 43727, 68 }, { 43879, 68 }, { 43910, 1 }, { 43977, 68 }, { 44003, 68 }, { 44080, 68 }, { 44082, 1 }, { 44136, 68 }, { 44169, 29 }, { 44186, 68 }, { 44339, 68 }, { 44350, 1 }, { 44356, 1 }, { 44430, 68 }, { 44440, 1 }, { 44530, 1 }, { 44538, 68 }, - { 44572, 68 }, { 44585, 68 }, { 44709, 68 }, { 44748, 68 }, { 44748, 68 }, { 44769, 68 }, { 44813, 68 }, { 44890, 68 }, { 45015, 68 }, { 45046, 4 }, { 45052, 68 }, { 45062, 68 }, { 45094, 6 }, { 45184, 68 }, { 45191, 68 }, { 45201, 3 }, { 45216, 3 }, { 45227, 68 }, { 45269, 1 }, { 45294, 68 }, { 45314, 68 }, { 45345, 8 }, - { 45352, 68 }, { 45365, 3 }, { 45378, 1 }, { 45392, 4 }, { 45405, 3 }, { 45410, 68 }, { 45448, 14 }, { 45450, 68 }, { 45457, 68 }, { 45466, 3 }, { 45481, 4 }, { 45486, 7 }, { 45533, 5 }, { 45576, 68 }, { 45649, 68 }, { 45917, 68 }, { 45919, 6 }, { 45919, 1 }, { 45930, 15 }, { 45930, 68 }, { 46001, 5 }, { 46036, 68 }, { 46054, 68 }, - { 46075, 68 }, { 46153, 68 }, { 46155, 68 }, { 46228, 68 }, { 46234, 68 }, { 46273, 68 }, { 46387, 68 }, { 46398, 68 }, { 46517, 68 }, { 46559, 68 }, { 46565, 1 }, { 46598, 68 }, { 46686, 68 }, { 46744, 68 }, { 46816, 3 }, { 46835, 68 }, { 46921, 68 }, { 46938, 68 }, { 46991, 68 }, { 47038, 68 }, { 47098, 3 }, { 47107, 68 }, - { 47201, 3 }, { 47327, 1 }, { 47327, 1 }, { 47338, 68 }, { 47395, 1 }, { 47499, 68 }, { 47504, 68 }, { 47515, 1 }, { 47516, 1 }, { 47600, 1 }, { 47604, 1 }, { 47707, 1 }, { 47728, 1 }, { 47748, 68 }, { 47763, 68 }, { 47807, 4 }, { 47814, 68 }, { 47822, 68 }, { 47834, 68 }, { 47843, 3 }, { 47886, 68 }, { 47893, 68 }, { 48066, 68 }, - { 48126, 68 }, { 48133, 1 }, { 48166, 68 }, { 48299, 1 }, { 48455, 68 }, { 48468, 68 }, { 48568, 68 }, { 48606, 68 }, { 48642, 68 }, { 48698, 68 }, { 48714, 68 }, { 48754, 68 }, { 48765, 3 }, { 48773, 5 }, { 48819, 68 }, { 48833, 68 }, { 48904, 68 }, { 49000, 1 }, { 49113, 168 }, { 49140, 68 }, { 49276, 68 }, { 49353, 68 }, - { 49411, 3 }, { 49418, 68 }, { 49540, 68 }, { 49544, 68 }, { 49584, 68 }, { 49602, 68 }, { 49784, 5 }, { 49822, 4 }, { 49822, 5 }, { 49828, 68 }, { 49866, 68 }, { 49922, 3 }, { 49959, 68 }, { 50045, 68 }, { 50134, 3 }, { 50140, 68 }, { 50237, 68 }, { 50247, 68 }, { 50266, 13 }, { 50290, 68 }, { 50312, 4 }, { 50314, 1 }, - { 50527, 68 }, { 50605, 1 }, { 50730, 68 }, { 50751, 68 }, { 50770, 68 }, { 50858, 68 }, { 50859, 68 }, { 50909, 68 }, { 50948, 3 }, { 51043, 68 }, { 51048, 68 }, { 51089, 68 }, { 51090, 68 }, { 51141, 68 }, { 51163, 68 }, { 51250, 68 }, { 51347, 68 }, { 51475, 68 }, { 51536, 68 }, { 51544, 68 }, { 51595, 68 }, { 51602, 19 }, - { 51643, 5 }, { 51702, 68 }, { 51702, 10 }, { 51764, 68 }, { 51793, 5 }, { 51812, 68 }, { 51839, 1 }, { 51938, 3 }, { 51941, 1 }, { 51967, 4 }, { 52049, 3 }, { 52074, 3 }, { 52098, 68 }, { 52118, 68 }, { 52119, 3 }, { 52227, 11 }, { 52246, 3 }, { 52282, 68 }, { 52451, 68 }, { 52583, 68 }, { 52601, 1 }, { 52605, 68 }, { 52615, 68 }, - { 52668, 68 }, { 52824, 68 }, { 53076, 1 }, { 53120, 1 }, { 53179, 68 }, { 53189, 68 }, { 53193, 1 }, { 53195, 68 }, { 53246, 68 }, { 53249, 68 }, { 53268, 1 }, { 53295, 68 }, { 53312, 68 }, { 53410, 68 }, { 53451, 68 }, { 53570, 68 }, { 53593, 68 }, { 53635, 68 }, { 53657, 68 }, { 53682, 3 }, { 53728, 5 }, { 53733, 68 }, - { 53753, 68 }, { 53787, 4 }, { 53807, 1 }, { 54008, 68 }, { 54059, 68 }, { 54060, 1 }, { 54080, 68 }, { 54090, 1 }, { 54138, 68 }, { 54149, 68 }, { 54168, 1 }, { 54171, 68 }, { 54216, 268 }, { 54233, 6 }, { 54434, 68 }, { 54534, 68 }, { 54562, 68 }, { 54763, 68 }, { 54791, 68 }, { 54816, 68 }, { 54909, 68 }, { 54916, 3 }, - { 54963, 68 }, { 54985, 68 }, { 54991, 3 }, { 55016, 3 }, { 55025, 3 }, { 55032, 68 }, { 55099, 68 }, { 55260, 68 }, { 55261, 68 }, { 55270, 3 }, { 55384, 68 }, { 55455, 68 }, { 55456, 68 }, { 55504, 3 }, { 55510, 68 }, { 55558, 68 }, { 55568, 68 }, { 55585, 68 }, { 55677, 68 }, { 55703, 68 }, { 55749, 68 }, { 55779, 68 }, - { 55789, 3 }, { 55792, 68 }, { 55830, 4 }, { 55835, 68 }, { 55879, 68 }, { 56076, 68 }, { 56118, 68 }, { 56314, 68 }, { 56392, 1 }, { 56411, 68 }, { 56459, 68 }, { 56553, 34 }, { 56575, 68 }, { 56733, 68 }, { 56762, 68 }, { 56793, 3 }, { 56877, 3 }, { 56927, 68 }, { 56981, 68 }, { 57014, 1 }, { 57149, 68 }, { 57162, 68 }, - { 57186, 68 }, { 57254, 68 }, { 57267, 1 }, { 57324, 68 }, { 57327, 68 }, { 57365, 4 }, { 57371, 68 }, { 57445, 68 }, { 57477, 68 }, { 57497, 68 }, { 57536, 68 }, { 57609, 68 }, { 57626, 68 }, { 57666, 68 }, { 57694, 68 }, { 57694, 68 }, { 57749, 68 }, { 57781, 7 }, { 57878, 68 }, { 57953, 68 }, { 58051, 68 }, { 58088, 68 }, - { 58097, 68 }, { 58142, 3 }, { 58142, 1 }, { 58197, 1 }, { 58221, 68 }, { 58222, 68 }, { 58244, 68 }, { 58290, 1 }, { 58296, 1 }, { 58325, 68 }, { 58378, 1 }, { 58389, 3 }, { 58430, 68 }, { 58454, 68 }, { 58551, 29 }, { 58563, 6 }, { 58681, 68 }, { 58751, 8 }, { 58752, 43 }, { 58790, 5 }, { 58846, 68 }, { 58879, 6 }, { 58953, 68 }, - { 58998, 68 }, { 59010, 1 }, { 59038, 5 }, { 59135, 68 }, { 59166, 68 }, { 59180, 68 }, { 59222, 68 }, { 59227, 68 }, { 59307, 68 }, { 59398, 3 }, { 59411, 68 }, { 59436, 3 }, { 59464, 68 }, { 59569, 68 }, { 59587, 68 }, { 59624, 3 }, { 59786, 68 }, { 59834, 68 }, { 59841, 68 }, { 59841, 1 }, { 59984, 68 }, { 59985, 68 }, - { 60003, 3 }, { 60045, 68 }, { 60097, 68 }, { 60148, 68 }, { 60172, 68 }, { 60203, 5 }, { 60565, 68 }, { 60625, 68 }, { 60743, 68 }, { 60781, 68 }, { 60892, 68 }, { 60977, 68 }, { 60979, 68 }, { 61021, 5 }, { 61021, 4 }, { 61026, 68 }, { 61139, 68 }, { 61165, 3 }, { 61204, 68 }, { 61207, 1 }, { 61248, 3 }, { 61257, 68 }, - { 61264, 6 }, { 61272, 3 }, { 61410, 68 }, { 61410, 3 }, { 61416, 68 }, { 61423, 1 }, { 61503, 68 }, { 61503, 68 }, { 61533, 68 }, { 61567, 68 }, { 61575, 68 }, { 61835, 1 }, { 61842, 1 }, { 61924, 68 }, { 61951, 6 }, { 61975, 68 }, { 61986, 3 }, { 62024, 1 }, { 62110, 68 }, { 62135, 68 }, { 62192, 68 }, { 62208, 68 }, - { 62399, 68 }, { 62400, 1 }, { 62414, 68 }, { 62423, 3 }, { 62456, 3 }, { 62459, 3 }, { 62478, 3 }, { 62484, 68 }, { 62510, 6 }, { 62511, 3 }, { 62565, 3 }, { 62610, 68 }, { 62875, 4 }, { 62896, 5 }, { 62898, 68 }, { 62904, 68 }, { 62938, 3 }, { 62943, 68 }, { 62977, 68 }, { 62989, 3 }, { 62998, 5 }, { 63069, 1 }, { 63093, 5 }, - { 63107, 68 }, { 63113, 1 }, { 63231, 4 }, { 63253, 68 }, { 63286, 4 }, { 63289, 68 }, { 63334, 1 }, { 63334, 4 }, { 63413, 68 }, { 63425, 68 }, { 63512, 10 }, { 63537, 1 }, { 63694, 1 }, { 63721, 4 }, { 63749, 68 }, { 63783, 17 }, { 63791, 3 }, { 63792, 68 }, { 63882, 25 }, { 63896, 1 }, { 63936, 68 }, { 63969, 3 }, { 63986, 68 }, - { 63988, 68 }, { 64009, 10 }, { 64018, 68 }, { 64032, 6 }, { 64125, 68 }, { 64195, 1 }, { 64221, 7 }, { 64390, 68 }, { 64459, 68 }, { 64568, 68 }, { 64784, 1 }, { 64789, 68 }, { 64829, 68 }, { 64848, 1 }, { 64914, 68 }, { 64928, 1 }, { 64939, 68 }, { 65026, 68 }, { 65057, 68 }, { 65070, 68 }, { 65193, 4 }, { 65235, 3 }, - { 65242, 68 }, { 65281, 68 }, { 65320, 68 }, { 65365, 1 }, { 65414, 68 }, { 65445, 68 }, { 65581, 68 }, { 65624, 1 }, { 65719, 68 }, { 65766, 68 }, { 65927, 68 }, { 66004, 1 }, { 66031, 68 }, { 66085, 1 }, { 66085, 68 }, { 66133, 68 }, { 66134, 68 }, { 66188, 1 }, { 66240, 68 }, { 66249, 68 }, { 66250, 68 }, { 66295, 68 }, - { 66342, 1 }, { 66352, 3 }, { 66388, 3 }, { 66432, 68 }, { 66437, 47 }, { 66497, 68 }, { 66517, 68 }, { 66526, 68 }, { 66546, 9 }, { 66605, 68 }, { 66753, 68 }, { 66792, 68 }, { 66796, 68 }, { 66828, 68 }, { 66899, 3 }, { 66970, 6 }, { 66981, 68 }, { 66983, 1 }, { 67009, 68 }, { 67017, 4 }, { 67115, 68 }, { 67117, 1 }, - { 67130, 6 }, { 67132, 7 }, { 67162, 68 }, { 67179, 6 }, { 67236, 68 }, { 67263, 3 }, { 67274, 68 }, { 67274, 68 }, { 67349, 3 }, { 67486, 68 }, { 67503, 3 }, { 67517, 1 }, { 67559, 1 }, { 67660, 68 }, { 67727, 68 }, { 67901, 68 }, { 67943, 4 }, { 67950, 68 }, { 67965, 3 }, { 68029, 68 }, { 68048, 68 }, { 68169, 68 }, { 68172, 1 }, - { 68258, 68 }, { 68288, 1 }, { 68359, 68 }, { 68441, 68 }, { 68484, 68 }, { 68488, 68 }, { 68525, 68 }, { 68535, 68 }, { 68575, 7 }, { 68575, 5 }, { 68583, 68 }, { 68588, 4 }, { 68593, 1 }, { 68597, 68 }, { 68636, 68 }, { 68636, 68 }, { 68667, 68 }, { 68785, 1 }, { 68914, 4 }, { 68915, 5 }, { 68940, 3 }, { 69010, 68 }, - { 69063, 68 }, { 69076, 68 }, { 69235, 68 }, { 69270, 68 }, { 69298, 1 }, { 69350, 5 }, { 69432, 68 }, { 69514, 68 }, { 69562, 3 }, { 69562, 4 }, { 69638, 1 }, { 69656, 68 }, { 69709, 68 }, { 69775, 68 }, { 69788, 68 }, { 70193, 68 }, { 70233, 68 }, { 70252, 68 }, { 70259, 68 }, { 70293, 3 }, { 70405, 3 }, { 70462, 68 }, - { 70515, 3 }, { 70518, 68 }, { 70535, 6 }, { 70547, 6 }, { 70577, 6 }, { 70631, 17 }, { 70667, 68 }, { 70680, 1 }, { 70694, 1 }, { 70898, 68 }, { 70916, 1 }, { 70936, 3 }, { 71033, 68 }, { 71126, 68 }, { 71158, 68 }, { 71162, 68 }, { 71421, 1 }, { 71441, 68 }, { 71557, 68 }, { 71789, 1 }, { 71816, 68 }, { 71850, 1 }, { 71869, 1 }, - { 71961, 68 }, { 71973, 4 }, { 72064, 68 }, { 72110, 68 }, { 72117, 3 }, { 72164, 68 }, { 72266, 68 }, { 72325, 68 }, { 72326, 1 }, { 72420, 68 }, { 72693, 68 }, { 72705, 1 }, { 72730, 68 }, { 72793, 68 }, { 72795, 1 }, { 72939, 1 }, { 72945, 3 }, { 72945, 68 }, { 73120, 1 }, { 73121, 5 }, { 73122, 4 }, { 73126, 1 }, { 73126, 1 }, - { 73196, 3 }, { 73219, 68 }, { 73241, 6 }, { 73272, 3 }, { 73354, 1 }, { 73368, 68 }, { 73467, 1 }, { 73517, 68 }, { 73554, 68 }, { 73678, 68 }, { 73838, 1 }, { 73881, 68 }, { 73958, 68 }, { 73985, 15 }, { 74092, 68 }, { 74205, 68 }, { 74245, 68 }, { 74277, 68 }, { 74286, 68 }, { 74353, 68 }, { 74403, 68 }, { 74428, 1 }, - { 74468, 68 }, { 74481, 3 }, { 74511, 68 }, { 74537, 68 }, { 74596, 68 }, { 74750, 68 }, { 74754, 68 }, { 74861, 68 }, { 74933, 4 }, { 74970, 1 }, { 75003, 3 }, { 75077, 1 }, { 75159, 68 }, { 75170, 68 }, { 75234, 45 }, { 75300, 3 }, { 75337, 68 }, { 75345, 68 }, { 75419, 1 }, { 75429, 68 }, { 75477, 1 }, { 75513, 68 }, - { 75536, 68 }, { 75536, 68 }, { 75539, 1 }, { 75551, 68 }, { 75561, 68 }, { 75565, 68 }, { 75590, 68 }, { 75623, 5 }, { 75773, 6 }, { 75777, 6 }, { 75785, 68 }, { 75791, 68 }, { 75804, 68 }, { 75862, 68 }, { 75924, 3 }, { 75927, 68 }, { 75996, 11 }, { 76000, 1 }, { 76006, 68 }, { 76020, 3 }, { 76110, 68 }, { 76126, 3 }, - { 76131, 68 }, { 76136, 68 }, { 76144, 68 }, { 76203, 68 }, { 76229, 3 }, { 76244, 15 }, { 76246, 68 }, { 76300, 1 }, { 76403, 3 }, { 76545, 68 }, { 76569, 68 }, { 76813, 68 }, { 76821, 68 }, { 76837, 68 }, { 76863, 68 }, { 77027, 68 }, { 77037, 65 }, { 77074, 3 }, { 77170, 68 }, { 77191, 68 }, { 77220, 68 }, { 77230, 68 }, - { 77261, 68 }, { 77277, 68 }, { 77309, 68 }, { 77314, 68 }, { 77412, 68 }, { 77419, 68 }, { 77457, 68 }, { 77633, 3 }, { 77714, 68 }, { 77855, 68 }, { 77857, 1 }, { 77876, 68 }, { 77895, 68 }, { 77916, 5 }, { 77947, 68 }, { 77948, 1 }, { 77966, 1 }, { 77996, 68 }, { 78025, 1 }, { 78064, 68 }, { 78100, 68 }, { 78113, 1 }, - { 78114, 3 }, { 78167, 68 }, { 78175, 68 }, { 78260, 68 }, { 78261, 1 }, { 78265, 68 }, { 78286, 1 }, { 78300, 68 }, { 78327, 3 }, { 78363, 1 }, { 78384, 68 }, { 78459, 68 }, { 78516, 68 }, { 78612, 68 }, { 78643, 68 }, { 78655, 68 }, { 78698, 1 }, { 78720, 3 }, { 78789, 76 }, { 78838, 5 }, { 78893, 1 }, { 78954, 7 }, - { 79007, 68 }, { 79132, 3 }, { 79193, 68 }, { 79193, 68 }, { 79226, 68 }, { 79411, 68 }, { 79422, 1 }, { 79502, 68 }, { 79593, 68 }, { 79622, 68 }, { 79657, 3 }, { 79771, 68 }, { 79866, 68 }, { 79909, 68 }, { 80005, 68 }, { 80032, 68 }, { 80060, 1 }, { 80132, 68 }, { 80149, 3 }, { 80251, 68 }, { 80363, 68 }, { 80379, 1 }, - { 80464, 68 }, { 80498, 68 }, { 80553, 68 }, { 80556, 3 }, { 80559, 1 }, { 80571, 68 }, { 80652, 1 }, { 80703, 68 }, { 80754, 68 }, { 80754, 68 }, { 80860, 68 }, { 81055, 68 }, { 81087, 4 }, { 81210, 68 }, { 81211, 1 }, { 81216, 1 }, { 81223, 1 }, { 81231, 1 }, { 81288, 68 }, { 81317, 68 }, { 81327, 65 }, { 81332, 68 }, - { 81376, 68 }, { 81469, 68 }, { 81579, 68 }, { 81617, 1 }, { 81630, 68 }, { 81666, 68 }, { 81800, 68 }, { 81832, 68 }, { 81848, 68 }, { 81869, 68 }, { 81941, 3 }, { 82177, 3 }, { 82179, 68 }, { 82180, 68 }, { 82182, 4 }, { 82185, 68 }, { 82195, 68 }, { 82238, 4 }, { 82265, 3 }, { 82295, 10 }, { 82299, 9 }, { 82367, 3 }, - { 82379, 3 }, { 82380, 1 }, { 82505, 68 }, { 82568, 68 }, { 82620, 1 }, { 82637, 5 }, { 82821, 68 }, { 82841, 68 }, { 82945, 1 }, { 83020, 168 }, { 83072, 68 }, { 83181, 68 }, { 83240, 68 }, { 83253, 3 }, { 83261, 68 }, { 83288, 68 }, { 83291, 4 }, { 83295, 3 }, { 83365, 68 }, { 83368, 68 }, { 83408, 68 }, { 83458, 68 }, - { 83470, 68 }, { 83471, 1 }, { 83637, 3 }, { 83693, 68 }, { 83703, 68 }, { 83732, 68 }, { 83745, 1 }, { 83800, 4 }, { 83801, 3 }, { 83856, 3 }, { 83863, 5 }, { 83867, 68 }, { 83868, 3 }, { 83898, 7 }, { 83900, 4 }, { 83901, 5 }, { 83989, 68 }, { 84049, 35 }, { 84086, 68 }, { 84089, 68 }, { 84115, 3 }, { 84130, 3 }, { 84132, 68 }, - { 84143, 54 }, { 84173, 68 }, { 84185, 5 }, { 84297, 68 }, { 84390, 68 }, { 84497, 4 }, { 84657, 68 }, { 84657, 68 }, { 84724, 68 }, { 84775, 68 }, { 84870, 68 }, { 84892, 68 }, { 84910, 3 }, { 84935, 3 }, { 85002, 68 }, { 85051, 68 }, { 85052, 68 }, { 85135, 25 }, { 85135, 68 }, { 85144, 68 }, { 85165, 3 }, { 85205, 68 }, - { 85232, 68 }, { 85281, 5 }, { 85423, 6 }, { 85539, 68 }, { 85582, 4 }, { 85609, 68 }, { 85701, 36 }, { 85705, 68 }, { 85824, 68 }, { 85824, 68 }, { 85858, 30 }, { 85858, 28 }, { 85904, 35 }, { 85910, 68 }, { 85913, 68 }, { 85926, 3 }, { 85942, 4 }, { 85969, 4 }, { 85996, 1 }, { 86013, 3 }, { 86034, 13 }, { 86068, 8 }, - { 86069, 8 }, { 86089, 8 }, { 86193, 13 }, { 86217, 7 }, { 86219, 68 }, { 86250, 68 }, { 86304, 16 }, { 86317, 68 }, { 86322, 4 }, { 86325, 68 }, { 86333, 68 }, { 86394, 68 }, { 86433, 68 }, { 86469, 3 }, { 86512, 4 }, { 86537, 68 }, { 86627, 68 }, { 86658, 68 }, { 86810, 68 }, { 86813, 68 }, { 86884, 68 }, { 86947, 68 }, - { 87003, 68 }, { 87010, 5 }, { 87019, 68 }, { 87027, 68 }, { 87105, 68 }, { 87107, 68 }, { 87183, 68 }, { 87273, 68 }, { 87358, 3 }, { 87388, 3 }, { 87503, 4 }, { 87639, 68 }, { 87649, 4 }, { 87722, 68 }, { 87829, 68 }, { 87829, 1 }, { 87863, 68 }, { 87894, 68 }, { 87988, 368 }, { 88035, 27 }, { 88059, 3 }, { 88094, 5 }, - { 88111, 21 }, { 88129, 68 }, { 88175, 5 }, { 88256, 68 }, { 88329, 76 }, { 88415, 3 }, { 88482, 68 }, { 88502, 1 }, { 88529, 68 }, { 88551, 3 }, { 88552, 1 }, { 88713, 68 }, { 88797, 68 }, { 88844, 27 }, { 88925, 5 }, { 88935, 68 }, { 88944, 1 }, { 89073, 68 }, { 89095, 3 }, { 89283, 68 }, { 89294, 3 }, { 89299, 68 }, - { 89324, 68 }, { 89368, 68 }, { 89387, 68 }, { 89464, 68 }, { 89607, 68 }, { 89737, 68 }, { 89791, 68 }, { 89794, 3 }, { 89840, 68 }, { 89849, 3 }, { 89859, 68 }, { 89905, 68 }, { 89952, 38 }, { 90030, 7 }, { 90030, 6 }, { 90031, 1 }, { 90072, 68 }, { 90090, 68 }, { 90146, 3 }, { 90202, 23 }, { 90302, 3 }, { 90328, 14 }, - { 90335, 14 }, { 90338, 8 }, { 90380, 68 }, { 90434, 1 }, { 90482, 68 }, { 90527, 9 }, { 90537, 68 }, { 90545, 68 }, { 90639, 5 }, { 90642, 68 }, { 90709, 68 }, { 90775, 1 }, { 90806, 68 }, { 90845, 19 }, { 90872, 4 }, { 90884, 68 }, { 90910, 68 }, { 90994, 5 }, { 91046, 8 }, { 91059, 8 }, { 91096, 39 }, { 91147, 68 }, - { 91168, 1 }, { 91493, 68 }, { 91513, 3 }, { 91618, 3 }, { 91653, 68 }, { 91817, 68 }, { 91831, 3 }, { 91833, 3 }, { 91885, 68 }, { 91919, 68 }, { 91934, 68 }, { 92245, 1 }, { 92284, 68 }, { 92292, 4 }, { 92369, 3 }, { 92388, 68 }, { 92426, 7 }, { 92720, 14 }, { 92720, 6 }, { 92729, 9 }, { 92733, 13 }, { 92735, 6 }, { 92786, 68 }, - { 92853, 31 }, { 92906, 68 }, { 93031, 7 }, { 93077, 68 }, { 93102, 68 }, { 93109, 68 }, { 93122, 3 }, { 93214, 68 }, { 93330, 68 }, { 93395, 68 }, { 93506, 68 }, { 93564, 9 }, { 93713, 9 }, { 93722, 4 }, { 93840, 68 }, { 93877, 4 }, { 93891, 3 }, { 93948, 68 }, { 93981, 68 }, { 94012, 3 }, { 94033, 68 }, { 94121, 68 }, - { 94165, 368 }, { 94181, 3 }, { 94210, 68 }, { 94216, 68 }, { 94230, 68 }, { 94333, 31 }, { 94433, 3 }, { 94497, 3 }, { 94609, 68 }, { 94623, 68 }, { 94763, 68 }, { 94780, 68 }, { 95287, 68 }, { 95348, 68 }, { 95433, 5 }, { 95446, 68 }, { 95493, 7 }, { 95517, 3 }, { 95580, 68 }, { 95610, 5 }, { 95620, 68 }, { 95678, 3 }, - { 95683, 68 }, { 95689, 68 }, { 95760, 68 }, { 95792, 68 }, { 95850, 68 }, { 95908, 68 }, { 95908, 68 }, { 95967, 68 }, { 96022, 3 }, { 96088, 65 }, { 96460, 68 }, { 96554, 68 }, { 96597, 68 }, { 96763, 68 }, { 96808, 68 }, { 96854, 1 }, { 96963, 1 }, { 97007, 3 }, { 97125, 1 }, { 97128, 68 }, { 97133, 3 }, { 97142, 3 }, - { 97156, 68 }, { 97223, 68 }, { 97244, 68 }, { 97303, 68 }, { 97355, 68 }, { 97356, 3 }, { 97393, 3 }, { 97409, 1 }, { 97451, 68 }, { 97539, 68 }, { 97546, 68 }, { 97553, 68 }, { 97627, 68 }, { 97640, 68 }, { 97650, 6 }, { 97675, 68 }, { 97685, 3 }, { 97773, 68 }, { 97802, 4 }, { 97826, 19 }, { 97860, 68 }, { 97956, 68 }, - { 97958, 68 }, { 97973, 3 }, { 97982, 68 }, { 98039, 68 }, { 98051, 68 }, { 98059, 68 }, { 98088, 68 }, { 98092, 4 }, { 98147, 68 }, { 98147, 68 }, { 98169, 68 }, { 98207, 65 }, { 98277, 1 }, { 98277, 268 }, { 98285, 68 }, { 98324, 3 }, { 98324, 3 }, { 98381, 31 }, { 98390, 68 }, { 98404, 68 }, { 98415, 4 }, { 98460, 68 }, - { 98462, 1 }, { 98475, 3 }, { 98485, 68 }, { 98640, 1 }, { 98798, 68 }, { 98800, 4 }, { 98821, 68 }, { 98895, 68 }, { 98936, 68 }, { 98950, 68 }, { 98980, 68 }, { 99033, 68 }, { 99045, 68 }, { 99135, 68 }, { 99315, 30 }, { 99324, 68 }, { 99346, 68 }, { 99418, 68 }, { 99505, 68 }, { 99557, 68 }, { 99559, 68 }, { 99586, 68 }, - { 99622, 68 }, { 99770, 1 }, { 99790, 68 }, { 99810, 68 }, { 99871, 1 }, { 99926, 68 }, { 99927, 68 }, { 99978, 68 }, { 99980, 68 }, { 100022, 3 }, { 100024, 1 }, { 100069, 68 }, { 100150, 68 }, { 100225, 63 }, { 100246, 1 }, { 100310, 68 }, { 100361, 68 }, { 100428, 1 }, { 100434, 68 }, { 100450, 4 }, { 100546, 68 }, - { 100551, 68 }, { 100551, 68 }, { 100554, 1 }, { 100597, 68 }, { 100676, 68 }, { 100693, 68 }, { 100827, 68 }, { 100928, 68 }, { 100928, 1 }, { 100935, 68 }, { 100937, 3 }, { 101034, 68 }, { 101041, 68 }, { 101154, 68 }, { 101200, 4 }, { 101250, 68 }, { 101352, 68 }, { 101403, 68 }, { 101430, 1 }, { 101508, 3 }, { 101509, 3 }, - { 101523, 10 }, { 101604, 68 }, { 101637, 68 }, { 101681, 4 }, { 101759, 1 }, { 101773, 1 }, { 101836, 1 }, { 101882, 4 }, { 101895, 68 }, { 101897, 68 }, { 101939, 68 }, { 101951, 6 }, { 101956, 5 }, { 102055, 1 }, { 102085, 68 }, { 102093, 67 }, { 102209, 68 }, { 102258, 6 }, { 102271, 68 }, { 102284, 68 }, { 102332, 68 }, - { 102354, 68 }, { 102366, 68 }, { 102424, 3 }, { 102456, 68 }, { 102496, 1 }, { 102497, 3 }, { 102519, 3 }, { 102554, 1 }, { 102610, 5 }, { 102657, 68 }, { 102661, 4 }, { 102695, 4 }, { 102707, 168 }, { 102910, 68 }, { 102930, 5 }, { 102937, 9 }, { 102938, 7 }, { 102965, 6 }, { 102969, 7 }, { 103031, 68 }, { 103062, 68 }, - { 103096, 68 }, { 103146, 68 }, { 103159, 68 }, { 103223, 68 }, { 103267, 68 }, { 103296, 68 }, { 103303, 68 }, { 103487, 68 }, { 103491, 68 }, { 103599, 68 }, { 103677, 68 }, { 103903, 1 }, { 104040, 68 }, { 104047, 1 }, { 104052, 68 }, { 104057, 4 }, { 104057, 68 }, { 104062, 98 }, { 104091, 68 }, { 104189, 3 }, { 104283, 8 }, - { 104288, 4 }, { 104305, 3 }, { 104445, 68 }, { 104472, 68 }, { 104475, 1 }, { 104497, 4 }, { 104548, 68 }, { 104582, 68 }, { 104626, 1 }, { 104716, 68 }, { 104826, 68 }, { 104849, 68 }, { 104872, 1 }, { 104945, 1 }, { 104948, 68 }, { 105066, 68 }, { 105071, 1 }, { 105198, 4 }, { 105198, 4 }, { 105203, 68 }, { 105256, 6 }, - { 105263, 68 }, { 105329, 68 }, { 105515, 68 }, { 105566, 68 }, { 105566, 68 }, { 105585, 68 }, { 105678, 68 }, { 105852, 68 }, { 105877, 68 }, { 105911, 68 }, { 106022, 1 }, { 106033, 68 }, { 106080, 68 }, { 106192, 68 }, { 106220, 3 }, { 106243, 68 }, { 106323, 11 }, { 106371, 68 }, { 106608, 68 }, { 106624, 87 }, { 106680, 3 }, - { 106688, 1 }, { 106800, 1 }, { 106800, 1 }, { 106821, 4 }, { 106853, 1 }, { 106930, 3 }, { 106937, 68 }, { 106955, 68 }, { 106996, 68 }, { 106996, 1 }, { 107148, 4 }, { 107213, 16 }, { 107213, 68 }, { 107243, 68 }, { 107360, 68 }, { 107408, 68 }, { 107509, 4 }, { 107572, 68 }, { 107592, 68 }, { 107644, 5 }, { 107679, 68 }, - { 107705, 3 }, { 107761, 4 }, { 107780, 68 }, { 107825, 68 }, { 108007, 68 }, { 108041, 4 }, { 108058, 68 }, { 108071, 1 }, { 108132, 68 }, { 108164, 68 }, { 108189, 68 }, { 108210, 68 }, { 108330, 68 }, { 108430, 68 }, { 108450, 68 }, { 108469, 68 }, { 108484, 68 }, { 108533, 68 }, { 108588, 68 }, { 108594, 68 }, { 108690, 68 }, - { 108785, 76 }, { 108814, 68 }, { 108818, 1 }, { 108820, 68 }, { 108889, 68 }, { 108951, 68 }, { 108959, 68 }, { 108963, 68 }, { 109034, 68 }, { 109172, 1 }, { 109176, 68 }, { 109195, 3 }, { 109229, 68 }, { 109256, 68 }, { 109290, 68 }, { 109304, 68 }, { 109333, 68 }, { 109343, 4 }, { 109347, 7 }, { 109387, 68 }, { 109421, 1 }, - { 109497, 68 }, { 109501, 3 }, { 109513, 68 }, { 109525, 3 }, { 109625, 4 }, { 109710, 68 }, { 109740, 68 }, { 109751, 68 }, { 109761, 68 }, { 109890, 8 }, { 109891, 4 }, { 109909, 68 }, { 109923, 1 }, { 110017, 68 }, { 110046, 68 }, { 110111, 68 }, { 110258, 68 }, { 110340, 68 }, { 110352, 68 }, { 110398, 68 }, { 110583, 68 }, - { 110600, 13 }, { 110626, 3 }, { 110709, 68 }, { 110772, 4 }, { 110773, 68 }, { 110813, 1 }, { 110890, 68 }, { 110898, 68 }, { 110954, 68 }, { 111120, 68 }, { 111132, 3 }, { 111163, 8 }, { 111224, 68 }, { 111340, 68 }, { 111398, 68 }, { 111555, 68 }, { 111597, 3 }, { 111607, 68 }, { 111655, 68 }, { 111691, 3 }, { 111835, 68 }, - { 111854, 68 }, { 111876, 16 }, { 111884, 1 }, { 111884, 56 }, { 111929, 68 }, { 111941, 68 }, { 111969, 68 }, { 112003, 68 }, { 112165, 68 }, { 112365, 68 }, { 112450, 1 }, { 112521, 68 }, { 112649, 4 }, { 112665, 68 }, { 112881, 1 }, { 112882, 68 }, { 112906, 68 }, { 112951, 68 }, { 112994, 68 }, { 112997, 68 }, { 113002, 68 }, - { 113056, 1 }, { 113077, 68 }, { 113208, 1 }, { 113320, 68 }, { 113326, 3 }, { 113375, 68 }, { 113530, 30 }, { 113530, 30 }, { 113537, 1 }, { 113563, 14 }, { 113592, 68 }, { 113637, 68 }, { 113768, 68 }, { 113850, 5 }, { 113892, 68 }, { 113916, 68 }, { 113965, 68 }, { 113976, 68 }, { 114037, 68 }, { 114149, 1 }, { 114158, 9 }, - { 114201, 68 }, { 114262, 68 }, { 114268, 4 }, { 114353, 68 }, { 114388, 68 }, { 114404, 68 }, { 114428, 5 }, { 114438, 68 }, { 114541, 68 }, { 114550, 68 }, { 114561, 68 }, { 114625, 3 }, { 114730, 68 }, { 114770, 1 }, { 114815, 4 }, { 114998, 68 }, { 115077, 68 }, { 115093, 68 }, { 115120, 68 }, { 115194, 68 }, { 115216, 3 }, - { 115299, 68 }, { 115391, 3 }, { 115410, 68 }, { 115542, 33 }, { 115581, 68 }, { 115618, 68 }, { 115645, 23 }, { 115647, 68 }, { 115697, 68 }, { 115725, 68 }, { 115740, 68 }, { 115757, 68 }, { 115763, 68 }, { 115770, 68 }, { 115787, 68 }, { 115916, 68 }, { 115928, 68 }, { 115962, 68 }, { 116020, 68 }, { 116022, 1 }, { 116089, 68 }, - { 116159, 1 }, { 116196, 68 }, { 116247, 68 }, { 116254, 7 }, { 116336, 68 }, { 116409, 68 }, { 116459, 68 }, { 116569, 68 }, { 116619, 68 }, { 116688, 68 }, { 116733, 68 }, { 116807, 3 }, { 116843, 68 }, { 116886, 1 }, { 116902, 68 }, { 116931, 68 }, { 116952, 68 }, { 116952, 68 }, { 117177, 68 }, { 117189, 68 }, { 117206, 68 }, - { 117260, 29 }, { 117271, 6 }, { 117276, 3 }, { 117276, 5 }, { 117278, 3 }, { 117278, 68 }, { 117359, 4 }, { 117380, 68 }, { 117414, 1 }, { 117503, 68 }, { 117517, 68 }, { 117530, 68 }, { 117574, 4 }, { 117575, 5 }, { 117577, 68 }, { 117606, 68 }, { 117645, 68 }, { 117655, 68 }, { 117692, 68 }, { 117705, 1 }, { 117731, 1 }, - { 117762, 4 }, { 117780, 68 }, { 117974, 1 }, { 118057, 1 }, { 118099, 68 }, { 118107, 68 }, { 118113, 68 }, { 118175, 68 }, { 118198, 68 }, { 118232, 45 }, { 118326, 1 }, { 118438, 31 }, { 118469, 68 }, { 118521, 31 }, { 118565, 68 }, { 118593, 68 }, { 118602, 68 }, { 118652, 68 }, { 118668, 68 }, { 118689, 3 }, { 118703, 14 }, - { 118705, 68 }, { 118813, 68 }, { 118825, 68 }, { 118894, 3 }, { 118915, 68 }, { 118962, 68 }, { 118986, 68 }, { 119045, 68 }, { 119054, 1 }, { 119054, 1 }, { 119119, 68 }, { 119149, 68 }, { 119206, 1 }, { 119316, 68 }, { 119387, 68 }, { 119404, 3 }, { 119516, 68 }, { 119520, 68 }, { 119571, 3 }, { 119573, 68 }, { 119610, 5 }, - { 119621, 68 }, { 119623, 4 }, { 119672, 68 }, { 119692, 3 }, { 119734, 68 }, { 119742, 1 }, { 119754, 1 }, { 119785, 68 }, { 120001, 68 }, { 120115, 4 }, { 120260, 68 }, { 120314, 68 }, { 120416, 68 }, { 120435, 1 }, { 120450, 3 }, { 120530, 68 }, { 120550, 5 }, { 120730, 68 }, { 120731, 68 }, { 120751, 3 }, { 120755, 68 }, - { 120869, 68 }, { 120988, 68 }, { 121061, 68 }, { 121177, 68 }, { 121212, 68 }, { 121214, 1 }, { 121286, 68 }, { 121331, 1 }, { 121344, 68 }, { 121407, 68 }, { 121424, 1 }, { 121491, 68 }, { 121568, 76 }, { 121588, 6 }, { 121651, 68 }, { 121676, 68 }, { 121785, 4 }, { 121830, 3 }, { 121919, 1 }, { 121951, 68 }, { 121991, 1 }, - { 122056, 68 }, { 122062, 68 }, { 122144, 68 }, { 122183, 1 }, { 122331, 68 }, { 122466, 68 }, { 122558, 68 }, { 122570, 68 }, { 122676, 68 }, { 122733, 68 }, { 122774, 6 }, { 122783, 68 }, { 122825, 68 }, { 122865, 68 }, { 122884, 68 }, { 122892, 68 }, { 122911, 68 }, { 122929, 68 }, { 122936, 68 }, { 123190, 68 }, { 123271, 68 }, - { 123271, 68 }, { 123302, 7 }, { 123391, 68 }, { 123394, 68 }, { 123416, 1 }, { 123708, 68 }, { 123752, 68 }, { 123761, 68 }, { 123783, 68 }, { 123794, 68 }, { 123817, 68 }, { 123820, 1 }, { 123823, 1 }, { 123857, 3 }, { 123886, 56 }, { 124023, 1 }, { 124029, 68 }, { 124042, 68 }, { 124056, 3 }, { 124071, 6 }, { 124105, 5 }, - { 124143, 68 }, { 124191, 68 }, { 124207, 1 }, { 124257, 68 }, { 124306, 3 }, { 124338, 68 }, { 124388, 8 }, { 124400, 68 }, { 124418, 68 }, { 124502, 68 }, { 124521, 1 }, { 124533, 68 }, { 124645, 68 }, { 124685, 1 }, { 124694, 68 }, { 124700, 1 }, { 124736, 68 }, { 124891, 7 }, { 124920, 68 }, { 124983, 68 }, { 125014, 68 }, - { 125038, 68 }, { 125084, 68 }, { 125162, 68 }, { 125193, 68 }, { 125285, 68 }, { 125368, 68 }, { 125409, 68 }, { 125570, 68 }, { 125601, 68 }, { 125641, 1 }, { 125721, 68 }, { 125731, 68 }, { 125803, 68 }, { 125904, 68 }, { 125973, 68 }, { 126018, 1 }, { 126034, 5 }, { 126094, 1 }, { 126144, 1 }, { 126195, 68 }, { 126297, 68 }, - { 126389, 68 }, { 126429, 68 }, { 126439, 68 }, { 126499, 68 }, { 126501, 1 }, { 126587, 68 }, { 126663, 68 }, { 126681, 68 }, { 126687, 1 }, { 126781, 68 }, { 126783, 68 }, { 126840, 8 }, { 126843, 68 }, { 126959, 68 }, { 127015, 68 }, { 127101, 68 }, { 127149, 68 }, { 127197, 54 }, { 127268, 68 }, { 127372, 68 }, { 127385, 68 }, - { 127473, 4 }, { 127539, 68 }, { 127598, 68 }, { 127613, 14 }, { 127683, 3 }, { 127684, 68 }, { 127697, 68 }, { 127698, 3 }, { 127773, 68 }, { 127781, 1 }, { 127839, 68 }, { 127905, 68 }, { 127949, 68 }, { 128035, 68 }, { 128046, 1 }, { 128167, 68 }, { 128271, 68 }, { 128307, 1 }, { 128320, 68 }, { 128330, 68 }, { 128375, 68 }, - { 128381, 4 }, { 128447, 68 }, { 128462, 68 }, { 128466, 3 }, { 128466, 68 }, { 128496, 68 }, { 128589, 68 }, { 128616, 3 }, { 128679, 1 }, { 128770, 1 }, { 128793, 68 }, { 128802, 68 }, { 128813, 68 }, { 128900, 68 }, { 128949, 68 }, { 129269, 68 }, { 129271, 3 }, { 129278, 68 }, { 129343, 1 }, { 129408, 67 }, { 129408, 1 }, - { 129421, 6 }, { 129461, 68 }, { 129469, 3 }, { 129482, 68 }, { 129502, 68 }, { 129512, 68 }, { 129551, 68 }, { 129629, 68 }, { 129632, 68 }, { 129679, 1 }, { 129725, 68 }, { 130007, 68 }, { 130018, 16 }, { 130057, 68 }, { 130071, 68 }, { 130087, 68 }, { 130188, 1 }, { 130202, 68 }, { 130316, 68 }, { 130328, 1 }, { 130466, 68 }, - { 130549, 68 }, { 130649, 68 }, { 130705, 3 }, { 130800, 68 }, { 130907, 68 }, { 130989, 68 }, { 131103, 68 }, { 131127, 68 }, { 131200, 5 }, { 131241, 6 }, { 131351, 68 }, { 131413, 68 }, { 131448, 68 }, { 131599, 68 }, { 131634, 1 }, { 131687, 68 }, { 131739, 68 }, { 131758, 68 }, { 131765, 68 }, { 131787, 3 }, { 131819, 3 }, - { 131868, 68 }, { 131886, 68 }, { 131901, 4 }, { 131977, 68 }, { 131990, 68 }, { 132035, 68 }, { 132035, 68 }, { 132043, 68 }, { 132173, 68 }, { 132181, 4 }, { 132181, 6 }, { 132194, 5 }, { 132252, 68 }, { 132262, 6 }, { 132271, 1 }, { 132285, 68 }, { 132328, 68 }, { 132335, 1 }, { 132337, 1 }, { 132389, 5 }, { 132430, 68 }, - { 132451, 68 }, { 132499, 87 }, { 132503, 1 }, { 132520, 4 }, { 132541, 4 }, { 132860, 68 }, { 132862, 4 }, { 132874, 168 }, { 132874, 13 }, { 132875, 168 }, { 132911, 68 }, { 132973, 68 }, { 133051, 68 }, { 133062, 68 }, { 133067, 68 }, { 133138, 68 }, { 133184, 68 }, { 133231, 68 }, { 133297, 3 }, { 133344, 68 }, { 133385, 4 }, - { 133408, 68 }, { 133464, 68 }, { 133522, 68 }, { 133631, 68 }, { 133631, 68 }, { 133702, 68 }, { 133705, 1 }, { 133721, 68 }, { 133746, 68 }, { 133773, 3 }, { 133819, 68 }, { 133843, 68 }, { 133929, 68 }, { 133946, 68 }, { 134113, 4 }, { 134151, 68 }, { 134289, 1 }, { 134385, 68 }, { 134429, 68 }, { 134506, 68 }, { 134511, 68 }, - { 134521, 68 }, { 134558, 1 }, { 134710, 68 }, { 134738, 68 }, { 134751, 3 }, { 134818, 68 }, { 134820, 4 }, { 134879, 68 }, { 134919, 68 }, { 134947, 68 }, { 134948, 3 }, { 135040, 3 }, { 135125, 10 }, { 135155, 68 }, { 135228, 68 }, { 135255, 68 }, { 135296, 3 }, { 135322, 68 }, { 135349, 68 }, { 135428, 3 }, { 135476, 1 }, - { 135503, 68 }, { 135524, 68 }, { 135550, 4 }, { 135594, 68 }, { 135597, 68 }, { 135624, 3 }, { 135741, 68 }, { 135753, 68 }, { 135842, 68 }, { 135853, 68 }, { 135896, 3 }, { 136004, 1 }, { 136061, 1 }, { 136068, 1 }, { 136106, 68 }, { 136145, 68 }, { 136145, 68 }, { 136173, 68 }, { 136186, 68 }, { 136196, 68 }, { 136201, 68 }, - { 136211, 68 }, { 136268, 68 }, { 136298, 68 }, { 136377, 68 }, { 136420, 68 }, { 136475, 23 }, { 136486, 1 }, { 136554, 68 }, { 136641, 68 }, { 136770, 1 }, { 136873, 68 }, { 136877, 1 }, { 136906, 68 }, { 137092, 68 }, { 137143, 68 }, { 137200, 3 }, { 137232, 68 }, { 137239, 68 }, { 137248, 68 }, { 137281, 1 }, { 137301, 68 }, - { 137314, 3 }, { 137352, 1 }, { 137365, 68 }, { 137375, 68 }, { 137411, 68 }, { 137424, 68 }, { 137516, 68 }, { 137532, 68 }, { 137593, 68 }, { 137600, 68 }, { 137658, 68 }, { 137703, 68 }, { 137766, 68 }, { 137791, 68 }, { 137801, 68 }, { 137864, 68 }, { 137870, 3 }, { 137931, 68 }, { 138009, 3 }, { 138013, 1 }, { 138013, 1 }, - { 138066, 68 }, { 138073, 68 }, { 138114, 68 }, { 138150, 68 }, { 138236, 68 }, { 138276, 68 }, { 138286, 68 }, { 138298, 3 }, { 138309, 1 }, { 138373, 3 }, { 138524, 68 }, { 138535, 1 }, { 138593, 4 }, { 138611, 1 }, { 138725, 68 }, { 138807, 68 }, { 138819, 3 }, { 138849, 5 }, { 138867, 68 }, { 138907, 68 }, { 138930, 3 }, - { 139026, 68 }, { 139102, 68 }, { 139108, 3 }, { 139184, 1 }, { 139209, 3 }, { 139282, 68 }, { 139289, 4 }, { 139382, 1 }, { 139421, 1 }, { 139436, 68 }, { 139450, 1 }, { 139523, 3 }, { 139533, 68 }, { 139590, 68 }, { 139590, 68 }, { 139722, 68 }, { 139785, 68 }, { 139785, 1 }, { 139798, 68 }, { 139813, 68 }, { 139868, 68 }, - { 139935, 3 }, { 139990, 3 }, { 140050, 68 }, { 140177, 68 }, { 140177, 4 }, { 140408, 68 }, { 140420, 3 }, { 140461, 68 }, { 140578, 15 }, { 140605, 1368 }, { 140662, 1 }, { 140755, 68 }, { 140786, 68 }, { 140846, 68 }, { 140874, 68 }, { 140959, 1 }, { 140973, 68 }, { 141128, 68 }, { 141132, 68 }, { 141257, 68 }, { 141290, 1 }, - { 141360, 68 }, { 141472, 68 }, { 141545, 68 }, { 141545, 68 }, { 141575, 1 }, { 141606, 5 }, { 141655, 68 }, { 141735, 68 }, { 141767, 5 }, { 141796, 68 }, { 141841, 68 }, { 141915, 68 }, { 141923, 1 }, { 141932, 68 }, { 141994, 68 }, { 142018, 68 }, { 142029, 3 }, { 142072, 68 }, { 142128, 68 }, { 142133, 1 }, { 142261, 68 }, - { 142304, 1 }, { 142400, 68 }, { 142401, 68 }, { 142409, 68 }, { 142479, 68 }, { 142522, 1 }, { 142552, 1 }, { 142589, 68 }, { 142596, 68 }, { 142753, 1 }, { 142766, 68 }, { 142796, 68 }, { 142836, 68 }, { 142871, 68 }, { 143058, 3 }, { 143059, 6 }, { 143063, 3 }, { 143065, 68 }, { 143141, 4 }, { 143173, 68 }, { 143374, 68 }, - { 143399, 68 }, { 143406, 68 }, { 143429, 3 }, { 143430, 68 }, { 143462, 1 }, { 143579, 68 }, { 143663, 68 }, { 143844, 3 }, { 143851, 68 }, { 143926, 68 }, { 143931, 68 }, { 144051, 6 }, { 144085, 10 }, { 144147, 68 }, { 144188, 4 }, { 144238, 4 }, { 144353, 68 }, { 144465, 68 }, { 144474, 68 }, { 144637, 68 }, { 144638, 1 }, - { 144648, 1 }, { 144661, 3 }, { 144812, 68 }, { 144847, 68 }, { 144901, 8 }, { 145058, 68 }, { 145122, 8 }, { 145134, 68 }, { 145150, 68 }, { 145299, 1 }, { 145313, 68 }, { 145314, 3 }, { 145374, 68 }, { 145412, 68 }, { 145432, 68 }, { 145446, 68 }, { 145534, 3 }, { 145592, 68 }, { 145614, 68 }, { 145648, 68 }, { 145721, 68 }, - { 145858, 1 }, { 145970, 3 }, { 145984, 3 }, { 146004, 68 }, { 146016, 3 }, { 146048, 68 }, { 146097, 3 }, { 146103, 68 }, { 146136, 68 }, { 146194, 3 }, { 146230, 1 }, { 146254, 68 }, { 146261, 4 }, { 146269, 4 }, { 146393, 68 }, { 146411, 3 }, { 146501, 68 }, { 146547, 68 }, { 146547, 68 }, { 146573, 68 }, { 146616, 68 }, - { 146622, 3 }, { 146728, 3 }, { 146781, 5 }, { 146805, 4 }, { 146921, 68 }, { 147002, 3 }, { 147072, 68 }, { 147159, 68 }, { 147170, 68 }, { 147203, 1 }, { 147245, 68 }, { 147278, 68 }, { 147422, 68 }, { 147471, 68 }, { 147491, 68 }, { 147607, 23 }, { 147693, 68 }, { 147763, 68 }, { 147775, 6 }, { 147776, 4 }, { 147824, 68 }, - { 147922, 68 }, { 147922, 68 }, { 147937, 68 }, { 147957, 68 }, { 147980, 68 }, { 148008, 68 }, { 148018, 68 }, { 148046, 3 }, { 148071, 4 }, { 148106, 3 }, { 148122, 68 }, { 148139, 68 }, { 148175, 68 }, { 164238, 18 }, { 164315, 28 }, { 164449, 28 }, { 164529, 28 }, { 164574, 348 }, { 164591, 968 }, { 164595, 28 }, - { 164611, 28 }, { 164623, 48 }, { 164632, 108 }, { 164691, 28 }, { 164706, 28 }, { 164755, 28 }, { 164761, 28 }, { 164973, 28 }, { 165030, 28 }, { 165090, 28 }, { 165099, 18 }, { 165126, 28 }, { 165188, 28 }, { 165205, 28 }, { 165275, 18 }, { 165347, 28 }, { 165381, 28 }, { 165562, 28 }, { 165563, 18 }, { 165594, 568 }, - { 165641, 868 }, { 165663, 68 }, { 165759, 28 }, { 165811, 28 }, { 165822, 18 }, { 165830, 18 }, { 165903, 18 }, { 165921, 28 }, { 165953, 18 }, { 166022, 18 }, { 166294, 28 }, { 166333, 28 }, { 166420, 28 }, { 166433, 28 }, { 166442, 18 }, { 166536, 28 }, { 166543, 28 }, { 166556, 28 }, { 166571, 28 }, { 166575, 18 }, - { 166588, 28 }, { 166601, 678 }, { 166663, 788 }, { 166692, 18 }, { 166710, 28 }, { 166759, 28 }, { 166785, 38 }, { 166842, 28 }, { 166843, 28 }, { 166864, 28 }, { 166902, 28 }, { 166996, 28 }, { 166999, 28 }, { 167038, 28 }, { 167112, 48 }, { 167112, 28 }, { 167177, 28 }, { 167180, 28 }, { 167229, 18 }, { 167298, 28 }, - { 167306, 48 }, { 167309, 38 }, { 167402, 28 }, { 167405, 878 }, { 167433, 568 }, { 167435, 18 }, { 167461, 38 }, { 167553, 38 }, { 167688, 58 }, { 167689, 28 }, { 167709, 28 }, { 167744, 28 }, { 167821, 28 }, { 167825, 28 }, { 167925, 108 }, { 167969, 28 }, { 168024, 28 }, { 168089, 28 }, { 168104, 28 }, { 168194, 28 }, - { 168305, 28 }, { 168316, 28 }, { 168366, 28 }, { 168423, 28 }, { 168568, 38 }, { 168582, 558 }, { 168615, 768 }, { 168618, 28 }, { 168638, 28 }, { 168671, 28 }, { 168736, 28 }, { 168747, 28 }, { 168750, 48 }, { 168808, 38 }, { 168814, 48 }, { 168820, 28 }, { 168914, 28 }, { 168968, 28 }, { 168979, 28 }, { 169006, 28 }, - { 169069, 28 }, { 169106, 38 }, { 169158, 28 }, { 169158, 28 }, { 169189, 28 }, { 169253, 28 }, { 169259, 18 }, { 169279, 658 }, { 169325, 568 }, { 169349, 28 }, { 169353, 28 }, { 169378, 28 }, { 169432, 28 }, { 169476, 18 }, { 169476, 18 }, { 169525, 28 }, { 169538, 78 }, { 169555, 28 }, { 169571, 28 }, { 169594, 48 }, - { 169687, 28 }, { 169799, 28 }, { 169831, 28 }, { 170042, 28 }, { 170061, 28 }, { 170065, 18 }, { 170128, 68 }, { 170148, 208 }, { 170215, 708 }, { 170256, 608 }, { 170266, 698 }, { 170275, 78 }, { 170277, 68 }, { 170500, 38 }, { 170516, 38 }, { 170601, 28 }, { 170666, 28 }, { 170668, 48 }, { 170668, 18 }, { 170716, 38 }, - { 170728, 38 }, { 170735, 58 }, { 170847, 38 }, { 170852, 98 }, { 170858, 438 }, { 170859, 568 }, { 170956, 568 }, { 170956, 18 }, { 170967, 28 }, { 171005, 28 }, { 171113, 28 }, { 171279, 28 }, { 171400, 28 }, { 171405, 28 }, { 171448, 18 }, { 171490, 28 }, { 171567, 328 }, { 171590, 28 }, { 171723, 28 }, { 171737, 38 }, - { 171958, 28 }, { 171967, 68 }, { 164238, 18 }, { 164315, 28 }, { 164449, 28 }, { 164529, 28 }, { 164574, 348 }, { 164591, 968 }, { 164595, 28 }, { 164611, 28 }, { 164623, 48 }, { 164632, 108 }, { 164691, 28 }, { 164706, 28 }, { 164755, 28 }, { 164761, 28 }, { 164973, 28 }, { 165030, 28 }, { 165090, 28 }, { 165099, 18 }, - { 165126, 28 }, { 165188, 28 }, { 165205, 28 }, { 165275, 18 }, { 165347, 28 }, { 165381, 28 }, { 165562, 28 }, { 165563, 18 }, { 165594, 568 }, { 165641, 868 }, { 165663, 68 }, { 165759, 28 }, { 165811, 28 }, { 165822, 18 }, { 165830, 18 }, { 165903, 18 }, { 165921, 28 }, { 165953, 18 }, { 166022, 18 }, { 166294, 28 }, - { 166333, 28 }, { 166420, 28 }, { 166433, 28 }, { 166442, 18 }, { 166536, 28 }, { 166543, 28 }, { 166556, 28 }, { 166571, 28 }, { 166575, 18 }, { 166588, 28 }, { 166601, 678 }, { 166663, 788 }, { 166692, 18 }, { 166710, 28 }, { 166759, 28 }, { 166785, 38 }, { 166842, 28 }, { 166843, 28 }, { 166864, 28 }, { 166902, 28 }, - { 166996, 28 }, { 166999, 28 }, { 167038, 28 }, { 167112, 48 }, { 167112, 28 }, { 167177, 28 }, { 167180, 28 }, { 167229, 18 }, { 167298, 28 }, { 167306, 48 }, { 167309, 38 }, { 167402, 28 }, { 167405, 878 }, { 167433, 568 }, { 167435, 18 }, { 167461, 38 }, { 167553, 38 }, { 167688, 58 }, { 167689, 28 }, { 167709, 28 }, - { 167744, 28 }, { 167821, 28 }, { 167825, 28 }, { 167925, 108 }, { 167969, 28 }, { 168024, 28 }, { 168089, 28 }, { 168104, 28 }, { 168194, 28 }, { 168305, 28 }, { 168316, 28 }, { 168366, 28 }, { 168423, 28 }, { 168568, 38 }, { 168582, 558 }, { 168615, 768 }, { 168618, 28 }, { 168638, 28 }, { 168671, 28 }, { 168736, 28 }, - { 168747, 28 }, { 168750, 48 }, { 168808, 38 }, { 168814, 48 }, { 168820, 28 }, { 168914, 28 }, { 168968, 28 }, { 168979, 28 }, { 169006, 28 }, { 169069, 28 }, { 169106, 38 }, { 169158, 28 }, { 169158, 28 }, { 169189, 28 }, { 169253, 28 }, { 169259, 18 }, { 169279, 658 }, { 169325, 568 }, { 169349, 28 }, { 169353, 28 }, - { 169378, 28 }, { 169432, 28 }, { 169476, 18 }, { 169476, 18 }, { 169525, 28 }, { 169538, 78 }, { 169555, 28 }, { 169571, 28 }, { 169594, 48 }, { 169687, 28 }, { 169799, 28 }, { 169831, 28 }, { 170042, 28 }, { 170061, 28 }, { 170065, 18 }, { 170128, 68 }, { 170148, 208 }, { 170215, 708 }, { 170256, 608 }, { 170266, 698 }, - { 170275, 78 }, { 170277, 68 }, { 170500, 38 }, { 170516, 38 }, { 170601, 28 }, { 170666, 28 }, { 170668, 48 }, { 170668, 18 }, { 170716, 38 }, { 170728, 38 }, { 170735, 58 }, { 170847, 38 }, { 170852, 98 }, { 170858, 438 }, { 170859, 568 }, { 170956, 568 }, { 170956, 18 }, { 170967, 28 }, { 171005, 28 }, { 171113, 28 }, - { 171279, 28 }, { 171400, 28 }, { 171405, 28 }, { 171448, 18 }, { 171490, 28 }, { 171567, 328 }, { 171590, 28 }, { 171723, 28 }, { 171737, 38 }, { 171958, 28 }, { 171967, 2 } }; - } - } \ No newline at end of file diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixTimer.java b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixTimer.java index e07afaef9..235be1c74 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixTimer.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixTimer.java @@ -15,8 +15,6 @@ */ package com.netflix.hystrix.util; -import static org.junit.Assert.*; - import java.lang.ref.Reference; import java.lang.ref.SoftReference; import java.util.concurrent.ScheduledFuture; @@ -26,7 +24,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,7 +63,7 @@ public static void reset() { } } - private AtomicReference executor = new AtomicReference(); + /* package */ AtomicReference executor = new AtomicReference(); /** * Add a {@link TimerListener} that will be executed until it is garbage collected or removed by clearing the returned {@link Reference}. @@ -142,8 +139,8 @@ protected void startThreadIfNeeded() { } } - private static class ScheduledExecutor { - private volatile ScheduledThreadPoolExecutor executor; + /* package */ static class ScheduledExecutor { + /* package */ volatile ScheduledThreadPoolExecutor executor; private volatile boolean initialized; /** @@ -190,210 +187,4 @@ public static interface TimerListener { public int getIntervalTimeInMilliseconds(); } - public static class UnitTest { - - @Test - public void testSingleCommandSingleInterval() { - HystrixTimer timer = HystrixTimer.getInstance(); - TestListener l1 = new TestListener(50, "A"); - timer.addTimerListener(l1); - - TestListener l2 = new TestListener(50, "B"); - timer.addTimerListener(l2); - - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - // we should have 7 or more 50ms ticks within 500ms - System.out.println("l1 ticks: " + l1.tickCount.get()); - System.out.println("l2 ticks: " + l2.tickCount.get()); - assertTrue(l1.tickCount.get() > 7); - assertTrue(l2.tickCount.get() > 7); - } - - @Test - public void testSingleCommandMultipleIntervals() { - HystrixTimer timer = HystrixTimer.getInstance(); - TestListener l1 = new TestListener(100, "A"); - timer.addTimerListener(l1); - - TestListener l2 = new TestListener(10, "B"); - timer.addTimerListener(l2); - - TestListener l3 = new TestListener(25, "C"); - timer.addTimerListener(l3); - - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - // we should have 3 or more 100ms ticks within 500ms - System.out.println("l1 ticks: " + l1.tickCount.get()); - assertTrue(l1.tickCount.get() >= 3); - // but it can't be more than 6 - assertTrue(l1.tickCount.get() < 6); - - // we should have 30 or more 10ms ticks within 500ms - System.out.println("l2 ticks: " + l2.tickCount.get()); - assertTrue(l2.tickCount.get() > 30); - assertTrue(l2.tickCount.get() < 550); - - // we should have 15-20 25ms ticks within 500ms - System.out.println("l3 ticks: " + l3.tickCount.get()); - assertTrue(l3.tickCount.get() > 14); - assertTrue(l3.tickCount.get() < 25); - } - - @Test - public void testSingleCommandRemoveListener() { - HystrixTimer timer = HystrixTimer.getInstance(); - TestListener l1 = new TestListener(50, "A"); - timer.addTimerListener(l1); - - TestListener l2 = new TestListener(50, "B"); - Reference l2ref = timer.addTimerListener(l2); - - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - // we should have 7 or more 50ms ticks within 500ms - System.out.println("l1 ticks: " + l1.tickCount.get()); - System.out.println("l2 ticks: " + l2.tickCount.get()); - assertTrue(l1.tickCount.get() > 7); - assertTrue(l2.tickCount.get() > 7); - - // remove l2 - l2ref.clear(); - - // reset counts - l1.tickCount.set(0); - l2.tickCount.set(0); - - // wait for time to pass again - try { - Thread.sleep(500); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - // we should have 7 or more 50ms ticks within 500ms - System.out.println("l1 ticks: " + l1.tickCount.get()); - System.out.println("l2 ticks: " + l2.tickCount.get()); - // l1 should continue ticking - assertTrue(l1.tickCount.get() > 7); - // we should have no ticks on l2 because we removed it - System.out.println("tickCount.get(): " + l2.tickCount.get() + " on l2: " + l2); - assertEquals(0, l2.tickCount.get()); - } - - @Test - public void testReset() { - HystrixTimer timer = HystrixTimer.getInstance(); - TestListener l1 = new TestListener(50, "A"); - timer.addTimerListener(l1); - - ScheduledExecutor ex = timer.executor.get(); - - assertFalse(ex.executor.isShutdown()); - - // perform reset which should shut it down - HystrixTimer.reset(); - - assertTrue(ex.executor.isShutdown()); - assertNull(timer.executor.get()); - - // assert it starts up again on use - TestListener l2 = new TestListener(50, "A"); - timer.addTimerListener(l2); - - ScheduledExecutor ex2 = timer.executor.get(); - - assertFalse(ex2.executor.isShutdown()); - - // reset again to shutdown what we just started - HystrixTimer.reset(); - // try resetting again to make sure it's idempotent (ie. doesn't blow up on an NPE) - HystrixTimer.reset(); - } - - private static class TestListener implements TimerListener { - - private final int interval; - AtomicInteger tickCount = new AtomicInteger(); - - TestListener(int interval, String value) { - this.interval = interval; - } - - @Override - public void tick() { - tickCount.incrementAndGet(); - } - - @Override - public int getIntervalTimeInMilliseconds() { - return interval; - } - - } - - public static void main(String args[]) { - PlayListener l1 = new PlayListener(); - PlayListener l2 = new PlayListener(); - PlayListener l3 = new PlayListener(); - PlayListener l4 = new PlayListener(); - PlayListener l5 = new PlayListener(); - - Reference ref = HystrixTimer.getInstance().addTimerListener(l1); - HystrixTimer.getInstance().addTimerListener(l2); - HystrixTimer.getInstance().addTimerListener(l3); - - HystrixTimer.getInstance().addTimerListener(l4); - - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - ref.clear(); - HystrixTimer.getInstance().addTimerListener(l5); - - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - System.out.println("counter: " + l1.counter); - System.out.println("counter: " + l2.counter); - System.out.println("counter: " + l3.counter); - System.out.println("counter: " + l4.counter); - System.out.println("counter: " + l5.counter); - - } - - public static class PlayListener implements TimerListener { - int counter = 0; - - @Override - public void tick() { - counter++; - } - - @Override - public int getIntervalTimeInMilliseconds() { - return 10; - } - - } - } } diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCircuitBreakerTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCircuitBreakerTest.java new file mode 100644 index 000000000..6ed2a0d05 --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCircuitBreakerTest.java @@ -0,0 +1,497 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.netflix.hystrix.HystrixCircuitBreaker.HystrixCircuitBreakerImpl; +import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifierDefault; +import com.netflix.hystrix.util.HystrixRollingNumberEvent; + +public class HystrixCircuitBreakerTest { + + /** + * A simple circuit breaker intended for unit testing of the {@link HystrixCommand} object, NOT production use. + *

+ * This uses simple logic to 'trip' the circuit after 3 subsequent failures and doesn't recover. + */ + public static class TestCircuitBreaker implements HystrixCircuitBreaker { + + final HystrixCommandMetrics metrics; + private boolean forceShortCircuit = false; + + public TestCircuitBreaker() { + this.metrics = getMetrics(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter()); + forceShortCircuit = false; + } + + public TestCircuitBreaker setForceShortCircuit(boolean value) { + this.forceShortCircuit = value; + return this; + } + + @Override + public boolean isOpen() { + if (forceShortCircuit) { + return true; + } else { + return metrics.getCumulativeCount(HystrixRollingNumberEvent.FAILURE) >= 3; + } + } + + @Override + public void markSuccess() { + // we don't need to do anything since we're going to permanently trip the circuit + } + + @Override + public boolean allowRequest() { + return !isOpen(); + } + + } + + private HystrixCommandKey key = CommandKeyForUnitTest.KEY_ONE; + + /** + * Test that if all 'marks' are successes during the test window that it does NOT trip the circuit. + * Test that if all 'marks' are failures during the test window that it trips the circuit. + */ + @Test + public void testTripCircuit() { + try { + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter(); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + metrics.markSuccess(1000); + metrics.markSuccess(1000); + metrics.markSuccess(1000); + metrics.markSuccess(1000); + + // this should still allow requests as everything has been successful + assertTrue(cb.allowRequest()); + assertFalse(cb.isOpen()); + + // fail + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + + // everything has failed in the test window so we should return false now + assertFalse(cb.allowRequest()); + assertTrue(cb.isOpen()); + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * Test that if the % of failures is higher than the threshold that the circuit trips. + */ + @Test + public void testTripCircuitOnFailuresAboveThreshold() { + try { + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter(); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + // this should start as allowing requests + assertTrue(cb.allowRequest()); + assertFalse(cb.isOpen()); + + // success with high latency + metrics.markSuccess(400); + metrics.markSuccess(400); + metrics.markFailure(10); + metrics.markSuccess(400); + metrics.markFailure(10); + metrics.markFailure(10); + metrics.markSuccess(400); + metrics.markFailure(10); + metrics.markFailure(10); + + // this should trip the circuit as the error percentage is above the threshold + assertFalse(cb.allowRequest()); + assertTrue(cb.isOpen()); + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * Test that if the % of failures is higher than the threshold that the circuit trips. + */ + @Test + public void testCircuitDoesNotTripOnFailuresBelowThreshold() { + try { + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter(); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + // this should start as allowing requests + assertTrue(cb.allowRequest()); + assertFalse(cb.isOpen()); + + // success with high latency + metrics.markSuccess(400); + metrics.markSuccess(400); + metrics.markFailure(10); + metrics.markSuccess(400); + metrics.markSuccess(40); + metrics.markSuccess(400); + metrics.markFailure(10); + metrics.markFailure(10); + + // this should remain open as the failure threshold is below the percentage limit + assertTrue(cb.allowRequest()); + assertFalse(cb.isOpen()); + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * Test that if all 'marks' are timeouts that it will trip the circuit. + */ + @Test + public void testTripCircuitOnTimeouts() { + try { + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter(); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + // this should start as allowing requests + assertTrue(cb.allowRequest()); + assertFalse(cb.isOpen()); + + // timeouts + metrics.markTimeout(2000); + metrics.markTimeout(2000); + metrics.markTimeout(2000); + metrics.markTimeout(2000); + + // everything has been a timeout so we should not allow any requests + assertFalse(cb.allowRequest()); + assertTrue(cb.isOpen()); + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * Test that if the % of timeouts is higher than the threshold that the circuit trips. + */ + @Test + public void testTripCircuitOnTimeoutsAboveThreshold() { + try { + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter(); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + // this should start as allowing requests + assertTrue(cb.allowRequest()); + assertFalse(cb.isOpen()); + + // success with high latency + metrics.markSuccess(400); + metrics.markSuccess(400); + metrics.markTimeout(10); + metrics.markSuccess(400); + metrics.markTimeout(10); + metrics.markTimeout(10); + metrics.markSuccess(400); + metrics.markTimeout(10); + metrics.markTimeout(10); + + // this should trip the circuit as the error percentage is above the threshold + assertFalse(cb.allowRequest()); + assertTrue(cb.isOpen()); + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * Test that on an open circuit that a single attempt will be allowed after a window of time to see if issues are resolved. + */ + @Test + public void testSingleTestOnOpenCircuitAfterTimeWindow() { + try { + int sleepWindow = 200; + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + // fail + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + + // everything has failed in the test window so we should return false now + assertFalse(cb.allowRequest()); + assertTrue(cb.isOpen()); + + // wait for sleepWindow to pass + Thread.sleep(sleepWindow + 50); + + // we should now allow 1 request + assertTrue(cb.allowRequest()); + // but the circuit should still be open + assertTrue(cb.isOpen()); + // and further requests are still blocked + assertFalse(cb.allowRequest()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * Test that an open circuit is closed after 1 success. + */ + @Test + public void testCircuitClosedAfterSuccess() { + try { + int sleepWindow = 200; + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + // fail + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markTimeout(1000); + + // everything has failed in the test window so we should return false now + assertFalse(cb.allowRequest()); + assertTrue(cb.isOpen()); + + // wait for sleepWindow to pass + Thread.sleep(sleepWindow + 50); + + // we should now allow 1 request + assertTrue(cb.allowRequest()); + // but the circuit should still be open + assertTrue(cb.isOpen()); + // and further requests are still blocked + assertFalse(cb.allowRequest()); + + // the 'singleTest' succeeds so should cause the circuit to be closed + metrics.markSuccess(500); + cb.markSuccess(); + + // all requests should be open again + assertTrue(cb.allowRequest()); + assertTrue(cb.allowRequest()); + assertTrue(cb.allowRequest()); + // and the circuit should be closed again + assertFalse(cb.isOpen()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * Test that an open circuit is closed after 1 success... when the sleepWindow is smaller than the statisticalWindow and 'failure' stats are still sticking around. + *

+ * This means that the statistical window needs to be cleared otherwise it will still calculate the failure percentage below the threshold and immediately open the circuit again. + */ + @Test + public void testCircuitClosedAfterSuccessAndClearsStatisticalWindow() { + try { + int statisticalWindow = 200; + int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow).withMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + // fail + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + + // everything has failed in the test window so we should return false now + assertFalse(cb.allowRequest()); + assertTrue(cb.isOpen()); + + // wait for sleepWindow to pass + Thread.sleep(sleepWindow + 50); + + // we should now allow 1 request + assertTrue(cb.allowRequest()); + // but the circuit should still be open + assertTrue(cb.isOpen()); + // and further requests are still blocked + assertFalse(cb.allowRequest()); + + // the 'singleTest' succeeds so should cause the circuit to be closed + metrics.markSuccess(500); + cb.markSuccess(); + + // all requests should be open again + assertTrue(cb.allowRequest()); + assertTrue(cb.allowRequest()); + assertTrue(cb.allowRequest()); + // and the circuit should be closed again + assertFalse(cb.isOpen()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * Over a period of several 'windows' a single attempt will be made and fail and then finally succeed and close the circuit. + *

+ * Ensure the circuit is kept open through the entire testing period and that only the single attempt in each window is made. + */ + @Test + public void testMultipleTimeWindowRetriesBeforeClosingCircuit() { + try { + int sleepWindow = 200; + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + // fail + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + + // everything has failed in the test window so we should return false now + assertFalse(cb.allowRequest()); + assertTrue(cb.isOpen()); + + // wait for sleepWindow to pass + Thread.sleep(sleepWindow + 50); + + // we should now allow 1 request + assertTrue(cb.allowRequest()); + // but the circuit should still be open + assertTrue(cb.isOpen()); + // and further requests are still blocked + assertFalse(cb.allowRequest()); + + // the 'singleTest' fails so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep + metrics.markFailure(1000); + + assertFalse(cb.allowRequest()); + assertFalse(cb.allowRequest()); + assertFalse(cb.allowRequest()); + + // wait for sleepWindow to pass + Thread.sleep(sleepWindow + 50); + + // we should now allow 1 request + assertTrue(cb.allowRequest()); + // but the circuit should still be open + assertTrue(cb.isOpen()); + // and further requests are still blocked + assertFalse(cb.allowRequest()); + + // the 'singleTest' fails again so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep + metrics.markFailure(1000); + + assertFalse(cb.allowRequest()); + assertFalse(cb.allowRequest()); + assertFalse(cb.allowRequest()); + + // wait for sleepWindow to pass + Thread.sleep(sleepWindow + 50); + + // we should now allow 1 request + assertTrue(cb.allowRequest()); + // but the circuit should still be open + assertTrue(cb.isOpen()); + // and further requests are still blocked + assertFalse(cb.allowRequest()); + + // now it finally succeeds + metrics.markSuccess(200); + cb.markSuccess(); + + // all requests should be open again + assertTrue(cb.allowRequest()); + assertTrue(cb.allowRequest()); + assertTrue(cb.allowRequest()); + // and the circuit should be closed again + assertFalse(cb.isOpen()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * When volume of reporting during a statistical window is lower than a defined threshold the circuit + * will not trip regardless of whatever statistics are calculated. + */ + @Test + public void testLowVolumeDoesNotTripCircuit() { + try { + int sleepWindow = 200; + int lowVolume = 5; + + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withCircuitBreakerSleepWindowInMilliseconds(sleepWindow).withCircuitBreakerRequestVolumeThreshold(lowVolume); + HystrixCommandMetrics metrics = getMetrics(properties); + HystrixCircuitBreaker cb = getCircuitBreaker(key, CommandOwnerForUnitTest.OWNER_TWO, metrics, properties); + + // fail + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + metrics.markFailure(1000); + + // even though it has all failed we won't trip the circuit because the volume is low + assertTrue(cb.allowRequest()); + assertFalse(cb.isOpen()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + } + + /** + * Utility method for creating {@link HystrixCommandMetrics} for unit tests. + */ + private static HystrixCommandMetrics getMetrics(HystrixCommandProperties.Setter properties) { + return new HystrixCommandMetrics(CommandKeyForUnitTest.KEY_ONE, CommandOwnerForUnitTest.OWNER_ONE, HystrixCommandPropertiesTest.asMock(properties), HystrixEventNotifierDefault.getInstance()); + } + + /** + * Utility method for creating {@link HystrixCircuitBreaker} for unit tests. + */ + private static HystrixCircuitBreaker getCircuitBreaker(HystrixCommandKey key, HystrixCommandGroupKey commandGroup, HystrixCommandMetrics metrics, HystrixCommandProperties.Setter properties) { + return new HystrixCircuitBreakerImpl(key, commandGroup, HystrixCommandPropertiesTest.asMock(properties), metrics); + } + + private static enum CommandOwnerForUnitTest implements HystrixCommandGroupKey { + OWNER_ONE, OWNER_TWO; + } + + private static enum ThreadPoolKeyForUnitTest implements HystrixThreadPoolKey { + THREAD_POOL_ONE, THREAD_POOL_TWO; + } + + private static enum CommandKeyForUnitTest implements HystrixCommandKey { + KEY_ONE, KEY_TWO; + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCollapserTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCollapserTest.java new file mode 100644 index 000000000..38fba326e --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCollapserTest.java @@ -0,0 +1,1287 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.netflix.hystrix.HystrixCollapser.CollapsedRequest; +import com.netflix.hystrix.HystrixCommandTest.TestHystrixCommand; +import com.netflix.hystrix.collapser.CollapserTimer; +import com.netflix.hystrix.collapser.RealCollapserTimer; +import com.netflix.hystrix.collapser.RequestCollapser; +import com.netflix.hystrix.collapser.RequestCollapserFactory; +import com.netflix.hystrix.strategy.HystrixPlugins; +import com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableHolder; +import com.netflix.hystrix.util.HystrixTimer.TimerListener; + +public class HystrixCollapserTest { + static AtomicInteger counter = new AtomicInteger(); + + @Before + public void init() { + counter.set(0); + // since we're going to modify properties of the same class between tests, wipe the cache each time + HystrixCollapser.reset(); + /* we must call this to simulate a new request lifecycle running and clearing caches */ + HystrixRequestContext.initializeContext(); + } + + @After + public void cleanup() { + // instead of storing the reference from initialize we'll just get the current state and shutdown + if (HystrixRequestContext.getContextForCurrentThread() != null) { + // it may be null if a test shuts the context down manually + HystrixRequestContext.getContextForCurrentThread().shutdown(); + } + } + + @Test + public void testTwoRequests() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); + Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); + timer.incrementTime(10); // let time pass that equals the default delay/period + + assertEquals("1", response1.get()); + assertEquals("2", response2.get()); + + assertEquals(1, counter.get()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testMultipleBatches() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); + Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); + timer.incrementTime(10); // let time pass that equals the default delay/period + + assertEquals("1", response1.get()); + assertEquals("2", response2.get()); + + assertEquals(1, counter.get()); + + // now request more + Future response3 = new TestRequestCollapser(timer, counter, 3).queue(); + timer.incrementTime(10); // let time pass that equals the default delay/period + + assertEquals("3", response3.get()); + + // we should have had it execute twice now + assertEquals(2, counter.get()); + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testMaxRequestsInBatch() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapser(timer, counter, 1, 2, 10).queue(); + Future response2 = new TestRequestCollapser(timer, counter, 2, 2, 10).queue(); + Future response3 = new TestRequestCollapser(timer, counter, 3, 2, 10).queue(); + timer.incrementTime(10); // let time pass that equals the default delay/period + + assertEquals("1", response1.get()); + assertEquals("2", response2.get()); + assertEquals("3", response3.get()); + + // we should have had it execute twice because the batch size was 2 + assertEquals(2, counter.get()); + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testRequestsOverTime() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); + timer.incrementTime(5); + Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); + timer.incrementTime(8); + // should execute here + Future response3 = new TestRequestCollapser(timer, counter, 3).queue(); + timer.incrementTime(6); + Future response4 = new TestRequestCollapser(timer, counter, 4).queue(); + timer.incrementTime(8); + // should execute here + Future response5 = new TestRequestCollapser(timer, counter, 5).queue(); + timer.incrementTime(10); + // should execute here + + // wait for all tasks to complete + assertEquals("1", response1.get()); + assertEquals("2", response2.get()); + assertEquals("3", response3.get()); + assertEquals("4", response4.get()); + assertEquals("5", response5.get()); + + System.out.println("number of executions: " + counter.get()); + assertEquals(3, counter.get()); + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testUnsubscribeOnOneDoesntKillBatch() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); + Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); + + // kill the first + response1.cancel(true); + + timer.incrementTime(10); // let time pass that equals the default delay/period + + // the first is cancelled so should return null + assertEquals(null, response1.get()); + // we should still get a response on the second + assertEquals("2", response2.get()); + + assertEquals(1, counter.get()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testShardedRequests() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestShardedRequestCollapser(timer, counter, "1a").queue(); + Future response2 = new TestShardedRequestCollapser(timer, counter, "2b").queue(); + Future response3 = new TestShardedRequestCollapser(timer, counter, "3b").queue(); + Future response4 = new TestShardedRequestCollapser(timer, counter, "4a").queue(); + timer.incrementTime(10); // let time pass that equals the default delay/period + + assertEquals("1a", response1.get()); + assertEquals("2b", response2.get()); + assertEquals("3b", response3.get()); + assertEquals("4a", response4.get()); + + /* we should get 2 batches since it gets sharded */ + assertEquals(2, counter.get()); + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testRequestScope() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapser(timer, counter, "1").queue(); + Future response2 = new TestRequestCollapser(timer, counter, "2").queue(); + + // simulate a new request + RequestCollapserFactory.resetRequest(); + + Future response3 = new TestRequestCollapser(timer, counter, "3").queue(); + Future response4 = new TestRequestCollapser(timer, counter, "4").queue(); + + timer.incrementTime(10); // let time pass that equals the default delay/period + + assertEquals("1", response1.get()); + assertEquals("2", response2.get()); + assertEquals("3", response3.get()); + assertEquals("4", response4.get()); + + // 2 different batches should execute, 1 per request + assertEquals(2, counter.get()); + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testGlobalScope() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestGloballyScopedRequestCollapser(timer, counter, "1").queue(); + Future response2 = new TestGloballyScopedRequestCollapser(timer, counter, "2").queue(); + + // simulate a new request + RequestCollapserFactory.resetRequest(); + + Future response3 = new TestGloballyScopedRequestCollapser(timer, counter, "3").queue(); + Future response4 = new TestGloballyScopedRequestCollapser(timer, counter, "4").queue(); + + timer.incrementTime(10); // let time pass that equals the default delay/period + + assertEquals("1", response1.get()); + assertEquals("2", response2.get()); + assertEquals("3", response3.get()); + assertEquals("4", response4.get()); + + // despite having cleared the cache in between we should have a single execution because this is on the global not request cache + assertEquals(1, counter.get()); + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testErrorHandlingViaFutureException() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapserWithFaultyCreateCommand(timer, counter, "1").queue(); + Future response2 = new TestRequestCollapserWithFaultyCreateCommand(timer, counter, "2").queue(); + timer.incrementTime(10); // let time pass that equals the default delay/period + + try { + response1.get(); + fail("we should have received an exception"); + } catch (ExecutionException e) { + // what we expect + } + try { + response2.get(); + fail("we should have received an exception"); + } catch (ExecutionException e) { + // what we expect + } + + assertEquals(0, counter.get()); + assertEquals(0, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testErrorHandlingWhenMapToResponseFails() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapserWithFaultyMapToResponse(timer, counter, "1").queue(); + Future response2 = new TestRequestCollapserWithFaultyMapToResponse(timer, counter, "2").queue(); + timer.incrementTime(10); // let time pass that equals the default delay/period + + try { + response1.get(); + fail("we should have received an exception"); + } catch (ExecutionException e) { + // what we expect + } + try { + response2.get(); + fail("we should have received an exception"); + } catch (ExecutionException e) { + // what we expect + } + + // the batch failed so no executions + assertEquals(0, counter.get()); + // but it still executed the command once + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testRequestVariableLifecycle1() throws Exception { + // simulate request lifecycle + HystrixRequestContext requestContext = HystrixRequestContext.initializeContext(); + + // do actual work + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapser(timer, counter, 1).queue(); + timer.incrementTime(5); + Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); + timer.incrementTime(8); + // should execute here + Future response3 = new TestRequestCollapser(timer, counter, 3).queue(); + timer.incrementTime(6); + Future response4 = new TestRequestCollapser(timer, counter, 4).queue(); + timer.incrementTime(8); + // should execute here + Future response5 = new TestRequestCollapser(timer, counter, 5).queue(); + timer.incrementTime(10); + // should execute here + + // wait for all tasks to complete + assertEquals("1", response1.get()); + assertEquals("2", response2.get()); + assertEquals("3", response3.get()); + assertEquals("4", response4.get()); + assertEquals("5", response5.get()); + + // each task should have been executed 3 times + for (TestCollapserTimer.ATask t : timer.tasks) { + assertEquals(3, t.task.count.get()); + } + + System.out.println("timer.tasks.size() A: " + timer.tasks.size()); + System.out.println("tasks in test: " + timer.tasks); + + // simulate request lifecycle + requestContext.shutdown(); + + System.out.println("timer.tasks.size() B: " + timer.tasks.size()); + + HystrixRequestVariableHolder> rv = RequestCollapserFactory.getRequestVariable(new TestRequestCollapser(timer, counter, 1).getCollapserKey().name()); + + assertNotNull(rv); + // they should have all been removed as part of ThreadContext.remove() + assertEquals(0, timer.tasks.size()); + } + + @Test + public void testRequestVariableLifecycle2() throws Exception { + // simulate request lifecycle + HystrixRequestContext requestContext = HystrixRequestContext.initializeContext(); + + final TestCollapserTimer timer = new TestCollapserTimer(); + final ConcurrentLinkedQueue> responses = new ConcurrentLinkedQueue>(); + ConcurrentLinkedQueue threads = new ConcurrentLinkedQueue(); + + // kick off work (simulating a single request with multiple threads) + for (int t = 0; t < 5; t++) { + Thread th = new Thread(new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + + @Override + public void run() { + for (int i = 0; i < 100; i++) { + responses.add(new TestRequestCollapser(timer, counter, 1).queue()); + } + } + })); + + threads.add(th); + th.start(); + } + + for (Thread th : threads) { + // wait for each thread to finish + th.join(); + } + + // we expect 5 threads * 100 responses each + assertEquals(500, responses.size()); + + for (Future f : responses) { + // they should not be done yet because the counter hasn't incremented + assertFalse(f.isDone()); + } + + timer.incrementTime(5); + Future response2 = new TestRequestCollapser(timer, counter, 2).queue(); + timer.incrementTime(8); + // should execute here + Future response3 = new TestRequestCollapser(timer, counter, 3).queue(); + timer.incrementTime(6); + Future response4 = new TestRequestCollapser(timer, counter, 4).queue(); + timer.incrementTime(8); + // should execute here + Future response5 = new TestRequestCollapser(timer, counter, 5).queue(); + timer.incrementTime(10); + // should execute here + + // wait for all tasks to complete + for (Future f : responses) { + assertEquals("1", f.get()); + } + assertEquals("2", response2.get()); + assertEquals("3", response3.get()); + assertEquals("4", response4.get()); + assertEquals("5", response5.get()); + + // each task should have been executed 3 times + for (TestCollapserTimer.ATask t : timer.tasks) { + assertEquals(3, t.task.count.get()); + } + + // simulate request lifecycle + requestContext.shutdown(); + + HystrixRequestVariableHolder> rv = RequestCollapserFactory.getRequestVariable(new TestRequestCollapser(timer, counter, 1).getCollapserKey().name()); + + assertNotNull(rv); + // they should have all been removed as part of ThreadContext.remove() + assertEquals(0, timer.tasks.size()); + } + + /** + * Test Request scoped caching of commands so that a 2nd duplicate call doesn't execute but returns the previous Future + */ + @Test + public void testRequestCache1() { + // simulate request lifecycle + HystrixRequestContext.initializeContext(); + + final TestCollapserTimer timer = new TestCollapserTimer(); + SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", true); + SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", true); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f1.get()); + assertEquals("A", f2.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // we should have executed a command once + assertEquals(1, counter.get()); + + Future f3 = command1.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // we should still have executed only one command + assertEquals(1, counter.get()); + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + HystrixCommand command = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[1])[0]; + System.out.println("command.getExecutionEvents(): " + command.getExecutionEvents()); + assertEquals(2, command.getExecutionEvents().size()); + assertTrue(command.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); + } + + /** + * Test Request scoped caching doesn't prevent different ones from executing + */ + @Test + public void testRequestCache2() { + // simulate request lifecycle + HystrixRequestContext.initializeContext(); + + final TestCollapserTimer timer = new TestCollapserTimer(); + SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", true); + SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", true); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // we should have executed a command once + assertEquals(1, counter.get()); + + Future f3 = command1.queue(); + Future f4 = command2.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f3.get()); + assertEquals("B", f4.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // we should still have executed only one command + assertEquals(1, counter.get()); + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + HystrixCommand command = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[1])[0]; + assertEquals(2, command.getExecutionEvents().size()); + assertTrue(command.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testRequestCache3() { + // simulate request lifecycle + HystrixRequestContext.initializeContext(); + + final TestCollapserTimer timer = new TestCollapserTimer(); + SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", true); + SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", true); + SuccessfulCacheableCollapsedCommand command3 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", true); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("B", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // we should have executed a command once + assertEquals(1, counter.get()); + + Future f4 = command1.queue(); + Future f5 = command2.queue(); + Future f6 = command3.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f4.get()); + assertEquals("B", f5.get()); + assertEquals("B", f6.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // we should still have executed only one command + assertEquals(1, counter.get()); + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + HystrixCommand command = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[1])[0]; + assertEquals(2, command.getExecutionEvents().size()); + assertTrue(command.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testNoRequestCache3() { + // simulate request lifecycle + HystrixRequestContext.initializeContext(); + + final TestCollapserTimer timer = new TestCollapserTimer(); + SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "A", false); + SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", false); + SuccessfulCacheableCollapsedCommand command3 = new SuccessfulCacheableCollapsedCommand(timer, counter, "B", false); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("B", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // we should have executed a command once + assertEquals(1, counter.get()); + + Future f4 = command1.queue(); + Future f5 = command2.queue(); + Future f6 = command3.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f4.get()); + assertEquals("B", f5.get()); + assertEquals("B", f6.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // request caching is turned off on this so we expect 2 command executions + assertEquals(2, counter.get()); + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + // we expect to see it with SUCCESS and COLLAPSED and both + HystrixCommand commandA = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[2])[0]; + assertEquals(2, commandA.getExecutionEvents().size()); + assertTrue(commandA.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(commandA.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); + + // we expect to see it with SUCCESS and COLLAPSED and both + HystrixCommand commandB = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[2])[1]; + assertEquals(2, commandB.getExecutionEvents().size()); + assertTrue(commandB.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(commandB.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); + } + + /** + * Test that a command that throws an Exception when cached will re-throw the exception. + */ + @Test + public void testRequestCacheWithException() { + // simulate request lifecycle + HystrixRequestContext.initializeContext(); + + ConcurrentLinkedQueue>> commands = new ConcurrentLinkedQueue>>(); + + final TestCollapserTimer timer = new TestCollapserTimer(); + // pass in 'null' which will cause an NPE to be thrown + SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, null, true, commands); + SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, null, true, commands); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f1.get()); + assertEquals("A", f2.get()); + fail("exception should have been thrown"); + } catch (Exception e) { + // expected + } + + // this should be 0 because we never complete execution + assertEquals(0, counter.get()); + + // it should have executed 1 command + assertEquals(1, commands.size()); + assertTrue(commands.peek().getExecutionEvents().contains(HystrixEventType.FAILURE)); + assertTrue(commands.peek().getExecutionEvents().contains(HystrixEventType.COLLAPSED)); + + SuccessfulCacheableCollapsedCommand command3 = new SuccessfulCacheableCollapsedCommand(timer, counter, null, true, commands); + Future f3 = command3.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f3.get()); + fail("exception should have been thrown"); + } catch (Exception e) { + // expected + } + + // this should be 0 because we never complete execution + assertEquals(0, counter.get()); + + // it should still be 1 ... no new executions + assertEquals(1, commands.size()); + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + HystrixCommand command = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[1])[0]; + assertEquals(2, command.getExecutionEvents().size()); + assertTrue(command.getExecutionEvents().contains(HystrixEventType.FAILURE)); + assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); + } + + /** + * Test that a command that times out will still be cached and when retrieved will re-throw the exception. + */ + @Test + public void testRequestCacheWithTimeout() { + // simulate request lifecycle + HystrixRequestContext.initializeContext(); + + ConcurrentLinkedQueue>> commands = new ConcurrentLinkedQueue>>(); + + final TestCollapserTimer timer = new TestCollapserTimer(); + // pass in 'null' which will cause an NPE to be thrown + SuccessfulCacheableCollapsedCommand command1 = new SuccessfulCacheableCollapsedCommand(timer, counter, "TIMEOUT", true, commands); + SuccessfulCacheableCollapsedCommand command2 = new SuccessfulCacheableCollapsedCommand(timer, counter, "TIMEOUT", true, commands); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f1.get()); + assertEquals("A", f2.get()); + fail("exception should have been thrown"); + } catch (Exception e) { + // expected + } + + // this should be 0 because we never complete execution + assertEquals(0, counter.get()); + + // it should have executed 1 command + assertEquals(1, commands.size()); + assertTrue(commands.peek().getExecutionEvents().contains(HystrixEventType.TIMEOUT)); + assertTrue(commands.peek().getExecutionEvents().contains(HystrixEventType.COLLAPSED)); + + Future f3 = command1.queue(); + + // increment past batch time so it executes + timer.incrementTime(15); + + try { + assertEquals("A", f3.get()); + fail("exception should have been thrown"); + } catch (Exception e) { + // expected + } + + // this should be 0 because we never complete execution + assertEquals(0, counter.get()); + + // it should still be 1 ... no new executions + assertEquals(1, commands.size()); + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test how the collapser behaves when the circuit is short-circuited + */ + @Test + public void testRequestWithCommandShortCircuited() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestRequestCollapserWithShortCircuitedCommand(timer, counter, "1").queue(); + Future response2 = new TestRequestCollapserWithShortCircuitedCommand(timer, counter, "2").queue(); + timer.incrementTime(10); // let time pass that equals the default delay/period + + try { + response1.get(); + fail("we should have received an exception"); + } catch (ExecutionException e) { + // e.printStackTrace(); + // what we expect + } + try { + response2.get(); + fail("we should have received an exception"); + } catch (ExecutionException e) { + // e.printStackTrace(); + // what we expect + } + + assertEquals(0, counter.get()); + // it will execute once (short-circuited) + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a Void response type - null being set as response. + * + * @throws Exception + */ + @Test + public void testVoidResponseTypeFireAndForgetCollapsing1() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestCollapserWithVoidResponseType(timer, counter, 1).queue(); + Future response2 = new TestCollapserWithVoidResponseType(timer, counter, 2).queue(); + timer.incrementTime(100); // let time pass that equals the default delay/period + + // normally someone wouldn't wait on these, but we need to make sure they do in fact return + // and not block indefinitely in case someone does call get() + assertEquals(null, response1.get()); + assertEquals(null, response2.get()); + + assertEquals(1, counter.get()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a Void response type - response never being set in mapResponseToRequest + * + * @throws Exception + */ + @Test + public void testVoidResponseTypeFireAndForgetCollapsing2() throws Exception { + TestCollapserTimer timer = new TestCollapserTimer(); + Future response1 = new TestCollapserWithVoidResponseTypeAndMissingMapResponseToRequests(timer, counter, 1).queue(); + Future response2 = new TestCollapserWithVoidResponseTypeAndMissingMapResponseToRequests(timer, counter, 2).queue(); + timer.incrementTime(100); // let time pass that equals the default delay/period + + // we will fetch one of these just so we wait for completion ... but expect an error + try { + assertEquals(null, response1.get()); + fail("expected an error as mapResponseToRequests did not set responses"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof IllegalStateException); + assertTrue(e.getCause().getMessage().startsWith("No response set by")); + } + + assertEquals(1, counter.get()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a Void response type with execute - response being set in mapResponseToRequest to null + * + * @throws Exception + */ + @Test + public void testVoidResponseTypeFireAndForgetCollapsing3() throws Exception { + CollapserTimer timer = new RealCollapserTimer(); + assertNull(new TestCollapserWithVoidResponseType(timer, counter, 1).execute()); + + assertEquals(1, counter.get()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + private static class TestRequestCollapser extends HystrixCollapser, String, String> { + + private final AtomicInteger count; + private final String value; + private ConcurrentLinkedQueue>> commandsExecuted; + + public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, int value) { + this(timer, counter, String.valueOf(value)); + } + + public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value) { + this(timer, counter, value, 10000, 10); + } + + public TestRequestCollapser(Scope scope, TestCollapserTimer timer, AtomicInteger counter, String value) { + this(scope, timer, counter, value, 10000, 10); + } + + public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value, ConcurrentLinkedQueue>> executionLog) { + this(timer, counter, value, 10000, 10, executionLog); + } + + public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, int value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds) { + this(timer, counter, String.valueOf(value), defaultMaxRequestsInBatch, defaultTimerDelayInMilliseconds); + } + + public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds) { + this(timer, counter, value, defaultMaxRequestsInBatch, defaultTimerDelayInMilliseconds, null); + } + + public TestRequestCollapser(Scope scope, TestCollapserTimer timer, AtomicInteger counter, String value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds) { + this(scope, timer, counter, value, defaultMaxRequestsInBatch, defaultTimerDelayInMilliseconds, null); + } + + public TestRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds, ConcurrentLinkedQueue>> executionLog) { + this(Scope.REQUEST, timer, counter, value, defaultMaxRequestsInBatch, defaultTimerDelayInMilliseconds, executionLog); + } + + public TestRequestCollapser(Scope scope, TestCollapserTimer timer, AtomicInteger counter, String value, int defaultMaxRequestsInBatch, int defaultTimerDelayInMilliseconds, ConcurrentLinkedQueue>> executionLog) { + // use a CollapserKey based on the CollapserTimer object reference so it's unique for each timer as we don't want caching + // of properties to occur and we're using the default HystrixProperty which typically does caching + super(collapserKeyFromString(timer), scope, timer, HystrixCollapserProperties.Setter().withMaxRequestsInBatch(defaultMaxRequestsInBatch).withTimerDelayInMilliseconds(defaultTimerDelayInMilliseconds)); + this.count = counter; + this.value = value; + this.commandsExecuted = executionLog; + } + + @Override + public String getRequestArgument() { + return value; + } + + @Override + public HystrixCommand> createCommand(final Collection> requests) { + /* return a mocked command */ + HystrixCommand> command = new TestCollapserCommand(requests); + if (commandsExecuted != null) { + commandsExecuted.add(command); + } + return command; + } + + @Override + public void mapResponseToRequests(List batchResponse, Collection> requests) { + // count how many times a batch is executed (this method is executed once per batch) + System.out.println("increment count: " + count.incrementAndGet()); + + // for simplicity I'll assume it's a 1:1 mapping between lists ... in real implementations they often need to index to maps + // to allow random access as the response size does not match the request size + if (batchResponse.size() != requests.size()) { + throw new RuntimeException("lists don't match in size => " + batchResponse.size() + " : " + requests.size()); + } + int i = 0; + for (CollapsedRequest request : requests) { + request.setResponse(batchResponse.get(i++)); + } + + } + + } + + /** + * Shard on the artificially provided 'type' variable. + */ + private static class TestShardedRequestCollapser extends TestRequestCollapser { + + public TestShardedRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value) { + super(timer, counter, value); + } + + @Override + protected Collection>> shardRequests(Collection> requests) { + Collection> typeA = new ArrayList>(); + Collection> typeB = new ArrayList>(); + + for (CollapsedRequest request : requests) { + if (request.getArgument().endsWith("a")) { + typeA.add(request); + } else if (request.getArgument().endsWith("b")) { + typeB.add(request); + } + } + + ArrayList>> shards = new ArrayList>>(); + shards.add(typeA); + shards.add(typeB); + return shards; + } + + } + + /** + * Test the global scope + */ + private static class TestGloballyScopedRequestCollapser extends TestRequestCollapser { + + public TestGloballyScopedRequestCollapser(TestCollapserTimer timer, AtomicInteger counter, String value) { + super(Scope.GLOBAL, timer, counter, value); + } + + } + + /** + * Throw an exception when creating a command. + */ + private static class TestRequestCollapserWithFaultyCreateCommand extends TestRequestCollapser { + + public TestRequestCollapserWithFaultyCreateCommand(TestCollapserTimer timer, AtomicInteger counter, String value) { + super(timer, counter, value); + } + + @Override + public HystrixCommand> createCommand(Collection> requests) { + throw new RuntimeException("some failure"); + } + + } + + /** + * Throw an exception when creating a command. + */ + private static class TestRequestCollapserWithShortCircuitedCommand extends TestRequestCollapser { + + public TestRequestCollapserWithShortCircuitedCommand(TestCollapserTimer timer, AtomicInteger counter, String value) { + super(timer, counter, value); + } + + @Override + public HystrixCommand> createCommand(Collection> requests) { + // args don't matter as it's short-circuited + return new ShortCircuitedCommand(); + } + + } + + /** + * Throw an exception when mapToResponse is invoked + */ + private static class TestRequestCollapserWithFaultyMapToResponse extends TestRequestCollapser { + + public TestRequestCollapserWithFaultyMapToResponse(TestCollapserTimer timer, AtomicInteger counter, String value) { + super(timer, counter, value); + } + + @Override + public void mapResponseToRequests(List batchResponse, Collection> requests) { + // pretend we blow up with an NPE + throw new NullPointerException("batchResponse was null and we blew up"); + } + + } + + private static class TestCollapserCommand extends TestHystrixCommand> { + + private final Collection> requests; + + TestCollapserCommand(Collection> requests) { + super(testPropsBuilder().setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(50))); + this.requests = requests; + } + + @Override + protected List run() { + System.out.println(">>> TestCollapserCommand run() ... batch size: " + requests.size()); + // simulate a batch request + ArrayList response = new ArrayList(); + for (CollapsedRequest request : requests) { + if (request.getArgument() == null) { + throw new NullPointerException("Simulated Error"); + } + if (request.getArgument() == "TIMEOUT") { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + response.add(request.getArgument()); + } + return response; + } + + } + + /** + * A Command implementation that supports caching. + */ + private static class SuccessfulCacheableCollapsedCommand extends TestRequestCollapser { + + private final boolean cacheEnabled; + + public SuccessfulCacheableCollapsedCommand(TestCollapserTimer timer, AtomicInteger counter, String value, boolean cacheEnabled) { + super(timer, counter, value); + this.cacheEnabled = cacheEnabled; + } + + public SuccessfulCacheableCollapsedCommand(TestCollapserTimer timer, AtomicInteger counter, String value, boolean cacheEnabled, ConcurrentLinkedQueue>> executionLog) { + super(timer, counter, value, executionLog); + this.cacheEnabled = cacheEnabled; + } + + @Override + public String getCacheKey() { + if (cacheEnabled) + return "aCacheKey_" + super.value; + else + return null; + } + } + + private static class ShortCircuitedCommand extends HystrixCommand> { + + protected ShortCircuitedCommand() { + super(HystrixCommand.Setter.withGroupKey( + HystrixCommandGroupKey.Factory.asKey("shortCircuitedCommand")) + .andCommandPropertiesDefaults(HystrixCommandPropertiesTest + .getUnitTestPropertiesSetter() + .withCircuitBreakerForceOpen(true))); + } + + @Override + protected List run() throws Exception { + System.out.println("*** execution (this shouldn't happen)"); + // this won't ever get called as we're forcing short-circuiting + ArrayList values = new ArrayList(); + values.add("hello"); + return values; + } + + } + + private static class FireAndForgetCommand extends HystrixCommand { + + protected FireAndForgetCommand(List values) { + super(HystrixCommand.Setter.withGroupKey( + HystrixCommandGroupKey.Factory.asKey("fireAndForgetCommand")) + .andCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter())); + } + + @Override + protected Void run() throws Exception { + System.out.println("*** FireAndForgetCommand execution: " + Thread.currentThread()); + return null; + } + + } + + private static class TestCollapserTimer implements CollapserTimer { + + private final ConcurrentLinkedQueue tasks = new ConcurrentLinkedQueue(); + + @Override + public Reference addListener(final TimerListener collapseTask) { + System.out.println("add listener: " + collapseTask); + tasks.add(new ATask(new TestTimerListener(collapseTask))); + + /** + * This is a hack that overrides 'clear' of a WeakReference to match the required API + * but then removes the strong-reference we have inside 'tasks'. + *

+ * We do this so our unit tests know if the WeakReference is cleared correctly, and if so then the ATack is removed from 'tasks' + */ + return new SoftReference(collapseTask) { + @Override + public void clear() { + System.out.println("tasks: " + tasks); + System.out.println("**** clear TimerListener: tasks.size => " + tasks.size()); + // super.clear(); + for (ATask t : tasks) { + if (t.task.actualListener.equals(collapseTask)) { + tasks.remove(t); + } + } + } + + }; + } + + /** + * Increment time by X. Note that incrementing by multiples of delay or period time will NOT execute multiple times. + *

+ * You must call incrementTime multiple times each increment being larger than 'period' on subsequent calls to cause multiple executions. + *

+ * This is because executing multiple times in a tight-loop would not achieve the correct behavior, such as batching, since it will all execute "now" not after intervals of time. + * + * @param timeInMilliseconds + */ + public synchronized void incrementTime(int timeInMilliseconds) { + for (ATask t : tasks) { + t.incrementTime(timeInMilliseconds); + } + } + + private static class ATask { + final TestTimerListener task; + final int delay = 10; + + // our relative time that we'll use + volatile int time = 0; + volatile int executionCount = 0; + + private ATask(TestTimerListener task) { + this.task = task; + } + + public synchronized void incrementTime(int timeInMilliseconds) { + time += timeInMilliseconds; + if (task != null) { + if (executionCount == 0) { + System.out.println("ExecutionCount 0 => Time: " + time + " Delay: " + delay); + if (time >= delay) { + // first execution, we're past the delay time + executeTask(); + } + } else { + System.out.println("ExecutionCount 1+ => Time: " + time + " Delay: " + delay); + if (time >= delay) { + // subsequent executions, we're past the interval time + executeTask(); + } + } + } + } + + private synchronized void executeTask() { + System.out.println("Executing task ..."); + task.tick(); + this.time = 0; // we reset time after each execution + this.executionCount++; + System.out.println("executionCount: " + executionCount); + } + } + + } + + private static class TestTimerListener implements TimerListener { + + private final TimerListener actualListener; + private final AtomicInteger count = new AtomicInteger(); + + public TestTimerListener(TimerListener actual) { + this.actualListener = actual; + } + + @Override + public void tick() { + count.incrementAndGet(); + actualListener.tick(); + } + + @Override + public int getIntervalTimeInMilliseconds() { + return 10; + } + + } + + private static HystrixCollapserKey collapserKeyFromString(final Object o) { + return new HystrixCollapserKey() { + + @Override + public String name() { + return String.valueOf(o); + } + + }; + } + + private static class TestCollapserWithVoidResponseType extends HystrixCollapser { + + private final AtomicInteger count; + private final Integer value; + + public TestCollapserWithVoidResponseType(CollapserTimer timer, AtomicInteger counter, int value) { + super(collapserKeyFromString(timer), Scope.REQUEST, timer, HystrixCollapserProperties.Setter().withMaxRequestsInBatch(1000).withTimerDelayInMilliseconds(50)); + this.count = counter; + this.value = value; + } + + @Override + public Integer getRequestArgument() { + return value; + } + + @Override + protected HystrixCommand createCommand(Collection> requests) { + + ArrayList args = new ArrayList(); + for (CollapsedRequest request : requests) { + args.add(request.getArgument()); + } + return new FireAndForgetCommand(args); + } + + @Override + protected void mapResponseToRequests(Void batchResponse, Collection> requests) { + count.incrementAndGet(); + for (CollapsedRequest r : requests) { + r.setResponse(null); + } + } + + } + + private static class TestCollapserWithVoidResponseTypeAndMissingMapResponseToRequests extends HystrixCollapser { + + private final AtomicInteger count; + private final Integer value; + + public TestCollapserWithVoidResponseTypeAndMissingMapResponseToRequests(CollapserTimer timer, AtomicInteger counter, int value) { + super(collapserKeyFromString(timer), Scope.REQUEST, timer, HystrixCollapserProperties.Setter().withMaxRequestsInBatch(1000).withTimerDelayInMilliseconds(50)); + this.count = counter; + this.value = value; + } + + @Override + public Integer getRequestArgument() { + return value; + } + + @Override + protected HystrixCommand createCommand(Collection> requests) { + + ArrayList args = new ArrayList(); + for (CollapsedRequest request : requests) { + args.add(request.getArgument()); + } + return new FireAndForgetCommand(args); + } + + @Override + protected void mapResponseToRequests(Void batchResponse, Collection> requests) { + count.incrementAndGet(); + } + + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandMetricsTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandMetricsTest.java new file mode 100644 index 000000000..5e9cd4ed0 --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandMetricsTest.java @@ -0,0 +1,64 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.netflix.hystrix.HystrixCommandTest.CommandGroupForUnitTest; +import com.netflix.hystrix.HystrixCommandTest.CommandKeyForUnitTest; +import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifierDefault; + + +public class HystrixCommandMetricsTest { + + /** + * Testing the ErrorPercentage because this method could be easy to miss when making changes elsewhere. + */ + @Test + public void testGetErrorPercentage() { + + try { + HystrixCommandProperties.Setter properties = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter(); + HystrixCommandMetrics metrics = getMetrics(properties); + + metrics.markSuccess(100); + assertEquals(0, metrics.getHealthCounts().getErrorPercentage()); + + metrics.markFailure(1000); + assertEquals(50, metrics.getHealthCounts().getErrorPercentage()); + + metrics.markSuccess(100); + metrics.markSuccess(100); + assertEquals(25, metrics.getHealthCounts().getErrorPercentage()); + + metrics.markTimeout(5000); + metrics.markTimeout(5000); + assertEquals(50, metrics.getHealthCounts().getErrorPercentage()); + + metrics.markSuccess(100); + metrics.markSuccess(100); + metrics.markSuccess(100); + + // latent + metrics.markSuccess(5000); + + // 6 success + 1 latent success + 1 failure + 2 timeout = 10 total + // latent success not considered error + // error percentage = 1 failure + 2 timeout / 10 + assertEquals(30, metrics.getHealthCounts().getErrorPercentage()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Error occurred: " + e.getMessage()); + } + + } + + /** + * Utility method for creating {@link HystrixCommandMetrics} for unit tests. + */ + private static HystrixCommandMetrics getMetrics(HystrixCommandProperties.Setter properties) { + return new HystrixCommandMetrics(CommandKeyForUnitTest.KEY_ONE, CommandGroupForUnitTest.OWNER_ONE, HystrixCommandPropertiesTest.asMock(properties), HystrixEventNotifierDefault.getInstance()); + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandPropertiesTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandPropertiesTest.java new file mode 100644 index 000000000..47d6b8ccd --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandPropertiesTest.java @@ -0,0 +1,332 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import org.junit.After; +import org.junit.Test; + +import com.netflix.config.ConfigurationManager; +import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; +import com.netflix.hystrix.HystrixCommandProperties.Setter; +import com.netflix.hystrix.strategy.properties.HystrixProperty; + +public class HystrixCommandPropertiesTest { + + /** + * Utility method for creating baseline properties for unit tests. + */ + /* package */static HystrixCommandProperties.Setter getUnitTestPropertiesSetter() { + return new HystrixCommandProperties.Setter() + .withExecutionIsolationThreadTimeoutInMilliseconds(1000)// when an execution will be timed out + .withExecutionIsolationStrategy(ExecutionIsolationStrategy.THREAD) // we want thread execution by default in tests + .withExecutionIsolationThreadInterruptOnTimeout(true) + .withCircuitBreakerForceOpen(false) // we don't want short-circuiting by default + .withCircuitBreakerErrorThresholdPercentage(40) // % of 'marks' that must be failed to trip the circuit + .withMetricsRollingStatisticalWindowInMilliseconds(5000)// milliseconds back that will be tracked + .withMetricsRollingStatisticalWindowBuckets(5) // buckets + .withCircuitBreakerRequestVolumeThreshold(0) // in testing we will not have a threshold unless we're specifically testing that feature + .withCircuitBreakerSleepWindowInMilliseconds(5000000) // milliseconds after tripping circuit before allowing retry (by default set VERY long as we want it to effectively never allow a singleTest for most unit tests) + .withCircuitBreakerEnabled(true) + .withRequestLogEnabled(true) + .withExecutionIsolationSemaphoreMaxConcurrentRequests(20) + .withFallbackIsolationSemaphoreMaxConcurrentRequests(10) + .withFallbackEnabled(true) + .withCircuitBreakerForceClosed(false) + .withMetricsRollingPercentileEnabled(true) + .withRequestCacheEnabled(true) + .withMetricsRollingPercentileWindowInMilliseconds(60000) + .withMetricsRollingPercentileWindowBuckets(12) + .withMetricsRollingPercentileBucketSize(1000) + .withMetricsHealthSnapshotIntervalInMilliseconds(0); + } + + /** + * Return a static representation of the properties with values from the Builder so that UnitTests can create properties that are not affected by the actual implementations which pick up their + * values dynamically. + * + * @param builder + * @return HystrixCommandProperties + */ + /* package */static HystrixCommandProperties asMock(final Setter builder) { + return new HystrixCommandProperties(TestKey.TEST) { + + @Override + public HystrixProperty circuitBreakerEnabled() { + return HystrixProperty.Factory.asProperty(builder.getCircuitBreakerEnabled()); + } + + @Override + public HystrixProperty circuitBreakerErrorThresholdPercentage() { + return HystrixProperty.Factory.asProperty(builder.getCircuitBreakerErrorThresholdPercentage()); + } + + @Override + public HystrixProperty circuitBreakerForceClosed() { + return HystrixProperty.Factory.asProperty(builder.getCircuitBreakerForceClosed()); + } + + @Override + public HystrixProperty circuitBreakerForceOpen() { + return HystrixProperty.Factory.asProperty(builder.getCircuitBreakerForceOpen()); + } + + @Override + public HystrixProperty circuitBreakerRequestVolumeThreshold() { + return HystrixProperty.Factory.asProperty(builder.getCircuitBreakerRequestVolumeThreshold()); + } + + @Override + public HystrixProperty circuitBreakerSleepWindowInMilliseconds() { + return HystrixProperty.Factory.asProperty(builder.getCircuitBreakerSleepWindowInMilliseconds()); + } + + @Override + public HystrixProperty executionIsolationSemaphoreMaxConcurrentRequests() { + return HystrixProperty.Factory.asProperty(builder.getExecutionIsolationSemaphoreMaxConcurrentRequests()); + } + + @Override + public HystrixProperty executionIsolationStrategy() { + return HystrixProperty.Factory.asProperty(builder.getExecutionIsolationStrategy()); + } + + @Override + public HystrixProperty executionIsolationThreadInterruptOnTimeout() { + return HystrixProperty.Factory.asProperty(builder.getExecutionIsolationThreadInterruptOnTimeout()); + } + + @Override + public HystrixProperty executionIsolationThreadPoolKeyOverride() { + return HystrixProperty.Factory.nullProperty(); + } + + @Override + public HystrixProperty executionIsolationThreadTimeoutInMilliseconds() { + return HystrixProperty.Factory.asProperty(builder.getExecutionIsolationThreadTimeoutInMilliseconds()); + } + + @Override + public HystrixProperty fallbackIsolationSemaphoreMaxConcurrentRequests() { + return HystrixProperty.Factory.asProperty(builder.getFallbackIsolationSemaphoreMaxConcurrentRequests()); + } + + @Override + public HystrixProperty fallbackEnabled() { + return HystrixProperty.Factory.asProperty(builder.getFallbackEnabled()); + } + + @Override + public HystrixProperty metricsHealthSnapshotIntervalInMilliseconds() { + return HystrixProperty.Factory.asProperty(builder.getMetricsHealthSnapshotIntervalInMilliseconds()); + } + + @Override + public HystrixProperty metricsRollingPercentileBucketSize() { + return HystrixProperty.Factory.asProperty(builder.getMetricsRollingPercentileBucketSize()); + } + + @Override + public HystrixProperty metricsRollingPercentileEnabled() { + return HystrixProperty.Factory.asProperty(builder.getMetricsRollingPercentileEnabled()); + } + + @Override + public HystrixProperty metricsRollingPercentileWindow() { + return HystrixProperty.Factory.asProperty(builder.getMetricsRollingPercentileWindowInMilliseconds()); + } + + @Override + public HystrixProperty metricsRollingPercentileWindowBuckets() { + return HystrixProperty.Factory.asProperty(builder.getMetricsRollingPercentileWindowBuckets()); + } + + @Override + public HystrixProperty metricsRollingStatisticalWindowInMilliseconds() { + return HystrixProperty.Factory.asProperty(builder.getMetricsRollingStatisticalWindowInMilliseconds()); + } + + @Override + public HystrixProperty metricsRollingStatisticalWindowBuckets() { + return HystrixProperty.Factory.asProperty(builder.getMetricsRollingStatisticalWindowBuckets()); + } + + @Override + public HystrixProperty requestCacheEnabled() { + return HystrixProperty.Factory.asProperty(builder.getRequestCacheEnabled()); + } + + @Override + public HystrixProperty requestLogEnabled() { + return HystrixProperty.Factory.asProperty(builder.getRequestLogEnabled()); + } + + }; + } + + // NOTE: We use "unitTestPrefix" as a prefix so we can't end up pulling in external properties that change unit test behavior + + public enum TestKey implements HystrixCommandKey { + TEST; + } + + private static class TestPropertiesCommand extends HystrixCommandProperties { + + protected TestPropertiesCommand(HystrixCommandKey key, Setter builder, String propertyPrefix) { + super(key, builder, propertyPrefix); + } + + } + + @After + public void cleanup() { + ConfigurationManager.getConfigInstance().clear(); + } + + @Test + public void testBooleanBuilderOverride1() { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, + new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(true), "unitTestPrefix"); + + // the builder override should take precedence over the default + assertEquals(true, properties.circuitBreakerForceClosed().get()); + } + + @Test + public void testBooleanBuilderOverride2() { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, + new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(false), "unitTestPrefix"); + + // the builder override should take precedence over the default + assertEquals(false, properties.circuitBreakerForceClosed().get()); + } + + @Test + public void testBooleanCodeDefault() { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); + assertEquals(HystrixCommandProperties.default_circuitBreakerForceClosed, properties.circuitBreakerForceClosed().get()); + } + + @Test + public void testBooleanGlobalDynamicOverrideOfCodeDefault() throws Exception { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed", true); + + // the global dynamic property should take precedence over the default + assertEquals(true, properties.circuitBreakerForceClosed().get()); + + // cleanup + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed"); + } + + @Test + public void testBooleanInstanceBuilderOverrideOfGlobalDynamicOverride1() throws Exception { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, + new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(true), "unitTestPrefix"); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed", false); + + // the builder injected should take precedence over the global dynamic property + assertEquals(true, properties.circuitBreakerForceClosed().get()); + + // cleanup + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed"); + } + + @Test + public void testBooleanInstanceBuilderOverrideOfGlobalDynamicOverride2() throws Exception { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, + new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(false), "unitTestPrefix"); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed", true); + + // the builder injected should take precedence over the global dynamic property + assertEquals(false, properties.circuitBreakerForceClosed().get()); + + // cleanup + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed"); + } + + @Test + public void testBooleanInstanceDynamicOverrideOfEverything() throws Exception { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, + new HystrixCommandProperties.Setter().withCircuitBreakerForceClosed(false), "unitTestPrefix"); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed", false); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.TEST.circuitBreaker.forceClosed", true); + + // the instance specific dynamic property should take precedence over everything + assertEquals(true, properties.circuitBreakerForceClosed().get()); + + // cleanup + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.circuitBreaker.forceClosed"); + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.TEST.circuitBreaker.forceClosed"); + } + + @Test + public void testIntegerBuilderOverride() { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, + new HystrixCommandProperties.Setter().withMetricsRollingStatisticalWindowInMilliseconds(5000), "unitTestPrefix"); + + // the builder override should take precedence over the default + assertEquals(5000, properties.metricsRollingStatisticalWindowInMilliseconds().get().intValue()); + } + + @Test + public void testIntegerCodeDefault() { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); + assertEquals(HystrixCommandProperties.default_metricsRollingStatisticalWindow, properties.metricsRollingStatisticalWindowInMilliseconds().get()); + } + + @Test + public void testIntegerGlobalDynamicOverrideOfCodeDefault() throws Exception { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.metrics.rollingStats.timeInMilliseconds", 1234); + + // the global dynamic property should take precedence over the default + assertEquals(1234, properties.metricsRollingStatisticalWindowInMilliseconds().get().intValue()); + + // cleanup + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.metrics.rollingStats.timeInMilliseconds"); + } + + @Test + public void testIntegerInstanceBuilderOverrideOfGlobalDynamicOverride() throws Exception { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, + new HystrixCommandProperties.Setter().withMetricsRollingStatisticalWindowInMilliseconds(5000), "unitTestPrefix"); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.rollingStats.timeInMilliseconds", 3456); + + // the builder injected should take precedence over the global dynamic property + assertEquals(5000, properties.metricsRollingStatisticalWindowInMilliseconds().get().intValue()); + + // cleanup + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.rollingStats.timeInMilliseconds"); + } + + @Test + public void testIntegerInstanceDynamicOverrideOfEverything() throws Exception { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, + new HystrixCommandProperties.Setter().withMetricsRollingStatisticalWindowInMilliseconds(5000), "unitTestPrefix"); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.metrics.rollingStats.timeInMilliseconds", 1234); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.TEST.metrics.rollingStats.timeInMilliseconds", 3456); + + // the instance specific dynamic property should take precedence over everything + assertEquals(3456, properties.metricsRollingStatisticalWindowInMilliseconds().get().intValue()); + + // cleanup + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.metrics.rollingStats.timeInMilliseconds"); + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.TEST.metrics.rollingStats.timeInMilliseconds"); + } + + @Test + public void testThreadPoolOnlyHasInstanceOverride() throws Exception { + HystrixCommandProperties properties = new TestPropertiesCommand(TestKey.TEST, new HystrixCommandProperties.Setter(), "unitTestPrefix"); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.default.threadPoolKeyOverride", 1234); + // it should be null + assertEquals(null, properties.executionIsolationThreadPoolKeyOverride().get()); + ConfigurationManager.getConfigInstance().setProperty("unitTestPrefix.command.TEST.threadPoolKeyOverride", "testPool"); + // now it should have a value + assertEquals("testPool", properties.executionIsolationThreadPoolKeyOverride().get()); + + // cleanup + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.default.threadPoolKeyOverride"); + ConfigurationManager.getConfigInstance().clearProperty("unitTestPrefix.command.TEST.threadPoolKeyOverride"); + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java new file mode 100644 index 000000000..0f4d63bfc --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java @@ -0,0 +1,5285 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import rx.Observer; +import rx.Scheduler; +import rx.functions.Action1; +import rx.observers.TestSubscriber; +import rx.schedulers.Schedulers; + +import com.netflix.config.ConfigurationManager; +import com.netflix.hystrix.HystrixCircuitBreakerTest.TestCircuitBreaker; +import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; +import com.netflix.hystrix.HystrixExecutableBase.TryableSemaphore; +import com.netflix.hystrix.exception.HystrixBadRequestException; +import com.netflix.hystrix.exception.HystrixRuntimeException; +import com.netflix.hystrix.exception.HystrixRuntimeException.FailureType; +import com.netflix.hystrix.strategy.HystrixPlugins; +import com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable; +import com.netflix.hystrix.strategy.concurrency.HystrixContextScheduler; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; +import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; +import com.netflix.hystrix.strategy.properties.HystrixProperty; +import com.netflix.hystrix.util.HystrixRollingNumberEvent; + +public class HystrixCommandTest { + + @Before + public void prepareForTest() { + /* we must call this to simulate a new request lifecycle running and clearing caches */ + HystrixRequestContext.initializeContext(); + } + + @After + public void cleanup() { + // instead of storing the reference from initialize we'll just get the current state and shutdown + if (HystrixRequestContext.getContextForCurrentThread() != null) { + // it could have been set NULL by the test + HystrixRequestContext.getContextForCurrentThread().shutdown(); + } + + // force properties to be clean as well + ConfigurationManager.getConfigInstance().clear(); + + HystrixCommandKey key = Hystrix.getCurrentThreadExecutingCommand(); + if (key != null) { + System.out.println("WARNING: Hystrix.getCurrentThreadExecutingCommand() should be null but got: " + key + ". Can occur when calling queue() and never retrieving."); + } + } + + /** + * Test a successful command execution. + */ + @Test + public void testExecutionSuccess() { + try { + TestHystrixCommand command = new SuccessfulTestCommand(); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(true, command.execute()); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + + assertEquals(null, command.getFailedExecutionException()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + } + + /** + * Test that a command can not be executed multiple times. + */ + @Test + public void testExecutionMultipleTimes() { + SuccessfulTestCommand command = new SuccessfulTestCommand(); + assertFalse(command.isExecutionComplete()); + // first should succeed + assertEquals(true, command.execute()); + assertTrue(command.isExecutionComplete()); + assertTrue(command.isExecutedInThread()); + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + try { + // second should fail + command.execute(); + fail("we should not allow this ... it breaks the state of request logs"); + } catch (IllegalStateException e) { + e.printStackTrace(); + // we want to get here + } + + try { + // queue should also fail + command.queue(); + fail("we should not allow this ... it breaks the state of request logs"); + } catch (IllegalStateException e) { + e.printStackTrace(); + // we want to get here + } + } + + /** + * Test a command execution that throws an HystrixException and didn't implement getFallback. + */ + @Test + public void testExecutionKnownFailureWithNoFallback() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + TestHystrixCommand command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (HystrixRuntimeException e) { + e.printStackTrace(); + assertNotNull(e.getFallbackException()); + assertNotNull(e.getImplementingClass()); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + } catch (Exception e) { + e.printStackTrace(); + fail("We should always get an HystrixRuntimeException when an error occurs."); + } + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution that throws an unknown exception (not HystrixException) and didn't implement getFallback. + */ + @Test + public void testExecutionUnknownFailureWithNoFallback() { + TestHystrixCommand command = new UnknownFailureTestCommandWithoutFallback(); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (HystrixRuntimeException e) { + e.printStackTrace(); + assertNotNull(e.getFallbackException()); + assertNotNull(e.getImplementingClass()); + + } catch (Exception e) { + e.printStackTrace(); + fail("We should always get an HystrixRuntimeException when an error occurs."); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution that fails but has a fallback. + */ + @Test + public void testExecutionFailureWithFallback() { + TestHystrixCommand command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); + try { + assertEquals(false, command.execute()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals("we failed with a simulated issue", command.getFailedExecutionException().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution that fails, has getFallback implemented but that fails as well. + */ + @Test + public void testExecutionFailureWithFallbackFailure() { + TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (HystrixRuntimeException e) { + System.out.println("------------------------------------------------"); + e.printStackTrace(); + System.out.println("------------------------------------------------"); + assertNotNull(e.getFallbackException()); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a successful command execution (asynchronously). + */ + @Test + public void testQueueSuccess() { + TestHystrixCommand command = new SuccessfulTestCommand(); + try { + Future future = command.queue(); + assertEquals(true, future.get()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution (asynchronously) that throws an HystrixException and didn't implement getFallback. + */ + @Test + public void testQueueKnownFailureWithNoFallback() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + TestHystrixCommand command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + + assertNotNull(de.getFallbackException()); + assertNotNull(de.getImplementingClass()); + } else { + fail("the cause should be HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution (asynchronously) that throws an unknown exception (not HystrixException) and didn't implement getFallback. + */ + @Test + public void testQueueUnknownFailureWithNoFallback() { + TestHystrixCommand command = new UnknownFailureTestCommandWithoutFallback(); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + assertNotNull(de.getFallbackException()); + assertNotNull(de.getImplementingClass()); + } else { + fail("the cause should be HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution (asynchronously) that fails but has a fallback. + */ + @Test + public void testQueueFailureWithFallback() { + TestHystrixCommand command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); + try { + Future future = command.queue(); + assertEquals(false, future.get()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution (asynchronously) that fails, has getFallback implemented but that fails as well. + */ + @Test + public void testQueueFailureWithFallbackFailure() { + TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + e.printStackTrace(); + assertNotNull(de.getFallbackException()); + } else { + fail("the cause should be HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a successful command execution. + */ + @Test + public void testObserveSuccess() { + try { + TestHystrixCommand command = new SuccessfulTestCommand(); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(true, command.observe().toBlockingObservable().single()); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + + assertEquals(null, command.getFailedExecutionException()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + } + + /** + * Test a successful command execution. + */ + @Test + public void testObserveOnScheduler() throws Exception { + for (int i = 0; i < 5; i++) { + final AtomicReference commandThread = new AtomicReference(); + final AtomicReference subscribeThread = new AtomicReference(); + + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder()) { + + @Override + protected Boolean run() { + commandThread.set(Thread.currentThread()); + return true; + } + }; + + final CountDownLatch latch = new CountDownLatch(1); + + command.toObservable(Schedulers.newThread()).subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + + } + + @Override + public void onError(Throwable e) { + latch.countDown(); + e.printStackTrace(); + + } + + @Override + public void onNext(Boolean args) { + subscribeThread.set(Thread.currentThread()); + } + }); + + if (!latch.await(2000, TimeUnit.MILLISECONDS)) { + fail("timed out"); + } + + assertNotNull(commandThread.get()); + assertNotNull(subscribeThread.get()); + + System.out.println("Command Thread: " + commandThread.get()); + System.out.println("Subscribe Thread: " + subscribeThread.get()); + + assertTrue(commandThread.get().getName().startsWith("hystrix-")); + assertFalse(subscribeThread.get().getName().startsWith("hystrix-")); + assertTrue(subscribeThread.get().getName().startsWith("Rx")); + } + } + + /** + * Test a successful command execution. + */ + @Test + public void testObserveOnComputationSchedulerByDefaultForThreadIsolation() throws Exception { + + final AtomicReference commandThread = new AtomicReference(); + final AtomicReference subscribeThread = new AtomicReference(); + + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder()) { + + @Override + protected Boolean run() { + commandThread.set(Thread.currentThread()); + return true; + } + }; + + final CountDownLatch latch = new CountDownLatch(1); + + command.toObservable().subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + + } + + @Override + public void onError(Throwable e) { + latch.countDown(); + e.printStackTrace(); + + } + + @Override + public void onNext(Boolean args) { + subscribeThread.set(Thread.currentThread()); + } + }); + + if (!latch.await(2000, TimeUnit.MILLISECONDS)) { + fail("timed out"); + } + + assertNotNull(commandThread.get()); + assertNotNull(subscribeThread.get()); + + System.out.println("Command Thread: " + commandThread.get()); + System.out.println("Subscribe Thread: " + subscribeThread.get()); + + assertTrue(commandThread.get().getName().startsWith("hystrix-")); + assertTrue(subscribeThread.get().getName().startsWith("RxComputationThreadPool")); + } + + /** + * Test a successful command execution. + */ + @Test + public void testObserveOnImmediateSchedulerByDefaultForSemaphoreIsolation() throws Exception { + + final AtomicReference commandThread = new AtomicReference(); + final AtomicReference subscribeThread = new AtomicReference(); + + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { + + @Override + protected Boolean run() { + commandThread.set(Thread.currentThread()); + return true; + } + }; + + final CountDownLatch latch = new CountDownLatch(1); + + command.toObservable().subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + + } + + @Override + public void onError(Throwable e) { + latch.countDown(); + e.printStackTrace(); + + } + + @Override + public void onNext(Boolean args) { + subscribeThread.set(Thread.currentThread()); + } + }); + + if (!latch.await(2000, TimeUnit.MILLISECONDS)) { + fail("timed out"); + } + + assertNotNull(commandThread.get()); + assertNotNull(subscribeThread.get()); + + System.out.println("Command Thread: " + commandThread.get()); + System.out.println("Subscribe Thread: " + subscribeThread.get()); + + String mainThreadName = Thread.currentThread().getName(); + + // semaphore should be on the calling thread + assertTrue(commandThread.get().getName().equals(mainThreadName)); + assertTrue(subscribeThread.get().getName().equals(mainThreadName)); + } + + /** + * Test that the circuit-breaker will 'trip' and prevent command execution on subsequent calls. + */ + @Test + public void testCircuitBreakerTripsAfterFailures() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + /* fail 3 times and then it should trip the circuit and stop executing */ + // failure 1 + KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt1.execute(); + assertTrue(attempt1.isResponseFromFallback()); + assertFalse(attempt1.isCircuitBreakerOpen()); + assertFalse(attempt1.isResponseShortCircuited()); + + // failure 2 + KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt2.execute(); + assertTrue(attempt2.isResponseFromFallback()); + assertFalse(attempt2.isCircuitBreakerOpen()); + assertFalse(attempt2.isResponseShortCircuited()); + + // failure 3 + KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt3.execute(); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + // it should now be 'open' and prevent further executions + assertTrue(attempt3.isCircuitBreakerOpen()); + + // attempt 4 + KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt4.execute(); + assertTrue(attempt4.isResponseFromFallback()); + // this should now be true as the response will be short-circuited + assertTrue(attempt4.isResponseShortCircuited()); + // this should remain open + assertTrue(attempt4.isCircuitBreakerOpen()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that the circuit-breaker will 'trip' and prevent command execution on subsequent calls. + */ + @Test + public void testCircuitBreakerTripsAfterFailuresViaQueue() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + /* fail 3 times and then it should trip the circuit and stop executing */ + // failure 1 + KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt1.queue().get(); + assertTrue(attempt1.isResponseFromFallback()); + assertFalse(attempt1.isCircuitBreakerOpen()); + assertFalse(attempt1.isResponseShortCircuited()); + + // failure 2 + KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt2.queue().get(); + assertTrue(attempt2.isResponseFromFallback()); + assertFalse(attempt2.isCircuitBreakerOpen()); + assertFalse(attempt2.isResponseShortCircuited()); + + // failure 3 + KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt3.queue().get(); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + // it should now be 'open' and prevent further executions + assertTrue(attempt3.isCircuitBreakerOpen()); + + // attempt 4 + KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt4.queue().get(); + assertTrue(attempt4.isResponseFromFallback()); + // this should now be true as the response will be short-circuited + assertTrue(attempt4.isResponseShortCircuited()); + // this should remain open + assertTrue(attempt4.isCircuitBreakerOpen()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received fallbacks."); + } + } + + /** + * Test that the circuit-breaker is shared across HystrixCommand objects with the same CommandKey. + *

+ * This will test HystrixCommand objects with a single circuit-breaker (as if each injected with same CommandKey) + *

+ * Multiple HystrixCommand objects with the same dependency use the same circuit-breaker. + */ + @Test + public void testCircuitBreakerAcrossMultipleCommandsButSameCircuitBreaker() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + /* fail 3 times and then it should trip the circuit and stop executing */ + // failure 1 + KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt1.execute(); + assertTrue(attempt1.isResponseFromFallback()); + assertFalse(attempt1.isCircuitBreakerOpen()); + assertFalse(attempt1.isResponseShortCircuited()); + + // failure 2 with a different command, same circuit breaker + KnownFailureTestCommandWithoutFallback attempt2 = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + attempt2.execute(); + } catch (Exception e) { + // ignore ... this doesn't have a fallback so will throw an exception + } + assertTrue(attempt2.isFailedExecution()); + assertFalse(attempt2.isResponseFromFallback()); // false because no fallback + assertFalse(attempt2.isCircuitBreakerOpen()); + assertFalse(attempt2.isResponseShortCircuited()); + + // failure 3 of the Hystrix, 2nd for this particular HystrixCommand + KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt3.execute(); + assertTrue(attempt2.isFailedExecution()); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + + // it should now be 'open' and prevent further executions + // after having 3 failures on the Hystrix that these 2 different HystrixCommand objects are for + assertTrue(attempt3.isCircuitBreakerOpen()); + + // attempt 4 + KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt4.execute(); + assertTrue(attempt4.isResponseFromFallback()); + // this should now be true as the response will be short-circuited + assertTrue(attempt4.isResponseShortCircuited()); + // this should remain open + assertTrue(attempt4.isCircuitBreakerOpen()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that the circuit-breaker is different between HystrixCommand objects with a different Hystrix. + */ + @Test + public void testCircuitBreakerAcrossMultipleCommandsAndDifferentDependency() { + TestCircuitBreaker circuitBreaker_one = new TestCircuitBreaker(); + TestCircuitBreaker circuitBreaker_two = new TestCircuitBreaker(); + /* fail 3 times, twice on one Hystrix, once on a different Hystrix ... circuit-breaker should NOT open */ + + // failure 1 + KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker_one); + attempt1.execute(); + assertTrue(attempt1.isResponseFromFallback()); + assertFalse(attempt1.isCircuitBreakerOpen()); + assertFalse(attempt1.isResponseShortCircuited()); + + // failure 2 with a different HystrixCommand implementation and different Hystrix + KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker_two); + attempt2.execute(); + assertTrue(attempt2.isResponseFromFallback()); + assertFalse(attempt2.isCircuitBreakerOpen()); + assertFalse(attempt2.isResponseShortCircuited()); + + // failure 3 but only 2nd of the Hystrix.ONE + KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker_one); + attempt3.execute(); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + + // it should remain 'closed' since we have only had 2 failures on Hystrix.ONE + assertFalse(attempt3.isCircuitBreakerOpen()); + + // this one should also remain closed as it only had 1 failure for Hystrix.TWO + assertFalse(attempt2.isCircuitBreakerOpen()); + + // attempt 4 (3rd attempt for Hystrix.ONE) + KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker_one); + attempt4.execute(); + // this should NOW flip to true as this is the 3rd failure for Hystrix.ONE + assertTrue(attempt3.isCircuitBreakerOpen()); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + + // Hystrix.TWO should still remain closed + assertFalse(attempt2.isCircuitBreakerOpen()); + + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(3, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(3, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker_one.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker_two.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that the circuit-breaker being disabled doesn't wreak havoc. + */ + @Test + public void testExecutionSuccessWithCircuitBreakerDisabled() { + TestHystrixCommand command = new TestCommandWithoutCircuitBreaker(); + try { + assertEquals(true, command.execute()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + + // we'll still get metrics ... just not the circuit breaker opening/closing + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution timeout where the command didn't implement getFallback. + */ + @Test + public void testExecutionTimeoutWithNoFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (Exception e) { + // e.printStackTrace(); + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be HystrixRuntimeException"); + } + } + // the time should be 50+ since we timeout at 50ms + assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); + + assertTrue(command.isResponseTimedOut()); + assertFalse(command.isResponseFromFallback()); + assertFalse(command.isResponseRejected()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution timeout where the command implemented getFallback. + */ + @Test + public void testExecutionTimeoutWithFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + assertEquals(false, command.execute()); + // the time should be 50+ since we timeout at 50ms + assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); + assertTrue(command.isResponseTimedOut()); + assertTrue(command.isResponseFromFallback()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a command execution timeout where the command implemented getFallback but it fails. + */ + @Test + public void testExecutionTimeoutFallbackFailure() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be HystrixRuntimeException"); + } + } + // the time should be 50+ since we timeout at 50ms + assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that the circuit-breaker counts a command execution timeout as a 'timeout' and not just failure. + */ + @Test + public void testCircuitBreakerOnExecutionTimeout() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + + command.execute(); + + assertTrue(command.isResponseFromFallback()); + assertFalse(command.isCircuitBreakerOpen()); + assertFalse(command.isResponseShortCircuited()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that the command finishing AFTER a timeout (because thread continues in background) does not register a SUCCESS + */ + @Test + public void testCountersOnExecutionTimeout() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + command.execute(); + + /* wait long enough for the command to have finished */ + Thread.sleep(200); + + /* response should still be the same as 'testCircuitBreakerOnExecutionTimeout' */ + assertTrue(command.isResponseFromFallback()); + assertFalse(command.isCircuitBreakerOpen()); + assertFalse(command.isResponseShortCircuited()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + assertFalse(command.isSuccessfulExecution()); + + /* failure and timeout count should be the same as 'testCircuitBreakerOnExecutionTimeout' */ + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + + /* we should NOT have a 'success' counter */ + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a queued command execution timeout where the command didn't implement getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testQueuedExecutionTimeoutWithNoFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + if (e instanceof ExecutionException && e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testQueuedExecutionTimeoutWithFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + assertEquals(false, command.queue().get()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback but it fails. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testQueuedExecutionTimeoutFallbackFailure() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e instanceof ExecutionException && e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + assertNotNull(de.getFallbackException()); + assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a queued command execution timeout where the command didn't implement getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testObservedExecutionTimeoutWithNoFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + try { + command.observe().toBlockingObservable().single(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testObservedExecutionTimeoutWithFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + assertEquals(false, command.observe().toBlockingObservable().single()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback but it fails. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testObservedExecutionTimeoutFallbackFailure() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE); + try { + command.observe().toBlockingObservable().single(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that the circuit-breaker counts a command execution timeout as a 'timeout' and not just failure. + */ + @Test + public void testShortCircuitFallbackCounter() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().setForceShortCircuit(true); + try { + new KnownFailureTestCommandWithFallback(circuitBreaker).execute(); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + + KnownFailureTestCommandWithFallback command = new KnownFailureTestCommandWithFallback(circuitBreaker); + command.execute(); + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + + // will be -1 because it never attempted execution + assertTrue(command.getExecutionTimeInMilliseconds() == -1); + assertTrue(command.isResponseShortCircuited()); + assertFalse(command.isResponseTimedOut()); + + // because it was short-circuited to a fallback we don't count an error + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test when a command fails to get queued up in the threadpool where the command didn't implement getFallback. + *

+ * We specifically want to protect against developers getting random thread exceptions and instead just correctly receiving HystrixRuntimeException when no fallback exists. + */ + @Test + public void testRejectedThreadWithNoFallback() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SingleThreadedPool pool = new SingleThreadedPool(1); + // fill up the queue + pool.queue.add(new Runnable() { + + @Override + public void run() { + System.out.println("**** queue filler1 ****"); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + }); + + Future f = null; + TestCommandRejection command = null; + try { + f = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_NOT_IMPLEMENTED).queue(); + command = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_NOT_IMPLEMENTED); + command.queue(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("command.getExecutionTimeInMilliseconds(): " + command.getExecutionTimeInMilliseconds()); + // will be -1 because it never attempted execution + assertTrue(command.getExecutionTimeInMilliseconds() == -1); + assertTrue(command.isResponseRejected()); + assertFalse(command.isResponseShortCircuited()); + assertFalse(command.isResponseTimedOut()); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + if (e instanceof HystrixRuntimeException && e.getCause() instanceof RejectedExecutionException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof RejectedExecutionException); + } else { + fail("the exception should be HystrixRuntimeException with cause as RejectedExecutionException"); + } + } + + try { + f.get(); + } catch (Exception e) { + e.printStackTrace(); + fail("The first one should succeed."); + } + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(50, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test when a command fails to get queued up in the threadpool where the command implemented getFallback. + *

+ * We specifically want to protect against developers getting random thread exceptions and instead just correctly receives a fallback. + */ + @Test + public void testRejectedThreadWithFallback() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SingleThreadedPool pool = new SingleThreadedPool(1); + // fill up the queue + pool.queue.add(new Runnable() { + + @Override + public void run() { + System.out.println("**** queue filler1 ****"); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + }); + + try { + TestCommandRejection command1 = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS); + command1.queue(); + TestCommandRejection command2 = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS); + assertEquals(false, command2.queue().get()); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertFalse(command1.isResponseRejected()); + assertFalse(command1.isResponseFromFallback()); + assertTrue(command2.isResponseRejected()); + assertTrue(command2.isResponseFromFallback()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test when a command fails to get queued up in the threadpool where the command implemented getFallback but it fails. + *

+ * We specifically want to protect against developers getting random thread exceptions and instead just correctly receives an HystrixRuntimeException. + */ + @Test + public void testRejectedThreadWithFallbackFailure() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SingleThreadedPool pool = new SingleThreadedPool(1); + // fill up the queue + pool.queue.add(new Runnable() { + + @Override + public void run() { + System.out.println("**** queue filler1 ****"); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + }); + + try { + new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_FAILURE).queue(); + assertEquals(false, new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_FAILURE).queue().get()); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + if (e instanceof HystrixRuntimeException && e.getCause() instanceof RejectedExecutionException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof RejectedExecutionException); + } else { + fail("the exception should be HystrixRuntimeException with cause as RejectedExecutionException"); + } + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that we can reject a thread using isQueueSpaceAvailable() instead of just when the pool rejects. + *

+ * For example, we have queue size set to 100 but want to reject when we hit 10. + *

+ * This allows us to use FastProperties to control our rejection point whereas we can't resize a queue after it's created. + */ + @Test + public void testRejectedThreadUsingQueueSize() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SingleThreadedPool pool = new SingleThreadedPool(10, 1); + // put 1 item in the queue + // the thread pool won't pick it up because we're bypassing the pool and adding to the queue directly so this will keep the queue full + pool.queue.add(new Runnable() { + + @Override + public void run() { + System.out.println("**** queue filler1 ****"); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + }); + + TestCommandRejection command = null; + try { + // this should fail as we already have 1 in the queue + command = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_NOT_IMPLEMENTED); + command.queue(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + + // will be -1 because it never attempted execution + assertTrue(command.getExecutionTimeInMilliseconds() == -1); + assertTrue(command.isResponseRejected()); + assertFalse(command.isResponseShortCircuited()); + assertFalse(command.isResponseTimedOut()); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + if (e instanceof HystrixRuntimeException && e.getCause() instanceof RejectedExecutionException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof RejectedExecutionException); + } else { + fail("the exception should be HystrixRuntimeException with cause as RejectedExecutionException"); + } + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * If it has been sitting in the queue, it should not execute if timed out by the time it hits the queue. + */ + @Test + public void testTimedOutCommandDoesNotExecute() { + SingleThreadedPool pool = new SingleThreadedPool(5); + + TestCircuitBreaker s1 = new TestCircuitBreaker(); + TestCircuitBreaker s2 = new TestCircuitBreaker(); + + // execution will take 100ms, thread pool has a 600ms timeout + CommandWithCustomThreadPool c1 = new CommandWithCustomThreadPool(s1, pool, 500, HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(600)); + // execution will take 200ms, thread pool has a 20ms timeout + CommandWithCustomThreadPool c2 = new CommandWithCustomThreadPool(s2, pool, 200, HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(20)); + // queue up c1 first + Future c1f = c1.queue(); + // now queue up c2 and wait on it + boolean receivedException = false; + try { + c2.queue().get(); + } catch (Exception e) { + // we expect to get an exception here + receivedException = true; + } + + if (!receivedException) { + fail("We expect to receive an exception for c2 as it's supposed to timeout."); + } + + // c1 will complete after 100ms + try { + c1f.get(); + } catch (Exception e1) { + e1.printStackTrace(); + fail("we should not have failed while getting c1"); + } + assertTrue("c1 is expected to executed but didn't", c1.didExecute); + + // c2 will timeout after 20 ms ... we'll wait longer than the 200ms time to make sure + // the thread doesn't keep running in the background and execute + try { + Thread.sleep(400); + } catch (Exception e) { + throw new RuntimeException("Failed to sleep"); + } + assertFalse("c2 is not expected to execute, but did", c2.didExecute); + + assertEquals(1, s1.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, s1.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, s1.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, s2.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, s2.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, s2.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, s2.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testFallbackSemaphore() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + // single thread should work + try { + boolean result = new TestSemaphoreCommandWithSlowFallback(circuitBreaker, 1, 200).queue().get(); + assertTrue(result); + } catch (Exception e) { + // we shouldn't fail on this one + throw new RuntimeException(e); + } + + // 2 threads, the second should be rejected by the fallback semaphore + boolean exceptionReceived = false; + Future result = null; + try { + System.out.println("c2 start: " + System.currentTimeMillis()); + result = new TestSemaphoreCommandWithSlowFallback(circuitBreaker, 1, 800).queue(); + System.out.println("c2 after queue: " + System.currentTimeMillis()); + // make sure that thread gets a chance to run before queuing the next one + Thread.sleep(50); + System.out.println("c3 start: " + System.currentTimeMillis()); + Future result2 = new TestSemaphoreCommandWithSlowFallback(circuitBreaker, 1, 200).queue(); + System.out.println("c3 after queue: " + System.currentTimeMillis()); + result2.get(); + } catch (Exception e) { + e.printStackTrace(); + exceptionReceived = true; + } + + try { + assertTrue(result.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + if (!exceptionReceived) { + fail("We expected an exception on the 2nd get"); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + // TestSemaphoreCommandWithSlowFallback always fails so all 3 should show failure + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + // the 1st thread executes single-threaded and gets a fallback, the next 2 are concurrent so only 1 of them is permitted by the fallback semaphore so 1 is rejected + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + // whenever a fallback_rejection occurs it is also a fallback_failure + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + // we should not have rejected any via the "execution semaphore" but instead via the "fallback semaphore" + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + // the rest should not be involved in this test + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testExecutionSemaphoreWithQueue() { + final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + // single thread should work + try { + boolean result = new TestSemaphoreCommand(circuitBreaker, 1, 200).queue().get(); + assertTrue(result); + } catch (Exception e) { + // we shouldn't fail on this one + throw new RuntimeException(e); + } + + final AtomicBoolean exceptionReceived = new AtomicBoolean(); + + final TryableSemaphore semaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(1)); + + Runnable r = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + + @Override + public void run() { + try { + new TestSemaphoreCommand(circuitBreaker, semaphore, 200).queue().get(); + } catch (Exception e) { + e.printStackTrace(); + exceptionReceived.set(true); + } + } + + }); + // 2 threads, the second should be rejected by the semaphore + Thread t1 = new Thread(r); + Thread t2 = new Thread(r); + + t1.start(); + t2.start(); + try { + t1.join(); + t2.join(); + } catch (Exception e) { + e.printStackTrace(); + fail("failed waiting on threads"); + } + + if (!exceptionReceived.get()) { + fail("We expected an exception on the 2nd get"); + } + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + // we don't have a fallback so threw an exception when rejected + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + // not a failure as the command never executed so can't fail + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + // no fallback failure as there isn't a fallback implemented + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + // we should have rejected via semaphore + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + // the rest should not be involved in this test + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testExecutionSemaphoreWithExecution() { + final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + // single thread should work + try { + TestSemaphoreCommand command = new TestSemaphoreCommand(circuitBreaker, 1, 200); + boolean result = command.execute(); + assertFalse(command.isExecutedInThread()); + assertTrue(result); + } catch (Exception e) { + // we shouldn't fail on this one + throw new RuntimeException(e); + } + + final ArrayBlockingQueue results = new ArrayBlockingQueue(2); + + final AtomicBoolean exceptionReceived = new AtomicBoolean(); + + final TryableSemaphore semaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(1)); + + Runnable r = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + + @Override + public void run() { + try { + results.add(new TestSemaphoreCommand(circuitBreaker, semaphore, 200).execute()); + } catch (Exception e) { + e.printStackTrace(); + exceptionReceived.set(true); + } + } + + }); + // 2 threads, the second should be rejected by the semaphore + Thread t1 = new Thread(r); + Thread t2 = new Thread(r); + + t1.start(); + t2.start(); + try { + t1.join(); + t2.join(); + } catch (Exception e) { + e.printStackTrace(); + fail("failed waiting on threads"); + } + + if (!exceptionReceived.get()) { + fail("We expected an exception on the 2nd get"); + } + + // only 1 value is expected as the other should have thrown an exception + assertEquals(1, results.size()); + // should contain only a true result + assertTrue(results.contains(Boolean.TRUE)); + assertFalse(results.contains(Boolean.FALSE)); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + // no failure ... we throw an exception because of rejection but the command does not fail execution + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + // there is no fallback implemented so no failure can occur on it + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + // we rejected via semaphore + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + // the rest should not be involved in this test + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testRejectedExecutionSemaphoreWithFallback() { + final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + final ArrayBlockingQueue results = new ArrayBlockingQueue(2); + + final AtomicBoolean exceptionReceived = new AtomicBoolean(); + + Runnable r = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + + @Override + public void run() { + try { + results.add(new TestSemaphoreCommandWithFallback(circuitBreaker, 1, 200, false).execute()); + } catch (Exception e) { + e.printStackTrace(); + exceptionReceived.set(true); + } + } + + }); + + // 2 threads, the second should be rejected by the semaphore and return fallback + Thread t1 = new Thread(r); + Thread t2 = new Thread(r); + + t1.start(); + t2.start(); + try { + t1.join(); + t2.join(); + } catch (Exception e) { + e.printStackTrace(); + fail("failed waiting on threads"); + } + + if (exceptionReceived.get()) { + fail("We should have received a fallback response"); + } + + // both threads should have returned values + assertEquals(2, results.size()); + // should contain both a true and false result + assertTrue(results.contains(Boolean.TRUE)); + assertTrue(results.contains(Boolean.FALSE)); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + // the rest should not be involved in this test + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + System.out.println("**** DONE"); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Tests that semaphores are counted separately for commands with unique keys + */ + @Test + public void testSemaphorePermitsInUse() { + final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + + // this semaphore will be shared across multiple command instances + final TryableSemaphore sharedSemaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(3)); + + // used to wait until all commands have started + final CountDownLatch startLatch = new CountDownLatch(sharedSemaphore.numberOfPermits.get() + 1); + + // used to signal that all command can finish + final CountDownLatch sharedLatch = new CountDownLatch(1); + + final Runnable sharedSemaphoreRunnable = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + public void run() { + try { + new LatchedSemaphoreCommand(circuitBreaker, sharedSemaphore, startLatch, sharedLatch).execute(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + + // creates group of threads each using command sharing a single semaphore + + // I create extra threads and commands so that I can verify that some of them fail to obtain a semaphore + final int sharedThreadCount = sharedSemaphore.numberOfPermits.get() * 2; + final Thread[] sharedSemaphoreThreads = new Thread[sharedThreadCount]; + for (int i = 0; i < sharedThreadCount; i++) { + sharedSemaphoreThreads[i] = new Thread(sharedSemaphoreRunnable); + } + + // creates thread using isolated semaphore + final TryableSemaphore isolatedSemaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(1)); + + final CountDownLatch isolatedLatch = new CountDownLatch(1); + + // tracks failures to obtain semaphores + final AtomicInteger failureCount = new AtomicInteger(); + + final Thread isolatedThread = new Thread(new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + public void run() { + try { + new LatchedSemaphoreCommand(circuitBreaker, isolatedSemaphore, startLatch, isolatedLatch).execute(); + } catch (Exception e) { + e.printStackTrace(); + failureCount.incrementAndGet(); + } + } + })); + + // verifies no permits in use before starting threads + assertEquals("wrong number of permits for shared semaphore", 0, sharedSemaphore.getNumberOfPermitsUsed()); + assertEquals("wrong number of permits for isolated semaphore", 0, isolatedSemaphore.getNumberOfPermitsUsed()); + + for (int i = 0; i < sharedThreadCount; i++) { + sharedSemaphoreThreads[i].start(); + } + isolatedThread.start(); + + // waits until all commands have started + try { + startLatch.await(1000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + // verifies that all semaphores are in use + assertEquals("wrong number of permits for shared semaphore", + sharedSemaphore.numberOfPermits.get().longValue(), sharedSemaphore.getNumberOfPermitsUsed()); + assertEquals("wrong number of permits for isolated semaphore", + isolatedSemaphore.numberOfPermits.get().longValue(), isolatedSemaphore.getNumberOfPermitsUsed()); + + // signals commands to finish + sharedLatch.countDown(); + isolatedLatch.countDown(); + + try { + for (int i = 0; i < sharedThreadCount; i++) { + sharedSemaphoreThreads[i].join(); + } + isolatedThread.join(); + } catch (Exception e) { + e.printStackTrace(); + fail("failed waiting on threads"); + } + + // verifies no permits in use after finishing threads + assertEquals("wrong number of permits for shared semaphore", 0, sharedSemaphore.getNumberOfPermitsUsed()); + assertEquals("wrong number of permits for isolated semaphore", 0, isolatedSemaphore.getNumberOfPermitsUsed()); + + // verifies that some executions failed + final int expectedFailures = sharedSemaphore.getNumberOfPermitsUsed(); + assertEquals("failures expected but did not happen", expectedFailures, failureCount.get()); + } + + /** + * Test that HystrixOwner can be passed in dynamically. + */ + @Test + public void testDynamicOwner() { + try { + TestHystrixCommand command = new DynamicOwnerTestCommand(CommandGroupForUnitTest.OWNER_ONE); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(true, command.execute()); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + } + + /** + * Test a successful command execution. + */ + @Test + public void testDynamicOwnerFails() { + try { + TestHystrixCommand command = new DynamicOwnerTestCommand(null); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(true, command.execute()); + fail("we should have thrown an exception as we need an owner"); + } catch (Exception e) { + // success if we get here + } + } + + /** + * Test that HystrixCommandKey can be passed in dynamically. + */ + @Test + public void testDynamicKey() { + try { + DynamicOwnerAndKeyTestCommand command1 = new DynamicOwnerAndKeyTestCommand(CommandGroupForUnitTest.OWNER_ONE, CommandKeyForUnitTest.KEY_ONE); + assertEquals(true, command1.execute()); + DynamicOwnerAndKeyTestCommand command2 = new DynamicOwnerAndKeyTestCommand(CommandGroupForUnitTest.OWNER_ONE, CommandKeyForUnitTest.KEY_TWO); + assertEquals(true, command2.execute()); + + // 2 different circuit breakers should be created + assertNotSame(command1.getCircuitBreaker(), command2.getCircuitBreaker()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + } + + /** + * Test Request scoped caching of commands so that a 2nd duplicate call doesn't execute but returns the previous Future + */ + @Test + public void testRequestCache1() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + + assertTrue(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("A", f2.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // the second one should not have executed as it should have received the cached value instead + assertFalse(command2.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command1.getExecutionTimeInMilliseconds() > -1); + assertFalse(command1.isResponseFromCache()); + + // the execution log for command2 should show it came from cache + assertEquals(2, command2.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + assertTrue(command2.getExecutionTimeInMilliseconds() == -1); + assertTrue(command2.isResponseFromCache()); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test Request scoped caching doesn't prevent different ones from executing + */ + @Test + public void testRequestCache2() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "B"); + + assertTrue(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command2.getExecutionTimeInMilliseconds() > -1); + assertFalse(command2.isResponseFromCache()); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testRequestCache3() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "B"); + SuccessfulCacheableCommand command3 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + + assertTrue(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("A", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // but the 3rd should come from cache + assertFalse(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show it came from cache + assertEquals(2, command3.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + assertTrue(command3.getExecutionTimeInMilliseconds() == -1); + assertTrue(command3.isResponseFromCache()); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test Request scoped caching of commands so that a 2nd duplicate call doesn't execute but returns the previous Future + */ + @Test + public void testRequestCacheWithSlowExecution() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SlowCacheableCommand command1 = new SlowCacheableCommand(circuitBreaker, "A", 200); + SlowCacheableCommand command2 = new SlowCacheableCommand(circuitBreaker, "A", 100); + SlowCacheableCommand command3 = new SlowCacheableCommand(circuitBreaker, "A", 100); + SlowCacheableCommand command4 = new SlowCacheableCommand(circuitBreaker, "A", 100); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + Future f4 = command4.queue(); + + try { + assertEquals("A", f2.get()); + assertEquals("A", f3.get()); + assertEquals("A", f4.get()); + + assertEquals("A", f1.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // the second one should not have executed as it should have received the cached value instead + assertFalse(command2.executed); + assertFalse(command3.executed); + assertFalse(command4.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command1.getExecutionTimeInMilliseconds() > -1); + assertFalse(command1.isResponseFromCache()); + + // the execution log for command2 should show it came from cache + assertEquals(2, command2.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + assertTrue(command2.getExecutionTimeInMilliseconds() == -1); + assertTrue(command2.isResponseFromCache()); + + assertTrue(command3.isResponseFromCache()); + assertTrue(command3.getExecutionTimeInMilliseconds() == -1); + assertTrue(command4.isResponseFromCache()); + assertTrue(command4.getExecutionTimeInMilliseconds() == -1); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + System.out.println("HystrixRequestLog: " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testNoRequestCache3() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, false, "A"); + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, false, "B"); + SuccessfulCacheableCommand command3 = new SuccessfulCacheableCommand(circuitBreaker, false, "A"); + + assertTrue(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("A", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // this should also execute since we disabled the cache + assertTrue(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show a SUCCESS + assertEquals(1, command3.getExecutionEvents().size()); + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testRequestCacheViaQueueSemaphore1() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); + SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "B"); + SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); + + assertFalse(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("A", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // but the 3rd should come from cache + assertFalse(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show it comes from cache + assertEquals(2, command3.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + + assertTrue(command3.isResponseFromCache()); + assertTrue(command3.getExecutionTimeInMilliseconds() == -1); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testNoRequestCacheViaQueueSemaphore1() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); + SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "B"); + SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); + + assertFalse(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("A", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // this should also execute because caching is disabled + assertTrue(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show a SUCCESS + assertEquals(1, command3.getExecutionEvents().size()); + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testRequestCacheViaExecuteSemaphore1() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); + SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "B"); + SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); + + assertFalse(command1.isCommandRunningInThread()); + + String f1 = command1.execute(); + String f2 = command2.execute(); + String f3 = command3.execute(); + + assertEquals("A", f1); + assertEquals("B", f2); + assertEquals("A", f3); + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // but the 3rd should come from cache + assertFalse(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show it comes from cache + assertEquals(2, command3.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testNoRequestCacheViaExecuteSemaphore1() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); + SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "B"); + SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); + + assertFalse(command1.isCommandRunningInThread()); + + String f1 = command1.execute(); + String f2 = command2.execute(); + String f3 = command3.execute(); + + assertEquals("A", f1); + assertEquals("B", f2); + assertEquals("A", f3); + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // this should also execute because caching is disabled + assertTrue(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show a SUCCESS + assertEquals(1, command3.getExecutionEvents().size()); + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testNoRequestCacheOnTimeoutThrowsException() throws Exception { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + NoRequestCacheTimeoutWithoutFallback r1 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + System.out.println("r1 value: " + r1.execute()); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r1.isResponseTimedOut()); + // what we want + } + + NoRequestCacheTimeoutWithoutFallback r2 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r2.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r2.isResponseTimedOut()); + // what we want + } + + NoRequestCacheTimeoutWithoutFallback r3 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + Future f3 = r3.queue(); + try { + f3.get(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (ExecutionException e) { + e.printStackTrace(); + assertTrue(r3.isResponseTimedOut()); + // what we want + } + + Thread.sleep(500); // timeout on command is set to 200ms + + NoRequestCacheTimeoutWithoutFallback r4 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r4.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r4.isResponseTimedOut()); + assertFalse(r4.isResponseFromFallback()); + // what we want + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testRequestCacheOnTimeoutCausesNullPointerException() throws Exception { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + // Expect it to time out - all results should be false + assertFalse(new RequestCacheNullPointerExceptionCase(circuitBreaker).execute()); + assertFalse(new RequestCacheNullPointerExceptionCase(circuitBreaker).execute()); // return from cache #1 + assertFalse(new RequestCacheNullPointerExceptionCase(circuitBreaker).execute()); // return from cache #2 + Thread.sleep(500); // timeout on command is set to 200ms + Boolean value = new RequestCacheNullPointerExceptionCase(circuitBreaker).execute(); // return from cache #3 + assertFalse(value); + RequestCacheNullPointerExceptionCase c = new RequestCacheNullPointerExceptionCase(circuitBreaker); + Future f = c.queue(); // return from cache #4 + // the bug is that we're getting a null Future back, rather than a Future that returns false + assertNotNull(f); + assertFalse(f.get()); + + assertTrue(c.isResponseFromFallback()); + assertTrue(c.isResponseTimedOut()); + assertFalse(c.isFailedExecution()); + assertFalse(c.isResponseShortCircuited()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(5, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + + HystrixCommand[] executeCommands = HystrixRequestLog.getCurrentRequest().getExecutedCommands().toArray(new HystrixCommand[] {}); + + System.out.println(":executeCommands[0].getExecutionEvents()" + executeCommands[0].getExecutionEvents()); + assertEquals(2, executeCommands[0].getExecutionEvents().size()); + assertTrue(executeCommands[0].getExecutionEvents().contains(HystrixEventType.FALLBACK_SUCCESS)); + assertTrue(executeCommands[0].getExecutionEvents().contains(HystrixEventType.TIMEOUT)); + assertTrue(executeCommands[0].getExecutionTimeInMilliseconds() > -1); + assertTrue(executeCommands[0].isResponseTimedOut()); + assertTrue(executeCommands[0].isResponseFromFallback()); + assertFalse(executeCommands[0].isResponseFromCache()); + + assertEquals(3, executeCommands[1].getExecutionEvents().size()); // it will include FALLBACK_SUCCESS/TIMEOUT + RESPONSE_FROM_CACHE + assertTrue(executeCommands[1].getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + assertTrue(executeCommands[1].getExecutionTimeInMilliseconds() == -1); + assertTrue(executeCommands[1].isResponseFromCache()); + assertTrue(executeCommands[1].isResponseTimedOut()); + assertTrue(executeCommands[1].isResponseFromFallback()); + } + + @Test + public void testRequestCacheOnTimeoutThrowsException() throws Exception { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + RequestCacheTimeoutWithoutFallback r1 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + System.out.println("r1 value: " + r1.execute()); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r1.isResponseTimedOut()); + // what we want + } + + RequestCacheTimeoutWithoutFallback r2 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r2.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r2.isResponseTimedOut()); + // what we want + } + + RequestCacheTimeoutWithoutFallback r3 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); + Future f3 = r3.queue(); + try { + f3.get(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (ExecutionException e) { + e.printStackTrace(); + assertTrue(r3.isResponseTimedOut()); + // what we want + } + + Thread.sleep(500); // timeout on command is set to 200ms + + RequestCacheTimeoutWithoutFallback r4 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r4.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r4.isResponseTimedOut()); + assertFalse(r4.isResponseFromFallback()); + // what we want + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testRequestCacheOnThreadRejectionThrowsException() throws Exception { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CountDownLatch completionLatch = new CountDownLatch(1); + RequestCacheThreadRejectionWithoutFallback r1 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); + try { + System.out.println("r1: " + r1.execute()); + // we should have thrown an exception + fail("expected a rejection"); + } catch (HystrixRuntimeException e) { + assertTrue(r1.isResponseRejected()); + // what we want + } + + RequestCacheThreadRejectionWithoutFallback r2 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); + try { + System.out.println("r2: " + r2.execute()); + // we should have thrown an exception + fail("expected a rejection"); + } catch (HystrixRuntimeException e) { + // e.printStackTrace(); + assertTrue(r2.isResponseRejected()); + // what we want + } + + RequestCacheThreadRejectionWithoutFallback r3 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); + try { + System.out.println("f3: " + r3.queue().get()); + // we should have thrown an exception + fail("expected a rejection"); + } catch (HystrixRuntimeException e) { + // e.printStackTrace(); + assertTrue(r3.isResponseRejected()); + // what we want + } + + // let the command finish (only 1 should actually be blocked on this due to the response cache) + completionLatch.countDown(); + + // then another after the command has completed + RequestCacheThreadRejectionWithoutFallback r4 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); + try { + System.out.println("r4: " + r4.execute()); + // we should have thrown an exception + fail("expected a rejection"); + } catch (HystrixRuntimeException e) { + // e.printStackTrace(); + assertTrue(r4.isResponseRejected()); + assertFalse(r4.isResponseFromFallback()); + // what we want + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that we can do basic execution without a RequestVariable being initialized. + */ + @Test + public void testBasicExecutionWorksWithoutRequestVariable() { + try { + /* force the RequestVariable to not be initialized */ + HystrixRequestContext.setContextOnCurrentThread(null); + + TestHystrixCommand command = new SuccessfulTestCommand(); + assertEquals(true, command.execute()); + + TestHystrixCommand command2 = new SuccessfulTestCommand(); + assertEquals(true, command2.queue().get()); + + // we should be able to execute without a RequestVariable if ... + // 1) We don't have a cacheKey + // 2) We don't ask for the RequestLog + // 3) We don't do collapsing + + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception => " + e.getMessage()); + } + } + + /** + * Test that if we try and execute a command with a cacheKey without initializing RequestVariable that it gives an error. + */ + @Test + public void testCacheKeyExecutionRequiresRequestVariable() { + try { + /* force the RequestVariable to not be initialized */ + HystrixRequestContext.setContextOnCurrentThread(null); + + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + + SuccessfulCacheableCommand command = new SuccessfulCacheableCommand(circuitBreaker, true, "one"); + assertEquals(true, command.execute()); + + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "two"); + assertEquals(true, command2.queue().get()); + + fail("We expect an exception because cacheKey requires RequestVariable."); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. + */ + @Test + public void testBadRequestExceptionViaExecuteInThread() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).execute(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (HystrixBadRequestException e) { + // success + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. + */ + @Test + public void testBadRequestExceptionViaQueueInThread() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).queue().get(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (ExecutionException e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixBadRequestException) { + // success + } else { + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test that BadRequestException behavior works the same on a cached response. + */ + @Test + public void testBadRequestExceptionViaQueueInThreadOnResponseFromCache() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + + // execute once to cache the value + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).execute(); + } catch (Throwable e) { + // ignore + } + + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).queue().get(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (ExecutionException e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixBadRequestException) { + // success + } else { + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. + */ + @Test + public void testBadRequestExceptionViaExecuteInSemaphore() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.SEMAPHORE).execute(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (HystrixBadRequestException e) { + // success + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. + */ + @Test + public void testBadRequestExceptionViaQueueInSemaphore() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.SEMAPHORE).queue().get(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (ExecutionException e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixBadRequestException) { + // success + } else { + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test a checked Exception being thrown + */ + @Test + public void testCheckedExceptionViaExecute() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithCheckedException command = new CommandWithCheckedException(circuitBreaker); + try { + command.execute(); + fail("we expect to receive a " + Exception.class.getSimpleName()); + } catch (Exception e) { + assertEquals("simulated checked exception message", e.getCause().getMessage()); + } + + assertEquals("simulated checked exception message", command.getFailedExecutionException().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + } + + /** + * Test a java.lang.Error being thrown + * + * @throws InterruptedException + */ + @Test + public void testCheckedExceptionViaObserve() throws InterruptedException { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithCheckedException command = new CommandWithCheckedException(circuitBreaker); + final AtomicReference t = new AtomicReference(); + final CountDownLatch latch = new CountDownLatch(1); + try { + command.observe().subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + } + + @Override + public void onError(Throwable e) { + t.set(e); + latch.countDown(); + } + + @Override + public void onNext(Boolean args) { + + } + + }); + } catch (Exception e) { + e.printStackTrace(); + fail("we should not get anything thrown, it should be emitted via the Observer#onError method"); + } + + latch.await(1, TimeUnit.SECONDS); + assertNotNull(t.get()); + t.get().printStackTrace(); + + assertTrue(t.get() instanceof HystrixRuntimeException); + assertEquals("simulated checked exception message", t.get().getCause().getMessage()); + assertEquals("simulated checked exception message", command.getFailedExecutionException().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + } + + /** + * Test a java.lang.Error being thrown + */ + @Test + public void testErrorThrownViaExecute() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithErrorThrown command = new CommandWithErrorThrown(circuitBreaker); + try { + command.execute(); + fail("we expect to receive a " + Error.class.getSimpleName()); + } catch (Exception e) { + // the actual error is an extra cause level deep because Hystrix needs to wrap Throwable/Error as it's public + // methods only support Exception and it's not a strong enough reason to break backwards compatibility and jump to version 2.x + // so HystrixRuntimeException -> wrapper Exception -> actual Error + assertEquals("simulated java.lang.Error message", e.getCause().getCause().getMessage()); + } + + assertEquals("simulated java.lang.Error message", command.getFailedExecutionException().getCause().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + } + + /** + * Test a java.lang.Error being thrown + */ + @Test + public void testErrorThrownViaQueue() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithErrorThrown command = new CommandWithErrorThrown(circuitBreaker); + try { + command.queue().get(); + fail("we expect to receive an Exception"); + } catch (Exception e) { + // one cause down from ExecutionException to HystrixRuntime + // then the actual error is an extra cause level deep because Hystrix needs to wrap Throwable/Error as it's public + // methods only support Exception and it's not a strong enough reason to break backwards compatibility and jump to version 2.x + // so ExecutionException -> HystrixRuntimeException -> wrapper Exception -> actual Error + assertEquals("simulated java.lang.Error message", e.getCause().getCause().getCause().getMessage()); + } + + assertEquals("simulated java.lang.Error message", command.getFailedExecutionException().getCause().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + } + + /** + * Test a java.lang.Error being thrown + * + * @throws InterruptedException + */ + @Test + public void testErrorThrownViaObserve() throws InterruptedException { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithErrorThrown command = new CommandWithErrorThrown(circuitBreaker); + final AtomicReference t = new AtomicReference(); + final CountDownLatch latch = new CountDownLatch(1); + try { + command.observe().subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + } + + @Override + public void onError(Throwable e) { + t.set(e); + latch.countDown(); + } + + @Override + public void onNext(Boolean args) { + + } + + }); + } catch (Exception e) { + e.printStackTrace(); + fail("we should not get anything thrown, it should be emitted via the Observer#onError method"); + } + + latch.await(1, TimeUnit.SECONDS); + assertNotNull(t.get()); + t.get().printStackTrace(); + + assertTrue(t.get() instanceof HystrixRuntimeException); + // the actual error is an extra cause level deep because Hystrix needs to wrap Throwable/Error as it's public + // methods only support Exception and it's not a strong enough reason to break backwards compatibility and jump to version 2.x + assertEquals("simulated java.lang.Error message", t.get().getCause().getCause().getMessage()); + assertEquals("simulated java.lang.Error message", command.getFailedExecutionException().getCause().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + } + + /** + * Execution hook on successful execution + */ + @Test + public void testExecutionHookSuccessfulCommand() { + /* test with execute() */ + TestHystrixCommand command = new SuccessfulTestCommand(); + command.execute(); + + System.out.println("hook: " + command.builder.executionHook); + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from execute() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + + /* test with queue() */ + command = new SuccessfulTestCommand(); + try { + command.queue().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on successful execution with "fire and forget" approach + */ + @Test + public void testExecutionHookSuccessfulCommandViaFireAndForget() { + TestHystrixCommand command = new SuccessfulTestCommand(); + try { + // do not block on "get()" ... fire this asynchronously + command.queue(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // wait for command to execute without calling get on the future + while (!command.isExecutionComplete()) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new RuntimeException("interrupted"); + } + } + + /* + * All the hooks should still work even though we didn't call get() on the future + */ + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on successful execution with multiple get() calls to Future + */ + @Test + public void testExecutionHookSuccessfulCommandWithMultipleGetsOnFuture() { + TestHystrixCommand command = new SuccessfulTestCommand(); + try { + Future f = command.queue(); + f.get(); + f.get(); + f.get(); + f.get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + /* + * Despite multiple calls to get() we should only have 1 call to the hooks. + */ + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on failed execution without a fallback + */ + @Test + public void testExecutionHookRunFailureWithoutFallback() { + /* test with execute() */ + TestHystrixCommand command = new UnknownFailureTestCommandWithoutFallback(); + try { + command.execute(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since fallback is not implemented + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since it's not implemented and throws an exception + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response from execute() since we do not have a fallback and run() failed + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception since run() failed + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // run() failure + assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + + /* test with queue() */ + command = new UnknownFailureTestCommandWithoutFallback(); + try { + command.queue().get(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since fallback is not implemented + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since it's not implemented and throws an exception + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response from queue() since we do not have a fallback and run() failed + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception since run() failed + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // run() failure + assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + + } + + /** + * Execution hook on failed execution with a fallback + */ + @Test + public void testExecutionHookRunFailureWithFallback() { + /* test with execute() */ + TestHystrixCommand command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); + command.execute(); + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response from run since run() failed + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception since run() failed + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // a response since fallback is implemented + assertNotNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it's implemented and succeeds + assertNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from execute() since we expect a fallback despite failure of run() + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception because we expect a fallback + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + + /* test with queue() */ + command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); + try { + command.queue().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response from run since run() failed + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception since run() failed + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // a response since fallback is implemented + assertNotNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it's implemented and succeeds + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since we expect a fallback despite failure of run() + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception because we expect a fallback + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on failed execution with a fallback failure + */ + @Test + public void testExecutionHookRunFailureWithFallbackFailure() { + /* test with execute() */ + TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); + try { + command.execute(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response because run() and fallback fail + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception because run() and fallback fail + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since fallback fails + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since it's implemented but fails + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because run() and fallback fail + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception because run() and fallback fail + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // run() failure + assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + + /* test with queue() */ + command = new KnownFailureTestCommandWithFallbackFailure(); + try { + command.queue().get(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response because run() and fallback fail + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception because run() and fallback fail + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since fallback fails + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since it's implemented but fails + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because run() and fallback fail + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception because run() and fallback fail + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // run() failure + assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + assertEquals(1, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on timeout without a fallback + */ + @Test + public void testExecutionHookTimeoutWithoutFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + try { + command.queue().get(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response because of timeout and no fallback + assertNull(command.builder.executionHook.runSuccessResponse); + // we should not have an exception because run() didn't fail, it timed out + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run due to timeout + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since no fallback + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since no fallback implementation + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // execution occurred + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because of timeout and no fallback + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception because of timeout and no fallback + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // timeout failure + assertEquals(FailureType.TIMEOUT, command.builder.executionHook.endExecuteFailureType); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + + // we need to wait for the thread to complete before the onThreadComplete hook will be called + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + // ignore + } + assertEquals(1, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on timeout with a fallback + */ + @Test + public void testExecutionHookTimeoutWithFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + command.queue().get(); + } catch (Exception e) { + throw new RuntimeException("not expecting", e); + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response because of timeout + assertNull(command.builder.executionHook.runSuccessResponse); + // we should not have an exception because run() didn't fail, it timed out + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run due to timeout + assertEquals(1, command.builder.executionHook.startFallback.get()); + // response since we have a fallback + assertNotNull(command.builder.executionHook.fallbackSuccessResponse); + // null since fallback succeeds + assertNull(command.builder.executionHook.fallbackFailureException); + + // execution occurred + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response because of fallback + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception because of fallback + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(1, command.builder.executionHook.threadStart.get()); + + // we need to wait for the thread to complete before the onThreadComplete hook will be called + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + // ignore + } + assertEquals(1, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on rejected with a fallback + */ + @Test + public void testExecutionHookRejectedWithFallback() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SingleThreadedPool pool = new SingleThreadedPool(1); + + try { + // fill the queue + new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS).queue(); + new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS).queue(); + } catch (Exception e) { + // ignore + } + + TestCommandRejection command = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS); + try { + // now execute one that will be rejected + command.queue().get(); + } catch (Exception e) { + throw new RuntimeException("not expecting", e); + } + + assertTrue(command.isResponseRejected()); + + // the run() method should not run as we're rejected + assertEquals(0, command.builder.executionHook.startRun.get()); + // we should not have a response because of rejection + assertNull(command.builder.executionHook.runSuccessResponse); + // we should not have an exception because we didn't run + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run due to rejection + assertEquals(1, command.builder.executionHook.startFallback.get()); + // response since we have a fallback + assertNotNull(command.builder.executionHook.fallbackSuccessResponse); + // null since fallback succeeds + assertNull(command.builder.executionHook.fallbackFailureException); + + // execution occurred + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response because of fallback + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception because of fallback + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(0, command.builder.executionHook.threadStart.get()); + assertEquals(0, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on short-circuit with a fallback + */ + @Test + public void testExecutionHookShortCircuitedWithFallbackViaQueue() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().setForceShortCircuit(true); + KnownFailureTestCommandWithoutFallback command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + // now execute one that will be short-circuited + command.queue().get(); + fail("we expect an error as there is no fallback"); + } catch (Exception e) { + // expecting + } + + assertTrue(command.isResponseShortCircuited()); + + // the run() method should not run as we're rejected + assertEquals(0, command.builder.executionHook.startRun.get()); + // we should not have a response because of rejection + assertNull(command.builder.executionHook.runSuccessResponse); + // we should not have an exception because we didn't run + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run due to rejection + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since we don't have a fallback + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since fallback fails and throws an exception + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // execution occurred + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because fallback fails + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we won't have an exception because short-circuit doesn't have one + assertNull(command.builder.executionHook.endExecuteFailureException); + // but we do expect to receive a onError call with FailureType.SHORTCIRCUIT + assertEquals(FailureType.SHORTCIRCUIT, command.builder.executionHook.endExecuteFailureType); + + // thread execution + assertEquals(0, command.builder.executionHook.threadStart.get()); + assertEquals(0, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on short-circuit with a fallback + */ + @Test + public void testExecutionHookShortCircuitedWithFallbackViaExecute() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().setForceShortCircuit(true); + KnownFailureTestCommandWithoutFallback command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + // now execute one that will be short-circuited + command.execute(); + fail("we expect an error as there is no fallback"); + } catch (Exception e) { + // expecting + } + + assertTrue(command.isResponseShortCircuited()); + + // the run() method should not run as we're rejected + assertEquals(0, command.builder.executionHook.startRun.get()); + // we should not have a response because of rejection + assertNull(command.builder.executionHook.runSuccessResponse); + // we should not have an exception because we didn't run + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run due to rejection + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since we don't have a fallback + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since fallback fails and throws an exception + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // execution occurred + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because fallback fails + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we won't have an exception because short-circuit doesn't have one + assertNull(command.builder.executionHook.endExecuteFailureException); + // but we do expect to receive a onError call with FailureType.SHORTCIRCUIT + assertEquals(FailureType.SHORTCIRCUIT, command.builder.executionHook.endExecuteFailureType); + + // thread execution + assertEquals(0, command.builder.executionHook.threadStart.get()); + assertEquals(0, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on successful execution with semaphore isolation + */ + @Test + public void testExecutionHookSuccessfulCommandWithSemaphoreIsolation() { + /* test with execute() */ + TestSemaphoreCommand command = new TestSemaphoreCommand(new TestCircuitBreaker(), 1, 10); + command.execute(); + + assertFalse(command.isExecutedInThread()); + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from execute() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(0, command.builder.executionHook.threadStart.get()); + assertEquals(0, command.builder.executionHook.threadComplete.get()); + + /* test with queue() */ + command = new TestSemaphoreCommand(new TestCircuitBreaker(), 1, 10); + try { + command.queue().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertFalse(command.isExecutedInThread()); + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(0, command.builder.executionHook.threadStart.get()); + assertEquals(0, command.builder.executionHook.threadComplete.get()); + } + + /** + * Execution hook on successful execution with semaphore isolation + */ + @Test + public void testExecutionHookFailureWithSemaphoreIsolation() { + /* test with execute() */ + final TryableSemaphore semaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(0)); + + TestSemaphoreCommand command = new TestSemaphoreCommand(new TestCircuitBreaker(), semaphore, 200); + try { + command.execute(); + fail("we expect a failure"); + } catch (Exception e) { + // expected + } + + assertFalse(command.isExecutedInThread()); + assertTrue(command.isResponseRejected()); + + // the run() method should not run as we are rejected + assertEquals(0, command.builder.executionHook.startRun.get()); + // null as run() does not get invoked + assertNull(command.builder.executionHook.runSuccessResponse); + // null as run() does not get invoked + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should run because of rejection + assertEquals(1, command.builder.executionHook.startFallback.get()); + // null since there is no fallback + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since the fallback is not implemented + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response since fallback has nothing + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we won't have an exception because rejection doesn't have one + assertNull(command.builder.executionHook.endExecuteFailureException); + // but we do expect to receive a onError call with FailureType.SHORTCIRCUIT + assertEquals(FailureType.REJECTED_SEMAPHORE_EXECUTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + assertEquals(0, command.builder.executionHook.threadStart.get()); + assertEquals(0, command.builder.executionHook.threadComplete.get()); + } + + /** + * Test a command execution that fails but has a fallback. + */ + @Test + public void testExecutionFailureWithFallbackImplementedButDisabled() { + TestHystrixCommand commandEnabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), true); + try { + assertEquals(false, commandEnabled.execute()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + TestHystrixCommand commandDisabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), false); + try { + assertEquals(false, commandDisabled.execute()); + fail("expect exception thrown"); + } catch (Exception e) { + // expected + } + + assertEquals("we failed with a simulated issue", commandDisabled.getFailedExecutionException().getMessage()); + + assertTrue(commandDisabled.isFailedExecution()); + + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, commandDisabled.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + @Test + public void testExecutionTimeoutValue() { + HystrixCommand.Setter properties = HystrixCommand.Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestKey")) + .andCommandPropertiesDefaults(HystrixCommandProperties.Setter() + .withExecutionIsolationThreadTimeoutInMilliseconds(50)); + + HystrixCommand command = new HystrixCommand(properties) { + @Override + protected String run() throws Exception { + Thread.sleep(3000); + // should never reach here + return "hello"; + } + + @Override + protected String getFallback() { + if (isResponseTimedOut()) { + return "timed-out"; + } else { + return "abc"; + } + } + }; + + String value = command.execute(); + assertTrue(command.isResponseTimedOut()); + assertEquals("expected fallback value", "timed-out", value); + + } + + /** + * See https://github.com/Netflix/Hystrix/issues/212 + */ + @Test + public void testObservableTimeoutNoFallbackThreadContext() { + TestSubscriber ts = new TestSubscriber(); + + final AtomicReference onErrorThread = new AtomicReference(); + final AtomicBoolean isRequestContextInitialized = new AtomicBoolean(); + + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + command.toObservable().doOnError(new Action1() { + + @Override + public void call(Throwable t1) { + System.out.println("onError: " + t1); + System.out.println("onError Thread: " + Thread.currentThread()); + System.out.println("ThreadContext in onError: " + HystrixRequestContext.isCurrentThreadInitialized()); + onErrorThread.set(Thread.currentThread()); + isRequestContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + } + + }).subscribe(ts); + + ts.awaitTerminalEvent(); + + assertTrue(isRequestContextInitialized.get()); + assertTrue(onErrorThread.get().getName().startsWith("RxComputationThreadPool")); + + List errors = ts.getOnErrorEvents(); + assertEquals(1, errors.size()); + Throwable e = errors.get(0); + if (errors.get(0) instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + /* ******************************************************************************** */ + /* ******************************************************************************** */ + /* private HystrixCommand class implementations for unit testing */ + /* ******************************************************************************** */ + /* ******************************************************************************** */ + + /** + * Used by UnitTest command implementations to provide base defaults for constructor and a builder pattern for the arguments being passed in. + */ + /* package */static abstract class TestHystrixCommand extends HystrixCommand { + + final TestCommandBuilder builder; + + TestHystrixCommand(TestCommandBuilder builder) { + super(builder.owner, builder.dependencyKey, builder.threadPoolKey, builder.circuitBreaker, builder.threadPool, + builder.commandPropertiesDefaults, builder.threadPoolPropertiesDefaults, builder.metrics, + builder.fallbackSemaphore, builder.executionSemaphore, TEST_PROPERTIES_FACTORY, builder.executionHook); + this.builder = builder; + } + + static TestCommandBuilder testPropsBuilder() { + return new TestCommandBuilder(); + } + + static class TestCommandBuilder { + TestCircuitBreaker _cb = new TestCircuitBreaker(); + HystrixCommandGroupKey owner = CommandGroupForUnitTest.OWNER_ONE; + HystrixCommandKey dependencyKey = null; + HystrixThreadPoolKey threadPoolKey = null; + HystrixCircuitBreaker circuitBreaker = _cb; + HystrixThreadPool threadPool = null; + HystrixCommandProperties.Setter commandPropertiesDefaults = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter(); + HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults = HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder(); + HystrixCommandMetrics metrics = _cb.metrics; + TryableSemaphore fallbackSemaphore = null; + TryableSemaphore executionSemaphore = null; + TestExecutionHook executionHook = new TestExecutionHook(); + + TestCommandBuilder setOwner(HystrixCommandGroupKey owner) { + this.owner = owner; + return this; + } + + TestCommandBuilder setCommandKey(HystrixCommandKey dependencyKey) { + this.dependencyKey = dependencyKey; + return this; + } + + TestCommandBuilder setThreadPoolKey(HystrixThreadPoolKey threadPoolKey) { + this.threadPoolKey = threadPoolKey; + return this; + } + + TestCommandBuilder setCircuitBreaker(HystrixCircuitBreaker circuitBreaker) { + this.circuitBreaker = circuitBreaker; + return this; + } + + TestCommandBuilder setThreadPool(HystrixThreadPool threadPool) { + this.threadPool = threadPool; + return this; + } + + TestCommandBuilder setCommandPropertiesDefaults(HystrixCommandProperties.Setter commandPropertiesDefaults) { + this.commandPropertiesDefaults = commandPropertiesDefaults; + return this; + } + + TestCommandBuilder setThreadPoolPropertiesDefaults(HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults) { + this.threadPoolPropertiesDefaults = threadPoolPropertiesDefaults; + return this; + } + + TestCommandBuilder setMetrics(HystrixCommandMetrics metrics) { + this.metrics = metrics; + return this; + } + + TestCommandBuilder setFallbackSemaphore(TryableSemaphore fallbackSemaphore) { + this.fallbackSemaphore = fallbackSemaphore; + return this; + } + + TestCommandBuilder setExecutionSemaphore(TryableSemaphore executionSemaphore) { + this.executionSemaphore = executionSemaphore; + return this; + } + + } + + } + + /** + * Successful execution - no fallback implementation. + */ + private static class SuccessfulTestCommand extends TestHystrixCommand { + + public SuccessfulTestCommand() { + this(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter()); + } + + public SuccessfulTestCommand(HystrixCommandProperties.Setter properties) { + super(testPropsBuilder().setCommandPropertiesDefaults(properties)); + } + + @Override + protected Boolean run() { + return true; + } + + } + + /** + * Successful execution - no fallback implementation. + */ + private static class DynamicOwnerTestCommand extends TestHystrixCommand { + + public DynamicOwnerTestCommand(HystrixCommandGroupKey owner) { + super(testPropsBuilder().setOwner(owner)); + } + + @Override + protected Boolean run() { + System.out.println("successfully executed"); + return true; + } + + } + + /** + * Successful execution - no fallback implementation. + */ + private static class DynamicOwnerAndKeyTestCommand extends TestHystrixCommand { + + public DynamicOwnerAndKeyTestCommand(HystrixCommandGroupKey owner, HystrixCommandKey key) { + super(testPropsBuilder().setOwner(owner).setCommandKey(key).setCircuitBreaker(null).setMetrics(null)); + // we specifically are NOT passing in a circuit breaker here so we test that it creates a new one correctly based on the dynamic key + } + + @Override + protected Boolean run() { + System.out.println("successfully executed"); + return true; + } + + } + + /** + * Failed execution with unknown exception (not HystrixException) - no fallback implementation. + */ + private static class UnknownFailureTestCommandWithoutFallback extends TestHystrixCommand { + + private UnknownFailureTestCommandWithoutFallback() { + super(testPropsBuilder()); + } + + @Override + protected Boolean run() { + System.out.println("*** simulated failed execution ***"); + throw new RuntimeException("we failed with an unknown issue"); + } + + } + + /** + * Failed execution with known exception (HystrixException) - no fallback implementation. + */ + private static class KnownFailureTestCommandWithoutFallback extends TestHystrixCommand { + + private KnownFailureTestCommandWithoutFallback(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + } + + @Override + protected Boolean run() { + System.out.println("*** simulated failed execution *** ==> " + Thread.currentThread()); + throw new RuntimeException("we failed with a simulated issue"); + } + + } + + /** + * Failed execution - fallback implementation successfully returns value. + */ + private static class KnownFailureTestCommandWithFallback extends TestHystrixCommand { + + public KnownFailureTestCommandWithFallback(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + } + + public KnownFailureTestCommandWithFallback(TestCircuitBreaker circuitBreaker, boolean fallbackEnabled) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withFallbackEnabled(fallbackEnabled))); + } + + @Override + protected Boolean run() { + System.out.println("*** simulated failed execution ***"); + throw new RuntimeException("we failed with a simulated issue"); + } + + @Override + protected Boolean getFallback() { + return false; + } + } + + /** + * Failed execution - fallback implementation throws exception. + */ + private static class KnownFailureTestCommandWithFallbackFailure extends TestHystrixCommand { + + private KnownFailureTestCommandWithFallbackFailure() { + super(testPropsBuilder()); + } + + @Override + protected Boolean run() { + System.out.println("*** simulated failed execution ***"); + throw new RuntimeException("we failed with a simulated issue"); + } + + @Override + protected Boolean getFallback() { + throw new RuntimeException("failed while getting fallback"); + } + } + + /** + * A Command implementation that supports caching. + */ + private static class SuccessfulCacheableCommand extends TestHystrixCommand { + + private final boolean cacheEnabled; + private volatile boolean executed = false; + private final String value; + + public SuccessfulCacheableCommand(TestCircuitBreaker circuitBreaker, boolean cacheEnabled, String value) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + this.value = value; + this.cacheEnabled = cacheEnabled; + } + + @Override + protected String run() { + executed = true; + System.out.println("successfully executed"); + return value; + } + + public boolean isCommandRunningInThread() { + return super.getProperties().executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD); + } + + @Override + public String getCacheKey() { + if (cacheEnabled) + return value; + else + return null; + } + } + + /** + * A Command implementation that supports caching. + */ + private static class SuccessfulCacheableCommandViaSemaphore extends TestHystrixCommand { + + private final boolean cacheEnabled; + private volatile boolean executed = false; + private final String value; + + public SuccessfulCacheableCommandViaSemaphore(TestCircuitBreaker circuitBreaker, boolean cacheEnabled, String value) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))); + this.value = value; + this.cacheEnabled = cacheEnabled; + } + + @Override + protected String run() { + executed = true; + System.out.println("successfully executed"); + return value; + } + + public boolean isCommandRunningInThread() { + return super.getProperties().executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD); + } + + @Override + public String getCacheKey() { + if (cacheEnabled) + return value; + else + return null; + } + } + + /** + * A Command implementation that supports caching and execution takes a while. + *

+ * Used to test scenario where Futures are returned with a backing call still executing. + */ + private static class SlowCacheableCommand extends TestHystrixCommand { + + private final String value; + private final int duration; + private volatile boolean executed = false; + + public SlowCacheableCommand(TestCircuitBreaker circuitBreaker, String value, int duration) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + this.value = value; + this.duration = duration; + } + + @Override + protected String run() { + executed = true; + try { + Thread.sleep(duration); + } catch (Exception e) { + e.printStackTrace(); + } + System.out.println("successfully executed"); + return value; + } + + @Override + public String getCacheKey() { + return value; + } + } + + /** + * Successful execution - no fallback implementation, circuit-breaker disabled. + */ + private static class TestCommandWithoutCircuitBreaker extends TestHystrixCommand { + + private TestCommandWithoutCircuitBreaker() { + super(testPropsBuilder().setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withCircuitBreakerEnabled(false))); + } + + @Override + protected Boolean run() { + System.out.println("successfully executed"); + return true; + } + + } + + /** + * This should timeout. + */ + private static class TestCommandWithTimeout extends TestHystrixCommand { + + private final long timeout; + + private final static int FALLBACK_NOT_IMPLEMENTED = 1; + private final static int FALLBACK_SUCCESS = 2; + private final static int FALLBACK_FAILURE = 3; + + private final int fallbackBehavior; + + private TestCommandWithTimeout(long timeout, int fallbackBehavior) { + super(testPropsBuilder().setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds((int) timeout))); + this.timeout = timeout; + this.fallbackBehavior = fallbackBehavior; + } + + @Override + protected Boolean run() { + System.out.println("***** running"); + try { + Thread.sleep(timeout * 10); + } catch (InterruptedException e) { + e.printStackTrace(); + // ignore and sleep some more to simulate a dependency that doesn't obey interrupts + try { + Thread.sleep(timeout * 2); + } catch (Exception e2) { + // ignore + } + System.out.println("after interruption with extra sleep"); + } + return true; + } + + @Override + protected Boolean getFallback() { + if (fallbackBehavior == FALLBACK_SUCCESS) { + return false; + } else if (fallbackBehavior == FALLBACK_FAILURE) { + throw new RuntimeException("failed on fallback"); + } else { + // FALLBACK_NOT_IMPLEMENTED + return super.getFallback(); + } + } + } + + /** + * Threadpool with 1 thread, queue of size 1 + */ + private static class SingleThreadedPool implements HystrixThreadPool { + + final LinkedBlockingQueue queue; + final ThreadPoolExecutor pool; + private final int rejectionQueueSizeThreshold; + + public SingleThreadedPool(int queueSize) { + this(queueSize, 100); + } + + public SingleThreadedPool(int queueSize, int rejectionQueueSizeThreshold) { + queue = new LinkedBlockingQueue(queueSize); + pool = new ThreadPoolExecutor(1, 1, 1, TimeUnit.MINUTES, queue); + this.rejectionQueueSizeThreshold = rejectionQueueSizeThreshold; + } + + @Override + public ThreadPoolExecutor getExecutor() { + return pool; + } + + @Override + public Scheduler getScheduler() { + return new HystrixContextScheduler(HystrixPlugins.getInstance().getConcurrencyStrategy(), this); + } + + @Override + public void markThreadExecution() { + // not used for this test + } + + @Override + public void markThreadCompletion() { + // not used for this test + } + + @Override + public boolean isQueueSpaceAvailable() { + return queue.size() < rejectionQueueSizeThreshold; + } + + } + + /** + * This has a ThreadPool that has a single thread and queueSize of 1. + */ + private static class TestCommandRejection extends TestHystrixCommand { + + private final static int FALLBACK_NOT_IMPLEMENTED = 1; + private final static int FALLBACK_SUCCESS = 2; + private final static int FALLBACK_FAILURE = 3; + + private final int fallbackBehavior; + + private final int sleepTime; + + private TestCommandRejection(TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int sleepTime, int timeout, int fallbackBehavior) { + super(testPropsBuilder().setThreadPool(threadPool).setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(timeout))); + this.fallbackBehavior = fallbackBehavior; + this.sleepTime = sleepTime; + } + + @Override + protected Boolean run() { + System.out.println(">>> TestCommandRejection running"); + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return true; + } + + @Override + protected Boolean getFallback() { + if (fallbackBehavior == FALLBACK_SUCCESS) { + return false; + } else if (fallbackBehavior == FALLBACK_FAILURE) { + throw new RuntimeException("failed on fallback"); + } else { + // FALLBACK_NOT_IMPLEMENTED + return super.getFallback(); + } + } + } + + /** + * Command that receives a custom thread-pool, sleepTime, timeout + */ + private static class CommandWithCustomThreadPool extends TestHystrixCommand { + + public boolean didExecute = false; + + private final int sleepTime; + + private CommandWithCustomThreadPool(TestCircuitBreaker circuitBreaker, HystrixThreadPool threadPool, int sleepTime, HystrixCommandProperties.Setter properties) { + super(testPropsBuilder().setThreadPool(threadPool).setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics).setCommandPropertiesDefaults(properties)); + this.sleepTime = sleepTime; + } + + @Override + protected Boolean run() { + System.out.println("**** Executing CommandWithCustomThreadPool. Execution => " + sleepTime); + didExecute = true; + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return true; + } + } + + /** + * The run() will fail and getFallback() take a long time. + */ + private static class TestSemaphoreCommandWithSlowFallback extends TestHystrixCommand { + + private final long fallbackSleep; + + private TestSemaphoreCommandWithSlowFallback(TestCircuitBreaker circuitBreaker, int fallbackSemaphoreExecutionCount, long fallbackSleep) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withFallbackIsolationSemaphoreMaxConcurrentRequests(fallbackSemaphoreExecutionCount))); + this.fallbackSleep = fallbackSleep; + } + + @Override + protected Boolean run() { + throw new RuntimeException("run fails"); + } + + @Override + protected Boolean getFallback() { + try { + Thread.sleep(fallbackSleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return true; + } + } + + private static class NoRequestCacheTimeoutWithoutFallback extends TestHystrixCommand { + public NoRequestCacheTimeoutWithoutFallback(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(200))); + + // we want it to timeout + } + + @Override + protected Boolean run() { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + System.out.println(">>>> Sleep Interrupted: " + e.getMessage()); + // e.printStackTrace(); + } + return true; + } + + @Override + public String getCacheKey() { + return null; + } + } + + /** + * The run() will take time. No fallback implementation. + */ + private static class TestSemaphoreCommand extends TestHystrixCommand { + + private final long executionSleep; + + private TestSemaphoreCommand(TestCircuitBreaker circuitBreaker, int executionSemaphoreCount, long executionSleep) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter() + .withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE) + .withExecutionIsolationSemaphoreMaxConcurrentRequests(executionSemaphoreCount))); + this.executionSleep = executionSleep; + } + + private TestSemaphoreCommand(TestCircuitBreaker circuitBreaker, TryableSemaphore semaphore, long executionSleep) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter() + .withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE)) + .setExecutionSemaphore(semaphore)); + this.executionSleep = executionSleep; + } + + @Override + protected Boolean run() { + try { + Thread.sleep(executionSleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return true; + } + } + + /** + * Semaphore based command that allows caller to use latches to know when it has started and signal when it + * would like the command to finish + */ + private static class LatchedSemaphoreCommand extends TestHystrixCommand { + + private final CountDownLatch startLatch, waitLatch; + + /** + * + * @param circuitBreaker + * @param semaphore + * @param startLatch + * this command calls {@link java.util.concurrent.CountDownLatch#countDown()} immediately + * upon running + * @param waitLatch + * this command calls {@link java.util.concurrent.CountDownLatch#await()} once it starts + * to run. The caller can use the latch to signal the command to finish + */ + private LatchedSemaphoreCommand(TestCircuitBreaker circuitBreaker, TryableSemaphore semaphore, + CountDownLatch startLatch, CountDownLatch waitLatch) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE)) + .setExecutionSemaphore(semaphore)); + this.startLatch = startLatch; + this.waitLatch = waitLatch; + } + + @Override + protected Boolean run() { + // signals caller that run has started + this.startLatch.countDown(); + + try { + // waits for caller to countDown latch + this.waitLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + return false; + } + return true; + } + } + + /** + * The run() will take time. Contains fallback. + */ + private static class TestSemaphoreCommandWithFallback extends TestHystrixCommand { + + private final long executionSleep; + private final Boolean fallback; + + private TestSemaphoreCommandWithFallback(TestCircuitBreaker circuitBreaker, int executionSemaphoreCount, long executionSleep, Boolean fallback) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE).withExecutionIsolationSemaphoreMaxConcurrentRequests(executionSemaphoreCount))); + this.executionSleep = executionSleep; + this.fallback = fallback; + } + + @Override + protected Boolean run() { + try { + Thread.sleep(executionSleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return true; + } + + @Override + protected Boolean getFallback() { + return fallback; + } + + } + + private static class RequestCacheNullPointerExceptionCase extends TestHystrixCommand { + public RequestCacheNullPointerExceptionCase(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(200))); + // we want it to timeout + } + + @Override + protected Boolean run() { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return true; + } + + @Override + protected Boolean getFallback() { + return false; + } + + @Override + public String getCacheKey() { + return "A"; + } + } + + private static class RequestCacheTimeoutWithoutFallback extends TestHystrixCommand { + public RequestCacheTimeoutWithoutFallback(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationThreadTimeoutInMilliseconds(200))); + // we want it to timeout + } + + @Override + protected Boolean run() { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + System.out.println(">>>> Sleep Interrupted: " + e.getMessage()); + // e.printStackTrace(); + } + return true; + } + + @Override + public String getCacheKey() { + return "A"; + } + } + + private static class RequestCacheThreadRejectionWithoutFallback extends TestHystrixCommand { + + final CountDownLatch completionLatch; + + public RequestCacheThreadRejectionWithoutFallback(TestCircuitBreaker circuitBreaker, CountDownLatch completionLatch) { + super(testPropsBuilder() + .setCircuitBreaker(circuitBreaker) + .setMetrics(circuitBreaker.metrics) + .setThreadPool(new HystrixThreadPool() { + + @Override + public ThreadPoolExecutor getExecutor() { + return null; + } + + @Override + public void markThreadExecution() { + + } + + @Override + public void markThreadCompletion() { + + } + + @Override + public boolean isQueueSpaceAvailable() { + // always return false so we reject everything + return false; + } + + @Override + public Scheduler getScheduler() { + return new HystrixContextScheduler(HystrixPlugins.getInstance().getConcurrencyStrategy(), this); + } + + })); + this.completionLatch = completionLatch; + } + + @Override + protected Boolean run() { + try { + if (completionLatch.await(1000, TimeUnit.MILLISECONDS)) { + throw new RuntimeException("timed out waiting on completionLatch"); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return true; + } + + @Override + public String getCacheKey() { + return "A"; + } + } + + private static class BadRequestCommand extends TestHystrixCommand { + + public BadRequestCommand(TestCircuitBreaker circuitBreaker, ExecutionIsolationStrategy isolationType) { + super(testPropsBuilder() + .setCircuitBreaker(circuitBreaker) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(isolationType))); + } + + @Override + protected Boolean run() { + throw new HystrixBadRequestException("Message to developer that they passed in bad data or something like that."); + } + + @Override + protected Boolean getFallback() { + return false; + } + + @Override + protected String getCacheKey() { + return "one"; + } + + } + + private static class CommandWithErrorThrown extends TestHystrixCommand { + + public CommandWithErrorThrown(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder() + .setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + } + + @Override + protected Boolean run() throws Exception { + throw new Error("simulated java.lang.Error message"); + } + + } + + private static class CommandWithCheckedException extends TestHystrixCommand { + + public CommandWithCheckedException(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder() + .setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + } + + @Override + protected Boolean run() throws Exception { + throw new IOException("simulated checked exception message"); + } + + } + + enum CommandKeyForUnitTest implements HystrixCommandKey { + KEY_ONE, KEY_TWO; + } + + enum CommandGroupForUnitTest implements HystrixCommandGroupKey { + OWNER_ONE, OWNER_TWO; + } + + enum ThreadPoolKeyForUnitTest implements HystrixThreadPoolKey { + THREAD_POOL_ONE, THREAD_POOL_TWO; + } + + private static HystrixPropertiesStrategy TEST_PROPERTIES_FACTORY = new TestPropertiesFactory(); + + private static class TestPropertiesFactory extends HystrixPropertiesStrategy { + + @Override + public HystrixCommandProperties getCommandProperties(HystrixCommandKey commandKey, HystrixCommandProperties.Setter builder) { + if (builder == null) { + builder = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter(); + } + return HystrixCommandPropertiesTest.asMock(builder); + } + + @Override + public HystrixThreadPoolProperties getThreadPoolProperties(HystrixThreadPoolKey threadPoolKey, HystrixThreadPoolProperties.Setter builder) { + if (builder == null) { + builder = HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder(); + } + return HystrixThreadPoolProperties.Setter.asMock(builder); + } + + @Override + public HystrixCollapserProperties getCollapserProperties(HystrixCollapserKey collapserKey, HystrixCollapserProperties.Setter builder) { + throw new IllegalStateException("not expecting collapser properties"); + } + + @Override + public String getCommandPropertiesCacheKey(HystrixCommandKey commandKey, HystrixCommandProperties.Setter builder) { + return null; + } + + @Override + public String getThreadPoolPropertiesCacheKey(HystrixThreadPoolKey threadPoolKey, com.netflix.hystrix.HystrixThreadPoolProperties.Setter builder) { + return null; + } + + @Override + public String getCollapserPropertiesCacheKey(HystrixCollapserKey collapserKey, com.netflix.hystrix.HystrixCollapserProperties.Setter builder) { + return null; + } + + } + + private static class TestExecutionHook extends HystrixCommandExecutionHook { + + AtomicInteger startExecute = new AtomicInteger(); + + @Override + public void onStart(HystrixCommand commandInstance) { + super.onStart(commandInstance); + startExecute.incrementAndGet(); + } + + Object endExecuteSuccessResponse = null; + + @Override + public T onComplete(HystrixCommand commandInstance, T response) { + endExecuteSuccessResponse = response; + return super.onComplete(commandInstance, response); + } + + Exception endExecuteFailureException = null; + FailureType endExecuteFailureType = null; + + @Override + public Exception onError(HystrixCommand commandInstance, FailureType failureType, Exception e) { + endExecuteFailureException = e; + endExecuteFailureType = failureType; + return super.onError(commandInstance, failureType, e); + } + + AtomicInteger startRun = new AtomicInteger(); + + @Override + public void onRunStart(HystrixCommand commandInstance) { + super.onRunStart(commandInstance); + startRun.incrementAndGet(); + } + + Object runSuccessResponse = null; + + @Override + public T onRunSuccess(HystrixCommand commandInstance, T response) { + runSuccessResponse = response; + return super.onRunSuccess(commandInstance, response); + } + + Exception runFailureException = null; + + @Override + public Exception onRunError(HystrixCommand commandInstance, Exception e) { + runFailureException = e; + return super.onRunError(commandInstance, e); + } + + AtomicInteger startFallback = new AtomicInteger(); + + @Override + public void onFallbackStart(HystrixCommand commandInstance) { + super.onFallbackStart(commandInstance); + startFallback.incrementAndGet(); + } + + Object fallbackSuccessResponse = null; + + @Override + public T onFallbackSuccess(HystrixCommand commandInstance, T response) { + fallbackSuccessResponse = response; + return super.onFallbackSuccess(commandInstance, response); + } + + Exception fallbackFailureException = null; + + @Override + public Exception onFallbackError(HystrixCommand commandInstance, Exception e) { + fallbackFailureException = e; + return super.onFallbackError(commandInstance, e); + } + + AtomicInteger threadStart = new AtomicInteger(); + + @Override + public void onThreadStart(HystrixCommand commandInstance) { + super.onThreadStart(commandInstance); + threadStart.incrementAndGet(); + } + + AtomicInteger threadComplete = new AtomicInteger(); + + @Override + public void onThreadComplete(HystrixCommand commandInstance) { + super.onThreadComplete(commandInstance); + threadComplete.incrementAndGet(); + } + + } +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixObservableCommandTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixObservableCommandTest.java new file mode 100644 index 000000000..93bcb79bf --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixObservableCommandTest.java @@ -0,0 +1,6810 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import rx.Notification; +import rx.Observable; +import rx.Observable.OnSubscribe; +import rx.Observer; +import rx.Scheduler; +import rx.Subscriber; +import rx.functions.Action1; +import rx.functions.Func1; +import rx.observers.TestSubscriber; +import rx.schedulers.Schedulers; + +import com.netflix.config.ConfigurationManager; +import com.netflix.hystrix.HystrixCircuitBreakerTest.TestCircuitBreaker; +import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; +import com.netflix.hystrix.HystrixExecutableBase.TryableSemaphore; +import com.netflix.hystrix.exception.HystrixBadRequestException; +import com.netflix.hystrix.exception.HystrixRuntimeException; +import com.netflix.hystrix.exception.HystrixRuntimeException.FailureType; +import com.netflix.hystrix.strategy.HystrixPlugins; +import com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable; +import com.netflix.hystrix.strategy.concurrency.HystrixContextScheduler; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; +import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; +import com.netflix.hystrix.strategy.properties.HystrixProperty; +import com.netflix.hystrix.util.HystrixRollingNumberEvent; + +public class HystrixObservableCommandTest { + + @Before + public void prepareForTest() { + /* we must call this to simulate a new request lifecycle running and clearing caches */ + HystrixRequestContext.initializeContext(); + } + + @After + public void cleanup() { + // instead of storing the reference from initialize we'll just get the current state and shutdown + if (HystrixRequestContext.getContextForCurrentThread() != null) { + // it could have been set NULL by the test + HystrixRequestContext.getContextForCurrentThread().shutdown(); + } + + // force properties to be clean as well + ConfigurationManager.getConfigInstance().clear(); + + //TODO commented out as it has issues when built from command-line even though it works from IDE + // HystrixCommandKey key = Hystrix.getCurrentThreadExecutingCommand(); + // if (key != null) { + // throw new IllegalStateException("should be null but got: " + key); + // } + } + + /** + * Test a successful command execution. + */ + @Test + public void testExecutionSuccess() { + try { + TestHystrixCommand command = new SuccessfulTestCommand(); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(true, command.execute()); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + + assertEquals(null, command.getFailedExecutionException()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + } + + /** + * Test that a command can not be executed multiple times. + */ + @Test + public void testExecutionMultipleTimes() { + SuccessfulTestCommand command = new SuccessfulTestCommand(); + assertFalse(command.isExecutionComplete()); + // first should succeed + assertEquals(true, command.execute()); + System.out.println(">> completed, checking metrics"); + assertTrue(command.isExecutionComplete()); + assertFalse(command.isExecutedInThread()); + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + try { + // second should fail + command.execute(); + fail("we should not allow this ... it breaks the state of request logs"); + } catch (IllegalStateException e) { + e.printStackTrace(); + // we want to get here + } + + try { + // queue should also fail + command.queue(); + fail("we should not allow this ... it breaks the state of request logs"); + } catch (IllegalStateException e) { + e.printStackTrace(); + // we want to get here + } + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution that throws an HystrixException and didn't implement getFallback. + */ + @Test + public void testExecutionKnownFailureWithNoFallback() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + TestHystrixCommand command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (HystrixRuntimeException e) { + e.printStackTrace(); + assertNotNull(e.getFallbackException()); + assertNotNull(e.getImplementingClass()); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + } catch (Exception e) { + e.printStackTrace(); + fail("We should always get an HystrixRuntimeException when an error occurs."); + } + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution that throws an unknown exception (not HystrixException) and didn't implement getFallback. + */ + @Test + public void testExecutionUnknownFailureWithNoFallback() { + TestHystrixCommand command = new UnknownFailureTestCommandWithoutFallback(); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (HystrixRuntimeException e) { + e.printStackTrace(); + assertNotNull(e.getFallbackException()); + assertNotNull(e.getImplementingClass()); + + } catch (Exception e) { + e.printStackTrace(); + fail("We should always get an HystrixRuntimeException when an error occurs."); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution that fails but has a fallback. + */ + @Test + public void testExecutionFailureWithFallback() { + TestHystrixCommand command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); + try { + assertEquals(false, command.execute()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals("we failed with a simulated issue", command.getFailedExecutionException().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution that fails, has getFallback implemented but that fails as well. + */ + @Test + public void testExecutionFailureWithFallbackFailure() { + TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (HystrixRuntimeException e) { + System.out.println("------------------------------------------------"); + e.printStackTrace(); + System.out.println("------------------------------------------------"); + assertNotNull(e.getFallbackException()); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a successful command execution (asynchronously). + */ + @Test + public void testQueueSuccess() { + TestHystrixCommand command = new SuccessfulTestCommand(); + try { + Future future = command.queue(); + assertEquals(true, future.get()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution (asynchronously) that throws an HystrixException and didn't implement getFallback. + */ + @Test + public void testQueueKnownFailureWithNoFallback() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + TestHystrixCommand command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + + assertNotNull(de.getFallbackException()); + assertNotNull(de.getImplementingClass()); + } else { + fail("the cause should be HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution (asynchronously) that throws an unknown exception (not HystrixException) and didn't implement getFallback. + */ + @Test + public void testQueueUnknownFailureWithNoFallback() { + TestHystrixCommand command = new UnknownFailureTestCommandWithoutFallback(); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + assertNotNull(de.getFallbackException()); + assertNotNull(de.getImplementingClass()); + } else { + fail("the cause should be HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution (asynchronously) that fails but has a fallback. + */ + @Test + public void testQueueFailureWithFallback() { + TestHystrixCommand command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); + try { + Future future = command.queue(); + assertEquals(false, future.get()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution (asynchronously) that fails, has getFallback implemented but that fails as well. + */ + @Test + public void testQueueFailureWithFallbackFailure() { + TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + e.printStackTrace(); + assertNotNull(de.getFallbackException()); + } else { + fail("the cause should be HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a successful command execution. + */ + @Test + public void testObserveSuccess() { + try { + TestHystrixCommand command = new SuccessfulTestCommand(); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + + assertEquals(true, command.observe().toBlockingObservable().single()); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + + assertEquals(null, command.getFailedExecutionException()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + } + + /** + * Test a successful command execution. + * + * @Test + * public void testObserveOnScheduler() throws Exception { + * + * System.out.println("test observeOn begins"); + * final AtomicReference commandThread = new AtomicReference(); + * final AtomicReference subscribeThread = new AtomicReference(); + * + * TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder()) { + * @Override + * protected Observable run() { + * commandThread.set(Thread.currentThread()); + * return Observable.just(true); + * } + * }; + * + * final CountDownLatch latch = new CountDownLatch(1); + * + * Scheduler customScheduler = new Scheduler() { + * + * private final Scheduler self = this; + * @Override + * public Subscription schedule(T state, Func2 action) { + * return schedule(state, action, 0, TimeUnit.MILLISECONDS); + * } + * @Override + * public Subscription schedule(final T state, final Func2 action, long delayTime, TimeUnit unit) { + * new Thread("RxScheduledThread") { + * @Override + * public void run() { + * System.out.println("in schedule"); + * action.call(self, state); + * } + * }.start(); + * + * // not testing unsubscribe behavior + * return Subscriptions.empty(); + * } + * + * }; + * + * command.toObservable(customScheduler).subscribe(new Observer() { + * @Override + * public void onCompleted() { + * latch.countDown(); + * + * } + * @Override + * public void onError(Throwable e) { + * latch.countDown(); + * e.printStackTrace(); + * + * } + * @Override + * public void onNext(Boolean args) { + * subscribeThread.set(Thread.currentThread()); + * } + * }); + * + * if (!latch.await(2000, TimeUnit.MILLISECONDS)) { + * fail("timed out"); + * } + * + * assertNotNull(commandThread.get()); + * assertNotNull(subscribeThread.get()); + * + * System.out.println("subscribeThread: " + subscribeThread.get().getName()); + * assertTrue(commandThread.get().getName().startsWith("main")); + * //assertTrue(subscribeThread.get().getName().equals("RxScheduledThread")); + * assertTrue(subscribeThread.get().getName().equals("main")); + * } + */ + /** + * Test a successful command execution. + * + * @Test + * public void testObserveOnComputationSchedulerByDefaultForThreadIsolation() throws Exception { + * + * final AtomicReference commandThread = new AtomicReference(); + * final AtomicReference subscribeThread = new AtomicReference(); + * + * TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder()) { + * @Override + * protected Observable run() { + * commandThread.set(Thread.currentThread()); + * return Observable.just(true); + * } + * }; + * + * final CountDownLatch latch = new CountDownLatch(1); + * + * command.toObservable().subscribe(new Observer() { + * @Override + * public void onCompleted() { + * latch.countDown(); + * + * } + * @Override + * public void onError(Throwable e) { + * latch.countDown(); + * e.printStackTrace(); + * + * } + * @Override + * public void onNext(Boolean args) { + * subscribeThread.set(Thread.currentThread()); + * } + * }); + * + * if (!latch.await(2000, TimeUnit.MILLISECONDS)) { + * fail("timed out"); + * } + * + * assertNotNull(commandThread.get()); + * assertNotNull(subscribeThread.get()); + * + * System.out.println("Command Thread: " + commandThread.get()); + * System.out.println("Subscribe Thread: " + subscribeThread.get()); + * + * //assertTrue(commandThread.get().getName().startsWith("hystrix-")); + * //assertTrue(subscribeThread.get().getName().startsWith("RxComputationThreadPool")); + * + * assertTrue(commandThread.get().getName().startsWith("main")); + * assertTrue(subscribeThread.get().getName().startsWith("main")); + * } + */ + + /** + * Test a successful command execution. + */ + @Test + public void testObserveOnImmediateSchedulerByDefaultForSemaphoreIsolation() throws Exception { + + final AtomicReference commandThread = new AtomicReference(); + final AtomicReference subscribeThread = new AtomicReference(); + + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { + + @Override + protected Observable run() { + commandThread.set(Thread.currentThread()); + + Func1> f1 = new Func1>() { + + @Override + public List call(Integer t1) { + + return null; + } + + }; + Observable.from(f1); + + return Observable.just(true); + } + }; + + final CountDownLatch latch = new CountDownLatch(1); + + command.toObservable().subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + + } + + @Override + public void onError(Throwable e) { + latch.countDown(); + e.printStackTrace(); + + } + + @Override + public void onNext(Boolean args) { + subscribeThread.set(Thread.currentThread()); + } + }); + + if (!latch.await(2000, TimeUnit.MILLISECONDS)) { + fail("timed out"); + } + + assertNotNull(commandThread.get()); + assertNotNull(subscribeThread.get()); + + System.out.println("Command Thread: " + commandThread.get()); + System.out.println("Subscribe Thread: " + subscribeThread.get()); + + String mainThreadName = Thread.currentThread().getName(); + + // semaphore should be on the calling thread + assertTrue(commandThread.get().getName().equals(mainThreadName)); + System.out.println("testObserveOnImmediateSchedulerByDefaultForSemaphoreIsolation: " + subscribeThread.get() + " => " + mainThreadName); + assertTrue(subscribeThread.get().getName().equals(mainThreadName)); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test that the circuit-breaker will 'trip' and prevent command execution on subsequent calls. + */ + @Test + public void testCircuitBreakerTripsAfterFailures() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + /* fail 3 times and then it should trip the circuit and stop executing */ + // failure 1 + KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt1.execute(); + assertTrue(attempt1.isResponseFromFallback()); + assertFalse(attempt1.isCircuitBreakerOpen()); + assertFalse(attempt1.isResponseShortCircuited()); + + // failure 2 + KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt2.execute(); + assertTrue(attempt2.isResponseFromFallback()); + assertFalse(attempt2.isCircuitBreakerOpen()); + assertFalse(attempt2.isResponseShortCircuited()); + + // failure 3 + KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt3.execute(); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + // it should now be 'open' and prevent further executions + assertTrue(attempt3.isCircuitBreakerOpen()); + + // attempt 4 + KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt4.execute(); + assertTrue(attempt4.isResponseFromFallback()); + // this should now be true as the response will be short-circuited + assertTrue(attempt4.isResponseShortCircuited()); + // this should remain open + assertTrue(attempt4.isCircuitBreakerOpen()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + /** + * Test that the circuit-breaker will 'trip' and prevent command execution on subsequent calls. + */ + @Test + public void testCircuitBreakerTripsAfterFailuresViaQueue() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + /* fail 3 times and then it should trip the circuit and stop executing */ + // failure 1 + KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt1.queue().get(); + assertTrue(attempt1.isResponseFromFallback()); + assertFalse(attempt1.isCircuitBreakerOpen()); + assertFalse(attempt1.isResponseShortCircuited()); + + // failure 2 + KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt2.queue().get(); + assertTrue(attempt2.isResponseFromFallback()); + assertFalse(attempt2.isCircuitBreakerOpen()); + assertFalse(attempt2.isResponseShortCircuited()); + + // failure 3 + KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt3.queue().get(); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + // it should now be 'open' and prevent further executions + assertTrue(attempt3.isCircuitBreakerOpen()); + + // attempt 4 + KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt4.queue().get(); + assertTrue(attempt4.isResponseFromFallback()); + // this should now be true as the response will be short-circuited + assertTrue(attempt4.isResponseShortCircuited()); + // this should remain open + assertTrue(attempt4.isCircuitBreakerOpen()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received fallbacks."); + } + } + + /** + * Test that the circuit-breaker is shared across HystrixCommand objects with the same CommandKey. + *

+ * This will test HystrixCommand objects with a single circuit-breaker (as if each injected with same CommandKey) + *

+ * Multiple HystrixCommand objects with the same dependency use the same circuit-breaker. + */ + @Test + public void testCircuitBreakerAcrossMultipleCommandsButSameCircuitBreaker() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + /* fail 3 times and then it should trip the circuit and stop executing */ + // failure 1 + KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt1.execute(); + assertTrue(attempt1.isResponseFromFallback()); + assertFalse(attempt1.isCircuitBreakerOpen()); + assertFalse(attempt1.isResponseShortCircuited()); + + // failure 2 with a different command, same circuit breaker + KnownFailureTestCommandWithoutFallback attempt2 = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + attempt2.execute(); + } catch (Exception e) { + // ignore ... this doesn't have a fallback so will throw an exception + } + assertTrue(attempt2.isFailedExecution()); + assertFalse(attempt2.isResponseFromFallback()); // false because no fallback + assertFalse(attempt2.isCircuitBreakerOpen()); + assertFalse(attempt2.isResponseShortCircuited()); + + // failure 3 of the Hystrix, 2nd for this particular HystrixCommand + KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt3.execute(); + assertTrue(attempt2.isFailedExecution()); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + + // it should now be 'open' and prevent further executions + // after having 3 failures on the Hystrix that these 2 different HystrixCommand objects are for + assertTrue(attempt3.isCircuitBreakerOpen()); + + // attempt 4 + KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker); + attempt4.execute(); + assertTrue(attempt4.isResponseFromFallback()); + // this should now be true as the response will be short-circuited + assertTrue(attempt4.isResponseShortCircuited()); + // this should remain open + assertTrue(attempt4.isCircuitBreakerOpen()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + /** + * Test that the circuit-breaker is different between HystrixCommand objects with a different Hystrix. + */ + @Test + public void testCircuitBreakerAcrossMultipleCommandsAndDifferentDependency() { + TestCircuitBreaker circuitBreaker_one = new TestCircuitBreaker(); + TestCircuitBreaker circuitBreaker_two = new TestCircuitBreaker(); + /* fail 3 times, twice on one Hystrix, once on a different Hystrix ... circuit-breaker should NOT open */ + + // failure 1 + KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker_one); + attempt1.execute(); + assertTrue(attempt1.isResponseFromFallback()); + assertFalse(attempt1.isCircuitBreakerOpen()); + assertFalse(attempt1.isResponseShortCircuited()); + + // failure 2 with a different HystrixCommand implementation and different Hystrix + KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker_two); + attempt2.execute(); + assertTrue(attempt2.isResponseFromFallback()); + assertFalse(attempt2.isCircuitBreakerOpen()); + assertFalse(attempt2.isResponseShortCircuited()); + + // failure 3 but only 2nd of the Hystrix.ONE + KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker_one); + attempt3.execute(); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + + // it should remain 'closed' since we have only had 2 failures on Hystrix.ONE + assertFalse(attempt3.isCircuitBreakerOpen()); + + // this one should also remain closed as it only had 1 failure for Hystrix.TWO + assertFalse(attempt2.isCircuitBreakerOpen()); + + // attempt 4 (3rd attempt for Hystrix.ONE) + KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker_one); + attempt4.execute(); + // this should NOW flip to true as this is the 3rd failure for Hystrix.ONE + assertTrue(attempt3.isCircuitBreakerOpen()); + assertTrue(attempt3.isResponseFromFallback()); + assertFalse(attempt3.isResponseShortCircuited()); + + // Hystrix.TWO should still remain closed + assertFalse(attempt2.isCircuitBreakerOpen()); + + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(3, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(3, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker_one.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker_one.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker_two.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker_two.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + /** + * Test that the circuit-breaker being disabled doesn't wreak havoc. + */ + @Test + public void testExecutionSuccessWithCircuitBreakerDisabled() { + TestHystrixCommand command = new TestCommandWithoutCircuitBreaker(); + try { + assertEquals(true, command.execute()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + + // we'll still get metrics ... just not the circuit breaker opening/closing + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + /** + * Test a command execution timeout where the command didn't implement getFallback. + */ + @Test + public void testExecutionTimeoutWithNoFallbackUsingSemaphoreIsolation() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED, ExecutionIsolationStrategy.SEMAPHORE); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (Exception e) { + // e.printStackTrace(); + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be HystrixRuntimeException"); + } + } + // the time should be 50+ since we timeout at 50ms + assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); + + assertTrue(command.isResponseTimedOut()); + assertFalse(command.isResponseFromFallback()); + assertFalse(command.isResponseRejected()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution timeout where the command implemented getFallback. + */ + @Test + public void testExecutionTimeoutWithFallbackUsingSemaphoreIsolation() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS, ExecutionIsolationStrategy.SEMAPHORE); + try { + assertEquals(false, command.execute()); + // the time should be 50+ since we timeout at 50ms + assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); + assertTrue(command.isResponseTimedOut()); + assertTrue(command.isResponseFromFallback()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution timeout where the command implemented getFallback but it fails. + */ + @Test + public void testExecutionTimeoutFallbackFailureUsingSemaphoreIsolation() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE, ExecutionIsolationStrategy.SEMAPHORE); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be HystrixRuntimeException"); + } + } + // the time should be 50+ since we timeout at 50ms + assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution timeout where the command didn't implement getFallback. + */ + @Test + public void testExecutionTimeoutWithNoFallbackUsingThreadIsolation() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED, ExecutionIsolationStrategy.THREAD); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (Exception e) { + // e.printStackTrace(); + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be HystrixRuntimeException"); + } + } + // the time should be 50+ since we timeout at 50ms + assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); + + assertTrue(command.isResponseTimedOut()); + assertFalse(command.isResponseFromFallback()); + assertFalse(command.isResponseRejected()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // thread isolated + assertTrue(command.isExecutedInThread()); + } + + /** + * Test a command execution timeout where the command implemented getFallback. + */ + @Test + public void testExecutionTimeoutWithFallbackUsingThreadIsolation() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS, ExecutionIsolationStrategy.THREAD); + try { + assertEquals(false, command.execute()); + // the time should be 50+ since we timeout at 50ms + assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); + assertTrue(command.isResponseTimedOut()); + assertTrue(command.isResponseFromFallback()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // thread isolated + assertTrue(command.isExecutedInThread()); + } + + /** + * Test a command execution timeout where the command implemented getFallback but it fails. + */ + @Test + public void testExecutionTimeoutFallbackFailureUsingThreadIsolation() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE, ExecutionIsolationStrategy.THREAD); + try { + command.execute(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be HystrixRuntimeException"); + } + } + // the time should be 50+ since we timeout at 50ms + assertTrue("Execution Time is: " + command.getExecutionTimeInMilliseconds(), command.getExecutionTimeInMilliseconds() >= 50); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // thread isolated + assertTrue(command.isExecutedInThread()); + } + + /** + * Test that the circuit-breaker counts a command execution timeout as a 'timeout' and not just failure. + */ + @Test + public void testCircuitBreakerOnExecutionTimeout() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + + command.execute(); + + assertTrue(command.isResponseFromFallback()); + assertFalse(command.isCircuitBreakerOpen()); + assertFalse(command.isResponseShortCircuited()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test that the command finishing AFTER a timeout (because thread continues in background) does not register a SUCCESS + */ + @Test + public void testCountersOnExecutionTimeout() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + command.execute(); + + /* wait long enough for the command to have finished */ + Thread.sleep(200); + + /* response should still be the same as 'testCircuitBreakerOnExecutionTimeout' */ + assertTrue(command.isResponseFromFallback()); + assertFalse(command.isCircuitBreakerOpen()); + assertFalse(command.isResponseShortCircuited()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + assertFalse(command.isSuccessfulExecution()); + + /* failure and timeout count should be the same as 'testCircuitBreakerOnExecutionTimeout' */ + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + + /* we should NOT have a 'success' counter */ + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a queued command execution timeout where the command didn't implement getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testQueuedExecutionTimeoutWithNoFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + if (e instanceof ExecutionException && e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testQueuedExecutionTimeoutWithFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + assertEquals(false, command.queue().get()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback but it fails. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testQueuedExecutionTimeoutFallbackFailure() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE); + try { + command.queue().get(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e instanceof ExecutionException && e.getCause() instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e.getCause(); + assertNotNull(de.getFallbackException()); + assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a queued command execution timeout where the command didn't implement getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testObservedExecutionTimeoutWithNoFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + try { + command.observe().toBlockingObservable().single(); + fail("we shouldn't get here"); + } catch (Exception e) { + e.printStackTrace(); + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testObservedExecutionTimeoutWithFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + assertEquals(false, command.observe().toBlockingObservable().single()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a queued command execution timeout where the command implemented getFallback but it fails. + *

+ * We specifically want to protect against developers queuing commands and using queue().get() without a timeout (such as queue().get(3000, TimeUnit.Milliseconds)) and ending up blocking + * indefinitely by skipping the timeout protection of the execute() command. + */ + @Test + public void testObservedExecutionTimeoutFallbackFailure() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_FAILURE); + try { + command.observe().toBlockingObservable().single(); + fail("we shouldn't get here"); + } catch (Exception e) { + if (e instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertFalse(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + } + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test that the circuit-breaker counts a command execution timeout as a 'timeout' and not just failure. + */ + @Test + public void testShortCircuitFallbackCounter() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().setForceShortCircuit(true); + try { + new KnownFailureTestCommandWithFallback(circuitBreaker).execute(); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + + KnownFailureTestCommandWithFallback command = new KnownFailureTestCommandWithFallback(circuitBreaker); + command.execute(); + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + + // will be -1 because it never attempted execution + assertEquals(-1, command.getExecutionTimeInMilliseconds()); + assertTrue(command.isResponseShortCircuited()); + assertFalse(command.isResponseTimedOut()); + + // because it was short-circuited to a fallback we don't count an error + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + @Test + public void testExecutionSemaphoreWithQueue() { + final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + // single thread should work + try { + boolean result = new TestSemaphoreCommand(circuitBreaker, 1, 200).queue().get(); + assertTrue(result); + } catch (Exception e) { + // we shouldn't fail on this one + throw new RuntimeException(e); + } + + final AtomicBoolean exceptionReceived = new AtomicBoolean(); + + final TryableSemaphore semaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(1)); + + Runnable r = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + + @Override + public void run() { + try { + new TestSemaphoreCommand(circuitBreaker, semaphore, 200).queue().get(); + } catch (Exception e) { + e.printStackTrace(); + exceptionReceived.set(true); + } + } + + }); + // 2 threads, the second should be rejected by the semaphore + Thread t1 = new Thread(r); + Thread t2 = new Thread(r); + + t1.start(); + t2.start(); + try { + t1.join(); + t2.join(); + } catch (Exception e) { + e.printStackTrace(); + fail("failed waiting on threads"); + } + + if (!exceptionReceived.get()) { + fail("We expected an exception on the 2nd get"); + } + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + // we don't have a fallback so threw an exception when rejected + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + // not a failure as the command never executed so can't fail + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + // no fallback failure as there isn't a fallback implemented + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + // we should have rejected via semaphore + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + // the rest should not be involved in this test + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + @Test + public void testExecutionSemaphoreWithExecution() { + final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + // single thread should work + try { + TestSemaphoreCommand command = new TestSemaphoreCommand(circuitBreaker, 1, 200); + boolean result = command.execute(); + assertFalse(command.isExecutedInThread()); + assertTrue(result); + } catch (Exception e) { + // we shouldn't fail on this one + throw new RuntimeException(e); + } + + final ArrayBlockingQueue results = new ArrayBlockingQueue(2); + + final AtomicBoolean exceptionReceived = new AtomicBoolean(); + + final TryableSemaphore semaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(1)); + + Runnable r = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + + @Override + public void run() { + try { + results.add(new TestSemaphoreCommand(circuitBreaker, semaphore, 200).execute()); + } catch (Exception e) { + e.printStackTrace(); + exceptionReceived.set(true); + } + } + + }); + // 2 threads, the second should be rejected by the semaphore + Thread t1 = new Thread(r); + Thread t2 = new Thread(r); + + t1.start(); + t2.start(); + try { + t1.join(); + t2.join(); + } catch (Exception e) { + e.printStackTrace(); + fail("failed waiting on threads"); + } + + if (!exceptionReceived.get()) { + fail("We expected an exception on the 2nd get"); + } + + // only 1 value is expected as the other should have thrown an exception + assertEquals(1, results.size()); + // should contain only a true result + assertTrue(results.contains(Boolean.TRUE)); + assertFalse(results.contains(Boolean.FALSE)); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + // no failure ... we throw an exception because of rejection but the command does not fail execution + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + // there is no fallback implemented so no failure can occur on it + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + // we rejected via semaphore + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + // the rest should not be involved in this test + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + @Test + public void testRejectedExecutionSemaphoreWithFallback() { + final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + final ArrayBlockingQueue results = new ArrayBlockingQueue(2); + + final AtomicBoolean exceptionReceived = new AtomicBoolean(); + + Runnable r = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + + @Override + public void run() { + try { + results.add(new TestSemaphoreCommandWithFallback(circuitBreaker, 1, 200, false).execute()); + } catch (Exception e) { + e.printStackTrace(); + exceptionReceived.set(true); + } + } + + }); + + // 2 threads, the second should be rejected by the semaphore and return fallback + Thread t1 = new Thread(r); + Thread t2 = new Thread(r); + + t1.start(); + t2.start(); + try { + t1.join(); + t2.join(); + } catch (Exception e) { + e.printStackTrace(); + fail("failed waiting on threads"); + } + + if (exceptionReceived.get()) { + fail("We should have received a fallback response"); + } + + // both threads should have returned values + assertEquals(2, results.size()); + // should contain both a true and false result + assertTrue(results.contains(Boolean.TRUE)); + assertTrue(results.contains(Boolean.FALSE)); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + // the rest should not be involved in this test + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + System.out.println("**** DONE"); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + /** + * Tests that semaphores are counted separately for commands with unique keys + */ + @Test + public void testSemaphorePermitsInUse() { + final TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + + // this semaphore will be shared across multiple command instances + final TryableSemaphore sharedSemaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(3)); + + // used to wait until all commands have started + final CountDownLatch startLatch = new CountDownLatch(sharedSemaphore.numberOfPermits.get() + 1); + + // used to signal that all command can finish + final CountDownLatch sharedLatch = new CountDownLatch(1); + + final Runnable sharedSemaphoreRunnable = new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + public void run() { + try { + new LatchedSemaphoreCommand(circuitBreaker, sharedSemaphore, startLatch, sharedLatch).execute(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + + // creates group of threads each using command sharing a single semaphore + + // I create extra threads and commands so that I can verify that some of them fail to obtain a semaphore + final int sharedThreadCount = sharedSemaphore.numberOfPermits.get() * 2; + final Thread[] sharedSemaphoreThreads = new Thread[sharedThreadCount]; + for (int i = 0; i < sharedThreadCount; i++) { + sharedSemaphoreThreads[i] = new Thread(sharedSemaphoreRunnable); + } + + // creates thread using isolated semaphore + final TryableSemaphore isolatedSemaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(1)); + + final CountDownLatch isolatedLatch = new CountDownLatch(1); + + // tracks failures to obtain semaphores + final AtomicInteger failureCount = new AtomicInteger(); + + final Thread isolatedThread = new Thread(new HystrixContextRunnable(HystrixPlugins.getInstance().getConcurrencyStrategy(), new Runnable() { + public void run() { + try { + new LatchedSemaphoreCommand(circuitBreaker, isolatedSemaphore, startLatch, isolatedLatch).execute(); + } catch (Exception e) { + e.printStackTrace(); + failureCount.incrementAndGet(); + } + } + })); + + // verifies no permits in use before starting threads + assertEquals("wrong number of permits for shared semaphore", 0, sharedSemaphore.getNumberOfPermitsUsed()); + assertEquals("wrong number of permits for isolated semaphore", 0, isolatedSemaphore.getNumberOfPermitsUsed()); + + for (int i = 0; i < sharedThreadCount; i++) { + sharedSemaphoreThreads[i].start(); + } + isolatedThread.start(); + + // waits until all commands have started + try { + startLatch.await(1000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + // verifies that all semaphores are in use + assertEquals("wrong number of permits for shared semaphore", + sharedSemaphore.numberOfPermits.get().longValue(), sharedSemaphore.getNumberOfPermitsUsed()); + assertEquals("wrong number of permits for isolated semaphore", + isolatedSemaphore.numberOfPermits.get().longValue(), isolatedSemaphore.getNumberOfPermitsUsed()); + + // signals commands to finish + sharedLatch.countDown(); + isolatedLatch.countDown(); + + try { + for (int i = 0; i < sharedThreadCount; i++) { + sharedSemaphoreThreads[i].join(); + } + isolatedThread.join(); + } catch (Exception e) { + e.printStackTrace(); + fail("failed waiting on threads"); + } + + // verifies no permits in use after finishing threads + assertEquals("wrong number of permits for shared semaphore", 0, sharedSemaphore.getNumberOfPermitsUsed()); + assertEquals("wrong number of permits for isolated semaphore", 0, isolatedSemaphore.getNumberOfPermitsUsed()); + + // verifies that some executions failed + final int expectedFailures = sharedSemaphore.getNumberOfPermitsUsed(); + assertEquals("failures expected but did not happen", expectedFailures, failureCount.get()); + } + + /** + * Test that HystrixOwner can be passed in dynamically. + */ + @Test + public void testDynamicOwner() { + try { + TestHystrixCommand command = new DynamicOwnerTestCommand(CommandGroupForUnitTest.OWNER_ONE); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(true, command.execute()); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + } + + /** + * Test a successful command execution. + */ + @Test + public void testDynamicOwnerFails() { + try { + TestHystrixCommand command = new DynamicOwnerTestCommand(null); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(true, command.execute()); + fail("we should have thrown an exception as we need an owner"); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } catch (Exception e) { + // success if we get here + } + } + + /** + * Test that HystrixCommandKey can be passed in dynamically. + */ + @Test + public void testDynamicKey() { + try { + DynamicOwnerAndKeyTestCommand command1 = new DynamicOwnerAndKeyTestCommand(CommandGroupForUnitTest.OWNER_ONE, CommandKeyForUnitTest.KEY_ONE); + assertEquals(true, command1.execute()); + DynamicOwnerAndKeyTestCommand command2 = new DynamicOwnerAndKeyTestCommand(CommandGroupForUnitTest.OWNER_ONE, CommandKeyForUnitTest.KEY_TWO); + assertEquals(true, command2.execute()); + + // 2 different circuit breakers should be created + assertNotSame(command1.getCircuitBreaker(), command2.getCircuitBreaker()); + + // semaphore isolated + assertFalse(command1.isExecutedInThread()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception."); + } + } + + /** + * Test Request scoped caching of commands so that a 2nd duplicate call doesn't execute but returns the previous Future + */ + @Test + public void testRequestCache1UsingThreadIsolation() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + + assertTrue(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("A", f2.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // the second one should not have executed as it should have received the cached value instead + assertFalse(command2.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command1.getExecutionTimeInMilliseconds() > -1); + assertFalse(command1.isResponseFromCache()); + + // the execution log for command2 should show it came from cache + assertEquals(2, command2.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + assertTrue(command2.getExecutionTimeInMilliseconds() == -1); + assertTrue(command2.isResponseFromCache()); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + /** + * Test Request scoped caching doesn't prevent different ones from executing + */ + @Test + public void testRequestCache2UsingThreadIsolation() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "B"); + + assertTrue(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command2.getExecutionTimeInMilliseconds() > -1); + assertFalse(command2.isResponseFromCache()); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(2, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testRequestCache3UsingThreadIsolation() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "B"); + SuccessfulCacheableCommand command3 = new SuccessfulCacheableCommand(circuitBreaker, true, "A"); + + assertTrue(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("A", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // but the 3rd should come from cache + assertFalse(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show it came from cache + assertEquals(2, command3.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + assertTrue(command3.getExecutionTimeInMilliseconds() == -1); + assertTrue(command3.isResponseFromCache()); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + System.out.println("executedCommand: " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); + assertEquals(3, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + /** + * Test Request scoped caching of commands so that a 2nd duplicate call doesn't execute but returns the previous Future + */ + @Test + public void testRequestCacheWithSlowExecution() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SlowCacheableCommand command1 = new SlowCacheableCommand(circuitBreaker, "A", 200); + SlowCacheableCommand command2 = new SlowCacheableCommand(circuitBreaker, "A", 100); + SlowCacheableCommand command3 = new SlowCacheableCommand(circuitBreaker, "A", 100); + SlowCacheableCommand command4 = new SlowCacheableCommand(circuitBreaker, "A", 100); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + Future f4 = command4.queue(); + + try { + assertEquals("A", f2.get()); + assertEquals("A", f3.get()); + assertEquals("A", f4.get()); + + assertEquals("A", f1.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // the second one should not have executed as it should have received the cached value instead + assertFalse(command2.executed); + assertFalse(command3.executed); + assertFalse(command4.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command1.getExecutionTimeInMilliseconds() > -1); + assertFalse(command1.isResponseFromCache()); + + // the execution log for command2 should show it came from cache + assertEquals(2, command2.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + assertTrue(command2.getExecutionTimeInMilliseconds() == -1); + assertTrue(command2.isResponseFromCache()); + + assertTrue(command3.isResponseFromCache()); + assertTrue(command3.getExecutionTimeInMilliseconds() == -1); + assertTrue(command4.isResponseFromCache()); + assertTrue(command4.getExecutionTimeInMilliseconds() == -1); + + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + System.out.println("HystrixRequestLog: " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString()); + + // semaphore isolated + assertFalse(command1.isExecutedInThread()); + assertFalse(command2.isExecutedInThread()); + assertFalse(command3.isExecutedInThread()); + assertFalse(command4.isExecutedInThread()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testNoRequestCache3UsingThreadIsolation() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommand command1 = new SuccessfulCacheableCommand(circuitBreaker, false, "A"); + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, false, "B"); + SuccessfulCacheableCommand command3 = new SuccessfulCacheableCommand(circuitBreaker, false, "A"); + + assertTrue(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("A", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // this should also execute since we disabled the cache + assertTrue(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show a SUCCESS + assertEquals(1, command3.getExecutionEvents().size()); + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // thread isolated + assertTrue(command1.isExecutedInThread()); + assertTrue(command2.isExecutedInThread()); + assertTrue(command3.isExecutedInThread()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testRequestCacheViaQueueUsingSemaphoreIsolation() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); + SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "B"); + SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); + + assertFalse(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("A", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // but the 3rd should come from cache + assertFalse(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show it comes from cache + assertEquals(2, command3.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + + assertTrue(command3.isResponseFromCache()); + assertTrue(command3.getExecutionTimeInMilliseconds() == -1); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command1.isExecutedInThread()); + assertFalse(command2.isExecutedInThread()); + assertFalse(command3.isExecutedInThread()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testNoRequestCacheViaQueueUsingSemaphoreIsolation() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); + SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "B"); + SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); + + assertFalse(command1.isCommandRunningInThread()); + + Future f1 = command1.queue(); + Future f2 = command2.queue(); + Future f3 = command3.queue(); + + try { + assertEquals("A", f1.get()); + assertEquals("B", f2.get()); + assertEquals("A", f3.get()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // this should also execute because caching is disabled + assertTrue(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show a SUCCESS + assertEquals(1, command3.getExecutionEvents().size()); + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command1.isExecutedInThread()); + assertFalse(command2.isExecutedInThread()); + assertFalse(command3.isExecutedInThread()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testRequestCacheViaExecuteUsingSemaphoreIsolation() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); + SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "B"); + SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, true, "A"); + + assertFalse(command1.isCommandRunningInThread()); + + String f1 = command1.execute(); + String f2 = command2.execute(); + String f3 = command3.execute(); + + assertEquals("A", f1); + assertEquals("B", f2); + assertEquals("A", f3); + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // but the 3rd should come from cache + assertFalse(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show it comes from cache + assertEquals(2, command3.getExecutionEvents().size()); // it will include the SUCCESS + RESPONSE_FROM_CACHE + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + + assertEquals(2, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command1.isExecutedInThread()); + assertFalse(command2.isExecutedInThread()); + assertFalse(command3.isExecutedInThread()); + } + + /** + * Test Request scoped caching with a mixture of commands + */ + @Test + public void testNoRequestCacheViaExecuteUsingSemaphoreIsolation() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + SuccessfulCacheableCommandViaSemaphore command1 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); + SuccessfulCacheableCommandViaSemaphore command2 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "B"); + SuccessfulCacheableCommandViaSemaphore command3 = new SuccessfulCacheableCommandViaSemaphore(circuitBreaker, false, "A"); + + assertFalse(command1.isCommandRunningInThread()); + + String f1 = command1.execute(); + String f2 = command2.execute(); + String f3 = command3.execute(); + + assertEquals("A", f1); + assertEquals("B", f2); + assertEquals("A", f3); + + assertTrue(command1.executed); + // both should execute as they are different + assertTrue(command2.executed); + // this should also execute because caching is disabled + assertTrue(command3.executed); + + // the execution log for command1 should show a SUCCESS + assertEquals(1, command1.getExecutionEvents().size()); + assertTrue(command1.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command2 should show a SUCCESS + assertEquals(1, command2.getExecutionEvents().size()); + assertTrue(command2.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + // the execution log for command3 should show a SUCCESS + assertEquals(1, command3.getExecutionEvents().size()); + assertTrue(command3.getExecutionEvents().contains(HystrixEventType.SUCCESS)); + + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(3, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command1.isExecutedInThread()); + assertFalse(command2.isExecutedInThread()); + assertFalse(command3.isExecutedInThread()); + } + + @Test + public void testNoRequestCacheOnTimeoutThrowsException() throws Exception { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + NoRequestCacheTimeoutWithoutFallback r1 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + System.out.println("r1 value: " + r1.execute()); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r1.isResponseTimedOut()); + // what we want + } + + NoRequestCacheTimeoutWithoutFallback r2 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r2.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r2.isResponseTimedOut()); + // what we want + } + + NoRequestCacheTimeoutWithoutFallback r3 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + Future f3 = r3.queue(); + try { + f3.get(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (ExecutionException e) { + e.printStackTrace(); + assertTrue(r3.isResponseTimedOut()); + // what we want + } + + Thread.sleep(500); // timeout on command is set to 200ms + + NoRequestCacheTimeoutWithoutFallback r4 = new NoRequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r4.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r4.isResponseTimedOut()); + assertFalse(r4.isResponseFromFallback()); + // what we want + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + @Test + public void testRequestCacheOnTimeoutCausesNullPointerException() throws Exception { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + // Expect it to time out - all results should be false + assertFalse(new RequestCacheNullPointerExceptionCase(circuitBreaker).execute()); + assertFalse(new RequestCacheNullPointerExceptionCase(circuitBreaker).execute()); // return from cache #1 + assertFalse(new RequestCacheNullPointerExceptionCase(circuitBreaker).execute()); // return from cache #2 + Thread.sleep(500); // timeout on command is set to 200ms + Boolean value = new RequestCacheNullPointerExceptionCase(circuitBreaker).execute(); // return from cache #3 + assertFalse(value); + RequestCacheNullPointerExceptionCase c = new RequestCacheNullPointerExceptionCase(circuitBreaker); + Future f = c.queue(); // return from cache #4 + // the bug is that we're getting a null Future back, rather than a Future that returns false + assertNotNull(f); + assertFalse(f.get()); + + assertTrue(c.isResponseFromFallback()); + assertTrue(c.isResponseTimedOut()); + assertFalse(c.isFailedExecution()); + assertFalse(c.isResponseShortCircuited()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(4, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(5, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + HystrixObservableCommand[] executeCommands = HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().toArray(new HystrixObservableCommand[] {}); + + System.out.println(":executeCommands[0].getExecutionEvents()" + executeCommands[0].getExecutionEvents()); + assertEquals(2, executeCommands[0].getExecutionEvents().size()); + assertTrue(executeCommands[0].getExecutionEvents().contains(HystrixEventType.FALLBACK_SUCCESS)); + assertTrue(executeCommands[0].getExecutionEvents().contains(HystrixEventType.TIMEOUT)); + assertTrue(executeCommands[0].getExecutionTimeInMilliseconds() > -1); + assertTrue(executeCommands[0].isResponseTimedOut()); + assertTrue(executeCommands[0].isResponseFromFallback()); + assertFalse(executeCommands[0].isResponseFromCache()); + + assertEquals(3, executeCommands[1].getExecutionEvents().size()); // it will include FALLBACK_SUCCESS/TIMEOUT + RESPONSE_FROM_CACHE + assertTrue(executeCommands[1].getExecutionEvents().contains(HystrixEventType.RESPONSE_FROM_CACHE)); + assertTrue(executeCommands[1].getExecutionTimeInMilliseconds() == -1); + assertTrue(executeCommands[1].isResponseFromCache()); + assertTrue(executeCommands[1].isResponseTimedOut()); + assertTrue(executeCommands[1].isResponseFromFallback()); + } + + // + @Test + public void testRequestCacheOnTimeoutThrowsException() throws Exception { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + RequestCacheTimeoutWithoutFallback r1 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + System.out.println("r1 value: " + r1.execute()); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r1.isResponseTimedOut()); + // what we want + } + + RequestCacheTimeoutWithoutFallback r2 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r2.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r2.isResponseTimedOut()); + // what we want + } + + RequestCacheTimeoutWithoutFallback r3 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); + Future f3 = r3.queue(); + try { + f3.get(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (ExecutionException e) { + e.printStackTrace(); + assertTrue(r3.isResponseTimedOut()); + // what we want + } + + Thread.sleep(500); // timeout on command is set to 200ms + + RequestCacheTimeoutWithoutFallback r4 = new RequestCacheTimeoutWithoutFallback(circuitBreaker); + try { + r4.execute(); + // we should have thrown an exception + fail("expected a timeout"); + } catch (HystrixRuntimeException e) { + assertTrue(r4.isResponseTimedOut()); + assertFalse(r4.isResponseFromFallback()); + // what we want + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(4, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + } + + @Test + public void testRequestCacheOnThreadRejectionThrowsException() throws Exception { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CountDownLatch completionLatch = new CountDownLatch(1); + RequestCacheThreadRejectionWithoutFallback r1 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); + try { + System.out.println("r1: " + r1.execute()); + // we should have thrown an exception + fail("expected a rejection"); + } catch (HystrixRuntimeException e) { + e.printStackTrace(); + assertTrue(r1.isResponseRejected()); + // what we want + } + + RequestCacheThreadRejectionWithoutFallback r2 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); + try { + System.out.println("r2: " + r2.execute()); + // we should have thrown an exception + fail("expected a rejection"); + } catch (HystrixRuntimeException e) { + // e.printStackTrace(); + assertTrue(r2.isResponseRejected()); + // what we want + } + + RequestCacheThreadRejectionWithoutFallback r3 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); + try { + System.out.println("f3: " + r3.queue().get()); + // we should have thrown an exception + fail("expected a rejection"); + } catch (HystrixRuntimeException e) { + // e.printStackTrace(); + assertTrue(r3.isResponseRejected()); + // what we want + } + + // let the command finish (only 1 should actually be blocked on this due to the response cache) + completionLatch.countDown(); + + // then another after the command has completed + RequestCacheThreadRejectionWithoutFallback r4 = new RequestCacheThreadRejectionWithoutFallback(circuitBreaker, completionLatch); + try { + System.out.println("r4: " + r4.execute()); + // we should have thrown an exception + fail("expected a rejection"); + } catch (HystrixRuntimeException e) { + // e.printStackTrace(); + assertTrue(r4.isResponseRejected()); + assertFalse(r4.isResponseFromFallback()); + // what we want + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(3, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, circuitBreaker.metrics.getHealthCounts().getErrorPercentage()); + + // assertEquals(4, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that we can do basic execution without a RequestVariable being initialized. + */ + @Test + public void testBasicExecutionWorksWithoutRequestVariable() { + try { + /* force the RequestVariable to not be initialized */ + HystrixRequestContext.setContextOnCurrentThread(null); + + TestHystrixCommand command = new SuccessfulTestCommand(); + assertEquals(true, command.execute()); + + TestHystrixCommand command2 = new SuccessfulTestCommand(); + assertEquals(true, command2.queue().get()); + + // we should be able to execute without a RequestVariable if ... + // 1) We don't have a cacheKey + // 2) We don't ask for the RequestLog + // 3) We don't do collapsing + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } catch (Exception e) { + e.printStackTrace(); + fail("We received an exception => " + e.getMessage()); + } + } + + /** + * Test that if we try and execute a command with a cacheKey without initializing RequestVariable that it gives an error. + */ + @Test + public void testCacheKeyExecutionRequiresRequestVariable() { + try { + /* force the RequestVariable to not be initialized */ + HystrixRequestContext.setContextOnCurrentThread(null); + + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + + SuccessfulCacheableCommand command = new SuccessfulCacheableCommand(circuitBreaker, true, "one"); + assertEquals(true, command.execute()); + + SuccessfulCacheableCommand command2 = new SuccessfulCacheableCommand(circuitBreaker, true, "two"); + assertEquals(true, command2.queue().get()); + + fail("We expect an exception because cacheKey requires RequestVariable."); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. + */ + @Test + public void testBadRequestExceptionViaExecuteInThread() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).execute(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (HystrixBadRequestException e) { + // success + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. + */ + @Test + public void testBadRequestExceptionViaQueueInThread() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).queue().get(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (ExecutionException e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixBadRequestException) { + // success + } else { + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test that BadRequestException behavior works the same on a cached response. + */ + @Test + public void testBadRequestExceptionViaQueueInThreadOnResponseFromCache() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + + // execute once to cache the value + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).execute(); + } catch (Throwable e) { + // ignore + } + + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.THREAD).queue().get(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (ExecutionException e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixBadRequestException) { + // success + } else { + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. + */ + @Test + public void testBadRequestExceptionViaExecuteInSemaphore() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.SEMAPHORE).execute(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (HystrixBadRequestException e) { + // success + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test that a BadRequestException can be thrown and not count towards errors and bypasses fallback. + */ + @Test + public void testBadRequestExceptionViaQueueInSemaphore() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + try { + new BadRequestCommand(circuitBreaker, ExecutionIsolationStrategy.SEMAPHORE).queue().get(); + fail("we expect to receive a " + HystrixBadRequestException.class.getSimpleName()); + } catch (ExecutionException e) { + e.printStackTrace(); + if (e.getCause() instanceof HystrixBadRequestException) { + // success + } else { + fail("We expect a " + HystrixBadRequestException.class.getSimpleName() + " but got a " + e.getClass().getSimpleName()); + } + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + } + + /** + * Test a checked Exception being thrown + */ + @Test + public void testCheckedExceptionViaExecute() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithCheckedException command = new CommandWithCheckedException(circuitBreaker); + try { + command.execute(); + fail("we expect to receive a " + Exception.class.getSimpleName()); + } catch (Exception e) { + assertEquals("simulated checked exception message", e.getCause().getMessage()); + } + + assertEquals("simulated checked exception message", command.getFailedExecutionException().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + } + + /** + * Test a java.lang.Error being thrown + * + * @throws InterruptedException + */ + @Test + public void testCheckedExceptionViaObserve() throws InterruptedException { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithCheckedException command = new CommandWithCheckedException(circuitBreaker); + final AtomicReference t = new AtomicReference(); + final CountDownLatch latch = new CountDownLatch(1); + try { + command.observe().subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + } + + @Override + public void onError(Throwable e) { + t.set(e); + latch.countDown(); + } + + @Override + public void onNext(Boolean args) { + + } + + }); + } catch (Exception e) { + e.printStackTrace(); + fail("we should not get anything thrown, it should be emitted via the Observer#onError method"); + } + + latch.await(1, TimeUnit.SECONDS); + assertNotNull(t.get()); + t.get().printStackTrace(); + + assertTrue(t.get() instanceof HystrixRuntimeException); + assertEquals("simulated checked exception message", t.get().getCause().getMessage()); + assertEquals("simulated checked exception message", command.getFailedExecutionException().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a java.lang.Error being thrown + */ + @Test + public void testErrorThrownViaExecute() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithErrorThrown command = new CommandWithErrorThrown(circuitBreaker); + try { + command.execute(); + fail("we expect to receive a " + Error.class.getSimpleName()); + } catch (Exception e) { + // the actual error is an extra cause level deep because Hystrix needs to wrap Throwable/Error as it's public + // methods only support Exception and it's not a strong enough reason to break backwards compatibility and jump to version 2.x + // so HystrixRuntimeException -> wrapper Exception -> actual Error + assertEquals("simulated java.lang.Error message", e.getCause().getCause().getMessage()); + } + + assertEquals("simulated java.lang.Error message", command.getFailedExecutionException().getCause().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a java.lang.Error being thrown + */ + @Test + public void testErrorThrownViaQueue() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithErrorThrown command = new CommandWithErrorThrown(circuitBreaker); + try { + command.queue().get(); + fail("we expect to receive an Exception"); + } catch (Exception e) { + // one cause down from ExecutionException to HystrixRuntime + // then the actual error is an extra cause level deep because Hystrix needs to wrap Throwable/Error as it's public + // methods only support Exception and it's not a strong enough reason to break backwards compatibility and jump to version 2.x + // so ExecutionException -> HystrixRuntimeException -> wrapper Exception -> actual Error + assertEquals("simulated java.lang.Error message", e.getCause().getCause().getCause().getMessage()); + } + + assertEquals("simulated java.lang.Error message", command.getFailedExecutionException().getCause().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a java.lang.Error being thrown + * + * @throws InterruptedException + */ + @Test + public void testErrorThrownViaObserve() throws InterruptedException { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + CommandWithErrorThrown command = new CommandWithErrorThrown(circuitBreaker); + final AtomicReference t = new AtomicReference(); + final CountDownLatch latch = new CountDownLatch(1); + try { + command.observe().subscribe(new Observer() { + + @Override + public void onCompleted() { + latch.countDown(); + } + + @Override + public void onError(Throwable e) { + t.set(e); + latch.countDown(); + } + + @Override + public void onNext(Boolean args) { + + } + + }); + } catch (Exception e) { + e.printStackTrace(); + fail("we should not get anything thrown, it should be emitted via the Observer#onError method"); + } + + latch.await(1, TimeUnit.SECONDS); + assertNotNull(t.get()); + t.get().printStackTrace(); + + assertTrue(t.get() instanceof HystrixRuntimeException); + // the actual error is an extra cause level deep because Hystrix needs to wrap Throwable/Error as it's public + // methods only support Exception and it's not a strong enough reason to break backwards compatibility and jump to version 2.x + assertEquals("simulated java.lang.Error message", t.get().getCause().getCause().getMessage()); + assertEquals("simulated java.lang.Error message", command.getFailedExecutionException().getCause().getMessage()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isFailedExecution()); + + assertEquals(0, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(1, circuitBreaker.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on successful execution + */ + @Test + public void testExecutionHookSuccessfulCommand() { + //test with execute() + TestHystrixCommand command = new SuccessfulTestCommand(); + command.execute(); + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from execute() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // test with queue() + command = new SuccessfulTestCommand(); + try { + command.queue().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on successful execution with "fire and forget" approach + */ + @Test + public void testExecutionHookSuccessfulCommandViaFireAndForget() { + TestHystrixCommand command = new SuccessfulTestCommand(); + try { + // do not block on "get()" ... fire this asynchronously + command.queue(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // wait for command to execute without calling get on the future + while (!command.isExecutionComplete()) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new RuntimeException("interrupted"); + } + } + + /* All the hooks should still work even though we didn't call get() on the future */ + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on successful execution with multiple get() calls to Future + */ + @Test + public void testExecutionHookSuccessfulCommandWithMultipleGetsOnFuture() { + TestHystrixCommand command = new SuccessfulTestCommand(); + try { + Future f = command.queue(); + f.get(); + f.get(); + f.get(); + f.get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + /* Despite multiple calls to get() we should only have 1 call to the hooks. */ + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on failed execution without a fallback + */ + @Test + public void testExecutionHookRunFailureWithoutFallback() { + // test with execute() + TestHystrixCommand command = new UnknownFailureTestCommandWithoutFallback(); + try { + command.execute(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since fallback is not implemented + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since it's not implemented and throws an exception + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response from execute() since we do not have a fallback and run() failed + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception since run() failed + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // run() failure + assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // test with queue() + command = new UnknownFailureTestCommandWithoutFallback(); + try { + command.queue().get(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since fallback is not implemented + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since it's not implemented and throws an exception + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response from queue() since we do not have a fallback and run() failed + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception since run() failed + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // run() failure + assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + + } + + /** + * Execution hook on failed execution with a fallback + */ + @Test + public void testExecutionHookRunFailureWithFallback() { + // test with execute() + TestHystrixCommand command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); + command.execute(); + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response from run since run() failed + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception since run() failed + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // a response since fallback is implemented + assertNotNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it's implemented and succeeds + assertNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from execute() since we expect a fallback despite failure of run() + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception because we expect a fallback + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // test with queue() + command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker()); + try { + command.queue().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response from run since run() failed + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception since run() failed + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // a response since fallback is implemented + assertNotNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it's implemented and succeeds + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since we expect a fallback despite failure of run() + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception because we expect a fallback + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on failed execution with a fallback failure + */ + @Test + public void testExecutionHookRunFailureWithFallbackFailure() { + // test with execute() + TestHystrixCommand command = new KnownFailureTestCommandWithFallbackFailure(); + try { + command.execute(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response because run() and fallback fail + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception because run() and fallback fail + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since fallback fails + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since it's implemented but fails + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because run() and fallback fail + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception because run() and fallback fail + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // run() failure + assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // test with queue() + command = new KnownFailureTestCommandWithFallbackFailure(); + try { + command.queue().get(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response because run() and fallback fail + assertNull(command.builder.executionHook.runSuccessResponse); + // we should have an exception because run() and fallback fail + assertNotNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run since run() failed + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since fallback fails + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since it's implemented but fails + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because run() and fallback fail + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception because run() and fallback fail + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // run() failure + assertEquals(FailureType.COMMAND_EXCEPTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on timeout without a fallback + */ + @Test + public void testExecutionHookTimeoutWithoutFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + try { + System.out.println("start at : " + System.currentTimeMillis()); + command.queue().get(); + fail("Expecting exception"); + } catch (Exception e) { + // ignore + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response because of timeout and no fallback + assertNull(command.builder.executionHook.runSuccessResponse); + // we should not have an exception because run() didn't fail, it timed out + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run due to timeout + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since no fallback + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since no fallback implementation + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // execution occurred + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because of timeout and no fallback + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should have an exception because of timeout and no fallback + assertNotNull(command.builder.executionHook.endExecuteFailureException); + // timeout failure + assertEquals(FailureType.TIMEOUT, command.builder.executionHook.endExecuteFailureType); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + + // we need to wait for the thread to complete before the onThreadComplete hook will be called + // try { + // Thread.sleep(400); + // } catch (InterruptedException e) { + // // ignore + // } + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on timeout with a fallback + */ + @Test + public void testExecutionHookTimeoutWithFallback() { + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + try { + command.queue().get(); + } catch (Exception e) { + throw new RuntimeException("not expecting", e); + } + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we should not have a response because of timeout + assertNull(command.builder.executionHook.runSuccessResponse); + // we should not have an exception because run() didn't fail, it timed out + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run due to timeout + assertEquals(1, command.builder.executionHook.startFallback.get()); + // response since we have a fallback + assertNotNull(command.builder.executionHook.fallbackSuccessResponse); + // null since fallback succeeds + assertNull(command.builder.executionHook.fallbackFailureException); + + // execution occurred + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response because of fallback + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception because of fallback + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + // assertEquals(1, command.builder.executionHook.threadStart.get()); + + // we need to wait for the thread to complete before the onThreadComplete hook will be called + // try { + // Thread.sleep(400); + // } catch (InterruptedException e) { + // // ignore + // } + // assertEquals(1, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on rejected with a fallback + */ + /* + * @Test + * public void testExecutionHookRejectedWithFallback() { + * TestCircuitBreaker circuitBreaker = new TestCircuitBreaker(); + * SingleThreadedPool pool = new SingleThreadedPool(1); + * + * try { + * // fill the queue + * new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS).queue(); + * new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS).queue(); + * } catch (Exception e) { + * // ignore + * } + * + * TestCommandRejection command = new TestCommandRejection(circuitBreaker, pool, 500, 600, TestCommandRejection.FALLBACK_SUCCESS); + * try { + * // now execute one that will be rejected + * command.queue().get(); + * } catch (Exception e) { + * throw new RuntimeException("not expecting", e); + * } + * + * assertTrue(command.isResponseRejected()); + * + * // the run() method should not run as we're rejected + * assertEquals(0, command.builder.executionHook.startRun.get()); + * // we should not have a response because of rejection + * assertNull(command.builder.executionHook.runSuccessResponse); + * // we should not have an exception because we didn't run + * assertNull(command.builder.executionHook.runFailureException); + * + * // the fallback() method should be run due to rejection + * assertEquals(1, command.builder.executionHook.startFallback.get()); + * // response since we have a fallback + * assertNotNull(command.builder.executionHook.fallbackSuccessResponse); + * // null since fallback succeeds + * assertNull(command.builder.executionHook.fallbackFailureException); + * + * // execution occurred + * assertEquals(1, command.builder.executionHook.startExecute.get()); + * // we should have a response because of fallback + * assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + * // we should not have an exception because of fallback + * assertNull(command.builder.executionHook.endExecuteFailureException); + * + * // thread execution + * // assertEquals(0, command.builder.executionHook.threadStart.get()); + * // assertEquals(0, command.builder.executionHook.threadComplete.get()); + * } + */ + /** + * Execution hook on short-circuit with a fallback + */ + @Test + public void testExecutionHookShortCircuitedWithFallbackViaQueue() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().setForceShortCircuit(true); + KnownFailureTestCommandWithoutFallback command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + // now execute one that will be short-circuited + command.queue().get(); + fail("we expect an error as there is no fallback"); + } catch (Exception e) { + // expecting + } + + assertTrue(command.isResponseShortCircuited()); + + // the run() method should not run as we're rejected + assertEquals(0, command.builder.executionHook.startRun.get()); + // we should not have a response because of rejection + assertNull(command.builder.executionHook.runSuccessResponse); + // we should not have an exception because we didn't run + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run due to rejection + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since we don't have a fallback + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since fallback fails and throws an exception + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // execution occurred + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because fallback fails + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we won't have an exception because short-circuit doesn't have one + assertNull(command.builder.executionHook.endExecuteFailureException); + // but we do expect to receive a onError call with FailureType.SHORTCIRCUIT + assertEquals(FailureType.SHORTCIRCUIT, command.builder.executionHook.endExecuteFailureType); + + // thread execution + // assertEquals(0, command.builder.executionHook.threadStart.get()); + // assertEquals(0, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on short-circuit with a fallback + */ + @Test + public void testExecutionHookShortCircuitedWithFallbackViaExecute() { + TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().setForceShortCircuit(true); + KnownFailureTestCommandWithoutFallback command = new KnownFailureTestCommandWithoutFallback(circuitBreaker); + try { + // now execute one that will be short-circuited + command.execute(); + fail("we expect an error as there is no fallback"); + } catch (Exception e) { + // expecting + } + + assertTrue(command.isResponseShortCircuited()); + + // the run() method should not run as we're rejected + assertEquals(0, command.builder.executionHook.startRun.get()); + // we should not have a response because of rejection + assertNull(command.builder.executionHook.runSuccessResponse); + // we should not have an exception because we didn't run + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should be run due to rejection + assertEquals(1, command.builder.executionHook.startFallback.get()); + // no response since we don't have a fallback + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since fallback fails and throws an exception + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // execution occurred + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response because fallback fails + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we won't have an exception because short-circuit doesn't have one + assertNull(command.builder.executionHook.endExecuteFailureException); + // but we do expect to receive a onError call with FailureType.SHORTCIRCUIT + assertEquals(FailureType.SHORTCIRCUIT, command.builder.executionHook.endExecuteFailureType); + + // thread execution + // assertEquals(0, command.builder.executionHook.threadStart.get()); + // assertEquals(0, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on successful execution with semaphore isolation + */ + @Test + public void testExecutionHookSuccessfulCommandWithSemaphoreIsolation() { + // test with execute() + TestSemaphoreCommand command = new TestSemaphoreCommand(new TestCircuitBreaker(), 1, 10); + command.execute(); + + assertFalse(command.isExecutedInThread()); + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from execute() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(0, command.builder.executionHook.threadStart.get()); + assertEquals(0, command.builder.executionHook.threadComplete.get()); + + // test with queue() + command = new TestSemaphoreCommand(new TestCircuitBreaker(), 1, 10); + try { + command.queue().get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertFalse(command.isExecutedInThread()); + + // the run() method should run as we're not short-circuited or rejected + assertEquals(1, command.builder.executionHook.startRun.get()); + // we expect a successful response from run() + assertNotNull(command.builder.executionHook.runSuccessResponse); + // we do not expect an exception + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should not be run as we were successful + assertEquals(0, command.builder.executionHook.startFallback.get()); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // null since it didn't run + assertNull(command.builder.executionHook.fallbackFailureException); + + // the queue() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should have a response from queue() since run() succeeded + assertNotNull(command.builder.executionHook.endExecuteSuccessResponse); + // we should not have an exception since run() succeeded + assertNull(command.builder.executionHook.endExecuteFailureException); + + // thread execution + assertEquals(0, command.builder.executionHook.threadStart.get()); + assertEquals(0, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Execution hook on successful execution with semaphore isolation + */ + @Test + public void testExecutionHookFailureWithSemaphoreIsolation() { + // test with execute() + final TryableSemaphore semaphore = + new TryableSemaphore(HystrixProperty.Factory.asProperty(0)); + + TestSemaphoreCommand command = new TestSemaphoreCommand(new TestCircuitBreaker(), semaphore, 200); + try { + command.execute(); + fail("we expect a failure"); + } catch (Exception e) { + // expected + } + + assertFalse(command.isExecutedInThread()); + assertTrue(command.isResponseRejected()); + + // the run() method should not run as we are rejected + assertEquals(0, command.builder.executionHook.startRun.get()); + // null as run() does not get invoked + assertNull(command.builder.executionHook.runSuccessResponse); + // null as run() does not get invoked + assertNull(command.builder.executionHook.runFailureException); + + // the fallback() method should run because of rejection + assertEquals(1, command.builder.executionHook.startFallback.get()); + // null since there is no fallback + assertNull(command.builder.executionHook.fallbackSuccessResponse); + // not null since the fallback is not implemented + assertNotNull(command.builder.executionHook.fallbackFailureException); + + // the execute() method was used + assertEquals(1, command.builder.executionHook.startExecute.get()); + // we should not have a response since fallback has nothing + assertNull(command.builder.executionHook.endExecuteSuccessResponse); + // we won't have an exception because rejection doesn't have one + assertNull(command.builder.executionHook.endExecuteFailureException); + // but we do expect to receive a onError call with FailureType.SHORTCIRCUIT + assertEquals(FailureType.REJECTED_SEMAPHORE_EXECUTION, command.builder.executionHook.endExecuteFailureType); + + // thread execution + assertEquals(0, command.builder.executionHook.threadStart.get()); + assertEquals(0, command.builder.executionHook.threadComplete.get()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * Test a command execution that fails but has a fallback. + */ + @Test + public void testExecutionFailureWithFallbackImplementedButDisabled() { + TestHystrixCommand commandEnabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), true); + try { + assertEquals(false, commandEnabled.execute()); + } catch (Exception e) { + e.printStackTrace(); + fail("We should have received a response from the fallback."); + } + + TestHystrixCommand commandDisabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), false); + try { + assertEquals(false, commandDisabled.execute()); + fail("expect exception thrown"); + } catch (Exception e) { + // expected + } + + assertEquals("we failed with a simulated issue", commandDisabled.getFailedExecutionException().getMessage()); + + assertTrue(commandDisabled.isFailedExecution()); + + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, commandDisabled.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, commandDisabled.builder.metrics.getHealthCounts().getErrorPercentage()); + + // assertEquals(2, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } + + /** + * Test that we can still use thread isolation if desired. + */ + @Test(timeout = 500) + public void testSynchronousExecutionTimeoutValueViaExecute() { + HystrixObservableCommand.Setter properties = HystrixObservableCommand.Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestKey")) + .andCommandPropertiesDefaults(HystrixCommandProperties.Setter() + .withExecutionIsolationStrategy(ExecutionIsolationStrategy.THREAD) + .withExecutionIsolationThreadTimeoutInMilliseconds(50)); + + System.out.println(">>>>> Begin: " + System.currentTimeMillis()); + + HystrixObservableCommand command = new HystrixObservableCommand(properties) { + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber t1) { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + t1.onNext("hello"); + t1.onCompleted(); + } + + }); + } + + @Override + protected Observable getFallback() { + if (isResponseTimedOut()) { + return Observable.from("timed-out"); + } else { + return Observable.from("abc"); + } + } + }; + + System.out.println(">>>>> Start: " + System.currentTimeMillis()); + String value = command.execute(); + System.out.println(">>>>> End: " + System.currentTimeMillis()); + assertTrue(command.isResponseTimedOut()); + assertEquals("expected fallback value", "timed-out", value); + + // Thread isolated + assertTrue(command.isExecutedInThread()); + } + + @Test(timeout = 500) + public void testSynchronousExecutionUsingThreadIsolationTimeoutValueViaObserve() { + HystrixObservableCommand.Setter properties = HystrixObservableCommand.Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestKey")) + .andCommandPropertiesDefaults(HystrixCommandProperties.Setter() + .withExecutionIsolationStrategy(ExecutionIsolationStrategy.THREAD) + .withExecutionIsolationThreadTimeoutInMilliseconds(50)); + + HystrixObservableCommand command = new HystrixObservableCommand(properties) { + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber t1) { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + t1.onNext("hello"); + t1.onCompleted(); + } + + }); + } + + @Override + protected Observable getFallback() { + if (isResponseTimedOut()) { + return Observable.from("timed-out"); + } else { + return Observable.from("abc"); + } + } + }; + + String value = command.observe().toBlockingObservable().last(); + assertTrue(command.isResponseTimedOut()); + assertEquals("expected fallback value", "timed-out", value); + + // Thread isolated + assertTrue(command.isExecutedInThread()); + } + + @Test(timeout = 500) + public void testAsyncExecutionTimeoutValueViaObserve() { + HystrixObservableCommand.Setter properties = HystrixObservableCommand.Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestKey")) + .andCommandPropertiesDefaults(HystrixCommandProperties.Setter() + .withExecutionIsolationThreadTimeoutInMilliseconds(50)); + + HystrixObservableCommand command = new HystrixObservableCommand(properties) { + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber t1) { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + System.out.println("********** interrupted on timeout"); + e.printStackTrace(); + } + // should never reach here + t1.onNext("hello"); + t1.onCompleted(); + } + }).subscribeOn(Schedulers.newThread()); + } + + @Override + protected Observable getFallback() { + if (isResponseTimedOut()) { + return Observable.from("timed-out"); + } else { + return Observable.from("abc"); + } + } + }; + + String value = command.observe().toBlockingObservable().last(); + assertTrue(command.isResponseTimedOut()); + assertEquals("expected fallback value", "timed-out", value); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * See https://github.com/Netflix/Hystrix/issues/212 + */ + @Test + public void testObservableTimeoutNoFallbackThreadContext() { + TestSubscriber ts = new TestSubscriber(); + + final AtomicReference onErrorThread = new AtomicReference(); + final AtomicBoolean isRequestContextInitialized = new AtomicBoolean(); + + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_NOT_IMPLEMENTED); + command.toObservable().doOnError(new Action1() { + + @Override + public void call(Throwable t1) { + System.out.println("onError: " + t1); + System.out.println("onError Thread: " + Thread.currentThread()); + System.out.println("ThreadContext in onError: " + HystrixRequestContext.isCurrentThreadInitialized()); + onErrorThread.set(Thread.currentThread()); + isRequestContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + } + + }).subscribe(ts); + + ts.awaitTerminalEvent(); + + assertTrue(isRequestContextInitialized.get()); + assertTrue(onErrorThread.get().getName().startsWith("RxComputationThreadPool")); + + List errors = ts.getOnErrorEvents(); + assertEquals(1, errors.size()); + Throwable e = errors.get(0); + if (errors.get(0) instanceof HystrixRuntimeException) { + HystrixRuntimeException de = (HystrixRuntimeException) e; + assertNotNull(de.getFallbackException()); + assertTrue(de.getFallbackException() instanceof UnsupportedOperationException); + assertNotNull(de.getImplementingClass()); + assertNotNull(de.getCause()); + assertTrue(de.getCause() instanceof TimeoutException); + } else { + fail("the exception should be ExecutionException with cause as HystrixRuntimeException"); + } + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + /** + * See https://github.com/Netflix/Hystrix/issues/212 + */ + @Test + public void testObservableTimeoutFallbackThreadContext() { + TestSubscriber ts = new TestSubscriber(); + + final AtomicReference onErrorThread = new AtomicReference(); + final AtomicBoolean isRequestContextInitialized = new AtomicBoolean(); + + TestHystrixCommand command = new TestCommandWithTimeout(50, TestCommandWithTimeout.FALLBACK_SUCCESS); + command.toObservable().doOnNext(new Action1() { + + @Override + public void call(Boolean t1) { + System.out.println("onNext: " + t1); + System.out.println("onNext Thread: " + Thread.currentThread()); + System.out.println("ThreadContext in onNext: " + HystrixRequestContext.isCurrentThreadInitialized()); + onErrorThread.set(Thread.currentThread()); + isRequestContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + } + + }).subscribe(ts); + + ts.awaitTerminalEvent(); + + System.out.println("events: " + ts.getOnNextEvents()); + + assertTrue(isRequestContextInitialized.get()); + assertTrue(onErrorThread.get().getName().startsWith("RxComputationThreadPool")); + + List onNexts = ts.getOnNextEvents(); + assertEquals(1, onNexts.size()); + assertFalse(onNexts.get(0)); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + // semaphore isolated + assertFalse(command.isExecutedInThread()); + } + + private RequestContextTestResults testRequestContextOnSuccess(ExecutionIsolationStrategy isolation, final Scheduler userScheduler) { + final RequestContextTestResults results = new RequestContextTestResults(); + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(isolation))) { + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + results.isContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.originThread.set(Thread.currentThread()); + s.onNext(true); + s.onCompleted(); + } + + }).subscribeOn(userScheduler); + } + + }; + + results.command = command; + + command.toObservable().doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + results.isContextInitializedObserveOn.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.observeOnThread.set(Thread.currentThread()); + } + + }).subscribe(results.ts); + results.ts.awaitTerminalEvent(); + + System.out.println("Run => Initialized: " + results.isContextInitialized.get() + " Thread: " + results.originThread.get()); + System.out.println("Observed => Initialized: " + results.isContextInitializedObserveOn.get() + " Thread: " + results.observeOnThread.get()); + + assertEquals(1, results.ts.getOnNextEvents().size()); + assertTrue(results.ts.getOnNextEvents().get(0)); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertTrue(command.isSuccessfulExecution()); + + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(0, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + return results; + } + + private RequestContextTestResults testRequestContextOnGracefulFailure(ExecutionIsolationStrategy isolation, final Scheduler userScheduler) { + final RequestContextTestResults results = new RequestContextTestResults(); + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(isolation))) { + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + results.isContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.originThread.set(Thread.currentThread()); + s.onError(new RuntimeException("graceful onError")); + } + + }).subscribeOn(userScheduler); + } + + }; + + results.command = command; + + command.toObservable().doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + results.isContextInitializedObserveOn.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.observeOnThread.set(Thread.currentThread()); + } + + }).subscribe(results.ts); + results.ts.awaitTerminalEvent(); + + System.out.println("Run => Initialized: " + results.isContextInitialized.get() + " Thread: " + results.originThread.get()); + System.out.println("Observed => Initialized: " + results.isContextInitializedObserveOn.get() + " Thread: " + results.observeOnThread.get()); + + assertEquals(1, results.ts.getOnErrorEvents().size()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertFalse(command.isSuccessfulExecution()); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + return results; + } + + private RequestContextTestResults testRequestContextOnBadFailure(ExecutionIsolationStrategy isolation, final Scheduler userScheduler) { + final RequestContextTestResults results = new RequestContextTestResults(); + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(isolation))) { + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + results.isContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.originThread.set(Thread.currentThread()); + throw new RuntimeException("bad onError"); + } + + }).subscribeOn(userScheduler); + } + + }; + + results.command = command; + + command.toObservable().doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + results.isContextInitializedObserveOn.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.observeOnThread.set(Thread.currentThread()); + } + + }).subscribe(results.ts); + results.ts.awaitTerminalEvent(); + + System.out.println("Run => Initialized: " + results.isContextInitialized.get() + " Thread: " + results.originThread.get()); + System.out.println("Observed => Initialized: " + results.isContextInitializedObserveOn.get() + " Thread: " + results.observeOnThread.get()); + + assertEquals(1, results.ts.getOnErrorEvents().size()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertFalse(command.isSuccessfulExecution()); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + return results; + } + + private RequestContextTestResults testRequestContextOnFailureWithFallback(ExecutionIsolationStrategy isolation, final Scheduler userScheduler) { + final RequestContextTestResults results = new RequestContextTestResults(); + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(isolation))) { + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + s.onError(new RuntimeException("onError")); + } + + }).subscribeOn(userScheduler); + } + + @Override + protected Observable getFallback() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + results.isContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.originThread.set(Thread.currentThread()); + s.onNext(false); + s.onCompleted(); + } + + }).subscribeOn(userScheduler); + } + + }; + + results.command = command; + + command.toObservable().doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + results.isContextInitializedObserveOn.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.observeOnThread.set(Thread.currentThread()); + } + + }).subscribe(results.ts); + results.ts.awaitTerminalEvent(); + + System.out.println("Run => Initialized: " + results.isContextInitialized.get() + " Thread: " + results.originThread.get()); + System.out.println("Observed => Initialized: " + results.isContextInitializedObserveOn.get() + " Thread: " + results.observeOnThread.get()); + + assertEquals(0, results.ts.getOnErrorEvents().size()); + assertEquals(1, results.ts.getOnNextEvents().size()); + assertEquals(false, results.ts.getOnNextEvents().get(0)); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertFalse(command.isSuccessfulExecution()); + assertTrue(command.isFailedExecution()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + return results; + } + + private RequestContextTestResults testRequestContextOnRejectionWithFallback(ExecutionIsolationStrategy isolation, final Scheduler userScheduler) { + final RequestContextTestResults results = new RequestContextTestResults(); + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationSemaphoreMaxConcurrentRequests(0))) { + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + s.onError(new RuntimeException("onError")); + } + + }).subscribeOn(userScheduler); + } + + @Override + protected Observable getFallback() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + results.isContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.originThread.set(Thread.currentThread()); + s.onNext(false); + s.onCompleted(); + } + + }).subscribeOn(userScheduler); + } + + }; + + results.command = command; + + command.toObservable().doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + results.isContextInitializedObserveOn.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.observeOnThread.set(Thread.currentThread()); + } + + }).subscribe(results.ts); + results.ts.awaitTerminalEvent(); + + System.out.println("Run => Initialized: " + results.isContextInitialized.get() + " Thread: " + results.originThread.get()); + System.out.println("Observed => Initialized: " + results.isContextInitializedObserveOn.get() + " Thread: " + results.observeOnThread.get()); + + assertEquals(0, results.ts.getOnErrorEvents().size()); + assertEquals(1, results.ts.getOnNextEvents().size()); + assertEquals(false, results.ts.getOnNextEvents().get(0)); + + assertTrue(command.getExecutionTimeInMilliseconds() == -1); + assertFalse(command.isSuccessfulExecution()); + assertTrue(command.isResponseRejected()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + return results; + } + + private RequestContextTestResults testRequestContextOnShortCircuitedWithFallback(ExecutionIsolationStrategy isolation, final Scheduler userScheduler) { + final RequestContextTestResults results = new RequestContextTestResults(); + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder().setCircuitBreaker(new TestCircuitBreaker().setForceShortCircuit(true))) { + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + s.onError(new RuntimeException("onError")); + } + + }).subscribeOn(userScheduler); + } + + @Override + protected Observable getFallback() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + results.isContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.originThread.set(Thread.currentThread()); + s.onNext(false); + s.onCompleted(); + } + + }).subscribeOn(userScheduler); + } + + }; + + results.command = command; + + command.toObservable().doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + results.isContextInitializedObserveOn.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.observeOnThread.set(Thread.currentThread()); + } + + }).subscribe(results.ts); + results.ts.awaitTerminalEvent(); + + System.out.println("Run => Initialized: " + results.isContextInitialized.get() + " Thread: " + results.originThread.get()); + System.out.println("Observed => Initialized: " + results.isContextInitializedObserveOn.get() + " Thread: " + results.observeOnThread.get()); + + assertEquals(0, results.ts.getOnErrorEvents().size()); + assertEquals(1, results.ts.getOnNextEvents().size()); + assertEquals(false, results.ts.getOnNextEvents().get(0)); + + assertTrue(command.getExecutionTimeInMilliseconds() == -1); + assertFalse(command.isSuccessfulExecution()); + assertTrue(command.isResponseShortCircuited()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + return results; + } + + private RequestContextTestResults testRequestContextOnTimeout(ExecutionIsolationStrategy isolation, final Scheduler userScheduler) { + final RequestContextTestResults results = new RequestContextTestResults(); + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(isolation).withExecutionIsolationThreadTimeoutInMilliseconds(50))) { + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + results.isContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.originThread.set(Thread.currentThread()); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + // ignore the interrupted exception + } + } + + }).subscribeOn(userScheduler); + } + + }; + + results.command = command; + + command.toObservable().doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + results.isContextInitializedObserveOn.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.observeOnThread.set(Thread.currentThread()); + } + + }).subscribe(results.ts); + results.ts.awaitTerminalEvent(); + + System.out.println("Run => Initialized: " + results.isContextInitialized.get() + " Thread: " + results.originThread.get()); + System.out.println("Observed => Initialized: " + results.isContextInitializedObserveOn.get() + " Thread: " + results.observeOnThread.get()); + + assertEquals(1, results.ts.getOnErrorEvents().size()); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertFalse(command.isSuccessfulExecution()); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + return results; + } + + private RequestContextTestResults testRequestContextOnTimeoutWithFallback(ExecutionIsolationStrategy isolation, final Scheduler userScheduler) { + final RequestContextTestResults results = new RequestContextTestResults(); + TestHystrixCommand command = new TestHystrixCommand(TestHystrixCommand.testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(isolation).withExecutionIsolationThreadTimeoutInMilliseconds(50))) { + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + // ignore the interrupted exception + } + } + + }).subscribeOn(userScheduler); + } + + @Override + protected Observable getFallback() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + results.isContextInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.originThread.set(Thread.currentThread()); + s.onNext(false); + s.onCompleted(); + } + + }).subscribeOn(userScheduler); + } + + }; + + results.command = command; + + command.toObservable().doOnEach(new Action1>() { + + @Override + public void call(Notification n) { + System.out.println("timeoutWithFallback notification: " + n + " " + Thread.currentThread()); + results.isContextInitializedObserveOn.set(HystrixRequestContext.isCurrentThreadInitialized()); + results.observeOnThread.set(Thread.currentThread()); + } + + }).subscribe(results.ts); + results.ts.awaitTerminalEvent(); + + System.out.println("Fallback => Initialized: " + results.isContextInitialized.get() + " Thread: " + results.originThread.get()); + System.out.println("Observed => Initialized: " + results.isContextInitializedObserveOn.get() + " Thread: " + results.observeOnThread.get()); + + assertEquals(1, results.ts.getOnNextEvents().size()); + assertEquals(false, results.ts.getOnNextEvents().get(0)); + + assertTrue(command.getExecutionTimeInMilliseconds() > -1); + assertFalse(command.isSuccessfulExecution()); + assertTrue(command.isResponseTimedOut()); + + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED)); + assertEquals(1, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(0, command.builder.metrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE)); + + assertEquals(100, command.builder.metrics.getHealthCounts().getErrorPercentage()); + + assertEquals(1, HystrixRequestLog.getCurrentRequest().getAllExecutedCommands().size()); + + return results; + } + + private final class RequestContextTestResults { + volatile TestHystrixCommand command; + final AtomicReference originThread = new AtomicReference(); + final AtomicBoolean isContextInitialized = new AtomicBoolean(); + TestSubscriber ts = new TestSubscriber(); + final AtomicBoolean isContextInitializedObserveOn = new AtomicBoolean(); + final AtomicReference observeOnThread = new AtomicReference(); + } + + /* *************************************** testSuccessfuleRequestContext *********************************** */ + + /** + * Synchronous Observable and semaphore isolation. Only [Main] thread is involved in this. + */ + @Test + public void testSuccessfulRequestContextWithSemaphoreIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnSuccess(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().equals(Thread.currentThread())); // all synchronous + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().equals(Thread.currentThread())); // all synchronous + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation. User provided thread [RxNewThread] does everything. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testSuccessfulRequestContextWithSemaphoreIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnSuccess(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testSuccessfulRequestContextWithSemaphoreIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnSuccess(ExecutionIsolationStrategy.SEMAPHORE, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Synchronous Observable and thread isolation. Work done on [hystrix-OWNER_ONE] thread and then observed on [RxComputation] + */ + @Test + public void testSuccessfulRequestContextWithThreadIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnSuccess(ExecutionIsolationStrategy.THREAD, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().getName().startsWith("hystrix-OWNER_ONE")); // thread isolated on a HystrixThreadPool + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix thread + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and thread isolation. User provided thread [RxNetThread] executes Observable and then [RxComputation] observes the onNext calls. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testSuccessfulRequestContextWithThreadIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnSuccess(ExecutionIsolationStrategy.THREAD, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testSuccessfulRequestContextWithThreadIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnSuccess(ExecutionIsolationStrategy.THREAD, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /* *************************************** testGracefulFailureRequestContext *********************************** */ + + /** + * Synchronous Observable and semaphore isolation. Only [Main] thread is involved in this. + */ + @Test + public void testGracefulFailureRequestContextWithSemaphoreIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnGracefulFailure(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().equals(Thread.currentThread())); // all synchronous + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().equals(Thread.currentThread())); // all synchronous + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation. User provided thread [RxNewThread] does everything. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testGracefulFailureRequestContextWithSemaphoreIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnGracefulFailure(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testGracefulFailureRequestContextWithSemaphoreIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnGracefulFailure(ExecutionIsolationStrategy.SEMAPHORE, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Synchronous Observable and thread isolation. Work done on [hystrix-OWNER_ONE] thread and then observed on [RxComputation] + */ + @Test + public void testGracefulFailureRequestContextWithThreadIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnGracefulFailure(ExecutionIsolationStrategy.THREAD, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().getName().startsWith("hystrix-OWNER_ONE")); // thread isolated on a HystrixThreadPool + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix thread + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and thread isolation. User provided thread [RxNetThread] executes Observable and then [RxComputation] observes the onNext calls. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testGracefulFailureRequestContextWithThreadIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnGracefulFailure(ExecutionIsolationStrategy.THREAD, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testGracefulFailureRequestContextWithThreadIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnGracefulFailure(ExecutionIsolationStrategy.THREAD, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /* *************************************** testBadFailureRequestContext *********************************** */ + + /** + * Synchronous Observable and semaphore isolation. Only [Main] thread is involved in this. + */ + @Test + public void testBadFailureRequestContextWithSemaphoreIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnBadFailure(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().equals(Thread.currentThread())); // all synchronous + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().equals(Thread.currentThread())); // all synchronous + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation. User provided thread [RxNewThread] does everything. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testBadFailureRequestContextWithSemaphoreIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnBadFailure(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testBadFailureRequestContextWithSemaphoreIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnBadFailure(ExecutionIsolationStrategy.SEMAPHORE, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Synchronous Observable and thread isolation. Work done on [hystrix-OWNER_ONE] thread and then observed on [RxComputation] + */ + @Test + public void testBadFailureRequestContextWithThreadIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnBadFailure(ExecutionIsolationStrategy.THREAD, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().getName().startsWith("hystrix-OWNER_ONE")); // thread isolated on a HystrixThreadPool + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix thread + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and thread isolation. User provided thread [RxNetThread] executes Observable and then [RxComputation] observes the onNext calls. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testBadFailureRequestContextWithThreadIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnBadFailure(ExecutionIsolationStrategy.THREAD, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testBadFailureRequestContextWithThreadIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnBadFailure(ExecutionIsolationStrategy.THREAD, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /* *************************************** testFailureWithFallbackRequestContext *********************************** */ + + /** + * Synchronous Observable and semaphore isolation. Only [Main] thread is involved in this. + */ + @Test + public void testFailureWithFallbackRequestContextWithSemaphoreIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnFailureWithFallback(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().equals(Thread.currentThread())); // all synchronous + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().equals(Thread.currentThread())); // all synchronous + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation. User provided thread [RxNewThread] does everything. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testFailureWithFallbackRequestContextWithSemaphoreIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnFailureWithFallback(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testFailureWithFallbackRequestContextWithSemaphoreIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnFailureWithFallback(ExecutionIsolationStrategy.SEMAPHORE, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Synchronous Observable and thread isolation. Work done on [hystrix-OWNER_ONE] thread and then observed on [RxComputation] + */ + @Test + public void testFailureWithFallbackRequestContextWithThreadIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnFailureWithFallback(ExecutionIsolationStrategy.THREAD, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().getName().startsWith("hystrix-OWNER_ONE")); // thread isolated on a HystrixThreadPool + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix thread + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and thread isolation. User provided thread [RxNetThread] executes Observable and then [RxComputation] observes the onNext calls. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testFailureWithFallbackRequestContextWithThreadIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnFailureWithFallback(ExecutionIsolationStrategy.THREAD, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testFailureWithFallbackRequestContextWithThreadIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnFailureWithFallback(ExecutionIsolationStrategy.THREAD, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /* *************************************** testRejectionWithFallbackRequestContext *********************************** */ + + /** + * Synchronous Observable and semaphore isolation. Only [Main] thread is involved in this. + */ + @Test + public void testRejectionWithFallbackRequestContextWithSemaphoreIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnRejectionWithFallback(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().equals(Thread.currentThread())); // all synchronous + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().equals(Thread.currentThread())); // all synchronous + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation. User provided thread [RxNewThread] does everything. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testRejectionWithFallbackRequestContextWithSemaphoreIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnRejectionWithFallback(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testRejectionWithFallbackRequestContextWithSemaphoreIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnRejectionWithFallback(ExecutionIsolationStrategy.SEMAPHORE, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Synchronous Observable and thread isolation. Work done on [hystrix-OWNER_ONE] thread and then observed on [RxComputation] + */ + @Test + public void testRejectionWithFallbackRequestContextWithThreadIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnRejectionWithFallback(ExecutionIsolationStrategy.THREAD, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().equals(Thread.currentThread())); // fallback is performed by the calling thread + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().equals(Thread.currentThread())); // rejected so we stay on calling thread + + // thread isolated ... but rejected so not executed in a thread + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and thread isolation. User provided thread [RxNetThread] executes Observable and then [RxComputation] observes the onNext calls. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testRejectionWithFallbackRequestContextWithThreadIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnRejectionWithFallback(ExecutionIsolationStrategy.THREAD, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); + + // thread isolated ... but rejected so not executed in a thread + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testRejectionWithFallbackRequestContextWithThreadIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnRejectionWithFallback(ExecutionIsolationStrategy.THREAD, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler for getFallback + + // thread isolated ... but rejected so not executed in a thread + assertFalse(results.command.isExecutedInThread()); + } + + /* *************************************** testShortCircuitedWithFallbackRequestContext *********************************** */ + + /** + * Synchronous Observable and semaphore isolation. Only [Main] thread is involved in this. + */ + @Test + public void testShortCircuitedWithFallbackRequestContextWithSemaphoreIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnShortCircuitedWithFallback(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().equals(Thread.currentThread())); // all synchronous + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().equals(Thread.currentThread())); // all synchronous + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation. User provided thread [RxNewThread] does everything. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testShortCircuitedWithFallbackRequestContextWithSemaphoreIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnShortCircuitedWithFallback(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testShortCircuitedWithFallbackRequestContextWithSemaphoreIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnShortCircuitedWithFallback(ExecutionIsolationStrategy.SEMAPHORE, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Synchronous Observable and thread isolation. Work done on [hystrix-OWNER_ONE] thread and then observed on [RxComputation] + */ + @Test + public void testShortCircuitedWithFallbackRequestContextWithThreadIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnShortCircuitedWithFallback(ExecutionIsolationStrategy.THREAD, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().equals(Thread.currentThread())); // fallback is performed by the calling thread + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().equals(Thread.currentThread())); // rejected so we stay on calling thread + + // thread isolated ... but rejected so not executed in a thread + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and thread isolation. User provided thread [RxNetThread] executes Observable and then [RxComputation] observes the onNext calls. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testShortCircuitedWithFallbackRequestContextWithThreadIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnShortCircuitedWithFallback(ExecutionIsolationStrategy.THREAD, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // we capture and set the context once the user provided Observable emits + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler from getFallback + + // thread isolated ... but rejected so not executed in a thread + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testShortCircuitedWithFallbackRequestContextWithThreadIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnShortCircuitedWithFallback(ExecutionIsolationStrategy.THREAD, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler from getFallback + + // thread isolated ... but rejected so not executed in a thread + assertFalse(results.command.isExecutedInThread()); + } + + /* *************************************** testTimeoutRequestContext *********************************** */ + + /** + * Synchronous Observable and semaphore isolation. Only [Main] thread is involved in this. + */ + @Test + public void testTimeoutRequestContextWithSemaphoreIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnTimeout(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().equals(Thread.currentThread())); // all synchronous + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // timeout schedules on RxComputation since the original thread was timed out + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation. User provided thread [RxNewThread] does everything. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testTimeoutRequestContextWithSemaphoreIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnTimeout(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the timeout captures the context so it exists + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // timeout schedules on RxComputation since the original thread was timed out + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testTimeoutRequestContextWithSemaphoreIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnTimeout(ExecutionIsolationStrategy.SEMAPHORE, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // timeout schedules on RxComputation since the original thread was timed out + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Synchronous Observable and thread isolation. Work done on [hystrix-OWNER_ONE] thread and then observed on [RxComputation] + */ + @Test + public void testTimeoutRequestContextWithThreadIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnTimeout(ExecutionIsolationStrategy.THREAD, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().getName().startsWith("hystrix-OWNER_ONE")); // thread isolated on a HystrixThreadPool + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix thread + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and thread isolation. User provided thread [RxNetThread] executes Observable and then [RxComputation] observes the onNext calls. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testTimeoutRequestContextWithThreadIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnTimeout(ExecutionIsolationStrategy.THREAD, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the timeout captures the context so it exists + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testTimeoutRequestContextWithThreadIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnTimeout(ExecutionIsolationStrategy.THREAD, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix or user thread because of thread isolation choice + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /* *************************************** testTimeoutWithFallbackRequestContext *********************************** */ + + /** + * Synchronous Observable and semaphore isolation. Only [Main] thread is involved in this. + */ + @Test + public void testTimeoutWithFallbackRequestContextWithSemaphoreIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnTimeoutWithFallback(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().getName().startsWith("RxComputation")); // timeout uses RxComputation thread + //(this use case is a little odd as it should generally not be the case that we are "timing out" a synchronous observable on semaphore isolation) + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // timeout uses RxComputation thread + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation. User provided thread [RxNewThread] does everything. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testTimeoutWithFallbackRequestContextWithSemaphoreIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnTimeoutWithFallback(ExecutionIsolationStrategy.SEMAPHORE, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the timeout captures the context so it exists + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testTimeoutWithFallbackRequestContextWithSemaphoreIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnTimeoutWithFallback(ExecutionIsolationStrategy.SEMAPHORE, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // semaphore isolated + assertFalse(results.command.isExecutedInThread()); + } + + /** + * Synchronous Observable and thread isolation. Work done on [hystrix-OWNER_ONE] thread and then observed on [RxComputation] + */ + @Test + public void testTimeoutWithFallbackRequestContextWithThreadIsolatedSynchronousObservable() { + RequestContextTestResults results = testRequestContextOnTimeoutWithFallback(ExecutionIsolationStrategy.THREAD, Schedulers.immediate()); + + assertTrue(results.isContextInitialized.get()); + assertTrue(results.originThread.get().getName().startsWith("RxComputation")); // timeout uses RxComputation thread for fallback + + assertTrue(results.isContextInitializedObserveOn.get()); + assertTrue(results.observeOnThread.get().getName().startsWith("RxComputation")); // we observeOn Schedulers.computation() instead of using the hystrix thread + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and thread isolation. User provided thread [RxNetThread] executes Observable and then [RxComputation] observes the onNext calls. + * + * NOTE: RequestContext will NOT exist on that thread. + * + * An async Observable running on its own thread will not have access to the request context unless the user manages the context. + */ + @Test + public void testTimeoutWithFallbackRequestContextWithThreadIsolatedAsynchronousObservable() { + RequestContextTestResults results = testRequestContextOnTimeoutWithFallback(ExecutionIsolationStrategy.THREAD, Schedulers.newThread()); + + assertFalse(results.isContextInitialized.get()); // it won't have request context as it's on a user provided thread/scheduler + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the timeout captures the context so it exists + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /** + * Async Observable and semaphore isolation WITH functioning RequestContext + * + * Use HystrixContextScheduler to make the user provided scheduler capture context. + */ + @Test + public void testTimeoutWithFallbackRequestContextWithThreadIsolatedAsynchronousObservableAndCapturedContextScheduler() { + RequestContextTestResults results = testRequestContextOnTimeoutWithFallback(ExecutionIsolationStrategy.THREAD, new HystrixContextScheduler(Schedulers.newThread())); + + assertTrue(results.isContextInitialized.get()); // the user scheduler captures context + assertTrue(results.originThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + assertTrue(results.isContextInitializedObserveOn.get()); // the user scheduler captures context + assertTrue(results.observeOnThread.get().getName().startsWith("RxNewThread")); // the user provided thread/scheduler + + // thread isolated + assertTrue(results.command.isExecutedInThread()); + } + + /* ******************************************************************************** */ + /* ******************************************************************************** */ + /* private HystrixCommand class implementations for unit testing */ + /* ******************************************************************************** */ + /* ******************************************************************************** */ + + /** + * Used by UnitTest command implementations to provide base defaults for constructor and a builder pattern for the arguments being passed in. + */ + private static abstract class TestHystrixCommand extends HystrixObservableCommand { + + final TestCommandBuilder builder; + + TestHystrixCommand(TestCommandBuilder builder) { + super(builder.owner, builder.dependencyKey, builder.threadPoolKey, builder.circuitBreaker, builder.threadPool, + builder.commandPropertiesDefaults, builder.threadPoolPropertiesDefaults, builder.metrics, + builder.fallbackSemaphore, builder.executionSemaphore, TEST_PROPERTIES_FACTORY, builder.executionHook); + this.builder = builder; + } + + static TestCommandBuilder testPropsBuilder() { + return new TestCommandBuilder(); + } + + static class TestCommandBuilder { + TestCircuitBreaker _cb = new TestCircuitBreaker(); + HystrixCommandGroupKey owner = CommandGroupForUnitTest.OWNER_ONE; + HystrixCommandKey dependencyKey = null; + HystrixThreadPoolKey threadPoolKey = null; + HystrixCircuitBreaker circuitBreaker = _cb; + HystrixThreadPool threadPool = null; + HystrixCommandProperties.Setter commandPropertiesDefaults = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE); + HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults = HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder(); + HystrixCommandMetrics metrics = _cb.metrics; + TryableSemaphore fallbackSemaphore = null; + TryableSemaphore executionSemaphore = null; + TestExecutionHook executionHook = new TestExecutionHook(); + + TestCommandBuilder setOwner(HystrixCommandGroupKey owner) { + this.owner = owner; + return this; + } + + TestCommandBuilder setCommandKey(HystrixCommandKey dependencyKey) { + this.dependencyKey = dependencyKey; + return this; + } + + TestCommandBuilder setThreadPoolKey(HystrixThreadPoolKey threadPoolKey) { + this.threadPoolKey = threadPoolKey; + return this; + } + + TestCommandBuilder setCircuitBreaker(HystrixCircuitBreaker circuitBreaker) { + this.circuitBreaker = circuitBreaker; + return this; + } + + TestCommandBuilder setThreadPool(HystrixThreadPool threadPool) { + this.threadPool = threadPool; + return this; + } + + TestCommandBuilder setCommandPropertiesDefaults(HystrixCommandProperties.Setter commandPropertiesDefaults) { + this.commandPropertiesDefaults = commandPropertiesDefaults; + return this; + } + + TestCommandBuilder setThreadPoolPropertiesDefaults(HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults) { + this.threadPoolPropertiesDefaults = threadPoolPropertiesDefaults; + return this; + } + + TestCommandBuilder setMetrics(HystrixCommandMetrics metrics) { + this.metrics = metrics; + return this; + } + + TestCommandBuilder setFallbackSemaphore(TryableSemaphore fallbackSemaphore) { + this.fallbackSemaphore = fallbackSemaphore; + return this; + } + + TestCommandBuilder setExecutionSemaphore(TryableSemaphore executionSemaphore) { + this.executionSemaphore = executionSemaphore; + return this; + } + + } + + } + + /** + * Successful execution - no fallback implementation. + */ + private static class SuccessfulTestCommand extends TestHystrixCommand { + + public SuccessfulTestCommand() { + this(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE)); + } + + public SuccessfulTestCommand(HystrixCommandProperties.Setter properties) { + super(testPropsBuilder().setCommandPropertiesDefaults(properties)); + } + + @Override + protected Observable run() { + return Observable.just(true).subscribeOn(Schedulers.computation()); + } + + } + + /** + * Successful execution - no fallback implementation. + */ + private static class DynamicOwnerTestCommand extends TestHystrixCommand { + + public DynamicOwnerTestCommand(HystrixCommandGroupKey owner) { + super(testPropsBuilder().setOwner(owner)); + } + + @Override + protected Observable run() { + System.out.println("successfully executed"); + return Observable.just(true).subscribeOn(Schedulers.computation()); + } + + } + + /** + * Successful execution - no fallback implementation. + */ + private static class DynamicOwnerAndKeyTestCommand extends TestHystrixCommand { + + public DynamicOwnerAndKeyTestCommand(HystrixCommandGroupKey owner, HystrixCommandKey key) { + super(testPropsBuilder().setOwner(owner).setCommandKey(key).setCircuitBreaker(null).setMetrics(null)); + // we specifically are NOT passing in a circuit breaker here so we test that it creates a new one correctly based on the dynamic key + } + + @Override + protected Observable run() { + System.out.println("successfully executed"); + return Observable.just(true).subscribeOn(Schedulers.computation()); + } + + } + + /** + * Failed execution with unknown exception (not HystrixException) - no fallback implementation. + */ + private static class UnknownFailureTestCommandWithoutFallback extends TestHystrixCommand { + + private UnknownFailureTestCommandWithoutFallback() { + super(testPropsBuilder()); + } + + @Override + protected Observable run() { + // TODO duplicate with error inside async Observable + System.out.println("*** simulated failed execution ***"); + throw new RuntimeException("we failed with an unknown issue"); + } + + } + + /** + * Failed execution with known exception (HystrixException) - no fallback implementation. + */ + private static class KnownFailureTestCommandWithoutFallback extends TestHystrixCommand { + + private KnownFailureTestCommandWithoutFallback(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + } + + @Override + protected Observable run() { + // TODO duplicate with error inside async Observable + System.out.println("*** simulated failed execution ***"); + throw new RuntimeException("we failed with a simulated issue"); + } + + } + + /** + * Failed execution - fallback implementation successfully returns value. + */ + private static class KnownFailureTestCommandWithFallback extends TestHystrixCommand { + + public KnownFailureTestCommandWithFallback(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + } + + public KnownFailureTestCommandWithFallback(TestCircuitBreaker circuitBreaker, boolean fallbackEnabled) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withFallbackEnabled(fallbackEnabled).withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))); + } + + @Override + protected Observable run() { + // TODO duplicate with error inside async Observable + System.out.println("*** simulated failed execution ***"); + throw new RuntimeException("we failed with a simulated issue"); + } + + @Override + protected Observable getFallback() { + return Observable.from(false).subscribeOn(Schedulers.computation()); + } + } + + /** + * Failed execution - fallback implementation throws exception. + */ + private static class KnownFailureTestCommandWithFallbackFailure extends TestHystrixCommand { + + private KnownFailureTestCommandWithFallbackFailure() { + super(testPropsBuilder()); + } + + @Override + protected Observable run() { + System.out.println("*** simulated failed execution ***"); + throw new RuntimeException("we failed with a simulated issue"); + } + + @Override + protected Observable getFallback() { + // TODO duplicate with error inside async Observable + throw new RuntimeException("failed while getting fallback"); + } + } + + /** + * A Command implementation that supports caching. + */ + private static class SuccessfulCacheableCommand extends TestHystrixCommand { + + private final boolean cacheEnabled; + private volatile boolean executed = false; + private final String value; + + public SuccessfulCacheableCommand(TestCircuitBreaker circuitBreaker, boolean cacheEnabled, String value) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.THREAD))); + this.value = value; + this.cacheEnabled = cacheEnabled; + } + + @Override + protected Observable run() { + executed = true; + System.out.println("successfully executed"); + return Observable.from(value).subscribeOn(Schedulers.computation()); + } + + public boolean isCommandRunningInThread() { + return super.getProperties().executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD); + } + + @Override + public String getCacheKey() { + if (cacheEnabled) + return value; + else + return null; + } + } + + /** + * A Command implementation that supports caching. + */ + private static class SuccessfulCacheableCommandViaSemaphore extends TestHystrixCommand { + + private final boolean cacheEnabled; + private volatile boolean executed = false; + private final String value; + + public SuccessfulCacheableCommandViaSemaphore(TestCircuitBreaker circuitBreaker, boolean cacheEnabled, String value) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))); + this.value = value; + this.cacheEnabled = cacheEnabled; + } + + @Override + protected Observable run() { + executed = true; + System.out.println("successfully executed"); + return Observable.from(value).subscribeOn(Schedulers.computation()); + } + + public boolean isCommandRunningInThread() { + return super.getProperties().executionIsolationStrategy().get().equals(ExecutionIsolationStrategy.THREAD); + } + + @Override + public String getCacheKey() { + if (cacheEnabled) + return value; + else + return null; + } + } + + /** + * A Command implementation that supports caching and execution takes a while. + *

+ * Used to test scenario where Futures are returned with a backing call still executing. + */ + private static class SlowCacheableCommand extends TestHystrixCommand { + + private final String value; + private final int duration; + private volatile boolean executed = false; + + public SlowCacheableCommand(TestCircuitBreaker circuitBreaker, String value, int duration) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + this.value = value; + this.duration = duration; + } + + @Override + protected Observable run() { + executed = true; + return Observable.from(value).delay(duration, TimeUnit.MILLISECONDS).subscribeOn(Schedulers.computation()) + .doOnNext(new Action1() { + + @Override + public void call(String t1) { + System.out.println("successfully executed"); + } + + }); + } + + @Override + public String getCacheKey() { + return value; + } + } + + /** + * Successful execution - no fallback implementation, circuit-breaker disabled. + */ + private static class TestCommandWithoutCircuitBreaker extends TestHystrixCommand { + + private TestCommandWithoutCircuitBreaker() { + super(testPropsBuilder().setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE).withCircuitBreakerEnabled(false))); + } + + @Override + protected Observable run() { + System.out.println("successfully executed"); + return Observable.from(true).subscribeOn(Schedulers.computation()); + } + + } + + /** + * This should timeout. + */ + private static class TestCommandWithTimeout extends TestHystrixCommand { + + private final long timeout; + + private final static int FALLBACK_NOT_IMPLEMENTED = 1; + private final static int FALLBACK_SUCCESS = 2; + private final static int FALLBACK_FAILURE = 3; + + private final int fallbackBehavior; + + private TestCommandWithTimeout(long timeout, int fallbackBehavior) { + this(timeout, fallbackBehavior, ExecutionIsolationStrategy.SEMAPHORE); + } + + private TestCommandWithTimeout(long timeout, int fallbackBehavior, ExecutionIsolationStrategy isolationStrategy) { + super(testPropsBuilder().setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(isolationStrategy).withExecutionIsolationThreadTimeoutInMilliseconds((int) timeout))); + this.timeout = timeout; + this.fallbackBehavior = fallbackBehavior; + } + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber sub) { + System.out.println("***** running"); + try { + Thread.sleep(timeout * 10); + } catch (InterruptedException e) { + e.printStackTrace(); + // ignore and sleep some more to simulate a dependency that doesn't obey interrupts + try { + Thread.sleep(timeout * 2); + } catch (Exception e2) { + // ignore + } + System.out.println("after interruption with extra sleep"); + } + sub.onNext(true); + sub.onCompleted(); + } + + }).subscribeOn(Schedulers.computation()); + } + + @Override + protected Observable getFallback() { + if (fallbackBehavior == FALLBACK_SUCCESS) { + return Observable.from(false); + } else if (fallbackBehavior == FALLBACK_FAILURE) { + // TODO duplicate with error inside async Observable + throw new RuntimeException("failed on fallback"); + } else { + // FALLBACK_NOT_IMPLEMENTED + return super.getFallback(); + } + } + } + + private static class NoRequestCacheTimeoutWithoutFallback extends TestHystrixCommand { + public NoRequestCacheTimeoutWithoutFallback(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE).withExecutionIsolationThreadTimeoutInMilliseconds(200))); + + // we want it to timeout + } + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + System.out.println(">>>> Sleep Interrupted: " + e.getMessage()); + // e.printStackTrace(); + } + s.onNext(true); + s.onCompleted(); + } + + }).subscribeOn(Schedulers.computation()); + } + + @Override + public String getCacheKey() { + return null; + } + } + + /** + * The run() will take time. No fallback implementation. + */ + private static class TestSemaphoreCommand extends TestHystrixCommand { + + private final long executionSleep; + + private TestSemaphoreCommand(TestCircuitBreaker circuitBreaker, int executionSemaphoreCount, long executionSleep) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter() + .withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE) + .withExecutionIsolationSemaphoreMaxConcurrentRequests(executionSemaphoreCount))); + this.executionSleep = executionSleep; + } + + private TestSemaphoreCommand(TestCircuitBreaker circuitBreaker, TryableSemaphore semaphore, long executionSleep) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter() + .withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE)) + .setExecutionSemaphore(semaphore)); + this.executionSleep = executionSleep; + } + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + try { + Thread.sleep(executionSleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } + s.onNext(true); + s.onCompleted(); + } + + }).subscribeOn(Schedulers.computation()); + } + } + + /** + * Semaphore based command that allows caller to use latches to know when it has started and signal when it + * would like the command to finish + */ + private static class LatchedSemaphoreCommand extends TestHystrixCommand { + + private final CountDownLatch startLatch, waitLatch; + + /** + * + * @param circuitBreaker + * @param semaphore + * @param startLatch + * this command calls {@link java.util.concurrent.CountDownLatch#countDown()} immediately + * upon running + * @param waitLatch + * this command calls {@link java.util.concurrent.CountDownLatch#await()} once it starts + * to run. The caller can use the latch to signal the command to finish + */ + private LatchedSemaphoreCommand(TestCircuitBreaker circuitBreaker, TryableSemaphore semaphore, + CountDownLatch startLatch, CountDownLatch waitLatch) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE)) + .setExecutionSemaphore(semaphore)); + this.startLatch = startLatch; + this.waitLatch = waitLatch; + } + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + // signals caller that run has started + startLatch.countDown(); + + try { + // waits for caller to countDown latch + waitLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + s.onNext(false); + s.onCompleted(); + return; + } + + s.onNext(true); + s.onCompleted(); + } + + }).subscribeOn(Schedulers.computation()); + } + } + + /** + * The run() will take time. Contains fallback. + */ + private static class TestSemaphoreCommandWithFallback extends TestHystrixCommand { + + private final long executionSleep; + private final Observable fallback; + + private TestSemaphoreCommandWithFallback(TestCircuitBreaker circuitBreaker, int executionSemaphoreCount, long executionSleep, Boolean fallback) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE).withExecutionIsolationSemaphoreMaxConcurrentRequests(executionSemaphoreCount))); + this.executionSleep = executionSleep; + this.fallback = Observable.from(fallback); + } + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + try { + Thread.sleep(executionSleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + s.onNext(true); + s.onCompleted(); + } + + }).subscribeOn(Schedulers.computation()); + } + + @Override + protected Observable getFallback() { + return fallback; + } + + } + + private static class RequestCacheNullPointerExceptionCase extends TestHystrixCommand { + public RequestCacheNullPointerExceptionCase(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE).withExecutionIsolationThreadTimeoutInMilliseconds(200))); + // we want it to timeout + } + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + s.onNext(true); + s.onCompleted(); + } + + }).subscribeOn(Schedulers.computation()); + } + + @Override + protected Observable getFallback() { + return Observable.from(false).subscribeOn(Schedulers.computation()); + } + + @Override + public String getCacheKey() { + return "A"; + } + } + + private static class RequestCacheTimeoutWithoutFallback extends TestHystrixCommand { + public RequestCacheTimeoutWithoutFallback(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder().setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE).withExecutionIsolationThreadTimeoutInMilliseconds(200))); + // we want it to timeout + } + + @Override + protected Observable run() { + return Observable.create(new OnSubscribe() { + + @Override + public void call(Subscriber s) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + System.out.println(">>>> Sleep Interrupted: " + e.getMessage()); + // e.printStackTrace(); + } + + s.onNext(true); + s.onCompleted(); + } + + }).subscribeOn(Schedulers.computation()); + } + + @Override + public String getCacheKey() { + return "A"; + } + } + + private static class RequestCacheThreadRejectionWithoutFallback extends TestHystrixCommand { + + final CountDownLatch completionLatch; + + public RequestCacheThreadRejectionWithoutFallback(TestCircuitBreaker circuitBreaker, CountDownLatch completionLatch) { + super(testPropsBuilder() + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.THREAD)) + .setCircuitBreaker(circuitBreaker) + .setMetrics(circuitBreaker.metrics) + .setThreadPool(new HystrixThreadPool() { + + @Override + public ThreadPoolExecutor getExecutor() { + return null; + } + + @Override + public void markThreadExecution() { + + } + + @Override + public void markThreadCompletion() { + + } + + @Override + public boolean isQueueSpaceAvailable() { + // always return false so we reject everything + return false; + } + + @Override + public Scheduler getScheduler() { + return new HystrixContextScheduler(HystrixPlugins.getInstance().getConcurrencyStrategy(), this); + } + + })); + this.completionLatch = completionLatch; + } + + @Override + protected Observable run() { + try { + if (completionLatch.await(1000, TimeUnit.MILLISECONDS)) { + throw new RuntimeException("timed out waiting on completionLatch"); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return Observable.from(true); + } + + @Override + public String getCacheKey() { + return "A"; + } + } + + private static class BadRequestCommand extends TestHystrixCommand { + + public BadRequestCommand(TestCircuitBreaker circuitBreaker, ExecutionIsolationStrategy isolationType) { + super(testPropsBuilder() + .setCircuitBreaker(circuitBreaker) + .setCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE).withExecutionIsolationStrategy(isolationType))); + } + + @Override + protected Observable run() { + throw new HystrixBadRequestException("Message to developer that they passed in bad data or something like that."); + } + + @Override + protected Observable getFallback() { + return Observable.from(false).subscribeOn(Schedulers.computation()); + } + + @Override + protected String getCacheKey() { + return "one"; + } + + } + + private static class CommandWithErrorThrown extends TestHystrixCommand { + + public CommandWithErrorThrown(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder() + .setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + } + + @Override + protected Observable run() { + // TODO duplicate with error inside async Observable + throw new Error("simulated java.lang.Error message"); + } + + } + + private static class CommandWithCheckedException extends TestHystrixCommand { + + public CommandWithCheckedException(TestCircuitBreaker circuitBreaker) { + super(testPropsBuilder() + .setCircuitBreaker(circuitBreaker).setMetrics(circuitBreaker.metrics)); + } + + @Override + protected Observable run() { + return Observable.error(new IOException("simulated checked exception message")); + } + + } + + enum CommandKeyForUnitTest implements HystrixCommandKey { + KEY_ONE, KEY_TWO; + } + + enum CommandGroupForUnitTest implements HystrixCommandGroupKey { + OWNER_ONE, OWNER_TWO; + } + + enum ThreadPoolKeyForUnitTest implements HystrixThreadPoolKey { + THREAD_POOL_ONE, THREAD_POOL_TWO; + } + + private static HystrixPropertiesStrategy TEST_PROPERTIES_FACTORY = new TestPropertiesFactory(); + + private static class TestPropertiesFactory extends HystrixPropertiesStrategy { + + @Override + public HystrixCommandProperties getCommandProperties(HystrixCommandKey commandKey, HystrixCommandProperties.Setter builder) { + if (builder == null) { + builder = HystrixCommandPropertiesTest.getUnitTestPropertiesSetter(); + } + return HystrixCommandPropertiesTest.asMock(builder); + } + + @Override + public HystrixThreadPoolProperties getThreadPoolProperties(HystrixThreadPoolKey threadPoolKey, HystrixThreadPoolProperties.Setter builder) { + if (builder == null) { + builder = HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder(); + } + return HystrixThreadPoolProperties.Setter.asMock(builder); + } + + @Override + public HystrixCollapserProperties getCollapserProperties(HystrixCollapserKey collapserKey, HystrixCollapserProperties.Setter builder) { + throw new IllegalStateException("not expecting collapser properties"); + } + + @Override + public String getCommandPropertiesCacheKey(HystrixCommandKey commandKey, HystrixCommandProperties.Setter builder) { + return null; + } + + @Override + public String getThreadPoolPropertiesCacheKey(HystrixThreadPoolKey threadPoolKey, com.netflix.hystrix.HystrixThreadPoolProperties.Setter builder) { + return null; + } + + @Override + public String getCollapserPropertiesCacheKey(HystrixCollapserKey collapserKey, com.netflix.hystrix.HystrixCollapserProperties.Setter builder) { + return null; + } + + } + + private static class TestExecutionHook extends HystrixCommandExecutionHook { + + AtomicInteger startExecute = new AtomicInteger(); + + @Override + public void onStart(HystrixExecutable commandInstance) { + super.onStart(commandInstance); + startExecute.incrementAndGet(); + } + + Object endExecuteSuccessResponse = null; + + @Override + public T onComplete(HystrixExecutable commandInstance, T response) { + endExecuteSuccessResponse = response; + return super.onComplete(commandInstance, response); + } + + Exception endExecuteFailureException = null; + FailureType endExecuteFailureType = null; + + @Override + public Exception onError(HystrixExecutable commandInstance, FailureType failureType, Exception e) { + endExecuteFailureException = e; + endExecuteFailureType = failureType; + return super.onError(commandInstance, failureType, e); + } + + AtomicInteger startRun = new AtomicInteger(); + + @Override + public void onRunStart(HystrixExecutable commandInstance) { + super.onRunStart(commandInstance); + startRun.incrementAndGet(); + } + + Object runSuccessResponse = null; + + @Override + public T onRunSuccess(HystrixExecutable commandInstance, T response) { + runSuccessResponse = response; + return super.onRunSuccess(commandInstance, response); + } + + Exception runFailureException = null; + + @Override + public Exception onRunError(HystrixExecutable commandInstance, Exception e) { + runFailureException = e; + return super.onRunError(commandInstance, e); + } + + AtomicInteger startFallback = new AtomicInteger(); + + @Override + public void onFallbackStart(HystrixExecutable commandInstance) { + super.onFallbackStart(commandInstance); + startFallback.incrementAndGet(); + } + + Object fallbackSuccessResponse = null; + + @Override + public T onFallbackSuccess(HystrixExecutable commandInstance, T response) { + fallbackSuccessResponse = response; + return super.onFallbackSuccess(commandInstance, response); + } + + Exception fallbackFailureException = null; + + @Override + public Exception onFallbackError(HystrixExecutable commandInstance, Exception e) { + fallbackFailureException = e; + return super.onFallbackError(commandInstance, e); + } + + AtomicInteger threadStart = new AtomicInteger(); + + @Override + public void onThreadStart(HystrixExecutable commandInstance) { + super.onThreadStart(commandInstance); + threadStart.incrementAndGet(); + } + + AtomicInteger threadComplete = new AtomicInteger(); + + @Override + public void onThreadComplete(HystrixExecutable commandInstance) { + super.onThreadComplete(commandInstance); + threadComplete.incrementAndGet(); + } + + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestCacheTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestCacheTest.java new file mode 100644 index 000000000..fa3fc157c --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestCacheTest.java @@ -0,0 +1,84 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import rx.Observable; +import rx.Subscriber; + +import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; +import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategyDefault; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; + +public class HystrixRequestCacheTest { + + @Test + public void testCache() { + HystrixConcurrencyStrategy strategy = HystrixConcurrencyStrategyDefault.getInstance(); + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + try { + HystrixRequestCache cache1 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command1"), strategy); + cache1.putIfAbsent("valueA", new TestObservable("a1")); + cache1.putIfAbsent("valueA", new TestObservable("a2")); + cache1.putIfAbsent("valueB", new TestObservable("b1")); + + HystrixRequestCache cache2 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command2"), strategy); + cache2.putIfAbsent("valueA", new TestObservable("a3")); + + assertEquals("a1", cache1.get("valueA").toBlockingObservable().last()); + assertEquals("b1", cache1.get("valueB").toBlockingObservable().last()); + + assertEquals("a3", cache2.get("valueA").toBlockingObservable().last()); + assertNull(cache2.get("valueB")); + } catch (Exception e) { + fail("Exception: " + e.getMessage()); + e.printStackTrace(); + } finally { + context.shutdown(); + } + + context = HystrixRequestContext.initializeContext(); + try { + // with a new context the instance should have nothing in it + HystrixRequestCache cache = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command1"), strategy); + assertNull(cache.get("valueA")); + assertNull(cache.get("valueB")); + } finally { + context.shutdown(); + } + } + + @Test + public void testClearCache() { + HystrixConcurrencyStrategy strategy = HystrixConcurrencyStrategyDefault.getInstance(); + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + try { + HystrixRequestCache cache1 = HystrixRequestCache.getInstance(HystrixCommandKey.Factory.asKey("command1"), strategy); + cache1.putIfAbsent("valueA", new TestObservable("a1")); + assertEquals("a1", cache1.get("valueA").toBlockingObservable().last()); + cache1.clear("valueA"); + assertNull(cache1.get("valueA")); + } catch (Exception e) { + fail("Exception: " + e.getMessage()); + e.printStackTrace(); + } finally { + context.shutdown(); + } + } + + private static class TestObservable extends Observable { + public TestObservable(final String value) { + super(new OnSubscribe() { + + @Override + public void call(Subscriber observer) { + observer.onNext(value); + observer.onCompleted(); + } + + }); + } + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestLogTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestLogTest.java new file mode 100644 index 000000000..3763ba770 --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixRequestLogTest.java @@ -0,0 +1,190 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; + +public class HystrixRequestLogTest { + + private static final String DIGITS_REGEX = "\\[\\d+"; + + @Test + public void testSuccess() { + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + try { + new TestCommand("A", false, true).execute(); + String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); + // strip the actual count so we can compare reliably + log = log.replaceAll(DIGITS_REGEX, "["); + assertEquals("TestCommand[SUCCESS][ms]", log); + } finally { + context.shutdown(); + } + } + + @Test + public void testSuccessFromCache() { + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + try { + // 1 success + new TestCommand("A", false, true).execute(); + // 4 success from cache + new TestCommand("A", false, true).execute(); + new TestCommand("A", false, true).execute(); + new TestCommand("A", false, true).execute(); + new TestCommand("A", false, true).execute(); + String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); + // strip the actual count so we can compare reliably + log = log.replaceAll(DIGITS_REGEX, "["); + assertEquals("TestCommand[SUCCESS][ms], TestCommand[SUCCESS, RESPONSE_FROM_CACHE][ms]x4", log); + } finally { + context.shutdown(); + } + } + + @Test + public void testFailWithFallbackSuccess() { + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + try { + // 1 failure + new TestCommand("A", true, false).execute(); + // 4 failures from cache + new TestCommand("A", true, false).execute(); + new TestCommand("A", true, false).execute(); + new TestCommand("A", true, false).execute(); + new TestCommand("A", true, false).execute(); + String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); + // strip the actual count so we can compare reliably + log = log.replaceAll(DIGITS_REGEX, "["); + assertEquals("TestCommand[FAILURE, FALLBACK_SUCCESS][ms], TestCommand[FAILURE, FALLBACK_SUCCESS, RESPONSE_FROM_CACHE][ms]x4", log); + } finally { + context.shutdown(); + } + } + + @Test + public void testFailWithFallbackFailure() { + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + try { + // 1 failure + try { + new TestCommand("A", true, true).execute(); + } catch (Exception e) { + } + // 1 failure from cache + try { + new TestCommand("A", true, true).execute(); + } catch (Exception e) { + } + String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); + // strip the actual count so we can compare reliably + log = log.replaceAll(DIGITS_REGEX, "["); + assertEquals("TestCommand[FAILURE, FALLBACK_FAILURE][ms], TestCommand[FAILURE, FALLBACK_FAILURE, RESPONSE_FROM_CACHE][ms]", log); + } finally { + context.shutdown(); + } + } + + @Test + public void testMultipleCommands() { + + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + try { + + // 1 success + new TestCommand("GetData", "A", false, false).execute(); + + // 1 success + new TestCommand("PutData", "B", false, false).execute(); + + // 1 success + new TestCommand("GetValues", "C", false, false).execute(); + + // 1 success from cache + new TestCommand("GetValues", "C", false, false).execute(); + + // 1 failure + try { + new TestCommand("A", true, true).execute(); + } catch (Exception e) { + } + // 1 failure from cache + try { + new TestCommand("A", true, true).execute(); + } catch (Exception e) { + } + String log = HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString(); + // strip the actual count so we can compare reliably + log = log.replaceAll(DIGITS_REGEX, "["); + assertEquals("GetData[SUCCESS][ms], PutData[SUCCESS][ms], GetValues[SUCCESS][ms], GetValues[SUCCESS, RESPONSE_FROM_CACHE][ms], TestCommand[FAILURE, FALLBACK_FAILURE][ms], TestCommand[FAILURE, FALLBACK_FAILURE, RESPONSE_FROM_CACHE][ms]", log); + } finally { + context.shutdown(); + } + + } + + @Test + public void testMaxLimit() { + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + try { + for (int i = 0; i < HystrixRequestLog.MAX_STORAGE; i++) { + new TestCommand("A", false, true).execute(); + } + // then execute again some more + for (int i = 0; i < 10; i++) { + new TestCommand("A", false, true).execute(); + } + + assertEquals(HystrixRequestLog.MAX_STORAGE, HystrixRequestLog.getCurrentRequest().getExecutedCommands().size()); + } finally { + context.shutdown(); + } + } + + private static class TestCommand extends HystrixCommand { + + private final String value; + private final boolean fail; + private final boolean failOnFallback; + + public TestCommand(String commandName, String value, boolean fail, boolean failOnFallback) { + super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("RequestLogTestCommand")).andCommandKey(HystrixCommandKey.Factory.asKey(commandName))); + this.value = value; + this.fail = fail; + this.failOnFallback = failOnFallback; + } + + public TestCommand(String value, boolean fail, boolean failOnFallback) { + super(HystrixCommandGroupKey.Factory.asKey("RequestLogTestCommand")); + this.value = value; + this.fail = fail; + this.failOnFallback = failOnFallback; + } + + @Override + protected String run() { + if (fail) { + throw new RuntimeException("forced failure"); + } else { + return value; + } + } + + @Override + protected String getFallback() { + if (failOnFallback) { + throw new RuntimeException("forced fallback failure"); + } else { + return value + "-fallback"; + } + } + + @Override + protected String getCacheKey() { + return value; + } + + } +} \ No newline at end of file diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixTest.java new file mode 100644 index 000000000..9af48d5ea --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixTest.java @@ -0,0 +1,169 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.netflix.hystrix.HystrixCommand.Setter; +import com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy; + +public class HystrixTest { + @Test + public void testNotInThread() { + assertNull(Hystrix.getCurrentThreadExecutingCommand()); + } + + @Test + public void testInsideHystrixThread() { + + assertNull(Hystrix.getCurrentThreadExecutingCommand()); + + HystrixCommand command = new HystrixCommand(Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) + .andCommandKey(HystrixCommandKey.Factory.asKey("CommandName"))) { + + @Override + protected Boolean run() { + assertEquals("CommandName", Hystrix.getCurrentThreadExecutingCommand().name()); + + return Hystrix.getCurrentThreadExecutingCommand() != null; + } + + }; + + assertTrue(command.execute()); + assertNull(Hystrix.getCurrentThreadExecutingCommand()); + } + + @Test + public void testInsideNestedHystrixThread() { + + HystrixCommand command = new HystrixCommand(Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) + .andCommandKey(HystrixCommandKey.Factory.asKey("OuterCommand"))) { + + @Override + protected Boolean run() { + + assertEquals("OuterCommand", Hystrix.getCurrentThreadExecutingCommand().name()); + + if (Hystrix.getCurrentThreadExecutingCommand() == null) { + throw new RuntimeException("BEFORE expected it to run inside a thread"); + } + + HystrixCommand command2 = new HystrixCommand(Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) + .andCommandKey(HystrixCommandKey.Factory.asKey("InnerCommand"))) { + + @Override + protected Boolean run() { + assertEquals("InnerCommand", Hystrix.getCurrentThreadExecutingCommand().name()); + + return Hystrix.getCurrentThreadExecutingCommand() != null; + } + + }; + + if (Hystrix.getCurrentThreadExecutingCommand() == null) { + throw new RuntimeException("AFTER expected it to run inside a thread"); + } + + return command2.execute(); + } + + }; + + assertTrue(command.execute()); + + assertNull(Hystrix.getCurrentThreadExecutingCommand()); + } + + @Test + public void testInsideHystrixSemaphoreExecute() { + + HystrixCommand command = new HystrixCommand(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) + .andCommandKey(HystrixCommandKey.Factory.asKey("SemaphoreIsolatedCommandName")) + .andCommandPropertiesDefaults(HystrixCommandProperties.Setter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { + + @Override + protected Boolean run() { + assertEquals("SemaphoreIsolatedCommandName", Hystrix.getCurrentThreadExecutingCommand().name()); + + return Hystrix.getCurrentThreadExecutingCommand() != null; + } + + }; + + // it should be true for semaphore isolation as well + assertTrue(command.execute()); + // and then be null again once done + assertNull(Hystrix.getCurrentThreadExecutingCommand()); + } + + @Test + public void testInsideHystrixSemaphoreQueue() throws Exception { + + HystrixCommand command = new HystrixCommand(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) + .andCommandKey(HystrixCommandKey.Factory.asKey("SemaphoreIsolatedCommandName")) + .andCommandPropertiesDefaults(HystrixCommandProperties.Setter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { + + @Override + protected Boolean run() { + assertEquals("SemaphoreIsolatedCommandName", Hystrix.getCurrentThreadExecutingCommand().name()); + + return Hystrix.getCurrentThreadExecutingCommand() != null; + } + + }; + + // it should be true for semaphore isolation as well + assertTrue(command.queue().get()); + // and then be null again once done + assertNull(Hystrix.getCurrentThreadExecutingCommand()); + } + + @Test + public void testThreadNestedInsideHystrixSemaphore() { + + HystrixCommand command = new HystrixCommand(Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) + .andCommandKey(HystrixCommandKey.Factory.asKey("OuterSemaphoreCommand")) + .andCommandPropertiesDefaults(HystrixCommandProperties.Setter().withExecutionIsolationStrategy(ExecutionIsolationStrategy.SEMAPHORE))) { + + @Override + protected Boolean run() { + + assertEquals("OuterSemaphoreCommand", Hystrix.getCurrentThreadExecutingCommand().name()); + + if (Hystrix.getCurrentThreadExecutingCommand() == null) { + throw new RuntimeException("BEFORE expected it to run inside a semaphore"); + } + + HystrixCommand command2 = new HystrixCommand(Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TestUtil")) + .andCommandKey(HystrixCommandKey.Factory.asKey("InnerCommand"))) { + + @Override + protected Boolean run() { + assertEquals("InnerCommand", Hystrix.getCurrentThreadExecutingCommand().name()); + + return Hystrix.getCurrentThreadExecutingCommand() != null; + } + + }; + + if (Hystrix.getCurrentThreadExecutingCommand() == null) { + throw new RuntimeException("AFTER expected it to run inside a semaphore"); + } + + return command2.execute(); + } + + }; + + assertTrue(command.execute()); + + assertNull(Hystrix.getCurrentThreadExecutingCommand()); + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixThreadPoolTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixThreadPoolTest.java new file mode 100644 index 000000000..df8d66076 --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixThreadPoolTest.java @@ -0,0 +1,48 @@ +package com.netflix.hystrix; + +import static org.junit.Assert.*; + +import java.util.concurrent.TimeUnit; + +import org.junit.Test; + +import com.netflix.hystrix.HystrixThreadPool.Factory; + +public class HystrixThreadPoolTest { + + @Test + public void testShutdown() { + // other unit tests will probably have run before this so get the count + int count = Factory.threadPools.size(); + + HystrixThreadPool pool = Factory.getInstance(HystrixThreadPoolKey.Factory.asKey("threadPoolFactoryTest"), + HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder()); + + assertEquals(count + 1, Factory.threadPools.size()); + assertFalse(pool.getExecutor().isShutdown()); + + Factory.shutdown(); + + // ensure all pools were removed from the cache + assertEquals(0, Factory.threadPools.size()); + assertTrue(pool.getExecutor().isShutdown()); + } + + @Test + public void testShutdownWithWait() { + // other unit tests will probably have run before this so get the count + int count = Factory.threadPools.size(); + + HystrixThreadPool pool = Factory.getInstance(HystrixThreadPoolKey.Factory.asKey("threadPoolFactoryTest"), + HystrixThreadPoolProperties.Setter.getUnitTestPropertiesBuilder()); + + assertEquals(count + 1, Factory.threadPools.size()); + assertFalse(pool.getExecutor().isShutdown()); + + Factory.shutdown(1, TimeUnit.SECONDS); + + // ensure all pools were removed from the cache + assertEquals(0, Factory.threadPools.size()); + assertTrue(pool.getExecutor().isShutdown()); + } +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunctionTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunctionTest.java new file mode 100644 index 000000000..b340d5fba --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/collapser/CollapsedRequestObservableFunctionTest.java @@ -0,0 +1,165 @@ +package com.netflix.hystrix.collapser; + +import static org.junit.Assert.*; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import org.junit.Test; + +import rx.Observable; + +public class CollapsedRequestObservableFunctionTest { + @Test + public void testSetResponseSuccess() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setResponse("theResponse"); + + // fetch value + assertEquals("theResponse", v.get()); + } + + @Test + public void testSetNullResponseSuccess() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setResponse(null); + + // fetch value + assertEquals(null, v.get()); + } + + @Test + public void testSetException() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setException(new RuntimeException("anException")); + + // fetch value + try { + v.get(); + fail("expected exception"); + } catch (ExecutionException e) { + assertEquals("anException", e.getCause().getMessage()); + } + } + + @Test + public void testSetExceptionAfterResponse() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setResponse("theResponse"); + + try { + cr.setException(new RuntimeException("anException")); + fail("expected IllegalState"); + } catch (IllegalStateException e) { + + } + + assertEquals("theResponse", v.get()); + } + + @Test + public void testSetResponseAfterException() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setException(new RuntimeException("anException")); + + try { + cr.setResponse("theResponse"); + fail("expected IllegalState"); + } catch (IllegalStateException e) { + + } + + try { + v.get(); + fail("expected exception"); + } catch (ExecutionException e) { + assertEquals("anException", e.getCause().getMessage()); + } + } + + @Test + public void testSetResponseDuplicate() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setResponse("theResponse"); + + try { + cr.setResponse("theResponse2"); + fail("expected IllegalState"); + } catch (IllegalStateException e) { + + } + + assertEquals("theResponse", v.get()); + } + + @Test + public void testSetResponseAfterUnsubscribe() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future f = o.toBlockingObservable().toFuture(); + + // cancel/unsubscribe + f.cancel(true); + + try { + cr.setResponse("theResponse"); + } catch (IllegalStateException e) { + fail("this should have done nothing as it was unsubscribed already"); + } + + // if you fetch after canceling it should be null + assertEquals(null, f.get()); + } + + @Test + public void testSetExceptionAfterUnsubscribe() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future f = o.toBlockingObservable().toFuture(); + + // cancel/unsubscribe + f.cancel(true); + + try { + cr.setException(new RuntimeException("anException")); + } catch (IllegalStateException e) { + fail("this should have done nothing as it was unsubscribed already"); + } + + // if you fetch after canceling it should be null + assertEquals(null, f.get()); + } + + @Test + public void testUnsubscribeAfterSetResponse() throws InterruptedException, ExecutionException { + CollapsedRequestObservableFunction cr = new CollapsedRequestObservableFunction("hello"); + Observable o = Observable.create(cr); + Future v = o.toBlockingObservable().toFuture(); + + cr.setResponse("theResponse"); + + // unsubscribe after the value is sent + v.cancel(true); + + // still get value as it was set before canceling + assertEquals("theResponse", v.get()); + } +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/strategy/HystrixPluginsTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/HystrixPluginsTest.java new file mode 100644 index 000000000..99d4d46aa --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/HystrixPluginsTest.java @@ -0,0 +1,225 @@ +package com.netflix.hystrix.strategy; + +import static org.junit.Assert.*; + +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.After; +import org.junit.Test; + +import rx.functions.Action1; + +import com.netflix.hystrix.Hystrix; +import com.netflix.hystrix.HystrixCommand; +import com.netflix.hystrix.HystrixCommandGroupKey; +import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; +import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategyDefault; +import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; +import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifier; +import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifierDefault; +import com.netflix.hystrix.strategy.metrics.HystrixMetricsPublisher; +import com.netflix.hystrix.strategy.metrics.HystrixMetricsPublisherDefault; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategyDefault; + +public class HystrixPluginsTest { + @After + public void reset() { + // use private access to reset so we can test different initializations via the public static flow + HystrixPlugins.getInstance().concurrencyStrategy.set(null); + HystrixPlugins.getInstance().metricsPublisher.set(null); + HystrixPlugins.getInstance().notifier.set(null); + HystrixPlugins.getInstance().propertiesFactory.set(null); + } + + @Test + public void testEventNotifierDefaultImpl() { + HystrixEventNotifier impl = HystrixPlugins.getInstance().getEventNotifier(); + assertTrue(impl instanceof HystrixEventNotifierDefault); + } + + @Test + public void testEventNotifierViaRegisterMethod() { + HystrixPlugins.getInstance().registerEventNotifier(new HystrixEventNotifierTestImpl()); + HystrixEventNotifier impl = HystrixPlugins.getInstance().getEventNotifier(); + assertTrue(impl instanceof HystrixEventNotifierTestImpl); + } + + @Test + public void testEventNotifierViaProperty() { + try { + String fullClass = HystrixEventNotifierTestImpl.class.getName(); + System.setProperty("hystrix.plugin.HystrixEventNotifier.implementation", fullClass); + HystrixEventNotifier impl = HystrixPlugins.getInstance().getEventNotifier(); + assertTrue(impl instanceof HystrixEventNotifierTestImpl); + } finally { + System.clearProperty("hystrix.plugin.HystrixEventNotifier.implementation"); + } + } + + // inside UnitTest so it is stripped from Javadocs + public static class HystrixEventNotifierTestImpl extends HystrixEventNotifier { + // just use defaults + } + + @Test + public void testConcurrencyStrategyDefaultImpl() { + HystrixConcurrencyStrategy impl = HystrixPlugins.getInstance().getConcurrencyStrategy(); + assertTrue(impl instanceof HystrixConcurrencyStrategyDefault); + } + + @Test + public void testConcurrencyStrategyViaRegisterMethod() { + HystrixPlugins.getInstance().registerConcurrencyStrategy(new HystrixConcurrencyStrategyTestImpl()); + HystrixConcurrencyStrategy impl = HystrixPlugins.getInstance().getConcurrencyStrategy(); + assertTrue(impl instanceof HystrixConcurrencyStrategyTestImpl); + } + + @Test + public void testConcurrencyStrategyViaProperty() { + try { + String fullClass = HystrixConcurrencyStrategyTestImpl.class.getName(); + System.setProperty("hystrix.plugin.HystrixConcurrencyStrategy.implementation", fullClass); + HystrixConcurrencyStrategy impl = HystrixPlugins.getInstance().getConcurrencyStrategy(); + assertTrue(impl instanceof HystrixConcurrencyStrategyTestImpl); + } finally { + System.clearProperty("hystrix.plugin.HystrixConcurrencyStrategy.implementation"); + } + } + + // inside UnitTest so it is stripped from Javadocs + public static class HystrixConcurrencyStrategyTestImpl extends HystrixConcurrencyStrategy { + // just use defaults + } + + @Test + public void testMetricsPublisherDefaultImpl() { + HystrixMetricsPublisher impl = HystrixPlugins.getInstance().getMetricsPublisher(); + assertTrue(impl instanceof HystrixMetricsPublisherDefault); + } + + @Test + public void testMetricsPublisherViaRegisterMethod() { + HystrixPlugins.getInstance().registerMetricsPublisher(new HystrixMetricsPublisherTestImpl()); + HystrixMetricsPublisher impl = HystrixPlugins.getInstance().getMetricsPublisher(); + assertTrue(impl instanceof HystrixMetricsPublisherTestImpl); + } + + @Test + public void testMetricsPublisherViaProperty() { + try { + String fullClass = HystrixMetricsPublisherTestImpl.class.getName(); + System.setProperty("hystrix.plugin.HystrixMetricsPublisher.implementation", fullClass); + HystrixMetricsPublisher impl = HystrixPlugins.getInstance().getMetricsPublisher(); + assertTrue(impl instanceof HystrixMetricsPublisherTestImpl); + } finally { + System.clearProperty("hystrix.plugin.HystrixMetricsPublisher.implementation"); + } + } + + // inside UnitTest so it is stripped from Javadocs + public static class HystrixMetricsPublisherTestImpl extends HystrixMetricsPublisher { + // just use defaults + } + + @Test + public void testPropertiesStrategyDefaultImpl() { + HystrixPropertiesStrategy impl = HystrixPlugins.getInstance().getPropertiesStrategy(); + assertTrue(impl instanceof HystrixPropertiesStrategyDefault); + } + + @Test + public void testPropertiesStrategyViaRegisterMethod() { + HystrixPlugins.getInstance().registerPropertiesStrategy(new HystrixPropertiesStrategyTestImpl()); + HystrixPropertiesStrategy impl = HystrixPlugins.getInstance().getPropertiesStrategy(); + assertTrue(impl instanceof HystrixPropertiesStrategyTestImpl); + } + + @Test + public void testPropertiesStrategyViaProperty() { + try { + String fullClass = HystrixPropertiesStrategyTestImpl.class.getName(); + System.setProperty("hystrix.plugin.HystrixPropertiesStrategy.implementation", fullClass); + HystrixPropertiesStrategy impl = HystrixPlugins.getInstance().getPropertiesStrategy(); + assertTrue(impl instanceof HystrixPropertiesStrategyTestImpl); + } finally { + System.clearProperty("hystrix.plugin.HystrixPropertiesStrategy.implementation"); + } + } + + // inside UnitTest so it is stripped from Javadocs + public static class HystrixPropertiesStrategyTestImpl extends HystrixPropertiesStrategy { + // just use defaults + } + + @Test + public void testRequestContextViaPluginInTimeout() { + HystrixPlugins.getInstance().registerConcurrencyStrategy(new HystrixConcurrencyStrategy() { + @Override + public Callable wrapCallable(final Callable callable) { + return new RequestIdCallable(callable); + } + }); + + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + + testRequestIdThreadLocal.set("foobar"); + final AtomicReference valueInTimeout = new AtomicReference(); + + new DummyCommand().toObservable() + .doOnError(new Action1() { + @Override + public void call(Throwable throwable) { + System.out.println("initialized = " + HystrixRequestContext.isCurrentThreadInitialized()); + System.out.println("requestId (timeout) = " + testRequestIdThreadLocal.get()); + valueInTimeout.set(testRequestIdThreadLocal.get()); + } + }) + .materialize() + .toBlockingObservable().single(); + + context.shutdown(); + Hystrix.reset(); + + assertEquals("foobar", valueInTimeout.get()); + } + + private static class RequestIdCallable implements Callable { + private final Callable callable; + private final String requestId; + + public RequestIdCallable(Callable callable) { + this.callable = callable; + this.requestId = testRequestIdThreadLocal.get(); + } + + @Override + public T call() throws Exception { + String original = testRequestIdThreadLocal.get(); + testRequestIdThreadLocal.set(requestId); + try { + return callable.call(); + } finally { + testRequestIdThreadLocal.set(original); + } + } + } + + private static final ThreadLocal testRequestIdThreadLocal = new ThreadLocal(); + + public static class DummyCommand extends HystrixCommand { + + public DummyCommand() { + super(HystrixCommandGroupKey.Factory.asKey("Dummy")); + } + + @Override + protected Void run() throws Exception { + System.out.println("requestId (run) = " + testRequestIdThreadLocal.get()); + Thread.sleep(2000); + return null; + } + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/strategy/concurrency/HystrixConcurrencyStrategyTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/concurrency/HystrixConcurrencyStrategyTest.java new file mode 100644 index 000000000..f00cdb4ba --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/concurrency/HystrixConcurrencyStrategyTest.java @@ -0,0 +1,111 @@ +package com.netflix.hystrix.strategy.concurrency; + +import static org.junit.Assert.*; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import rx.functions.Action1; +import rx.functions.Func1; + +import com.netflix.config.ConfigurationManager; +import com.netflix.hystrix.HystrixCommand; +import com.netflix.hystrix.HystrixCommandGroupKey; +import com.netflix.hystrix.HystrixCommandProperties; +import com.netflix.hystrix.HystrixRequestLog; + +public class HystrixConcurrencyStrategyTest { + @Before + public void prepareForTest() { + /* we must call this to simulate a new request lifecycle running and clearing caches */ + HystrixRequestContext.initializeContext(); + } + + @After + public void cleanup() { + // instead of storing the reference from initialize we'll just get the current state and shutdown + if (HystrixRequestContext.getContextForCurrentThread() != null) { + // it could have been set NULL by the test + HystrixRequestContext.getContextForCurrentThread().shutdown(); + } + + // force properties to be clean as well + ConfigurationManager.getConfigInstance().clear(); + } + + /** + * If the RequestContext does not get transferred across threads correctly this blows up. + * No specific assertions are necessary. + */ + @Test + public void testRequestContextPropagatesAcrossObserveOnPool() { + new SimpleCommand().execute(); + new SimpleCommand().observe().map(new Func1() { + + @Override + public String call(String s) { + System.out.println("Map => Commands: " + HystrixRequestLog.getCurrentRequest().getExecutedCommands()); + return s; + } + }).toBlockingObservable().forEach(new Action1() { + + @Override + public void call(String s) { + System.out.println("Result [" + s + "] => Commands: " + HystrixRequestLog.getCurrentRequest().getExecutedCommands()); + } + }); + } + + private static class SimpleCommand extends HystrixCommand { + + public SimpleCommand() { + super(HystrixCommandGroupKey.Factory.asKey("SimpleCommand")); + } + + @Override + protected String run() throws Exception { + System.out.println("Executing => Commands: " + HystrixRequestLog.getCurrentRequest().getAllExecutedCommands()); + return "Hello"; + } + + } + + @Test + public void testThreadContextOnTimeout() { + final AtomicBoolean isInitialized = new AtomicBoolean(); + new TimeoutCommand().toObservable() + .doOnError(new Action1() { + @Override + public void call(Throwable throwable) { + isInitialized.set(HystrixRequestContext.isCurrentThreadInitialized()); + } + }) + .materialize() + .toBlockingObservable().single(); + + System.out.println("initialized = " + HystrixRequestContext.isCurrentThreadInitialized()); + System.out.println("initialized inside onError = " + isInitialized.get()); + assertEquals(true, isInitialized.get()); + } + + public static class TimeoutCommand extends HystrixCommand { + static final HystrixCommand.Setter properties = HystrixCommand.Setter + .withGroupKey(HystrixCommandGroupKey.Factory.asKey("TimeoutTest")) + .andCommandPropertiesDefaults(HystrixCommandProperties.Setter() + .withExecutionIsolationThreadTimeoutInMilliseconds(50)); + + public TimeoutCommand() { + super(properties); + } + + @Override + protected Void run() throws Exception { + Thread.sleep(500); + return null; + } + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/strategy/metrics/HystrixMetricsPublisherFactoryTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/metrics/HystrixMetricsPublisherFactoryTest.java new file mode 100644 index 000000000..352d7479a --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/metrics/HystrixMetricsPublisherFactoryTest.java @@ -0,0 +1,94 @@ +package com.netflix.hystrix.strategy.metrics; + +import static org.junit.Assert.*; + +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Test; + +import com.netflix.hystrix.HystrixCircuitBreaker; +import com.netflix.hystrix.HystrixCommandGroupKey; +import com.netflix.hystrix.HystrixCommandKey; +import com.netflix.hystrix.HystrixCommandMetrics; +import com.netflix.hystrix.HystrixCommandProperties; +import com.netflix.hystrix.HystrixThreadPoolKey; +import com.netflix.hystrix.HystrixThreadPoolMetrics; +import com.netflix.hystrix.HystrixThreadPoolProperties; + +public class HystrixMetricsPublisherFactoryTest { + /** + * Assert that we only call a publisher once for a given Command or ThreadPool key. + */ + @Test + public void testSingleInitializePerKey() { + final TestHystrixMetricsPublisher publisher = new TestHystrixMetricsPublisher(); + final HystrixMetricsPublisherFactory factory = new HystrixMetricsPublisherFactory(publisher); + ArrayList threads = new ArrayList(); + for (int i = 0; i < 20; i++) { + threads.add(new Thread(new Runnable() { + + @Override + public void run() { + factory.getPublisherForCommand(TestCommandKey.TEST_A, null, null, null, null); + factory.getPublisherForCommand(TestCommandKey.TEST_B, null, null, null, null); + factory.getPublisherForThreadPool(TestThreadPoolKey.TEST_A, null, null); + } + + })); + } + + // start them + for (Thread t : threads) { + t.start(); + } + + // wait for them to finish + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + // we should see 2 commands and 1 threadPool publisher created + assertEquals(2, publisher.commandCounter.get()); + assertEquals(1, publisher.threadCounter.get()); + } + + private static class TestHystrixMetricsPublisher extends HystrixMetricsPublisher { + + AtomicInteger commandCounter = new AtomicInteger(); + AtomicInteger threadCounter = new AtomicInteger(); + + @Override + public HystrixMetricsPublisherCommand getMetricsPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, HystrixCircuitBreaker circuitBreaker, HystrixCommandProperties properties) { + return new HystrixMetricsPublisherCommand() { + @Override + public void initialize() { + commandCounter.incrementAndGet(); + } + }; + } + + @Override + public HystrixMetricsPublisherThreadPool getMetricsPublisherForThreadPool(HystrixThreadPoolKey threadPoolKey, HystrixThreadPoolMetrics metrics, HystrixThreadPoolProperties properties) { + return new HystrixMetricsPublisherThreadPool() { + @Override + public void initialize() { + threadCounter.incrementAndGet(); + } + }; + } + + } + + private static enum TestCommandKey implements HystrixCommandKey { + TEST_A, TEST_B; + } + + private static enum TestThreadPoolKey implements HystrixThreadPoolKey { + TEST_A, TEST_B; + } +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/strategy/properties/HystrixPropertiesChainedArchaiusPropertyTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/properties/HystrixPropertiesChainedArchaiusPropertyTest.java new file mode 100644 index 000000000..b9ec9c87c --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/properties/HystrixPropertiesChainedArchaiusPropertyTest.java @@ -0,0 +1,212 @@ +package com.netflix.hystrix.strategy.properties; + +import static org.junit.Assert.*; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.After; +import org.junit.Test; + +import com.netflix.config.ConfigurationManager; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty.DynamicBooleanProperty; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty.DynamicIntegerProperty; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty.DynamicStringProperty; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty.IntegerProperty; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesChainedArchaiusProperty.StringProperty; + +public class HystrixPropertiesChainedArchaiusPropertyTest { + @After + public void cleanUp() { + // Tests which use ConfigurationManager.getConfigInstance() will leave the singleton in an initialize state, + // this will leave the singleton in a reasonable state between tests. + ConfigurationManager.getConfigInstance().clear(); + } + + @Test + public void testString() throws Exception { + + DynamicStringProperty pString = new DynamicStringProperty("defaultString", "default-default"); + HystrixPropertiesChainedArchaiusProperty.StringProperty fString = new HystrixPropertiesChainedArchaiusProperty.StringProperty("overrideString", pString); + + assertTrue("default-default".equals(fString.get())); + + ConfigurationManager.getConfigInstance().setProperty("defaultString", "default"); + assertTrue("default".equals(fString.get())); + + ConfigurationManager.getConfigInstance().setProperty("overrideString", "override"); + assertTrue("override".equals(fString.get())); + + ConfigurationManager.getConfigInstance().clearProperty("overrideString"); + assertTrue("default".equals(fString.get())); + + ConfigurationManager.getConfigInstance().clearProperty("defaultString"); + assertTrue("default-default".equals(fString.get())); + } + + @Test + public void testInteger() throws Exception { + + DynamicIntegerProperty pInt = new DynamicIntegerProperty("defaultInt", -1); + HystrixPropertiesChainedArchaiusProperty.IntegerProperty fInt = new HystrixPropertiesChainedArchaiusProperty.IntegerProperty("overrideInt", pInt); + + assertTrue(-1 == fInt.get()); + + ConfigurationManager.getConfigInstance().setProperty("defaultInt", 10); + assertTrue(10 == fInt.get()); + + ConfigurationManager.getConfigInstance().setProperty("overrideInt", 11); + assertTrue(11 == fInt.get()); + + ConfigurationManager.getConfigInstance().clearProperty("overrideInt"); + assertTrue(10 == fInt.get()); + + ConfigurationManager.getConfigInstance().clearProperty("defaultInt"); + assertTrue(-1 == fInt.get()); + } + + @Test + public void testBoolean() throws Exception { + + DynamicBooleanProperty pBoolean = new DynamicBooleanProperty("defaultBoolean", true); + HystrixPropertiesChainedArchaiusProperty.BooleanProperty fBoolean = new HystrixPropertiesChainedArchaiusProperty.BooleanProperty("overrideBoolean", pBoolean); + + System.out.println("pBoolean: " + pBoolean.get()); + System.out.println("fBoolean: " + fBoolean.get()); + + assertTrue(fBoolean.get()); + + ConfigurationManager.getConfigInstance().setProperty("defaultBoolean", Boolean.FALSE); + + System.out.println("pBoolean: " + pBoolean.get()); + System.out.println("fBoolean: " + fBoolean.get()); + + assertFalse(fBoolean.get()); + + ConfigurationManager.getConfigInstance().setProperty("overrideBoolean", Boolean.TRUE); + assertTrue(fBoolean.get()); + + ConfigurationManager.getConfigInstance().clearProperty("overrideBoolean"); + assertFalse(fBoolean.get()); + + ConfigurationManager.getConfigInstance().clearProperty("defaultBoolean"); + assertTrue(fBoolean.get()); + } + + @Test + public void testChainingString() throws Exception { + + DynamicStringProperty node1 = new DynamicStringProperty("node1", "v1"); + StringProperty node2 = new HystrixPropertiesChainedArchaiusProperty.StringProperty("node2", node1); + + HystrixPropertiesChainedArchaiusProperty.StringProperty node3 = new HystrixPropertiesChainedArchaiusProperty.StringProperty("node3", node2); + + assertTrue("" + node3.get(), "v1".equals(node3.get())); + + ConfigurationManager.getConfigInstance().setProperty("node1", "v11"); + assertTrue("v11".equals(node3.get())); + + ConfigurationManager.getConfigInstance().setProperty("node2", "v22"); + assertTrue("v22".equals(node3.get())); + + ConfigurationManager.getConfigInstance().clearProperty("node1"); + assertTrue("v22".equals(node3.get())); + + ConfigurationManager.getConfigInstance().setProperty("node3", "v33"); + assertTrue("v33".equals(node3.get())); + + ConfigurationManager.getConfigInstance().clearProperty("node2"); + assertTrue("v33".equals(node3.get())); + + ConfigurationManager.getConfigInstance().setProperty("node2", "v222"); + assertTrue("v33".equals(node3.get())); + + ConfigurationManager.getConfigInstance().clearProperty("node3"); + assertTrue("v222".equals(node3.get())); + + ConfigurationManager.getConfigInstance().clearProperty("node2"); + assertTrue("v1".equals(node3.get())); + + ConfigurationManager.getConfigInstance().setProperty("node2", "v2222"); + assertTrue("v2222".equals(node3.get())); + } + + @Test + public void testChainingInteger() throws Exception { + + DynamicIntegerProperty node1 = new DynamicIntegerProperty("node1", 1); + IntegerProperty node2 = new HystrixPropertiesChainedArchaiusProperty.IntegerProperty("node2", node1); + + HystrixPropertiesChainedArchaiusProperty.IntegerProperty node3 = new HystrixPropertiesChainedArchaiusProperty.IntegerProperty("node3", node2); + + assertTrue("" + node3.get(), 1 == node3.get()); + + ConfigurationManager.getConfigInstance().setProperty("node1", 11); + assertTrue(11 == node3.get()); + + ConfigurationManager.getConfigInstance().setProperty("node2", 22); + assertTrue(22 == node3.get()); + + ConfigurationManager.getConfigInstance().clearProperty("node1"); + assertTrue(22 == node3.get()); + + ConfigurationManager.getConfigInstance().setProperty("node3", 33); + assertTrue(33 == node3.get()); + + ConfigurationManager.getConfigInstance().clearProperty("node2"); + assertTrue(33 == node3.get()); + + ConfigurationManager.getConfigInstance().setProperty("node2", 222); + assertTrue(33 == node3.get()); + + ConfigurationManager.getConfigInstance().clearProperty("node3"); + assertTrue(222 == node3.get()); + + ConfigurationManager.getConfigInstance().clearProperty("node2"); + assertTrue(1 == node3.get()); + + ConfigurationManager.getConfigInstance().setProperty("node2", 2222); + assertTrue(2222 == node3.get()); + } + + @Test + public void testAddCallback() throws Exception { + + final DynamicStringProperty node1 = new DynamicStringProperty("n1", "n1"); + final HystrixPropertiesChainedArchaiusProperty.StringProperty node2 = new HystrixPropertiesChainedArchaiusProperty.StringProperty("n2", node1); + + final AtomicInteger callbackCount = new AtomicInteger(0); + + node2.addCallback(new Runnable() { + @Override + public void run() { + callbackCount.incrementAndGet(); + } + }); + + assertTrue(0 == callbackCount.get()); + + assertTrue("n1".equals(node2.get())); + assertTrue(0 == callbackCount.get()); + + ConfigurationManager.getConfigInstance().setProperty("n1", "n11"); + assertTrue("n11".equals(node2.get())); + assertTrue(0 == callbackCount.get()); + + ConfigurationManager.getConfigInstance().setProperty("n2", "n22"); + assertTrue("n22".equals(node2.get())); + assertTrue(1 == callbackCount.get()); + + ConfigurationManager.getConfigInstance().clearProperty("n1"); + assertTrue("n22".equals(node2.get())); + assertTrue(1 == callbackCount.get()); + + ConfigurationManager.getConfigInstance().setProperty("n2", "n222"); + assertTrue("n222".equals(node2.get())); + assertTrue(2 == callbackCount.get()); + + ConfigurationManager.getConfigInstance().clearProperty("n2"); + assertTrue("n1".equals(node2.get())); + assertTrue(3 == callbackCount.get()); + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/strategy/properties/HystrixPropertyTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/properties/HystrixPropertyTest.java new file mode 100644 index 000000000..eb9e987cb --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/strategy/properties/HystrixPropertyTest.java @@ -0,0 +1,76 @@ +package com.netflix.hystrix.strategy.properties; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.netflix.hystrix.strategy.properties.HystrixProperty.Factory; + +public class HystrixPropertyTest { + + @Test + public void testNested1() { + HystrixProperty a = Factory.asProperty("a"); + assertEquals("a", a.get()); + + HystrixProperty aWithDefault = Factory.asProperty(a, "b"); + assertEquals("a", aWithDefault.get()); + } + + @Test + public void testNested2() { + HystrixProperty nullValue = Factory.nullProperty(); + + HystrixProperty withDefault = Factory.asProperty(nullValue, "b"); + assertEquals("b", withDefault.get()); + } + + @Test + public void testNested3() { + HystrixProperty nullValue = Factory.nullProperty(); + HystrixProperty a = Factory.asProperty(nullValue, "a"); + + HystrixProperty withDefault = Factory.asProperty(a, "b"); + assertEquals("a", withDefault.get()); + } + + @Test + public void testNested4() { + HystrixProperty nullValue = Factory.nullProperty(); + HystrixProperty a = Factory.asProperty(nullValue, null); + + HystrixProperty withDefault = Factory.asProperty(a, "b"); + assertEquals("b", withDefault.get()); + } + + @Test + public void testNested5() { + HystrixProperty nullValue = Factory.nullProperty(); + HystrixProperty a = Factory.asProperty(nullValue, null); + + @SuppressWarnings("unchecked") + HystrixProperty withDefault = Factory.asProperty(a, Factory.asProperty("b")); + assertEquals("b", withDefault.get()); + } + + @Test + public void testSeries1() { + HystrixProperty nullValue = Factory.nullProperty(); + HystrixProperty a = Factory.asProperty(nullValue, null); + + @SuppressWarnings("unchecked") + HystrixProperty withDefault = Factory.asProperty(a, nullValue, nullValue, Factory.asProperty("b")); + assertEquals("b", withDefault.get()); + } + + @Test + public void testSeries2() { + HystrixProperty nullValue = Factory.nullProperty(); + HystrixProperty a = Factory.asProperty(nullValue, null); + + @SuppressWarnings("unchecked") + HystrixProperty withDefault = Factory.asProperty(a, nullValue, Factory.asProperty("b"), nullValue, Factory.asProperty("c")); + assertEquals("b", withDefault.get()); + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/util/ExceptionThreadingUtilityTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/util/ExceptionThreadingUtilityTest.java new file mode 100644 index 000000000..6aa68e84d --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/util/ExceptionThreadingUtilityTest.java @@ -0,0 +1,52 @@ +package com.netflix.hystrix.util; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class ExceptionThreadingUtilityTest { + private final Throwable ex1 = new Throwable("Ex1"); + private final Throwable ex2 = new Throwable("Ex2", ex1); + + public ExceptionThreadingUtilityTest() { + ex1.initCause(ex2); + } + + @Test + public void testAttachCallingThreadStackParentThenChild() { + ExceptionThreadingUtility.attachCallingThreadStack(ex1, ex2.getStackTrace()); + assertEquals("Ex1", ex1.getMessage()); + assertEquals("Ex2", ex1.getCause().getMessage()); + assertEquals("Ex2", ex2.getMessage()); + assertEquals("Ex1", ex2.getCause().getMessage()); + } + + @Test + public void testAttachCallingThreadStackChildThenParent() { + ExceptionThreadingUtility.attachCallingThreadStack(ex2, ex1.getStackTrace()); + assertEquals("Ex1", ex1.getMessage()); + assertEquals("Ex2", ex1.getCause().getMessage()); + assertEquals("Ex2", ex2.getMessage()); + assertEquals("Ex1", ex2.getCause().getMessage()); + } + + @Test + public void testAttachCallingThreadStackAddExceptionsToEachOther() { + ExceptionThreadingUtility.attachCallingThreadStack(ex1, ex2.getStackTrace()); + ExceptionThreadingUtility.attachCallingThreadStack(ex2, ex1.getStackTrace()); + assertEquals("Ex1", ex1.getMessage()); + assertEquals("Ex2", ex2.getMessage()); + assertEquals("Ex2", ex1.getCause().getMessage()); + assertEquals("Ex1", ex2.getCause().getMessage()); + } + + @Test + public void testAttachCallingThreadStackAddExceptionToItself() { + ExceptionThreadingUtility.attachCallingThreadStack(ex2, ex2.getStackTrace()); + assertEquals("Ex1", ex1.getMessage()); + assertEquals("Ex2", ex1.getCause().getMessage()); + assertEquals("Ex2", ex2.getMessage()); + assertEquals("Ex1", ex2.getCause().getMessage()); + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingNumberTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingNumberTest.java new file mode 100644 index 000000000..86d61d56f --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingNumberTest.java @@ -0,0 +1,678 @@ +package com.netflix.hystrix.util; + +import static org.junit.Assert.*; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Test; + +import com.netflix.hystrix.util.HystrixRollingNumber.Time; + +public class HystrixRollingNumberTest { + + @Test + public void testCreatesBuckets() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + // confirm the initial settings + assertEquals(200, counter.timeInMilliseconds.get().intValue()); + assertEquals(10, counter.numberOfBuckets.get().intValue()); + assertEquals(20, counter.getBucketSizeInMilliseconds()); + + // we start out with 0 buckets in the queue + assertEquals(0, counter.buckets.size()); + + // add a success in each interval which should result in all 10 buckets being created with 1 success in each + for (int i = 0; i < counter.numberOfBuckets.get(); i++) { + counter.increment(HystrixRollingNumberEvent.SUCCESS); + time.increment(counter.getBucketSizeInMilliseconds()); + } + + // confirm we have all 10 buckets + assertEquals(10, counter.buckets.size()); + + // add 1 more and we should still only have 10 buckets since that's the max + counter.increment(HystrixRollingNumberEvent.SUCCESS); + assertEquals(10, counter.buckets.size()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testResetBuckets() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // we start out with 0 buckets in the queue + assertEquals(0, counter.buckets.size()); + + // add 1 + counter.increment(HystrixRollingNumberEvent.SUCCESS); + + // confirm we have 1 bucket + assertEquals(1, counter.buckets.size()); + + // confirm we still have 1 bucket + assertEquals(1, counter.buckets.size()); + + // add 1 + counter.increment(HystrixRollingNumberEvent.SUCCESS); + + // we should now have a single bucket with no values in it instead of 2 or more buckets + assertEquals(1, counter.buckets.size()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testEmptyBucketsFillIn() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // add 1 + counter.increment(HystrixRollingNumberEvent.SUCCESS); + + // we should have 1 bucket + assertEquals(1, counter.buckets.size()); + + // wait past 3 bucket time periods (the 1st bucket then 2 empty ones) + time.increment(counter.getBucketSizeInMilliseconds() * 3); + + // add another + counter.increment(HystrixRollingNumberEvent.SUCCESS); + + // we should have 4 (1 + 2 empty + 1 new one) buckets + assertEquals(4, counter.buckets.size()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testIncrementInSingleBucket() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // increment + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.FAILURE); + counter.increment(HystrixRollingNumberEvent.FAILURE); + counter.increment(HystrixRollingNumberEvent.TIMEOUT); + + // we should have 1 bucket + assertEquals(1, counter.buckets.size()); + + // the count should be 4 + assertEquals(4, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SUCCESS).sum()); + assertEquals(2, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.FAILURE).sum()); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testTimeout() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // increment + counter.increment(HystrixRollingNumberEvent.TIMEOUT); + + // we should have 1 bucket + assertEquals(1, counter.buckets.size()); + + // the count should be 1 + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); + assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds() * 3); + + // incremenet again in latest bucket + counter.increment(HystrixRollingNumberEvent.TIMEOUT); + + // we should have 4 buckets + assertEquals(4, counter.buckets.size()); + + // the counts of the last bucket + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); + + // the total counts + assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testShortCircuited() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // increment + counter.increment(HystrixRollingNumberEvent.SHORT_CIRCUITED); + + // we should have 1 bucket + assertEquals(1, counter.buckets.size()); + + // the count should be 1 + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SHORT_CIRCUITED).sum()); + assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds() * 3); + + // incremenet again in latest bucket + counter.increment(HystrixRollingNumberEvent.SHORT_CIRCUITED); + + // we should have 4 buckets + assertEquals(4, counter.buckets.size()); + + // the counts of the last bucket + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SHORT_CIRCUITED).sum()); + + // the total counts + assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testThreadPoolRejection() { + testCounterType(HystrixRollingNumberEvent.THREAD_POOL_REJECTED); + } + + @Test + public void testFallbackSuccess() { + testCounterType(HystrixRollingNumberEvent.FALLBACK_SUCCESS); + } + + @Test + public void testFallbackFailure() { + testCounterType(HystrixRollingNumberEvent.FALLBACK_FAILURE); + } + + @Test + public void testExceptionThrow() { + testCounterType(HystrixRollingNumberEvent.EXCEPTION_THROWN); + } + + private void testCounterType(HystrixRollingNumberEvent type) { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // increment + counter.increment(type); + + // we should have 1 bucket + assertEquals(1, counter.buckets.size()); + + // the count should be 1 + assertEquals(1, counter.buckets.getLast().getAdder(type).sum()); + assertEquals(1, counter.getRollingSum(type)); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds() * 3); + + // increment again in latest bucket + counter.increment(type); + + // we should have 4 buckets + assertEquals(4, counter.buckets.size()); + + // the counts of the last bucket + assertEquals(1, counter.buckets.getLast().getAdder(type).sum()); + + // the total counts + assertEquals(2, counter.getRollingSum(type)); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testIncrementInMultipleBuckets() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // increment + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.FAILURE); + counter.increment(HystrixRollingNumberEvent.FAILURE); + counter.increment(HystrixRollingNumberEvent.TIMEOUT); + counter.increment(HystrixRollingNumberEvent.TIMEOUT); + counter.increment(HystrixRollingNumberEvent.SHORT_CIRCUITED); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds() * 3); + + // increment + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.FAILURE); + counter.increment(HystrixRollingNumberEvent.FAILURE); + counter.increment(HystrixRollingNumberEvent.FAILURE); + counter.increment(HystrixRollingNumberEvent.TIMEOUT); + counter.increment(HystrixRollingNumberEvent.SHORT_CIRCUITED); + + // we should have 4 buckets + assertEquals(4, counter.buckets.size()); + + // the counts of the last bucket + assertEquals(2, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SUCCESS).sum()); + assertEquals(3, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.FAILURE).sum()); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SHORT_CIRCUITED).sum()); + + // the total counts + assertEquals(6, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(5, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); + assertEquals(3, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + + // wait until window passes + time.increment(counter.timeInMilliseconds.get()); + + // increment + counter.increment(HystrixRollingNumberEvent.SUCCESS); + + // the total counts should now include only the last bucket after a reset since the window passed + assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); + assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testCounterRetrievalRefreshesBuckets() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // increment + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.SUCCESS); + counter.increment(HystrixRollingNumberEvent.FAILURE); + counter.increment(HystrixRollingNumberEvent.FAILURE); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds() * 3); + + // we should have 1 bucket since nothing has triggered the update of buckets in the elapsed time + assertEquals(1, counter.buckets.size()); + + // the total counts + assertEquals(4, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); + + // we should have 4 buckets as the counter 'gets' should have triggered the buckets being created to fill in time + assertEquals(4, counter.buckets.size()); + + // wait until window passes + time.increment(counter.timeInMilliseconds.get()); + + // the total counts should all be 0 (and the buckets cleared by the get, not only increment) + assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); + + // increment + counter.increment(HystrixRollingNumberEvent.SUCCESS); + + // the total counts should now include only the last bucket after a reset since the window passed + assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); + assertEquals(0, counter.getRollingSum(HystrixRollingNumberEvent.FAILURE)); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testUpdateMax1() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // increment + counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 10); + + // we should have 1 bucket + assertEquals(1, counter.buckets.size()); + + // the count should be 10 + assertEquals(10, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); + assertEquals(10, counter.getRollingMaxValue(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds() * 3); + + // increment again in latest bucket + counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 20); + + // we should have 4 buckets + assertEquals(4, counter.buckets.size()); + + // the max + assertEquals(20, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); + + // counts per bucket + long values[] = counter.getValues(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE); + assertEquals(10, values[0]); // oldest bucket + assertEquals(0, values[1]); + assertEquals(0, values[2]); + assertEquals(20, values[3]); // latest bucket + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testUpdateMax2() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + // increment + counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 10); + counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 30); + counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 20); + + // we should have 1 bucket + assertEquals(1, counter.buckets.size()); + + // the count should be 30 + assertEquals(30, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); + assertEquals(30, counter.getRollingMaxValue(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds() * 3); + + counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 30); + counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 30); + counter.updateRollingMax(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE, 50); + + // we should have 4 buckets + assertEquals(4, counter.buckets.size()); + + // the count + assertEquals(50, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); + assertEquals(50, counter.getValueOfLatestBucket(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); + + // values per bucket + long values[] = counter.getValues(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE); + assertEquals(30, values[0]); // oldest bucket + assertEquals(0, values[1]); + assertEquals(0, values[2]); + assertEquals(50, values[3]); // latest bucket + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testMaxValue() { + MockedTime time = new MockedTime(); + try { + HystrixRollingNumberEvent type = HystrixRollingNumberEvent.THREAD_MAX_ACTIVE; + + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + + counter.updateRollingMax(type, 10); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds()); + + counter.updateRollingMax(type, 30); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds()); + + counter.updateRollingMax(type, 40); + + // sleep to get to a new bucket + time.increment(counter.getBucketSizeInMilliseconds()); + + counter.updateRollingMax(type, 15); + + assertEquals(40, counter.getRollingMaxValue(type)); + + } catch (Exception e) { + e.printStackTrace(); + fail("Exception: " + e.getMessage()); + } + } + + @Test + public void testEmptySum() { + MockedTime time = new MockedTime(); + HystrixRollingNumberEvent type = HystrixRollingNumberEvent.COLLAPSED; + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + assertEquals(0, counter.getRollingSum(type)); + } + + @Test + public void testEmptyMax() { + MockedTime time = new MockedTime(); + HystrixRollingNumberEvent type = HystrixRollingNumberEvent.THREAD_MAX_ACTIVE; + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + assertEquals(0, counter.getRollingMaxValue(type)); + } + + @Test + public void testEmptyLatestValue() { + MockedTime time = new MockedTime(); + HystrixRollingNumberEvent type = HystrixRollingNumberEvent.THREAD_MAX_ACTIVE; + HystrixRollingNumber counter = new HystrixRollingNumber(time, 200, 10); + assertEquals(0, counter.getValueOfLatestBucket(type)); + } + + @Test + public void testRolling() { + MockedTime time = new MockedTime(); + HystrixRollingNumberEvent type = HystrixRollingNumberEvent.THREAD_MAX_ACTIVE; + HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); + // iterate over 20 buckets on a queue sized for 2 + for (int i = 0; i < 20; i++) { + // first bucket + counter.getCurrentBucket(); + try { + time.increment(counter.getBucketSizeInMilliseconds()); + } catch (Exception e) { + // ignore + } + + assertEquals(2, counter.getValues(type).length); + + counter.getValueOfLatestBucket(type); + + // System.out.println("Head: " + counter.buckets.state.get().head); + // System.out.println("Tail: " + counter.buckets.state.get().tail); + } + } + + @Test + public void testCumulativeCounterAfterRolling() { + MockedTime time = new MockedTime(); + HystrixRollingNumberEvent type = HystrixRollingNumberEvent.SUCCESS; + HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); + + assertEquals(0, counter.getCumulativeSum(type)); + + // iterate over 20 buckets on a queue sized for 2 + for (int i = 0; i < 20; i++) { + // first bucket + counter.increment(type); + try { + time.increment(counter.getBucketSizeInMilliseconds()); + } catch (Exception e) { + // ignore + } + + assertEquals(2, counter.getValues(type).length); + + counter.getValueOfLatestBucket(type); + + } + + // cumulative count should be 20 (for the number of loops above) regardless of buckets rolling + assertEquals(20, counter.getCumulativeSum(type)); + } + + @Test + public void testCumulativeCounterAfterRollingAndReset() { + MockedTime time = new MockedTime(); + HystrixRollingNumberEvent type = HystrixRollingNumberEvent.SUCCESS; + HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); + + assertEquals(0, counter.getCumulativeSum(type)); + + // iterate over 20 buckets on a queue sized for 2 + for (int i = 0; i < 20; i++) { + // first bucket + counter.increment(type); + try { + time.increment(counter.getBucketSizeInMilliseconds()); + } catch (Exception e) { + // ignore + } + + assertEquals(2, counter.getValues(type).length); + + counter.getValueOfLatestBucket(type); + + if (i == 5 || i == 15) { + // simulate a reset occurring every once in a while + // so we ensure the absolute sum is handling it okay + counter.reset(); + } + } + + // cumulative count should be 20 (for the number of loops above) regardless of buckets rolling + assertEquals(20, counter.getCumulativeSum(type)); + } + + @Test + public void testCumulativeCounterAfterRollingAndReset2() { + MockedTime time = new MockedTime(); + HystrixRollingNumberEvent type = HystrixRollingNumberEvent.SUCCESS; + HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); + + assertEquals(0, counter.getCumulativeSum(type)); + + counter.increment(type); + counter.increment(type); + counter.increment(type); + + // iterate over 20 buckets on a queue sized for 2 + for (int i = 0; i < 20; i++) { + try { + time.increment(counter.getBucketSizeInMilliseconds()); + } catch (Exception e) { + // ignore + } + + if (i == 5 || i == 15) { + // simulate a reset occurring every once in a while + // so we ensure the absolute sum is handling it okay + counter.reset(); + } + } + + // no increments during the loop, just some before and after + counter.increment(type); + counter.increment(type); + + // cumulative count should be 5 regardless of buckets rolling + assertEquals(5, counter.getCumulativeSum(type)); + } + + @Test + public void testCumulativeCounterAfterRollingAndReset3() { + MockedTime time = new MockedTime(); + HystrixRollingNumberEvent type = HystrixRollingNumberEvent.SUCCESS; + HystrixRollingNumber counter = new HystrixRollingNumber(time, 20, 2); + + assertEquals(0, counter.getCumulativeSum(type)); + + counter.increment(type); + counter.increment(type); + counter.increment(type); + + // iterate over 20 buckets on a queue sized for 2 + for (int i = 0; i < 20; i++) { + try { + time.increment(counter.getBucketSizeInMilliseconds()); + } catch (Exception e) { + // ignore + } + } + + // since we are rolling over the buckets it should reset naturally + + // no increments during the loop, just some before and after + counter.increment(type); + counter.increment(type); + + // cumulative count should be 5 regardless of buckets rolling + assertEquals(5, counter.getCumulativeSum(type)); + } + + private static class MockedTime implements Time { + + private AtomicInteger time = new AtomicInteger(0); + + @Override + public long getCurrentTimeInMillis() { + return time.get(); + } + + public void increment(int millis) { + time.addAndGet(millis); + } + + } + +} diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingPercentileTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingPercentileTest.java new file mode 100644 index 000000000..492a4f68f --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingPercentileTest.java @@ -0,0 +1,661 @@ +package com.netflix.hystrix.util; + +import static org.junit.Assert.*; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; + +import com.netflix.hystrix.strategy.properties.HystrixProperty; +import com.netflix.hystrix.util.HystrixRollingPercentile.PercentileSnapshot; +import com.netflix.hystrix.util.HystrixRollingPercentile.Time; + +public class HystrixRollingPercentileTest { + + private static final HystrixProperty timeInMilliseconds = HystrixProperty.Factory.asProperty(60000); + private static final HystrixProperty numberOfBuckets = HystrixProperty.Factory.asProperty(12); // 12 buckets at 5000ms each + private static final HystrixProperty bucketDataLength = HystrixProperty.Factory.asProperty(1000); + private static final HystrixProperty enabled = HystrixProperty.Factory.asProperty(true); + + @Test + public void testRolling() { + MockedTime time = new MockedTime(); + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); + p.addValue(1000); + p.addValue(1000); + p.addValue(1000); + p.addValue(2000); + + assertEquals(1, p.buckets.size()); + + // no bucket turnover yet so percentile not yet generated + assertEquals(0, p.getPercentile(50)); + + time.increment(6000); + + // still only 1 bucket until we touch it again + assertEquals(1, p.buckets.size()); + + // a bucket has been created so we have a new percentile + assertEquals(1000, p.getPercentile(50)); + + // now 2 buckets since getting a percentile causes bucket retrieval + assertEquals(2, p.buckets.size()); + + p.addValue(1000); + p.addValue(500); + + // should still be 2 buckets + assertEquals(2, p.buckets.size()); + + p.addValue(200); + p.addValue(200); + p.addValue(1600); + p.addValue(200); + p.addValue(1600); + p.addValue(1600); + + // we haven't progressed to a new bucket so the percentile should be the same and ignore the most recent bucket + assertEquals(1000, p.getPercentile(50)); + + // increment to another bucket so we include all of the above in the PercentileSnapshot + time.increment(6000); + + // the rolling version should have the same data as creating a snapshot like this + PercentileSnapshot ps = new PercentileSnapshot(1000, 1000, 1000, 2000, 1000, 500, 200, 200, 1600, 200, 1600, 1600); + + assertEquals(ps.getPercentile(0.15), p.getPercentile(0.15)); + assertEquals(ps.getPercentile(0.50), p.getPercentile(0.50)); + assertEquals(ps.getPercentile(0.90), p.getPercentile(0.90)); + assertEquals(ps.getPercentile(0.995), p.getPercentile(0.995)); + + System.out.println("100th: " + ps.getPercentile(100) + " " + p.getPercentile(100)); + System.out.println("99.5th: " + ps.getPercentile(99.5) + " " + p.getPercentile(99.5)); + System.out.println("99th: " + ps.getPercentile(99) + " " + p.getPercentile(99)); + System.out.println("90th: " + ps.getPercentile(90) + " " + p.getPercentile(90)); + System.out.println("50th: " + ps.getPercentile(50) + " " + p.getPercentile(50)); + System.out.println("10th: " + ps.getPercentile(10) + " " + p.getPercentile(10)); + + // mean = 1000+1000+1000+2000+1000+500+200+200+1600+200+1600+1600/12 + assertEquals(991, ps.getMean()); + } + + @Test + public void testValueIsZeroAfterRollingWindowPassesAndNoTraffic() { + MockedTime time = new MockedTime(); + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); + p.addValue(1000); + p.addValue(1000); + p.addValue(1000); + p.addValue(2000); + p.addValue(4000); + + assertEquals(1, p.buckets.size()); + + // no bucket turnover yet so percentile not yet generated + assertEquals(0, p.getPercentile(50)); + + time.increment(6000); + + // still only 1 bucket until we touch it again + assertEquals(1, p.buckets.size()); + + // a bucket has been created so we have a new percentile + assertEquals(1500, p.getPercentile(50)); + + // let 1 minute pass + time.increment(60000); + + // no data in a minute should mean all buckets are empty (or reset) so we should not have any percentiles + assertEquals(0, p.getPercentile(50)); + } + + @Test + public void testSampleDataOverTime1() { + System.out.println("\n\n***************************** testSampleDataOverTime1 \n"); + + MockedTime time = new MockedTime(); + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); + int previousTime = 0; + for (int i = 0; i < SampleDataHolder1.data.length; i++) { + int timeInMillisecondsSinceStart = SampleDataHolder1.data[i][0]; + int latency = SampleDataHolder1.data[i][1]; + time.increment(timeInMillisecondsSinceStart - previousTime); + previousTime = timeInMillisecondsSinceStart; + p.addValue(latency); + } + + System.out.println("0.01: " + p.getPercentile(0.01)); + System.out.println("Median: " + p.getPercentile(50)); + System.out.println("90th: " + p.getPercentile(90)); + System.out.println("99th: " + p.getPercentile(99)); + System.out.println("99.5th: " + p.getPercentile(99.5)); + System.out.println("99.99: " + p.getPercentile(99.99)); + + System.out.println("Median: " + p.getPercentile(50)); + System.out.println("Median: " + p.getPercentile(50)); + System.out.println("Median: " + p.getPercentile(50)); + + /* + * In a loop as a use case was found where very different values were calculated in subsequent requests. + */ + for (int i = 0; i < 10; i++) { + if (p.getPercentile(50) > 5) { + fail("We expect around 2 but got: " + p.getPercentile(50)); + } + + if (p.getPercentile(99.5) < 20) { + fail("We expect to see some high values over 20 but got: " + p.getPercentile(99.5)); + } + } + } + + @Test + public void testSampleDataOverTime2() { + System.out.println("\n\n***************************** testSampleDataOverTime2 \n"); + MockedTime time = new MockedTime(); + int previousTime = 0; + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); + for (int i = 0; i < SampleDataHolder2.data.length; i++) { + int timeInMillisecondsSinceStart = SampleDataHolder2.data[i][0]; + int latency = SampleDataHolder2.data[i][1]; + time.increment(timeInMillisecondsSinceStart - previousTime); + previousTime = timeInMillisecondsSinceStart; + p.addValue(latency); + } + + System.out.println("0.01: " + p.getPercentile(0.01)); + System.out.println("Median: " + p.getPercentile(50)); + System.out.println("90th: " + p.getPercentile(90)); + System.out.println("99th: " + p.getPercentile(99)); + System.out.println("99.5th: " + p.getPercentile(99.5)); + System.out.println("99.99: " + p.getPercentile(99.99)); + + if (p.getPercentile(50) > 90 || p.getPercentile(50) < 50) { + fail("We expect around 60-70 but got: " + p.getPercentile(50)); + } + + if (p.getPercentile(99) < 400) { + fail("We expect to see some high values over 400 but got: " + p.getPercentile(99)); + } + } + + public PercentileSnapshot getPercentileForValues(int... values) { + PercentileSnapshot p = new PercentileSnapshot(values); + return p; + } + + @Test + public void testPercentileAlgorithm_Median1() { + PercentileSnapshot list = new PercentileSnapshot(100, 100, 100, 100, 200, 200, 200, 300, 300, 300, 300); + Assert.assertEquals(200, list.getPercentile(50)); + } + + @Test + public void testPercentileAlgorithm_Median2() { + PercentileSnapshot list = new PercentileSnapshot(100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 500); + Assert.assertEquals(100, list.getPercentile(50)); + } + + @Test + public void testPercentileAlgorithm_Median3() { + PercentileSnapshot list = new PercentileSnapshot(50, 75, 100, 125, 160, 170, 180, 200, 210, 300, 500); + // list.addValue(50); // 1 + // list.addValue(75); // 2 + // list.addValue(100); // 3 + // list.addValue(125); // 4 + // list.addValue(160); // 5 + // list.addValue(170); // 6 + // list.addValue(180); // 7 + // list.addValue(200); // 8 + // list.addValue(210); // 9 + // list.addValue(300); // 10 + // list.addValue(500); // 11 + + Assert.assertEquals(175, list.getPercentile(50)); + } + + @Test + public void testPercentileAlgorithm_Median4() { + PercentileSnapshot list = new PercentileSnapshot(300, 75, 125, 500, 100, 160, 180, 200, 210, 50, 170); + // unsorted so it is expected to sort it for us + // list.addValue(300); // 10 + // list.addValue(75); // 2 + // list.addValue(125); // 4 + // list.addValue(500); // 11 + // list.addValue(100); // 3 + // list.addValue(160); // 5 + // list.addValue(180); // 7 + // list.addValue(200); // 8 + // list.addValue(210); // 9 + // list.addValue(50); // 1 + // list.addValue(170); // 6 + + Assert.assertEquals(175, list.getPercentile(50)); + } + + @Test + public void testPercentileAlgorithm_Extremes() { + PercentileSnapshot p = new PercentileSnapshot(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 800, 768, 657, 700, 867); + + System.out.println("0.01: " + p.getPercentile(0.01)); + System.out.println("10th: " + p.getPercentile(10)); + System.out.println("Median: " + p.getPercentile(50)); + System.out.println("75th: " + p.getPercentile(75)); + System.out.println("90th: " + p.getPercentile(90)); + System.out.println("99th: " + p.getPercentile(99)); + System.out.println("99.5th: " + p.getPercentile(99.5)); + System.out.println("99.99: " + p.getPercentile(99.99)); + Assert.assertEquals(2, p.getPercentile(50)); + Assert.assertEquals(2, p.getPercentile(10)); + Assert.assertEquals(2, p.getPercentile(75)); + if (p.getPercentile(95) < 600) { + fail("We expect the 90th to be over 600 to show the extremes but got: " + p.getPercentile(90)); + } + if (p.getPercentile(99) < 600) { + fail("We expect the 99th to be over 600 to show the extremes but got: " + p.getPercentile(99)); + } + } + + @Test + public void testPercentileAlgorithm_HighPercentile() { + PercentileSnapshot p = getPercentileForValues(1, 2, 3); + Assert.assertEquals(2, p.getPercentile(50)); + Assert.assertEquals(3, p.getPercentile(75)); + } + + @Test + public void testPercentileAlgorithm_LowPercentile() { + PercentileSnapshot p = getPercentileForValues(1, 2); + Assert.assertEquals(1, p.getPercentile(25)); + Assert.assertEquals(2, p.getPercentile(75)); + } + + @Test + public void testPercentileAlgorithm_Percentiles() { + PercentileSnapshot p = getPercentileForValues(10, 30, 20, 40); + Assert.assertEquals(22, p.getPercentile(30), 1.0e-5); + Assert.assertEquals(20, p.getPercentile(25), 1.0e-5); + Assert.assertEquals(40, p.getPercentile(75), 1.0e-5); + Assert.assertEquals(30, p.getPercentile(50), 1.0e-5); + + // invalid percentiles + Assert.assertEquals(10, p.getPercentile(-1)); + Assert.assertEquals(40, p.getPercentile(101)); + } + + @Test + public void testPercentileAlgorithm_NISTExample() { + PercentileSnapshot p = getPercentileForValues(951772, 951567, 951937, 951959, 951442, 950610, 951591, 951195, 951772, 950925, 951990, 951682); + Assert.assertEquals(951983, p.getPercentile(90)); + Assert.assertEquals(951990, p.getPercentile(100)); + } + + /** + * This code should work without throwing exceptions but the data returned will all be -1 since the rolling percentile is disabled. + */ + @Test + public void testDoesNothingWhenDisabled() { + MockedTime time = new MockedTime(); + int previousTime = 0; + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, HystrixProperty.Factory.asProperty(false)); + for (int i = 0; i < SampleDataHolder2.data.length; i++) { + int timeInMillisecondsSinceStart = SampleDataHolder2.data[i][0]; + int latency = SampleDataHolder2.data[i][1]; + time.increment(timeInMillisecondsSinceStart - previousTime); + previousTime = timeInMillisecondsSinceStart; + p.addValue(latency); + } + + assertEquals(-1, p.getPercentile(50)); + assertEquals(-1, p.getPercentile(75)); + assertEquals(-1, p.getMean()); + } + + private static class MockedTime implements Time { + + private AtomicInteger time = new AtomicInteger(0); + + @Override + public long getCurrentTimeInMillis() { + return time.get(); + } + + public void increment(int millis) { + time.addAndGet(millis); + } + + } + + /* sub-class to avoid 65k limit of a single class */ + private static class SampleDataHolder1 { + /* + * Array of [milliseconds, latency] + */ + private static int[][] data = new int[][] { + { 0, 3 }, { 43, 33 }, { 45, 11 }, { 45, 1 }, { 68, 13 }, { 88, 10 }, { 158, 2 }, { 158, 4 }, { 169, 12 }, { 267, 2 }, { 342, 2 }, { 438, 2 }, { 464, 7 }, { 504, 2 }, { 541, 6 }, { 541, 2 }, { 562, 2 }, { 581, 3 }, { 636, 2 }, { 778, 2 }, { 825, 1 }, { 859, 2 }, { 948, 1 }, { 1043, 2 }, { 1145, 2 }, { 1152, 1 }, { 1218, 5 }, + { 1229, 2 }, { 1259, 2 }, { 1333, 2 }, { 1349, 2 }, { 1392, 2 }, { 1468, 1 }, { 1551, 2 }, { 1586, 2 }, { 1685, 2 }, { 1696, 1 }, { 1807, 2 }, { 1817, 3 }, { 1817, 6 }, { 1847, 2 }, { 1870, 2 }, { 1939, 2 }, { 2050, 2 }, { 2129, 3 }, { 2141, 2 }, { 2265, 2 }, { 2414, 1 }, { 2693, 2 }, { 2703, 2 }, { 2791, 2 }, { 2838, 2 }, + { 2906, 2 }, { 2981, 2 }, { 3008, 2 }, { 3026, 4 }, { 3077, 2 }, { 3273, 2 }, { 3282, 2 }, { 3286, 2 }, { 3318, 3 }, { 3335, 5 }, { 3710, 2 }, { 3711, 1 }, { 3745, 2 }, { 3748, 4 }, { 3767, 3 }, { 3809, 3 }, { 3835, 35 }, { 4083, 1 }, { 4116, 2 }, { 4117, 1 }, { 4157, 1 }, { 4279, 2 }, { 4344, 2 }, { 4452, 2 }, { 4530, 2 }, + { 4583, 2 }, { 4647, 3 }, { 4758, 2 }, { 4776, 2 }, { 4793, 2 }, { 4901, 2 }, { 4909, 2 }, { 4962, 2 }, { 4984, 2 }, { 5022, 2 }, { 5139, 2 }, { 5166, 1 }, { 5174, 2 }, { 5187, 2 }, { 5225, 2 }, { 5234, 2 }, { 5263, 1 }, { 5325, 2 }, { 5355, 4 }, { 5407, 1 }, { 5414, 2 }, { 5589, 2 }, { 5595, 2 }, { 5747, 2 }, { 5780, 2 }, + { 5788, 2 }, { 5796, 2 }, { 5818, 2 }, { 5975, 1 }, { 6018, 1 }, { 6270, 2 }, { 6272, 2 }, { 6348, 2 }, { 6372, 2 }, { 6379, 2 }, { 6439, 2 }, { 6442, 2 }, { 6460, 2 }, { 6460, 2 }, { 6509, 2 }, { 6511, 1 }, { 6514, 4 }, { 6530, 8 }, { 6719, 2 }, { 6760, 2 }, { 6784, 2 }, { 6838, 1 }, { 6861, 2 }, { 6947, 2 }, { 7013, 2 }, + { 7075, 2 }, { 7122, 5 }, { 7130, 2 }, { 7209, 3 }, { 7259, 2 }, { 7309, 1 }, { 7315, 3 }, { 7322, 2 }, { 7348, 2 }, { 7420, 2 }, { 7461, 2 }, { 7545, 2 }, { 7554, 3 }, { 7630, 2 }, { 7666, 2 }, { 7815, 1 }, { 7972, 1 }, { 7972, 2 }, { 7988, 2 }, { 8049, 8 }, { 8254, 2 }, { 8269, 2 }, { 8352, 1 }, { 8378, 2 }, { 8526, 2 }, + { 8531, 2 }, { 8583, 2 }, { 8615, 2 }, { 8619, 3 }, { 8623, 2 }, { 8692, 1 }, { 8698, 2 }, { 8773, 2 }, { 8777, 3 }, { 8822, 2 }, { 8929, 2 }, { 8935, 2 }, { 9025, 2 }, { 9054, 2 }, { 9056, 1 }, { 9086, 2 }, { 9147, 3 }, { 9219, 2 }, { 9230, 3 }, { 9248, 2 }, { 9283, 2 }, { 9314, 2 }, { 9418, 1 }, { 9426, 2 }, { 9456, 1 }, + { 9594, 2 }, { 9628, 2 }, { 9642, 2 }, { 9646, 2 }, { 9686, 1 }, { 9709, 2 }, { 9771, 3 }, { 9782, 2 }, { 9884, 2 }, { 9914, 5 }, { 10004, 4 }, { 10033, 6 }, { 10052, 2 }, { 10086, 2 }, { 10168, 2 }, { 10176, 1 }, { 10228, 2 }, { 10312, 2 }, { 10372, 2 }, { 10622, 2 }, { 10685, 2 }, { 10687, 1 }, { 10787, 2 }, { 11010, 2 }, + { 11024, 2 }, { 11044, 2 }, { 11086, 2 }, { 11149, 1 }, { 11198, 2 }, { 11265, 2 }, { 11302, 2 }, { 11326, 2 }, { 11354, 2 }, { 11404, 1 }, { 11473, 2 }, { 11506, 2 }, { 11548, 4 }, { 11575, 2 }, { 11621, 4 }, { 11625, 3 }, { 11625, 1 }, { 11642, 4 }, { 11859, 5 }, { 11870, 2 }, { 11872, 3 }, { 11880, 7 }, { 11886, 3 }, + { 11905, 6 }, { 11880, 3 }, { 11912, 6 }, { 11916, 4 }, { 11916, 3 }, { 11965, 4 }, { 12068, 13 }, { 12106, 2 }, { 12120, 2 }, { 12221, 2 }, { 12257, 2 }, { 12361, 2 }, { 12411, 2 }, { 12473, 3 }, { 12554, 2 }, { 12583, 2 }, { 12654, 2 }, { 12665, 2 }, { 12744, 1 }, { 12775, 2 }, { 12858, 2 }, { 12993, 2 }, { 13007, 3 }, + { 13025, 4 }, { 13038, 2 }, { 13092, 4 }, { 13094, 5 }, { 13095, 1 }, { 13110, 2 }, { 13116, 1 }, { 13140, 2 }, { 13169, 1 }, { 13186, 2 }, { 13202, 2 }, { 13202, 1 }, { 13256, 2 }, { 13344, 2 }, { 13373, 2 }, { 13396, 3 }, { 13446, 2 }, { 13451, 3 }, { 13475, 2 }, { 13521, 1 }, { 13587, 2 }, { 13592, 2 }, { 13708, 3 }, + { 13711, 1 }, { 13741, 1 }, { 13757, 1 }, { 13847, 2 }, { 13881, 3 }, { 13915, 1 }, { 14005, 2 }, { 14028, 2 }, { 14037, 2 }, { 14074, 2 }, { 14135, 2 }, { 14176, 2 }, { 14227, 2 }, { 14228, 2 }, { 14271, 3 }, { 14279, 3 }, { 14493, 2 }, { 14535, 3 }, { 14535, 1 }, { 14680, 2 }, { 14717, 2 }, { 14725, 1 }, { 14790, 2 }, + { 14801, 1 }, { 14959, 2 }, { 15052, 2 }, { 15055, 1 }, { 15055, 1 }, { 15075, 2 }, { 15103, 8 }, { 15153, 16 }, { 15191, 2 }, { 15240, 2 }, { 15313, 2 }, { 15323, 2 }, { 15341, 1 }, { 15383, 2 }, { 15387, 2 }, { 15491, 2 }, { 15534, 2 }, { 15539, 2 }, { 15549, 2 }, { 15554, 1 }, { 15664, 1 }, { 15726, 2 }, { 15807, 2 }, + { 15842, 2 }, { 15897, 2 }, { 15913, 3 }, { 15925, 2 }, { 15935, 2 }, { 16131, 1 }, { 16211, 3 }, { 16249, 2 }, { 16268, 2 }, { 16307, 2 }, { 16398, 2 }, { 16498, 2 }, { 16518, 1 }, { 16552, 1 }, { 16571, 2 }, { 16592, 2 }, { 16601, 3 }, { 16638, 2 }, { 16698, 2 }, { 16712, 1 }, { 16767, 2 }, { 16789, 2 }, { 16992, 2 }, + { 17015, 2 }, { 17035, 2 }, { 17074, 3 }, { 17086, 3 }, { 17086, 1 }, { 17092, 1 }, { 17110, 4 }, { 17116, 3 }, { 17236, 2 }, { 17291, 2 }, { 17291, 2 }, { 17340, 2 }, { 17342, 1 }, { 17360, 3 }, { 17436, 3 }, { 17457, 2 }, { 17508, 1 }, { 17556, 2 }, { 17601, 2 }, { 17639, 2 }, { 17671, 2 }, { 17743, 2 }, { 17857, 2 }, + { 17915, 2 }, { 17992, 2 }, { 18077, 1 }, { 18088, 2 }, { 18158, 1 }, { 18239, 16 }, { 18242, 2 }, { 18252, 3 }, { 18299, 1 }, { 18405, 2 }, { 18433, 2 }, { 18444, 2 }, { 18490, 2 }, { 18497, 2 }, { 18516, 2 }, { 18540, 2 }, { 18598, 2 }, { 18649, 2 }, { 18658, 2 }, { 18683, 2 }, { 18728, 2 }, { 18767, 1 }, { 18821, 2 }, + { 18868, 2 }, { 18876, 2 }, { 18914, 14 }, { 19212, 1 }, { 19215, 1 }, { 19293, 2 }, { 19303, 2 }, { 19336, 2 }, { 19376, 2 }, { 19419, 2 }, { 19558, 2 }, { 19559, 1 }, { 19609, 2 }, { 19688, 2 }, { 19724, 2 }, { 19820, 1 }, { 19851, 2 }, { 19881, 2 }, { 19966, 2 }, { 19983, 3 }, { 19988, 4 }, { 20047, 1 }, { 20062, 2 }, + { 20091, 1 }, { 20152, 1 }, { 20183, 1 }, { 20208, 2 }, { 20346, 2 }, { 20386, 1 }, { 20459, 2 }, { 20505, 2 }, { 20520, 1 }, { 20560, 3 }, { 20566, 3 }, { 20566, 1 }, { 20610, 2 }, { 20652, 2 }, { 20694, 2 }, { 20740, 2 }, { 20756, 2 }, { 20825, 3 }, { 20895, 2 }, { 20959, 1 }, { 20995, 2 }, { 21017, 3 }, { 21039, 2 }, + { 21086, 1 }, { 21109, 3 }, { 21139, 3 }, { 21206, 2 }, { 21230, 2 }, { 21251, 3 }, { 21352, 2 }, { 21353, 2 }, { 21370, 3 }, { 21389, 1 }, { 21445, 3 }, { 21475, 2 }, { 21528, 2 }, { 21559, 3 }, { 21604, 2 }, { 21606, 1 }, { 21815, 2 }, { 21858, 3 }, { 21860, 3 }, { 22015, 2 }, { 22065, 2 }, { 22098, 5 }, { 22105, 2 }, + { 22158, 3 }, { 22197, 2 }, { 22254, 1 }, { 22353, 2 }, { 22404, 4 }, { 22422, 2 }, { 22569, 2 }, { 22634, 2 }, { 22639, 2 }, { 22861, 2 }, { 22868, 2 }, { 22876, 1 }, { 22902, 2 }, { 22925, 2 }, { 23080, 2 }, { 23085, 3 }, { 23089, 5 }, { 23329, 1 }, { 23349, 2 }, { 23559, 5 }, { 23567, 3 }, { 23574, 2 }, { 23584, 3 }, + { 23615, 3 }, { 23633, 2 }, { 23674, 2 }, { 23678, 1 }, { 23853, 2 }, { 23875, 2 }, { 24010, 4 }, { 24076, 2 }, { 24128, 6 }, { 24248, 2 }, { 24253, 2 }, { 24259, 1 }, { 24319, 2 }, { 24319, 1 }, { 24502, 3 }, { 24666, 2 }, { 24781, 3 }, { 24792, 2 }, { 24909, 2 }, { 24993, 2 }, { 25039, 1 }, { 25090, 3 }, { 25137, 1 }, + { 25138, 3 }, { 25140, 3 }, { 25155, 5 }, { 25411, 2 }, { 25460, 2 }, { 25564, 3 }, { 25586, 3 }, { 25630, 2 }, { 25765, 2 }, { 25789, 3 }, { 25803, 2 }, { 25851, 2 }, { 25872, 2 }, { 25887, 2 }, { 25981, 1 }, { 26016, 2 }, { 26019, 1 }, { 26029, 1 }, { 26104, 7 }, { 26144, 2 }, { 26275, 1 }, { 26295, 2 }, { 26298, 1 }, + { 26322, 2 }, { 26380, 2 }, { 26408, 4 }, { 26446, 1 }, { 26553, 1 }, { 26576, 1 }, { 26635, 1 }, { 26668, 2 }, { 26675, 2 }, { 26698, 4 }, { 26748, 9 }, { 26788, 2 }, { 26932, 2 }, { 26962, 2 }, { 27042, 2 }, { 27060, 2 }, { 27163, 3 }, { 27202, 2 }, { 27290, 2 }, { 27337, 3 }, { 27376, 2 }, { 27439, 2 }, { 27458, 4 }, + { 27515, 2 }, { 27518, 1 }, { 27541, 2 }, { 27585, 3 }, { 27633, 2 }, { 27695, 2 }, { 27702, 2 }, { 27861, 2 }, { 27924, 1 }, { 28025, 14 }, { 28058, 2 }, { 28143, 2 }, { 28215, 2 }, { 28240, 2 }, { 28241, 2 }, { 28285, 2 }, { 28324, 3 }, { 28378, 2 }, { 28514, 2 }, { 28529, 2 }, { 28538, 2 }, { 28565, 3 }, { 28697, 2 }, + { 28735, 2 }, { 28769, 2 }, { 28770, 4 }, { 28788, 4 }, { 28807, 3 }, { 28807, 4 }, { 28829, 1 }, { 28853, 2 }, { 28856, 7 }, { 28864, 2 }, { 28865, 3 }, { 28915, 2 }, { 28928, 2 }, { 28964, 2 }, { 28988, 1 }, { 29031, 2 }, { 29095, 2 }, { 29189, 2 }, { 29205, 1 }, { 29230, 1 }, { 29332, 2 }, { 29339, 2 }, { 29349, 5 }, + { 29449, 2 }, { 29471, 2 }, { 29578, 2 }, { 29859, 2 }, { 29878, 2 }, { 29947, 10 }, { 30083, 2 }, { 30121, 2 }, { 30128, 2 }, { 30155, 4 }, { 30157, 1 }, { 30272, 2 }, { 30281, 2 }, { 30286, 2 }, { 30305, 2 }, { 30408, 2 }, { 30444, 22 }, { 30612, 2 }, { 30628, 2 }, { 30747, 2 }, { 30783, 2 }, { 30808, 5 }, { 30868, 3 }, + { 30875, 2 }, { 30997, 2 }, { 31000, 2 }, { 31022, 3 }, { 31111, 1 }, { 31144, 2 }, { 31146, 3 }, { 31187, 2 }, { 31324, 2 }, { 31343, 2 }, { 31416, 2 }, { 31485, 2 }, { 31539, 2 }, { 31638, 2 }, { 31648, 2 }, { 31750, 2 }, { 31754, 2 }, { 31785, 10 }, { 31786, 5 }, { 31800, 2 }, { 31801, 4 }, { 31807, 7 }, { 31807, 3 }, + { 31807, 10 }, { 31808, 3 }, { 31808, 4 }, { 31818, 6 }, { 31825, 7 }, { 31838, 2 }, { 31911, 1 }, { 31974, 2 }, { 32010, 3 }, { 32031, 2 }, { 32040, 2 }, { 32063, 1 }, { 32078, 2 }, { 32156, 2 }, { 32198, 31 }, { 32257, 2 }, { 32257, 2 }, { 32265, 2 }, { 32330, 2 }, { 32369, 8 }, { 32404, 3 }, { 32425, 2 }, { 32432, 2 }, + { 32505, 2 }, { 32531, 2 }, { 32536, 2 }, { 32549, 2 }, { 32582, 3 }, { 32590, 4 }, { 32624, 2 }, { 32644, 2 }, { 32692, 2 }, { 32695, 4 }, { 32699, 3 }, { 32726, 4 }, { 32784, 2 }, { 32832, 2 }, { 32883, 6 }, { 32965, 4 }, { 33044, 2 }, { 33104, 2 }, { 33184, 2 }, { 33264, 1 }, { 33292, 2 }, { 33312, 1 }, { 33468, 2 }, + { 33471, 1 }, { 33565, 2 }, { 33627, 2 }, { 33659, 2 }, { 33709, 2 }, { 33766, 5 }, { 33836, 2 }, { 33875, 2 }, { 33954, 2 }, { 33959, 2 }, { 34050, 2 }, { 34090, 2 }, { 34168, 2 }, { 34233, 2 }, { 34461, 2 }, { 34462, 1 }, { 34463, 2 }, { 34472, 4 }, { 34500, 2 }, { 34520, 2 }, { 34544, 2 }, { 34614, 2 }, { 34662, 1 }, + { 34676, 2 }, { 34729, 4 }, { 34803, 2 }, { 34845, 2 }, { 34913, 2 }, { 34963, 6 }, { 35019, 2 }, { 35022, 2 }, { 35070, 2 }, { 35120, 2 }, { 35132, 2 }, { 35144, 2 }, { 35205, 2 }, { 35230, 3 }, { 35244, 2 }, { 35271, 4 }, { 35276, 2 }, { 35282, 2 }, { 35324, 3 }, { 35366, 3 }, { 35659, 2 }, { 35680, 2 }, { 35744, 2 }, + { 35758, 3 }, { 35796, 2 }, { 35830, 2 }, { 35841, 7 }, { 35843, 2 }, { 35856, 2 }, { 35914, 4 }, { 35929, 13 }, { 35993, 2 }, { 35997, 1 }, { 36046, 4 }, { 36046, 1 }, { 36051, 1 }, { 36111, 2 }, { 36208, 1 }, { 36208, 1 }, { 36306, 2 }, { 36325, 2 }, { 36386, 2 }, { 36405, 2 }, { 36443, 1 }, { 36455, 1 }, { 36538, 2 }, + { 36562, 2 }, { 36566, 2 }, { 36628, 2 }, { 36693, 2 }, { 36713, 2 }, { 36730, 2 }, { 36747, 2 }, { 36786, 2 }, { 36810, 1 }, { 36848, 2 }, { 36914, 1 }, { 36920, 2 }, { 36952, 2 }, { 37071, 2 }, { 37086, 1 }, { 37094, 3 }, { 37158, 3 }, { 37231, 2 }, { 37241, 2 }, { 37285, 2 }, { 37349, 2 }, { 37404, 2 }, { 37410, 1 }, + { 37433, 4 }, { 37615, 2 }, { 37659, 2 }, { 37742, 2 }, { 37773, 2 }, { 37867, 1 }, { 37890, 2 }, { 37960, 2 }, { 38042, 3 }, { 38241, 2 }, { 38400, 2 }, { 38461, 1 }, { 38551, 2 }, { 38611, 1 }, { 38657, 2 }, { 38729, 2 }, { 38748, 2 }, { 38815, 2 }, { 38852, 2 }, { 38890, 1 }, { 38954, 2 }, { 39119, 2 }, { 39162, 2 }, + { 39175, 3 }, { 39176, 2 }, { 39231, 2 }, { 39261, 2 }, { 39467, 2 }, { 39500, 2 }, { 39507, 2 }, { 39566, 2 }, { 39608, 2 }, { 39686, 6 }, { 39730, 2 }, { 39842, 1 }, { 39853, 1 }, { 39905, 2 }, { 39931, 2 }, { 39989, 2 }, { 40030, 2 }, { 40227, 2 }, { 40268, 2 }, { 40372, 2 }, { 40415, 1 }, { 40488, 3 }, { 40536, 2 }, + { 40676, 3 }, { 40677, 2 }, { 40755, 2 }, { 40842, 2 }, { 40849, 1 }, { 40870, 3 }, { 40873, 3 }, { 40972, 2 }, { 41033, 2 }, { 41190, 2 }, { 41273, 5 }, { 41273, 1 }, { 41293, 2 }, { 41367, 32 }, { 41376, 2 }, { 41420, 2 }, { 41473, 2 }, { 41473, 2 }, { 41493, 4 }, { 41521, 2 }, { 41533, 2 }, { 41554, 2 }, { 41568, 2 }, + { 41583, 3 }, { 41728, 2 }, { 41786, 2 }, { 41836, 1 }, { 41875, 2 }, { 41933, 2 }, { 42044, 2 }, { 42075, 2 }, { 42076, 2 }, { 42133, 2 }, { 42259, 29 }, { 42269, 3 }, { 42294, 2 }, { 42420, 2 }, { 42524, 2 }, { 42524, 1 }, { 42546, 1 }, { 42631, 2 }, { 42693, 2 }, { 42740, 2 }, { 42744, 4 }, { 42755, 1 }, { 42870, 2 }, + { 42894, 2 }, { 42939, 2 }, { 42973, 2 }, { 43016, 2 }, { 43070, 2 }, { 43105, 2 }, { 43115, 2 }, { 43375, 3 }, { 43387, 1 }, { 43424, 3 }, { 43448, 2 }, { 43480, 2 }, { 43498, 2 }, { 43651, 2 }, { 43727, 2 }, { 43879, 2 }, { 43910, 1 }, { 43977, 2 }, { 44003, 2 }, { 44080, 2 }, { 44082, 1 }, { 44136, 2 }, { 44169, 29 }, + { 44186, 2 }, { 44339, 2 }, { 44350, 1 }, { 44356, 1 }, { 44430, 2 }, { 44440, 1 }, { 44530, 1 }, { 44538, 2 }, { 44572, 2 }, { 44585, 2 }, { 44709, 2 }, { 44748, 2 }, { 44748, 2 }, { 44769, 2 }, { 44813, 2 }, { 44890, 2 }, { 45015, 2 }, { 45046, 4 }, { 45052, 2 }, { 45062, 2 }, { 45094, 6 }, { 45184, 2 }, { 45191, 2 }, + { 45201, 3 }, { 45216, 3 }, { 45227, 2 }, { 45269, 1 }, { 45294, 2 }, { 45314, 2 }, { 45345, 8 }, { 45352, 2 }, { 45365, 3 }, { 45378, 1 }, { 45392, 4 }, { 45405, 3 }, { 45410, 2 }, { 45448, 14 }, { 45450, 2 }, { 45457, 2 }, { 45466, 3 }, { 45481, 4 }, { 45486, 7 }, { 45533, 5 }, { 45576, 2 }, { 45649, 2 }, { 45917, 2 }, + { 45919, 6 }, { 45919, 1 }, { 45930, 15 }, { 45930, 2 }, { 46001, 5 }, { 46036, 2 }, { 46054, 2 }, { 46075, 2 }, { 46153, 2 }, { 46155, 2 }, { 46228, 2 }, { 46234, 2 }, { 46273, 2 }, { 46387, 2 }, { 46398, 2 }, { 46517, 2 }, { 46559, 2 }, { 46565, 1 }, { 46598, 2 }, { 46686, 2 }, { 46744, 2 }, { 46816, 3 }, { 46835, 2 }, + { 46921, 2 }, { 46938, 2 }, { 46991, 2 }, { 47038, 2 }, { 47098, 3 }, { 47107, 2 }, { 47201, 3 }, { 47327, 1 }, { 47327, 1 }, { 47338, 2 }, { 47395, 1 }, { 47499, 2 }, { 47504, 2 }, { 47515, 1 }, { 47516, 1 }, { 47600, 1 }, { 47604, 1 }, { 47707, 1 }, { 47728, 1 }, { 47748, 2 }, { 47763, 2 }, { 47807, 4 }, { 47814, 2 }, + { 47822, 2 }, { 47834, 2 }, { 47843, 3 }, { 47886, 2 }, { 47893, 2 }, { 48066, 2 }, { 48126, 2 }, { 48133, 1 }, { 48166, 2 }, { 48299, 1 }, { 48455, 2 }, { 48468, 2 }, { 48568, 2 }, { 48606, 2 }, { 48642, 2 }, { 48698, 2 }, { 48714, 2 }, { 48754, 2 }, { 48765, 3 }, { 48773, 5 }, { 48819, 2 }, { 48833, 2 }, { 48904, 2 }, + { 49000, 1 }, { 49113, 12 }, { 49140, 2 }, { 49276, 2 }, { 49353, 2 }, { 49411, 3 }, { 49418, 2 }, { 49540, 2 }, { 49544, 2 }, { 49584, 2 }, { 49602, 2 }, { 49784, 5 }, { 49822, 4 }, { 49822, 5 }, { 49828, 2 }, { 49866, 2 }, { 49922, 3 }, { 49959, 2 }, { 50045, 2 }, { 50134, 3 }, { 50140, 2 }, { 50237, 2 }, { 50247, 2 }, + { 50266, 13 }, { 50290, 2 }, { 50312, 4 }, { 50314, 1 }, { 50527, 2 }, { 50605, 1 }, { 50730, 2 }, { 50751, 2 }, { 50770, 2 }, { 50858, 2 }, { 50859, 2 }, { 50909, 2 }, { 50948, 3 }, { 51043, 2 }, { 51048, 2 }, { 51089, 2 }, { 51090, 2 }, { 51141, 2 }, { 51163, 2 }, { 51250, 2 }, { 51347, 2 }, { 51475, 2 }, { 51536, 2 }, + { 51544, 2 }, { 51595, 2 }, { 51602, 19 }, { 51643, 5 }, { 51702, 2 }, { 51702, 10 }, { 51764, 2 }, { 51793, 5 }, { 51812, 2 }, { 51839, 1 }, { 51938, 3 }, { 51941, 1 }, { 51967, 4 }, { 52049, 3 }, { 52074, 3 }, { 52098, 2 }, { 52118, 2 }, { 52119, 3 }, { 52227, 11 }, { 52246, 3 }, { 52282, 2 }, { 52451, 2 }, { 52583, 2 }, + { 52601, 1 }, { 52605, 2 }, { 52615, 2 }, { 52668, 2 }, { 52824, 2 }, { 53076, 1 }, { 53120, 1 }, { 53179, 2 }, { 53189, 2 }, { 53193, 1 }, { 53195, 2 }, { 53246, 2 }, { 53249, 2 }, { 53268, 1 }, { 53295, 2 }, { 53312, 2 }, { 53410, 2 }, { 53451, 2 }, { 53570, 2 }, { 53593, 2 }, { 53635, 2 }, { 53657, 2 }, { 53682, 3 }, + { 53728, 5 }, { 53733, 2 }, { 53753, 2 }, { 53787, 4 }, { 53807, 1 }, { 54008, 2 }, { 54059, 2 }, { 54060, 1 }, { 54080, 2 }, { 54090, 1 }, { 54138, 2 }, { 54149, 2 }, { 54168, 1 }, { 54171, 2 }, { 54216, 22 }, { 54233, 6 }, { 54434, 2 }, { 54534, 2 }, { 54562, 2 }, { 54763, 2 }, { 54791, 2 }, { 54816, 2 }, { 54909, 2 }, + { 54916, 3 }, { 54963, 2 }, { 54985, 2 }, { 54991, 3 }, { 55016, 3 }, { 55025, 3 }, { 55032, 2 }, { 55099, 2 }, { 55260, 2 }, { 55261, 2 }, { 55270, 3 }, { 55384, 2 }, { 55455, 2 }, { 55456, 2 }, { 55504, 3 }, { 55510, 2 }, { 55558, 2 }, { 55568, 2 }, { 55585, 2 }, { 55677, 2 }, { 55703, 2 }, { 55749, 2 }, { 55779, 2 }, + { 55789, 3 }, { 55792, 2 }, { 55830, 4 }, { 55835, 2 }, { 55879, 2 }, { 56076, 2 }, { 56118, 2 }, { 56314, 2 }, { 56392, 1 }, { 56411, 2 }, { 56459, 2 }, { 56553, 34 }, { 56575, 2 }, { 56733, 2 }, { 56762, 2 }, { 56793, 3 }, { 56877, 3 }, { 56927, 2 }, { 56981, 2 }, { 57014, 1 }, { 57149, 2 }, { 57162, 2 }, { 57186, 2 }, + { 57254, 2 }, { 57267, 1 }, { 57324, 2 }, { 57327, 2 }, { 57365, 4 }, { 57371, 2 }, { 57445, 2 }, { 57477, 2 }, { 57497, 2 }, { 57536, 2 }, { 57609, 2 }, { 57626, 2 }, { 57666, 2 }, { 57694, 2 }, { 57694, 2 }, { 57749, 2 }, { 57781, 7 }, { 57878, 2 }, { 57953, 2 }, { 58051, 2 }, { 58088, 2 }, { 58097, 2 }, { 58142, 3 }, + { 58142, 1 }, { 58197, 1 }, { 58221, 2 }, { 58222, 2 }, { 58244, 2 }, { 58290, 1 }, { 58296, 1 }, { 58325, 2 }, { 58378, 1 }, { 58389, 3 }, { 58430, 2 }, { 58454, 2 }, { 58551, 29 }, { 58563, 6 }, { 58681, 2 }, { 58751, 8 }, { 58752, 43 }, { 58790, 5 }, { 58846, 2 }, { 58879, 6 }, { 58953, 2 }, { 58998, 2 }, { 59010, 1 }, + { 59038, 5 }, { 59135, 2 }, { 59166, 2 }, { 59180, 2 }, { 59222, 2 }, { 59227, 2 }, { 59307, 2 }, { 59398, 3 }, { 59411, 2 }, { 59436, 3 }, { 59464, 2 }, { 59569, 2 }, { 59587, 2 }, { 59624, 3 }, { 59786, 2 }, { 59834, 2 }, { 59841, 2 }, { 59841, 1 }, { 59984, 2 }, { 59985, 2 }, { 60003, 3 }, { 60045, 2 }, { 60097, 2 }, + { 60148, 2 }, { 60172, 2 }, { 60203, 5 }, { 60565, 2 }, { 60625, 2 }, { 60743, 2 }, { 60781, 2 }, { 60892, 2 }, { 60977, 2 }, { 60979, 2 }, { 61021, 5 }, { 61021, 4 }, { 61026, 2 }, { 61139, 2 }, { 61165, 3 }, { 61204, 2 }, { 61207, 1 }, { 61248, 3 }, { 61257, 2 }, { 61264, 6 }, { 61272, 3 }, { 61410, 2 }, { 61410, 3 }, + { 61416, 2 }, { 61423, 1 }, { 61503, 2 }, { 61503, 2 }, { 61533, 2 }, { 61567, 2 }, { 61575, 2 }, { 61835, 1 }, { 61842, 1 }, { 61924, 2 }, { 61951, 6 }, { 61975, 2 }, { 61986, 3 }, { 62024, 1 }, { 62110, 2 }, { 62135, 2 }, { 62192, 2 }, { 62208, 2 }, { 62399, 2 }, { 62400, 1 }, { 62414, 2 }, { 62423, 3 }, { 62456, 3 }, + { 62459, 3 }, { 62478, 3 }, { 62484, 2 }, { 62510, 6 }, { 62511, 3 }, { 62565, 3 }, { 62610, 2 }, { 62875, 4 }, { 62896, 5 }, { 62898, 2 }, { 62904, 2 }, { 62938, 3 }, { 62943, 2 }, { 62977, 2 }, { 62989, 3 }, { 62998, 5 }, { 63069, 1 }, { 63093, 5 }, { 63107, 2 }, { 63113, 1 }, { 63231, 4 }, { 63253, 2 }, { 63286, 4 }, + { 63289, 2 }, { 63334, 1 }, { 63334, 4 }, { 63413, 2 }, { 63425, 2 }, { 63512, 10 }, { 63537, 1 }, { 63694, 1 }, { 63721, 4 }, { 63749, 2 }, { 63783, 17 }, { 63791, 3 }, { 63792, 2 }, { 63882, 25 }, { 63896, 1 }, { 63936, 2 }, { 63969, 3 }, { 63986, 2 }, { 63988, 2 }, { 64009, 10 }, { 64018, 2 }, { 64032, 6 }, { 64125, 2 }, + { 64195, 1 }, { 64221, 7 }, { 64390, 2 }, { 64459, 2 }, { 64568, 2 }, { 64784, 1 }, { 64789, 2 }, { 64829, 2 }, { 64848, 1 }, { 64914, 2 }, { 64928, 1 }, { 64939, 2 }, { 65026, 2 }, { 65057, 2 }, { 65070, 2 }, { 65193, 4 }, { 65235, 3 }, { 65242, 2 }, { 65281, 2 }, { 65320, 2 }, { 65365, 1 }, { 65414, 2 }, { 65445, 2 }, + { 65581, 2 }, { 65624, 1 }, { 65719, 2 }, { 65766, 2 }, { 65927, 2 }, { 66004, 1 }, { 66031, 2 }, { 66085, 1 }, { 66085, 2 }, { 66133, 2 }, { 66134, 2 }, { 66188, 1 }, { 66240, 2 }, { 66249, 2 }, { 66250, 2 }, { 66295, 2 }, { 66342, 1 }, { 66352, 3 }, { 66388, 3 }, { 66432, 2 }, { 66437, 47 }, { 66497, 2 }, { 66517, 2 }, + { 66526, 2 }, { 66546, 9 }, { 66605, 2 }, { 66753, 2 }, { 66792, 2 }, { 66796, 2 }, { 66828, 2 }, { 66899, 3 }, { 66970, 6 }, { 66981, 2 }, { 66983, 1 }, { 67009, 2 }, { 67017, 4 }, { 67115, 2 }, { 67117, 1 }, { 67130, 6 }, { 67132, 7 }, { 67162, 2 }, { 67179, 6 }, { 67236, 2 }, { 67263, 3 }, { 67274, 2 }, { 67274, 2 }, + { 67349, 3 }, { 67486, 2 }, { 67503, 3 }, { 67517, 1 }, { 67559, 1 }, { 67660, 2 }, { 67727, 2 }, { 67901, 2 }, { 67943, 4 }, { 67950, 2 }, { 67965, 3 }, { 68029, 2 }, { 68048, 2 }, { 68169, 2 }, { 68172, 1 }, { 68258, 2 }, { 68288, 1 }, { 68359, 2 }, { 68441, 2 }, { 68484, 2 }, { 68488, 2 }, { 68525, 2 }, { 68535, 2 }, + { 68575, 7 }, { 68575, 5 }, { 68583, 2 }, { 68588, 4 }, { 68593, 1 }, { 68597, 2 }, { 68636, 2 }, { 68636, 2 }, { 68667, 2 }, { 68785, 1 }, { 68914, 4 }, { 68915, 5 }, { 68940, 3 }, { 69010, 2 }, { 69063, 2 }, { 69076, 2 }, { 69235, 2 }, { 69270, 2 }, { 69298, 1 }, { 69350, 5 }, { 69432, 2 }, { 69514, 2 }, { 69562, 3 }, + { 69562, 4 }, { 69638, 1 }, { 69656, 2 }, { 69709, 2 }, { 69775, 2 }, { 69788, 2 }, { 70193, 2 }, { 70233, 2 }, { 70252, 2 }, { 70259, 2 }, { 70293, 3 }, { 70405, 3 }, { 70462, 2 }, { 70515, 3 }, { 70518, 2 }, { 70535, 6 }, { 70547, 6 }, { 70577, 6 }, { 70631, 17 }, { 70667, 2 }, { 70680, 1 }, { 70694, 1 }, { 70898, 2 }, + { 70916, 1 }, { 70936, 3 }, { 71033, 2 }, { 71126, 2 }, { 71158, 2 }, { 71162, 2 }, { 71421, 1 }, { 71441, 2 }, { 71557, 2 }, { 71789, 1 }, { 71816, 2 }, { 71850, 1 }, { 71869, 1 }, { 71961, 2 }, { 71973, 4 }, { 72064, 2 }, { 72110, 2 }, { 72117, 3 }, { 72164, 2 }, { 72266, 2 }, { 72325, 2 }, { 72326, 1 }, { 72420, 2 }, + { 72693, 2 }, { 72705, 1 }, { 72730, 2 }, { 72793, 2 }, { 72795, 1 }, { 72939, 1 }, { 72945, 3 }, { 72945, 2 }, { 73120, 1 }, { 73121, 5 }, { 73122, 4 }, { 73126, 1 }, { 73126, 1 }, { 73196, 3 }, { 73219, 2 }, { 73241, 6 }, { 73272, 3 }, { 73354, 1 }, { 73368, 2 }, { 73467, 1 }, { 73517, 2 }, { 73554, 2 }, { 73678, 2 }, + { 73838, 1 }, { 73881, 2 }, { 73958, 2 }, { 73985, 15 }, { 74092, 2 }, { 74205, 2 }, { 74245, 2 }, { 74277, 2 }, { 74286, 2 }, { 74353, 2 }, { 74403, 2 }, { 74428, 1 }, { 74468, 2 }, { 74481, 3 }, { 74511, 2 }, { 74537, 2 }, { 74596, 2 }, { 74750, 2 }, { 74754, 2 }, { 74861, 2 }, { 74933, 4 }, { 74970, 1 }, { 75003, 3 }, + { 75077, 1 }, { 75159, 2 }, { 75170, 2 }, { 75234, 2 }, { 75300, 3 }, { 75337, 2 }, { 75345, 2 }, { 75419, 1 }, { 75429, 2 }, { 75477, 1 }, { 75513, 2 }, { 75536, 2 }, { 75536, 2 }, { 75539, 1 }, { 75551, 2 }, { 75561, 2 }, { 75565, 2 }, { 75590, 2 }, { 75623, 5 }, { 75773, 6 }, { 75777, 6 }, { 75785, 2 }, { 75791, 2 }, + { 75804, 2 }, { 75862, 2 }, { 75924, 3 }, { 75927, 2 }, { 75996, 11 }, { 76000, 1 }, { 76006, 2 }, { 76020, 3 }, { 76110, 2 }, { 76126, 3 }, { 76131, 2 }, { 76136, 2 }, { 76144, 2 }, { 76203, 2 }, { 76229, 3 }, { 76244, 15 }, { 76246, 2 }, { 76300, 1 }, { 76403, 3 }, { 76545, 2 }, { 76569, 2 }, { 76813, 2 }, { 76821, 2 }, + { 76837, 2 }, { 76863, 2 }, { 77027, 2 }, { 77037, 2 }, { 77074, 3 }, { 77170, 2 }, { 77191, 2 }, { 77220, 2 }, { 77230, 2 }, { 77261, 2 }, { 77277, 2 }, { 77309, 2 }, { 77314, 2 }, { 77412, 2 }, { 77419, 2 }, { 77457, 2 }, { 77633, 3 }, { 77714, 2 }, { 77855, 2 }, { 77857, 1 }, { 77876, 2 }, { 77895, 2 }, { 77916, 5 }, + { 77947, 2 }, { 77948, 1 }, { 77966, 1 }, { 77996, 2 }, { 78025, 1 }, { 78064, 2 }, { 78100, 2 }, { 78113, 1 }, { 78114, 3 }, { 78167, 2 }, { 78175, 2 }, { 78260, 2 }, { 78261, 1 }, { 78265, 2 }, { 78286, 1 }, { 78300, 2 }, { 78327, 3 }, { 78363, 1 }, { 78384, 2 }, { 78459, 2 }, { 78516, 2 }, { 78612, 2 }, { 78643, 2 }, + { 78655, 2 }, { 78698, 1 }, { 78720, 3 }, { 78789, 3 }, { 78838, 5 }, { 78893, 1 }, { 78954, 7 }, { 79007, 2 }, { 79132, 3 }, { 79193, 2 }, { 79193, 2 }, { 79226, 2 }, { 79411, 2 }, { 79422, 1 }, { 79502, 2 }, { 79593, 2 }, { 79622, 2 }, { 79657, 3 }, { 79771, 2 }, { 79866, 2 }, { 79909, 2 }, { 80005, 2 }, { 80032, 2 }, + { 80060, 1 }, { 80132, 2 }, { 80149, 3 }, { 80251, 2 }, { 80363, 2 }, { 80379, 1 }, { 80464, 2 }, { 80498, 2 }, { 80553, 2 }, { 80556, 3 }, { 80559, 1 }, { 80571, 2 }, { 80652, 1 }, { 80703, 2 }, { 80754, 2 }, { 80754, 2 }, { 80860, 2 }, { 81055, 2 }, { 81087, 4 }, { 81210, 2 }, { 81211, 1 }, { 81216, 1 }, { 81223, 1 }, + { 81231, 1 }, { 81288, 2 }, { 81317, 2 }, { 81327, 3 }, { 81332, 2 }, { 81376, 2 }, { 81469, 2 }, { 81579, 2 }, { 81617, 1 }, { 81630, 2 }, { 81666, 2 }, { 81800, 2 }, { 81832, 2 }, { 81848, 2 }, { 81869, 2 }, { 81941, 3 }, { 82177, 3 }, { 82179, 2 }, { 82180, 2 }, { 82182, 4 }, { 82185, 2 }, { 82195, 2 }, { 82238, 4 }, + { 82265, 3 }, { 82295, 10 }, { 82299, 9 }, { 82367, 3 }, { 82379, 3 }, { 82380, 1 }, { 82505, 2 }, { 82568, 2 }, { 82620, 1 }, { 82637, 5 }, { 82821, 2 }, { 82841, 2 }, { 82945, 1 }, { 83020, 12 }, { 83072, 2 }, { 83181, 2 }, { 83240, 2 }, { 83253, 3 }, { 83261, 2 }, { 83288, 2 }, { 83291, 4 }, { 83295, 3 }, { 83365, 2 }, + { 83368, 2 }, { 83408, 2 }, { 83458, 2 }, { 83470, 2 }, { 83471, 1 }, { 83637, 3 }, { 83693, 2 }, { 83703, 2 }, { 83732, 2 }, { 83745, 1 }, { 83800, 4 }, { 83801, 3 }, { 83856, 3 }, { 83863, 5 }, { 83867, 2 }, { 83868, 3 }, { 83898, 7 }, { 83900, 4 }, { 83901, 5 }, { 83989, 2 }, { 84049, 35 }, { 84086, 2 }, { 84089, 2 }, + { 84115, 3 }, { 84130, 3 }, { 84132, 2 }, { 84143, 3 }, { 84173, 2 }, { 84185, 5 }, { 84297, 2 }, { 84390, 2 }, { 84497, 4 }, { 84657, 2 }, { 84657, 2 }, { 84724, 2 }, { 84775, 2 }, { 84870, 2 }, { 84892, 2 }, { 84910, 3 }, { 84935, 3 }, { 85002, 2 }, { 85051, 2 }, { 85052, 2 }, { 85135, 25 }, { 85135, 2 }, { 85144, 2 }, + { 85165, 3 }, { 85205, 2 }, { 85232, 2 }, { 85281, 5 }, { 85423, 6 }, { 85539, 2 }, { 85582, 4 }, { 85609, 2 }, { 85701, 36 }, { 85705, 2 }, { 85824, 2 }, { 85824, 2 }, { 85858, 30 }, { 85858, 28 }, { 85904, 35 }, { 85910, 2 }, { 85913, 2 }, { 85926, 3 }, { 85942, 4 }, { 85969, 4 }, { 85996, 1 }, { 86013, 3 }, { 86034, 13 }, + { 86068, 8 }, { 86069, 8 }, { 86089, 8 }, { 86193, 13 }, { 86217, 7 }, { 86219, 2 }, { 86250, 2 }, { 86304, 16 }, { 86317, 2 }, { 86322, 4 }, { 86325, 2 }, { 86333, 2 }, { 86394, 2 }, { 86433, 2 }, { 86469, 3 }, { 86512, 4 }, { 86537, 2 }, { 86627, 2 }, { 86658, 2 }, { 86810, 2 }, { 86813, 2 }, { 86884, 2 }, { 86947, 2 }, + { 87003, 2 }, { 87010, 5 }, { 87019, 2 }, { 87027, 2 }, { 87105, 2 }, { 87107, 2 }, { 87183, 2 }, { 87273, 2 }, { 87358, 3 }, { 87388, 3 }, { 87503, 4 }, { 87639, 2 }, { 87649, 4 }, { 87722, 2 }, { 87829, 2 }, { 87829, 1 }, { 87863, 2 }, { 87894, 2 }, { 87988, 32 }, { 88035, 27 }, { 88059, 3 }, { 88094, 5 }, { 88111, 21 }, + { 88129, 2 }, { 88175, 5 }, { 88256, 2 }, { 88329, 2 }, { 88415, 3 }, { 88482, 2 }, { 88502, 1 }, { 88529, 2 }, { 88551, 3 }, { 88552, 1 }, { 88713, 2 }, { 88797, 2 }, { 88844, 27 }, { 88925, 5 }, { 88935, 2 }, { 88944, 1 }, { 89073, 2 }, { 89095, 3 }, { 89283, 2 }, { 89294, 3 }, { 89299, 2 }, { 89324, 2 }, { 89368, 2 }, + { 89387, 2 }, { 89464, 2 }, { 89607, 2 }, { 89737, 2 }, { 89791, 2 }, { 89794, 3 }, { 89840, 2 }, { 89849, 3 }, { 89859, 2 }, { 89905, 2 }, { 89952, 38 }, { 90030, 7 }, { 90030, 6 }, { 90031, 1 }, { 90072, 2 }, { 90090, 2 }, { 90146, 3 }, { 90202, 23 }, { 90302, 3 }, { 90328, 14 }, { 90335, 14 }, { 90338, 8 }, { 90380, 2 }, + { 90434, 1 }, { 90482, 2 }, { 90527, 9 }, { 90537, 3 }, { 90545, 2 }, { 90639, 5 }, { 90642, 2 }, { 90709, 2 }, { 90775, 1 }, { 90806, 2 }, { 90845, 19 }, { 90872, 4 }, { 90884, 2 }, { 90910, 2 }, { 90994, 5 }, { 91046, 8 }, { 91059, 8 }, { 91096, 39 }, { 91147, 2 }, { 91168, 1 }, { 91493, 2 }, { 91513, 3 }, { 91618, 3 }, + { 91653, 2 }, { 91817, 2 }, { 91831, 3 }, { 91833, 3 }, { 91885, 2 }, { 91919, 2 }, { 91934, 2 }, { 92245, 1 }, { 92284, 2 }, { 92292, 4 }, { 92369, 3 }, { 92388, 2 }, { 92426, 7 }, { 92720, 14 }, { 92720, 6 }, { 92729, 9 }, { 92733, 13 }, { 92735, 6 }, { 92786, 2 }, { 92853, 31 }, { 92906, 2 }, { 93031, 7 }, { 93077, 2 }, + { 93102, 2 }, { 93109, 2 }, { 93122, 3 }, { 93214, 2 }, { 93330, 2 }, { 93395, 2 }, { 93506, 2 }, { 93564, 9 }, { 93713, 9 }, { 93722, 4 }, { 93840, 2 }, { 93877, 4 }, { 93891, 3 }, { 93948, 2 }, { 93981, 2 }, { 94012, 3 }, { 94033, 2 }, { 94121, 2 }, { 94165, 32 }, { 94181, 3 }, { 94210, 2 }, { 94216, 2 }, { 94230, 2 }, + { 94333, 31 }, { 94433, 3 }, { 94497, 3 }, { 94609, 2 }, { 94623, 2 }, { 94763, 2 }, { 94780, 2 }, { 95287, 2 }, { 95348, 2 }, { 95433, 5 }, { 95446, 2 }, { 95493, 7 }, { 95517, 3 }, { 95580, 2 }, { 95610, 5 }, { 95620, 2 }, { 95678, 3 }, { 95683, 2 }, { 95689, 2 }, { 95760, 2 }, { 95792, 2 }, { 95850, 2 }, { 95908, 2 }, + { 95908, 2 }, { 95967, 2 }, { 96022, 3 }, { 96088, 2 }, { 96460, 2 }, { 96554, 2 }, { 96597, 2 }, { 96763, 2 }, { 96808, 2 }, { 96854, 1 }, { 96963, 1 }, { 97007, 3 }, { 97125, 1 }, { 97128, 2 }, { 97133, 3 }, { 97142, 3 }, { 97156, 2 }, { 97223, 2 }, { 97244, 2 }, { 97303, 2 }, { 97355, 2 }, { 97356, 3 }, { 97393, 3 }, + { 97409, 1 }, { 97451, 2 }, { 97539, 2 }, { 97546, 2 }, { 97553, 2 }, { 97627, 2 }, { 97640, 2 }, { 97650, 6 }, { 97675, 2 }, { 97685, 3 }, { 97773, 2 }, { 97802, 4 }, { 97826, 19 }, { 97860, 2 }, { 97956, 2 }, { 97958, 2 }, { 97973, 3 }, { 97982, 2 }, { 98039, 2 }, { 98051, 2 }, { 98059, 2 }, { 98088, 2 }, { 98092, 4 }, + { 98147, 2 }, { 98147, 2 }, { 98169, 2 }, { 98207, 2 }, { 98277, 1 }, { 98277, 22 }, { 98285, 2 }, { 98324, 3 }, { 98324, 3 }, { 98381, 31 }, { 98390, 2 }, { 98404, 2 }, { 98415, 4 }, { 98460, 2 }, { 98462, 1 }, { 98475, 3 }, { 98485, 2 }, { 98640, 1 }, { 98798, 2 }, { 98800, 4 }, { 98821, 2 }, { 98895, 2 }, { 98936, 2 }, + { 98950, 2 }, { 98980, 2 }, { 99033, 2 }, { 99045, 2 }, { 99135, 2 }, { 99315, 30 }, { 99324, 2 }, { 99346, 2 }, { 99418, 2 }, { 99505, 2 }, { 99557, 2 }, { 99559, 2 }, { 99586, 2 }, { 99622, 2 }, { 99770, 1 }, { 99790, 2 }, { 99810, 2 }, { 99871, 1 }, { 99926, 2 }, { 99927, 2 }, { 99978, 2 }, { 99980, 2 }, { 100022, 3 }, + { 100024, 1 }, { 100069, 2 }, { 100150, 2 }, { 100225, 2 }, { 100246, 1 }, { 100310, 2 }, { 100361, 2 }, { 100428, 1 }, { 100434, 2 }, { 100450, 4 }, { 100546, 2 }, { 100551, 2 }, { 100551, 2 }, { 100554, 1 }, { 100597, 2 }, { 100676, 2 }, { 100693, 2 }, { 100827, 2 }, { 100928, 2 }, { 100928, 1 }, { 100935, 2 }, { 100937, 3 }, + { 101034, 2 }, { 101041, 2 }, { 101154, 2 }, { 101200, 4 }, { 101250, 2 }, { 101352, 2 }, { 101403, 2 }, { 101430, 1 }, { 101508, 3 }, { 101509, 3 }, { 101523, 10 }, { 101604, 2 }, { 101637, 2 }, { 101681, 4 }, { 101759, 1 }, { 101773, 1 }, { 101836, 1 }, { 101882, 4 }, { 101895, 2 }, { 101897, 2 }, { 101939, 2 }, { 101951, 6 }, + { 101956, 5 }, { 102055, 1 }, { 102085, 2 }, { 102093, 2 }, { 102209, 2 }, { 102258, 6 }, { 102271, 2 }, { 102284, 2 }, { 102332, 2 }, { 102354, 2 }, { 102366, 2 }, { 102424, 3 }, { 102456, 2 }, { 102496, 1 }, { 102497, 3 }, { 102519, 3 }, { 102554, 1 }, { 102610, 5 }, { 102657, 2 }, { 102661, 4 }, { 102695, 4 }, { 102707, 12 }, + { 102910, 2 }, { 102930, 5 }, { 102937, 9 }, { 102938, 7 }, { 102965, 6 }, { 102969, 7 }, { 103031, 2 }, { 103062, 2 }, { 103096, 2 }, { 103146, 2 }, { 103159, 2 }, { 103223, 2 }, { 103267, 2 }, { 103296, 2 }, { 103303, 2 }, { 103487, 2 }, { 103491, 2 }, { 103599, 2 }, { 103677, 2 }, { 103903, 1 }, { 104040, 2 }, { 104047, 1 }, + { 104052, 2 }, { 104057, 4 }, { 104057, 2 }, { 104062, 4 }, { 104091, 2 }, { 104189, 3 }, { 104283, 8 }, { 104288, 4 }, { 104305, 3 }, { 104445, 2 }, { 104472, 2 }, { 104475, 1 }, { 104497, 4 }, { 104548, 2 }, { 104582, 2 }, { 104626, 1 }, { 104716, 2 }, { 104826, 2 }, { 104849, 2 }, { 104872, 1 }, { 104945, 1 }, { 104948, 2 }, + { 105066, 2 }, { 105071, 1 }, { 105198, 4 }, { 105198, 4 }, { 105203, 2 }, { 105256, 6 }, { 105263, 2 }, { 105329, 2 }, { 105515, 2 }, { 105566, 2 }, { 105566, 2 }, { 105585, 2 }, { 105678, 2 }, { 105852, 2 }, { 105877, 2 }, { 105911, 2 }, { 106022, 1 }, { 106033, 2 }, { 106080, 2 }, { 106192, 2 }, { 106220, 3 }, { 106243, 2 }, + { 106323, 11 }, { 106371, 2 }, { 106608, 2 }, { 106624, 2 }, { 106680, 3 }, { 106688, 1 }, { 106800, 1 }, { 106800, 1 }, { 106821, 4 }, { 106853, 1 }, { 106930, 3 }, { 106937, 2 }, { 106955, 2 }, { 106996, 2 }, { 106996, 1 }, { 107148, 4 }, { 107213, 16 }, { 107213, 2 }, { 107243, 2 }, { 107360, 2 }, { 107408, 2 }, { 107509, 4 }, + { 107572, 2 }, { 107592, 2 }, { 107644, 5 }, { 107679, 2 }, { 107705, 3 }, { 107761, 4 }, { 107780, 2 }, { 107825, 2 }, { 108007, 2 }, { 108041, 4 }, { 108058, 2 }, { 108071, 1 }, { 108132, 2 }, { 108164, 2 }, { 108189, 2 }, { 108210, 2 }, { 108330, 2 }, { 108430, 2 }, { 108450, 2 }, { 108469, 2 }, { 108484, 2 }, { 108533, 2 }, + { 108588, 2 }, { 108594, 2 }, { 108690, 2 }, { 108785, 1 }, { 108814, 2 }, { 108818, 1 }, { 108820, 2 }, { 108889, 2 }, { 108951, 2 }, { 108959, 2 }, { 108963, 2 }, { 109034, 2 }, { 109172, 1 }, { 109176, 2 }, { 109195, 3 }, { 109229, 2 }, { 109256, 2 }, { 109290, 2 }, { 109304, 2 }, { 109333, 2 }, { 109343, 4 }, { 109347, 7 }, + { 109387, 2 }, { 109421, 1 }, { 109497, 2 }, { 109501, 3 }, { 109513, 2 }, { 109525, 3 }, { 109625, 4 }, { 109710, 2 }, { 109740, 2 }, { 109751, 2 }, { 109761, 2 }, { 109890, 8 }, { 109891, 4 }, { 109909, 2 }, { 109923, 1 }, { 110017, 2 }, { 110046, 2 }, { 110111, 2 }, { 110258, 2 }, { 110340, 2 }, { 110352, 2 }, { 110398, 2 }, + { 110583, 2 }, { 110600, 13 }, { 110626, 3 }, { 110709, 2 }, { 110772, 4 }, { 110773, 2 }, { 110813, 1 }, { 110890, 2 }, { 110898, 2 }, { 110954, 2 }, { 111120, 2 }, { 111132, 3 }, { 111163, 8 }, { 111224, 2 }, { 111340, 2 }, { 111398, 2 }, { 111555, 2 }, { 111597, 3 }, { 111607, 2 }, { 111655, 2 }, { 111691, 3 }, { 111835, 2 }, + { 111854, 2 }, { 111876, 16 }, { 111884, 1 }, { 111884, 2 }, { 111929, 2 }, { 111941, 2 }, { 111969, 2 }, { 112003, 2 }, { 112165, 2 }, { 112365, 2 }, { 112450, 1 }, { 112521, 2 }, { 112649, 4 }, { 112665, 2 }, { 112881, 1 }, { 112882, 2 }, { 112906, 2 }, { 112951, 2 }, { 112994, 2 }, { 112997, 2 }, { 113002, 2 }, { 113056, 1 }, + { 113077, 2 }, { 113208, 1 }, { 113320, 2 }, { 113326, 3 }, { 113375, 2 }, { 113530, 30 }, { 113530, 30 }, { 113537, 1 }, { 113563, 14 }, { 113592, 2 }, { 113637, 2 }, { 113768, 2 }, { 113850, 5 }, { 113892, 2 }, { 113916, 2 }, { 113965, 2 }, { 113976, 2 }, { 114037, 2 }, { 114149, 1 }, { 114158, 9 }, { 114201, 2 }, { 114262, 2 }, + { 114268, 4 }, { 114353, 2 }, { 114388, 2 }, { 114404, 2 }, { 114428, 5 }, { 114438, 2 }, { 114541, 2 }, { 114550, 2 }, { 114561, 2 }, { 114625, 3 }, { 114730, 2 }, { 114770, 1 }, { 114815, 4 }, { 114998, 2 }, { 115077, 2 }, { 115093, 2 }, { 115120, 2 }, { 115194, 2 }, { 115216, 3 }, { 115299, 2 }, { 115391, 3 }, { 115410, 2 }, + { 115542, 33 }, { 115581, 2 }, { 115618, 2 }, { 115645, 5 }, { 115647, 2 }, { 115697, 2 }, { 115725, 2 }, { 115740, 2 }, { 115757, 2 }, { 115763, 2 }, { 115770, 2 }, { 115787, 2 }, { 115916, 2 }, { 115928, 2 }, { 115962, 2 }, { 116020, 2 }, { 116022, 1 }, { 116089, 2 }, { 116159, 1 }, { 116196, 2 }, { 116247, 2 }, { 116254, 7 }, + { 116336, 2 }, { 116409, 2 }, { 116459, 2 }, { 116569, 2 }, { 116619, 2 }, { 116688, 2 }, { 116733, 2 }, { 116807, 3 }, { 116843, 2 }, { 116886, 1 }, { 116902, 2 }, { 116931, 2 }, { 116952, 2 }, { 116952, 2 }, { 117177, 2 }, { 117189, 2 }, { 117206, 2 }, { 117260, 29 }, { 117271, 6 }, { 117276, 3 }, { 117276, 5 }, { 117278, 3 }, + { 117278, 2 }, { 117359, 4 }, { 117380, 2 }, { 117414, 1 }, { 117503, 2 }, { 117517, 2 }, { 117530, 2 }, { 117574, 4 }, { 117575, 5 }, { 117577, 2 }, { 117606, 2 }, { 117645, 2 }, { 117655, 2 }, { 117692, 2 }, { 117705, 1 }, { 117731, 1 }, { 117762, 4 }, { 117780, 2 }, { 117974, 1 }, { 118057, 1 }, { 118099, 2 }, { 118107, 2 }, + { 118113, 2 }, { 118175, 2 }, { 118198, 2 }, { 118232, 2 }, { 118326, 1 }, { 118438, 31 }, { 118469, 2 }, { 118521, 31 }, { 118565, 2 }, { 118593, 2 }, { 118602, 2 }, { 118652, 2 }, { 118668, 2 }, { 118689, 3 }, { 118703, 14 }, { 118705, 2 }, { 118813, 2 }, { 118825, 2 }, { 118894, 3 }, { 118915, 2 }, { 118962, 2 }, { 118986, 2 }, + { 119045, 2 }, { 119054, 1 }, { 119054, 1 }, { 119119, 2 }, { 119149, 2 }, { 119206, 1 }, { 119316, 2 }, { 119387, 2 }, { 119404, 3 }, { 119516, 2 }, { 119520, 2 }, { 119571, 3 }, { 119573, 2 }, { 119610, 5 }, { 119621, 2 }, { 119623, 4 }, { 119672, 2 }, { 119692, 3 }, { 119734, 2 }, { 119742, 1 }, { 119754, 1 }, { 119785, 2 }, + { 120001, 2 }, { 120115, 4 }, { 120260, 2 }, { 120314, 2 }, { 120416, 2 }, { 120435, 1 }, { 120450, 3 }, { 120530, 2 }, { 120550, 5 }, { 120730, 2 }, { 120731, 2 }, { 120751, 3 }, { 120755, 2 }, { 120869, 2 }, { 120988, 2 }, { 121061, 2 }, { 121177, 2 }, { 121212, 2 }, { 121214, 1 }, { 121286, 2 }, { 121331, 1 }, { 121344, 2 }, + { 121407, 2 }, { 121424, 1 }, { 121491, 2 }, { 121568, 1 }, { 121588, 6 }, { 121651, 2 }, { 121676, 2 }, { 121785, 4 }, { 121830, 3 }, { 121919, 1 }, { 121951, 2 }, { 121991, 1 }, { 122056, 2 }, { 122062, 2 }, { 122144, 2 }, { 122183, 1 }, { 122331, 2 }, { 122466, 2 }, { 122558, 2 }, { 122570, 2 }, { 122676, 2 }, { 122733, 2 }, + { 122774, 6 }, { 122783, 2 }, { 122825, 2 }, { 122865, 2 }, { 122884, 2 }, { 122892, 2 }, { 122911, 2 }, { 122929, 2 }, { 122936, 2 }, { 123190, 2 }, { 123271, 2 }, { 123271, 2 }, { 123302, 7 }, { 123391, 2 }, { 123394, 2 }, { 123416, 1 }, { 123708, 2 }, { 123752, 2 }, { 123761, 2 }, { 123783, 2 }, { 123794, 2 }, { 123817, 2 }, + { 123820, 1 }, { 123823, 1 }, { 123857, 3 }, { 123886, 2 }, { 124023, 1 }, { 124029, 2 }, { 124042, 2 }, { 124056, 3 }, { 124071, 6 }, { 124105, 5 }, { 124143, 2 }, { 124191, 2 }, { 124207, 1 }, { 124257, 2 }, { 124306, 3 }, { 124338, 2 }, { 124388, 8 }, { 124400, 2 }, { 124418, 2 }, { 124502, 2 }, { 124521, 1 }, { 124533, 2 }, + { 124645, 2 }, { 124685, 1 }, { 124694, 2 }, { 124700, 1 }, { 124736, 2 }, { 124891, 7 }, { 124920, 2 }, { 124983, 2 }, { 125014, 2 }, { 125038, 2 }, { 125084, 2 }, { 125162, 2 }, { 125193, 2 }, { 125285, 2 }, { 125368, 2 }, { 125409, 2 }, { 125570, 2 }, { 125601, 2 }, { 125641, 1 }, { 125721, 2 }, { 125731, 2 }, { 125803, 2 }, + { 125904, 2 }, { 125973, 2 }, { 126018, 1 }, { 126034, 5 }, { 126094, 1 }, { 126144, 1 }, { 126195, 2 }, { 126297, 2 }, { 126389, 2 }, { 126429, 2 }, { 126439, 2 }, { 126499, 2 }, { 126501, 1 }, { 126587, 2 }, { 126663, 2 }, { 126681, 2 }, { 126687, 1 }, { 126781, 2 }, { 126783, 2 }, { 126840, 8 }, { 126843, 2 }, { 126959, 2 }, + { 127015, 2 }, { 127101, 2 }, { 127149, 2 }, { 127197, 3 }, { 127268, 2 }, { 127372, 2 }, { 127385, 2 }, { 127473, 4 }, { 127539, 2 }, { 127598, 2 }, { 127613, 14 }, { 127683, 3 }, { 127684, 2 }, { 127697, 2 }, { 127698, 3 }, { 127773, 2 }, { 127781, 1 }, { 127839, 2 }, { 127905, 2 }, { 127949, 2 }, { 128035, 2 }, { 128046, 1 }, + { 128167, 2 }, { 128271, 2 }, { 128307, 1 }, { 128320, 2 }, { 128330, 2 }, { 128375, 2 }, { 128381, 4 }, { 128447, 2 }, { 128462, 2 }, { 128466, 3 }, { 128466, 2 }, { 128496, 2 }, { 128589, 2 }, { 128616, 3 }, { 128679, 1 }, { 128770, 1 }, { 128793, 2 }, { 128802, 2 }, { 128813, 2 }, { 128900, 2 }, { 128949, 2 }, { 129269, 2 }, + { 129271, 3 }, { 129278, 2 }, { 129343, 1 }, { 129408, 2 }, { 129408, 1 }, { 129421, 6 }, { 129461, 2 }, { 129469, 3 }, { 129482, 2 }, { 129502, 2 }, { 129512, 2 }, { 129551, 2 }, { 129629, 2 }, { 129632, 2 }, { 129679, 1 }, { 129725, 2 }, { 130007, 2 }, { 130018, 16 }, { 130057, 2 }, { 130071, 2 }, { 130087, 2 }, { 130188, 1 }, + { 130202, 2 }, { 130316, 2 }, { 130328, 1 }, { 130466, 2 }, { 130549, 2 }, { 130649, 2 }, { 130705, 3 }, { 130800, 2 }, { 130907, 2 }, { 130989, 2 }, { 131103, 2 }, { 131127, 2 }, { 131200, 5 }, { 131241, 6 }, { 131351, 2 }, { 131413, 2 }, { 131448, 2 }, { 131599, 2 }, { 131634, 1 }, { 131687, 2 }, { 131739, 2 }, { 131758, 2 }, + { 131765, 2 }, { 131787, 3 }, { 131819, 3 }, { 131868, 2 }, { 131886, 2 }, { 131901, 4 }, { 131977, 2 }, { 131990, 2 }, { 132035, 2 }, { 132035, 2 }, { 132043, 2 }, { 132173, 2 }, { 132181, 4 }, { 132181, 6 }, { 132194, 5 }, { 132252, 2 }, { 132262, 6 }, { 132271, 1 }, { 132285, 2 }, { 132328, 2 }, { 132335, 1 }, { 132337, 1 }, + { 132389, 5 }, { 132430, 2 }, { 132451, 2 }, { 132499, 4 }, { 132503, 1 }, { 132520, 4 }, { 132541, 4 }, { 132860, 2 }, { 132862, 4 }, { 132874, 12 }, { 132874, 13 }, { 132875, 12 }, { 132911, 2 }, { 132973, 2 }, { 133051, 2 }, { 133062, 2 }, { 133067, 2 }, { 133138, 2 }, { 133184, 2 }, { 133231, 2 }, { 133297, 3 }, { 133344, 2 }, + { 133385, 4 }, { 133408, 2 }, { 133464, 2 }, { 133522, 2 }, { 133631, 2 }, { 133631, 2 }, { 133702, 2 }, { 133705, 1 }, { 133721, 2 }, { 133746, 2 }, { 133773, 3 }, { 133819, 2 }, { 133843, 2 }, { 133929, 2 }, { 133946, 2 }, { 134113, 4 }, { 134151, 2 }, { 134289, 1 }, { 134385, 2 }, { 134429, 2 }, { 134506, 2 }, { 134511, 2 }, + { 134521, 2 }, { 134558, 1 }, { 134710, 2 }, { 134738, 2 }, { 134751, 3 }, { 134818, 2 }, { 134820, 4 }, { 134879, 2 }, { 134919, 2 }, { 134947, 2 }, { 134948, 3 }, { 135040, 3 }, { 135125, 10 }, { 135155, 2 }, { 135228, 2 }, { 135255, 2 }, { 135296, 3 }, { 135322, 2 }, { 135349, 2 }, { 135428, 3 }, { 135476, 1 }, { 135503, 2 }, + { 135524, 2 }, { 135550, 4 }, { 135594, 2 }, { 135597, 2 }, { 135624, 3 }, { 135741, 2 }, { 135753, 2 }, { 135842, 2 }, { 135853, 2 }, { 135896, 3 }, { 136004, 1 }, { 136061, 1 }, { 136068, 1 }, { 136106, 2 }, { 136145, 2 }, { 136145, 2 }, { 136173, 2 }, { 136186, 2 }, { 136196, 2 }, { 136201, 2 }, { 136211, 2 }, { 136268, 2 }, + { 136298, 2 }, { 136377, 2 }, { 136420, 2 }, { 136475, 2 }, { 136486, 1 }, { 136554, 2 }, { 136641, 2 }, { 136770, 1 }, { 136873, 2 }, { 136877, 1 }, { 136906, 2 }, { 137092, 2 }, { 137143, 2 }, { 137200, 3 }, { 137232, 2 }, { 137239, 2 }, { 137248, 2 }, { 137281, 1 }, { 137301, 2 }, { 137314, 3 }, { 137352, 1 }, { 137365, 2 }, + { 137375, 2 }, { 137411, 2 }, { 137424, 2 }, { 137516, 2 }, { 137532, 2 }, { 137593, 2 }, { 137600, 2 }, { 137658, 2 }, { 137703, 2 }, { 137766, 2 }, { 137791, 2 }, { 137801, 2 }, { 137864, 2 }, { 137870, 3 }, { 137931, 2 }, { 138009, 3 }, { 138013, 1 }, { 138013, 1 }, { 138066, 2 }, { 138073, 2 }, { 138114, 2 }, { 138150, 2 }, + { 138236, 2 }, { 138276, 2 }, { 138286, 2 }, { 138298, 3 }, { 138309, 1 }, { 138373, 3 }, { 138524, 2 }, { 138535, 1 }, { 138593, 4 }, { 138611, 1 }, { 138725, 2 }, { 138807, 2 }, { 138819, 3 }, { 138849, 5 }, { 138867, 2 }, { 138907, 2 }, { 138930, 3 }, { 139026, 2 }, { 139102, 2 }, { 139108, 3 }, { 139184, 1 }, { 139209, 3 }, + { 139282, 2 }, { 139289, 4 }, { 139382, 1 }, { 139421, 1 }, { 139436, 2 }, { 139450, 1 }, { 139523, 3 }, { 139533, 2 }, { 139590, 2 }, { 139590, 2 }, { 139722, 2 }, { 139785, 2 }, { 139785, 1 }, { 139798, 2 }, { 139813, 2 }, { 139868, 2 }, { 139935, 3 }, { 139990, 3 }, { 140050, 2 }, { 140177, 2 }, { 140177, 4 }, { 140408, 2 }, + { 140420, 3 }, { 140461, 2 }, { 140578, 15 }, { 140605, 1 }, { 140662, 1 }, { 140755, 2 }, { 140786, 2 }, { 140846, 2 }, { 140874, 2 }, { 140959, 1 }, { 140973, 2 }, { 141128, 2 }, { 141132, 2 }, { 141257, 2 }, { 141290, 1 }, { 141360, 2 }, { 141472, 2 }, { 141545, 2 }, { 141545, 2 }, { 141575, 1 }, { 141606, 5 }, { 141655, 2 }, + { 141735, 2 }, { 141767, 5 }, { 141796, 2 }, { 141841, 2 }, { 141915, 2 }, { 141923, 1 }, { 141932, 2 }, { 141994, 2 }, { 142018, 2 }, { 142029, 3 }, { 142072, 2 }, { 142128, 2 }, { 142133, 1 }, { 142261, 2 }, { 142304, 1 }, { 142400, 2 }, { 142401, 2 }, { 142409, 2 }, { 142479, 2 }, { 142522, 1 }, { 142552, 1 }, { 142589, 2 }, + { 142596, 2 }, { 142753, 1 }, { 142766, 2 }, { 142796, 2 }, { 142836, 2 }, { 142871, 2 }, { 143058, 3 }, { 143059, 6 }, { 143063, 3 }, { 143065, 2 }, { 143141, 4 }, { 143173, 2 }, { 143374, 2 }, { 143399, 2 }, { 143406, 2 }, { 143429, 3 }, { 143430, 2 }, { 143462, 1 }, { 143579, 2 }, { 143663, 2 }, { 143844, 3 }, { 143851, 2 }, + { 143926, 2 }, { 143931, 2 }, { 144051, 6 }, { 144085, 10 }, { 144147, 2 }, { 144188, 4 }, { 144238, 4 }, { 144353, 2 }, { 144465, 2 }, { 144474, 2 }, { 144637, 2 }, { 144638, 1 }, { 144648, 1 }, { 144661, 3 }, { 144812, 2 }, { 144847, 2 }, { 144901, 8 }, { 145058, 2 }, { 145122, 8 }, { 145134, 2 }, { 145150, 2 }, { 145299, 1 }, + { 145313, 2 }, { 145314, 3 }, { 145374, 2 }, { 145412, 2 }, { 145432, 2 }, { 145446, 2 }, { 145534, 3 }, { 145592, 2 }, { 145614, 2 }, { 145648, 2 }, { 145721, 2 }, { 145858, 1 }, { 145970, 3 }, { 145984, 3 }, { 146004, 2 }, { 146016, 3 }, { 146048, 2 }, { 146097, 3 }, { 146103, 2 }, { 146136, 2 }, { 146194, 3 }, { 146230, 1 }, + { 146254, 2 }, { 146261, 4 }, { 146269, 4 }, { 146393, 2 }, { 146411, 3 }, { 146501, 2 }, { 146547, 2 }, { 146547, 2 }, { 146573, 2 }, { 146616, 2 }, { 146622, 3 }, { 146728, 3 }, { 146781, 5 }, { 146805, 4 }, { 146921, 2 }, { 147002, 3 }, { 147072, 2 }, { 147159, 2 }, { 147170, 2 }, { 147203, 1 }, { 147245, 2 }, { 147278, 2 }, + { 147422, 2 }, { 147471, 2 }, { 147491, 2 }, { 147607, 4 }, { 147693, 2 }, { 147763, 2 }, { 147775, 6 }, { 147776, 4 }, { 147824, 2 }, { 147922, 2 }, { 147922, 2 }, { 147937, 2 }, { 147957, 2 }, { 147980, 2 }, { 148008, 2 }, { 148018, 2 }, { 148046, 3 }, { 148071, 4 }, { 148106, 3 }, { 148122, 2 }, { 148139, 2 }, { 148175, 2 }, + { 148318, 2 }, { 148514, 2 }, { 148528, 2 }, { 148539, 2 }, { 148545, 2 }, { 148564, 2 }, { 148569, 2 }, { 148607, 3 }, { 148712, 2 }, { 148751, 2 }, { 148792, 4 }, { 148807, 2 }, { 148818, 2 }, { 148846, 9 }, { 148848, 2 }, { 148851, 2 }, { 148861, 3 }, { 148924, 32 }, { 148934, 2 }, { 149037, 1 }, { 149127, 3 }, { 149132, 2 }, + { 149181, 1 }, { 149181, 2 }, { 149206, 2 }, { 149216, 7 }, { 149240, 4 }, { 149240, 1 }, { 149279, 1 }, { 149280, 3 }, { 149292, 2 }, { 149314, 2 }, { 149344, 2 }, { 149364, 4 }, { 149388, 2 }, { 149438, 2 }, { 149520, 2 }, { 149566, 2 }, { 149630, 2 }, { 149682, 2 }, { 149691, 1 }, { 149703, 2 }, { 149775, 2 }, { 149796, 1 }, + { 149863, 1 }, { 149884, 2 }, { 149888, 1 }, { 149983, 2 }, { 150078, 3 }, { 150083, 6 }, { 150175, 1 }, { 150235, 2 }, { 150238, 2 }, { 150298, 3 }, { 150321, 2 }, { 150382, 2 }, { 150510, 4 }, { 150574, 2 }, { 150619, 5 }, { 150645, 2 }, { 150694, 2 }, { 150732, 8 }, { 150764, 2 }, { 150813, 2 }, { 150871, 2 }, { 150879, 2 }, + { 150888, 1 }, { 150920, 2 }, { 151009, 2 }, { 151013, 2 }, { 151019, 2 }, { 151063, 2 }, { 151067, 2 }, { 151125, 1 }, { 151151, 5 }, { 151172, 2 }, { 151197, 4 }, { 151228, 70 }, { 151292, 2 }, { 151354, 2 }, { 151392, 2 }, { 151396, 1 }, { 151412, 2 }, { 151514, 2 }, { 151529, 2 }, { 151567, 2 }, { 151589, 2 }, { 151641, 2 }, + { 151672, 2 }, { 151730, 2 }, { 151748, 2 }, { 151770, 1 }, { 151788, 2 }, { 151795, 2 }, { 151805, 2 }, { 152046, 5 }, { 152048, 3 }, { 152054, 2 }, { 152057, 3 }, { 152058, 2 }, { 152075, 2 }, { 152090, 9 }, { 152205, 2 }, { 152440, 2 }, { 152480, 2 }, { 152484, 2 }, { 152601, 2 }, { 152714, 2 }, { 152801, 2 }, { 152819, 10 }, + { 152825, 2 }, { 152896, 10 }, { 152937, 2 }, { 152938, 2 }, { 152939, 3 }, { 153042, 2 }, { 153069, 2 }, { 153099, 2 }, { 153164, 2 }, { 153234, 2 }, { 153266, 2 }, { 153345, 2 }, { 153420, 2 }, { 153479, 2 }, { 153488, 4 }, { 153502, 2 }, { 153663, 3 }, { 153740, 1 }, { 153780, 2 }, { 153824, 6 }, { 153938, 2 }, { 153985, 2 }, + { 154022, 2 }, { 154022, 1 }, { 154072, 4 }, { 154109, 2 }, { 154189, 2 }, { 154222, 2 }, { 154228, 2 }, { 154265, 15 }, { 154324, 2 }, { 154350, 2 }, { 154375, 2 }, { 154396, 2 }, { 154431, 2 }, { 154463, 2 }, { 154475, 3 }, { 154510, 1 }, { 154518, 1 }, { 154529, 2 }, { 154710, 2 }, { 154742, 2 }, { 154792, 2 }, { 154871, 4 }, + { 154960, 2 }, { 154961, 2 }, { 154964, 2 }, { 154989, 3 }, { 155002, 3 }, { 155079, 2 }, { 155105, 2 }, { 155107, 2 }, { 155258, 1 }, { 155328, 2 }, { 155328, 2 }, { 155347, 2 }, { 155369, 2 }, { 155447, 2 }, { 155482, 2 }, { 155508, 2 }, { 155531, 2 }, { 155553, 1 }, { 155647, 3 }, { 155659, 9 }, { 155859, 2 }, { 155960, 2 }, + { 156009, 2 }, { 156062, 2 }, { 156143, 3 }, { 156217, 3 }, { 156252, 7 }, { 156260, 2 }, { 156274, 2 }, { 156339, 2 }, { 156354, 3 }, { 156406, 2 }, { 156550, 2 }, { 156694, 2 }, { 156730, 12 }, { 156795, 4 }, { 156806, 3 }, { 156818, 5 }, { 156835, 3 }, { 156850, 3 }, { 156861, 4 }, { 156877, 2 }, { 156915, 6 }, { 156967, 2 }, + { 157046, 2 }, { 157208, 3 }, { 157260, 2 }, { 157364, 2 }, { 157365, 1 }, { 157371, 2 }, { 157440, 2 }, { 157453, 2 }, { 157482, 1 }, { 157505, 2 }, { 157511, 4 }, { 157522, 2 }, { 157562, 2 }, { 157562, 2 }, { 157702, 2 }, { 157734, 3 }, { 157807, 2 }, { 157851, 2 }, { 157882, 2 }, { 157957, 2 }, { 158227, 2 }, { 158284, 2 }, + { 158292, 2 }, { 158310, 3 }, { 158310, 3 }, { 158330, 2 }, { 158358, 1 }, { 158493, 2 }, { 158596, 2 }, { 158736, 2 }, { 158812, 1 }, { 158830, 2 }, { 158846, 1 }, { 158884, 1 }, { 158918, 2 }, { 159003, 2 }, { 159056, 2 }, { 159189, 2 }, { 159372, 2 }, { 159373, 2 }, { 159410, 3 }, { 159421, 4 }, { 159429, 2 }, { 159505, 2 }, + { 159559, 1 }, { 159574, 2 }, { 159587, 4 }, { 159683, 2 }, { 159745, 1 }, { 159748, 3 }, { 159858, 2 }, { 159945, 1 }, { 159971, 4 }, { 159982, 2 }, { 160079, 13 }, { 160084, 2 }, { 160085, 4 }, { 160104, 9 }, { 160197, 2 }, { 160295, 2 }, { 160365, 2 }, { 160372, 1 }, { 160392, 3 }, { 160408, 1 }, { 160446, 1 }, { 160540, 2 }, + { 160599, 1 }, { 160604, 2 }, { 160745, 3 }, { 160752, 2 }, { 160794, 2 }, { 160826, 2 }, { 160846, 2 }, { 160871, 2 }, { 160957, 2 }, { 160986, 2 }, { 161053, 2 }, { 161133, 3 }, { 161133, 1 }, { 161162, 3 }, { 161247, 2 }, { 161270, 2 }, { 161292, 6 }, { 161370, 2 }, { 161420, 2 }, { 161446, 2 }, { 161487, 2 }, { 161511, 3 }, + { 161512, 1 }, { 161580, 2 }, { 161782, 12 }, { 161784, 8 }, { 161786, 2 }, { 161786, 8 }, { 161787, 7 }, { 161795, 2 }, { 161825, 7 }, { 161833, 4 }, { 161892, 2 }, { 161930, 2 }, { 161992, 2 }, { 162054, 2 }, { 162176, 2 }, { 162183, 2 }, { 162219, 2 }, { 162245, 2 }, { 162288, 2 }, { 162361, 3 }, { 162370, 2 }, { 162388, 2 }, + { 162434, 2 }, { 162447, 2 }, { 162524, 2 }, { 162542, 3 }, { 162562, 2 }, { 162594, 2 }, { 162646, 1 }, { 162662, 2 }, { 162761, 2 }, { 162780, 2 }, { 162802, 2 }, { 162820, 3 }, { 162824, 2 }, { 162908, 2 }, { 162968, 2 }, { 163171, 2 }, { 163190, 2 }, { 163288, 1 }, { 163288, 1 }, { 163397, 2 }, { 163405, 6 }, { 163414, 1 }, + { 163506, 1 }, { 163558, 2 }, { 163565, 1 }, { 163568, 3 }, { 163619, 2 }, { 163633, 2 }, { 163678, 2 }, { 163745, 3 }, { 163765, 4 }, { 163773, 2 }, { 163793, 2 }, { 163878, 7 }, { 163949, 2 }, { 163975, 2 }, { 163991, 2 }, { 164016, 2 }, { 164020, 3 }, { 164068, 1 }, { 164076, 4 }, { 164082, 3 }, { 164176, 2 }, { 164236, 2 }, + { 164238, 1 }, { 164315, 2 }, { 164449, 2 }, { 164529, 2 }, { 164574, 4 }, { 164591, 2 }, { 164595, 2 }, { 164611, 2 }, { 164623, 4 }, { 164632, 10 }, { 164691, 2 }, { 164706, 2 }, { 164755, 2 }, { 164761, 2 }, { 164973, 2 }, { 165030, 2 }, { 165090, 2 }, { 165099, 1 }, { 165126, 2 }, { 165188, 2 }, { 165205, 2 }, { 165275, 1 }, + { 165347, 2 }, { 165381, 2 }, { 165562, 2 }, { 165563, 1 }, { 165594, 2 }, { 165641, 2 }, { 165663, 6 }, { 165759, 2 }, { 165811, 2 }, { 165822, 1 }, { 165830, 1 }, { 165903, 1 }, { 165921, 2 }, { 165953, 1 }, { 166022, 1 }, { 166294, 2 }, { 166333, 2 }, { 166420, 2 }, { 166433, 2 }, { 166442, 1 }, { 166536, 2 }, { 166543, 2 }, + { 166556, 2 }, { 166571, 2 }, { 166575, 1 }, { 166588, 2 }, { 166601, 2 }, { 166663, 3 }, { 166692, 1 }, { 166710, 2 }, { 166759, 2 }, { 166785, 3 }, { 166842, 2 }, { 166843, 2 }, { 166864, 2 }, { 166902, 2 }, { 166996, 2 }, { 166999, 2 }, { 167038, 2 }, { 167112, 4 }, { 167112, 2 }, { 167177, 2 }, { 167180, 2 }, { 167229, 1 }, + { 167298, 2 }, { 167306, 4 }, { 167309, 3 }, { 167402, 2 }, { 167405, 2 }, { 167433, 2 }, { 167435, 1 }, { 167461, 3 }, { 167553, 3 }, { 167688, 5 }, { 167689, 2 }, { 167709, 2 }, { 167744, 2 }, { 167821, 2 }, { 167825, 2 }, { 167925, 10 }, { 167969, 2 }, { 168024, 2 }, { 168089, 2 }, { 168104, 2 }, { 168194, 2 }, { 168305, 2 }, + { 168316, 2 }, { 168366, 2 }, { 168423, 2 }, { 168568, 3 }, { 168582, 2 }, { 168615, 3 }, { 168618, 2 }, { 168638, 2 }, { 168671, 2 }, { 168736, 2 }, { 168747, 2 }, { 168750, 4 }, { 168808, 3 }, { 168814, 4 }, { 168820, 2 }, { 168914, 2 }, { 168968, 2 }, { 168979, 2 }, { 169006, 2 }, { 169069, 2 }, { 169106, 3 }, { 169158, 2 }, + { 169158, 2 }, { 169189, 2 }, { 169253, 2 }, { 169259, 1 }, { 169279, 1 }, { 169325, 8 }, { 169349, 2 }, { 169353, 2 }, { 169378, 2 }, { 169432, 2 }, { 169476, 1 }, { 169476, 1 }, { 169525, 2 }, { 169538, 7 }, { 169555, 2 }, { 169571, 2 }, { 169594, 4 }, { 169687, 2 }, { 169799, 2 }, { 169831, 2 }, { 170042, 2 }, { 170061, 2 }, + { 170065, 1 }, { 170128, 6 }, { 170148, 20 }, { 170215, 70 }, { 170256, 60 }, { 170266, 69 }, { 170275, 7 }, { 170277, 6 }, { 170500, 3 }, { 170516, 3 }, { 170601, 2 }, { 170666, 2 }, { 170668, 4 }, { 170668, 1 }, { 170716, 3 }, { 170728, 3 }, { 170735, 5 }, { 170847, 3 }, { 170852, 9 }, { 170858, 2 }, { 170859, 3 }, { 170956, 2 }, + { 170956, 1 }, { 170967, 2 }, { 171005, 2 }, { 171113, 2 }, { 171279, 2 }, { 171400, 2 }, { 171405, 2 }, { 171448, 1 }, { 171490, 2 }, { 171567, 32 }, { 171590, 2 }, { 171723, 2 }, { 171737, 3 }, { 171958, 2 }, { 171967, 2 } + }; + } + + /* sub-class to avoid 65k limit of a single class */ + private static class SampleDataHolder2 { + /* + * Array of [milliseconds, latency] + */ + private static int[][] data = new int[][] { { 0, 3 }, { 43, 33 }, { 45, 11 }, { 45, 1 }, { 68, 13 }, { 88, 10 }, { 158, 68 }, { 158, 4 }, { 169, 168 }, { 267, 68 }, { 342, 68 }, { 438, 68 }, { 464, 7 }, { 504, 68 }, { 541, 6 }, { 541, 68 }, { 562, 68 }, { 581, 3 }, { 636, 68 }, { 778, 68 }, { 825, 1 }, { 859, 68 }, + { 948, 1 }, { 1043, 68 }, { 1145, 68 }, { 1152, 1 }, { 1218, 5 }, { 1229, 68 }, { 1259, 68 }, { 1333, 68 }, { 1349, 68 }, { 1392, 68 }, { 1468, 1 }, { 1551, 68 }, { 1586, 68 }, { 1685, 68 }, { 1696, 1 }, { 1807, 68 }, { 1817, 3 }, { 1817, 6 }, { 1847, 68 }, { 1870, 68 }, { 1939, 68 }, { 2050, 68 }, { 2129, 3 }, { 2141, 68 }, + { 2265, 68 }, { 2414, 1 }, { 2693, 68 }, { 2703, 68 }, { 2791, 68 }, { 2838, 68 }, { 2906, 68 }, { 2981, 68 }, { 3008, 68 }, { 3026, 4 }, { 3077, 68 }, { 3273, 68 }, { 3282, 68 }, { 3286, 68 }, { 3318, 3 }, { 3335, 5 }, { 3710, 68 }, { 3711, 1 }, { 3745, 68 }, { 3748, 4 }, { 3767, 3 }, { 3809, 3 }, { 3835, 35 }, { 4083, 1 }, + { 4116, 68 }, { 4117, 1 }, { 4157, 1 }, { 4279, 68 }, { 4344, 68 }, { 4452, 68 }, { 4530, 68 }, { 4583, 68 }, { 4647, 3 }, { 4758, 68 }, { 4776, 68 }, { 4793, 68 }, { 4901, 68 }, { 4909, 68 }, { 4962, 68 }, { 4984, 68 }, { 5022, 68 }, { 5139, 68 }, { 5166, 1 }, { 5174, 68 }, { 5187, 68 }, { 5225, 68 }, { 5234, 68 }, { 5263, 1 }, + { 5325, 68 }, { 5355, 4 }, { 5407, 1 }, { 5414, 68 }, { 5589, 68 }, { 5595, 68 }, { 5747, 68 }, { 5780, 68 }, { 5788, 68 }, { 5796, 68 }, { 5818, 68 }, { 5975, 1 }, { 6018, 1 }, { 6270, 68 }, { 6272, 68 }, { 6348, 68 }, { 6372, 68 }, { 6379, 68 }, { 6439, 68 }, { 6442, 68 }, { 6460, 68 }, { 6460, 68 }, { 6509, 68 }, { 6511, 1 }, + { 6514, 4 }, { 6530, 8 }, { 6719, 68 }, { 6760, 68 }, { 6784, 68 }, { 6838, 1 }, { 6861, 68 }, { 6947, 68 }, { 7013, 68 }, { 7075, 68 }, { 7122, 5 }, { 7130, 68 }, { 7209, 3 }, { 7259, 68 }, { 7309, 1 }, { 7315, 3 }, { 7322, 68 }, { 7348, 68 }, { 7420, 68 }, { 7461, 68 }, { 7545, 68 }, { 7554, 3 }, { 7630, 68 }, { 7666, 68 }, + { 7815, 1 }, { 7972, 1 }, { 7972, 68 }, { 7988, 68 }, { 8049, 8 }, { 8254, 68 }, { 8269, 68 }, { 8352, 1 }, { 8378, 68 }, { 8526, 68 }, { 8531, 68 }, { 8583, 68 }, { 8615, 68 }, { 8619, 3 }, { 8623, 68 }, { 8692, 1 }, { 8698, 68 }, { 8773, 68 }, { 8777, 3 }, { 8822, 68 }, { 8929, 68 }, { 8935, 68 }, { 9025, 68 }, { 9054, 68 }, + { 9056, 1 }, { 9086, 68 }, { 9147, 3 }, { 9219, 68 }, { 9230, 3 }, { 9248, 68 }, { 9283, 68 }, { 9314, 68 }, { 9418, 1 }, { 9426, 68 }, { 9456, 1 }, { 9594, 68 }, { 9628, 68 }, { 9642, 68 }, { 9646, 68 }, { 9686, 1 }, { 9709, 68 }, { 9771, 3 }, { 9782, 68 }, { 9884, 68 }, { 9914, 5 }, { 10004, 4 }, { 10033, 6 }, { 10052, 68 }, + { 10086, 68 }, { 10168, 68 }, { 10176, 1 }, { 10228, 68 }, { 10312, 68 }, { 10372, 68 }, { 10622, 68 }, { 10685, 68 }, { 10687, 1 }, { 10787, 68 }, { 11010, 68 }, { 11024, 68 }, { 11044, 68 }, { 11086, 68 }, { 11149, 1 }, { 11198, 68 }, { 11265, 68 }, { 11302, 68 }, { 11326, 68 }, { 11354, 68 }, { 11404, 1 }, { 11473, 68 }, + { 11506, 68 }, { 11548, 4 }, { 11575, 68 }, { 11621, 4 }, { 11625, 3 }, { 11625, 1 }, { 11642, 4 }, { 11859, 5 }, { 11870, 68 }, { 11872, 3 }, { 11880, 7 }, { 11886, 3 }, { 11905, 6 }, { 11880, 3 }, { 11912, 6 }, { 11916, 4 }, { 11916, 3 }, { 11965, 4 }, { 12068, 13 }, { 12106, 68 }, { 12120, 68 }, { 12221, 68 }, { 12257, 68 }, + { 12361, 68 }, { 12411, 68 }, { 12473, 3 }, { 12554, 68 }, { 12583, 68 }, { 12654, 68 }, { 12665, 68 }, { 12744, 1 }, { 12775, 68 }, { 12858, 68 }, { 12993, 68 }, { 13007, 3 }, { 13025, 4 }, { 13038, 68 }, { 13092, 4 }, { 13094, 5 }, { 13095, 1 }, { 13110, 68 }, { 13116, 1 }, { 13140, 68 }, { 13169, 1 }, { 13186, 68 }, + { 13202, 68 }, { 13202, 1 }, { 13256, 68 }, { 13344, 68 }, { 13373, 68 }, { 13396, 3 }, { 13446, 68 }, { 13451, 3 }, { 13475, 68 }, { 13521, 1 }, { 13587, 68 }, { 13592, 68 }, { 13708, 3 }, { 13711, 1 }, { 13741, 1 }, { 13757, 1 }, { 13847, 68 }, { 13881, 3 }, { 13915, 1 }, { 14005, 68 }, { 14028, 68 }, { 14037, 68 }, + { 14074, 68 }, { 14135, 68 }, { 14176, 68 }, { 14227, 68 }, { 14228, 68 }, { 14271, 3 }, { 14279, 3 }, { 14493, 68 }, { 14535, 3 }, { 14535, 1 }, { 14680, 68 }, { 14717, 68 }, { 14725, 1 }, { 14790, 68 }, { 14801, 1 }, { 14959, 68 }, { 15052, 68 }, { 15055, 1 }, { 15055, 1 }, { 15075, 68 }, { 15103, 8 }, { 15153, 16 }, + { 15191, 68 }, { 15240, 68 }, { 15313, 68 }, { 15323, 68 }, { 15341, 1 }, { 15383, 68 }, { 15387, 68 }, { 15491, 68 }, { 15534, 68 }, { 15539, 68 }, { 15549, 68 }, { 15554, 1 }, { 15664, 1 }, { 15726, 68 }, { 15807, 68 }, { 15842, 68 }, { 15897, 68 }, { 15913, 3 }, { 15925, 68 }, { 15935, 68 }, { 16131, 1 }, { 16211, 3 }, + { 16249, 68 }, { 16268, 68 }, { 16307, 68 }, { 16398, 68 }, { 16498, 68 }, { 16518, 1 }, { 16552, 1 }, { 16571, 68 }, { 16592, 68 }, { 16601, 3 }, { 16638, 68 }, { 16698, 68 }, { 16712, 1 }, { 16767, 68 }, { 16789, 68 }, { 16992, 68 }, { 17015, 68 }, { 17035, 68 }, { 17074, 3 }, { 17086, 3 }, { 17086, 1 }, { 17092, 1 }, + { 17110, 4 }, { 17116, 3 }, { 17236, 68 }, { 17291, 68 }, { 17291, 68 }, { 17340, 68 }, { 17342, 1 }, { 17360, 3 }, { 17436, 3 }, { 17457, 68 }, { 17508, 1 }, { 17556, 68 }, { 17601, 68 }, { 17639, 68 }, { 17671, 68 }, { 17743, 68 }, { 17857, 68 }, { 17915, 68 }, { 17992, 68 }, { 18077, 1 }, { 18088, 68 }, { 18158, 1 }, + { 18239, 16 }, { 18242, 68 }, { 18252, 3 }, { 18299, 1 }, { 18405, 68 }, { 18433, 68 }, { 18444, 68 }, { 18490, 68 }, { 18497, 68 }, { 18516, 68 }, { 18540, 68 }, { 18598, 68 }, { 18649, 68 }, { 18658, 68 }, { 18683, 68 }, { 18728, 68 }, { 18767, 1 }, { 18821, 68 }, { 18868, 68 }, { 18876, 68 }, { 18914, 14 }, { 19212, 1 }, + { 19215, 1 }, { 19293, 68 }, { 19303, 68 }, { 19336, 68 }, { 19376, 68 }, { 19419, 68 }, { 19558, 68 }, { 19559, 1 }, { 19609, 68 }, { 19688, 68 }, { 19724, 68 }, { 19820, 1 }, { 19851, 68 }, { 19881, 68 }, { 19966, 68 }, { 19983, 3 }, { 19988, 4 }, { 20047, 1 }, { 20062, 68 }, { 20091, 1 }, { 20152, 1 }, { 20183, 1 }, + { 20208, 68 }, { 20346, 68 }, { 20386, 1 }, { 20459, 68 }, { 20505, 68 }, { 20520, 1 }, { 20560, 3 }, { 20566, 3 }, { 20566, 1 }, { 20610, 68 }, { 20652, 68 }, { 20694, 68 }, { 20740, 68 }, { 20756, 68 }, { 20825, 3 }, { 20895, 68 }, { 20959, 1 }, { 20995, 68 }, { 21017, 3 }, { 21039, 68 }, { 21086, 1 }, { 21109, 3 }, { 21139, 3 }, + { 21206, 68 }, { 21230, 68 }, { 21251, 3 }, { 21352, 68 }, { 21353, 68 }, { 21370, 3 }, { 21389, 1 }, { 21445, 3 }, { 21475, 68 }, { 21528, 68 }, { 21559, 3 }, { 21604, 68 }, { 21606, 1 }, { 21815, 68 }, { 21858, 3 }, { 21860, 3 }, { 22015, 68 }, { 22065, 68 }, { 22098, 5 }, { 22105, 68 }, { 22158, 3 }, { 22197, 68 }, { 22254, 1 }, + { 22353, 68 }, { 22404, 4 }, { 22422, 68 }, { 22569, 68 }, { 22634, 68 }, { 22639, 68 }, { 22861, 68 }, { 22868, 68 }, { 22876, 1 }, { 22902, 68 }, { 22925, 68 }, { 23080, 68 }, { 23085, 3 }, { 23089, 5 }, { 23329, 1 }, { 23349, 68 }, { 23559, 5 }, { 23567, 3 }, { 23574, 68 }, { 23584, 3 }, { 23615, 3 }, { 23633, 68 }, + { 23674, 68 }, { 23678, 1 }, { 23853, 68 }, { 23875, 68 }, { 24010, 4 }, { 24076, 68 }, { 24128, 6 }, { 24248, 68 }, { 24253, 68 }, { 24259, 1 }, { 24319, 68 }, { 24319, 1 }, { 24502, 3 }, { 24666, 68 }, { 24781, 3 }, { 24792, 68 }, { 24909, 68 }, { 24993, 68 }, { 25039, 1 }, { 25090, 3 }, { 25137, 1 }, { 25138, 3 }, { 25140, 3 }, + { 25155, 5 }, { 25411, 68 }, { 25460, 68 }, { 25564, 3 }, { 25586, 3 }, { 25630, 68 }, { 25765, 68 }, { 25789, 3 }, { 25803, 68 }, { 25851, 68 }, { 25872, 68 }, { 25887, 68 }, { 25981, 1 }, { 26016, 68 }, { 26019, 1 }, { 26029, 1 }, { 26104, 7 }, { 26144, 68 }, { 26275, 1 }, { 26295, 68 }, { 26298, 1 }, { 26322, 68 }, + { 26380, 68 }, { 26408, 4 }, { 26446, 1 }, { 26553, 1 }, { 26576, 1 }, { 26635, 1 }, { 26668, 68 }, { 26675, 68 }, { 26698, 4 }, { 26748, 9 }, { 26788, 68 }, { 26932, 68 }, { 26962, 68 }, { 27042, 68 }, { 27060, 68 }, { 27163, 3 }, { 27202, 68 }, { 27290, 68 }, { 27337, 3 }, { 27376, 68 }, { 27439, 68 }, { 27458, 4 }, + { 27515, 68 }, { 27518, 1 }, { 27541, 68 }, { 27585, 3 }, { 27633, 68 }, { 27695, 68 }, { 27702, 68 }, { 27861, 68 }, { 27924, 1 }, { 28025, 14 }, { 28058, 68 }, { 28143, 68 }, { 28215, 68 }, { 28240, 68 }, { 28241, 68 }, { 28285, 68 }, { 28324, 3 }, { 28378, 68 }, { 28514, 68 }, { 28529, 68 }, { 28538, 68 }, { 28565, 3 }, + { 28697, 68 }, { 28735, 68 }, { 28769, 68 }, { 28770, 4 }, { 28788, 4 }, { 28807, 3 }, { 28807, 4 }, { 28829, 1 }, { 28853, 68 }, { 28856, 7 }, { 28864, 68 }, { 28865, 3 }, { 28915, 68 }, { 28928, 68 }, { 28964, 68 }, { 28988, 1 }, { 29031, 68 }, { 29095, 68 }, { 29189, 68 }, { 29205, 1 }, { 29230, 1 }, { 29332, 68 }, + { 29339, 68 }, { 29349, 5 }, { 29449, 68 }, { 29471, 68 }, { 29578, 68 }, { 29859, 68 }, { 29878, 68 }, { 29947, 10 }, { 30083, 68 }, { 30121, 68 }, { 30128, 68 }, { 30155, 4 }, { 30157, 1 }, { 30272, 68 }, { 30281, 68 }, { 30286, 68 }, { 30305, 68 }, { 30408, 68 }, { 30444, 268 }, { 30612, 68 }, { 30628, 68 }, { 30747, 68 }, + { 30783, 68 }, { 30808, 5 }, { 30868, 3 }, { 30875, 68 }, { 30997, 68 }, { 31000, 68 }, { 31022, 3 }, { 31111, 1 }, { 31144, 68 }, { 31146, 3 }, { 31187, 68 }, { 31324, 68 }, { 31343, 68 }, { 31416, 68 }, { 31485, 68 }, { 31539, 68 }, { 31638, 68 }, { 31648, 68 }, { 31750, 68 }, { 31754, 68 }, { 31785, 10 }, { 31786, 5 }, + { 31800, 68 }, { 31801, 4 }, { 31807, 7 }, { 31807, 3 }, { 31807, 10 }, { 31808, 3 }, { 31808, 4 }, { 31818, 6 }, { 31825, 7 }, { 31838, 68 }, { 31911, 1 }, { 31974, 68 }, { 32010, 3 }, { 32031, 68 }, { 32040, 68 }, { 32063, 1 }, { 32078, 68 }, { 32156, 68 }, { 32198, 31 }, { 32257, 68 }, { 32257, 68 }, { 32265, 68 }, + { 32330, 68 }, { 32369, 8 }, { 32404, 3 }, { 32425, 68 }, { 32432, 68 }, { 32505, 68 }, { 32531, 68 }, { 32536, 68 }, { 32549, 68 }, { 32582, 3 }, { 32590, 4 }, { 32624, 68 }, { 32644, 68 }, { 32692, 68 }, { 32695, 4 }, { 32699, 3 }, { 32726, 4 }, { 32784, 68 }, { 32832, 68 }, { 32883, 6 }, { 32965, 4 }, { 33044, 68 }, + { 33104, 68 }, { 33184, 68 }, { 33264, 1 }, { 33292, 68 }, { 33312, 1 }, { 33468, 68 }, { 33471, 1 }, { 33565, 68 }, { 33627, 68 }, { 33659, 68 }, { 33709, 68 }, { 33766, 5 }, { 33836, 68 }, { 33875, 68 }, { 33954, 68 }, { 33959, 68 }, { 34050, 68 }, { 34090, 68 }, { 34168, 68 }, { 34233, 68 }, { 34461, 68 }, { 34462, 1 }, + { 34463, 68 }, { 34472, 4 }, { 34500, 68 }, { 34520, 68 }, { 34544, 68 }, { 34614, 68 }, { 34662, 1 }, { 34676, 68 }, { 34729, 4 }, { 34803, 68 }, { 34845, 68 }, { 34913, 68 }, { 34963, 6 }, { 35019, 68 }, { 35022, 68 }, { 35070, 68 }, { 35120, 68 }, { 35132, 68 }, { 35144, 68 }, { 35205, 68 }, { 35230, 3 }, { 35244, 68 }, + { 35271, 4 }, { 35276, 68 }, { 35282, 68 }, { 35324, 3 }, { 35366, 3 }, { 35659, 68 }, { 35680, 68 }, { 35744, 68 }, { 35758, 3 }, { 35796, 68 }, { 35830, 68 }, { 35841, 7 }, { 35843, 68 }, { 35856, 68 }, { 35914, 4 }, { 35929, 13 }, { 35993, 68 }, { 35997, 1 }, { 36046, 4 }, { 36046, 1 }, { 36051, 1 }, { 36111, 68 }, { 36208, 1 }, + { 36208, 1 }, { 36306, 68 }, { 36325, 68 }, { 36386, 68 }, { 36405, 68 }, { 36443, 1 }, { 36455, 1 }, { 36538, 68 }, { 36562, 68 }, { 36566, 68 }, { 36628, 68 }, { 36693, 68 }, { 36713, 68 }, { 36730, 68 }, { 36747, 68 }, { 36786, 68 }, { 36810, 1 }, { 36848, 68 }, { 36914, 1 }, { 36920, 68 }, { 36952, 68 }, { 37071, 68 }, + { 37086, 1 }, { 37094, 3 }, { 37158, 3 }, { 37231, 68 }, { 37241, 68 }, { 37285, 68 }, { 37349, 68 }, { 37404, 68 }, { 37410, 1 }, { 37433, 4 }, { 37615, 68 }, { 37659, 68 }, { 37742, 68 }, { 37773, 68 }, { 37867, 1 }, { 37890, 68 }, { 37960, 68 }, { 38042, 3 }, { 38241, 68 }, { 38400, 68 }, { 38461, 1 }, { 38551, 68 }, + { 38611, 1 }, { 38657, 68 }, { 38729, 68 }, { 38748, 68 }, { 38815, 68 }, { 38852, 68 }, { 38890, 1 }, { 38954, 68 }, { 39119, 68 }, { 39162, 68 }, { 39175, 3 }, { 39176, 68 }, { 39231, 68 }, { 39261, 68 }, { 39467, 68 }, { 39500, 68 }, { 39507, 68 }, { 39566, 68 }, { 39608, 68 }, { 39686, 6 }, { 39730, 68 }, { 39842, 1 }, + { 39853, 1 }, { 39905, 68 }, { 39931, 68 }, { 39989, 68 }, { 40030, 68 }, { 40227, 68 }, { 40268, 68 }, { 40372, 68 }, { 40415, 1 }, { 40488, 3 }, { 40536, 68 }, { 40676, 3 }, { 40677, 68 }, { 40755, 68 }, { 40842, 68 }, { 40849, 1 }, { 40870, 3 }, { 40873, 3 }, { 40972, 68 }, { 41033, 68 }, { 41190, 68 }, { 41273, 5 }, + { 41273, 1 }, { 41293, 68 }, { 41367, 368 }, { 41376, 68 }, { 41420, 68 }, { 41473, 68 }, { 41473, 68 }, { 41493, 4 }, { 41521, 68 }, { 41533, 68 }, { 41554, 68 }, { 41568, 68 }, { 41583, 3 }, { 41728, 68 }, { 41786, 68 }, { 41836, 1 }, { 41875, 68 }, { 41933, 68 }, { 42044, 68 }, { 42075, 68 }, { 42076, 68 }, { 42133, 68 }, + { 42259, 29 }, { 42269, 3 }, { 42294, 68 }, { 42420, 68 }, { 42524, 68 }, { 42524, 1 }, { 42546, 1 }, { 42631, 68 }, { 42693, 68 }, { 42740, 68 }, { 42744, 4 }, { 42755, 1 }, { 42870, 68 }, { 42894, 68 }, { 42939, 68 }, { 42973, 68 }, { 43016, 68 }, { 43070, 68 }, { 43105, 68 }, { 43115, 68 }, { 43375, 3 }, { 43387, 1 }, + { 43424, 3 }, { 43448, 68 }, { 43480, 68 }, { 43498, 68 }, { 43651, 68 }, { 43727, 68 }, { 43879, 68 }, { 43910, 1 }, { 43977, 68 }, { 44003, 68 }, { 44080, 68 }, { 44082, 1 }, { 44136, 68 }, { 44169, 29 }, { 44186, 68 }, { 44339, 68 }, { 44350, 1 }, { 44356, 1 }, { 44430, 68 }, { 44440, 1 }, { 44530, 1 }, { 44538, 68 }, + { 44572, 68 }, { 44585, 68 }, { 44709, 68 }, { 44748, 68 }, { 44748, 68 }, { 44769, 68 }, { 44813, 68 }, { 44890, 68 }, { 45015, 68 }, { 45046, 4 }, { 45052, 68 }, { 45062, 68 }, { 45094, 6 }, { 45184, 68 }, { 45191, 68 }, { 45201, 3 }, { 45216, 3 }, { 45227, 68 }, { 45269, 1 }, { 45294, 68 }, { 45314, 68 }, { 45345, 8 }, + { 45352, 68 }, { 45365, 3 }, { 45378, 1 }, { 45392, 4 }, { 45405, 3 }, { 45410, 68 }, { 45448, 14 }, { 45450, 68 }, { 45457, 68 }, { 45466, 3 }, { 45481, 4 }, { 45486, 7 }, { 45533, 5 }, { 45576, 68 }, { 45649, 68 }, { 45917, 68 }, { 45919, 6 }, { 45919, 1 }, { 45930, 15 }, { 45930, 68 }, { 46001, 5 }, { 46036, 68 }, { 46054, 68 }, + { 46075, 68 }, { 46153, 68 }, { 46155, 68 }, { 46228, 68 }, { 46234, 68 }, { 46273, 68 }, { 46387, 68 }, { 46398, 68 }, { 46517, 68 }, { 46559, 68 }, { 46565, 1 }, { 46598, 68 }, { 46686, 68 }, { 46744, 68 }, { 46816, 3 }, { 46835, 68 }, { 46921, 68 }, { 46938, 68 }, { 46991, 68 }, { 47038, 68 }, { 47098, 3 }, { 47107, 68 }, + { 47201, 3 }, { 47327, 1 }, { 47327, 1 }, { 47338, 68 }, { 47395, 1 }, { 47499, 68 }, { 47504, 68 }, { 47515, 1 }, { 47516, 1 }, { 47600, 1 }, { 47604, 1 }, { 47707, 1 }, { 47728, 1 }, { 47748, 68 }, { 47763, 68 }, { 47807, 4 }, { 47814, 68 }, { 47822, 68 }, { 47834, 68 }, { 47843, 3 }, { 47886, 68 }, { 47893, 68 }, { 48066, 68 }, + { 48126, 68 }, { 48133, 1 }, { 48166, 68 }, { 48299, 1 }, { 48455, 68 }, { 48468, 68 }, { 48568, 68 }, { 48606, 68 }, { 48642, 68 }, { 48698, 68 }, { 48714, 68 }, { 48754, 68 }, { 48765, 3 }, { 48773, 5 }, { 48819, 68 }, { 48833, 68 }, { 48904, 68 }, { 49000, 1 }, { 49113, 168 }, { 49140, 68 }, { 49276, 68 }, { 49353, 68 }, + { 49411, 3 }, { 49418, 68 }, { 49540, 68 }, { 49544, 68 }, { 49584, 68 }, { 49602, 68 }, { 49784, 5 }, { 49822, 4 }, { 49822, 5 }, { 49828, 68 }, { 49866, 68 }, { 49922, 3 }, { 49959, 68 }, { 50045, 68 }, { 50134, 3 }, { 50140, 68 }, { 50237, 68 }, { 50247, 68 }, { 50266, 13 }, { 50290, 68 }, { 50312, 4 }, { 50314, 1 }, + { 50527, 68 }, { 50605, 1 }, { 50730, 68 }, { 50751, 68 }, { 50770, 68 }, { 50858, 68 }, { 50859, 68 }, { 50909, 68 }, { 50948, 3 }, { 51043, 68 }, { 51048, 68 }, { 51089, 68 }, { 51090, 68 }, { 51141, 68 }, { 51163, 68 }, { 51250, 68 }, { 51347, 68 }, { 51475, 68 }, { 51536, 68 }, { 51544, 68 }, { 51595, 68 }, { 51602, 19 }, + { 51643, 5 }, { 51702, 68 }, { 51702, 10 }, { 51764, 68 }, { 51793, 5 }, { 51812, 68 }, { 51839, 1 }, { 51938, 3 }, { 51941, 1 }, { 51967, 4 }, { 52049, 3 }, { 52074, 3 }, { 52098, 68 }, { 52118, 68 }, { 52119, 3 }, { 52227, 11 }, { 52246, 3 }, { 52282, 68 }, { 52451, 68 }, { 52583, 68 }, { 52601, 1 }, { 52605, 68 }, { 52615, 68 }, + { 52668, 68 }, { 52824, 68 }, { 53076, 1 }, { 53120, 1 }, { 53179, 68 }, { 53189, 68 }, { 53193, 1 }, { 53195, 68 }, { 53246, 68 }, { 53249, 68 }, { 53268, 1 }, { 53295, 68 }, { 53312, 68 }, { 53410, 68 }, { 53451, 68 }, { 53570, 68 }, { 53593, 68 }, { 53635, 68 }, { 53657, 68 }, { 53682, 3 }, { 53728, 5 }, { 53733, 68 }, + { 53753, 68 }, { 53787, 4 }, { 53807, 1 }, { 54008, 68 }, { 54059, 68 }, { 54060, 1 }, { 54080, 68 }, { 54090, 1 }, { 54138, 68 }, { 54149, 68 }, { 54168, 1 }, { 54171, 68 }, { 54216, 268 }, { 54233, 6 }, { 54434, 68 }, { 54534, 68 }, { 54562, 68 }, { 54763, 68 }, { 54791, 68 }, { 54816, 68 }, { 54909, 68 }, { 54916, 3 }, + { 54963, 68 }, { 54985, 68 }, { 54991, 3 }, { 55016, 3 }, { 55025, 3 }, { 55032, 68 }, { 55099, 68 }, { 55260, 68 }, { 55261, 68 }, { 55270, 3 }, { 55384, 68 }, { 55455, 68 }, { 55456, 68 }, { 55504, 3 }, { 55510, 68 }, { 55558, 68 }, { 55568, 68 }, { 55585, 68 }, { 55677, 68 }, { 55703, 68 }, { 55749, 68 }, { 55779, 68 }, + { 55789, 3 }, { 55792, 68 }, { 55830, 4 }, { 55835, 68 }, { 55879, 68 }, { 56076, 68 }, { 56118, 68 }, { 56314, 68 }, { 56392, 1 }, { 56411, 68 }, { 56459, 68 }, { 56553, 34 }, { 56575, 68 }, { 56733, 68 }, { 56762, 68 }, { 56793, 3 }, { 56877, 3 }, { 56927, 68 }, { 56981, 68 }, { 57014, 1 }, { 57149, 68 }, { 57162, 68 }, + { 57186, 68 }, { 57254, 68 }, { 57267, 1 }, { 57324, 68 }, { 57327, 68 }, { 57365, 4 }, { 57371, 68 }, { 57445, 68 }, { 57477, 68 }, { 57497, 68 }, { 57536, 68 }, { 57609, 68 }, { 57626, 68 }, { 57666, 68 }, { 57694, 68 }, { 57694, 68 }, { 57749, 68 }, { 57781, 7 }, { 57878, 68 }, { 57953, 68 }, { 58051, 68 }, { 58088, 68 }, + { 58097, 68 }, { 58142, 3 }, { 58142, 1 }, { 58197, 1 }, { 58221, 68 }, { 58222, 68 }, { 58244, 68 }, { 58290, 1 }, { 58296, 1 }, { 58325, 68 }, { 58378, 1 }, { 58389, 3 }, { 58430, 68 }, { 58454, 68 }, { 58551, 29 }, { 58563, 6 }, { 58681, 68 }, { 58751, 8 }, { 58752, 43 }, { 58790, 5 }, { 58846, 68 }, { 58879, 6 }, { 58953, 68 }, + { 58998, 68 }, { 59010, 1 }, { 59038, 5 }, { 59135, 68 }, { 59166, 68 }, { 59180, 68 }, { 59222, 68 }, { 59227, 68 }, { 59307, 68 }, { 59398, 3 }, { 59411, 68 }, { 59436, 3 }, { 59464, 68 }, { 59569, 68 }, { 59587, 68 }, { 59624, 3 }, { 59786, 68 }, { 59834, 68 }, { 59841, 68 }, { 59841, 1 }, { 59984, 68 }, { 59985, 68 }, + { 60003, 3 }, { 60045, 68 }, { 60097, 68 }, { 60148, 68 }, { 60172, 68 }, { 60203, 5 }, { 60565, 68 }, { 60625, 68 }, { 60743, 68 }, { 60781, 68 }, { 60892, 68 }, { 60977, 68 }, { 60979, 68 }, { 61021, 5 }, { 61021, 4 }, { 61026, 68 }, { 61139, 68 }, { 61165, 3 }, { 61204, 68 }, { 61207, 1 }, { 61248, 3 }, { 61257, 68 }, + { 61264, 6 }, { 61272, 3 }, { 61410, 68 }, { 61410, 3 }, { 61416, 68 }, { 61423, 1 }, { 61503, 68 }, { 61503, 68 }, { 61533, 68 }, { 61567, 68 }, { 61575, 68 }, { 61835, 1 }, { 61842, 1 }, { 61924, 68 }, { 61951, 6 }, { 61975, 68 }, { 61986, 3 }, { 62024, 1 }, { 62110, 68 }, { 62135, 68 }, { 62192, 68 }, { 62208, 68 }, + { 62399, 68 }, { 62400, 1 }, { 62414, 68 }, { 62423, 3 }, { 62456, 3 }, { 62459, 3 }, { 62478, 3 }, { 62484, 68 }, { 62510, 6 }, { 62511, 3 }, { 62565, 3 }, { 62610, 68 }, { 62875, 4 }, { 62896, 5 }, { 62898, 68 }, { 62904, 68 }, { 62938, 3 }, { 62943, 68 }, { 62977, 68 }, { 62989, 3 }, { 62998, 5 }, { 63069, 1 }, { 63093, 5 }, + { 63107, 68 }, { 63113, 1 }, { 63231, 4 }, { 63253, 68 }, { 63286, 4 }, { 63289, 68 }, { 63334, 1 }, { 63334, 4 }, { 63413, 68 }, { 63425, 68 }, { 63512, 10 }, { 63537, 1 }, { 63694, 1 }, { 63721, 4 }, { 63749, 68 }, { 63783, 17 }, { 63791, 3 }, { 63792, 68 }, { 63882, 25 }, { 63896, 1 }, { 63936, 68 }, { 63969, 3 }, { 63986, 68 }, + { 63988, 68 }, { 64009, 10 }, { 64018, 68 }, { 64032, 6 }, { 64125, 68 }, { 64195, 1 }, { 64221, 7 }, { 64390, 68 }, { 64459, 68 }, { 64568, 68 }, { 64784, 1 }, { 64789, 68 }, { 64829, 68 }, { 64848, 1 }, { 64914, 68 }, { 64928, 1 }, { 64939, 68 }, { 65026, 68 }, { 65057, 68 }, { 65070, 68 }, { 65193, 4 }, { 65235, 3 }, + { 65242, 68 }, { 65281, 68 }, { 65320, 68 }, { 65365, 1 }, { 65414, 68 }, { 65445, 68 }, { 65581, 68 }, { 65624, 1 }, { 65719, 68 }, { 65766, 68 }, { 65927, 68 }, { 66004, 1 }, { 66031, 68 }, { 66085, 1 }, { 66085, 68 }, { 66133, 68 }, { 66134, 68 }, { 66188, 1 }, { 66240, 68 }, { 66249, 68 }, { 66250, 68 }, { 66295, 68 }, + { 66342, 1 }, { 66352, 3 }, { 66388, 3 }, { 66432, 68 }, { 66437, 47 }, { 66497, 68 }, { 66517, 68 }, { 66526, 68 }, { 66546, 9 }, { 66605, 68 }, { 66753, 68 }, { 66792, 68 }, { 66796, 68 }, { 66828, 68 }, { 66899, 3 }, { 66970, 6 }, { 66981, 68 }, { 66983, 1 }, { 67009, 68 }, { 67017, 4 }, { 67115, 68 }, { 67117, 1 }, + { 67130, 6 }, { 67132, 7 }, { 67162, 68 }, { 67179, 6 }, { 67236, 68 }, { 67263, 3 }, { 67274, 68 }, { 67274, 68 }, { 67349, 3 }, { 67486, 68 }, { 67503, 3 }, { 67517, 1 }, { 67559, 1 }, { 67660, 68 }, { 67727, 68 }, { 67901, 68 }, { 67943, 4 }, { 67950, 68 }, { 67965, 3 }, { 68029, 68 }, { 68048, 68 }, { 68169, 68 }, { 68172, 1 }, + { 68258, 68 }, { 68288, 1 }, { 68359, 68 }, { 68441, 68 }, { 68484, 68 }, { 68488, 68 }, { 68525, 68 }, { 68535, 68 }, { 68575, 7 }, { 68575, 5 }, { 68583, 68 }, { 68588, 4 }, { 68593, 1 }, { 68597, 68 }, { 68636, 68 }, { 68636, 68 }, { 68667, 68 }, { 68785, 1 }, { 68914, 4 }, { 68915, 5 }, { 68940, 3 }, { 69010, 68 }, + { 69063, 68 }, { 69076, 68 }, { 69235, 68 }, { 69270, 68 }, { 69298, 1 }, { 69350, 5 }, { 69432, 68 }, { 69514, 68 }, { 69562, 3 }, { 69562, 4 }, { 69638, 1 }, { 69656, 68 }, { 69709, 68 }, { 69775, 68 }, { 69788, 68 }, { 70193, 68 }, { 70233, 68 }, { 70252, 68 }, { 70259, 68 }, { 70293, 3 }, { 70405, 3 }, { 70462, 68 }, + { 70515, 3 }, { 70518, 68 }, { 70535, 6 }, { 70547, 6 }, { 70577, 6 }, { 70631, 17 }, { 70667, 68 }, { 70680, 1 }, { 70694, 1 }, { 70898, 68 }, { 70916, 1 }, { 70936, 3 }, { 71033, 68 }, { 71126, 68 }, { 71158, 68 }, { 71162, 68 }, { 71421, 1 }, { 71441, 68 }, { 71557, 68 }, { 71789, 1 }, { 71816, 68 }, { 71850, 1 }, { 71869, 1 }, + { 71961, 68 }, { 71973, 4 }, { 72064, 68 }, { 72110, 68 }, { 72117, 3 }, { 72164, 68 }, { 72266, 68 }, { 72325, 68 }, { 72326, 1 }, { 72420, 68 }, { 72693, 68 }, { 72705, 1 }, { 72730, 68 }, { 72793, 68 }, { 72795, 1 }, { 72939, 1 }, { 72945, 3 }, { 72945, 68 }, { 73120, 1 }, { 73121, 5 }, { 73122, 4 }, { 73126, 1 }, { 73126, 1 }, + { 73196, 3 }, { 73219, 68 }, { 73241, 6 }, { 73272, 3 }, { 73354, 1 }, { 73368, 68 }, { 73467, 1 }, { 73517, 68 }, { 73554, 68 }, { 73678, 68 }, { 73838, 1 }, { 73881, 68 }, { 73958, 68 }, { 73985, 15 }, { 74092, 68 }, { 74205, 68 }, { 74245, 68 }, { 74277, 68 }, { 74286, 68 }, { 74353, 68 }, { 74403, 68 }, { 74428, 1 }, + { 74468, 68 }, { 74481, 3 }, { 74511, 68 }, { 74537, 68 }, { 74596, 68 }, { 74750, 68 }, { 74754, 68 }, { 74861, 68 }, { 74933, 4 }, { 74970, 1 }, { 75003, 3 }, { 75077, 1 }, { 75159, 68 }, { 75170, 68 }, { 75234, 45 }, { 75300, 3 }, { 75337, 68 }, { 75345, 68 }, { 75419, 1 }, { 75429, 68 }, { 75477, 1 }, { 75513, 68 }, + { 75536, 68 }, { 75536, 68 }, { 75539, 1 }, { 75551, 68 }, { 75561, 68 }, { 75565, 68 }, { 75590, 68 }, { 75623, 5 }, { 75773, 6 }, { 75777, 6 }, { 75785, 68 }, { 75791, 68 }, { 75804, 68 }, { 75862, 68 }, { 75924, 3 }, { 75927, 68 }, { 75996, 11 }, { 76000, 1 }, { 76006, 68 }, { 76020, 3 }, { 76110, 68 }, { 76126, 3 }, + { 76131, 68 }, { 76136, 68 }, { 76144, 68 }, { 76203, 68 }, { 76229, 3 }, { 76244, 15 }, { 76246, 68 }, { 76300, 1 }, { 76403, 3 }, { 76545, 68 }, { 76569, 68 }, { 76813, 68 }, { 76821, 68 }, { 76837, 68 }, { 76863, 68 }, { 77027, 68 }, { 77037, 65 }, { 77074, 3 }, { 77170, 68 }, { 77191, 68 }, { 77220, 68 }, { 77230, 68 }, + { 77261, 68 }, { 77277, 68 }, { 77309, 68 }, { 77314, 68 }, { 77412, 68 }, { 77419, 68 }, { 77457, 68 }, { 77633, 3 }, { 77714, 68 }, { 77855, 68 }, { 77857, 1 }, { 77876, 68 }, { 77895, 68 }, { 77916, 5 }, { 77947, 68 }, { 77948, 1 }, { 77966, 1 }, { 77996, 68 }, { 78025, 1 }, { 78064, 68 }, { 78100, 68 }, { 78113, 1 }, + { 78114, 3 }, { 78167, 68 }, { 78175, 68 }, { 78260, 68 }, { 78261, 1 }, { 78265, 68 }, { 78286, 1 }, { 78300, 68 }, { 78327, 3 }, { 78363, 1 }, { 78384, 68 }, { 78459, 68 }, { 78516, 68 }, { 78612, 68 }, { 78643, 68 }, { 78655, 68 }, { 78698, 1 }, { 78720, 3 }, { 78789, 76 }, { 78838, 5 }, { 78893, 1 }, { 78954, 7 }, + { 79007, 68 }, { 79132, 3 }, { 79193, 68 }, { 79193, 68 }, { 79226, 68 }, { 79411, 68 }, { 79422, 1 }, { 79502, 68 }, { 79593, 68 }, { 79622, 68 }, { 79657, 3 }, { 79771, 68 }, { 79866, 68 }, { 79909, 68 }, { 80005, 68 }, { 80032, 68 }, { 80060, 1 }, { 80132, 68 }, { 80149, 3 }, { 80251, 68 }, { 80363, 68 }, { 80379, 1 }, + { 80464, 68 }, { 80498, 68 }, { 80553, 68 }, { 80556, 3 }, { 80559, 1 }, { 80571, 68 }, { 80652, 1 }, { 80703, 68 }, { 80754, 68 }, { 80754, 68 }, { 80860, 68 }, { 81055, 68 }, { 81087, 4 }, { 81210, 68 }, { 81211, 1 }, { 81216, 1 }, { 81223, 1 }, { 81231, 1 }, { 81288, 68 }, { 81317, 68 }, { 81327, 65 }, { 81332, 68 }, + { 81376, 68 }, { 81469, 68 }, { 81579, 68 }, { 81617, 1 }, { 81630, 68 }, { 81666, 68 }, { 81800, 68 }, { 81832, 68 }, { 81848, 68 }, { 81869, 68 }, { 81941, 3 }, { 82177, 3 }, { 82179, 68 }, { 82180, 68 }, { 82182, 4 }, { 82185, 68 }, { 82195, 68 }, { 82238, 4 }, { 82265, 3 }, { 82295, 10 }, { 82299, 9 }, { 82367, 3 }, + { 82379, 3 }, { 82380, 1 }, { 82505, 68 }, { 82568, 68 }, { 82620, 1 }, { 82637, 5 }, { 82821, 68 }, { 82841, 68 }, { 82945, 1 }, { 83020, 168 }, { 83072, 68 }, { 83181, 68 }, { 83240, 68 }, { 83253, 3 }, { 83261, 68 }, { 83288, 68 }, { 83291, 4 }, { 83295, 3 }, { 83365, 68 }, { 83368, 68 }, { 83408, 68 }, { 83458, 68 }, + { 83470, 68 }, { 83471, 1 }, { 83637, 3 }, { 83693, 68 }, { 83703, 68 }, { 83732, 68 }, { 83745, 1 }, { 83800, 4 }, { 83801, 3 }, { 83856, 3 }, { 83863, 5 }, { 83867, 68 }, { 83868, 3 }, { 83898, 7 }, { 83900, 4 }, { 83901, 5 }, { 83989, 68 }, { 84049, 35 }, { 84086, 68 }, { 84089, 68 }, { 84115, 3 }, { 84130, 3 }, { 84132, 68 }, + { 84143, 54 }, { 84173, 68 }, { 84185, 5 }, { 84297, 68 }, { 84390, 68 }, { 84497, 4 }, { 84657, 68 }, { 84657, 68 }, { 84724, 68 }, { 84775, 68 }, { 84870, 68 }, { 84892, 68 }, { 84910, 3 }, { 84935, 3 }, { 85002, 68 }, { 85051, 68 }, { 85052, 68 }, { 85135, 25 }, { 85135, 68 }, { 85144, 68 }, { 85165, 3 }, { 85205, 68 }, + { 85232, 68 }, { 85281, 5 }, { 85423, 6 }, { 85539, 68 }, { 85582, 4 }, { 85609, 68 }, { 85701, 36 }, { 85705, 68 }, { 85824, 68 }, { 85824, 68 }, { 85858, 30 }, { 85858, 28 }, { 85904, 35 }, { 85910, 68 }, { 85913, 68 }, { 85926, 3 }, { 85942, 4 }, { 85969, 4 }, { 85996, 1 }, { 86013, 3 }, { 86034, 13 }, { 86068, 8 }, + { 86069, 8 }, { 86089, 8 }, { 86193, 13 }, { 86217, 7 }, { 86219, 68 }, { 86250, 68 }, { 86304, 16 }, { 86317, 68 }, { 86322, 4 }, { 86325, 68 }, { 86333, 68 }, { 86394, 68 }, { 86433, 68 }, { 86469, 3 }, { 86512, 4 }, { 86537, 68 }, { 86627, 68 }, { 86658, 68 }, { 86810, 68 }, { 86813, 68 }, { 86884, 68 }, { 86947, 68 }, + { 87003, 68 }, { 87010, 5 }, { 87019, 68 }, { 87027, 68 }, { 87105, 68 }, { 87107, 68 }, { 87183, 68 }, { 87273, 68 }, { 87358, 3 }, { 87388, 3 }, { 87503, 4 }, { 87639, 68 }, { 87649, 4 }, { 87722, 68 }, { 87829, 68 }, { 87829, 1 }, { 87863, 68 }, { 87894, 68 }, { 87988, 368 }, { 88035, 27 }, { 88059, 3 }, { 88094, 5 }, + { 88111, 21 }, { 88129, 68 }, { 88175, 5 }, { 88256, 68 }, { 88329, 76 }, { 88415, 3 }, { 88482, 68 }, { 88502, 1 }, { 88529, 68 }, { 88551, 3 }, { 88552, 1 }, { 88713, 68 }, { 88797, 68 }, { 88844, 27 }, { 88925, 5 }, { 88935, 68 }, { 88944, 1 }, { 89073, 68 }, { 89095, 3 }, { 89283, 68 }, { 89294, 3 }, { 89299, 68 }, + { 89324, 68 }, { 89368, 68 }, { 89387, 68 }, { 89464, 68 }, { 89607, 68 }, { 89737, 68 }, { 89791, 68 }, { 89794, 3 }, { 89840, 68 }, { 89849, 3 }, { 89859, 68 }, { 89905, 68 }, { 89952, 38 }, { 90030, 7 }, { 90030, 6 }, { 90031, 1 }, { 90072, 68 }, { 90090, 68 }, { 90146, 3 }, { 90202, 23 }, { 90302, 3 }, { 90328, 14 }, + { 90335, 14 }, { 90338, 8 }, { 90380, 68 }, { 90434, 1 }, { 90482, 68 }, { 90527, 9 }, { 90537, 68 }, { 90545, 68 }, { 90639, 5 }, { 90642, 68 }, { 90709, 68 }, { 90775, 1 }, { 90806, 68 }, { 90845, 19 }, { 90872, 4 }, { 90884, 68 }, { 90910, 68 }, { 90994, 5 }, { 91046, 8 }, { 91059, 8 }, { 91096, 39 }, { 91147, 68 }, + { 91168, 1 }, { 91493, 68 }, { 91513, 3 }, { 91618, 3 }, { 91653, 68 }, { 91817, 68 }, { 91831, 3 }, { 91833, 3 }, { 91885, 68 }, { 91919, 68 }, { 91934, 68 }, { 92245, 1 }, { 92284, 68 }, { 92292, 4 }, { 92369, 3 }, { 92388, 68 }, { 92426, 7 }, { 92720, 14 }, { 92720, 6 }, { 92729, 9 }, { 92733, 13 }, { 92735, 6 }, { 92786, 68 }, + { 92853, 31 }, { 92906, 68 }, { 93031, 7 }, { 93077, 68 }, { 93102, 68 }, { 93109, 68 }, { 93122, 3 }, { 93214, 68 }, { 93330, 68 }, { 93395, 68 }, { 93506, 68 }, { 93564, 9 }, { 93713, 9 }, { 93722, 4 }, { 93840, 68 }, { 93877, 4 }, { 93891, 3 }, { 93948, 68 }, { 93981, 68 }, { 94012, 3 }, { 94033, 68 }, { 94121, 68 }, + { 94165, 368 }, { 94181, 3 }, { 94210, 68 }, { 94216, 68 }, { 94230, 68 }, { 94333, 31 }, { 94433, 3 }, { 94497, 3 }, { 94609, 68 }, { 94623, 68 }, { 94763, 68 }, { 94780, 68 }, { 95287, 68 }, { 95348, 68 }, { 95433, 5 }, { 95446, 68 }, { 95493, 7 }, { 95517, 3 }, { 95580, 68 }, { 95610, 5 }, { 95620, 68 }, { 95678, 3 }, + { 95683, 68 }, { 95689, 68 }, { 95760, 68 }, { 95792, 68 }, { 95850, 68 }, { 95908, 68 }, { 95908, 68 }, { 95967, 68 }, { 96022, 3 }, { 96088, 65 }, { 96460, 68 }, { 96554, 68 }, { 96597, 68 }, { 96763, 68 }, { 96808, 68 }, { 96854, 1 }, { 96963, 1 }, { 97007, 3 }, { 97125, 1 }, { 97128, 68 }, { 97133, 3 }, { 97142, 3 }, + { 97156, 68 }, { 97223, 68 }, { 97244, 68 }, { 97303, 68 }, { 97355, 68 }, { 97356, 3 }, { 97393, 3 }, { 97409, 1 }, { 97451, 68 }, { 97539, 68 }, { 97546, 68 }, { 97553, 68 }, { 97627, 68 }, { 97640, 68 }, { 97650, 6 }, { 97675, 68 }, { 97685, 3 }, { 97773, 68 }, { 97802, 4 }, { 97826, 19 }, { 97860, 68 }, { 97956, 68 }, + { 97958, 68 }, { 97973, 3 }, { 97982, 68 }, { 98039, 68 }, { 98051, 68 }, { 98059, 68 }, { 98088, 68 }, { 98092, 4 }, { 98147, 68 }, { 98147, 68 }, { 98169, 68 }, { 98207, 65 }, { 98277, 1 }, { 98277, 268 }, { 98285, 68 }, { 98324, 3 }, { 98324, 3 }, { 98381, 31 }, { 98390, 68 }, { 98404, 68 }, { 98415, 4 }, { 98460, 68 }, + { 98462, 1 }, { 98475, 3 }, { 98485, 68 }, { 98640, 1 }, { 98798, 68 }, { 98800, 4 }, { 98821, 68 }, { 98895, 68 }, { 98936, 68 }, { 98950, 68 }, { 98980, 68 }, { 99033, 68 }, { 99045, 68 }, { 99135, 68 }, { 99315, 30 }, { 99324, 68 }, { 99346, 68 }, { 99418, 68 }, { 99505, 68 }, { 99557, 68 }, { 99559, 68 }, { 99586, 68 }, + { 99622, 68 }, { 99770, 1 }, { 99790, 68 }, { 99810, 68 }, { 99871, 1 }, { 99926, 68 }, { 99927, 68 }, { 99978, 68 }, { 99980, 68 }, { 100022, 3 }, { 100024, 1 }, { 100069, 68 }, { 100150, 68 }, { 100225, 63 }, { 100246, 1 }, { 100310, 68 }, { 100361, 68 }, { 100428, 1 }, { 100434, 68 }, { 100450, 4 }, { 100546, 68 }, + { 100551, 68 }, { 100551, 68 }, { 100554, 1 }, { 100597, 68 }, { 100676, 68 }, { 100693, 68 }, { 100827, 68 }, { 100928, 68 }, { 100928, 1 }, { 100935, 68 }, { 100937, 3 }, { 101034, 68 }, { 101041, 68 }, { 101154, 68 }, { 101200, 4 }, { 101250, 68 }, { 101352, 68 }, { 101403, 68 }, { 101430, 1 }, { 101508, 3 }, { 101509, 3 }, + { 101523, 10 }, { 101604, 68 }, { 101637, 68 }, { 101681, 4 }, { 101759, 1 }, { 101773, 1 }, { 101836, 1 }, { 101882, 4 }, { 101895, 68 }, { 101897, 68 }, { 101939, 68 }, { 101951, 6 }, { 101956, 5 }, { 102055, 1 }, { 102085, 68 }, { 102093, 67 }, { 102209, 68 }, { 102258, 6 }, { 102271, 68 }, { 102284, 68 }, { 102332, 68 }, + { 102354, 68 }, { 102366, 68 }, { 102424, 3 }, { 102456, 68 }, { 102496, 1 }, { 102497, 3 }, { 102519, 3 }, { 102554, 1 }, { 102610, 5 }, { 102657, 68 }, { 102661, 4 }, { 102695, 4 }, { 102707, 168 }, { 102910, 68 }, { 102930, 5 }, { 102937, 9 }, { 102938, 7 }, { 102965, 6 }, { 102969, 7 }, { 103031, 68 }, { 103062, 68 }, + { 103096, 68 }, { 103146, 68 }, { 103159, 68 }, { 103223, 68 }, { 103267, 68 }, { 103296, 68 }, { 103303, 68 }, { 103487, 68 }, { 103491, 68 }, { 103599, 68 }, { 103677, 68 }, { 103903, 1 }, { 104040, 68 }, { 104047, 1 }, { 104052, 68 }, { 104057, 4 }, { 104057, 68 }, { 104062, 98 }, { 104091, 68 }, { 104189, 3 }, { 104283, 8 }, + { 104288, 4 }, { 104305, 3 }, { 104445, 68 }, { 104472, 68 }, { 104475, 1 }, { 104497, 4 }, { 104548, 68 }, { 104582, 68 }, { 104626, 1 }, { 104716, 68 }, { 104826, 68 }, { 104849, 68 }, { 104872, 1 }, { 104945, 1 }, { 104948, 68 }, { 105066, 68 }, { 105071, 1 }, { 105198, 4 }, { 105198, 4 }, { 105203, 68 }, { 105256, 6 }, + { 105263, 68 }, { 105329, 68 }, { 105515, 68 }, { 105566, 68 }, { 105566, 68 }, { 105585, 68 }, { 105678, 68 }, { 105852, 68 }, { 105877, 68 }, { 105911, 68 }, { 106022, 1 }, { 106033, 68 }, { 106080, 68 }, { 106192, 68 }, { 106220, 3 }, { 106243, 68 }, { 106323, 11 }, { 106371, 68 }, { 106608, 68 }, { 106624, 87 }, { 106680, 3 }, + { 106688, 1 }, { 106800, 1 }, { 106800, 1 }, { 106821, 4 }, { 106853, 1 }, { 106930, 3 }, { 106937, 68 }, { 106955, 68 }, { 106996, 68 }, { 106996, 1 }, { 107148, 4 }, { 107213, 16 }, { 107213, 68 }, { 107243, 68 }, { 107360, 68 }, { 107408, 68 }, { 107509, 4 }, { 107572, 68 }, { 107592, 68 }, { 107644, 5 }, { 107679, 68 }, + { 107705, 3 }, { 107761, 4 }, { 107780, 68 }, { 107825, 68 }, { 108007, 68 }, { 108041, 4 }, { 108058, 68 }, { 108071, 1 }, { 108132, 68 }, { 108164, 68 }, { 108189, 68 }, { 108210, 68 }, { 108330, 68 }, { 108430, 68 }, { 108450, 68 }, { 108469, 68 }, { 108484, 68 }, { 108533, 68 }, { 108588, 68 }, { 108594, 68 }, { 108690, 68 }, + { 108785, 76 }, { 108814, 68 }, { 108818, 1 }, { 108820, 68 }, { 108889, 68 }, { 108951, 68 }, { 108959, 68 }, { 108963, 68 }, { 109034, 68 }, { 109172, 1 }, { 109176, 68 }, { 109195, 3 }, { 109229, 68 }, { 109256, 68 }, { 109290, 68 }, { 109304, 68 }, { 109333, 68 }, { 109343, 4 }, { 109347, 7 }, { 109387, 68 }, { 109421, 1 }, + { 109497, 68 }, { 109501, 3 }, { 109513, 68 }, { 109525, 3 }, { 109625, 4 }, { 109710, 68 }, { 109740, 68 }, { 109751, 68 }, { 109761, 68 }, { 109890, 8 }, { 109891, 4 }, { 109909, 68 }, { 109923, 1 }, { 110017, 68 }, { 110046, 68 }, { 110111, 68 }, { 110258, 68 }, { 110340, 68 }, { 110352, 68 }, { 110398, 68 }, { 110583, 68 }, + { 110600, 13 }, { 110626, 3 }, { 110709, 68 }, { 110772, 4 }, { 110773, 68 }, { 110813, 1 }, { 110890, 68 }, { 110898, 68 }, { 110954, 68 }, { 111120, 68 }, { 111132, 3 }, { 111163, 8 }, { 111224, 68 }, { 111340, 68 }, { 111398, 68 }, { 111555, 68 }, { 111597, 3 }, { 111607, 68 }, { 111655, 68 }, { 111691, 3 }, { 111835, 68 }, + { 111854, 68 }, { 111876, 16 }, { 111884, 1 }, { 111884, 56 }, { 111929, 68 }, { 111941, 68 }, { 111969, 68 }, { 112003, 68 }, { 112165, 68 }, { 112365, 68 }, { 112450, 1 }, { 112521, 68 }, { 112649, 4 }, { 112665, 68 }, { 112881, 1 }, { 112882, 68 }, { 112906, 68 }, { 112951, 68 }, { 112994, 68 }, { 112997, 68 }, { 113002, 68 }, + { 113056, 1 }, { 113077, 68 }, { 113208, 1 }, { 113320, 68 }, { 113326, 3 }, { 113375, 68 }, { 113530, 30 }, { 113530, 30 }, { 113537, 1 }, { 113563, 14 }, { 113592, 68 }, { 113637, 68 }, { 113768, 68 }, { 113850, 5 }, { 113892, 68 }, { 113916, 68 }, { 113965, 68 }, { 113976, 68 }, { 114037, 68 }, { 114149, 1 }, { 114158, 9 }, + { 114201, 68 }, { 114262, 68 }, { 114268, 4 }, { 114353, 68 }, { 114388, 68 }, { 114404, 68 }, { 114428, 5 }, { 114438, 68 }, { 114541, 68 }, { 114550, 68 }, { 114561, 68 }, { 114625, 3 }, { 114730, 68 }, { 114770, 1 }, { 114815, 4 }, { 114998, 68 }, { 115077, 68 }, { 115093, 68 }, { 115120, 68 }, { 115194, 68 }, { 115216, 3 }, + { 115299, 68 }, { 115391, 3 }, { 115410, 68 }, { 115542, 33 }, { 115581, 68 }, { 115618, 68 }, { 115645, 23 }, { 115647, 68 }, { 115697, 68 }, { 115725, 68 }, { 115740, 68 }, { 115757, 68 }, { 115763, 68 }, { 115770, 68 }, { 115787, 68 }, { 115916, 68 }, { 115928, 68 }, { 115962, 68 }, { 116020, 68 }, { 116022, 1 }, { 116089, 68 }, + { 116159, 1 }, { 116196, 68 }, { 116247, 68 }, { 116254, 7 }, { 116336, 68 }, { 116409, 68 }, { 116459, 68 }, { 116569, 68 }, { 116619, 68 }, { 116688, 68 }, { 116733, 68 }, { 116807, 3 }, { 116843, 68 }, { 116886, 1 }, { 116902, 68 }, { 116931, 68 }, { 116952, 68 }, { 116952, 68 }, { 117177, 68 }, { 117189, 68 }, { 117206, 68 }, + { 117260, 29 }, { 117271, 6 }, { 117276, 3 }, { 117276, 5 }, { 117278, 3 }, { 117278, 68 }, { 117359, 4 }, { 117380, 68 }, { 117414, 1 }, { 117503, 68 }, { 117517, 68 }, { 117530, 68 }, { 117574, 4 }, { 117575, 5 }, { 117577, 68 }, { 117606, 68 }, { 117645, 68 }, { 117655, 68 }, { 117692, 68 }, { 117705, 1 }, { 117731, 1 }, + { 117762, 4 }, { 117780, 68 }, { 117974, 1 }, { 118057, 1 }, { 118099, 68 }, { 118107, 68 }, { 118113, 68 }, { 118175, 68 }, { 118198, 68 }, { 118232, 45 }, { 118326, 1 }, { 118438, 31 }, { 118469, 68 }, { 118521, 31 }, { 118565, 68 }, { 118593, 68 }, { 118602, 68 }, { 118652, 68 }, { 118668, 68 }, { 118689, 3 }, { 118703, 14 }, + { 118705, 68 }, { 118813, 68 }, { 118825, 68 }, { 118894, 3 }, { 118915, 68 }, { 118962, 68 }, { 118986, 68 }, { 119045, 68 }, { 119054, 1 }, { 119054, 1 }, { 119119, 68 }, { 119149, 68 }, { 119206, 1 }, { 119316, 68 }, { 119387, 68 }, { 119404, 3 }, { 119516, 68 }, { 119520, 68 }, { 119571, 3 }, { 119573, 68 }, { 119610, 5 }, + { 119621, 68 }, { 119623, 4 }, { 119672, 68 }, { 119692, 3 }, { 119734, 68 }, { 119742, 1 }, { 119754, 1 }, { 119785, 68 }, { 120001, 68 }, { 120115, 4 }, { 120260, 68 }, { 120314, 68 }, { 120416, 68 }, { 120435, 1 }, { 120450, 3 }, { 120530, 68 }, { 120550, 5 }, { 120730, 68 }, { 120731, 68 }, { 120751, 3 }, { 120755, 68 }, + { 120869, 68 }, { 120988, 68 }, { 121061, 68 }, { 121177, 68 }, { 121212, 68 }, { 121214, 1 }, { 121286, 68 }, { 121331, 1 }, { 121344, 68 }, { 121407, 68 }, { 121424, 1 }, { 121491, 68 }, { 121568, 76 }, { 121588, 6 }, { 121651, 68 }, { 121676, 68 }, { 121785, 4 }, { 121830, 3 }, { 121919, 1 }, { 121951, 68 }, { 121991, 1 }, + { 122056, 68 }, { 122062, 68 }, { 122144, 68 }, { 122183, 1 }, { 122331, 68 }, { 122466, 68 }, { 122558, 68 }, { 122570, 68 }, { 122676, 68 }, { 122733, 68 }, { 122774, 6 }, { 122783, 68 }, { 122825, 68 }, { 122865, 68 }, { 122884, 68 }, { 122892, 68 }, { 122911, 68 }, { 122929, 68 }, { 122936, 68 }, { 123190, 68 }, { 123271, 68 }, + { 123271, 68 }, { 123302, 7 }, { 123391, 68 }, { 123394, 68 }, { 123416, 1 }, { 123708, 68 }, { 123752, 68 }, { 123761, 68 }, { 123783, 68 }, { 123794, 68 }, { 123817, 68 }, { 123820, 1 }, { 123823, 1 }, { 123857, 3 }, { 123886, 56 }, { 124023, 1 }, { 124029, 68 }, { 124042, 68 }, { 124056, 3 }, { 124071, 6 }, { 124105, 5 }, + { 124143, 68 }, { 124191, 68 }, { 124207, 1 }, { 124257, 68 }, { 124306, 3 }, { 124338, 68 }, { 124388, 8 }, { 124400, 68 }, { 124418, 68 }, { 124502, 68 }, { 124521, 1 }, { 124533, 68 }, { 124645, 68 }, { 124685, 1 }, { 124694, 68 }, { 124700, 1 }, { 124736, 68 }, { 124891, 7 }, { 124920, 68 }, { 124983, 68 }, { 125014, 68 }, + { 125038, 68 }, { 125084, 68 }, { 125162, 68 }, { 125193, 68 }, { 125285, 68 }, { 125368, 68 }, { 125409, 68 }, { 125570, 68 }, { 125601, 68 }, { 125641, 1 }, { 125721, 68 }, { 125731, 68 }, { 125803, 68 }, { 125904, 68 }, { 125973, 68 }, { 126018, 1 }, { 126034, 5 }, { 126094, 1 }, { 126144, 1 }, { 126195, 68 }, { 126297, 68 }, + { 126389, 68 }, { 126429, 68 }, { 126439, 68 }, { 126499, 68 }, { 126501, 1 }, { 126587, 68 }, { 126663, 68 }, { 126681, 68 }, { 126687, 1 }, { 126781, 68 }, { 126783, 68 }, { 126840, 8 }, { 126843, 68 }, { 126959, 68 }, { 127015, 68 }, { 127101, 68 }, { 127149, 68 }, { 127197, 54 }, { 127268, 68 }, { 127372, 68 }, { 127385, 68 }, + { 127473, 4 }, { 127539, 68 }, { 127598, 68 }, { 127613, 14 }, { 127683, 3 }, { 127684, 68 }, { 127697, 68 }, { 127698, 3 }, { 127773, 68 }, { 127781, 1 }, { 127839, 68 }, { 127905, 68 }, { 127949, 68 }, { 128035, 68 }, { 128046, 1 }, { 128167, 68 }, { 128271, 68 }, { 128307, 1 }, { 128320, 68 }, { 128330, 68 }, { 128375, 68 }, + { 128381, 4 }, { 128447, 68 }, { 128462, 68 }, { 128466, 3 }, { 128466, 68 }, { 128496, 68 }, { 128589, 68 }, { 128616, 3 }, { 128679, 1 }, { 128770, 1 }, { 128793, 68 }, { 128802, 68 }, { 128813, 68 }, { 128900, 68 }, { 128949, 68 }, { 129269, 68 }, { 129271, 3 }, { 129278, 68 }, { 129343, 1 }, { 129408, 67 }, { 129408, 1 }, + { 129421, 6 }, { 129461, 68 }, { 129469, 3 }, { 129482, 68 }, { 129502, 68 }, { 129512, 68 }, { 129551, 68 }, { 129629, 68 }, { 129632, 68 }, { 129679, 1 }, { 129725, 68 }, { 130007, 68 }, { 130018, 16 }, { 130057, 68 }, { 130071, 68 }, { 130087, 68 }, { 130188, 1 }, { 130202, 68 }, { 130316, 68 }, { 130328, 1 }, { 130466, 68 }, + { 130549, 68 }, { 130649, 68 }, { 130705, 3 }, { 130800, 68 }, { 130907, 68 }, { 130989, 68 }, { 131103, 68 }, { 131127, 68 }, { 131200, 5 }, { 131241, 6 }, { 131351, 68 }, { 131413, 68 }, { 131448, 68 }, { 131599, 68 }, { 131634, 1 }, { 131687, 68 }, { 131739, 68 }, { 131758, 68 }, { 131765, 68 }, { 131787, 3 }, { 131819, 3 }, + { 131868, 68 }, { 131886, 68 }, { 131901, 4 }, { 131977, 68 }, { 131990, 68 }, { 132035, 68 }, { 132035, 68 }, { 132043, 68 }, { 132173, 68 }, { 132181, 4 }, { 132181, 6 }, { 132194, 5 }, { 132252, 68 }, { 132262, 6 }, { 132271, 1 }, { 132285, 68 }, { 132328, 68 }, { 132335, 1 }, { 132337, 1 }, { 132389, 5 }, { 132430, 68 }, + { 132451, 68 }, { 132499, 87 }, { 132503, 1 }, { 132520, 4 }, { 132541, 4 }, { 132860, 68 }, { 132862, 4 }, { 132874, 168 }, { 132874, 13 }, { 132875, 168 }, { 132911, 68 }, { 132973, 68 }, { 133051, 68 }, { 133062, 68 }, { 133067, 68 }, { 133138, 68 }, { 133184, 68 }, { 133231, 68 }, { 133297, 3 }, { 133344, 68 }, { 133385, 4 }, + { 133408, 68 }, { 133464, 68 }, { 133522, 68 }, { 133631, 68 }, { 133631, 68 }, { 133702, 68 }, { 133705, 1 }, { 133721, 68 }, { 133746, 68 }, { 133773, 3 }, { 133819, 68 }, { 133843, 68 }, { 133929, 68 }, { 133946, 68 }, { 134113, 4 }, { 134151, 68 }, { 134289, 1 }, { 134385, 68 }, { 134429, 68 }, { 134506, 68 }, { 134511, 68 }, + { 134521, 68 }, { 134558, 1 }, { 134710, 68 }, { 134738, 68 }, { 134751, 3 }, { 134818, 68 }, { 134820, 4 }, { 134879, 68 }, { 134919, 68 }, { 134947, 68 }, { 134948, 3 }, { 135040, 3 }, { 135125, 10 }, { 135155, 68 }, { 135228, 68 }, { 135255, 68 }, { 135296, 3 }, { 135322, 68 }, { 135349, 68 }, { 135428, 3 }, { 135476, 1 }, + { 135503, 68 }, { 135524, 68 }, { 135550, 4 }, { 135594, 68 }, { 135597, 68 }, { 135624, 3 }, { 135741, 68 }, { 135753, 68 }, { 135842, 68 }, { 135853, 68 }, { 135896, 3 }, { 136004, 1 }, { 136061, 1 }, { 136068, 1 }, { 136106, 68 }, { 136145, 68 }, { 136145, 68 }, { 136173, 68 }, { 136186, 68 }, { 136196, 68 }, { 136201, 68 }, + { 136211, 68 }, { 136268, 68 }, { 136298, 68 }, { 136377, 68 }, { 136420, 68 }, { 136475, 23 }, { 136486, 1 }, { 136554, 68 }, { 136641, 68 }, { 136770, 1 }, { 136873, 68 }, { 136877, 1 }, { 136906, 68 }, { 137092, 68 }, { 137143, 68 }, { 137200, 3 }, { 137232, 68 }, { 137239, 68 }, { 137248, 68 }, { 137281, 1 }, { 137301, 68 }, + { 137314, 3 }, { 137352, 1 }, { 137365, 68 }, { 137375, 68 }, { 137411, 68 }, { 137424, 68 }, { 137516, 68 }, { 137532, 68 }, { 137593, 68 }, { 137600, 68 }, { 137658, 68 }, { 137703, 68 }, { 137766, 68 }, { 137791, 68 }, { 137801, 68 }, { 137864, 68 }, { 137870, 3 }, { 137931, 68 }, { 138009, 3 }, { 138013, 1 }, { 138013, 1 }, + { 138066, 68 }, { 138073, 68 }, { 138114, 68 }, { 138150, 68 }, { 138236, 68 }, { 138276, 68 }, { 138286, 68 }, { 138298, 3 }, { 138309, 1 }, { 138373, 3 }, { 138524, 68 }, { 138535, 1 }, { 138593, 4 }, { 138611, 1 }, { 138725, 68 }, { 138807, 68 }, { 138819, 3 }, { 138849, 5 }, { 138867, 68 }, { 138907, 68 }, { 138930, 3 }, + { 139026, 68 }, { 139102, 68 }, { 139108, 3 }, { 139184, 1 }, { 139209, 3 }, { 139282, 68 }, { 139289, 4 }, { 139382, 1 }, { 139421, 1 }, { 139436, 68 }, { 139450, 1 }, { 139523, 3 }, { 139533, 68 }, { 139590, 68 }, { 139590, 68 }, { 139722, 68 }, { 139785, 68 }, { 139785, 1 }, { 139798, 68 }, { 139813, 68 }, { 139868, 68 }, + { 139935, 3 }, { 139990, 3 }, { 140050, 68 }, { 140177, 68 }, { 140177, 4 }, { 140408, 68 }, { 140420, 3 }, { 140461, 68 }, { 140578, 15 }, { 140605, 1368 }, { 140662, 1 }, { 140755, 68 }, { 140786, 68 }, { 140846, 68 }, { 140874, 68 }, { 140959, 1 }, { 140973, 68 }, { 141128, 68 }, { 141132, 68 }, { 141257, 68 }, { 141290, 1 }, + { 141360, 68 }, { 141472, 68 }, { 141545, 68 }, { 141545, 68 }, { 141575, 1 }, { 141606, 5 }, { 141655, 68 }, { 141735, 68 }, { 141767, 5 }, { 141796, 68 }, { 141841, 68 }, { 141915, 68 }, { 141923, 1 }, { 141932, 68 }, { 141994, 68 }, { 142018, 68 }, { 142029, 3 }, { 142072, 68 }, { 142128, 68 }, { 142133, 1 }, { 142261, 68 }, + { 142304, 1 }, { 142400, 68 }, { 142401, 68 }, { 142409, 68 }, { 142479, 68 }, { 142522, 1 }, { 142552, 1 }, { 142589, 68 }, { 142596, 68 }, { 142753, 1 }, { 142766, 68 }, { 142796, 68 }, { 142836, 68 }, { 142871, 68 }, { 143058, 3 }, { 143059, 6 }, { 143063, 3 }, { 143065, 68 }, { 143141, 4 }, { 143173, 68 }, { 143374, 68 }, + { 143399, 68 }, { 143406, 68 }, { 143429, 3 }, { 143430, 68 }, { 143462, 1 }, { 143579, 68 }, { 143663, 68 }, { 143844, 3 }, { 143851, 68 }, { 143926, 68 }, { 143931, 68 }, { 144051, 6 }, { 144085, 10 }, { 144147, 68 }, { 144188, 4 }, { 144238, 4 }, { 144353, 68 }, { 144465, 68 }, { 144474, 68 }, { 144637, 68 }, { 144638, 1 }, + { 144648, 1 }, { 144661, 3 }, { 144812, 68 }, { 144847, 68 }, { 144901, 8 }, { 145058, 68 }, { 145122, 8 }, { 145134, 68 }, { 145150, 68 }, { 145299, 1 }, { 145313, 68 }, { 145314, 3 }, { 145374, 68 }, { 145412, 68 }, { 145432, 68 }, { 145446, 68 }, { 145534, 3 }, { 145592, 68 }, { 145614, 68 }, { 145648, 68 }, { 145721, 68 }, + { 145858, 1 }, { 145970, 3 }, { 145984, 3 }, { 146004, 68 }, { 146016, 3 }, { 146048, 68 }, { 146097, 3 }, { 146103, 68 }, { 146136, 68 }, { 146194, 3 }, { 146230, 1 }, { 146254, 68 }, { 146261, 4 }, { 146269, 4 }, { 146393, 68 }, { 146411, 3 }, { 146501, 68 }, { 146547, 68 }, { 146547, 68 }, { 146573, 68 }, { 146616, 68 }, + { 146622, 3 }, { 146728, 3 }, { 146781, 5 }, { 146805, 4 }, { 146921, 68 }, { 147002, 3 }, { 147072, 68 }, { 147159, 68 }, { 147170, 68 }, { 147203, 1 }, { 147245, 68 }, { 147278, 68 }, { 147422, 68 }, { 147471, 68 }, { 147491, 68 }, { 147607, 23 }, { 147693, 68 }, { 147763, 68 }, { 147775, 6 }, { 147776, 4 }, { 147824, 68 }, + { 147922, 68 }, { 147922, 68 }, { 147937, 68 }, { 147957, 68 }, { 147980, 68 }, { 148008, 68 }, { 148018, 68 }, { 148046, 3 }, { 148071, 4 }, { 148106, 3 }, { 148122, 68 }, { 148139, 68 }, { 148175, 68 }, { 164238, 18 }, { 164315, 28 }, { 164449, 28 }, { 164529, 28 }, { 164574, 348 }, { 164591, 968 }, { 164595, 28 }, + { 164611, 28 }, { 164623, 48 }, { 164632, 108 }, { 164691, 28 }, { 164706, 28 }, { 164755, 28 }, { 164761, 28 }, { 164973, 28 }, { 165030, 28 }, { 165090, 28 }, { 165099, 18 }, { 165126, 28 }, { 165188, 28 }, { 165205, 28 }, { 165275, 18 }, { 165347, 28 }, { 165381, 28 }, { 165562, 28 }, { 165563, 18 }, { 165594, 568 }, + { 165641, 868 }, { 165663, 68 }, { 165759, 28 }, { 165811, 28 }, { 165822, 18 }, { 165830, 18 }, { 165903, 18 }, { 165921, 28 }, { 165953, 18 }, { 166022, 18 }, { 166294, 28 }, { 166333, 28 }, { 166420, 28 }, { 166433, 28 }, { 166442, 18 }, { 166536, 28 }, { 166543, 28 }, { 166556, 28 }, { 166571, 28 }, { 166575, 18 }, + { 166588, 28 }, { 166601, 678 }, { 166663, 788 }, { 166692, 18 }, { 166710, 28 }, { 166759, 28 }, { 166785, 38 }, { 166842, 28 }, { 166843, 28 }, { 166864, 28 }, { 166902, 28 }, { 166996, 28 }, { 166999, 28 }, { 167038, 28 }, { 167112, 48 }, { 167112, 28 }, { 167177, 28 }, { 167180, 28 }, { 167229, 18 }, { 167298, 28 }, + { 167306, 48 }, { 167309, 38 }, { 167402, 28 }, { 167405, 878 }, { 167433, 568 }, { 167435, 18 }, { 167461, 38 }, { 167553, 38 }, { 167688, 58 }, { 167689, 28 }, { 167709, 28 }, { 167744, 28 }, { 167821, 28 }, { 167825, 28 }, { 167925, 108 }, { 167969, 28 }, { 168024, 28 }, { 168089, 28 }, { 168104, 28 }, { 168194, 28 }, + { 168305, 28 }, { 168316, 28 }, { 168366, 28 }, { 168423, 28 }, { 168568, 38 }, { 168582, 558 }, { 168615, 768 }, { 168618, 28 }, { 168638, 28 }, { 168671, 28 }, { 168736, 28 }, { 168747, 28 }, { 168750, 48 }, { 168808, 38 }, { 168814, 48 }, { 168820, 28 }, { 168914, 28 }, { 168968, 28 }, { 168979, 28 }, { 169006, 28 }, + { 169069, 28 }, { 169106, 38 }, { 169158, 28 }, { 169158, 28 }, { 169189, 28 }, { 169253, 28 }, { 169259, 18 }, { 169279, 658 }, { 169325, 568 }, { 169349, 28 }, { 169353, 28 }, { 169378, 28 }, { 169432, 28 }, { 169476, 18 }, { 169476, 18 }, { 169525, 28 }, { 169538, 78 }, { 169555, 28 }, { 169571, 28 }, { 169594, 48 }, + { 169687, 28 }, { 169799, 28 }, { 169831, 28 }, { 170042, 28 }, { 170061, 28 }, { 170065, 18 }, { 170128, 68 }, { 170148, 208 }, { 170215, 708 }, { 170256, 608 }, { 170266, 698 }, { 170275, 78 }, { 170277, 68 }, { 170500, 38 }, { 170516, 38 }, { 170601, 28 }, { 170666, 28 }, { 170668, 48 }, { 170668, 18 }, { 170716, 38 }, + { 170728, 38 }, { 170735, 58 }, { 170847, 38 }, { 170852, 98 }, { 170858, 438 }, { 170859, 568 }, { 170956, 568 }, { 170956, 18 }, { 170967, 28 }, { 171005, 28 }, { 171113, 28 }, { 171279, 28 }, { 171400, 28 }, { 171405, 28 }, { 171448, 18 }, { 171490, 28 }, { 171567, 328 }, { 171590, 28 }, { 171723, 28 }, { 171737, 38 }, + { 171958, 28 }, { 171967, 68 }, { 164238, 18 }, { 164315, 28 }, { 164449, 28 }, { 164529, 28 }, { 164574, 348 }, { 164591, 968 }, { 164595, 28 }, { 164611, 28 }, { 164623, 48 }, { 164632, 108 }, { 164691, 28 }, { 164706, 28 }, { 164755, 28 }, { 164761, 28 }, { 164973, 28 }, { 165030, 28 }, { 165090, 28 }, { 165099, 18 }, + { 165126, 28 }, { 165188, 28 }, { 165205, 28 }, { 165275, 18 }, { 165347, 28 }, { 165381, 28 }, { 165562, 28 }, { 165563, 18 }, { 165594, 568 }, { 165641, 868 }, { 165663, 68 }, { 165759, 28 }, { 165811, 28 }, { 165822, 18 }, { 165830, 18 }, { 165903, 18 }, { 165921, 28 }, { 165953, 18 }, { 166022, 18 }, { 166294, 28 }, + { 166333, 28 }, { 166420, 28 }, { 166433, 28 }, { 166442, 18 }, { 166536, 28 }, { 166543, 28 }, { 166556, 28 }, { 166571, 28 }, { 166575, 18 }, { 166588, 28 }, { 166601, 678 }, { 166663, 788 }, { 166692, 18 }, { 166710, 28 }, { 166759, 28 }, { 166785, 38 }, { 166842, 28 }, { 166843, 28 }, { 166864, 28 }, { 166902, 28 }, + { 166996, 28 }, { 166999, 28 }, { 167038, 28 }, { 167112, 48 }, { 167112, 28 }, { 167177, 28 }, { 167180, 28 }, { 167229, 18 }, { 167298, 28 }, { 167306, 48 }, { 167309, 38 }, { 167402, 28 }, { 167405, 878 }, { 167433, 568 }, { 167435, 18 }, { 167461, 38 }, { 167553, 38 }, { 167688, 58 }, { 167689, 28 }, { 167709, 28 }, + { 167744, 28 }, { 167821, 28 }, { 167825, 28 }, { 167925, 108 }, { 167969, 28 }, { 168024, 28 }, { 168089, 28 }, { 168104, 28 }, { 168194, 28 }, { 168305, 28 }, { 168316, 28 }, { 168366, 28 }, { 168423, 28 }, { 168568, 38 }, { 168582, 558 }, { 168615, 768 }, { 168618, 28 }, { 168638, 28 }, { 168671, 28 }, { 168736, 28 }, + { 168747, 28 }, { 168750, 48 }, { 168808, 38 }, { 168814, 48 }, { 168820, 28 }, { 168914, 28 }, { 168968, 28 }, { 168979, 28 }, { 169006, 28 }, { 169069, 28 }, { 169106, 38 }, { 169158, 28 }, { 169158, 28 }, { 169189, 28 }, { 169253, 28 }, { 169259, 18 }, { 169279, 658 }, { 169325, 568 }, { 169349, 28 }, { 169353, 28 }, + { 169378, 28 }, { 169432, 28 }, { 169476, 18 }, { 169476, 18 }, { 169525, 28 }, { 169538, 78 }, { 169555, 28 }, { 169571, 28 }, { 169594, 48 }, { 169687, 28 }, { 169799, 28 }, { 169831, 28 }, { 170042, 28 }, { 170061, 28 }, { 170065, 18 }, { 170128, 68 }, { 170148, 208 }, { 170215, 708 }, { 170256, 608 }, { 170266, 698 }, + { 170275, 78 }, { 170277, 68 }, { 170500, 38 }, { 170516, 38 }, { 170601, 28 }, { 170666, 28 }, { 170668, 48 }, { 170668, 18 }, { 170716, 38 }, { 170728, 38 }, { 170735, 58 }, { 170847, 38 }, { 170852, 98 }, { 170858, 438 }, { 170859, 568 }, { 170956, 568 }, { 170956, 18 }, { 170967, 28 }, { 171005, 28 }, { 171113, 28 }, + { 171279, 28 }, { 171400, 28 }, { 171405, 28 }, { 171448, 18 }, { 171490, 28 }, { 171567, 328 }, { 171590, 28 }, { 171723, 28 }, { 171737, 38 }, { 171958, 28 }, { 171967, 2 } }; + } +} \ No newline at end of file diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixTimerTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixTimerTest.java new file mode 100644 index 000000000..680f0697c --- /dev/null +++ b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixTimerTest.java @@ -0,0 +1,221 @@ +package com.netflix.hystrix.util; + +import static org.junit.Assert.*; + +import java.lang.ref.Reference; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Test; + +import com.netflix.hystrix.util.HystrixTimer.ScheduledExecutor; +import com.netflix.hystrix.util.HystrixTimer.TimerListener; + + +public class HystrixTimerTest { + + @Test + public void testSingleCommandSingleInterval() { + HystrixTimer timer = HystrixTimer.getInstance(); + TestListener l1 = new TestListener(50, "A"); + timer.addTimerListener(l1); + + TestListener l2 = new TestListener(50, "B"); + timer.addTimerListener(l2); + + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + // we should have 7 or more 50ms ticks within 500ms + System.out.println("l1 ticks: " + l1.tickCount.get()); + System.out.println("l2 ticks: " + l2.tickCount.get()); + assertTrue(l1.tickCount.get() > 7); + assertTrue(l2.tickCount.get() > 7); + } + + @Test + public void testSingleCommandMultipleIntervals() { + HystrixTimer timer = HystrixTimer.getInstance(); + TestListener l1 = new TestListener(100, "A"); + timer.addTimerListener(l1); + + TestListener l2 = new TestListener(10, "B"); + timer.addTimerListener(l2); + + TestListener l3 = new TestListener(25, "C"); + timer.addTimerListener(l3); + + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + // we should have 3 or more 100ms ticks within 500ms + System.out.println("l1 ticks: " + l1.tickCount.get()); + assertTrue(l1.tickCount.get() >= 3); + // but it can't be more than 6 + assertTrue(l1.tickCount.get() < 6); + + // we should have 30 or more 10ms ticks within 500ms + System.out.println("l2 ticks: " + l2.tickCount.get()); + assertTrue(l2.tickCount.get() > 30); + assertTrue(l2.tickCount.get() < 550); + + // we should have 15-20 25ms ticks within 500ms + System.out.println("l3 ticks: " + l3.tickCount.get()); + assertTrue(l3.tickCount.get() > 14); + assertTrue(l3.tickCount.get() < 25); + } + + @Test + public void testSingleCommandRemoveListener() { + HystrixTimer timer = HystrixTimer.getInstance(); + TestListener l1 = new TestListener(50, "A"); + timer.addTimerListener(l1); + + TestListener l2 = new TestListener(50, "B"); + Reference l2ref = timer.addTimerListener(l2); + + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + // we should have 7 or more 50ms ticks within 500ms + System.out.println("l1 ticks: " + l1.tickCount.get()); + System.out.println("l2 ticks: " + l2.tickCount.get()); + assertTrue(l1.tickCount.get() > 7); + assertTrue(l2.tickCount.get() > 7); + + // remove l2 + l2ref.clear(); + + // reset counts + l1.tickCount.set(0); + l2.tickCount.set(0); + + // wait for time to pass again + try { + Thread.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + // we should have 7 or more 50ms ticks within 500ms + System.out.println("l1 ticks: " + l1.tickCount.get()); + System.out.println("l2 ticks: " + l2.tickCount.get()); + // l1 should continue ticking + assertTrue(l1.tickCount.get() > 7); + // we should have no ticks on l2 because we removed it + System.out.println("tickCount.get(): " + l2.tickCount.get() + " on l2: " + l2); + assertEquals(0, l2.tickCount.get()); + } + + @Test + public void testReset() { + HystrixTimer timer = HystrixTimer.getInstance(); + TestListener l1 = new TestListener(50, "A"); + timer.addTimerListener(l1); + + ScheduledExecutor ex = timer.executor.get(); + + assertFalse(ex.executor.isShutdown()); + + // perform reset which should shut it down + HystrixTimer.reset(); + + assertTrue(ex.executor.isShutdown()); + assertNull(timer.executor.get()); + + // assert it starts up again on use + TestListener l2 = new TestListener(50, "A"); + timer.addTimerListener(l2); + + ScheduledExecutor ex2 = timer.executor.get(); + + assertFalse(ex2.executor.isShutdown()); + + // reset again to shutdown what we just started + HystrixTimer.reset(); + // try resetting again to make sure it's idempotent (ie. doesn't blow up on an NPE) + HystrixTimer.reset(); + } + + private static class TestListener implements TimerListener { + + private final int interval; + AtomicInteger tickCount = new AtomicInteger(); + + TestListener(int interval, String value) { + this.interval = interval; + } + + @Override + public void tick() { + tickCount.incrementAndGet(); + } + + @Override + public int getIntervalTimeInMilliseconds() { + return interval; + } + + } + + public static void main(String args[]) { + PlayListener l1 = new PlayListener(); + PlayListener l2 = new PlayListener(); + PlayListener l3 = new PlayListener(); + PlayListener l4 = new PlayListener(); + PlayListener l5 = new PlayListener(); + + Reference ref = HystrixTimer.getInstance().addTimerListener(l1); + HystrixTimer.getInstance().addTimerListener(l2); + HystrixTimer.getInstance().addTimerListener(l3); + + HystrixTimer.getInstance().addTimerListener(l4); + + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + ref.clear(); + HystrixTimer.getInstance().addTimerListener(l5); + + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + System.out.println("counter: " + l1.counter); + System.out.println("counter: " + l2.counter); + System.out.println("counter: " + l3.counter); + System.out.println("counter: " + l4.counter); + System.out.println("counter: " + l5.counter); + + } + + public static class PlayListener implements TimerListener { + int counter = 0; + + @Override + public void tick() { + counter++; + } + + @Override + public int getIntervalTimeInMilliseconds() { + return 10; + } + + } + + +} diff --git a/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandCollapserGetValueForKey.java b/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandCollapserGetValueForKey.java index 85f36573e..bddd4246d 100644 --- a/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandCollapserGetValueForKey.java +++ b/hystrix-examples/src/main/java/com/netflix/hystrix/examples/basic/CommandCollapserGetValueForKey.java @@ -98,18 +98,34 @@ public void testCollapser() throws Exception { assertEquals("ValueForKey: 3", f3.get()); assertEquals("ValueForKey: 4", f4.get()); + int numExecuted = HystrixRequestLog.getCurrentRequest().getExecutedCommands().size(); + + System.err.println("num executed: " + numExecuted); + // assert that the batch command 'GetValueForKey' was in fact executed and that it executed only // once or twice (due to non-determinism of scheduler since this example uses the real timer) - if (HystrixRequestLog.getCurrentRequest().getExecutedCommands().size() > 2) { + if (numExecuted > 2) { fail("some of the commands should have been collapsed"); } + + System.err.println("HystrixRequestLog.getCurrentRequest().getExecutedCommands(): " + HystrixRequestLog.getCurrentRequest().getExecutedCommands()); + System.err.println("HystrixRequestLog.getCurrentRequest().getAllExecutedCommands(): " + HystrixRequestLog.getCurrentRequest().getAllExecutedCommands()); + + int numLogs = 0; for (HystrixCommand command : HystrixRequestLog.getCurrentRequest().getExecutedCommands()) { + numLogs++; + // assert the command is the one we're expecting assertEquals("GetValueForKey", command.getCommandKey().name()); + + System.err.println(command.getCommandKey().name() + " => command.getExecutionEvents(): " + command.getExecutionEvents()); + // confirm that it was a COLLAPSED command execution assertTrue(command.getExecutionEvents().contains(HystrixEventType.COLLAPSED)); assertTrue(command.getExecutionEvents().contains(HystrixEventType.SUCCESS)); } + + assertEquals(numExecuted, numLogs); } finally { context.shutdown(); }