From f7abada1a5e1e79eee10376940187a6f7dc3cd79 Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Mon, 20 Dec 2021 11:25:40 -0800 Subject: [PATCH] [pulsar-broker] load-balancer support disabling max-session for bundle split (#13108) --- conf/broker.conf | 1 + .../java/org/apache/pulsar/broker/ServiceConfiguration.java | 1 + .../pulsar/broker/loadbalance/impl/BundleSplitterTask.java | 3 ++- .../broker/loadbalance/impl/SimpleLoadManagerImpl.java | 3 ++- .../apache/pulsar/broker/loadbalance/LoadBalancerTest.java | 5 +++++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 80eb86e9d9b9a..9b67442663c18 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1074,6 +1074,7 @@ loadBalancerAutoUnloadSplitBundlesEnabled=true loadBalancerNamespaceBundleMaxTopics=1000 # maximum sessions (producers + consumers) in a bundle, otherwise bundle split will be triggered +# (disable threshold check with value -1) loadBalancerNamespaceBundleMaxSessions=1000 # maximum msgRate (in + out) in a bundle, otherwise bundle split will be triggered diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 21c82b6c4e6b4..ba1db65af5e39 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1953,6 +1953,7 @@ public class ServiceConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_LOAD_BALANCER, doc = "maximum sessions (producers + consumers) in a bundle, otherwise bundle split will be triggered" + + "(disable threshold check with value -1)" ) private int loadBalancerNamespaceBundleMaxSessions = 1000; @FieldContext( diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java index fa48618bd205c..751203ca12458 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTask.java @@ -84,7 +84,8 @@ public Set findBundlesToSplit(final LoadData loadData, final PulsarServi totalMessageRate = longTermData.totalMsgRate(); totalMessageThroughput = longTermData.totalMsgThroughput(); } - if (stats.topics > maxBundleTopics || stats.consumerCount + stats.producerCount > maxBundleSessions + if (stats.topics > maxBundleTopics || (maxBundleSessions > 0 && (stats.consumerCount + + stats.producerCount > maxBundleSessions)) || totalMessageRate > maxBundleMsgRate || totalMessageThroughput > maxBundleBandwidth) { final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle); try { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index e1829e68aed0e..017a0401b2397 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -1372,7 +1372,8 @@ public void doNamespaceBundleSplit() throws Exception { double totalBandwidth = stats.msgThroughputIn + stats.msgThroughputOut; boolean needSplit = false; - if (stats.topics > maxBundleTopics || totalSessions > maxBundleSessions || totalMsgRate > maxBundleMsgRate + if (stats.topics > maxBundleTopics || (maxBundleSessions > 0 + && totalSessions > maxBundleSessions) || totalMsgRate > maxBundleMsgRate || totalBandwidth > maxBundleBandwidth) { if (stats.topics <= 1) { log.info("Unable to split hot namespace bundle {} since there is only one topic.", bundleName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java index 5a6b7845fe7f4..a64f28384d3fb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java @@ -662,6 +662,11 @@ public void testNamespaceBundleAutoSplit() throws Exception { isAutoUnooadSplitBundleEnabled, null); verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-10", "0x00000000_0x02000000", isAutoUnooadSplitBundleEnabled, null); + // disable max session + bundleStats.put("pulsar/use/primary-ns-03/0x00000000_0x80000000", + newBundleStats(2, -1, 0, 0, 0, 0, 0)); + verify(namespaceAdmin, times(0)).splitNamespaceBundle("pulsar/use/primary-ns-12", "0x00000000_0x80000000", + isAutoUnooadSplitBundleEnabled, null); } /*