Skip to content

Commit

Permalink
Add static node settings to set default values for max merged segment
Browse files Browse the repository at this point in the history 
sizes
  • Loading branch information
@tlrx
tlrx committed Nov 15, 2023
1 parent aaa6e78 commit a4f1cdc
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 11 deletions.
5 changes: 4 additions & 1 deletion server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexingPressure;
import org.elasticsearch.index.MergePolicyConfig;
import org.elasticsearch.indices.IndexingMemoryController;
import org.elasticsearch.indices.IndicesQueryCache;
import org.elasticsearch.indices.IndicesRequestCache;
Expand Down Expand Up @@ -577,6 +578,8 @@ public void apply(Settings value, Settings current, Settings previous) {
IndicesClusterStateService.SHARD_LOCK_RETRY_TIMEOUT_SETTING,
IngestSettings.GROK_WATCHDOG_INTERVAL,
IngestSettings.GROK_WATCHDOG_MAX_EXECUTION_TIME,
TDigestExecutionHint.SETTING
TDigestExecutionHint.SETTING,
MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT_SETTING,
MergePolicyConfig.DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT_SETTING
).filter(Objects::nonNull).collect(Collectors.toSet());
}
24 changes: 20 additions & 4 deletions server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,17 +111,33 @@ public final class MergePolicyConfig {
private final Logger logger;
private final boolean mergesEnabled;
private volatile Type mergePolicyType;
private final ByteSizeValue defaultMaxMergedSegment;
private final ByteSizeValue defaultMaxTimeBasedMergedSegment;

public static final double DEFAULT_EXPUNGE_DELETES_ALLOWED = 10d;
public static final ByteSizeValue DEFAULT_FLOOR_SEGMENT = new ByteSizeValue(2, ByteSizeUnit.MB);
public static final int DEFAULT_MAX_MERGE_AT_ONCE = 10;
public static final ByteSizeValue DEFAULT_MAX_MERGED_SEGMENT = new ByteSizeValue(5, ByteSizeUnit.GB);
public static final Setting<ByteSizeValue> DEFAULT_MAX_MERGED_SEGMENT_SETTING = Setting.byteSizeSetting(
"merge.policy.default.max_merged_segment",
DEFAULT_MAX_MERGED_SEGMENT,
ByteSizeValue.ofBytes(1L),
ByteSizeValue.ofBytes(Long.MAX_VALUE),
Setting.Property.NodeScope
);
/**
* Time-based data generally gets rolled over, so there is not much value in enforcing a maximum segment size, which has the side effect
* of merging fewer segments together than the merge factor, which in-turn increases write amplification. So we set an arbitrarily high
* roof that serves as a protection that we expect to never hit.
*/
public static final ByteSizeValue DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT = new ByteSizeValue(100, ByteSizeUnit.GB);
public static final Setting<ByteSizeValue> DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT_SETTING = Setting.byteSizeSetting(
"merge.policy.default.max_time_based_merged_segment",
DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT,
ByteSizeValue.ofBytes(1L),
ByteSizeValue.ofBytes(Long.MAX_VALUE),
Setting.Property.NodeScope
);
public static final double DEFAULT_SEGMENTS_PER_TIER = 10.0d;
/**
* A default value for {@link LogByteSizeMergePolicy}'s merge factor: 32. This default value differs from the Lucene default of 10 in
Expand Down Expand Up @@ -262,8 +278,8 @@ MergePolicy getMergePolicy(MergePolicyConfig config, boolean isTimeBasedIndex) {
double forceMergeDeletesPctAllowed = indexSettings.getValue(INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING); // percentage
ByteSizeValue floorSegment = indexSettings.getValue(INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING);
int maxMergeAtOnce = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING);
// TODO is this really a good default number for max_merge_segment, what happens for large indices,
// won't they end up with many segments?
this.defaultMaxMergedSegment = DEFAULT_MAX_MERGED_SEGMENT_SETTING.get(indexSettings.getNodeSettings());
this.defaultMaxTimeBasedMergedSegment = DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT_SETTING.get(indexSettings.getNodeSettings());
ByteSizeValue maxMergedSegment = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING);
double segmentsPerTier = indexSettings.getValue(INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING);
int mergeFactor = indexSettings.getValue(INDEX_MERGE_POLICY_MERGE_FACTOR_SETTING);
Expand Down Expand Up @@ -315,8 +331,8 @@ void setMergeFactor(int mergeFactor) {
void setMaxMergedSegment(ByteSizeValue maxMergedSegment) {
// We use 0 as a placeholder for "unset".
if (maxMergedSegment.getBytes() == 0) {
tieredMergePolicy.setMaxMergedSegmentMB(DEFAULT_MAX_MERGED_SEGMENT.getMbFrac());
timeBasedMergePolicy.setMaxMergeMB(DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT.getMbFrac());
tieredMergePolicy.setMaxMergedSegmentMB(defaultMaxMergedSegment.getMbFrac());
timeBasedMergePolicy.setMaxMergeMB(defaultMaxTimeBasedMergedSegment.getMbFrac());
} else {
tieredMergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
timeBasedMergePolicy.setMaxMergeMB(maxMergedSegment.getMbFrac());
Expand Down
58 changes: 52 additions & 6 deletions server/src/test/java/org/elasticsearch/index/MergePolicyConfigTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,11 @@ private void assertCompoundThreshold(Settings settings, double noCFSRatio, ByteS
}

private static IndexSettings indexSettings(Settings settings) {
return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY);
return indexSettings(settings, Settings.EMPTY);
}

private static IndexSettings indexSettings(Settings indexSettings, Settings nodeSettings) {
return new IndexSettings(newIndexMeta("test", indexSettings), nodeSettings);
}

public void testNoMerges() {
Expand Down Expand Up @@ -118,7 +122,7 @@ public void testUpdateSettings() throws IOException {
assertThat(indexSettings.getMergePolicy(randomBoolean()), Matchers.instanceOf(LogByteSizeMergePolicy.class));
}

public void testTieredMergePolicySettingsUpdate() throws IOException {
public void testTieredMergePolicySettingsUpdate() {
IndexSettings indexSettings = indexSettings(Settings.EMPTY);
assertEquals(
((TieredMergePolicy) indexSettings.getMergePolicy(false)).getForceMergeDeletesPctAllowed(),
Expand Down Expand Up @@ -353,10 +357,6 @@ public Settings build(boolean value) {
return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
}

private Settings build(ByteSizeValue value) {
return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
}

public void testCompoundFileConfiguredByByteSize() throws IOException {
for (boolean isTimeSeriesIndex : new boolean[] { false, true }) {
try (Directory dir = newDirectory()) {
Expand Down Expand Up @@ -394,4 +394,50 @@ public void testCompoundFileConfiguredByByteSize() throws IOException {
}
}
}

public void testDefaultMaxMergedSegment() {
var indexSettings = indexSettings(Settings.EMPTY);
{
TieredMergePolicy tieredPolicy = (TieredMergePolicy) new MergePolicyConfig(logger, indexSettings).getMergePolicy(false);
assertEquals(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), tieredPolicy.getMaxMergedSegmentMB(), 0.0d);
}
{
LogByteSizeMergePolicy timePolicy = (LogByteSizeMergePolicy) new MergePolicyConfig(logger, indexSettings).getMergePolicy(true);
assertEquals(MergePolicyConfig.DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT.getMbFrac(), timePolicy.getMaxMergeMB(), 0.0d);
}
}

public void testDefaultMaxMergedSegmentWithNodeOverrides() {
var maxMergedSegmentSize = ByteSizeValue.ofBytes(randomLongBetween(1L, Long.MAX_VALUE));
{
var indexSettings = indexSettings(
Settings.EMPTY,
Settings.builder().put(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT_SETTING.getKey(), maxMergedSegmentSize).build()
);
TieredMergePolicy tieredPolicy = (TieredMergePolicy) new MergePolicyConfig(logger, indexSettings).getMergePolicy(false);
assertEquals(
maxMergedSegmentSize.getBytes() == 0L
? MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac()
: maxMergedSegmentSize.getMbFrac(),
tieredPolicy.getMaxMergedSegmentMB(),
0.0d
);
}
{
var indexSettings = indexSettings(
Settings.EMPTY,
Settings.builder()
.put(MergePolicyConfig.DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT_SETTING.getKey(), maxMergedSegmentSize)
.build()
);
LogByteSizeMergePolicy timePolicy = (LogByteSizeMergePolicy) new MergePolicyConfig(logger, indexSettings).getMergePolicy(true);
assertEquals(
maxMergedSegmentSize.getBytes() == 0L
? MergePolicyConfig.DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT.getMbFrac()
: maxMergedSegmentSize.getMbFrac(),
timePolicy.getMaxMergeMB(),
0.0d
);
}
}
}

0 comments on commit a4f1cdc

Please sign in to comment.