Skip to content

Commit

Permalink
Merge branch 'main' into fix_constant_keyword
Browse files Browse the repository at this point in the history
Signed-off-by: kkewwei <kewei.11@bytedance.com>
  • Loading branch information
kkewwei committed Jul 15, 2024
2 parents 1b4b111 + 6b8b3ef commit ccd4794
Show file tree
Hide file tree
Showing 21 changed files with 871 additions and 120 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Workload Management] Add QueryGroup schema ([13669](https://github.com/opensearch-project/OpenSearch/pull/13669))
- Add batching supported processor base type AbstractBatchingProcessor ([#14554](https://github.com/opensearch-project/OpenSearch/pull/14554))
- Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
- Add `strict_allow_templates` dynamic mapping option ([#14555](https://github.com/opensearch-project/OpenSearch/pull/14555))
- Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
- Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
- Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
Expand Down Expand Up @@ -65,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Handle NPE in GetResult if "found" field is missing ([#14552](https://github.com/opensearch-project/OpenSearch/pull/14552))
- Refactoring FilterPath.parse by using an iterative approach ([#14200](https://github.com/opensearch-project/OpenSearch/pull/14200))
- Refactoring Grok.validatePatternBank by using an iterative approach ([#14206](https://github.com/opensearch-project/OpenSearch/pull/14206))
- Update help output for _cat ([#14722](https://github.com/opensearch-project/OpenSearch/pull/14722))
- Fix constant_keyword field type ([#14651](https://github.com/opensearch-project/OpenSearch/pull/14651))

### Security
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,50 +81,52 @@ public void apply(Project project) {
// tests
Configuration testConfig = project.getConfigurations().create("restTestConfig");
project.getConfigurations().create("restTests");

if (BuildParams.isInternal()) {
// core
Dependency restTestdependency = project.getDependencies().project(new HashMap<String, String>() {
{
put("path", ":rest-api-spec");
put("configuration", "restTests");
}
});
testConfig.withDependencies(s -> s.add(restTestdependency));
} else {
Dependency dependency = project.getDependencies().create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
testConfig.withDependencies(s -> s.add(dependency));
}

Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
.register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
task.includeCore.set(extension.restTests.getIncludeCore());
task.coreConfig = testConfig;
task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
if (BuildParams.isInternal()) {
// core
Dependency restTestdependency = project.getDependencies().project(new HashMap<String, String>() {
{
put("path", ":rest-api-spec");
put("configuration", "restTests");
}
});
project.getDependencies().add(task.coreConfig.getName(), restTestdependency);
} else {
Dependency dependency = project.getDependencies()
.create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
project.getDependencies().add(task.coreConfig.getName(), dependency);
}
task.dependsOn(task.coreConfig);
});

// api
Configuration specConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity
project.getConfigurations().create("restSpecs");

if (BuildParams.isInternal()) {
Dependency restSpecDependency = project.getDependencies().project(new HashMap<String, String>() {
{
put("path", ":rest-api-spec");
put("configuration", "restSpecs");
}
});
specConfig.withDependencies(s -> s.add(restSpecDependency));
} else {
Dependency dependency = project.getDependencies().create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
specConfig.withDependencies(s -> s.add(dependency));
}

Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
.register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
task.includeCore.set(extension.restApi.getIncludeCore());
task.dependsOn(copyRestYamlTestTask);
task.coreConfig = specConfig;
task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
if (BuildParams.isInternal()) {
Dependency restSpecDependency = project.getDependencies().project(new HashMap<String, String>() {
{
put("path", ":rest-api-spec");
put("configuration", "restSpecs");
}
});
project.getDependencies().add(task.coreConfig.getName(), restSpecDependency);
} else {
Dependency dependency = project.getDependencies()
.create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
project.getDependencies().add(task.coreConfig.getName(), dependency);
}
task.dependsOn(task.coreConfig);
});

Expand Down
Binary file modified gradle/wrapper/gradle-wrapper.jar
Binary file not shown.
4 changes: 2 additions & 2 deletions gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961
distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
7 changes: 5 additions & 2 deletions gradlew
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#

##############################################################################
#
Expand Down Expand Up @@ -55,7 +57,7 @@
# Darwin, MinGW, and NonStop.
#
# (3) This script is generated from the Groovy template
# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# within the Gradle project.
#
# You can find Gradle at https://github.com/gradle/gradle/.
Expand Down Expand Up @@ -84,7 +86,8 @@ done
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
' "$PWD" ) || exit

# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
Expand Down
2 changes: 2 additions & 0 deletions gradlew.bat
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@rem SPDX-License-Identifier: Apache-2.0
@rem

@if "%DEBUG%"=="" @echo off
@rem ##########################################################################
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
---
"Index documents with setting dynamic parameter to strict_allow_templates in the mapping of the index":
- skip:
version: " - 2.99.99"
reason: "introduced in 3.0.0"

- do:
indices.create:
index: test_1
body:
mappings:
dynamic: strict_allow_templates
dynamic_templates: [
{
strings: {
"match": "stringField*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
object: {
"match": "objectField*",
"match_mapping_type": "object",
"mapping": {
"type": "object",
"properties": {
"bar1": {
"type": "keyword"
},
"bar2": {
"type": "text"
}
}
}
}
},
{
boolean: {
"match": "booleanField*",
"match_mapping_type": "boolean",
"mapping": {
"type": "boolean"
}
}
},
{
double: {
"match": "doubleField*",
"match_mapping_type": "double",
"mapping": {
"type": "double"
}
}
},
{
long: {
"match": "longField*",
"match_mapping_type": "long",
"mapping": {
"type": "long"
}
}
},
{
array: {
"match": "arrayField*",
"mapping": {
"type": "keyword"
}
}
},
{
date: {
"match": "dateField*",
"match_mapping_type": "date",
"mapping": {
"type": "date"
}
}
}
]
properties:
test1:
type: text

- do:
catch: /mapping set to strict_allow_templates, dynamic introduction of \[test2\] within \[\_doc\] is not allowed/
index:
index: test_1
id: 1
body: {
stringField: bar,
objectField: {
bar1: "bar1",
bar2: "bar2"
},
test1: test1,
test2: test2
}

- do:
index:
index: test_1
id: 1
body: {
stringField: bar,
objectField: {
bar1: "bar1",
bar2: "bar2"
},
booleanField: true,
doubleField: 1.0,
longField: 100,
arrayField: ["1","2"],
dateField: "2024-06-25T05:11:51.243Z",
test1: test1
}

- do:
get:
index: test_1
id: 1
- match: { _source: {
stringField: bar,
objectField: {
bar1: "bar1",
bar2: "bar2"
},
booleanField: true,
doubleField: 1.0,
longField: 100,
arrayField: [ "1","2" ],
dateField: "2024-06-25T05:11:51.243Z",
test1: test1
}
}

- do:
indices.get_mapping: {
index: test_1
}

- match: {test_1.mappings.dynamic: strict_allow_templates}
- match: {test_1.mappings.properties.stringField.type: keyword}
- match: {test_1.mappings.properties.objectField.properties.bar1.type: keyword}
- match: {test_1.mappings.properties.objectField.properties.bar2.type: text}
- match: {test_1.mappings.properties.booleanField.type: boolean}
- match: {test_1.mappings.properties.doubleField.type: double}
- match: {test_1.mappings.properties.longField.type: long}
- match: {test_1.mappings.properties.arrayField.type: keyword}
- match: {test_1.mappings.properties.dateField.type: date}
- match: {test_1.mappings.properties.test1.type: text}
Original file line number Diff line number Diff line change
Expand Up @@ -159,3 +159,34 @@ setup:
indices.get_mapping: {}

- match: {test_index1.mappings.properties.text.type: text}

---
"post a mapping with setting dynamic to strict_allow_templates":
- skip:
version: " - 2.99.99"
reason: "introduced in 3.0.0"
- do:
indices.put_mapping:
index: test_index1
body:
dynamic: strict_allow_templates
dynamic_templates: [
{
strings: {
"match": "foo*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
]
properties:
test1:
type: text

- do:
indices.get_mapping: {}

- match: {test_index1.mappings.dynamic: strict_allow_templates}
- match: {test_index1.mappings.properties.test1.type: text}
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* PrimaryShardBatchAllocator is similar to {@link org.opensearch.gateway.PrimaryShardAllocator} only difference is
Expand Down Expand Up @@ -82,6 +84,7 @@ public AllocateUnassignedDecision makeAllocationDecision(ShardRouting unassigned
* @param allocation the allocation state container object
*/
public void allocateUnassignedBatch(List<ShardRouting> shardRoutings, RoutingAllocation allocation) {
logger.trace("Starting shard allocation execution for unassigned primary shards: {}", shardRoutings.size());
HashMap<ShardId, AllocateUnassignedDecision> ineligibleShardAllocationDecisions = new HashMap<>();
List<ShardRouting> eligibleShards = new ArrayList<>();
List<ShardRouting> inEligibleShards = new ArrayList<>();
Expand All @@ -99,13 +102,13 @@ public void allocateUnassignedBatch(List<ShardRouting> shardRoutings, RoutingAll
// only fetch data for eligible shards
final FetchResult<NodeGatewayStartedShardsBatch> shardsState = fetchData(eligibleShards, inEligibleShards, allocation);

Set<ShardRouting> batchShardRoutingSet = new HashSet<>(shardRoutings);
RoutingNodes.UnassignedShards.UnassignedIterator iterator = allocation.routingNodes().unassigned().iterator();
while (iterator.hasNext()) {
ShardRouting unassignedShard = iterator.next();
AllocateUnassignedDecision allocationDecision;

if (shardRoutings.contains(unassignedShard)) {
assert unassignedShard.primary();
if (unassignedShard.primary() && batchShardRoutingSet.contains(unassignedShard)) {
if (ineligibleShardAllocationDecisions.containsKey(unassignedShard.shardId())) {
allocationDecision = ineligibleShardAllocationDecisions.get(unassignedShard.shardId());
} else {
Expand All @@ -115,6 +118,7 @@ public void allocateUnassignedBatch(List<ShardRouting> shardRoutings, RoutingAll
executeDecision(unassignedShard, allocationDecision, allocation, iterator);
}
}
logger.trace("Finished shard allocation execution for unassigned primary shards: {}", shardRoutings.size());
}

/**
Expand Down
Loading

0 comments on commit ccd4794

Please sign in to comment.