Skip to content

Commit

Permalink
Mappings: Remove type level default analyzers
Browse files Browse the repository at this point in the history 
closes #8874
  • Loading branch information
@rjernst
rjernst committed Jan 27, 2015
1 parent 6f894b1 commit cff0ec3
Show file tree
Hide file tree
Showing 17 changed files with 157 additions and 261 deletions.
54 changes: 35 additions & 19 deletions dev-tools/create-bwc-index.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,16 +131,32 @@ def create_client(http_port, timeout=30):
time.sleep(1)
assert False, 'Timed out waiting for node for %s seconds' % timeout

def generate_index(client):
def generate_index(client, version):
client.indices.delete(index='test', ignore=404)
num_shards = random.randint(1, 10)
num_replicas = random.randint(0, 1)
logging.info('Create single shard test index')

mappings = {}
if not version.startswith('2.'):
# TODO: we need better "before/onOr/after" logic in python

# backcompat test for legacy type level analyzer settings, see #8874
mappings['analyzer_type1'] = {
'analyzer': 'standard',
}
mappings['analyzer_type2'] = {
'index_analyzer': 'standard',
'search_analyzer': 'keyword',
'search_quote_analyzer': 'english',
}

client.indices.create(index='test', body={
'settings': {
'number_of_shards': 1,
'number_of_replicas': 0
}
},
'mappings': mappings
})
health = client.cluster.health(wait_for_status='green', wait_for_relocating_shards=0)
assert health['timed_out'] == False, 'cluster health timed out %s' % health
Expand All @@ -152,32 +168,32 @@ def generate_index(client):

def snapshot_index(client, cfg):
# Add bogus persistent settings to make sure they can be restored
client.cluster.put_settings(body = {
client.cluster.put_settings(body={
'persistent': {
'cluster.routing.allocation.exclude.version_attr' : cfg.version
'cluster.routing.allocation.exclude.version_attr': cfg.version
}
})
client.indices.put_template(name = 'template_' + cfg.version.lower(), order = 0, body = {
"template" : "te*",
"settings" : {
client.indices.put_template(name='template_' + cfg.version.lower(), order=0, body={
"template": "te*",
"settings": {
"number_of_shards" : 1
},
"mappings" : {
"type1" : {
"_source" : { "enabled" : False }
"mappings": {
"type1": {
"_source": { "enabled" : False }
}
},
"aliases" : {
"alias1" : {},
"alias2" : {
"filter" : {
"term" : {"version" : cfg.version }
"aliases": {
"alias1": {},
"alias2": {
"filter": {
"term": {"version" : cfg.version }
},
"routing" : "kimchy"
"routing": "kimchy"
},
"{index}-alias" : {}
"{index}-alias": {}
}
});
})
client.snapshot.create_repository(repository='test_repo', body={
'type': 'fs',
'settings': {
Expand Down Expand Up @@ -243,7 +259,7 @@ def main():
try:
node = start_node(cfg.version, cfg.release_dir, cfg.data_dir, cfg.tcp_port, cfg.http_port)
client = create_client(cfg.http_port)
generate_index(client)
generate_index(client, cfg.version)
if cfg.snapshot_supported:
snapshot_index(client, cfg)
finally:
Expand Down
34 changes: 18 additions & 16 deletions src/main/java/org/elasticsearch/index/engine/Engine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,7 @@ static enum Origin {
static abstract class IndexingOperation implements Operation {

private final DocumentMapper docMapper;
private final Analyzer analyzer;
private final Term uid;
private final ParsedDocument doc;
private long version;
Expand All @@ -296,8 +297,9 @@ static abstract class IndexingOperation implements Operation {
private final long startTime;
private long endTime;

public IndexingOperation(DocumentMapper docMapper, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime, boolean canHaveDuplicates) {
public IndexingOperation(DocumentMapper docMapper, Analyzer analyzer, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime, boolean canHaveDuplicates) {
this.docMapper = docMapper;
this.analyzer = analyzer;
this.uid = uid;
this.doc = doc;
this.version = version;
Expand All @@ -307,8 +309,8 @@ public IndexingOperation(DocumentMapper docMapper, Term uid, ParsedDocument doc,
this.canHaveDuplicates = canHaveDuplicates;
}

public IndexingOperation(DocumentMapper docMapper, Term uid, ParsedDocument doc) {
this(docMapper, uid, doc, Versions.MATCH_ANY, VersionType.INTERNAL, Origin.PRIMARY, System.nanoTime(), true);
public IndexingOperation(DocumentMapper docMapper, Analyzer analyzer, Term uid, ParsedDocument doc) {
this(docMapper, analyzer, uid, doc, Versions.MATCH_ANY, VersionType.INTERNAL, Origin.PRIMARY, System.nanoTime(), true);
}

public DocumentMapper docMapper() {
Expand Down Expand Up @@ -374,7 +376,7 @@ public List<Document> docs() {
}

public Analyzer analyzer() {
return docMapper.mappers().indexAnalyzer();
return this.analyzer;
}

public BytesReference source() {
Expand Down Expand Up @@ -403,17 +405,17 @@ public long endTime() {
static final class Create extends IndexingOperation {
private final boolean autoGeneratedId;

public Create(DocumentMapper docMapper, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime, boolean canHaveDuplicates, boolean autoGeneratedId) {
super(docMapper, uid, doc, version, versionType, origin, startTime, canHaveDuplicates);
public Create(DocumentMapper docMapper, Analyzer analyzer, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime, boolean canHaveDuplicates, boolean autoGeneratedId) {
super(docMapper, analyzer, uid, doc, version, versionType, origin, startTime, canHaveDuplicates);
this.autoGeneratedId = autoGeneratedId;
}

public Create(DocumentMapper docMapper, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime) {
this(docMapper, uid, doc, version, versionType, origin, startTime, true, false);
public Create(DocumentMapper docMapper, Analyzer analyzer, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime) {
this(docMapper, analyzer, uid, doc, version, versionType, origin, startTime, true, false);
}

public Create(DocumentMapper docMapper, Term uid, ParsedDocument doc) {
super(docMapper, uid, doc);
public Create(DocumentMapper docMapper,Analyzer analyzer, Term uid, ParsedDocument doc) {
super(docMapper, analyzer, uid, doc);
autoGeneratedId = false;
}

Expand All @@ -431,16 +433,16 @@ public boolean autoGeneratedId() {
static final class Index extends IndexingOperation {
private boolean created;

public Index(DocumentMapper docMapper, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime, boolean canHaveDuplicates) {
super(docMapper, uid, doc, version, versionType, origin, startTime, canHaveDuplicates);
public Index(DocumentMapper docMapper, Analyzer analyzer, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime, boolean canHaveDuplicates) {
super(docMapper, analyzer, uid, doc, version, versionType, origin, startTime, canHaveDuplicates);
}

public Index(DocumentMapper docMapper, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime) {
super(docMapper, uid, doc, version, versionType, origin, startTime, true);
public Index(DocumentMapper docMapper, Analyzer analyzer, Term uid, ParsedDocument doc, long version, VersionType versionType, Origin origin, long startTime) {
super(docMapper, analyzer, uid, doc, version, versionType, origin, startTime, true);
}

public Index(DocumentMapper docMapper, Term uid, ParsedDocument doc) {
super(docMapper, uid, doc);
public Index(DocumentMapper docMapper, Analyzer analyzer, Term uid, ParsedDocument doc) {
super(docMapper, analyzer, uid, doc);
}

@Override
Expand Down
7 changes: 5 additions & 2 deletions src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.google.common.collect.ForwardingSet;
import com.google.common.collect.Maps;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.analysis.FieldNameAnalyzer;

import java.util.Collection;
Expand All @@ -42,8 +43,10 @@ public final class DocumentFieldMappers extends ForwardingSet<FieldMapper<?>> {
private final FieldNameAnalyzer searchAnalyzer;
private final FieldNameAnalyzer searchQuoteAnalyzer;

public DocumentFieldMappers(DocumentMapper docMapper) {
this(new FieldMappersLookup(), new FieldNameAnalyzer(docMapper.indexAnalyzer()), new FieldNameAnalyzer(docMapper.searchAnalyzer()), new FieldNameAnalyzer(docMapper.searchQuotedAnalyzer()));
public DocumentFieldMappers(AnalysisService analysisService) {
this(new FieldMappersLookup(), new FieldNameAnalyzer(analysisService.defaultIndexAnalyzer()),
new FieldNameAnalyzer(analysisService.defaultSearchAnalyzer()),
new FieldNameAnalyzer(analysisService.defaultSearchQuoteAnalyzer()));
}

private DocumentFieldMappers(FieldMappersLookup fieldMappers, FieldNameAnalyzer indexAnalyzer, FieldNameAnalyzer searchAnalyzer, FieldNameAnalyzer searchQuoteAnalyzer) {
Expand Down
80 changes: 2 additions & 78 deletions src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,12 +166,6 @@ public static class Builder {

private Map<Class<? extends RootMapper>, RootMapper> rootMappers = new LinkedHashMap<>();

private NamedAnalyzer indexAnalyzer;

private NamedAnalyzer searchAnalyzer;

private NamedAnalyzer searchQuoteAnalyzer;

private List<SourceTransform> sourceTransforms;

private final String index;
Expand Down Expand Up @@ -228,36 +222,6 @@ public Builder put(RootMapper.Builder mapper) {
return this;
}

public Builder indexAnalyzer(NamedAnalyzer indexAnalyzer) {
this.indexAnalyzer = indexAnalyzer;
return this;
}

public boolean hasIndexAnalyzer() {
return indexAnalyzer != null;
}

public Builder searchAnalyzer(NamedAnalyzer searchAnalyzer) {
this.searchAnalyzer = searchAnalyzer;
if (this.searchQuoteAnalyzer == null) {
this.searchQuoteAnalyzer = searchAnalyzer;
}
return this;
}

public Builder searchQuoteAnalyzer(NamedAnalyzer searchQuoteAnalyzer) {
this.searchQuoteAnalyzer = searchQuoteAnalyzer;
return this;
}

public boolean hasSearchAnalyzer() {
return searchAnalyzer != null;
}

public boolean hasSearchQuoteAnalyzer() {
return searchQuoteAnalyzer != null;
}

public Builder transform(ScriptService scriptService, String script, ScriptType scriptType, String language, Map<String, Object> parameters) {
if (sourceTransforms == null) {
sourceTransforms = new ArrayList<>();
Expand All @@ -268,8 +232,7 @@ public Builder transform(ScriptService scriptService, String script, ScriptType

public DocumentMapper build(DocumentMapperParser docMapperParser) {
Preconditions.checkNotNull(rootObjectMapper, "Mapper builder must have the root object mapper set");
return new DocumentMapper(index, indexSettings, docMapperParser, rootObjectMapper, meta,
indexAnalyzer, searchAnalyzer, searchQuoteAnalyzer, rootMappers, sourceTransforms);
return new DocumentMapper(index, indexSettings, docMapperParser, rootObjectMapper, meta, rootMappers, sourceTransforms);
}
}

Expand Down Expand Up @@ -300,11 +263,6 @@ protected ParseContext.InternalParseContext initialValue() {
private final RootMapper[] rootMappersOrdered;
private final RootMapper[] rootMappersNotIncludedInObject;

private final NamedAnalyzer indexAnalyzer;

private final NamedAnalyzer searchAnalyzer;
private final NamedAnalyzer searchQuoteAnalyzer;

private volatile DocumentFieldMappers fieldMappers;

private volatile ImmutableMap<String, ObjectMapper> objectMappers = ImmutableMap.of();
Expand All @@ -324,7 +282,6 @@ protected ParseContext.InternalParseContext initialValue() {
public DocumentMapper(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser,
RootObjectMapper rootObjectMapper,
ImmutableMap<String, Object> meta,
NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer,
Map<Class<? extends RootMapper>, RootMapper> rootMappers, List<SourceTransform> sourceTransforms) {
this.index = index;
this.indexSettings = indexSettings;
Expand All @@ -345,10 +302,6 @@ public DocumentMapper(String index, @Nullable Settings indexSettings, DocumentMa
}
this.rootMappersNotIncludedInObject = rootMappersNotIncludedInObjectLst.toArray(new RootMapper[rootMappersNotIncludedInObjectLst.size()]);

this.indexAnalyzer = indexAnalyzer;
this.searchAnalyzer = searchAnalyzer;
this.searchQuoteAnalyzer = searchQuoteAnalyzer != null ? searchQuoteAnalyzer : searchAnalyzer;

this.typeFilter = typeMapper().termFilter(type, null);

if (rootMapper(ParentFieldMapper.class).active()) {
Expand All @@ -370,7 +323,7 @@ public DocumentMapper(String index, @Nullable Settings indexSettings, DocumentMa
// now traverse and get all the statically defined ones
rootObjectMapper.traverse(fieldMappersAgg);

this.fieldMappers = new DocumentFieldMappers(this).copyAndAllAll(fieldMappersAgg.mappers);
this.fieldMappers = new DocumentFieldMappers(docMapperParser.analysisService).copyAndAllAll(fieldMappersAgg.mappers);

final Map<String, ObjectMapper> objectMappers = Maps.newHashMap();
rootObjectMapper.traverse(new ObjectMapperListener() {
Expand Down Expand Up @@ -470,18 +423,6 @@ public BoostFieldMapper boostFieldMapper() {
return rootMapper(BoostFieldMapper.class);
}

public Analyzer indexAnalyzer() {
return this.indexAnalyzer;
}

public Analyzer searchAnalyzer() {
return this.searchAnalyzer;
}

public Analyzer searchQuotedAnalyzer() {
return this.searchQuoteAnalyzer;
}

public Filter typeFilter() {
return this.typeFilter;
}
Expand Down Expand Up @@ -765,23 +706,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
rootObjectMapper.toXContent(builder, params, new ToXContent() {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (indexAnalyzer != null && searchAnalyzer != null && indexAnalyzer.name().equals(searchAnalyzer.name()) && !indexAnalyzer.name().startsWith("_")) {
if (!indexAnalyzer.name().equals("default")) {
// same analyzers, output it once
builder.field("analyzer", indexAnalyzer.name());
}
} else {
if (indexAnalyzer != null && !indexAnalyzer.name().startsWith("_")) {
if (!indexAnalyzer.name().equals("default")) {
builder.field("index_analyzer", indexAnalyzer.name());
}
}
if (searchAnalyzer != null && !searchAnalyzer.name().startsWith("_")) {
if (!searchAnalyzer.name().equals("default")) {
builder.field("search_analyzer", searchAnalyzer.name());
}
}
}
if (sourceTransforms != null) {
if (sourceTransforms.size() == 1) {
builder.field("transform");
Expand Down
Loading

0 comments on commit cff0ec3

Please sign in to comment.