Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SQL: Fix esType for DATETIME/DATE and INTERVALS #38179

Merged
merged 6 commits into from
Feb 5, 2019
Merged

SQL: Fix esType for DATETIME/DATE and INTERVALS #38179

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5c7ff99
SQL: Fix esType for DATETIME/DATE and INTERVALS
matriv Feb 1, 2019
508f53f
Address comment
matriv Feb 5, 2019
7dd0a81
Merge remote-tracking branch 'upstream/master' into mt/fix-38051
matriv Feb 5, 2019
ea18004
Fix: use correct field
matriv Feb 5, 2019
3057fd7
Address comment
matriv Feb 5, 2019
4fed2fb
fixup! Address comment
matriv Feb 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ else if (DataTypes.isUnsupported(fa.dataType())) {
// compound fields
else if (allowCompound == false && fa.dataType().isPrimitive() == false) {
named = u.withUnresolvedMessage(
"Cannot use field [" + fa.name() + "] type [" + fa.dataType().esType + "] only its subfields");
"Cannot use field [" + fa.name() + "] type [" + fa.dataType().esSQLType + "] only its subfields");
}
}
return named;
Expand Down Expand Up @@ -1159,4 +1159,4 @@ protected boolean skipResolved() {
return true;
}
}
}
}
4 changes: 2 additions & 2 deletions x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -660,7 +660,7 @@ private static void validateInExpression(LogicalPlan p, Set<Failure> localFailur
for (Expression value : in.list()) {
if (areTypesCompatible(dt, value.dataType()) == false) {
localFailures.add(fail(value, "expected data type [%s], value provided is of type [%s]",
dt.esType, value.dataType().esType));
dt.esSQLType, value.dataType().esSQLType));
return;
}
}
Expand All @@ -681,7 +681,7 @@ private static void validateConditional(LogicalPlan p, Set<Failure> localFailure
} else {
if (areTypesCompatible(dt, child.dataType()) == false) {
localFailures.add(fail(child, "expected data type [%s], value provided is of type [%s]",
dt.esType, child.dataType().esType));
dt.esSQLType, child.dataType().esSQLType));
return;
}
}
Expand Down
6 changes: 3 additions & 3 deletions ...lugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,12 @@ private static void sorting(QueryContainer container, SearchSourceBuilder source

sortBuilder = fieldSort(fa.name())
.missing(as.missing().position())
.unmappedType(fa.dataType().esType);
.unmappedType(fa.dataType().esType());

if (fa.isNested()) {
FieldSortBuilder fieldSort = fieldSort(fa.name())
.missing(as.missing().position())
.unmappedType(fa.dataType().esType);
.unmappedType(fa.dataType().esType());

NestedSortBuilder newSort = new NestedSortBuilder(fa.nestedParent().name());
NestedSortBuilder nestedSort = fieldSort.getNestedSort();
Expand Down Expand Up @@ -175,4 +175,4 @@ private static void disableSource(SearchSourceBuilder builder) {
builder.storedFields(NO_STORED_FIELD);
}
}
}
}
2 changes: 1 addition & 1 deletion ...c/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public String getWriteableName() {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(fieldName);
out.writeOptionalString(dataType == null ? null : dataType.esType);
out.writeOptionalString(dataType == null ? null : dataType.esType());
out.writeBoolean(useDocValue);
out.writeOptionalString(hitName);
}
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ public static TypeResolution typeMustBe(Expression e,
paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT),
acceptedTypesForErrorMsg(acceptedTypes),
Expressions.name(e),
e.dataType().esType));
e.dataType().esSQLType));
}

private static String acceptedTypesForErrorMsg(String... acceptedTypes) {
Expand Down
4 changes: 2 additions & 2 deletions .../sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/TopHits.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ protected TypeResolution resolveType() {
((FieldAttribute) field()).exactAttribute();
} catch (MappingException ex) {
return new TypeResolution(format(null, "[{}] cannot operate on first argument field of data type [{}]",
functionName(), field().dataType().esType));
functionName(), field().dataType().esSQLType));
}

if (orderField() != null) {
Expand All @@ -59,7 +59,7 @@ protected TypeResolution resolveType() {
((FieldAttribute) orderField()).exactAttribute();
} catch (MappingException ex) {
return new TypeResolution(format(null, "[{}] cannot operate on second argument field of data type [{}]",
functionName(), orderField().dataType().esType));
functionName(), orderField().dataType().esSQLType));
}
}
return TypeResolution.TYPE_RESOLVED;
Expand Down
4 changes: 2 additions & 2 deletions ...c/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@

import org.elasticsearch.xpack.sql.expression.Expression;
import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation;
import org.elasticsearch.xpack.sql.tree.Source;
import org.elasticsearch.xpack.sql.tree.NodeInfo;
import org.elasticsearch.xpack.sql.tree.Source;
import org.elasticsearch.xpack.sql.type.DataTypes;

import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
Expand Down Expand Up @@ -36,7 +36,7 @@ protected Sub replaceChildren(Expression newLeft, Expression newRight) {
protected TypeResolution resolveWithIntervals() {
if (right().dataType().isDateBased() && DataTypes.isInterval(left().dataType())) {
return new TypeResolution(format(null, "Cannot subtract a {}[{}] from an interval[{}]; do you mean the reverse?",
right().dataType().esType, right().source().text(), left().source().text()));
right().dataType().esSQLType, right().source().text(), left().source().text()));
}
return TypeResolution.TYPE_RESOLVED;
}
Expand Down
2 changes: 1 addition & 1 deletion ...lugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ private void fillInRows(Map<String, EsField> mapping, String prefix, List<List<?
String name = e.getKey();
if (dt != null) {
// show only fields that exist in ES
rows.add(asList(prefix != null ? prefix + "." + name : name, dt.sqlName(), dt.esType));
rows.add(asList(prefix != null ? prefix + "." + name : name, dt.sqlName(), dt.esType()));
if (field.getProperties().isEmpty() == false) {
String newPrefix = prefix != null ? prefix + "." + name : name;
fillInRows(field.getProperties(), newPrefix, rows);
Expand Down
5 changes: 2 additions & 3 deletions ...in/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.sql.DatabaseMetaData;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -143,7 +142,7 @@ static void fillInRows(String clusterName, String indexName, Map<String, EsField
indexName,
name,
odbcCompatible(type.sqlType.getVendorTypeNumber(), isOdbcClient),
type.esType.toUpperCase(Locale.ROOT),
type.toString(),
type.displaySize,
// TODO: is the buffer_length correct?
type.size,
Expand Down Expand Up @@ -208,4 +207,4 @@ public boolean equals(Object obj) {
&& Objects.equals(pattern, other.pattern)
&& Objects.equals(columnPattern, other.columnPattern);
}
}
}
7 changes: 3 additions & 4 deletions ...ugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,14 @@
import org.elasticsearch.xpack.sql.session.Rows;
import org.elasticsearch.xpack.sql.session.SchemaRowSet;
import org.elasticsearch.xpack.sql.session.SqlSession;
import org.elasticsearch.xpack.sql.tree.Source;
import org.elasticsearch.xpack.sql.tree.NodeInfo;
import org.elasticsearch.xpack.sql.tree.Source;
import org.elasticsearch.xpack.sql.type.DataType;
import org.elasticsearch.xpack.sql.type.DataTypes;

import java.sql.DatabaseMetaData;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.stream.Stream;

import static java.util.Arrays.asList;
Expand Down Expand Up @@ -81,7 +80,7 @@ public final void execute(SqlSession session, ActionListener<SchemaRowSet> liste
List<List<?>> rows = values
// sort by SQL int type (that's what the JDBC/ODBC specs want) followed by name
.sorted(Comparator.comparing((DataType t) -> t.sqlType.getVendorTypeNumber()).thenComparing(DataType::sqlName))
.map(t -> asList(t.esType.toUpperCase(Locale.ROOT),
.map(t -> asList(t.toString(),
t.sqlType.getVendorTypeNumber(),
//https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/column-size?view=sql-server-2017
t.defaultPrecision,
Expand Down Expand Up @@ -132,4 +131,4 @@ public boolean equals(Object obj) {

return type.equals(((SysTypes) obj).type);
}
}
}
4 changes: 2 additions & 2 deletions .../plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,9 @@ static SqlQueryResponse createResponse(SqlQueryRequest request, SchemaRowSet row
List<ColumnInfo> columns = new ArrayList<>(rowSet.columnCount());
for (Schema.Entry entry : rowSet.schema()) {
if (Mode.isDriver(request.mode())) {
columns.add(new ColumnInfo("", entry.name(), entry.type().esType, entry.type().displaySize));
columns.add(new ColumnInfo("", entry.name(), entry.type().esSQLType, entry.type().displaySize));
} else {
columns.add(new ColumnInfo("", entry.name(), entry.type().esType));
columns.add(new ColumnInfo("", entry.name(), entry.type().esSQLType));
}
}
columns = unmodifiableList(columns);
Expand Down
4 changes: 2 additions & 2 deletions x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/TopHitsAgg.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,13 @@ AggregationBuilder toBuilder() {
new FieldSortBuilder(sortField)
.order(sortOrder)
.missing(LAST.position())
.unmappedType(sortFieldDataType.esType));
.unmappedType(sortFieldDataType.esType()));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is there still a sortFieldDataType? I recall discussing about removing it in the date PR.

Copy link
Contributor Author

@matriv matriv Feb 1, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the date PR? You mean in the FIRST/LAST PR? I checked again, but don't get, and sorry maybe I missed something, but we need it for this the unmappedType call.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#37936 (comment)

There's no need for sortFieldDataType - you know it already null (hence the FieldSortBuilder), simply do sortField.dataType()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I forgot why I didn't do it after all:
We need the field name to be extracted from the field, which has some small logic and is something that we do in the QueryTranslator for other cases as well, so I didn't want to pass the field and do the name extraction inside the TopHitsAgg.

}
sortBuilderList.add(
new FieldSortBuilder(fieldName())
.order(sortOrder)
.missing(LAST.position())
.unmappedType(fieldDataType.esType));
.unmappedType(fieldDataType.esType()));

return topHits(id()).docValueField(fieldName(), fieldDataType.format()).sorts(sortBuilderList).size(1);
}
Expand Down
2 changes: 1 addition & 1 deletion ...lugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/TopHitsAggRef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,6 @@ public DataType fieldDataType() {

@Override
public String toString() {
return ">" + name + "[" + fieldDataType.esType + "]";
return ">" + name + "[" + fieldDataType.esSQLType + "]";
}
}
19 changes: 14 additions & 5 deletions x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.sql.type;

import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
import org.elasticsearch.xpack.sql.util.DateUtils;

import java.sql.JDBCType;
Expand Down Expand Up @@ -126,9 +127,9 @@ public enum DataType {


/**
* Elasticsearch type name
* Lowercase type name
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure "Lowercase" is the right description for this field, I mean it is a true statement, but doesn't help much with the purpose of the field.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1.
The purpose of a comment is to explain the non-obvious:

/** an integer */ vs /** array index */
public int i;

*/
public final String esType;
public final String esSQLType;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I may be wrong, but why not esSqlType name?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure


/**
* Compatible JDBC type
Expand Down Expand Up @@ -176,7 +177,7 @@ public enum DataType {

DataType(SQLType sqlType, int size, int defaultPrecision, int displaySize, boolean isInteger,
boolean isRational, boolean defaultDocValues) {
this.esType = name().toLowerCase(Locale.ROOT);
this.esSQLType = name().toLowerCase(Locale.ROOT);
this.sqlType = sqlType;
this.size = size;
this.defaultPrecision = defaultPrecision;
Expand All @@ -186,6 +187,16 @@ public enum DataType {
this.defaultDocValues = defaultDocValues;
}

public String esType() {
if (this == DATE || DataTypes.isInterval(this)) {
throw new SqlIllegalArgumentException("{} doesn't have a corresponding ES data type", this);
}
if (this == DATETIME) {
return "date";
}
return esSQLType;
}

public String sqlName() {
return sqlType.getName();
}
Expand Down Expand Up @@ -228,8 +239,6 @@ public static DataType fromOdbcType(String odbcType) {

/**
* Creates returns DataType enum corresponding to the specified es type
* <p>
* For any dataType DataType.fromTypeName(dataType.esType) == dataType
*/
public static DataType fromTypeName(String esType) {
String uppercase = esType.toUpperCase(Locale.ROOT);
Expand Down
4 changes: 2 additions & 2 deletions x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/Schema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,9 @@ public String toString() {
}
sb.append(names.get(i));
sb.append(":");
sb.append(types.get(i).esType);
sb.append(types.get(i).esSQLType);
}
sb.append("]");
return sb.toString();
}
}
}
4 changes: 2 additions & 2 deletions ...ugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ public static Map<String, Map<String, FieldCapabilities>> fromMappings(EsIndex..
if (entry.getValue().size() > 1) {
for (EsIndex index : indices) {
EsField field = index.mapping().get(fieldName);
UpdateableFieldCapabilities fieldCaps = (UpdateableFieldCapabilities) caps.get(field.getDataType().esType);
UpdateableFieldCapabilities fieldCaps = (UpdateableFieldCapabilities) caps.get(field.getDataType().esSQLType);
fieldCaps.indices.add(index.name());
}
//TODO: what about nonAgg/SearchIndices?
Expand All @@ -171,7 +171,7 @@ private static void addFieldCaps(String parent, EsField field, String indexName,
map = new HashMap<>();
merged.put(fieldName, map);
}
FieldCapabilities caps = map.computeIfAbsent(field.getDataType().esType,
FieldCapabilities caps = map.computeIfAbsent(field.getDataType().esSQLType,
esType -> new UpdateableFieldCapabilities(fieldName, esType,
isSearchable(field.getDataType()),
isAggregatable(field.getDataType())));
Expand Down
16 changes: 8 additions & 8 deletions x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/ParameterTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public class ParameterTests extends ESTestCase {
public void testSingleParameter() {
Expression expression = new SqlParser().createExpression("a = \n?",
Collections.singletonList(
new SqlTypedParamValue(DataType.KEYWORD.esType, "foo")
new SqlTypedParamValue(DataType.KEYWORD.esSQLType, "foo")
));
logger.info(expression);
assertThat(expression, instanceOf(Equals.class));
Expand All @@ -42,10 +42,10 @@ public void testSingleParameter() {

public void testMultipleParameters() {
Expression expression = new SqlParser().createExpression("(? + ? * ?) - ?", Arrays.asList(
new SqlTypedParamValue(DataType.LONG.esType, 1L),
new SqlTypedParamValue(DataType.LONG.esType, 2L),
new SqlTypedParamValue(DataType.LONG.esType, 3L),
new SqlTypedParamValue(DataType.LONG.esType, 4L)
new SqlTypedParamValue(DataType.LONG.esSQLType, 1L),
new SqlTypedParamValue(DataType.LONG.esSQLType, 2L),
new SqlTypedParamValue(DataType.LONG.esSQLType, 3L),
new SqlTypedParamValue(DataType.LONG.esSQLType, 4L)
));
assertThat(expression, instanceOf(Sub.class));
Sub sub = (Sub) expression;
Expand All @@ -62,9 +62,9 @@ public void testMultipleParameters() {
public void testNotEnoughParameters() {
ParsingException ex = expectThrows(ParsingException.class,
() -> new SqlParser().createExpression("(? + ? * ?) - ?", Arrays.asList(
new SqlTypedParamValue(DataType.LONG.esType, 1L),
new SqlTypedParamValue(DataType.LONG.esType, 2L),
new SqlTypedParamValue(DataType.LONG.esType, 3L)
new SqlTypedParamValue(DataType.LONG.esSQLType, 1L),
new SqlTypedParamValue(DataType.LONG.esSQLType, 2L),
new SqlTypedParamValue(DataType.LONG.esSQLType, 3L)
)));
assertThat(ex.getMessage(), containsString("Not enough actual parameters"));
}
Expand Down
6 changes: 3 additions & 3 deletions ...ck/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,9 @@ public void testFunctionWithFunctionWithArg() {
public void testFunctionWithFunctionWithArgAndParams() {
String e = "POWER(?, {fn POWER({fn ABS(?)}, {fN ABS(?)})})";
Function f = (Function) parser.createExpression(e,
asList(new SqlTypedParamValue(DataType.LONG.esType, 1),
new SqlTypedParamValue(DataType.LONG.esType, 1),
new SqlTypedParamValue(DataType.LONG.esType, 1)));
asList(new SqlTypedParamValue(DataType.LONG.esSQLType, 1),
new SqlTypedParamValue(DataType.LONG.esSQLType, 1),
new SqlTypedParamValue(DataType.LONG.esSQLType, 1)));

assertEquals(e, f.sourceText());
assertEquals(2, f.arguments().size());
Expand Down
5 changes: 2 additions & 3 deletions ...plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/LikeEscapingParsingTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,10 @@
import org.elasticsearch.xpack.sql.type.DataType;

import static java.util.Collections.singletonList;
import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;

import static org.elasticsearch.common.logging.LoggerMessageFormat.format;

public class LikeEscapingParsingTests extends ESTestCase {

private final SqlParser parser = new SqlParser();
Expand All @@ -33,7 +32,7 @@ private LikePattern like(String pattern) {
Expression exp = null;
boolean parameterized = randomBoolean();
if (parameterized) {
exp = parser.createExpression("exp LIKE ?", singletonList(new SqlTypedParamValue(DataType.KEYWORD.esType, pattern)));
exp = parser.createExpression("exp LIKE ?", singletonList(new SqlTypedParamValue(DataType.KEYWORD.esSQLType, pattern)));
} else {
exp = parser.createExpression(format(null, "exp LIKE '{}'", pattern));
}
Expand Down
2 changes: 1 addition & 1 deletion ...ql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ public void testSysTablesTypesEnumerationWoString() throws Exception {
}

private SqlTypedParamValue param(Object value) {
return new SqlTypedParamValue(DataTypes.fromJava(value).esType, value);
return new SqlTypedParamValue(DataTypes.fromJava(value).esSQLType, value);
}

private Tuple<Command, SqlSession> sql(String sql, List<SqlTypedParamValue> params) {
Expand Down
Loading