From 6d9d5e397b9f2b008a6023dd2d65aa071efbdba2 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Mon, 17 Dec 2018 18:58:06 +0200 Subject: [PATCH] SQL: Fix translation of LIKE/RLIKE keywords (#36672) * SQL: Fix translation of LIKE/RLIKE keywords Refactor Like/RLike functions to simplify internals and improve query translation when chained or within a script context. Fix #36039 Fix #36584 --- .../sql/qa/src/main/resources/agg.sql-spec | 2 + .../qa/src/main/resources/datetime.sql-spec | 5 +- .../sql/qa/src/main/resources/filter.sql-spec | 2 + .../whitelist/InternalSqlScriptUtils.java | 1 + .../sql/expression/predicate/regex/Like.java | 20 ++--- .../predicate/regex/LikePattern.java | 24 +----- .../sql/expression/predicate/regex/RLike.java | 21 ++--- .../predicate/regex/RegexMatch.java | 29 +++++-- .../expression/predicate/regex/RegexPipe.java | 34 -------- .../predicate/regex/RegexProcessor.java | 68 +++++++-------- .../xpack/sql/parser/ExpressionBuilder.java | 4 +- .../xpack/sql/planner/QueryTranslator.java | 83 +++++++++++++++---- .../xpack/sql/optimizer/OptimizerTests.java | 6 +- .../xpack/sql/tree/NodeSubclassTests.java | 13 ++- 14 files changed, 158 insertions(+), 154 deletions(-) delete mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexPipe.java diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec index 9adbe79edc685..149e23f771349 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec @@ -280,6 +280,8 @@ aggMaxWithAlias SELECT gender g, MAX(emp_no) m FROM "test_emp" GROUP BY g ORDER BY gender; aggMaxOnDate SELECT gender, MAX(birth_date) m FROM "test_emp" GROUP BY gender ORDER BY gender; +aggAvgAndMaxWithLikeFilter +SELECT CAST(AVG(salary) AS LONG) AS avg, CAST(SUM(salary) AS LONG) AS s FROM "test_emp" WHERE first_name LIKE 'G%'; // Conditional MAX aggMaxWithHaving diff --git a/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec index 0f8a16b9e7bb4..4b12d2de58fc7 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec @@ -119,7 +119,10 @@ SELECT DAY_OF_WEEK(birth_date) day, COUNT(*) c FROM test_emp WHERE DAY_OF_WEEK(b currentTimestampYear SELECT YEAR(CURRENT_TIMESTAMP()) AS result; -currentTimestampMonth +// +// H2 uses the local timezone instead of the specified one +// +currentTimestampMonth-Ignore SELECT MONTH(CURRENT_TIMESTAMP()) AS result; currentTimestampHour-Ignore diff --git a/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec index cfbff2ada573e..af81b060ebd3f 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec @@ -49,6 +49,8 @@ whereFieldWithNotEqualsOnString SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND gender <> 'M'; whereFieldWithLikeMatch SELECT last_name l FROM "test_emp" WHERE emp_no < 10003 AND last_name LIKE 'K%'; +whereFieldWithNotLikeMatch +SELECT last_name l FROM "test_emp" WHERE emp_no < 10020 AND first_name NOT LIKE 'Ma%'; whereFieldWithOrderNot SELECT last_name l FROM "test_emp" WHERE NOT emp_no < 10003 ORDER BY emp_no LIMIT 5; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java index cdc773a91af7a..a67da8d6efd0b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java @@ -165,6 +165,7 @@ public static Object nullif(Object left, Object right) { // Regex // public static Boolean regex(String value, String pattern) { + // TODO: this needs to be improved to avoid creating the pattern on every call return RegexOperation.match(value, pattern); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java index a5c8028f6709a..9dc3c69fd2971 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/Like.java @@ -11,26 +11,24 @@ public class Like extends RegexMatch { - public Like(Location location, Expression left, LikePattern right) { - super(location, left, right); - } + private final LikePattern pattern; - @Override - protected NodeInfo info() { - return NodeInfo.create(this, Like::new, left(), pattern()); + public Like(Location location, Expression left, LikePattern pattern) { + super(location, left, pattern.asJavaRegex()); + this.pattern = pattern; } public LikePattern pattern() { - return (LikePattern) right(); + return pattern; } @Override - protected Like replaceChildren(Expression newLeft, Expression newRight) { - return new Like(location(), newLeft, (LikePattern) newRight); + protected NodeInfo info() { + return NodeInfo.create(this, Like::new, field(), pattern); } @Override - protected String asString(Expression pattern) { - return ((LikePattern) pattern).asJavaRegex(); + protected Like replaceChild(Expression newLeft) { + return new Like(location(), newLeft, pattern); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/LikePattern.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/LikePattern.java index bde8129f8e72a..d07df617df9f0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/LikePattern.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/LikePattern.java @@ -5,10 +5,6 @@ */ package org.elasticsearch.xpack.sql.expression.predicate.regex; -import org.elasticsearch.xpack.sql.expression.LeafExpression; -import org.elasticsearch.xpack.sql.tree.Location; -import org.elasticsearch.xpack.sql.tree.NodeInfo; -import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.util.StringUtils; import java.util.Objects; @@ -21,7 +17,7 @@ * * To prevent conflicts with ES, the string and char must be validated to not contain '*'. */ -public class LikePattern extends LeafExpression { +public class LikePattern { private final String pattern; private final char escape; @@ -30,8 +26,7 @@ public class LikePattern extends LeafExpression { private final String wildcard; private final String indexNameWildcard; - public LikePattern(Location location, String pattern, char escape) { - super(location); + public LikePattern(String pattern, char escape) { this.pattern = pattern; this.escape = escape; // early initialization to force string validation @@ -40,11 +35,6 @@ public LikePattern(Location location, String pattern, char escape) { this.indexNameWildcard = StringUtils.likeToIndexWildcard(pattern, escape); } - @Override - protected NodeInfo info() { - return NodeInfo.create(this, LikePattern::new, pattern, escape); - } - public String pattern() { return pattern; } @@ -74,16 +64,6 @@ public String asIndexNameWildcard() { return indexNameWildcard; } - @Override - public boolean nullable() { - return false; - } - - @Override - public DataType dataType() { - return DataType.KEYWORD; - } - @Override public int hashCode() { return Objects.hash(pattern, escape); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java index 346c3062bfaa8..a09586fd35fb4 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RLike.java @@ -6,28 +6,29 @@ package org.elasticsearch.xpack.sql.expression.predicate.regex; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.Literal; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.tree.NodeInfo; public class RLike extends RegexMatch { - public RLike(Location location, Expression left, Literal right) { - super(location, left, right); + private final String pattern; + + public RLike(Location location, Expression left, String pattern) { + super(location, left, pattern); + this.pattern = pattern; } - @Override - protected NodeInfo info() { - return NodeInfo.create(this, RLike::new, left(), (Literal) right()); + public String pattern() { + return pattern; } @Override - protected RLike replaceChildren(Expression newLeft, Expression newRight) { - return new RLike(location(), newLeft, (Literal) newRight); + protected NodeInfo info() { + return NodeInfo.create(this, RLike::new, field(), pattern); } @Override - protected String asString(Expression pattern) { - return pattern.fold().toString(); + protected RLike replaceChild(Expression newChild) { + return new RLike(location(), newChild, pattern); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java index e1e410064924c..f9390fdfa4514 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexMatch.java @@ -7,15 +7,19 @@ package org.elasticsearch.xpack.sql.expression.predicate.regex; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.predicate.BinaryPredicate; +import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexProcessor.RegexOperation; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; -public abstract class RegexMatch extends BinaryPredicate { +public abstract class RegexMatch extends UnaryScalarFunction { - protected RegexMatch(Location location, Expression value, Expression pattern) { - super(location, value, pattern, RegexOperation.INSTANCE); + private final String pattern; + + protected RegexMatch(Location location, Expression value, String pattern) { + super(location, value); + this.pattern = pattern; } @Override @@ -23,18 +27,25 @@ public DataType dataType() { return DataType.BOOLEAN; } + @Override + public boolean nullable() { + return field().nullable() && pattern != null; + } + @Override public boolean foldable() { // right() is not directly foldable in any context but Like can fold it. - return left().foldable(); + return field().foldable(); } @Override public Boolean fold() { - Object val = left().fold(); - val = val != null ? val.toString() : val; - return function().apply((String) val, asString(right())); + Object val = field().fold(); + return RegexOperation.match(val, pattern); } - protected abstract String asString(Expression pattern); + @Override + protected Processor makeProcessor() { + return new RegexProcessor(pattern); + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexPipe.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexPipe.java deleted file mode 100644 index 7ce8b2b0fc9b4..0000000000000 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexPipe.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.expression.predicate.regex; - -import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.gen.pipeline.BinaryPipe; -import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe; -import org.elasticsearch.xpack.sql.tree.Location; -import org.elasticsearch.xpack.sql.tree.NodeInfo; - -public class RegexPipe extends BinaryPipe { - - public RegexPipe(Location location, Expression expression, Pipe left, Pipe right) { - super(location, expression, left, right); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, RegexPipe::new, expression(), left(), right()); - } - - @Override - protected BinaryPipe replaceChildren(Pipe left, Pipe right) { - return new RegexPipe(location(), expression(), left, right); - } - - @Override - public RegexProcessor asProcessor() { - return new RegexProcessor(left().asProcessor(), right().asProcessor()); - } -} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexProcessor.java index 16f6f0a694966..7f9a2ed76235a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexProcessor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/regex/RegexProcessor.java @@ -7,79 +7,71 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; -import org.elasticsearch.xpack.sql.expression.gen.processor.BinaryProcessor; import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; -import org.elasticsearch.xpack.sql.expression.predicate.PredicateBiFunction; import java.io.IOException; import java.util.Objects; import java.util.regex.Pattern; -public class RegexProcessor extends BinaryProcessor { +public class RegexProcessor implements Processor { - public static class RegexOperation implements PredicateBiFunction { + public static class RegexOperation { - public static final RegexOperation INSTANCE = new RegexOperation(); + public static Boolean match(Object value, Pattern pattern) { + if (pattern == null) { + return Boolean.TRUE; + } - @Override - public String name() { - return symbol(); - } + if (value == null) { + return null; + } - @Override - public String symbol() { - return "REGEX"; + return pattern.matcher(value.toString()).matches(); } - @Override - public Boolean doApply(String value, String pattern) { - return match(value, pattern); - } + public static Boolean match(Object value, String pattern) { + if (pattern == null) { + return Boolean.TRUE; + } - public static Boolean match(Object value, Object pattern) { - if (value == null || pattern == null) { + if (value == null) { return null; } - Pattern p = Pattern.compile(pattern.toString()); - return p.matcher(value.toString()).matches(); + return Pattern.compile(pattern).matcher(value.toString()).matches(); } } public static final String NAME = "rgx"; - public RegexProcessor(Processor value, Processor pattern) { - super(value, pattern); - } + private Pattern pattern; - public RegexProcessor(StreamInput in) throws IOException { - super(in); + public RegexProcessor(String pattern) { + this.pattern = pattern != null ? Pattern.compile(pattern) : null; } @Override - protected Boolean doProcess(Object value, Object pattern) { - return RegexOperation.match(value, pattern); + public String getWriteableName() { + return NAME; } - @Override - protected void checkParameter(Object param) { - if (!(param instanceof String || param instanceof Character)) { - throw new SqlIllegalArgumentException("A string/char is required; received [{}]", param); - } + public RegexProcessor(StreamInput in) throws IOException { + this(in.readOptionalString()); } @Override - public String getWriteableName() { - return NAME; + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(pattern != null ? pattern.toString() : null); } @Override - protected void doWrite(StreamOutput out) throws IOException {} + public Object process(Object input) { + return RegexOperation.match(input, pattern); + } @Override public int hashCode() { - return Objects.hash(left(), right()); + return Objects.hash(pattern); } @Override @@ -93,6 +85,6 @@ public boolean equals(Object obj) { } RegexProcessor other = (RegexProcessor) obj; - return Objects.equals(left(), other.left()) && Objects.equals(right(), other.right()); + return Objects.equals(pattern, other.pattern); } } \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index cd1cb189b6aa2..f7d659a2933da 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -232,7 +232,7 @@ public Expression visitPredicated(PredicatedContext ctx) { e = new Like(loc, exp, visitPattern(pCtx.pattern())); break; case SqlBaseParser.RLIKE: - e = new RLike(loc, exp, new Literal(source(pCtx.regex), string(pCtx.regex), DataType.KEYWORD)); + e = new RLike(loc, exp, string(pCtx.regex)); break; case SqlBaseParser.NULL: // shortcut to avoid double negation later on (since there's no IsNull (missing in ES is a negated exists)) @@ -301,7 +301,7 @@ public LikePattern visitPattern(PatternContext ctx) { } } - return new LikePattern(source(ctx), pattern, escape); + return new LikePattern(pattern, escape); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java index a757bde89e857..af180aae90bdc 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeHistogramFunction; +import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.sql.expression.literal.Intervals; import org.elasticsearch.xpack.sql.expression.predicate.Range; import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MatchQueryPredicate; @@ -103,7 +104,6 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.sql.expression.Foldables.doubleValuesOf; -import static org.elasticsearch.xpack.sql.expression.Foldables.stringValueOf; import static org.elasticsearch.xpack.sql.expression.Foldables.valueOf; final class QueryTranslator { @@ -121,7 +121,8 @@ private QueryTranslator(){} new Likes(), new StringQueries(), new Matches(), - new MultiMatches() + new MultiMatches(), + new Scalars() ); private static final List> AGG_TRANSLATORS = Arrays.asList( @@ -447,13 +448,13 @@ protected QueryTranslation asQuery(RegexMatch e, boolean onAggs) { boolean inexact = true; String target = null; - if (e.left() instanceof FieldAttribute) { - FieldAttribute fa = (FieldAttribute) e.left(); + if (e.field() instanceof FieldAttribute) { + FieldAttribute fa = (FieldAttribute) e.field(); inexact = fa.isInexact(); target = nameOf(inexact ? fa : fa.exactAttribute()); } else { throw new SqlIllegalArgumentException("Scalar function ({}) not allowed (yet) as arguments for LIKE", - Expressions.name(e.left())); + Expressions.name(e.field())); } if (e instanceof Like) { @@ -462,21 +463,21 @@ protected QueryTranslation asQuery(RegexMatch e, boolean onAggs) { q = new QueryStringQuery(e.location(), p.asLuceneWildcard(), target); } else { - q = new WildcardQuery(e.location(), nameOf(e.left()), p.asLuceneWildcard()); + q = new WildcardQuery(e.location(), nameOf(e.field()), p.asLuceneWildcard()); } } if (e instanceof RLike) { - String pattern = stringValueOf(e.right()); + String pattern = ((RLike) e).pattern(); if (inexact) { q = new QueryStringQuery(e.location(), "/" + pattern + "/", target); } else { - q = new RegexQuery(e.location(), nameOf(e.left()), pattern); + q = new RegexQuery(e.location(), nameOf(e.field()), pattern); } } - return q != null ? new QueryTranslation(wrapIfNested(q, e.left())) : null; + return q != null ? new QueryTranslation(wrapIfNested(q, e.field())) : null; } } @@ -529,8 +530,16 @@ protected QueryTranslation asQuery(Not not, boolean onAggs) { if (onAggs) { aggFilter = new AggFilter(not.id().toString(), not.asScript()); } else { - query = handleQuery(not, not.field(), - () -> new NotQuery(not.location(), toQuery(not.field(), false).query)); + Expression e = not.field(); + Query wrappedQuery = toQuery(not.field(), false).query; + Query q = wrappedQuery instanceof ScriptQuery ? new ScriptQuery(not.location(), + not.asScript()) : new NotQuery(not.location(), wrappedQuery); + + if (e instanceof FieldAttribute) { + query = wrapIfNested(q, e); + } + + query = q; } return new QueryTranslation(query, aggFilter); @@ -547,8 +556,14 @@ protected QueryTranslation asQuery(IsNotNull isNotNull, boolean onAggs) { if (onAggs) { aggFilter = new AggFilter(isNotNull.id().toString(), isNotNull.asScript()); } else { - query = handleQuery(isNotNull, isNotNull.field(), - () -> new ExistsQuery(isNotNull.location(), nameOf(isNotNull.field()))); + Query q = null; + if (isNotNull.field() instanceof FieldAttribute) { + q = new ExistsQuery(isNotNull.location(), nameOf(isNotNull.field())); + } else { + q = new ScriptQuery(isNotNull.location(), isNotNull.asScript()); + } + final Query qu = q; + query = handleQuery(isNotNull, isNotNull.field(), () -> qu); } return new QueryTranslation(query, aggFilter); @@ -565,8 +580,15 @@ protected QueryTranslation asQuery(IsNull isNull, boolean onAggs) { if (onAggs) { aggFilter = new AggFilter(isNull.id().toString(), isNull.asScript()); } else { - query = handleQuery(isNull, isNull.field(), - () -> new NotQuery(isNull.location(), new ExistsQuery(isNull.location(), nameOf(isNull.field())))); + Query q = null; + if (isNull.field() instanceof FieldAttribute) { + q = new NotQuery(isNull.location(), new ExistsQuery(isNull.location(), nameOf(isNull.field()))); + } else { + q = new ScriptQuery(isNull.location(), isNull.asScript()); + } + final Query qu = q; + + query = handleQuery(isNull, isNull.field(), () -> qu); } return new QueryTranslation(query, aggFilter); @@ -678,7 +700,14 @@ protected QueryTranslation asQuery(In in, boolean onAggs) { aggFilter = new AggFilter(at.id().toString(), in.asScript()); } else { - query = handleQuery(in, ne, () -> new TermsQuery(in.location(), ne.name(), in.list())); + Query q = null; + if (in.value() instanceof FieldAttribute) { + q = new TermsQuery(in.location(), ne.name(), in.list()); + } else { + q = new ScriptQuery(in.location(), in.asScript()); + } + Query qu = q; + query = handleQuery(in, ne, () -> qu); } return new QueryTranslation(query, aggFilter); } @@ -719,6 +748,25 @@ protected QueryTranslation asQuery(Range r, boolean onAggs) { } } } + + static class Scalars extends ExpressionTranslator { + + @Override + protected QueryTranslation asQuery(ScalarFunction f, boolean onAggs) { + ScriptTemplate script = f.asScript(); + + Query query = null; + AggFilter aggFilter = null; + + if (onAggs) { + aggFilter = new AggFilter(f.id().toString(), script); + } else { + query = handleQuery(f, f, () -> new ScriptQuery(f.location(), script)); + } + + return new QueryTranslation(query, aggFilter); + } + } // @@ -862,8 +910,9 @@ public QueryTranslation translate(Expression exp, boolean onAggs) { protected static Query handleQuery(ScalarFunction sf, Expression field, Supplier query) { + Query q = query.get(); if (field instanceof FieldAttribute) { - return wrapIfNested(query.get(), field); + return wrapIfNested(q, field); } return new ScriptQuery(sf.location(), sf.asScript()); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java index 8c8a64c79f298..2412342c69c6e 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java @@ -322,10 +322,10 @@ public void testConstantNot() { public void testConstantFoldingLikes() { assertEquals(Literal.TRUE, - new ConstantFolding().rule(new Like(EMPTY, Literal.of(EMPTY, "test_emp"), new LikePattern(EMPTY, "test%", (char) 0))) + new ConstantFolding().rule(new Like(EMPTY, Literal.of(EMPTY, "test_emp"), new LikePattern("test%", (char) 0))) .canonical()); assertEquals(Literal.TRUE, - new ConstantFolding().rule(new RLike(EMPTY, Literal.of(EMPTY, "test_emp"), Literal.of(EMPTY, "test.emp"))).canonical()); + new ConstantFolding().rule(new RLike(EMPTY, Literal.of(EMPTY, "test_emp"), "test.emp")).canonical()); } public void testConstantFoldingDatetime() { @@ -419,7 +419,7 @@ public void testGenericNullableExpression() { // comparison assertNullLiteral(rule.rule(new GreaterThan(EMPTY, getFieldAttribute(), Literal.NULL))); // regex - assertNullLiteral(rule.rule(new RLike(EMPTY, getFieldAttribute(), Literal.NULL))); + assertNullLiteral(rule.rule(new RLike(EMPTY, Literal.NULL, "123"))); } public void testSimplifyCoalesceNulls() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java index 963498bb9b6b2..cc91cdf6eabd7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.sql.expression.predicate.fulltext.FullTextPredicate; import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.InPipe; +import org.elasticsearch.xpack.sql.expression.predicate.regex.Like; import org.elasticsearch.xpack.sql.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.sql.tree.NodeTests.ChildrenAreAProperty; import org.elasticsearch.xpack.sql.tree.NodeTests.Dummy; @@ -449,14 +450,12 @@ public boolean equals(Object obj) { } return b.toString(); } - } else if (toBuildClass == LikePattern.class) { - /* - * The pattern and escape character have to be valid together - * so we pick an escape character that isn't used - */ - if (argClass == char.class) { - return randomFrom('\\', '|', '/', '`'); + } else if (toBuildClass == Like.class) { + + if (argClass == LikePattern.class) { + return new LikePattern(randomAlphaOfLength(16), randomFrom('\\', '|', '/', '`')); } + } else if (toBuildClass == Histogram.class) { if (argClass == Expression.class) { return LiteralTests.randomLiteral();