Merge branch 'master' into OctEscChar

vesoft-inc · May 27, 2021 · b6c98c4 · b6c98c4
2 parents e683555 + e6e3d6f
commit b6c98c4
Show file tree

Hide file tree

Showing 17 changed files with 190 additions and 67 deletions.
diff --git a/src/context/test/CMakeLists.txt b/src/context/test/CMakeLists.txt
@@ -29,6 +29,7 @@ SET(CONTEXT_TEST_LIBS
     $<TARGET_OBJECTS:common_graph_obj>
     $<TARGET_OBJECTS:common_ft_es_graph_adapter_obj>
     $<TARGET_OBJECTS:common_ws_common_obj>
+    $<TARGET_OBJECTS:common_version_obj>
     $<TARGET_OBJECTS:util_obj>
     $<TARGET_OBJECTS:context_obj>
     $<TARGET_OBJECTS:expr_visitor_obj>

diff --git a/src/executor/admin/ShowHostsExecutor.cpp b/src/executor/admin/ShowHostsExecutor.cpp
@@ -24,7 +24,7 @@ folly::Future<Status> ShowHostsExecutor::showHosts() {
     static constexpr char kPartitionDelimeter[] = ", ";
 
     auto *shNode = asNode<ShowHosts>(node());
-    auto makeTranditionalResult = [&](const std::vector<meta::cpp2::HostItem> &hostVec) -> DataSet {
+    auto makeTraditionalResult = [&](const std::vector<meta::cpp2::HostItem> &hostVec) -> DataSet {
         DataSet v({"Host",
                    "Port",
                    "Status",
@@ -128,13 +128,15 @@ folly::Future<Status> ShowHostsExecutor::showHosts() {
     };
 
     auto makeGitInfoResult = [&](const std::vector<meta::cpp2::HostItem> &hostVec) -> DataSet {
-        DataSet v({"Host", "Port", "Status", "Role", "Git Info Sha"});
+        DataSet v({"Host", "Port", "Status", "Role", "Git Info Sha", "Version"});
         for (const auto &host : hostVec) {
             nebula::Row r({host.get_hostAddr().host,
                            host.get_hostAddr().port,
                            apache::thrift::util::enumNameSafe(host.get_status()),
                            apache::thrift::util::enumNameSafe(host.get_role()),
                            host.get_git_info_sha()});
+            // empty for non-versioned
+            r.emplace_back(host.version_ref().has_value() ? Value(*host.version_ref()) : Value());
             v.emplace_back(std::move(r));
         }   // row loop
         return v;
@@ -151,7 +153,7 @@ folly::Future<Status> ShowHostsExecutor::showHosts() {
             }
             auto value = std::move(resp).value();
             if (type == meta::cpp2::ListHostType::ALLOC) {
-                return finish(makeTranditionalResult(value));
+                return finish(makeTraditionalResult(value));
             }
             return finish(makeGitInfoResult(value));
         });

diff --git a/src/executor/test/CMakeLists.txt b/src/executor/test/CMakeLists.txt
@@ -33,6 +33,7 @@ SET(EXEC_QUERY_TEST_OBJS
     $<TARGET_OBJECTS:common_time_utils_obj>
     $<TARGET_OBJECTS:common_ft_es_graph_adapter_obj>
     $<TARGET_OBJECTS:common_ws_common_obj>
+    $<TARGET_OBJECTS:common_version_obj>
     $<TARGET_OBJECTS:session_obj>
     $<TARGET_OBJECTS:graph_flags_obj>
     $<TARGET_OBJECTS:parser_obj>

diff --git a/src/optimizer/rule/PushFilterDownAggregateRule.cpp b/src/optimizer/rule/PushFilterDownAggregateRule.cpp
@@ -43,18 +43,21 @@ StatusOr<OptRule::TransformResult> PushFilterDownAggregateRule::transform(
     DCHECK_EQ(deps.size(), 1);
     auto aggGroupNode = deps.front().node;
     auto* oldAggNode = aggGroupNode->node();
-    DCHECK(oldFilterNode->kind() == PlanNode::Kind::kFilter);
-    DCHECK(oldAggNode->kind() == PlanNode::Kind::kAggregate);
+    DCHECK_EQ(oldFilterNode->kind(), PlanNode::Kind::kFilter);
+    DCHECK_EQ(oldAggNode->kind(), PlanNode::Kind::kAggregate);
     auto* newFilterNode = static_cast<graph::Filter*>(oldFilterNode->clone());
     auto* newAggNode = static_cast<graph::Aggregate*>(oldAggNode->clone());
     const auto* condition = newFilterNode->condition();
     auto& groupItems = newAggNode->groupItems();
 
     // Check expression recursively to ensure no aggregate items in the filter
     auto varProps = graph::ExpressionUtils::collectAll(condition, {Expression::Kind::kVarProperty});
+    if (varProps.empty()) {
+        return TransformResult::noTransform();
+    }
     std::vector<std::string> propNames;
     for (auto* expr : varProps) {
-        DCHECK(expr->kind() == Expression::Kind::kVarProperty);
+        DCHECK_EQ(expr->kind(), Expression::Kind::kVarProperty);
         propNames.emplace_back(*static_cast<const VariablePropertyExpression*>(expr)->prop());
     }
     std::unordered_map<std::string, Expression*> rewriteMap;
@@ -92,7 +95,7 @@ StatusOr<OptRule::TransformResult> PushFilterDownAggregateRule::transform(
     // Exchange planNode
     newAggNode->setOutputVar(oldFilterNode->outputVar());
     newFilterNode->setInputVar(oldAggNode->inputVar());
-    DCHECK(oldAggNode->outputVar() == oldFilterNode->inputVar());
+    DCHECK_EQ(oldAggNode->outputVar(), oldFilterNode->inputVar());
     newAggNode->setInputVar(oldAggNode->outputVar());
     newFilterNode->setOutputVar(oldAggNode->outputVar());
 

diff --git a/src/optimizer/rule/PushFilterDownLeftJoinRule.cpp b/src/optimizer/rule/PushFilterDownLeftJoinRule.cpp
@@ -53,6 +53,9 @@ StatusOr<OptRule::TransformResult> PushFilterDownLeftJoinRule::transform(
     // split the `condition` based on whether the varPropExpr comes from the left child
     auto picker = [&leftVarColNames](const Expression* e) -> bool {
         auto varProps = graph::ExpressionUtils::collectAll(e, {Expression::Kind::kVarProperty});
+        if (varProps.empty()) {
+            return false;
+        }
         std::vector<std::string> propNames;
         for (auto* expr : varProps) {
             DCHECK(expr->kind() == Expression::Kind::kVarProperty);
@@ -118,6 +121,7 @@ StatusOr<OptRule::TransformResult> PushFilterDownLeftJoinRule::transform(
         result.newGroupNodes.emplace_back(newAboveFilterGroupNode);
     } else {
         newLeftJoinNode->setOutputVar(oldFilterNode->outputVar());
+        newLeftJoinNode->setColNames(oldLeftJoinNode->colNames());
         auto newLeftJoinGroupNode =
             OptGroupNode::create(octx, newLeftJoinNode, filterGroupNode->group());
         newLeftJoinGroupNode->setDeps({newFilterGroup});

diff --git a/src/optimizer/rule/PushFilterDownProjectRule.cpp b/src/optimizer/rule/PushFilterDownProjectRule.cpp
@@ -34,60 +34,118 @@ const Pattern& PushFilterDownProjectRule::pattern() const {
 StatusOr<OptRule::TransformResult> PushFilterDownProjectRule::transform(
     OptContext* octx,
     const MatchedResult& matched) const {
-    auto filterGroupNode = matched.node;
-    auto oldFilterNode = filterGroupNode->node();
-    auto projGroupNode = matched.dependencies.front().node;
-    auto oldProjNode = projGroupNode->node();
-
-    auto newFilterNode = static_cast<graph::Filter*>(oldFilterNode->clone());
-    auto newProjNode = static_cast<graph::Project*>(oldProjNode->clone());
-    const auto condition = newFilterNode->condition();
+    const auto* filterGroupNode = matched.node;
+    const auto* filterNode = filterGroupNode->node();
+    DCHECK_EQ(filterNode->kind(), PlanNode::Kind::kFilter);
+    auto* oldFilterNode = static_cast<const graph::Filter*>(filterNode);
+    const auto* projGroupNode = matched.dependencies.front().node;
+    const auto* projNode = projGroupNode->node();
+    DCHECK_EQ(projNode->kind(), PlanNode::Kind::kProject);
+    const auto* oldProjNode = static_cast<const graph::Project*>(projNode);
+    const auto* condition = oldFilterNode->condition();
+    auto objPool = octx->qctx()->objPool();
+
+    auto projColNames = oldProjNode->colNames();
+    auto projColumns = oldProjNode->columns()->columns();
+    std::unordered_map<std::string, Expression*> rewriteMap;
+    // split the `condition` based on whether the propExprs comes from the left child
+    auto picker = [&projColumns, &projColNames, &rewriteMap](const Expression* e) -> bool {
+        auto varProps = graph::ExpressionUtils::collectAll(e,
+                                                           {Expression::Kind::kTagProperty,
+                                                            Expression::Kind::kEdgeProperty,
+                                                            Expression::Kind::kInputProperty,
+                                                            Expression::Kind::kVarProperty,
+                                                            Expression::Kind::kDstProperty,
+                                                            Expression::Kind::kSrcProperty});
+        if (varProps.empty()) {
+            return false;
+        }
+        std::vector<std::string> propNames;
+        for (auto* expr : varProps) {
+            DCHECK(graph::ExpressionUtils::isPropertyExpr(expr));
+            propNames.emplace_back(*static_cast<const PropertyExpression*>(expr)->prop());
+        }
+        for (size_t i = 0; i < projColNames.size(); ++i) {
+            auto column = projColumns[i];
+            auto colName = projColNames[i];
+            auto iter =
+                std::find_if(propNames.begin(), propNames.end(), [&colName](const auto& name) {
+                    return !colName.compare(name);
+                });
+            if (iter == propNames.end()) continue;
+            if (graph::ExpressionUtils::isPropertyExpr(column->expr())) {
+                if (column->alias()) {
+                    rewriteMap[colName] = column->expr();
+                }
+                continue;
+            } else {
+                return false;
+            }
+        }
+        return true;
+    };
+    std::unique_ptr<Expression> filterPicked;
+    std::unique_ptr<Expression> filterUnpicked;
+    graph::ExpressionUtils::splitFilter(condition, picker, &filterPicked, &filterUnpicked);
 
-    auto varProps = graph::ExpressionUtils::collectAll(condition, {Expression::Kind::kVarProperty});
-    if (varProps.empty()) {
+    if (!filterPicked) {
         return TransformResult::noTransform();
     }
-    std::vector<std::string> propNames;
-    for (auto expr : varProps) {
-        DCHECK(expr->kind() == Expression::Kind::kVarProperty);
-        propNames.emplace_back(*static_cast<const VariablePropertyExpression*>(expr)->prop());
-    }
-
-    auto projColNames = newProjNode->colNames();
-    auto projColumns = newProjNode->columns()->columns();
-    for (size_t i = 0; i < projColNames.size(); ++i) {
-        auto column = projColumns[i];
-        auto colName = projColNames[i];
-        auto iter = std::find_if(propNames.begin(), propNames.end(), [&colName](const auto& name) {
-            return !colName.compare(name);
-        });
-        if (iter == propNames.end()) continue;
-        if (!column->alias() && column->expr()->kind() == Expression::Kind::kVarProperty) {
-            continue;
-        } else {
-            // project column contains computing expression, need to rewrite
-            return TransformResult::noTransform();
+    // Rewrite PropertyExpr in filter's condition
+    auto matcher = [&rewriteMap](const Expression* e) -> bool {
+        if (!graph::ExpressionUtils::isPropertyExpr(e)) {
+            return false;
         }
-    }
-
-    // Exchange planNode
-    newProjNode->setOutputVar(oldFilterNode->outputVar());
-    newFilterNode->setInputVar(oldProjNode->inputVar());
-    newProjNode->setInputVar(oldProjNode->outputVar());
-    newFilterNode->setOutputVar(oldProjNode->outputVar());
-
-    // Push down filter's optGroup and embed newProjGroupNode into old filter's Group
-    auto newProjGroupNode = OptGroupNode::create(octx, newProjNode, filterGroupNode->group());
-    auto newFilterGroup = OptGroup::create(octx);
-    auto newFilterGroupNode = newFilterGroup->makeGroupNode(newFilterNode);
-    newProjGroupNode->dependsOn(newFilterGroup);
+        auto* propName = static_cast<const PropertyExpression*>(e)->prop();
+        return rewriteMap[*propName];
+    };
+    auto rewriter = [&rewriteMap](const Expression* e) -> Expression* {
+        DCHECK(graph::ExpressionUtils::isPropertyExpr(e));
+        auto* propName = static_cast<const PropertyExpression*>(e)->prop();
+        return rewriteMap[*propName]->clone().release();
+    };
+    auto* newFilterPicked = rewriteMap.empty()
+                                ? filterPicked.release()
+                                : graph::RewriteVisitor::transform(
+                                      filterPicked.get(), std::move(matcher), std::move(rewriter));
+
+    // produce new Filter node below
+    auto* newBelowFilterNode = graph::Filter::make(octx->qctx(),
+                                                   const_cast<graph::PlanNode*>(oldProjNode->dep()),
+                                                   objPool->add(newFilterPicked));
+    newBelowFilterNode->setInputVar(oldProjNode->inputVar());
+    auto newBelowFilterGroup = OptGroup::create(octx);
+    auto newFilterGroupNode = newBelowFilterGroup->makeGroupNode(newBelowFilterNode);
     for (auto dep : projGroupNode->dependencies()) {
         newFilterGroupNode->dependsOn(dep);
     }
 
+    // produce new Proj node
+    auto* newProjNode = static_cast<graph::Project*>(oldProjNode->clone());
+    newProjNode->setInputVar(newBelowFilterNode->outputVar());
+
     TransformResult result;
     result.eraseAll = true;
-    result.newGroupNodes.emplace_back(newProjGroupNode);
+    if (filterUnpicked) {
+        // produce new Filter node above
+        auto* newAboveFilterNode =
+            graph::Filter::make(octx->qctx(), newProjNode, objPool->add(filterUnpicked.release()));
+        newAboveFilterNode->setOutputVar(oldFilterNode->outputVar());
+        auto newAboveFilterGroupNode =
+            OptGroupNode::create(octx, newAboveFilterNode, filterGroupNode->group());
+
+        auto newProjGroup = OptGroup::create(octx);
+        auto newProjGroupNode = newProjGroup->makeGroupNode(newProjNode);
+        newProjGroupNode->setDeps({newBelowFilterGroup});
+        newAboveFilterGroupNode->setDeps({newProjGroup});
+        result.newGroupNodes.emplace_back(newAboveFilterGroupNode);
+    } else {
+        newProjNode->setOutputVar(oldFilterNode->outputVar());
+        auto newProjGroupNode = OptGroupNode::create(octx, newProjNode, filterGroupNode->group());
+        newProjGroupNode->setDeps({newBelowFilterGroup});
+        result.newGroupNodes.emplace_back(newProjGroupNode);
+    }
+
     return result;
 }
 
@@ -97,4 +155,3 @@ std::string PushFilterDownProjectRule::toString() const {
 
 }   // namespace opt
 }   // namespace nebula
-
diff --git a/src/optimizer/test/CMakeLists.txt b/src/optimizer/test/CMakeLists.txt
@@ -31,6 +31,7 @@ set(OPTIMIZER_TEST_LIB
     $<TARGET_OBJECTS:common_graph_obj>
     $<TARGET_OBJECTS:common_ft_es_graph_adapter_obj>
     $<TARGET_OBJECTS:common_ws_common_obj>
+    $<TARGET_OBJECTS:common_version_obj>
     $<TARGET_OBJECTS:idgenerator_obj>
     $<TARGET_OBJECTS:expr_visitor_obj>
     $<TARGET_OBJECTS:session_obj>

diff --git a/src/parser/test/CMakeLists.txt b/src/parser/test/CMakeLists.txt
@@ -31,6 +31,7 @@ set(PARSER_TEST_LIBS
     $<TARGET_OBJECTS:common_time_utils_obj>
     $<TARGET_OBJECTS:common_ft_es_graph_adapter_obj>
     $<TARGET_OBJECTS:common_ws_common_obj>
+    $<TARGET_OBJECTS:common_version_obj>
     $<TARGET_OBJECTS:session_obj>
     $<TARGET_OBJECTS:graph_flags_obj>
     $<TARGET_OBJECTS:graph_auth_obj>

diff --git a/src/planner/plan/Query.h b/src/planner/plan/Query.h
@@ -1010,10 +1010,12 @@ class Join : public SingleDependencyNode {
     }
 
     void setLeftVar(std::pair<std::string, int64_t> lvar) {
+        setInputVar(lvar.first, 0);
         leftVar_ = lvar;
     }
 
     void setRightVar(std::pair<std::string, int64_t> rvar) {
+        setInputVar(rvar.first, 1);
         rightVar_ = rvar;
     }
 
@@ -1061,7 +1063,7 @@ class LeftJoin final : public Join {
 public:
     static LeftJoin* make(QueryContext* qctx,
                           PlanNode* input,
-                          std::pair<std::string, int64_t> leftVar ,
+                          std::pair<std::string, int64_t> leftVar,
                           std::pair<std::string, int64_t> rightVar,
                           std::vector<Expression*> hashKeys = {},
                           std::vector<Expression*> probeKeys = {}) {

diff --git a/src/util/ExpressionUtils.cpp b/src/util/ExpressionUtils.cpp
@@ -17,6 +17,18 @@
 namespace nebula {
 namespace graph {
 
+bool ExpressionUtils::isPropertyExpr(const Expression *expr) {
+    auto kind = expr->kind();
+
+    return std::unordered_set<Expression::Kind>{Expression::Kind::kTagProperty,
+                                                Expression::Kind::kEdgeProperty,
+                                                Expression::Kind::kInputProperty,
+                                                Expression::Kind::kVarProperty,
+                                                Expression::Kind::kDstProperty,
+                                                Expression::Kind::kSrcProperty}
+        .count(kind);
+}
+
 const Expression *ExpressionUtils::findAny(const Expression *self,
                                      const std::unordered_set<Expression::Kind> &expected) {
     auto finder = [&expected](const Expression *expr) -> bool {

diff --git a/src/util/ExpressionUtils.h b/src/util/ExpressionUtils.h
@@ -33,6 +33,8 @@ class ExpressionUtils {
         return expected.find(expr->kind()) != expected.end();
     }
 
+    static bool isPropertyExpr(const Expression* expr);
+
     static const Expression* findAny(const Expression* self,
                                      const std::unordered_set<Expression::Kind>& expected);
 

diff --git a/src/util/test/CMakeLists.txt b/src/util/test/CMakeLists.txt
@@ -31,6 +31,7 @@ nebula_add_test(
         $<TARGET_OBJECTS:common_graph_obj>
         $<TARGET_OBJECTS:common_ft_es_graph_adapter_obj>
         $<TARGET_OBJECTS:common_ws_common_obj>
+        $<TARGET_OBJECTS:common_version_obj>
         $<TARGET_OBJECTS:idgenerator_obj>
         $<TARGET_OBJECTS:expr_visitor_obj>
         $<TARGET_OBJECTS:session_obj>

diff --git a/src/validator/YieldValidator.cpp b/src/validator/YieldValidator.cpp
@@ -247,7 +247,6 @@ Status YieldValidator::toPlan() {
         }
     }
 
-
     if (yield->yield()->isDistinct()) {
         auto dedup = Dedup::make(qctx_, dedupDep);
         dedup->setColNames(dedupDep->colNames());

diff --git a/src/validator/test/CMakeLists.txt b/src/validator/test/CMakeLists.txt
@@ -50,6 +50,7 @@ set(VALIDATOR_TEST_LIBS
     $<TARGET_OBJECTS:common_graph_obj>
     $<TARGET_OBJECTS:common_ft_es_graph_adapter_obj>
     $<TARGET_OBJECTS:common_ws_common_obj>
+    $<TARGET_OBJECTS:common_version_obj>
 )
 
 nebula_add_test(

diff --git a/src/visitor/test/CMakeLists.txt b/src/visitor/test/CMakeLists.txt
@@ -54,6 +54,7 @@ nebula_add_test(
         $<TARGET_OBJECTS:common_graph_obj>
         $<TARGET_OBJECTS:common_ft_es_graph_adapter_obj>
         $<TARGET_OBJECTS:common_ws_common_obj>
+        $<TARGET_OBJECTS:common_version_obj>
     LIBRARIES
         gtest
         ${THRIFT_LIBRARIES}