Rewrite lookup index selection implementation (#1188)

* Add specified index scan nodes * Refactor lookup validator * Fix getcontext in validator * Refactor index scan plan node * construct IndexScan plan node * Add opt rule for indexscan and filter * Select index * Fix file edge scan rule * Fix crash * Add lookup tests * Fix bug * Resolve conflicts * Cleanup * Rename priority to score * Refactor tck test cases * More cases * Cleanup * Fix logicalor and lookup on tag and edge * fix nullptr initialize bug * Fix invalid column name * Fix bug * Fix lookup column errors * Add more or expr cases * Limit IndexScanRule usage * drop space * Fix IndexFullScanRule * Fix multiple range error * Fix failed tests * Fix comment * Cleanup and comment * more comments for logical or expression * Fix lookup validator unit tests Move comment to header file * Fix debug option * cleanup Makefile * Fix tck cases about examples usage * improve slow query test cases * format and fix optrule match * cleanup * Define extern string const values * Fix reclaim secs Co-authored-by: cpw <13495049+CPWstatic@users.noreply.github.com>
vesoft-inc · Jul 5, 2021 · 98ecfc1 · 98ecfc1
1 parent bc55dde
commit 98ecfc1
Show file tree

Hide file tree

Showing 71 changed files with 2,891 additions and 778 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -138,17 +138,17 @@ jobs:
       - name: Pytest
         run: |
           make up
-          make RM_DIR=false J=${{ steps.cmake.outputs.j }} test
+          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} test
           make down
         working-directory: tests/
         timeout-minutes: 15
       - name: TCK
         run: |
           make up
-          make RM_DIR=false J=${{ steps.cmake.outputs.j }} tck
+          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} tck
           make down
         working-directory: tests/
-        timeout-minutes: 20
+        timeout-minutes: 25
       - name: Sanitizer
         if: ${{ always() }}
         run: |

diff --git a/src/context/ast/AstContext.h b/src/context/ast/AstContext.h
@@ -12,11 +12,14 @@
 
 namespace nebula {
 namespace graph {
+
 struct AstContext {
     QueryContext*   qctx;
     Sentence*       sentence;
     SpaceInfo       space;
 };
+
 }  // namespace graph
 }  // namespace nebula
+
 #endif  // CONTEXT_ASTCONTEXT_H_
diff --git a/src/context/ast/QueryAstContext.h b/src/context/ast/QueryAstContext.h
@@ -106,6 +106,16 @@ struct GoContext final : AstContext {
     std::string                 inputVarName;
 };
 
+struct LookupContext final : public AstContext {
+    bool isEdge{false};
+    bool dedup{false};
+    bool isEmptyResultSet{false};
+    int32_t schemaId{-1};
+    int32_t limit{-1};
+    Expression* filter{nullptr};
+    // order by
+};
+
 }  // namespace graph
 }  // namespace nebula
 #endif  // CONTEXT_AST_QUERYASTCONTEXT_H_
diff --git a/src/executor/Executor.cpp b/src/executor/Executor.cpp
@@ -181,7 +181,13 @@ Executor *Executor::makeExecutor(QueryContext *qctx, const PlanNode *node) {
         case PlanNode::Kind::kUnwind: {
             return pool->add(new UnwindExecutor(node, qctx));
         }
-        case PlanNode::Kind::kIndexScan: {
+        case PlanNode::Kind::kIndexScan:
+        case PlanNode::Kind::kEdgeIndexFullScan:
+        case PlanNode::Kind::kEdgeIndexPrefixScan:
+        case PlanNode::Kind::kEdgeIndexRangeScan:
+        case PlanNode::Kind::kTagIndexFullScan:
+        case PlanNode::Kind::kTagIndexPrefixScan:
+        case PlanNode::Kind::kTagIndexRangeScan: {
             return pool->add(new IndexScanExecutor(node, qctx));
         }
         case PlanNode::Kind::kStart: {

diff --git a/src/executor/query/IndexScanExecutor.cpp b/src/executor/query/IndexScanExecutor.cpp
@@ -29,10 +29,10 @@ folly::Future<Status> IndexScanExecutor::indexScan() {
         return finish(ResultBuilder().value(Value(std::move(dataSet))).finish());
     }
     return storageClient->lookupIndex(lookup->space(),
-                                     *lookup->queryContext(),
+                                      lookup->queryContext(),
                                       lookup->isEdge(),
                                       lookup->schemaId(),
-                                     *lookup->returnColumns())
+                                      lookup->returnColumns())
         .via(runner())
         .thenValue([this](StorageRpcResponse<LookupIndexResp> &&rpcResp) {
             return handleResp(std::move(rpcResp));

diff --git a/src/optimizer/CMakeLists.txt b/src/optimizer/CMakeLists.txt
@@ -24,6 +24,14 @@ nebula_add_library(
     rule/PushFilterDownAggregateRule.cpp
     rule/PushFilterDownProjectRule.cpp
     rule/PushFilterDownLeftJoinRule.cpp
+    rule/PushFilterDownEdgeIndexScanRule.cpp
+    rule/PushFilterDownTagIndexScanRule.cpp
+    rule/UnionAllIndexScanBaseRule.cpp
+    rule/UnionAllTagIndexScanRule.cpp
+    rule/UnionAllEdgeIndexScanRule.cpp
+    rule/IndexFullScanBaseRule.cpp
+    rule/TagIndexFullScanRule.cpp
+    rule/EdgeIndexFullScanRule.cpp
 )
 
 nebula_add_subdirectory(test)
diff --git a/src/optimizer/OptRule.cpp b/src/optimizer/OptRule.cpp
@@ -12,9 +12,26 @@
 #include "optimizer/OptGroup.h"
 #include "planner/plan/PlanNode.h"
 
+using nebula::graph::PlanNode;
+
 namespace nebula {
 namespace opt {
 
+const PlanNode *MatchedResult::planNode(const std::vector<int32_t> &pos) const {
+    if (pos.empty()) {
+        return DCHECK_NOTNULL(node)->node();
+    }
+
+    DCHECK_EQ(pos[0], 0);
+
+    const MatchedResult *result = this;
+    for (size_t i = 1; i < pos.size(); ++i) {
+        DCHECK_LT(pos[i], result->dependencies.size());
+        result = &result->dependencies[pos[i]];
+    }
+    return DCHECK_NOTNULL(result->node)->node();
+}
+
 Pattern Pattern::create(graph::PlanNode::Kind kind, std::initializer_list<Pattern> patterns) {
     Pattern pattern;
     pattern.kind_ = kind;

diff --git a/src/optimizer/OptRule.h b/src/optimizer/OptRule.h
@@ -19,6 +19,7 @@ namespace nebula {
 
 namespace graph {
 class QueryContext;
+class PlanNode;
 }   // namespace graph
 
 namespace opt {
@@ -30,6 +31,16 @@ class OptGroup;
 struct MatchedResult {
     const OptGroupNode *node{nullptr};
     std::vector<MatchedResult> dependencies;
+
+    // params       | plan node
+    // -------------+------------
+    // {}           | this->node
+    // {0}          | this->node
+    // {1}          | error
+    // {0, 1}       | this->dependencies[1]
+    // {0, 1, 0}    | this->dependencies[1].dependencies[0]
+    // {0, 1, 0, 1} | this->dependencies[1].dependencies[0].dependencies[1]
+    const graph::PlanNode *planNode(const std::vector<int32_t> &pos = {}) const;
 };
 
 class Pattern final {