Cherry pick v3.0.0 0127 (#3841)

* Fix `Match cannot be followed by use space` (#3748)

* fix bug

* add tck

* remove set

* fix

* rm header

* fix

* remove changes in match

* fix

* enhance annotation

* fix review

* fix

* fix review

* remove redundant

Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com>
Co-authored-by: cpw <13495049+CPWstatic@users.noreply.github.com>

* raft: fix always loop when no log need to send (#3819)

Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com>
Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com>

Co-authored-by: jakevin <30525741+jackwener@users.noreply.github.com>
Co-authored-by: cpw <13495049+CPWstatic@users.noreply.github.com>
Co-authored-by: liwenhui-soul <38217397+liwenhui-soul@users.noreply.github.com>
Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com>

src/graph/planner/SequentialPlanner.cpp

@@ -24,6 +24,10 @@ StatusOr<SubPlan> SequentialPlanner::transform(AstContext* astCtx) {
   subPlan.root = validators.back()->root();
   ifBuildDataCollect(subPlan, qctx);
   for (auto iter = validators.begin(); iter < validators.end() - 1; ++iter) {
+    // Remove left tail kStart plannode before append plan.
+    // It allows that kUse sentence to append kMatch Sentence.
+    // For example: Use ...; Match ...
+    rmLeftTailStartNode((iter + 1)->get(), iter->get()->sentence()->kind());
     NG_RETURN_IF_ERROR((iter + 1)->get()->appendPlan(iter->get()->root()));
   }
   if (validators.front()->tail()->isSingleInput()) {
@@ -59,5 +63,27 @@ void SequentialPlanner::ifBuildDataCollect(SubPlan& subPlan, QueryContext* qctx)
       break;
   }
 }
+
+// When appending plans, it need to remove left tail plannode.
+// Because the left tail plannode is StartNode which needs to be removed,
+// and remain one size for add dependency
+// TODO: It's a temporary solution, remove it after Execute multiple sequences one by one.
+void SequentialPlanner::rmLeftTailStartNode(Validator* validator, Sentence::Kind appendPlanKind) {
+  if (appendPlanKind != Sentence::Kind::kUse ||
+      validator->tail()->kind() != PlanNode::Kind::kStart ||
+      validator->root()->dependencies().size() == 0UL) {
+    return;
+  }
+
+  PlanNode* node = validator->root();
+  while (node->dependencies()[0]->dependencies().size() > 0UL) {
+    node = const_cast<PlanNode*>(node->dependencies()[0]);
+  }
+  if (node->dependencies().size() == 1UL) {
+    // Remain one size for add dependency
+    node->dependencies()[0] = nullptr;
+    validator->setTail(node);
+  }
+}
 }  // namespace graph
 }  // namespace nebula

src/graph/planner/SequentialPlanner.h

@@ -8,6 +8,7 @@
 
 #include "graph/context/QueryContext.h"
 #include "graph/planner/Planner.h"
+#include "graph/validator/Validator.h"
 
 namespace nebula {
 namespace graph {
@@ -27,6 +28,8 @@ class SequentialPlanner final : public Planner {
 
   void ifBuildDataCollect(SubPlan& subPlan, QueryContext* qctx);
 
+  void rmLeftTailStartNode(Validator* validator, Sentence::Kind appendPlanKind);
+
  private:
   SequentialPlanner() = default;
 };

src/graph/planner/plan/PlanNode.h

@@ -238,6 +238,10 @@ class PlanNode {
     return dependencies_;
   }
 
+  auto& dependencies() {
+    return dependencies_;
+  }
+
   const PlanNode* dep(size_t index = 0) const {
     DCHECK_LT(index, dependencies_.size());
     return dependencies_.at(index);

src/graph/validator/SequentialValidator.cpp

@@ -48,5 +48,6 @@ const Sentence* SequentialValidator::getFirstSentence(const Sentence* sentence)
   auto pipe = static_cast<const PipedSentence*>(sentence);
   return getFirstSentence(pipe->left());
 }
+
 }  // namespace graph
 }  // namespace nebula

src/graph/validator/Validator.cpp

@@ -8,6 +8,7 @@
 #include <thrift/lib/cpp/util/EnumUtils.h>
 
 #include "common/function/FunctionManager.h"
+#include "graph/planner/plan/PlanNode.h"
 #include "graph/planner/plan/Query.h"
 #include "graph/util/ExpressionUtils.h"
 #include "graph/util/SchemaUtil.h"
@@ -299,8 +300,9 @@ std::vector<std::string> Validator::getOutColNames() const {
 Status Validator::appendPlan(PlanNode* node, PlanNode* appended) {
   DCHECK(node != nullptr);
   DCHECK(appended != nullptr);
+
   if (!node->isSingleInput()) {
-    return Status::SemanticError("%s not support to append an input.",
+    return Status::SemanticError("PlanNode(%s) not support to append an input.",
                                  PlanNode::toString(node->kind()));
   }
   static_cast<SingleDependencyNode*>(node)->dependsOn(appended);

src/graph/validator/Validator.h

@@ -87,6 +87,10 @@ class Validator {
     exprProps_ = exprProps;
   }
 
+  void setTail(PlanNode* tail) {
+    tail_ = tail;
+  }
+
   const std::set<std::string>& userDefinedVarNameList() const {
     return userDefinedVarNameList_;
   }

src/kvstore/raftex/Host.cpp

@@ -270,11 +270,29 @@ Host::prepareAppendLogRequest() {
   VLOG(2) << idStr_ << "Prepare AppendLogs request from Log " << lastLogIdSent_ + 1 << " to "
           << logIdToSend_;
 
+  auto makeReq = [this]() -> std::shared_ptr<cpp2::AppendLogRequest> {
+    auto req = std::make_shared<cpp2::AppendLogRequest>();
+    req->space_ref() = part_->spaceId();
+    req->part_ref() = part_->partitionId();
+    req->current_term_ref() = logTermToSend_;
+    req->committed_log_id_ref() = committedLogId_;
+    req->leader_addr_ref() = part_->address().host;
+    req->leader_port_ref() = part_->address().port;
+    req->last_log_term_sent_ref() = lastLogTermSent_;
+    req->last_log_id_sent_ref() = lastLogIdSent_;
+    return req;
+  };
+
   // We need to use lastLogIdSent_ + 1 to check whether need to send snapshot
   if (UNLIKELY(lastLogIdSent_ + 1 < part_->wal()->firstLogId())) {
     return startSendSnapshot();
   }
 
+  if (lastLogIdSent_ == logIdToSend_) {
+    auto req = makeReq();
+    return req;
+  }
+
   if (lastLogIdSent_ + 1 > part_->wal()->lastLogId()) {
     LOG_IF(INFO, FLAGS_trace_raft)
         << idStr_ << "My lastLogId in wal is " << part_->wal()->lastLogId()
@@ -285,16 +303,7 @@ Host::prepareAppendLogRequest() {
 
   auto it = part_->wal()->iterator(lastLogIdSent_ + 1, logIdToSend_);
   if (it->valid()) {
-    auto req = std::make_shared<cpp2::AppendLogRequest>();
-    req->space_ref() = part_->spaceId();
-    req->part_ref() = part_->partitionId();
-    req->current_term_ref() = logTermToSend_;
-    req->committed_log_id_ref() = committedLogId_;
-    req->leader_addr_ref() = part_->address().host;
-    req->leader_port_ref() = part_->address().port;
-    req->last_log_term_sent_ref() = lastLogTermSent_;
-    req->last_log_id_sent_ref() = lastLogIdSent_;
-
+    auto req = makeReq();
     std::vector<cpp2::RaftLogEntry> logs;
     for (size_t cnt = 0; it->valid() && cnt < FLAGS_max_appendlog_batch_size; ++(*it), ++cnt) {
       cpp2::RaftLogEntry entry;

tests/tck/features/match/Base.feature

@@ -699,3 +699,13 @@ Feature: Basic match
     Then the result should be, in any order:
       | v                                                                                                          |
       | ("Tim Duncan":bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"}) |
+
+  Scenario: Sequential sentence
+    When executing query:
+      """
+      USE nba;
+      MATCH (n:player) RETURN n LIMIT 1;
+      """
+    Then the result should be, in any order:
+      | n                                                   |
+      | ("Boris Diaw" :player{age: 36, name: "Boris Diaw"}) |

tests/tck/features/match/MultiLineMultiQueryParts.feature

@@ -0,0 +1,225 @@
+# Copyright (c) 2021 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License.
+Feature: Multi Line Multi Query Parts
+
+  Background:
+    Given a graph with space named "nba"
+
+  Scenario: Multi Line Multi Path Patterns
+    When executing query:
+      """
+      USE nba;
+      MATCH (m)-[]-(n), (n)-[]-(l) WHERE id(m)=="Tim Duncan"
+      RETURN m.player.name AS n1, n.player.name AS n2,
+      CASE WHEN l.team.name is not null THEN l.team.name
+      WHEN l.player.name is not null THEN l.player.name ELSE "null" END AS n3 ORDER BY n1, n2, n3 LIMIT 10
+      """
+    Then the result should be, in order:
+      | n1           | n2            | n3           |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics"    |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons"    |
+      | "Tim Duncan" | "Aron Baynes" | "Spurs"      |
+      | "Tim Duncan" | "Aron Baynes" | "Tim Duncan" |
+      | "Tim Duncan" | "Boris Diaw"  | "Hawks"      |
+      | "Tim Duncan" | "Boris Diaw"  | "Hornets"    |
+      | "Tim Duncan" | "Boris Diaw"  | "Jazz"       |
+      | "Tim Duncan" | "Boris Diaw"  | "Spurs"      |
+      | "Tim Duncan" | "Boris Diaw"  | "Suns"       |
+      | "Tim Duncan" | "Boris Diaw"  | "Tim Duncan" |
+    When executing query:
+      """
+      USE nba;
+      MATCH (m)-[]-(n), (n)-[]-(l) WHERE id(n)=="Tim Duncan"
+      RETURN m.player.name AS n1, n.player.name AS n2, l.player.name AS n3 ORDER BY n1, n2, n3 LIMIT 10
+      """
+    Then the result should be, in order:
+      | n1            | n2           | n3                  |
+      | "Aron Baynes" | "Tim Duncan" | "Aron Baynes"       |
+      | "Aron Baynes" | "Tim Duncan" | "Boris Diaw"        |
+      | "Aron Baynes" | "Tim Duncan" | "Danny Green"       |
+      | "Aron Baynes" | "Tim Duncan" | "Danny Green"       |
+      | "Aron Baynes" | "Tim Duncan" | "Dejounte Murray"   |
+      | "Aron Baynes" | "Tim Duncan" | "LaMarcus Aldridge" |
+      | "Aron Baynes" | "Tim Duncan" | "LaMarcus Aldridge" |
+      | "Aron Baynes" | "Tim Duncan" | "Manu Ginobili"     |
+      | "Aron Baynes" | "Tim Duncan" | "Manu Ginobili"     |
+      | "Aron Baynes" | "Tim Duncan" | "Manu Ginobili"     |
+    When executing query:
+      """
+      USE nba;
+      MATCH (m)-[]-(n), (n)-[]-(l), (l)-[]-(h) WHERE id(m)=="Tim Duncan"
+      RETURN m.player.name AS n1, n.player.name AS n2, l.team.name AS n3, h.player.name AS n4
+      ORDER BY n1, n2, n3, n4 LIMIT 10
+      """
+    Then the result should be, in order:
+      | n1           | n2            | n3        | n4                |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Aron Baynes"     |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Kyrie Irving"    |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Rajon Rondo"     |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Ray Allen"       |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Shaquile O'Neal" |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons" | "Aron Baynes"     |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons" | "Blake Griffin"   |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons" | "Grant Hill"      |
+      | "Tim Duncan" | "Aron Baynes" | "Spurs"   | "Aron Baynes"     |
+      | "Tim Duncan" | "Aron Baynes" | "Spurs"   | "Boris Diaw"      |
+
+  Scenario: Multi Line Multi Match
+    When executing query:
+      """
+      USE nba;
+      MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan"
+      MATCH (n)-[]-(l)
+      RETURN m.player.name AS n1, n.player.name AS n2,
+      CASE WHEN l.player.name is not null THEN l.player.name
+      WHEN l.team.name is not null THEN l.team.name ELSE "null" END AS n3 ORDER BY n1, n2, n3 LIMIT 10
+      """
+    Then the result should be, in order:
+      | n1           | n2            | n3           |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics"    |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons"    |
+      | "Tim Duncan" | "Aron Baynes" | "Spurs"      |
+      | "Tim Duncan" | "Aron Baynes" | "Tim Duncan" |
+      | "Tim Duncan" | "Boris Diaw"  | "Hawks"      |
+      | "Tim Duncan" | "Boris Diaw"  | "Hornets"    |
+      | "Tim Duncan" | "Boris Diaw"  | "Jazz"       |
+      | "Tim Duncan" | "Boris Diaw"  | "Spurs"      |
+      | "Tim Duncan" | "Boris Diaw"  | "Suns"       |
+      | "Tim Duncan" | "Boris Diaw"  | "Tim Duncan" |
+    When executing query:
+      """
+      USE nba;
+      MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan"
+      MATCH (n)-[]-(l), (l)-[]-(h)
+      RETURN m.player.name AS n1, n.player.name AS n2, l.team.name AS n3, h.player.name AS n4
+      ORDER BY n1, n2, n3, n4 LIMIT 10
+      """
+    Then the result should be, in order:
+      | n1           | n2            | n3        | n4                |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Aron Baynes"     |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Kyrie Irving"    |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Rajon Rondo"     |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Ray Allen"       |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Shaquile O'Neal" |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons" | "Aron Baynes"     |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons" | "Blake Griffin"   |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons" | "Grant Hill"      |
+      | "Tim Duncan" | "Aron Baynes" | "Spurs"   | "Aron Baynes"     |
+      | "Tim Duncan" | "Aron Baynes" | "Spurs"   | "Boris Diaw"      |
+    When executing query:
+      """
+      USE nba;
+      MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan"
+      MATCH (n)-[]-(l)
+      MATCH (l)-[]-(h)
+      RETURN m.player.name AS n1, n.player.name AS n2, l.team.name AS n3, h.player.name AS n4
+      ORDER BY n1, n2, n3, n4 LIMIT 10
+      """
+    Then the result should be, in order:
+      | n1           | n2            | n3        | n4                |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Aron Baynes"     |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Kyrie Irving"    |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Rajon Rondo"     |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Ray Allen"       |
+      | "Tim Duncan" | "Aron Baynes" | "Celtics" | "Shaquile O'Neal" |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons" | "Aron Baynes"     |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons" | "Blake Griffin"   |
+      | "Tim Duncan" | "Aron Baynes" | "Pistons" | "Grant Hill"      |
+      | "Tim Duncan" | "Aron Baynes" | "Spurs"   | "Aron Baynes"     |
+      | "Tim Duncan" | "Aron Baynes" | "Spurs"   | "Boris Diaw"      |
+    When executing query:
+      """
+      USE nba;
+      MATCH (v:player{name:"Tony Parker"})
+      WITH v AS a
+      MATCH p=(o:player{name:"Tim Duncan"})-[]->(a)
+      RETURN o.player.name
+      """
+    Then the result should be, in order:
+      | o.player.name |
+      | "Tim Duncan"  |
+      | "Tim Duncan"  |
+
+  Scenario: Multi Line Optional Match
+    When executing query:
+      """
+      USE nba;
+      MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan"
+      OPTIONAL MATCH (n)<-[:serve]-(l)
+      RETURN m.player.name AS n1, n.player.name AS n2, l AS n3 ORDER BY n1, n2, n3 LIMIT 10
+      """
+    Then the result should be, in order:
+      | n1           | n2                  | n3   |
+      | "Tim Duncan" | "Aron Baynes"       | NULL |
+      | "Tim Duncan" | "Boris Diaw"        | NULL |
+      | "Tim Duncan" | "Danny Green"       | NULL |
+      | "Tim Duncan" | "Danny Green"       | NULL |
+      | "Tim Duncan" | "Dejounte Murray"   | NULL |
+      | "Tim Duncan" | "LaMarcus Aldridge" | NULL |
+      | "Tim Duncan" | "LaMarcus Aldridge" | NULL |
+      | "Tim Duncan" | "Manu Ginobili"     | NULL |
+      | "Tim Duncan" | "Manu Ginobili"     | NULL |
+      | "Tim Duncan" | "Manu Ginobili"     | NULL |
+
+  Scenario: Multi Line Multi Query Parts
+    When executing query:
+      """
+      USE nba;
+      MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan"
+      WITH n, n.player.name AS n1 ORDER BY n1 LIMIT 10
+      MATCH (n)-[]-(l)
+      RETURN n.player.name AS n1,
+      CASE WHEN l.player.name is not null THEN l.player.name
+      WHEN l.team.name is not null THEN l.team.name ELSE "null" END AS n2 ORDER BY n1, n2 LIMIT 10
+      """
+    Then the result should be, in order:
+      | n1            | n2           |
+      | "Aron Baynes" | "Celtics"    |
+      | "Aron Baynes" | "Pistons"    |
+      | "Aron Baynes" | "Spurs"      |
+      | "Aron Baynes" | "Tim Duncan" |
+      | "Boris Diaw"  | "Hawks"      |
+      | "Boris Diaw"  | "Hornets"    |
+      | "Boris Diaw"  | "Jazz"       |
+      | "Boris Diaw"  | "Spurs"      |
+      | "Boris Diaw"  | "Suns"       |
+      | "Boris Diaw"  | "Tim Duncan" |
+    When executing query:
+      """
+      USE nba;
+      MATCH (m:player{name:"Tim Duncan"})-[:like]-(n)--()
+      WITH  m,count(*) AS lcount
+      MATCH (m)--(n)
+      RETURN count(*) AS scount, lcount
+      """
+    Then the result should be, in order:
+      | scount | lcount |
+      | 19     | 110    |
+    When executing query:
+      """
+      USE nba;
+      MATCH (m:player{name:"Tim Duncan"})-[:like]-(n)--()
+      WITH  m,n
+      MATCH (m)--(n)
+      RETURN count(*) AS scount
+      """
+    Then the result should be, in order:
+      | scount |
+      | 270    |
+
+  Scenario: Multi Line Some Erros
+    When executing query:
+      """
+      USE nba;
+      MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan"
+      WITH n, n.player.name AS n1 ORDER BY n1 LIMIT 10
+      RETURN m
+      """
+    Then a SemanticError should be raised at runtime: Alias used but not defined: `m'
+    When executing query:
+      """
+      USE nba;
+      MATCH (v:player)-[e]-(v:team) RETURN v, e
+      """
+    Then a SemanticError should be raised at runtime: `v': Redefined alias in a single path pattern.