Cherry pick v3.0.0 (0117-0124) (#3820)

* drop host only when it registe heartbeat (#3739) * Fix race of create tag and edge (#3735) * fix rece * polish * fix review * Add version for `show hosts` (#3702) * version * tck * pytest * pytest Co-authored-by: cpw <13495049+CPWstatic@users.noreply.github.com> * Update client version to v3.0.0 (#3741) * Update python package name to v3 * Update client version white list * Update go mod to v3 * Change the python package name used in tests * Update client version white list * Fix create fulltext index failed (#3747) * fix bug: the same tagId/edgetype under different spaces, failed to create fulltext indexes * fix bug: the same tagId/edgetype under different spaces, failed to create fulltext indexes * fix writternBy delete by optimizer (#3746) * fix bugs * add tck features * style Co-authored-by: cpw <13495049+CPWstatic@users.noreply.github.com> * fix with & optimizer rule error (#3736) * fix error * fix collapseProjectRule Co-authored-by: cpw <13495049+CPWstatic@users.noreply.github.com> * fix upgrade bug (#3772) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * fix 3727 (#3737) Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> * Balance fix (#3668) * fix core balance in zone removing the only host in zone * fix balance plan that make redundant part * fix deadlock when runInternal and finish callback called in the same thread * fix transleader return TERM_OUT_OF_DATE * continue running balance task from last status * fix deadlock when removing part Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * Add snapshot for prefix iterate. (#3717) * Add snapshot for prefix iterate. * Add header comment * Add happy test * Fix lint * Modify engine snap argument position * Fix other subclass * Fix code Co-authored-by: yaphet <4414314+darionyaphet@users.noreply.github.com> Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> * suppress undesired ASAN error (#3705) * suppress undesired error * fix unstable test case * fix upgrade (#3787) Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> * Create CODEOWNERS (#3770) Confirm with sophie.xie * disable data balance (#3756) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * Fix iterator with snapshot(parameter position) (#3785) Co-authored-by: Harris.Chu <1726587+HarrisChu@users.noreply.github.com> Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> Co-authored-by: yaphet <4414314+darionyaphet@users.noreply.github.com> Co-authored-by: jakevin <30525741+jackwener@users.noreply.github.com> Co-authored-by: cpw <13495049+CPWstatic@users.noreply.github.com> Co-authored-by: Yichen Wang <18348405+Aiee@users.noreply.github.com> Co-authored-by: panda-sheep <59197347+panda-sheep@users.noreply.github.com> Co-authored-by: endy.li <25311962+heroicNeZha@users.noreply.github.com> Co-authored-by: jimingquan <mingquan.ji@vesoft.com> Co-authored-by: hs.zhang <22708345+cangfengzhs@users.noreply.github.com> Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> Co-authored-by: liwenhui-soul <38217397+liwenhui-soul@users.noreply.github.com> Co-authored-by: Alex Xing <90179377+SuperYoko@users.noreply.github.com> Co-authored-by: Harris.Chu <1726587+HarrisChu@users.noreply.github.com>
vesoft-inc · Jan 26, 2022 · caf0815 · caf0815
1 parent e7863df
commit caf0815
Show file tree

Hide file tree

Showing 95 changed files with 1,048 additions and 268 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,6 @@
+#Require an approved review in PRs including files with a designated code owner.
+/conf/ @vesoft-inc/tech-committee-reviewers
+/src/kvstore/raftex/ @critical27 @sherman-the-tank
+/cmake/ @sherman-the-tank @yixinglu
+*.thrift @vesoft-inc/tech-committee-reviewers
+*.yy @CPWstatic
diff --git a/cmake/ThriftGenerate.cmake b/cmake/ThriftGenerate.cmake
@@ -86,7 +86,7 @@ add_custom_command(
     --gen "js:node:"
     --gen "csharp"
     --gen "java:hashcode"
-    --gen "go:thrift_import=github.com/facebook/fbthrift/thrift/lib/go/thrift,package_prefix=github.com/vesoft-inc/nebula-go/v2/,use_context"
+    --gen "go:thrift_import=github.com/facebook/fbthrift/thrift/lib/go/thrift,package_prefix=github.com/vesoft-inc/nebula-go/v3/,use_context"
     -o "." "${file_path}/${file_name}.thrift"
   COMMAND
     mkdir -p "./gen-rust/${file_name}"

diff --git a/src/codec/RowReaderV2.h b/src/codec/RowReaderV2.h
@@ -26,7 +26,7 @@ class RowReaderV2 : public RowReader {
   FRIEND_TEST(ScanEdgePropBench, ProcessEdgeProps);
 
  public:
-  virtual ~RowReaderV2() = default;
+  ~RowReaderV2() override = default;
 
   Value getValueByName(const std::string& prop) const noexcept override;
   Value getValueByIndex(const int64_t index) const noexcept override;

diff --git a/src/common/base/SanitizerOptions.cpp b/src/common/base/SanitizerOptions.cpp
@@ -21,7 +21,9 @@ const char* __asan_default_options() {
            "fast_unwind_on_malloc=0 \n"
            "detect_stack_use_after_return=1 \n"
            "alloc_dealloc_mismatch=1 \n"
-           "new_delete_type_mismatch=1 \n"
+           // todo(doodle): Reopen when https://github.com/vesoft-inc/nebula/issues/3690
+           // addressed throughly
+           "new_delete_type_mismatch=0 \n"
            "strict_init_order=1 \n"
            "intercept_tls_get_addr=1 \n"
            "symbolize_inline_frames=1 \n"

diff --git a/src/common/expression/PropertyExpression.h b/src/common/expression/PropertyExpression.h
@@ -155,6 +155,10 @@ class LabelTagPropertyExpression final : public PropertyExpression {
     return label_;
   }
 
+  void setLabel(Expression* label) {
+    label_ = label;
+  }
+
  private:
   LabelTagPropertyExpression(ObjectPool* pool,
                              Expression* label = nullptr,

diff --git a/src/graph/executor/admin/ShowHostsExecutor.cpp b/src/graph/executor/admin/ShowHostsExecutor.cpp
@@ -30,7 +30,8 @@ folly::Future<Status> ShowHostsExecutor::showHosts() {
                "Status",
                "Leader count",
                "Leader distribution",
-               "Partition distribution"});
+               "Partition distribution",
+               "Version"});
 
     std::map<std::string, int64_t> leaderPartsCount;
     std::map<std::string, int64_t> allPartsCount;
@@ -82,6 +83,7 @@ folly::Future<Status> ShowHostsExecutor::showHosts() {
       r.emplace_back(leaderCount);
       r.emplace_back(leaders.str());
       r.emplace_back(parts.str());
+      r.emplace_back(host.version_ref().has_value() ? Value(*host.version_ref()) : Value());
       v.emplace_back(std::move(r));
     }  // row loop
     {
@@ -148,7 +150,7 @@ folly::Future<Status> ShowHostsExecutor::showHosts() {
           LOG(ERROR) << resp.status();
           return resp.status();
         }
-        auto value = std::move(resp).value();
+        auto value = std::forward<decltype(resp)>(resp).value();
         if (type == meta::cpp2::ListHostType::ALLOC) {
           return finish(makeTraditionalResult(value));
         }

diff --git a/src/graph/optimizer/OptGroup.cpp b/src/graph/optimizer/OptGroup.cpp
@@ -45,6 +45,7 @@ void OptGroup::addGroupNode(OptGroupNode *groupNode) {
   DCHECK(groupNode != nullptr);
   DCHECK(groupNode->group() == this);
   groupNodes_.emplace_back(groupNode);
+  groupNode->node()->updateSymbols();
 }
 
 OptGroupNode *OptGroup::makeGroupNode(PlanNode *node) {

diff --git a/src/graph/planner/match/MatchPlanner.h b/src/graph/planner/match/MatchPlanner.h
@@ -14,7 +14,7 @@ namespace graph {
 class MatchPlanner final : public Planner {
  public:
   static std::unique_ptr<MatchPlanner> make() {
-    return std::unique_ptr<MatchPlanner>(new MatchPlanner());
+    return std::make_unique<MatchPlanner>();
   }
 
   static bool match(AstContext* astCtx);

diff --git a/src/graph/planner/plan/PlanNode.cpp b/src/graph/planner/plan/PlanNode.cpp
@@ -370,6 +370,15 @@ void PlanNode::releaseSymbols() {
   }
 }
 
+void PlanNode::updateSymbols() {
+  auto symTbl = qctx_->symTable();
+  for (auto out : outputVars_) {
+    if (out != nullptr) {
+      symTbl->updateWrittenBy(out->name, out->name, this);
+    }
+  }
+}
+
 std::ostream& operator<<(std::ostream& os, PlanNode::Kind kind) {
   os << PlanNode::toString(kind);
   return os;

diff --git a/src/graph/planner/plan/PlanNode.h b/src/graph/planner/plan/PlanNode.h
@@ -269,6 +269,8 @@ class PlanNode {
 
   void releaseSymbols();
 
+  void updateSymbols();
+
   static const char* toString(Kind kind);
   std::string toString() const;
 

diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp
@@ -56,7 +56,7 @@ DEFINE_bool(enable_optimizer, false, "Whether to enable optimizer");
 DEFINE_uint32(ft_request_retry_times, 3, "Retry times if fulltext request failed");
 DEFINE_bool(enable_client_white_list, true, "Turn on/off the client white list.");
 DEFINE_string(client_white_list,
-              nebula::getOriginVersion() + ":2.5.0:2.5.1:2.6.0",
+              nebula::getOriginVersion() + ":3.0.0",
               "A white list for different client versions, separate with colon.");
 
 #endif

diff --git a/src/graph/service/test/StandAloneTestGraphFlags.cpp b/src/graph/service/test/StandAloneTestGraphFlags.cpp
@@ -9,5 +9,5 @@
 DEFINE_uint32(ft_request_retry_times, 3, "Retry times if fulltext request failed");
 DEFINE_bool(enable_client_white_list, true, "Turn on/off the client white list.");
 DEFINE_string(client_white_list,
-              nebula::getOriginVersion() + ":2.5.0:2.5.1:2.6.0",
+              nebula::getOriginVersion() + ":3.0.0",
               "A white list for different client versions, separate with colon.");
diff --git a/src/graph/util/ExpressionUtils.cpp b/src/graph/util/ExpressionUtils.cpp
@@ -126,18 +126,24 @@ bool ExpressionUtils::isEvaluableExpr(const Expression *expr, const QueryContext
 }
 
 // rewrite Attribute to LabelTagProp
-Expression *ExpressionUtils::rewriteAttr2LabelTagProp(const Expression *expr) {
+Expression *ExpressionUtils::rewriteAttr2LabelTagProp(
+    const Expression *expr, const std::unordered_map<std::string, AliasType> &aliasTypeMap) {
   ObjectPool *pool = expr->getObjPool();
 
-  auto matcher = [](const Expression *e) -> bool {
-    if (e->kind() == Expression::Kind::kAttribute) {
-      auto attrExpr = static_cast<const AttributeExpression *>(e);
-      if (attrExpr->left()->kind() == Expression::Kind::kLabelAttribute &&
-          attrExpr->right()->kind() == Expression::Kind::kConstant) {
-        return true;
-      }
+  auto matcher = [&aliasTypeMap](const Expression *e) -> bool {
+    if (e->kind() != Expression::Kind::kAttribute) {
+      return false;
     }
-    return false;
+    auto attrExpr = static_cast<const AttributeExpression *>(e);
+    if (attrExpr->left()->kind() != Expression::Kind::kLabelAttribute) {
+      return false;
+    }
+    auto label = static_cast<const LabelAttributeExpression *>(attrExpr->left())->left()->name();
+    auto iter = aliasTypeMap.find(label);
+    if (iter == aliasTypeMap.end() || iter->second != AliasType::kNode) {
+      return false;
+    }
+    return true;
   };
 
   auto rewriter = [pool](const Expression *e) -> Expression * {

diff --git a/src/graph/util/ExpressionUtils.h b/src/graph/util/ExpressionUtils.h
@@ -15,14 +15,14 @@
 #include "common/expression/PropertyExpression.h"
 #include "common/expression/TypeCastingExpression.h"
 #include "common/expression/UnaryExpression.h"
+#include "graph/context/ast/CypherAstContext.h"
 #include "graph/visitor/EvaluableExprVisitor.h"
 #include "graph/visitor/FindVisitor.h"
 #include "graph/visitor/RewriteVisitor.h"
 
 namespace nebula {
 class ObjectPool;
 namespace graph {
-
 class ExpressionUtils {
  public:
   explicit ExpressionUtils(...) = delete;
@@ -55,7 +55,8 @@ class ExpressionUtils {
 
   static bool isEvaluableExpr(const Expression* expr, const QueryContext* qctx = nullptr);
 
-  static Expression* rewriteAttr2LabelTagProp(const Expression* expr);
+  static Expression* rewriteAttr2LabelTagProp(
+      const Expression* expr, const std::unordered_map<std::string, AliasType>& aliasTypeMap);
 
   static Expression* rewriteLabelAttr2TagProp(const Expression* expr);
 

diff --git a/src/graph/validator/AdminJobValidator.cpp b/src/graph/validator/AdminJobValidator.cpp
@@ -11,6 +11,10 @@ namespace nebula {
 namespace graph {
 
 Status AdminJobValidator::validateImpl() {
+  if (sentence_->getCmd() == meta::cpp2::AdminCmd::DATA_BALANCE ||
+      sentence_->getCmd() == meta::cpp2::AdminCmd::ZONE_BALANCE) {
+    return Status::SemanticError("Data balance not support");
+  }
   if (sentence_->getOp() == meta::cpp2::AdminJobOp::ADD) {
     auto cmd = sentence_->getCmd();
     if (requireSpace()) {

diff --git a/src/graph/validator/MatchValidator.cpp b/src/graph/validator/MatchValidator.cpp
@@ -273,7 +273,8 @@ Status MatchValidator::validateFilter(const Expression *filter,
   auto transformRes = ExpressionUtils::filterTransform(filter);
   NG_RETURN_IF_ERROR(transformRes);
   // rewrite Attribute to LabelTagProperty
-  whereClauseCtx.filter = ExpressionUtils::rewriteAttr2LabelTagProp(transformRes.value());
+  whereClauseCtx.filter = ExpressionUtils::rewriteAttr2LabelTagProp(
+      transformRes.value(), whereClauseCtx.aliasesAvailable);
 
   auto typeStatus = deduceExprType(whereClauseCtx.filter);
   NG_RETURN_IF_ERROR(typeStatus);
@@ -383,7 +384,8 @@ Status MatchValidator::validateReturn(MatchReturn *ret,
           ExpressionUtils::hasAny(column->expr(), {Expression::Kind::kAggregate})) {
         retClauseCtx.yield->hasAgg_ = true;
       }
-      column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(column->expr()));
+      column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(
+          column->expr(), retClauseCtx.yield->aliasesAvailable));
       exprs.push_back(column->expr());
       columns->addColumn(column->clone().release());
     }
@@ -459,7 +461,8 @@ Status MatchValidator::validateWith(const WithClause *with,
   }
   if (with->returnItems()->columns()) {
     for (auto *column : with->returnItems()->columns()->columns()) {
-      column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(column->expr()));
+      column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(
+          column->expr(), withClauseCtx.yield->aliasesAvailable));
       columns->addColumn(column->clone().release());
     }
   }
@@ -819,7 +822,7 @@ Status MatchValidator::checkAlias(
       auto name = static_cast<const VariablePropertyExpression *>(labelExpr)->prop();
       auto res = getAliasType(aliasesAvailable, name);
       NG_RETURN_IF_ERROR(res);
-      if (res.value() != AliasType::kNode) {
+      if (res.value() == AliasType::kEdge || res.value() == AliasType::kPath) {
         return Status::SemanticError("The type of `%s' should be tag", name.c_str());
       }
       return Status::OK();

diff --git a/src/graph/visitor/RewriteVisitor.cpp b/src/graph/visitor/RewriteVisitor.cpp
@@ -237,6 +237,18 @@ void RewriteVisitor::visit(PathBuildExpression *expr) {
   }
 }
 
+void RewriteVisitor::visit(LabelTagPropertyExpression *expr) {
+  if (!care(expr->kind())) {
+    return;
+  }
+  auto label = expr->label();
+  if (matcher_(label)) {
+    expr->setLabel(rewriter_(label));
+  } else {
+    label->accept(this);
+  }
+}
+
 void RewriteVisitor::visit(AttributeExpression *expr) {
   if (!care(expr->kind())) {
     return;

diff --git a/src/graph/visitor/RewriteVisitor.h b/src/graph/visitor/RewriteVisitor.h
@@ -67,12 +67,12 @@ class RewriteVisitor final : public ExprVisitorImpl {
   void visit(RelationalExpression*) override;
   void visit(SubscriptExpression*) override;
   void visit(PathBuildExpression*) override;
+  void visit(LabelTagPropertyExpression*) override;
   void visit(SubscriptRangeExpression*) override;
   void visit(ConstantExpression*) override {}
   void visit(LabelExpression*) override {}
   void visit(UUIDExpression*) override {}
   void visit(LabelAttributeExpression*) override {}
-  void visit(LabelTagPropertyExpression*) override {}
   void visit(VariableExpression*) override {}
   void visit(VersionedVariableExpression*) override {}
   void visit(TagPropertyExpression*) override {}

diff --git a/src/interface/common.thrift b/src/interface/common.thrift
@@ -10,7 +10,7 @@ namespace java com.vesoft.nebula
 namespace go nebula
 namespace js nebula
 namespace csharp nebula
-namespace py nebula2.common
+namespace py nebula3.common
 
 cpp_include "common/thrift/ThriftTypes.h"
 cpp_include "common/datatypes/DateOps-inl.h"
@@ -34,7 +34,7 @@ cpp_include "common/datatypes/DurationOps-inl.h"
  *
  */
 
-const binary (cpp.type = "char const *") version = "2.6.0"
+const binary (cpp.type = "char const *") version = "3.0.0"
 
 typedef i64 (cpp.type = "nebula::ClusterID") ClusterID
 typedef i32 (cpp.type = "nebula::GraphSpaceID") GraphSpaceID

diff --git a/src/interface/graph.thrift b/src/interface/graph.thrift
@@ -9,7 +9,7 @@ namespace java com.vesoft.nebula.graph
 namespace go nebula.graph
 namespace js nebula.graph
 namespace csharp nebula.graph
-namespace py nebula2.graph
+namespace py nebula3.graph
 
 include "common.thrift"
 

diff --git a/src/interface/meta.thrift b/src/interface/meta.thrift
@@ -8,7 +8,7 @@ namespace java com.vesoft.nebula.meta
 namespace go nebula.meta
 namespace js nebula.meta
 namespace csharp nebula.meta
-namespace py nebula2.meta
+namespace py nebula3.meta
 
 include "common.thrift"
 

diff --git a/src/interface/storage.thrift b/src/interface/storage.thrift
@@ -9,7 +9,7 @@ namespace java com.vesoft.nebula.storage
 namespace go nebula.storage
 namespace csharp nebula.storage
 namespace js nebula.storage
-namespace py nebula2.storage
+namespace py nebula3.storage
 
 include "common.thrift"
 include "meta.thrift"

diff --git a/src/kvstore/KVEngine.h b/src/kvstore/KVEngine.h
@@ -49,6 +49,18 @@ class KVEngine {
                                                    bool sync,
                                                    bool wait) = 0;
 
+  /**
+   * @brief Get the Snapshot from kv engine.
+   *
+   * @return const void* snapshot pointer.
+   */
+  virtual const void* GetSnapshot() = 0;
+  /**
+   * @brief Release snapshot from kv engine.
+   *
+   * @param snapshot
+   */
+  virtual void ReleaseSnapshot(const void* snapshot) = 0;
   // Read a single key
   virtual nebula::cpp2::ErrorCode get(const std::string& key, std::string* value) = 0;
 
@@ -62,9 +74,17 @@ class KVEngine {
                                         const std::string& end,
                                         std::unique_ptr<KVIterator>* iter) = 0;
 
-  // Get all results with 'prefix' str as prefix.
+  /**
+   * @brief Get all results with 'prefix' str as prefix.
+   *
+   * @param prefix Prefix string.
+   * @param snapshot Snapshot from kv engine. nullptr means no snapshot.
+   * @param iter Iterator for this prefix range.
+   * @return nebula::cpp2::ErrorCode
+   */
   virtual nebula::cpp2::ErrorCode prefix(const std::string& prefix,
-                                         std::unique_ptr<KVIterator>* iter) = 0;
+                                         std::unique_ptr<KVIterator>* iter,
+                                         const void* snapshot = nullptr) = 0;
 
   // Get all results with 'prefix' str as prefix starting form 'start'
   virtual nebula::cpp2::ErrorCode rangeWithPrefix(const std::string& start,