Merge branch 'master' into format_subgraph

.github/workflows/pull_request.yml

@@ -6,6 +6,11 @@ on:
     branches:
       - master
       - 'v[0-9]+.*'
+    paths-ignore:
+      - 'docs/**'
+      - 'LICENSES/**'
+      - 'LICENSE'
+      - '**.md'
 
 defaults:
   run:
@@ -121,20 +126,26 @@ jobs:
         run: ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure
         working-directory: build/
         timeout-minutes: 20
-      - name: Pytest
+      - name: Setup cluster
         run: |
           make up
+        working-directory: tests/
+        timeout-minutes: 2
+      - name: Pytest
+        run: |
           make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} test
-          make down
         working-directory: tests/
         timeout-minutes: 15
       - name: TCK
         run: |
-          make up
           make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} tck
-          make down
         working-directory: tests/
         timeout-minutes: 30
+      - name: Down cluster
+        run: |
+          make down
+        working-directory: tests/
+        timeout-minutes: 2
       - name: Sanitizer
         if: ${{ always() }}
         run: |

.gitignore

@@ -51,6 +51,7 @@ cmake-build*/
 .cache/
 core.*
 workspace.*
+.metals/
 
 #py
 *.egg-info

cmake/CPackage.cmake

@@ -59,14 +59,15 @@ macro(package to_one name home_page scripts_dir)
             OUTPUT_STRIP_TRAILING_WHITESPACE
         )
         string(REPLACE "." "" HOST_SYSTEM_VER ${HOST_SYSTEM_VER})
-        string(CONCAT HOST_SYSTEM_VER ${HOST_SYSTEM_NAME} ${HOST_SYSTEM_VER})
         if (${HOST_SYSTEM_NAME} MATCHES "Ubuntu")
+            string(CONCAT HOST_SYSTEM_VER "ubuntu" ${HOST_SYSTEM_VER})
             # the ubuntu need to modify the architecture name
             if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "x86_64")
                 set(CMAKE_HOST_SYSTEM_PROCESSOR "amd64")
             endif()
         # Adapt the Kylin system
         elseif (${HOST_SYSTEM_NAME} MATCHES "Kylin" AND ${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "aarch64")
+            string(CONCAT HOST_SYSTEM_VER ${HOST_SYSTEM_NAME} ${HOST_SYSTEM_VER})
             set(CMAKE_HOST_SYSTEM_PROCESSOR "arm64")
         endif()
     elseif (EXISTS "/etc/issue")

conf/nebula-storaged.conf.default

@@ -100,9 +100,3 @@
 --rocksdb_column_family_options={"write_buffer_size":"67108864","max_write_buffer_number":"4","max_bytes_for_level_base":"268435456"}
 # rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma
 --rocksdb_block_based_table_options={"block_size":"8192"}
-
-############## Runtime global Options ##############
-# Index rebuild batch number
---rebuild_index_batch_num=256
-# Index rebuild processing interval, microsecond.
---rebuild_index_process_interval=3000

conf/nebula-storaged.conf.production

@@ -100,10 +100,7 @@
 --enable_rocksdb_prefix_filtering=false
 
 ############### misc ####################
---max_handlers_per_req=1
-
-############## Runtime global Options ##############
-# Index rebuild batch number
---rebuild_index_batch_num=256
-# Index rebuild processing interval, microsecond.
---rebuild_index_process_interval=3000
+--snapshot_part_rate_limit=8388608
+--snapshot_batch_size=1048576
+--rebuild_index_part_rate_limit=4194304
+--rebuild_index_batch_size=1048576

src/clients/meta/MetaClient.cpp

@@ -20,6 +20,7 @@
 #include "common/meta/NebulaSchemaProvider.h"
 #include "common/network/NetworkUtils.h"
 #include "common/stats/StatsManager.h"
+#include "common/time/TimeUtils.h"
 #include "version/Version.h"
 #include "webservice/Common.h"
 
@@ -673,6 +674,10 @@ Status MetaClient::handleResponse(const RESP& resp) {
       return Status::Error("Edge not existed!");
     case nebula::cpp2::ErrorCode::E_INDEX_NOT_FOUND:
       return Status::Error("Index not existed!");
+    case nebula::cpp2::ErrorCode::E_STATS_NOT_FOUND:
+      return Status::Error(
+          "There is no any stats info to show, please execute "
+          "`submit job stats' firstly!");
     case nebula::cpp2::ErrorCode::E_EDGE_PROP_NOT_FOUND:
       return Status::Error("Edge prop not existed!");
     case nebula::cpp2::ErrorCode::E_TAG_PROP_NOT_FOUND:
@@ -2322,6 +2327,7 @@ folly::Future<StatusOr<bool>> MetaClient::heartbeat() {
             LOG(FATAL) << "Can't persist the clusterId in file " << FLAGS_cluster_id_path;
           }
         }
+        heartbeatTime_ = time::WallClock::fastNowInMilliSec();
         metadLastUpdateTime_ = resp.get_last_update_time_in_ms();
         VLOG(1) << "Metad last update time: " << metadLastUpdateTime_;
         return true;  // resp.code == nebula::cpp2::ErrorCode::SUCCEEDED
@@ -3323,7 +3329,7 @@ StatusOr<std::pair<std::string, cpp2::FTIndex>> MetaClient::getFTIndexBySpaceSch
   }
   folly::RWSpinLock::ReadHolder holder(localCacheLock_);
   for (auto it = fulltextIndexMap_.begin(); it != fulltextIndexMap_.end(); ++it) {
-    auto id = it->second.get_depend_schema().getType() == cpp2::SchemaID::Type::edge_type
+    auto id = it->second.get_depend_schema().getType() == nebula::cpp2::SchemaID::Type::edge_type
                   ? it->second.get_depend_schema().get_edge_type()
                   : it->second.get_depend_schema().get_tag_id();
     if (it->second.get_space_id() == spaceId && id == schemaId) {

src/clients/meta/MetaClient.h

@@ -599,6 +599,10 @@ class MetaClient {
 
   folly::Future<StatusOr<bool>> ingest(GraphSpaceID spaceId);
 
+  HostAddr getMetaLeader() { return leader_; }
+
+  int64_t HeartbeatTime() { return heartbeatTime_; }
+
  protected:
   // Return true if load succeeded.
   bool loadData();
@@ -738,6 +742,7 @@ class MetaClient {
   bool skipConfig_ = false;
   MetaClientOptions options_;
   std::vector<HostAddr> storageHosts_;
+  int64_t heartbeatTime_;
 };
 
 }  // namespace meta

src/common/thrift/ThriftClientManager-inl.h

@@ -86,7 +86,7 @@ std::shared_ptr<ClientType> ThriftClientManager<ClientType>::client(const HostAd
   std::shared_ptr<ClientType> client(
       new ClientType(std::move(headerClientChannel)),
       [evb](auto* p) { evb->runImmediatelyOrRunInEventBaseThreadAndWait([p] { delete p; }); });
-  clientMap_->emplace(std::make_pair(resolved, evb), client);
+  clientMap_->emplace(std::make_pair(host, evb), client);
   return client;
 }
 

src/graph/context/Iterator.cpp

@@ -309,6 +309,10 @@ const Value& GetNeighborsIter::getEdgeProp(const std::string& edge, const std::s
     return Value::kNullValue;
   }
 
+  if (noEdge_) {
+    return Value::kEmpty;
+  }
+
   auto& currentEdge = currentEdgeName();
   if (edge != "*" && (currentEdge.compare(1, std::string::npos, edge) != 0)) {
     VLOG(1) << "Current edge: " << currentEdgeName() << " Wanted: " << edge;
@@ -386,6 +390,10 @@ Value GetNeighborsIter::getEdge() const {
     return Value::kNullValue;
   }
 
+  if (noEdge_) {
+    return Value::kEmpty;
+  }
+
   Edge edge;
   auto edgeName = currentEdgeName().substr(1, std::string::npos);
   edge.name = edgeName;

src/graph/executor/CMakeLists.txt

@@ -57,6 +57,7 @@ nebula_add_library(
     admin/BalanceLeadersExecutor.cpp
     admin/ShowBalanceExecutor.cpp
     admin/ShowHostsExecutor.cpp
+    admin/ShowMetaLeaderExecutor.cpp
     admin/SpaceExecutor.cpp
     admin/SnapshotExecutor.cpp
     admin/ListenerExecutor.cpp

src/graph/executor/Executor.cpp

@@ -38,6 +38,7 @@
 #include "graph/executor/admin/SessionExecutor.h"
 #include "graph/executor/admin/ShowBalanceExecutor.h"
 #include "graph/executor/admin/ShowHostsExecutor.h"
+#include "graph/executor/admin/ShowMetaLeaderExecutor.h"
 #include "graph/executor/admin/ShowQueriesExecutor.h"
 #include "graph/executor/admin/ShowStatsExecutor.h"
 #include "graph/executor/admin/ShowTSClientsExecutor.h"
@@ -411,6 +412,9 @@ Executor *Executor::makeExecutor(QueryContext *qctx, const PlanNode *node) {
     case PlanNode::Kind::kShowHosts: {
       return pool->add(new ShowHostsExecutor(node, qctx));
     }
+    case PlanNode::Kind::kShowMetaLeader: {
+      return pool->add(new ShowMetaLeaderExecutor(node, qctx));
+    }
     case PlanNode::Kind::kShowParts: {
       return pool->add(new ShowPartsExecutor(node, qctx));
     }

src/graph/executor/admin/ShowMetaLeaderExecutor.cpp

@@ -0,0 +1,34 @@
+/* Copyright (c) 2020 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#include "graph/executor/admin/ShowMetaLeaderExecutor.h"
+
+#include <thrift/lib/cpp/util/EnumUtils.h>
+
+#include "common/time/TimeUtils.h"
+#include "graph/context/QueryContext.h"
+#include "graph/planner/plan/Admin.h"
+#include "graph/util/ScopedTimer.h"
+
+namespace nebula {
+namespace graph {
+
+folly::Future<Status> ShowMetaLeaderExecutor::execute() {
+  SCOPED_TIMER(&execTime_);
+  auto metaLeader = qctx()->getMetaClient()->getMetaLeader();
+  auto now = time::WallClock::fastNowInMilliSec();
+  auto lastHeartBeatTime = qctx()->getMetaClient()->HeartbeatTime();
+  auto intervalMs = now - lastHeartBeatTime;
+  DataSet ds({"Meta Leader", "secs from last heart beat"});
+  auto strLeader = folly::sformat("{}:{}", metaLeader.host, metaLeader.port);
+  nebula::Row r({std::move(strLeader), intervalMs / 1000});
+  ds.emplace_back(std::move(r));
+  finish(ds);
+  return Status::OK();
+}
+
+}  // namespace graph
+}  // namespace nebula

src/graph/executor/admin/ShowMetaLeaderExecutor.h

@@ -0,0 +1,23 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#pragma once
+
+#include "graph/executor/Executor.h"
+
+namespace nebula {
+namespace graph {
+
+class ShowMetaLeaderExecutor final : public Executor {
+ public:
+  ShowMetaLeaderExecutor(const PlanNode *node, QueryContext *qctx)
+      : Executor("ShowMetaLeaderExecutor", node, qctx) {}
+
+  folly::Future<Status> execute() override;
+};
+
+}  // namespace graph
+}  // namespace nebula

src/graph/executor/maintain/FTIndexExecutor.cpp

@@ -75,7 +75,7 @@ folly::Future<Status> ShowFTIndexesExecutor::execute() {
             continue;
           }
           auto shmId = index.second.get_depend_schema();
-          auto isEdge = shmId.getType() == meta::cpp2::SchemaID::Type::edge_type;
+          auto isEdge = shmId.getType() == nebula::cpp2::SchemaID::Type::edge_type;
           auto shmNameRet =
               isEdge ? this->qctx_->schemaMng()->toEdgeName(spaceId, shmId.get_edge_type())
                      : this->qctx_->schemaMng()->toTagName(spaceId, shmId.get_tag_id());

src/graph/planner/plan/Admin.h

@@ -65,6 +65,17 @@ class ShowHosts final : public SingleDependencyNode {
   meta::cpp2::ListHostType type_;
 };
 
+class ShowMetaLeaderNode final : public SingleDependencyNode {
+ public:
+  static ShowMetaLeaderNode* make(QueryContext* qctx, PlanNode* dep) {
+    return qctx->objPool()->add(new ShowMetaLeaderNode(qctx, dep));
+  }
+
+ private:
+  ShowMetaLeaderNode(QueryContext* qctx, PlanNode* dep)
+      : SingleDependencyNode(qctx, Kind::kShowMetaLeader, dep) {}
+};
+
 class CreateSpace final : public SingleDependencyNode {
  public:
   static CreateSpace* make(QueryContext* qctx,

src/graph/planner/plan/PlanNode.cpp

@@ -211,6 +211,8 @@ const char* PlanNode::toString(PlanNode::Kind kind) {
       return "UpdateEdge";
     case Kind::kShowHosts:
       return "ShowHosts";
+    case Kind::kShowMetaLeader:
+      return "ShowMetaLeader";
     case Kind::kShowParts:
       return "ShowParts";
     case Kind::kShowCharset:

src/graph/planner/plan/PlanNode.h

@@ -143,6 +143,7 @@ class PlanNode {
     kShowConfigs,
     kSetConfig,
     kGetConfig,
+    kShowMetaLeader,
 
     // zone related
     kShowGroups,

src/graph/service/PermissionCheck.cpp

@@ -164,6 +164,7 @@ Status PermissionCheck::permissionCheck(ClientSession *session,
     case Sentence::Kind::kShowCollation:
     case Sentence::Kind::kShowGroups:
     case Sentence::Kind::kShowZones:
+    case Sentence::Kind::kShowMetaLeader:
     case Sentence::Kind::kShowHosts: {
       /**
        * all roles can be show for above operations.

src/graph/validator/AdminValidator.cpp

@@ -298,6 +298,15 @@ Status ShowHostsValidator::toPlan() {
   return Status::OK();
 }
 
+Status ShowMetaLeaderValidator::validateImpl() { return Status::OK(); }
+
+Status ShowMetaLeaderValidator::toPlan() {
+  auto *node = ShowMetaLeaderNode::make(qctx_, nullptr);
+  root_ = node;
+  tail_ = root_;
+  return Status::OK();
+}
+
 Status ShowPartsValidator::validateImpl() { return Status::OK(); }
 
 Status ShowPartsValidator::toPlan() {

src/graph/validator/AdminValidator.h

@@ -165,6 +165,18 @@ class ShowHostsValidator final : public Validator {
   Status toPlan() override;
 };
 
+class ShowMetaLeaderValidator final : public Validator {
+ public:
+  ShowMetaLeaderValidator(Sentence* sentence, QueryContext* ctx) : Validator(sentence, ctx) {
+    setNoSpaceRequired();
+  }
+
+ private:
+  Status validateImpl() override;
+
+  Status toPlan() override;
+};
+
 class ShowPartsValidator final : public Validator {
  public:
   ShowPartsValidator(Sentence* sentence, QueryContext* context) : Validator(sentence, context) {}

src/graph/validator/MaintainValidator.cpp

@@ -611,7 +611,7 @@ Status CreateFTIndexValidator::validateImpl() {
                     ? qctx_->schemaMng()->toEdgeType(space.id, *sentence->schemaName())
                     : qctx_->schemaMng()->toTagID(space.id, *sentence->schemaName());
   NG_RETURN_IF_ERROR(status);
-  meta::cpp2::SchemaID id;
+  nebula::cpp2::SchemaID id;
   if (sentence->isEdge()) {
     id.set_edge_type(status.value());
   } else {