Feature/arena allocator

src/common/base/Arena.cpp

@@ -0,0 +1,46 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#include "common/base/Arena.h"
+
+#include <cstdint>
+
+namespace nebula {
+
+void* Arena::allocateAligned(const std::size_t alloc) {
+  DCHECK_NE(alloc, 0);  // don't allow zero sized allocation
+  // replace the modulo operation by bit and
+  static_assert(kAlignment && !(kAlignment & (kAlignment - 1)), "Align must be power of 2.");
+  const std::size_t pad =
+      kAlignment - (reinterpret_cast<uintptr_t>(currentPtr_) & (kAlignment - 1));
+  const std::size_t consumption = alloc + pad;
+  if (UNLIKELY(consumption > kMaxChunkSize)) {
+    DLOG(FATAL) << "Arena can't allocate so large memory.";
+    return nullptr;
+  }
+  if (LIKELY(consumption <= availableSize_)) {
+    void* ptr = currentPtr_ + pad;
+    currentPtr_ += consumption;
+#ifndef NDEBUG
+    allocatedSize_ += consumption;
+#endif
+    availableSize_ -= consumption;
+    return ptr;
+  } else {
+    newChunk(std::max(alloc, kMinChunkSize));
+    // The new operator will allocate the aligned memory
+    DCHECK_EQ(reinterpret_cast<uintptr_t>(currentPtr_) & (kAlignment - 1), 0);
+    void* ptr = currentPtr_;
+    currentPtr_ += alloc;
+#ifndef NDEBUG
+    allocatedSize_ += alloc;
+#endif
+    availableSize_ -= alloc;
+    return ptr;
+  }
+}
+
+}  // namespace nebula

src/common/base/Arena.h

@@ -0,0 +1,92 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#pragma once
+
+#include <folly/Likely.h>
+
+#include <boost/core/noncopyable.hpp>
+#include <cstddef>
+#include <limits>
+#include <type_traits>
+
+#include "common/base/Logging.h"
+#include "common/cpp/helpers.h"
+
+namespace nebula {
+
+// MT-unsafe arena allocator
+// It's optimized for many small objects construct/destruct
+class Arena : public boost::noncopyable, cpp::NonMovable {
+ public:
+  ~Arena() {
+    while (LIKELY(currentChunk_ != nullptr)) {
+      auto *prev = currentChunk_->prev;
+      delete[] currentChunk_;
+      currentChunk_ = prev;
+    }
+#ifndef NDEBUG
+    allocatedSize_ = 0;
+#endif
+    availableSize_ = 0;
+    currentPtr_ = nullptr;
+  }
+
+  // The CPU access memory with the alignment,
+  // So construct object from alignment address will reduce the CPU access count then
+  // speed up read/write
+  void *allocateAligned(const std::size_t alloc);
+
+#ifndef NDEBUG
+  std::size_t allocatedSize() const {
+    return allocatedSize_;
+  }
+#endif
+
+  std::size_t availableSize() const {
+    return availableSize_;
+  }
+
+ private:
+  static constexpr std::size_t kMinChunkSize = 4096;
+  static constexpr std::size_t kMaxChunkSize = std::numeric_limits<uint16_t>::max();
+  static constexpr std::size_t kAlignment = std::alignment_of<std::max_align_t>::value;
+
+  struct Chunk {
+    explicit Chunk(Chunk *p) : prev{p} {}
+
+    union {
+      Chunk *prev{nullptr};
+      std::byte aligned[kAlignment];
+    };
+  };
+
+  // allocate new chunk
+  // The current pointer will keep alignment
+  void newChunk(std::size_t size) {
+    DCHECK_NE(size, 0);
+    std::byte *ptr = new std::byte[size + sizeof(Chunk)];
+    currentChunk_ = new (ptr) Chunk(currentChunk_);
+    availableSize_ = size;
+    currentPtr_ = (ptr + sizeof(Chunk));
+  }
+
+  Chunk *currentChunk_{nullptr};
+// These are debug info
+// Remove to speed up in Release build
+#ifndef NDEBUG
+  // total size allocated
+  std::size_t allocatedSize_{0};
+#endif
+  // total size which available to allocate
+  std::size_t availableSize_{0};
+  // The total chunks size
+  // = allocatedSize_ + availableSize_ + Memory Deprecated (Size can't fit allocation)
+  // Current pointer to available memory address
+  std::byte *currentPtr_{nullptr};
+};
+
+}  // namespace nebula

src/common/base/CMakeLists.txt

@@ -13,6 +13,7 @@ nebula_add_library(
     Status.cpp
     SanitizerOptions.cpp
     SignalHandler.cpp
+    Arena.cpp
     ${gdb_debug_script}
 )
 

src/common/base/ObjectPool.h

@@ -13,6 +13,7 @@
 #include <list>
 #include <type_traits>
 
+#include "common/base/Arena.h"
 #include "common/base/Logging.h"
 #include "common/cpp/helpers.h"
 
@@ -26,26 +27,19 @@ class ObjectPool final : private boost::noncopyable, private cpp::NonMovable {
  public:
   ObjectPool() {}
 
-  ~ObjectPool() = default;
+  ~ObjectPool() {
+    clear();
+  }
 
   void clear() {
     SLGuard g(lock_);
     objects_.clear();
   }
 
-  template <typename T>
-  T *add(T *obj) {
-    if constexpr (std::is_base_of<Expression, T>::value) {
-      VLOG(3) << "New expression added into pool: " << obj->toString();
-    }
-    SLGuard g(lock_);
-    objects_.emplace_back(obj);
-    return obj;
-  }
-
   template <typename T, typename... Args>
   T *makeAndAdd(Args &&... args) {
-    return add(new T(std::forward<Args>(args)...));
+    void *ptr = arena_.allocateAligned(sizeof(T));
+    return add(new (ptr) T(std::forward<Args>(args)...));
   }
 
   bool empty() const {
@@ -58,7 +52,7 @@ class ObjectPool final : private boost::noncopyable, private cpp::NonMovable {
    public:
     template <typename T>
     explicit OwnershipHolder(T *obj)
-        : obj_(obj), deleteFn_([](void *p) { delete reinterpret_cast<T *>(p); }) {}
+        : obj_(obj), deleteFn_([](void *p) { reinterpret_cast<T *>(p)->~T(); }) {}
 
     ~OwnershipHolder() {
       deleteFn_(obj_);
@@ -69,7 +63,18 @@ class ObjectPool final : private boost::noncopyable, private cpp::NonMovable {
     std::function<void(void *)> deleteFn_;
   };
 
+  template <typename T>
+  T *add(T *obj) {
+    if constexpr (std::is_base_of<Expression, T>::value) {
+      VLOG(3) << "New expression added into pool: " << obj->toString();
+    }
+    SLGuard g(lock_);
+    objects_.emplace_back(obj);
+    return obj;
+  }
+
   std::list<OwnershipHolder> objects_;
+  Arena arena_;
 
   folly::SpinLock lock_;
 };

src/common/base/test/ArenaBenchmark.cpp

@@ -0,0 +1,90 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#include <folly/Benchmark.h>
+#include <folly/init/Init.h>
+#include <folly/memory/Arena.h>
+
+#include <string>
+#include <type_traits>
+
+#include "common/base/Arena.h"
+#include "common/expression/LabelExpression.h"
+
+namespace nebula {
+
+class TestExpr : public LabelExpression {
+ public:
+  explicit TestExpr(const std::string &name = "")
+      : LabelExpression(reinterpret_cast<ObjectPool *>(1), name) {}
+};
+
+BENCHMARK(DefaultAllocator, iters) {
+  std::size_t round = iters * 1000;
+  for (std::size_t _ = 0; _ < round; ++_) {
+    auto *expr = new TestExpr("Label");
+    delete expr;
+  }
+}
+
+BENCHMARK_RELATIVE(ArenaAllocator, iters) {
+  std::size_t round = iters * 1000;
+  Arena a;
+  for (std::size_t _ = 0; _ < round; ++_) {
+    auto *ptr = a.allocateAligned(sizeof(TestExpr));
+    auto *expr = new (ptr) TestExpr("Label");
+    expr->~TestExpr();
+  }
+}
+
+BENCHMARK_RELATIVE(FollyArenaAllocator, iters) {
+  std::size_t round = iters * 1000;
+  folly::SysArena a;
+  for (std::size_t _ = 0; _ < round; ++_) {
+    auto *ptr = a.allocate(sizeof(TestExpr));
+    auto *expr = new (ptr) TestExpr("Label");
+    expr->~TestExpr();
+  }
+}
+
+BENCHMARK_DRAW_LINE();
+
+}  // namespace nebula
+
+int main(int argc, char **argv) {
+  folly::init(&argc, &argv, true);
+
+  folly::runBenchmarks();
+  return 0;
+}
+
+// CPU info
+// Brand Raw: Intel(R) Xeon(R) CPU E5-2690 v2 @ 3.00GHz
+// Hz Advertised Friendly: 3.0000 GHz
+// Hz Actual Friendly: 3.2942 GHz
+// Hz Advertised: (3000000000, 0)
+// Hz Actual: (3294220000, 0)
+// Arch: X86_64
+// Bits: 64
+// Count: 40
+// Arch String Raw: x86_64
+// L1 Data Cache Size: 32768
+// L1 Instruction Cache Size: 32768
+// L2 Cache Size: 262144
+// L2 Cache Line Size: 256
+// L2 Cache Associativity: 6
+// L3 Cache Size: 26214400
+//
+// Build in Release mode
+//
+// ============================================================================
+// /home/shylock.huang/nebula/src/common/base/test/ArenaBenchmark.cpprelative  time/iter  iters/s
+// ============================================================================
+// DefaultAllocator                                            36.59us   27.33K
+// ArenaAllocator                                   145.89%    25.08us   39.87K
+// FollyArenaAllocator                              138.96%    26.33us   37.98K
+// ----------------------------------------------------------------------------
+// ============================================================================

src/common/base/test/ArenaTest.cpp

@@ -0,0 +1,46 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#include <gtest/gtest.h>
+
+#include <type_traits>
+
+#include "common/base/Arena.h"
+
+namespace nebula {
+
+TEST(ArenaTest, Basic) {
+  Arena a;
+
+  for (int i = 1; i < 4096; i += 8) {
+    void *ptr = a.allocateAligned(i);
+    EXPECT_EQ(reinterpret_cast<uintptr_t>(ptr) % std::alignment_of<std::max_align_t>::value, 0);
+  }
+}
+
+TEST(ArenaTest, Construct) {
+  Arena a;
+  {
+    void *ptr = a.allocateAligned(sizeof(std::string));
+    auto *obj = new (ptr) std::string("Hello World!");
+    EXPECT_EQ(*obj, "Hello World!");
+    obj->~basic_string();
+  }
+  {
+    void *ptr = a.allocateAligned(sizeof(int));
+    auto *obj = new (ptr) int(3);  // NOLINT
+    EXPECT_EQ(*obj, 3);
+  }
+  {
+    for (std::size_t i = 0; i < 1024; ++i) {
+      void *ptr = a.allocateAligned(sizeof(int));
+      auto *obj = new (ptr) int(i);  // NOLINT
+      EXPECT_EQ(*obj, i);
+    }
+  }
+}
+
+}  // namespace nebula

src/common/base/test/CMakeLists.txt

@@ -97,5 +97,35 @@ target_compile_options(range_vs_transform_bm PRIVATE -O3)
 nebula_add_test(
     NAME object_pool_test
     SOURCES ObjectPoolTest.cpp
+    OBJECTS $<TARGET_OBJECTS:base_obj>
     LIBRARIES gtest gtest_main
 )
+
+nebula_add_executable(
+    NAME arena_bm
+    SOURCES ArenaBenchmark.cpp
+    OBJECTS
+        $<TARGET_OBJECTS:base_obj>
+        $<TARGET_OBJECTS:datatypes_obj>
+        $<TARGET_OBJECTS:expression_obj>
+        $<TARGET_OBJECTS:function_manager_obj>
+        $<TARGET_OBJECTS:agg_function_manager_obj>
+        $<TARGET_OBJECTS:time_obj>
+        $<TARGET_OBJECTS:time_utils_obj>
+        $<TARGET_OBJECTS:ast_match_path_obj>
+        $<TARGET_OBJECTS:wkt_wkb_io_obj>
+        $<TARGET_OBJECTS:datetime_parser_obj>
+    LIBRARIES
+        follybenchmark
+        ${THRIFT_LIBRARIES}
+)
+
+nebula_add_test(
+    NAME arena_test
+    SOURCES ArenaTest.cpp
+    OBJECTS
+        $<TARGET_OBJECTS:base_obj>
+    LIBRARIES
+        gtest
+        gtest_main
+)