Support breakpad (#2536)

* Introduce the breakpad.

* Setup breakpad.

* Introduce breakpad to meta and storage.

* pkg support dump syms.

* Forbid arm build with breakpad.

* Update package script.

* Update cmake macros.

* Update release.yaml

* Fix release.

* Address comment.

* Fix _strip_unnecessary_binaries.

* Using self-hosted runner and fix tag-name action.

* Fix typo.

.github/actions/tagname-action/action.yml

@@ -7,6 +7,9 @@ outputs:
   tagnum:
     description: 'tag number'
     value: ${{ steps.tag.outputs.tagnum }}
+  majorver:
+    description: 'major version'
+    value: ${{ steps.tag.outputs.majorver }}
 runs:
   using: "composite"
   steps:

.github/workflows/release.yml

@@ -12,28 +12,28 @@ defaults:
 jobs:
   package:
     name: build package
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     strategy:
       fail-fast: false
       matrix:
         os:
           - ubuntu1604
           - ubuntu1804
           - ubuntu2004
-          # - centos6
           - centos7
           - centos8
     container:
       image: vesoft/nebula-dev:${{ matrix.os }}
     env:
       BUILD_DIR: ./pkg-build
       CPACK_DIR: ./pkg-build/cpack_output
+      SYMS_DIR: ./pkg-build/symbols
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/tagname-action
         id: tag
       - name: package
-        run: ./package/package.sh -b ${{ steps.tag.outputs.tag }}
+        run: ./package/package.sh -b ${{ steps.tag.outputs.tag }} -t RelWithDebInfo -r OFF -p ON -s TRUE
       - name: output some vars
         run: |
           tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
@@ -46,6 +46,14 @@ jobs:
           bucket: nebula-graph
           asset-path: ${{ env.CPACK_DIR }}
           target-path: package/${{ steps.tag.outputs.tagnum }}
+      - uses: ./.github/actions/upload-to-oss-action
+        with:
+          key-id: ${{ secrets.OSS_ID }}
+          key-secret: ${{ secrets.OSS_SECRET }}
+          endpoint: ${{ secrets.OSS_ENDPOINT }}
+          bucket: nebula-graph
+          asset-path: ${{ env.SYMS_DIR }}
+          target-path: package/${{ steps.tag.outputs.tagnum }}/symbols
 
   docker_build:
     name: docker-build

cmake/FindBreakpad.cmake

@@ -0,0 +1,33 @@
+# - Try to find Breakpad includes dirs and libraries
+#
+# Usage of this module as follows:
+#
+#     find_package(Breakpad)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+# Variables defined by this module:
+#
+#  Breakpad_FOUND            System has breakpad, include and lib dirs found
+#  Breakpad_INCLUDE_DIR      The breakpad includes directories.
+#  Breakpad_LIBRARY          The breakpad library.
+
+find_path(Breakpad_INCLUDE_DIR NAMES breakpad)
+find_library(Breakpad_LIBRARY NAMES libbreakpad_client.a)
+
+message(STATUS "BREAKPAD: ${Breakpad_INCLUDE_DIR}")
+
+if(Breakpad_INCLUDE_DIR AND Breakpad_LIBRARY)
+    set(Breakpad_FOUND TRUE)
+    mark_as_advanced(
+        Breakpad_INCLUDE_DIR
+        Breakpad_LIBRARY
+    )
+endif()
+
+if(NOT Breakpad_FOUND)
+    message(FATAL_ERROR "Breakpad doesn't exist")
+endif()
+
+

cmake/nebula/GeneralCompilerConfig.cmake

@@ -21,7 +21,10 @@ add_compile_options(-Wnon-virtual-dtor)
 add_compile_options(-Woverloaded-virtual)
 add_compile_options(-Wignored-qualifiers)
 
+# For s2
 add_definitions(-DS2_USE_GLOG)
+# For breakpad
+add_definitions(-D__STDC_FORMAT_MACROS)
 
 include_directories(AFTER ${CMAKE_SOURCE_DIR}/src)
 include_directories(AFTER ${CMAKE_CURRENT_BINARY_DIR}/src)

cmake/nebula/NebulaCMakeMacros.cmake

@@ -103,6 +103,7 @@ macro(nebula_link_libraries target)
         -pthread
         z
         ${COVERAGES}
+        ${Breakpad_LIBRARY}
     )
 endmacro(nebula_link_libraries)
 

cmake/nebula/ThirdPartyConfig.cmake

@@ -106,6 +106,9 @@ find_package(FLEX REQUIRED)
 find_package(LibLZMA REQUIRED)
 find_package(Fizz REQUIRED)
 find_package(Sodium REQUIRED)
+if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "x86_64")
+    find_package(Breakpad REQUIRED)
+endif()
 
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L ${NEBULA_THIRDPARTY_ROOT}/lib")
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L ${NEBULA_THIRDPARTY_ROOT}/lib64")
@@ -150,4 +153,8 @@ else()
     set(JEMALLOC_LIB jemalloc)
 endif()
 
+if (Breakpad_FOUND)
+    include_directories(AFTER SYSTEM ${Breakpad_INCLUDE_DIR}/breakpad)
+endif()
+
 message(">>>> Configuring third party for '${PROJECT_NAME}' done <<<<")

docker/Dockerfile.graphd

@@ -5,7 +5,7 @@ COPY . /home/nebula/BUILD
 ARG BRANCH=master
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -g OFF
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
 
 FROM centos:7
 

docker/Dockerfile.metad

@@ -5,7 +5,7 @@ ARG BRANCH=master
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH}
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
 
 FROM centos:7
 

docker/Dockerfile.storaged

@@ -5,7 +5,7 @@ ARG BRANCH=master
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH}
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
 
 FROM centos:7
 

docker/Dockerfile.tools

@@ -5,7 +5,7 @@ ARG BRANCH=master
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH}
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
 
 FROM centos:7
 

package/package.sh

@@ -10,6 +10,8 @@
 #   -d: Whether to enable sanitizer, default OFF
 #   -t: Build type, default Release
 #   -j: Number of threads, default $(nproc)
+#   -r: Whether enable compressed debug info, default ON
+#   -p: Whether dump the symbols from binary by dump_syms
 #
 # usage: ./package.sh -v <version> -n <ON/OFF> -s <TRUE/FALSE> -b <BRANCH>
 #
@@ -19,16 +21,20 @@ set -e
 version=""
 package_one=ON
 strip_enable="FALSE"
-usage="Usage: ${0} -v <version> -n <ON/OFF> -s <TRUE/FALSE> -b <BRANCH> -g <ON/OFF> -j <jobs>"
+usage="Usage: ${0} -v <version> -n <ON/OFF> -s <TRUE/FALSE> -b <BRANCH> -g <ON/OFF> -j <jobs> -t <BUILD TYPE>"
 project_dir="$(cd "$(dirname "$0")" && pwd)/.."
 build_dir=${project_dir}/pkg-build
 enablesanitizer="OFF"
 static_sanitizer="OFF"
 build_type="Release"
 branch="master"
 jobs=$(nproc)
+enable_compressed_debug_info=ON
+dump_symbols=OFF
+dump_syms_tool_dir=
+system_name=
 
-while getopts v:n:s:b:d:t:j:g: opt;
+while getopts v:n:s:b:d:t:r:p:j: opt;
 do
     case $opt in
         v)
@@ -56,6 +62,12 @@ do
         j)
             jobs=$OPTARG
             ;;
+        r)
+            enable_compressed_debug_info=$OPTARG
+            ;;
+        p)
+            dump_symbols=$OPTARG
+            ;;
         ?)
             echo "Invalid option, use default arguments"
             ;;
@@ -80,7 +92,18 @@ if [[ $strip_enable != TRUE ]] && [[ $strip_enable != FALSE ]]; then
     exit 1
 fi
 
-echo "current version is [ $version ], strip enable is [$strip_enable], enablesanitizer is [$enablesanitizer], static_sanitizer is [$static_sanitizer]"
+cat << EOF
+Configuration for this shell:
+version: $version
+strip_enable: $strip_enable
+enablesanitizer: $enablesanitizer
+static_sanitizer: $static_sanitizer
+build_type: $build_type
+branch: $branch
+enable_compressed_debug_info: $enable_compressed_debug_info
+dump_symbols: $dump_symbols
+
+EOF
 
 function _build_graph {
     pushd ${build_dir}
@@ -93,6 +116,7 @@ function _build_graph {
           -DCMAKE_INSTALL_PREFIX=/usr/local/nebula \
           -DENABLE_TESTING=OFF \
           -DENABLE_PACK_ONE=${package_one} \
+          -DENABLE_COMPRESSED_DEBUG_INFO=${enable_compressed_debug_info} \
           ${project_dir}
 
     if ! ( make -j ${jobs} ); then
@@ -133,14 +157,59 @@ function package {
         mkdir -p ${outputDir}
         for pkg_name in $(ls ./*nebula*-${version}*); do
             mv ${pkg_name} ${outputDir}/
-            echo "####### taget package file is ${outputDir}/${pkg_name}"
+            echo "####### target package file is ${outputDir}/${pkg_name}"
         done
     fi
 
     popd
 }
 
+function _find_dump_syms_tool {
+    if [[ -x ${build_dir}/third-party/install/bin/dump_syms ]]; then
+        dump_syms_tool_dir=${build_dir}/third-party/install/bin
+    elif [[ -x /opt/vesoft/third-party/2.0/bin/dump_syms ]]; then
+        dump_syms_tool_dir=/opt/vesoft/third-party/2.0/bin
+    else
+        echo ">>> Failed to find the dump_syms tool <<<"
+        exit 1
+    fi
+}
+
+# This is only for releasing the disk resources.
+function _strip_unnecessary_binaries {
+    for bin in $(ls -1 -F ${build_dir}/bin/ | grep -v [/$] | sed -e '/nebula-metad/d;/nebula-graphd/d;/nebula-storaged/d'); do
+        if ! (strip ${build_dir}/bin/${bin}); then
+            echo ">>> strip ${bin} faild: $?. <<<"
+            exit 1
+        fi
+    done
+}
+
+function dump_syms {
+    _strip_unnecessary_binaries
+    _find_dump_syms_tool
+    dump_syms=${dump_syms_tool_dir}/dump_syms
+
+    syms_dir=${build_dir}/symbols/
+    rm -rf ${syms_dir} && mkdir -p ${syms_dir}
+
+    pack=`ls ${build_dir}/cpack_output/`
+    tmp=${pack#nebula-graph}
+    ver=${tmp%.*}
+
+    for bin in nebula-graphd nebula-storaged nebula-metad; do
+        if ! (${dump_syms} ${build_dir}/bin/${bin} > ${syms_dir}/${bin}${ver}.sym); then
+            echo ">>> dump ${bin} symbols faild: $?. <<<"
+            exit 1
+        fi
+    done
+}
 
 # The main
 build $version $enablesanitizer $static_sanitizer $build_type $branch
+rm -rf ${build_dir}/src
 package $strip_enable
+if [[ $dump_symbols == ON ]]; then
+    echo ">>> start dump symbols <<<"
+    dump_syms
+fi

src/daemons/CMakeLists.txt

@@ -50,6 +50,7 @@ nebula_add_executable(
     SOURCES
         StorageDaemon.cpp
         SetupLogging.cpp
+        SetupBreakpad.cpp
     OBJECTS
         $<TARGET_OBJECTS:storage_server>
         $<TARGET_OBJECTS:internal_storage_service_handler>
@@ -76,6 +77,7 @@ nebula_add_executable(
     SOURCES
         MetaDaemon.cpp
         SetupLogging.cpp
+        SetupBreakpad.cpp
     OBJECTS
         $<TARGET_OBJECTS:meta_service_handler>
         $<TARGET_OBJECTS:meta_http_handler>
@@ -97,6 +99,7 @@ nebula_add_executable(
     SOURCES
         GraphDaemon.cpp
         SetupLogging.cpp
+        SetupBreakpad.cpp
     OBJECTS
         $<TARGET_OBJECTS:stats_def_obj>
         $<TARGET_OBJECTS:util_obj>

src/daemons/GraphDaemon.cpp

@@ -12,7 +12,6 @@
 
 #include "common/base/Base.h"
 #include "common/base/SignalHandler.h"
-#include "common/base/Status.h"
 #include "common/fs/FileUtils.h"
 #include "common/network/NetworkUtils.h"
 #include "common/process/ProcessUtils.h"
@@ -26,6 +25,7 @@
 
 using nebula::ProcessUtils;
 using nebula::Status;
+using nebula::StatusOr;
 using nebula::fs::FileUtils;
 using nebula::graph::GraphService;
 using nebula::network::NetworkUtils;
@@ -37,6 +37,9 @@ static Status setupSignalHandler();
 extern Status setupLogging();
 static void printHelp(const char *prog);
 static void setupThreadManager();
+#if defined(__x86_64__)
+extern Status setupBreakpad();
+#endif
 
 DECLARE_string(flagfile);
 
@@ -71,6 +74,14 @@ int main(int argc, char *argv[]) {
     return EXIT_FAILURE;
   }
 
+#if defined(__x86_64__)
+  status = setupBreakpad();
+  if (!status.ok()) {
+    LOG(ERROR) << status;
+    return EXIT_FAILURE;
+  }
+#endif
+
   // Detect if the server has already been started
   auto pidPath = FLAGS_pid_file;
   status = ProcessUtils::isPidAvailable(pidPath);