refactor: chain storage stub for validator node

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 
 defaults:
-  rust_image: &rust_image quay.io/tarilabs/rust_tari-build-with-deps:nightly-2021-08-17
+  rust_image: &rust_image quay.io/tarilabs/rust_tari-build-with-deps:nightly-2021-09-18
 
 commands:
   test:
@@ -76,13 +76,13 @@ commands:
       - run:
           name: Cargo fmt
           command: |
-            TOOLCHAIN=$(cat rust-toolchain)
+            TOOLCHAIN=$(awk '/channel = /{print $NF}' rust-toolchain.toml)
             rustup component add --toolchain $TOOLCHAIN rustfmt
             cargo fmt --all -- --check
       - run:
           name: Run clippy (main source)
           command: |
-            TOOLCHAIN=$(cat rust-toolchain)
+            TOOLCHAIN=$(awk '/channel = /{print $NF}' rust-toolchain.toml)
             rustup component add --toolchain $TOOLCHAIN clippy
             cargo clippy -- -D warnings -W clippy::cognitive_complexity
           when: always
@@ -134,15 +134,15 @@ commands:
       - run:
           name: Run cucumber scenarios
           no_output_timeout: 20m
-          command: cd integration_tests && mkdir -p cucumber_output && node_modules/.bin/cucumber-js --profile "ci" --tags "not @long-running and not @broken and not @wallet-ffi" --format json:cucumber_output/tests.cucumber --exit --retry 2 --retryTagFilter "@flaky and not @broken"
+          command: cd integration_tests && mkdir -p cucumber_output && node_modules/.bin/cucumber-js --profile "ci" --tags "not @long-running and not @broken and not @wallet-ffi" --format json:cucumber_output/tests.cucumber --exit --retry 3 --retry-tag-filter "@flaky and not @broken"
       - run:
           name: Generate report
           command: cd integration_tests && node ./generate_report.js
           when: always
       # Below step requires NodeJS v12 to run correctly, see explanation in WalletFFI.feature
       - run:
           name: Run FFI wallet library cucumber scenarios
-          command: cd integration_tests && mkdir -p cucumber_output && node_modules/.bin/cucumber-js --profile "ci"  --tags "not @long-running and not @broken and not @flaky and @wallet-ffi" --format json:cucumber_output/tests_ffi.cucumber --exit
+          command: cd integration_tests && mkdir -p cucumber_output && node_modules/.bin/cucumber-js --profile "ci"  --tags "not @long-running and not @broken and @wallet-ffi" --format json:cucumber_output/tests_ffi.cucumber --exit --retry 3 --retry-tag-filter "@flaky and not @broken"
           when: always
       - run:
           name: Generate report (ffi)

.github/wip_integration_tests.yml

@@ -23,7 +23,7 @@ jobs:
       - name: toolchain
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-08-17
+          toolchain: nightly-2021-08-18
           components: clippy, rustfmt
           override: true
       - name: dependencies

.github/workflows/audit.yml

@@ -7,7 +7,7 @@ jobs:
   security_audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - uses: actions-rs/audit-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/base_node_binaries.yml

@@ -61,7 +61,7 @@ jobs:
       - name: Setup Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-08-17
+          toolchain: nightly-2021-09-18
           components: rustfmt
           override: true
 
@@ -87,7 +87,7 @@ jobs:
       #        sudo apt-get -y upgrade
       - name: Install macOS dependencies
         if: startsWith(runner.os,'macOS')
-        run: brew install cmake zip
+        run: brew install cmake zip coreutils
 
       - name: Install Windows dependencies
         if: startsWith(runner.os,'Windows')
@@ -141,13 +141,13 @@ jobs:
           #cargo build --release --bin tari_base_node --features ${{ matrix.features}}
           cargo build --release
 
-      #    - name: Prepare Archives
-      #      shell: bash
-      #      run: |
-      #        ls -la "$GITHUB_WORKSPACE"
-      #        mkdir -p "$GITHUB_WORKSPACE${TBN_DIST}/archives"
-      #        export distDir="$GITHUB_WORKSPACE${TBN_DIST}/archives"
-      #        "$GITHUB_WORKSPACE/scripts/build_dists_tarball.sh" noBuild
+        #  - name: Prepare Archives
+        #    shell: bash
+        #    run: |
+        #      ls -la "$GITHUB_WORKSPACE"
+        #      mkdir -p "$GITHUB_WORKSPACE${TBN_DIST}/archives"
+        #      export distDir="$GITHUB_WORKSPACE${TBN_DIST}/archives"
+        #      "$GITHUB_WORKSPACE/scripts/build_dists_tarball.sh" noBuild
 
       - name: Prepare binaries
         shell: bash
@@ -162,19 +162,59 @@ jobs:
           #BINFILE="${TBN_FILENAME}-${VERSION}-${VSHA_SHORT}-${{ matrix.os }}-${{ matrix.target_cpu }}-${{ matrix.features }}${TBN_EXT}"
           BINFILE="${TBN_FILENAME}-${VERSION}-${VSHA_SHORT}-${{ matrix.os }}-${{ matrix.target_cpu }}${TBN_EXT}"
           echo "BINFILE=${BINFILE}" >> $GITHUB_ENV
-          echo "Copying files for ${BINFILE} too $(pwd)"
+          echo "Copying files for ${BINFILE} to $(pwd)"
           #cp -v "$GITHUB_WORKSPACE/target/release/${TBN_FILENAME}${TBN_EXT}" "./${BINFILE}"
           ls -la "$GITHUB_WORKSPACE/target/release/"
           cp -v "$GITHUB_WORKSPACE/target/release/tari_base_node${TBN_EXT}" .
           cp -v "$GITHUB_WORKSPACE/target/release/tari_console_wallet${TBN_EXT}" .
           cp -v "$GITHUB_WORKSPACE/target/release/tari_merge_mining_proxy${TBN_EXT}" .
           cp -v "$GITHUB_WORKSPACE/target/release/tari_mining_node${TBN_EXT}" .
 
+      - name: Build the macos pkg
+        if: startsWith(runner.os,'macOS')
+        env:
+          MACOS_KEYCHAIN_PASS: ${{ secrets.MACOS_KEYCHAIN_PASS }}
+          MACOS_APPLICATION_CERT: ${{ secrets.MACOS_APPLICATION_CERT }}
+          MACOS_APPLICATION_PASS: ${{ secrets.MACOS_APPLICATION_PASS }}
+          MACOS_INSTALLER_CERT: ${{ secrets.MACOS_INSTALLER_CERT }}
+          MACOS_INSTALLER_PASS: ${{ secrets.MACOS_INSTALLER_PASS }}
+        run: |
+          echo $MACOS_APPLICATION_CERT | base64 --decode > application.p12
+          echo $MACOS_INSTALLER_CERT | base64 --decode > installer.p12
+          security create-keychain -p $MACOS_KEYCHAIN_PASS build.keychain
+          security default-keychain -s build.keychain
+          security unlock-keychain -p $MACOS_KEYCHAIN_PASS build.keychain
+          security import application.p12 -k build.keychain -P $MACOS_APPLICATION_PASS -T /usr/bin/codesign
+          security import installer.p12 -k build.keychain -P $MACOS_INSTALLER_PASS -T /usr/bin/pkgbuild
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PASS build.keychain
+          cd buildtools
+          ./create_osx_install_zip.sh unused nozip
+          FILES=("tari_base_node" "tari_console_wallet" "tari_mining_node" "tari_merge_mining_proxy")
+          for FILE in "${FILES[@]}"; do
+            codesign --force -s "Developer ID Application: Tari Labs, LLC (8XGMD9X2H2)" "/tmp/tari_testnet/runtime/$FILE" -v
+            codesign --verify --deep --display --verbose=4 "/tmp/tari_testnet/runtime/$FILE"
+          done
+          pkgbuild --root /tmp/tari_testnet \
+          --identifier "com.tarilabs.pkg" \
+          --version "$VERSION" \
+          --install-location "/tmp/tari" \
+          --scripts "/tmp/tari_testnet/scripts" \
+          --sign "Developer ID Installer: Tari Labs, LLC (8XGMD9X2H2)" \
+          "${{ github.workspace }}${{ env.TBN_DIST }}/tari-${{ env.VERSION }}.pkg"
+      - name: Artifact macos pkg
+        if: startsWith(runner.os,'macOS')
+        uses: actions/upload-artifact@v2
+        with:
+          name: tari-${{ env.VERSION }}.pkg
+          path: "${{ github.workspace }}${{ env.TBN_DIST }}/tari-${{ env.VERSION }}.pkg"
+
+      # unlike inno script studio, iscc.exe doesn't run the [precompile] step generate_config.bat
       - name: Build the windows installer
         shell: cmd
         if: startsWith(runner.os,'Windows')
         run: |
           cd buildtools
+          call generate_config.bat
           "%programfiles(x86)%\Inno Setup 6\iscc.exe" "/DMyAppVersion=${{ env.VERSION }}-${{ env.VSHA_SHORT }}-release" "windows_inno_installer.iss"
 
       - name: Upload artifact for Windows installer

.github/workflows/ci.yml

@@ -9,6 +9,10 @@ on:
 
 name: CI
 
+env:
+  CARGO_HTTP_MULTIPLEXING: false
+  toolchain: nightly-2021-09-18
+
 jobs:
   clippy:
     name: clippy
@@ -27,9 +31,17 @@ jobs:
       - name: toolchain
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-08-17
+          toolchain: ${{ env.toolchain }}
           components: clippy, rustfmt
           override: true
+      - name: Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-${{ runner.cpu-model }}-${{ env.toolchain }}-${{ hashFiles('**/Cargo.lock') }}
       - name: cargo fmt
         uses: actions-rs/cargo@v1
         with:
@@ -39,86 +51,25 @@ jobs:
       #   uses: actions-rs/cargo@v1
       #   with:
       #     command: check
-      - name: cargo clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: -- -D warnings -W clippy::cognitive_complexity
-      - name: cargo clippy --all-targets
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets -- -D warnings
-  build:
-    name: build
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-18.04, macos-10.15, windows-2019]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: ubuntu dependencies
-        if: startsWith(matrix.os,'ubuntu')
-        run: |
-          sudo apt-get update && \
-          sudo apt-get -y install \
-            build-essential \
-            libgtk-3-dev \
-            libwebkit2gtk-4.0-dev \
-            libsoup2.4-dev
-
-      - name: macOS dependencies
-        if: startsWith(matrix.os,'macos')
-        run: brew install cmake zip
-
-      - name: windows dependencies
-        if: startsWith(matrix.os,'windows')
-        run: |
-          vcpkg.exe install sqlite3:x64-windows zlib:x64-windows
-          choco upgrade llvm zip psutils strawberryperl -y
-
-      - name: windows env
-        if: startsWith(matrix.os,'Windows')
-        shell: bash
-        run: |
-          echo "SQLITE3_LIB_DIR=C:\vcpkg\installed\x64-windows\lib" >> $GITHUB_ENV
-          echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $GITHUB_ENV
-
-      - name: toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2021-08-17
-          components: clippy, rustfmt
-          override: true
-
-      - name: cargo build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release
-
-      - name: cargo build wallet
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release -p tari_wallet
-
-      - name: cargo build wallet ffi
-        uses: actions-rs/cargo@v1
+      - name: Clippy check
+        uses: actions-rs/clippy-check@v1
         with:
-          command: build
-          args: --release -p tari_wallet_ffi
-
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features
   test:
     name: test
     runs-on: ubuntu-18.04
     steps:
       - name: checkout
         uses: actions/checkout@v2
-
+      - name: Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-${{ runner.cpu-model }}-${{ env.toolchain }}-${{ hashFiles('**/Cargo.lock') }}
       - name: ubuntu dependencies
         run: |
           sudo apt-get update && \
@@ -131,7 +82,7 @@ jobs:
       - name: toolchain
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-08-17
+          toolchain: ${{ env.toolchain }}
           components: clippy, rustfmt
           override: true
 

.github/workflows/libwallet.yml

@@ -7,6 +7,8 @@ on:
     #   - development
     tags:
       - "libwallet-*"
+    schedule:
+      - cron: "05 00 * * *"
 jobs:
   android:
     runs-on: ubuntu-latest
@@ -28,6 +30,7 @@ jobs:
           path: ${{ github.workspace }}/libwallet/
       # Copy tarballs to S3
       - name: Sync to S3
+        if: ${{ startsWith(github.ref, 'refs/tags/v') }}
         continue-on-error: true  # Don't break if s3 upload fails
         uses: jakejarvis/s3-sync-action@v0.5.1
         with:
@@ -45,13 +48,13 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-08-17
+          toolchain: nightly-2021-09-18
           target: aarch64-apple-ios
           components: rustfmt
           override: true
       - uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly-2021-08-17
+          toolchain: nightly-2021-09-18
           target: x86_64-apple-ios
           components: rustfmt
           override: true