From 24ddd394503b50ed38aaa4dbdb84ea045e09ef1f Mon Sep 17 00:00:00 2001 From: Cy Rossignol Date: Sun, 22 Sep 2019 13:00:55 -0500 Subject: [PATCH 1/7] Port Bitcoin allocators implementation This refactors the parts of the application that depend on the secure allocator and zero-after-free allocator. It replaces the structure to copy Bitcoin's file hierarchy. This updates the secure allocator with the newer memory swap prevention mechanism. --- src/Makefile.am | 11 +- src/allocators.cpp | 64 ---- src/allocators.h | 221 -------------- src/crypter.cpp | 20 +- src/crypter.h | 27 +- src/key.h | 2 +- src/qt/walletmodel.h | 2 +- src/serialize.h | 3 +- src/support/allocators/secure.h | 62 ++++ src/support/allocators/zeroafterfree.h | 48 +++ src/support/cleanse.cpp | 35 +++ src/support/cleanse.h | 15 + src/support/lockedpool.cpp | 404 +++++++++++++++++++++++++ src/support/lockedpool.h | 240 +++++++++++++++ src/test/allocator_tests.cpp | 305 +++++++++++++------ src/util.cpp | 2 - src/util/memory.h | 19 ++ 17 files changed, 1070 insertions(+), 410 deletions(-) delete mode 100644 src/allocators.cpp delete mode 100644 src/allocators.h create mode 100644 src/support/allocators/secure.h create mode 100644 src/support/allocators/zeroafterfree.h create mode 100644 src/support/cleanse.cpp create mode 100644 src/support/cleanse.h create mode 100644 src/support/lockedpool.cpp create mode 100644 src/support/lockedpool.h create mode 100644 src/util/memory.h diff --git a/src/Makefile.am b/src/Makefile.am index a62ccee43b..ddd5b42515 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -60,9 +60,8 @@ GRIDCOIN_CORE_H = \ addrdb.h \ addrman.h \ alert.h \ - allocators.h \ - attributes.h \ appcache.h \ + attributes.h \ backup.h \ banman.h \ base58.h \ @@ -117,6 +116,10 @@ GRIDCOIN_CORE_H = \ scrypt.h \ serialize.h \ strlcpy.h \ + support/allocators/secure.h \ + support/allocators/zeroafterfree.h \ + support/cleanse.h \ + support/lockedpool.h \ sync.h \ tally.h \ threadsafety.h \ @@ -125,6 +128,7 @@ GRIDCOIN_CORE_H = \ txdb-leveldb.h \ ui_interface.h \ uint256.h \ + util/memory.h \ util.h \ version.h \ walletdb.h \ @@ -133,7 +137,6 @@ GRIDCOIN_CORE_H = \ GRIDCOIN_CORE_CPP = addrdb.cpp \ addrman.cpp \ alert.cpp \ - allocators.cpp \ appcache.cpp \ backup.cpp \ banman.cpp \ @@ -205,6 +208,8 @@ libgridcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libgridcoin_util_a_SOURCES = $(GRIDCOIN_CORE_CPP) \ neuralnet/neuralnet_stub.cpp \ neuralnet/neuralnet.cpp \ + support/cleanse.cpp \ + support/lockedpool.cpp \ $(GRIDCOIN_CORE_H) if TARGET_WINDOWS diff --git a/src/allocators.cpp b/src/allocators.cpp deleted file mode 100644 index b239b623d8..0000000000 --- a/src/allocators.cpp +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2009-2013 The Bitcoin developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include "allocators.h" - -#ifdef WIN32 -#ifdef _WIN32_WINNT -#undef _WIN32_WINNT -#endif -#define _WIN32_WINNT 0x0501 -#define WIN32_LEAN_AND_MEAN 1 -#ifndef NOMINMAX -#define NOMINMAX -#endif -#include -// This is used to attempt to keep keying material out of swap -// Note that VirtualLock does not provide this as a guarantee on Windows, -// but, in practice, memory that has been VirtualLock'd almost never gets written to -// the pagefile except in rare circumstances where memory is extremely low. -#else -#include -#include // for PAGESIZE -#include // for sysconf -#endif - -/** Determine system page size in bytes */ -static inline size_t GetSystemPageSize() -{ - size_t page_size; -#if defined(WIN32) - SYSTEM_INFO sSysInfo; - GetSystemInfo(&sSysInfo); - page_size = sSysInfo.dwPageSize; -#elif defined(PAGESIZE) // defined in limits.h - page_size = PAGESIZE; -#else // assume some POSIX OS - page_size = sysconf(_SC_PAGESIZE); -#endif - return page_size; -} - -bool MemoryPageLocker::Lock(const void *addr, size_t len) -{ -#ifdef WIN32 - return VirtualLock(const_cast(addr), len); -#else - return mlock(addr, len) == 0; -#endif -} - -bool MemoryPageLocker::Unlock(const void *addr, size_t len) -{ -#ifdef WIN32 - return VirtualUnlock(const_cast(addr), len); -#else - return munlock(addr, len) == 0; -#endif -} - -LockedPageManager::LockedPageManager() : LockedPageManagerBase(GetSystemPageSize()) -{ -} - diff --git a/src/allocators.h b/src/allocators.h deleted file mode 100644 index fd6f51b27e..0000000000 --- a/src/allocators.h +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2012 The Bitcoin developers -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_ALLOCATORS_H -#define BITCOIN_ALLOCATORS_H - -#include -#include -#include -#include -#include // for OPENSSL_cleanse() - - -/** - * Thread-safe class to keep track of locked (ie, non-swappable) memory pages. - * - * Memory locks do not stack, that is, pages which have been locked several times by calls to mlock() - * will be unlocked by a single call to munlock(). This can result in keying material ending up in swap when - * those functions are used naively. This class simulates stacking memory locks by keeping a counter per page. - * - * @note By using a map from each page base address to lock count, this class is optimized for - * small objects that span up to a few pages, mostly smaller than a page. To support large allocations, - * something like an interval tree would be the preferred data structure. - */ -template class LockedPageManagerBase -{ -public: - LockedPageManagerBase(size_t page_size): - page_size(page_size) - { - // Determine bitmask for extracting page from address - assert(!(page_size & (page_size-1))); // size must be power of two - page_mask = ~(page_size - 1); - } - - // For all pages in affected range, increase lock count - void LockRange(void *p, size_t size) - { - boost::mutex::scoped_lock lock(mutex); - if(!size) return; - const size_t base_addr = reinterpret_cast(p); - const size_t start_page = base_addr & page_mask; - const size_t end_page = (base_addr + size - 1) & page_mask; - for(size_t page = start_page; page <= end_page; page += page_size) - { - Histogram::iterator it = histogram.find(page); - if(it == histogram.end()) // Newly locked page - { - locker.Lock(reinterpret_cast(page), page_size); - histogram.insert(std::make_pair(page, 1)); - } - else // Page was already locked; increase counter - { - it->second += 1; - } - } - } - - // For all pages in affected range, decrease lock count - void UnlockRange(void *p, size_t size) - { - boost::mutex::scoped_lock lock(mutex); - if(!size) return; - const size_t base_addr = reinterpret_cast(p); - const size_t start_page = base_addr & page_mask; - const size_t end_page = (base_addr + size - 1) & page_mask; - for(size_t page = start_page; page <= end_page; page += page_size) - { - Histogram::iterator it = histogram.find(page); - assert(it != histogram.end()); // Cannot unlock an area that was not locked - // Decrease counter for page, when it is zero, the page will be unlocked - it->second -= 1; - if(it->second == 0) // Nothing on the page anymore that keeps it locked - { - // Unlock page and remove the count from histogram - locker.Unlock(reinterpret_cast(page), page_size); - histogram.erase(it); - } - } - } - - // Get number of locked pages for diagnostics - int GetLockedPageCount() - { - boost::mutex::scoped_lock lock(mutex); - return histogram.size(); - } - -private: - Locker locker; - boost::mutex mutex; - size_t page_size, page_mask; - // map of page base address to lock count - typedef std::map Histogram; - Histogram histogram; -}; - - -/** - * OS-dependent memory page locking/unlocking. - * Defined as policy class to make stubbing for test possible. - */ -class MemoryPageLocker -{ -public: - /** Lock memory pages. - * addr and len must be a multiple of the system page size - */ - bool Lock(const void *addr, size_t len); - /** Unlock memory pages. - * addr and len must be a multiple of the system page size - */ - bool Unlock(const void *addr, size_t len); -}; - -/** - * Singleton class to keep track of locked (ie, non-swappable) memory pages, for use in - * std::allocator templates. - */ -class LockedPageManager: public LockedPageManagerBase -{ -public: - static LockedPageManager instance; // instantiated in util.cpp -private: - LockedPageManager(); -}; - -// -// Functions for directly locking/unlocking memory objects. -// Intended for non-dynamically allocated structures. -// -template void LockObject(const T &t) { - LockedPageManager::instance.LockRange((void*)(&t), sizeof(T)); -} - -template void UnlockObject(const T &t) { - OPENSSL_cleanse((void*)(&t), sizeof(T)); - LockedPageManager::instance.UnlockRange((void*)(&t), sizeof(T)); -} - -// -// Allocator that locks its contents from being paged -// out of memory and clears its contents before deletion. -// -template -struct secure_allocator : public std::allocator -{ - // MSVC8 default copy constructor is broken - typedef std::allocator base; - typedef typename base::size_type size_type; - typedef typename base::difference_type difference_type; - typedef typename base::pointer pointer; - typedef typename base::const_pointer const_pointer; - typedef typename base::reference reference; - typedef typename base::const_reference const_reference; - typedef typename base::value_type value_type; - secure_allocator() throw() {} - secure_allocator(const secure_allocator& a) throw() : base(a) {} - template - secure_allocator(const secure_allocator& a) throw() : base(a) {} - ~secure_allocator() throw() {} - template struct rebind - { typedef secure_allocator<_Other> other; }; - - T* allocate(std::size_t n, const void *hint = 0) - { - T *p; - p = std::allocator::allocate(n, hint); - if (p != NULL) - LockedPageManager::instance.LockRange(p, sizeof(T) * n); - return p; - } - - void deallocate(T* p, std::size_t n) - { - if (p != NULL) - { - OPENSSL_cleanse(p, sizeof(T) * n); - LockedPageManager::instance.UnlockRange(p, sizeof(T) * n); - } - std::allocator::deallocate(p, n); - } -}; - - -// -// Allocator that clears its contents before deletion. -// -template -struct zero_after_free_allocator : public std::allocator -{ - // MSVC8 default copy constructor is broken - typedef std::allocator base; - typedef typename base::size_type size_type; - typedef typename base::difference_type difference_type; - typedef typename base::pointer pointer; - typedef typename base::const_pointer const_pointer; - typedef typename base::reference reference; - typedef typename base::const_reference const_reference; - typedef typename base::value_type value_type; - zero_after_free_allocator() throw() {} - zero_after_free_allocator(const zero_after_free_allocator& a) throw() : base(a) {} - template - zero_after_free_allocator(const zero_after_free_allocator& a) throw() : base(a) {} - ~zero_after_free_allocator() throw() {} - template struct rebind - { typedef zero_after_free_allocator<_Other> other; }; - - void deallocate(T* p, std::size_t n) - { - if (p != NULL) - OPENSSL_cleanse(p, sizeof(T) * n); - std::allocator::deallocate(p, n); - } -}; - -// This is exactly like std::string, but with a custom allocator. -typedef std::basic_string, secure_allocator > SecureString; - -#endif diff --git a/src/crypter.cpp b/src/crypter.cpp index 07b6a187bc..98d57945b2 100644 --- a/src/crypter.cpp +++ b/src/crypter.cpp @@ -19,7 +19,7 @@ bool CCrypter::SetKeyFromPassphrase(const SecureString& strKeyData, const std::v if (nDerivationMethod == 0) { i = EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha512(), &chSalt[0], - (unsigned char *)&strKeyData[0], strKeyData.size(), nRounds, chKey, chIV); + (unsigned char *)&strKeyData[0], strKeyData.size(), nRounds, vchKey.data(), vchIV.data()); } if (nDerivationMethod == 1) @@ -28,15 +28,15 @@ bool CCrypter::SetKeyFromPassphrase(const SecureString& strKeyData, const std::v uint256 scryptHash = scrypt_salted_multiround_hash((const void*)strKeyData.c_str(), strKeyData.size(), &chSalt[0], 8, nRounds); i = EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha512(), &chSalt[0], - (unsigned char *)&scryptHash, sizeof scryptHash, nRounds, chKey, chIV); + (unsigned char *)&scryptHash, sizeof scryptHash, nRounds, vchKey.data(), vchIV.data()); OPENSSL_cleanse(&scryptHash, sizeof scryptHash); } if (i != (int)WALLET_CRYPTO_KEY_SIZE) { - OPENSSL_cleanse(&chKey, sizeof chKey); - OPENSSL_cleanse(&chIV, sizeof chIV); + OPENSSL_cleanse(vchKey.data(), vchKey.size()); + OPENSSL_cleanse(vchIV.data(), vchIV.size()); return false; } @@ -49,14 +49,14 @@ bool CCrypter::SetKey(const CKeyingMaterial& chNewKey, const std::vector &vchCiphertext) +bool CCrypter::Encrypt(const CKeyingMaterial& vchPlaintext, std::vector &vchCiphertext) const { if (!fKeySet) return false; @@ -73,7 +73,7 @@ bool CCrypter::Encrypt(const CKeyingMaterial& vchPlaintext, std::vector& vchCiphertext, CKeyingMaterial& vchPlaintext) +bool CCrypter::Decrypt(const std::vector& vchCiphertext, CKeyingMaterial& vchPlaintext) const { if (!fKeySet) return false; @@ -101,7 +101,7 @@ bool CCrypter::Decrypt(const std::vector& vchCiphertext, CKeyingM if(!ctx) throw std::runtime_error("Error allocating cipher context"); - if (fOk) fOk = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, chKey, chIV); + if (fOk) fOk = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, vchKey.data(), vchIV.data()); if (fOk) fOk = EVP_DecryptUpdate(ctx, &vchPlaintext[0], &nPLen, &vchCiphertext[0], nLen); if (fOk) fOk = EVP_DecryptFinal_ex(ctx, (&vchPlaintext[0])+nPLen, &nFLen); EVP_CIPHER_CTX_free(ctx); diff --git a/src/crypter.h b/src/crypter.h index c1eb07e376..a703c454fd 100644 --- a/src/crypter.h +++ b/src/crypter.h @@ -4,7 +4,7 @@ #ifndef __CRYPTER_H__ #define __CRYPTER_H__ -#include "allocators.h" /* for SecureString */ +#include "support/allocators/secure.h" /* for SecureString */ #include "key.h" const unsigned int WALLET_CRYPTO_KEY_SIZE = 32; @@ -83,43 +83,38 @@ typedef std::vector > CKeyingMate class CCrypter { private: - unsigned char chKey[WALLET_CRYPTO_KEY_SIZE]; - unsigned char chIV[WALLET_CRYPTO_KEY_SIZE]; + std::vector> vchKey; + std::vector> vchIV; bool fKeySet; public: bool SetKeyFromPassphrase(const SecureString &strKeyData, const std::vector& chSalt, const unsigned int nRounds, const unsigned int nDerivationMethod); - bool Encrypt(const CKeyingMaterial& vchPlaintext, std::vector &vchCiphertext); - bool Decrypt(const std::vector& vchCiphertext, CKeyingMaterial& vchPlaintext); + bool Encrypt(const CKeyingMaterial& vchPlaintext, std::vector &vchCiphertext) const; + bool Decrypt(const std::vector& vchCiphertext, CKeyingMaterial& vchPlaintext) const; bool SetKey(const CKeyingMaterial& chNewKey, const std::vector& chNewIV); void CleanKey() { - OPENSSL_cleanse(&chKey, sizeof chKey); - OPENSSL_cleanse(&chIV, sizeof chIV); + memory_cleanse(vchKey.data(), vchKey.size()); + memory_cleanse(vchIV.data(), vchIV.size()); fKeySet = false; } CCrypter() { fKeySet = false; - - // Try to keep the key data out of swap (and be a bit over-careful to keep the IV that we don't even use out of swap) - // Note that this does nothing about suspend-to-disk (which will put all our key data on disk) - // Note as well that at no point in this program is any attempt made to prevent stealing of keys by reading the memory of the running process. - LockedPageManager::instance.LockRange(&chKey[0], sizeof chKey); - LockedPageManager::instance.LockRange(&chIV[0], sizeof chIV); + vchKey.resize(WALLET_CRYPTO_KEY_SIZE); + // TODO: Bitcoin uses an IV size of 16 defined by WALLET_CRYPTO_IV_SIZE. + vchIV.resize(WALLET_CRYPTO_KEY_SIZE); } ~CCrypter() { CleanKey(); - - LockedPageManager::instance.UnlockRange(&chKey[0], sizeof chKey); - LockedPageManager::instance.UnlockRange(&chIV[0], sizeof chIV); } }; + bool EncryptSecret(CKeyingMaterial& vMasterKey, const CSecret &vchPlaintext, const uint256& nIV, std::vector &vchCiphertext); bool DecryptSecret(const CKeyingMaterial& vMasterKey, const std::vector &vchCiphertext, const uint256& nIV, CSecret &vchPlaintext); #endif diff --git a/src/key.h b/src/key.h index db7d2e324d..f31f88472d 100644 --- a/src/key.h +++ b/src/key.h @@ -8,8 +8,8 @@ #include #include -#include "allocators.h" #include "serialize.h" +#include "support/allocators/secure.h" #include "uint256.h" #include "util.h" diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 881d0cd0af..a4deffcf1f 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -5,7 +5,7 @@ #include #include -#include "allocators.h" /* for SecureString */ +#include "support/allocators/secure.h" /* for SecureString */ class OptionsModel; class AddressTableModel; diff --git a/src/serialize.h b/src/serialize.h index d3198d7ddc..29276d47fb 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -19,7 +20,7 @@ #include -#include "allocators.h" +#include "support/allocators/zeroafterfree.h" #include "version.h" class CAutoFile; diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h new file mode 100644 index 0000000000..57f5b1f733 --- /dev/null +++ b/src/support/allocators/secure.h @@ -0,0 +1,62 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SUPPORT_ALLOCATORS_SECURE_H +#define BITCOIN_SUPPORT_ALLOCATORS_SECURE_H + +#include +#include + +#include + +// +// Allocator that locks its contents from being paged +// out of memory and clears its contents before deletion. +// +template +struct secure_allocator : public std::allocator { + // MSVC8 default copy constructor is broken + typedef std::allocator base; + typedef typename base::size_type size_type; + typedef typename base::difference_type difference_type; + typedef typename base::pointer pointer; + typedef typename base::const_pointer const_pointer; + typedef typename base::reference reference; + typedef typename base::const_reference const_reference; + typedef typename base::value_type value_type; + secure_allocator() noexcept {} + secure_allocator(const secure_allocator& a) noexcept : base(a) {} + template + secure_allocator(const secure_allocator& a) noexcept : base(a) + { + } + ~secure_allocator() noexcept {} + template + struct rebind { + typedef secure_allocator<_Other> other; + }; + + T* allocate(std::size_t n, const void* hint = 0) + { + T* allocation = static_cast(LockedPoolManager::Instance().alloc(sizeof(T) * n)); + if (!allocation) { + throw std::bad_alloc(); + } + return allocation; + } + + void deallocate(T* p, std::size_t n) + { + if (p != nullptr) { + memory_cleanse(p, sizeof(T) * n); + } + LockedPoolManager::Instance().free(p); + } +}; + +// This is exactly like std::string, but with a custom allocator. +typedef std::basic_string, secure_allocator > SecureString; + +#endif // BITCOIN_SUPPORT_ALLOCATORS_SECURE_H diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h new file mode 100644 index 0000000000..c7ed5ef308 --- /dev/null +++ b/src/support/allocators/zeroafterfree.h @@ -0,0 +1,48 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H +#define BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H + +#include + +#include +#include + +template +struct zero_after_free_allocator : public std::allocator { + // MSVC8 default copy constructor is broken + typedef std::allocator base; + typedef typename base::size_type size_type; + typedef typename base::difference_type difference_type; + typedef typename base::pointer pointer; + typedef typename base::const_pointer const_pointer; + typedef typename base::reference reference; + typedef typename base::const_reference const_reference; + typedef typename base::value_type value_type; + zero_after_free_allocator() noexcept {} + zero_after_free_allocator(const zero_after_free_allocator& a) noexcept : base(a) {} + template + zero_after_free_allocator(const zero_after_free_allocator& a) noexcept : base(a) + { + } + ~zero_after_free_allocator() noexcept {} + template + struct rebind { + typedef zero_after_free_allocator<_Other> other; + }; + + void deallocate(T* p, std::size_t n) + { + if (p != nullptr) + memory_cleanse(p, sizeof(T) * n); + std::allocator::deallocate(p, n); + } +}; + +// Byte-vector that clears its contents before deletion. +typedef std::vector > CSerializeData; + +#endif // BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H diff --git a/src/support/cleanse.cpp b/src/support/cleanse.cpp new file mode 100644 index 0000000000..ecb00510f7 --- /dev/null +++ b/src/support/cleanse.cpp @@ -0,0 +1,35 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include + +#if defined(_MSC_VER) +#include // For SecureZeroMemory. +#endif + +void memory_cleanse(void *ptr, size_t len) +{ +#if defined(_MSC_VER) + /* SecureZeroMemory is guaranteed not to be optimized out by MSVC. */ + SecureZeroMemory(ptr, len); +#else + std::memset(ptr, 0, len); + + /* Memory barrier that scares the compiler away from optimizing out the memset. + * + * Quoting Adam Langley in commit ad1907fe73334d6c696c8539646c21b11178f20f + * in BoringSSL (ISC License): + * As best as we can tell, this is sufficient to break any optimisations that + * might try to eliminate "superfluous" memsets. + * This method is used in memzero_explicit() the Linux kernel, too. Its advantage is that it + * is pretty efficient because the compiler can still implement the memset() efficiently, + * just not remove it entirely. See "Dead Store Elimination (Still) Considered Harmful" by + * Yang et al. (USENIX Security 2017) for more background. + */ + __asm__ __volatile__("" : : "r"(ptr) : "memory"); +#endif +} diff --git a/src/support/cleanse.h b/src/support/cleanse.h new file mode 100644 index 0000000000..b03520315d --- /dev/null +++ b/src/support/cleanse.h @@ -0,0 +1,15 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SUPPORT_CLEANSE_H +#define BITCOIN_SUPPORT_CLEANSE_H + +#include + +/** Secure overwrite a buffer (possibly containing secret data) with zero-bytes. The write + * operation will not be optimized out by the compiler. */ +void memory_cleanse(void *ptr, size_t len); + +#endif // BITCOIN_SUPPORT_CLEANSE_H diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp new file mode 100644 index 0000000000..b3b843b189 --- /dev/null +++ b/src/support/lockedpool.cpp @@ -0,0 +1,404 @@ +// Copyright (c) 2016-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN 1 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include // for mmap +#include // for getrlimit +#include // for PAGESIZE +#include // for sysconf +#endif + +#include + +LockedPoolManager* LockedPoolManager::_instance = nullptr; +std::once_flag LockedPoolManager::init_flag; + +/*******************************************************************************/ +// Utilities +// +/** Align up to power of 2 */ +static inline size_t align_up(size_t x, size_t align) +{ + return (x + align - 1) & ~(align - 1); +} + +/*******************************************************************************/ +// Implementation: Arena + +Arena::Arena(void *base_in, size_t size_in, size_t alignment_in): + base(static_cast(base_in)), end(static_cast(base_in) + size_in), alignment(alignment_in) +{ + // Start with one free chunk that covers the entire arena + auto it = size_to_free_chunk.emplace(size_in, base); + chunks_free.emplace(base, it); + chunks_free_end.emplace(base + size_in, it); +} + +Arena::~Arena() +{ +} + +void* Arena::alloc(size_t size) +{ + // Round to next multiple of alignment + size = align_up(size, alignment); + + // Don't handle zero-sized chunks + if (size == 0) + return nullptr; + + // Pick a large enough free-chunk. Returns an iterator pointing to the first element that is not less than key. + // This allocation strategy is best-fit. According to "Dynamic Storage Allocation: A Survey and Critical Review", + // Wilson et. al. 1995, http://www.scs.stanford.edu/14wi-cs140/sched/readings/wilson.pdf, best-fit and first-fit + // policies seem to work well in practice. + auto size_ptr_it = size_to_free_chunk.lower_bound(size); + if (size_ptr_it == size_to_free_chunk.end()) + return nullptr; + + // Create the used-chunk, taking its space from the end of the free-chunk + const size_t size_remaining = size_ptr_it->first - size; + auto allocated = chunks_used.emplace(size_ptr_it->second + size_remaining, size).first; + chunks_free_end.erase(size_ptr_it->second + size_ptr_it->first); + if (size_ptr_it->first == size) { + // whole chunk is used up + chunks_free.erase(size_ptr_it->second); + } else { + // still some memory left in the chunk + auto it_remaining = size_to_free_chunk.emplace(size_remaining, size_ptr_it->second); + chunks_free[size_ptr_it->second] = it_remaining; + chunks_free_end.emplace(size_ptr_it->second + size_remaining, it_remaining); + } + size_to_free_chunk.erase(size_ptr_it); + + return reinterpret_cast(allocated->first); +} + +void Arena::free(void *ptr) +{ + // Freeing the nullptr pointer is OK. + if (ptr == nullptr) { + return; + } + + // Remove chunk from used map + auto i = chunks_used.find(static_cast(ptr)); + if (i == chunks_used.end()) { + throw std::runtime_error("Arena: invalid or double free"); + } + std::pair freed = *i; + chunks_used.erase(i); + + // coalesce freed with previous chunk + auto prev = chunks_free_end.find(freed.first); + if (prev != chunks_free_end.end()) { + freed.first -= prev->second->first; + freed.second += prev->second->first; + size_to_free_chunk.erase(prev->second); + chunks_free_end.erase(prev); + } + + // coalesce freed with chunk after freed + auto next = chunks_free.find(freed.first + freed.second); + if (next != chunks_free.end()) { + freed.second += next->second->first; + size_to_free_chunk.erase(next->second); + chunks_free.erase(next); + } + + // Add/set space with coalesced free chunk + auto it = size_to_free_chunk.emplace(freed.second, freed.first); + chunks_free[freed.first] = it; + chunks_free_end[freed.first + freed.second] = it; +} + +Arena::Stats Arena::stats() const +{ + Arena::Stats r{ 0, 0, 0, chunks_used.size(), chunks_free.size() }; + for (const auto& chunk: chunks_used) + r.used += chunk.second; + for (const auto& chunk: chunks_free) + r.free += chunk.second->first; + r.total = r.used + r.free; + return r; +} + +#ifdef ARENA_DEBUG +static void printchunk(char* base, size_t sz, bool used) { + std::cout << + "0x" << std::hex << std::setw(16) << std::setfill('0') << base << + " 0x" << std::hex << std::setw(16) << std::setfill('0') << sz << + " 0x" << used << std::endl; +} +void Arena::walk() const +{ + for (const auto& chunk: chunks_used) + printchunk(chunk.first, chunk.second, true); + std::cout << std::endl; + for (const auto& chunk: chunks_free) + printchunk(chunk.first, chunk.second, false); + std::cout << std::endl; +} +#endif + +/*******************************************************************************/ +// Implementation: Win32LockedPageAllocator + +#ifdef WIN32 +/** LockedPageAllocator specialized for Windows. + */ +class Win32LockedPageAllocator: public LockedPageAllocator +{ +public: + Win32LockedPageAllocator(); + void* AllocateLocked(size_t len, bool *lockingSuccess) override; + void FreeLocked(void* addr, size_t len) override; + size_t GetLimit() override; +private: + size_t page_size; +}; + +Win32LockedPageAllocator::Win32LockedPageAllocator() +{ + // Determine system page size in bytes + SYSTEM_INFO sSysInfo; + GetSystemInfo(&sSysInfo); + page_size = sSysInfo.dwPageSize; +} +void *Win32LockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) +{ + len = align_up(len, page_size); + void *addr = VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + if (addr) { + // VirtualLock is used to attempt to keep keying material out of swap. Note + // that it does not provide this as a guarantee, but, in practice, memory + // that has been VirtualLock'd almost never gets written to the pagefile + // except in rare circumstances where memory is extremely low. + *lockingSuccess = VirtualLock(const_cast(addr), len) != 0; + } + return addr; +} +void Win32LockedPageAllocator::FreeLocked(void* addr, size_t len) +{ + len = align_up(len, page_size); + memory_cleanse(addr, len); + VirtualUnlock(const_cast(addr), len); +} + +size_t Win32LockedPageAllocator::GetLimit() +{ + // TODO is there a limit on Windows, how to get it? + return std::numeric_limits::max(); +} +#endif + +/*******************************************************************************/ +// Implementation: PosixLockedPageAllocator + +#ifndef WIN32 +/** LockedPageAllocator specialized for OSes that don't try to be + * special snowflakes. + */ +class PosixLockedPageAllocator: public LockedPageAllocator +{ +public: + PosixLockedPageAllocator(); + void* AllocateLocked(size_t len, bool *lockingSuccess) override; + void FreeLocked(void* addr, size_t len) override; + size_t GetLimit() override; +private: + size_t page_size; +}; + +PosixLockedPageAllocator::PosixLockedPageAllocator() +{ + // Determine system page size in bytes +#if defined(PAGESIZE) // defined in limits.h + page_size = PAGESIZE; +#else // assume some POSIX OS + page_size = sysconf(_SC_PAGESIZE); +#endif +} + +// Some systems (at least OS X) do not define MAP_ANONYMOUS yet and define +// MAP_ANON which is deprecated +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) +{ + void *addr; + len = align_up(len, page_size); + addr = mmap(nullptr, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { + return nullptr; + } + if (addr) { + *lockingSuccess = mlock(addr, len) == 0; + } + return addr; +} +void PosixLockedPageAllocator::FreeLocked(void* addr, size_t len) +{ + len = align_up(len, page_size); + memory_cleanse(addr, len); + munlock(addr, len); + munmap(addr, len); +} +size_t PosixLockedPageAllocator::GetLimit() +{ +#ifdef RLIMIT_MEMLOCK + struct rlimit rlim; + if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) { + if (rlim.rlim_cur != RLIM_INFINITY) { + return rlim.rlim_cur; + } + } +#endif + return std::numeric_limits::max(); +} +#endif + +/*******************************************************************************/ +// Implementation: LockedPool + +LockedPool::LockedPool(std::unique_ptr allocator_in, LockingFailed_Callback lf_cb_in): + allocator(std::move(allocator_in)), lf_cb(lf_cb_in), cumulative_bytes_locked(0) +{ +} + +LockedPool::~LockedPool() +{ +} +void* LockedPool::alloc(size_t size) +{ + std::lock_guard lock(mutex); + + // Don't handle impossible sizes + if (size == 0 || size > ARENA_SIZE) + return nullptr; + + // Try allocating from each current arena + for (auto &arena: arenas) { + void *addr = arena.alloc(size); + if (addr) { + return addr; + } + } + // If that fails, create a new one + if (new_arena(ARENA_SIZE, ARENA_ALIGN)) { + return arenas.back().alloc(size); + } + return nullptr; +} + +void LockedPool::free(void *ptr) +{ + std::lock_guard lock(mutex); + // TODO we can do better than this linear search by keeping a map of arena + // extents to arena, and looking up the address. + for (auto &arena: arenas) { + if (arena.addressInArena(ptr)) { + arena.free(ptr); + return; + } + } + throw std::runtime_error("LockedPool: invalid address not pointing to any arena"); +} + +LockedPool::Stats LockedPool::stats() const +{ + std::lock_guard lock(mutex); + LockedPool::Stats r{0, 0, 0, cumulative_bytes_locked, 0, 0}; + for (const auto &arena: arenas) { + Arena::Stats i = arena.stats(); + r.used += i.used; + r.free += i.free; + r.total += i.total; + r.chunks_used += i.chunks_used; + r.chunks_free += i.chunks_free; + } + return r; +} + +bool LockedPool::new_arena(size_t size, size_t align) +{ + bool locked; + // If this is the first arena, handle this specially: Cap the upper size + // by the process limit. This makes sure that the first arena will at least + // be locked. An exception to this is if the process limit is 0: + // in this case no memory can be locked at all so we'll skip past this logic. + if (arenas.empty()) { + size_t limit = allocator->GetLimit(); + if (limit > 0) { + size = std::min(size, limit); + } + } + void *addr = allocator->AllocateLocked(size, &locked); + if (!addr) { + return false; + } + if (locked) { + cumulative_bytes_locked += size; + } else if (lf_cb) { // Call the locking-failed callback if locking failed + if (!lf_cb()) { // If the callback returns false, free the memory and fail, otherwise consider the user warned and proceed. + allocator->FreeLocked(addr, size); + return false; + } + } + arenas.emplace_back(allocator.get(), addr, size, align); + return true; +} + +LockedPool::LockedPageArena::LockedPageArena(LockedPageAllocator *allocator_in, void *base_in, size_t size_in, size_t align_in): + Arena(base_in, size_in, align_in), base(base_in), size(size_in), allocator(allocator_in) +{ +} +LockedPool::LockedPageArena::~LockedPageArena() +{ + allocator->FreeLocked(base, size); +} + +/*******************************************************************************/ +// Implementation: LockedPoolManager +// +LockedPoolManager::LockedPoolManager(std::unique_ptr allocator_in): + LockedPool(std::move(allocator_in), &LockedPoolManager::LockingFailed) +{ +} + +bool LockedPoolManager::LockingFailed() +{ + // TODO: log something but how? without including util.h + return true; +} + +void LockedPoolManager::CreateInstance() +{ + // Using a local static instance guarantees that the object is initialized + // when it's first needed and also deinitialized after all objects that use + // it are done with it. I can think of one unlikely scenario where we may + // have a static deinitialization order/problem, but the check in + // LockedPoolManagerBase's destructor helps us detect if that ever happens. +#ifdef WIN32 + std::unique_ptr allocator(new Win32LockedPageAllocator()); +#else + std::unique_ptr allocator(new PosixLockedPageAllocator()); +#endif + static LockedPoolManager instance(std::move(allocator)); + LockedPoolManager::_instance = &instance; +} diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h new file mode 100644 index 0000000000..b420c909fc --- /dev/null +++ b/src/support/lockedpool.h @@ -0,0 +1,240 @@ +// Copyright (c) 2016-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SUPPORT_LOCKEDPOOL_H +#define BITCOIN_SUPPORT_LOCKEDPOOL_H + +#include +#include +#include +#include +#include +#include + +/** + * OS-dependent allocation and deallocation of locked/pinned memory pages. + * Abstract base class. + */ +class LockedPageAllocator +{ +public: + virtual ~LockedPageAllocator() {} + /** Allocate and lock memory pages. + * If len is not a multiple of the system page size, it is rounded up. + * Returns nullptr in case of allocation failure. + * + * If locking the memory pages could not be accomplished it will still + * return the memory, however the lockingSuccess flag will be false. + * lockingSuccess is undefined if the allocation fails. + */ + virtual void* AllocateLocked(size_t len, bool *lockingSuccess) = 0; + + /** Unlock and free memory pages. + * Clear the memory before unlocking. + */ + virtual void FreeLocked(void* addr, size_t len) = 0; + + /** Get the total limit on the amount of memory that may be locked by this + * process, in bytes. Return size_t max if there is no limit or the limit + * is unknown. Return 0 if no memory can be locked at all. + */ + virtual size_t GetLimit() = 0; +}; + +/* An arena manages a contiguous region of memory by dividing it into + * chunks. + */ +class Arena +{ +public: + Arena(void *base, size_t size, size_t alignment); + virtual ~Arena(); + + Arena(const Arena& other) = delete; // non construction-copyable + Arena& operator=(const Arena&) = delete; // non copyable + + /** Memory statistics. */ + struct Stats + { + size_t used; + size_t free; + size_t total; + size_t chunks_used; + size_t chunks_free; + }; + + /** Allocate size bytes from this arena. + * Returns pointer on success, or 0 if memory is full or + * the application tried to allocate 0 bytes. + */ + void* alloc(size_t size); + + /** Free a previously allocated chunk of memory. + * Freeing the zero pointer has no effect. + * Raises std::runtime_error in case of error. + */ + void free(void *ptr); + + /** Get arena usage statistics */ + Stats stats() const; + +#ifdef ARENA_DEBUG + void walk() const; +#endif + + /** Return whether a pointer points inside this arena. + * This returns base <= ptr < (base+size) so only use it for (inclusive) + * chunk starting addresses. + */ + bool addressInArena(void *ptr) const { return ptr >= base && ptr < end; } +private: + typedef std::multimap SizeToChunkSortedMap; + /** Map to enable O(log(n)) best-fit allocation, as it's sorted by size */ + SizeToChunkSortedMap size_to_free_chunk; + + typedef std::unordered_map ChunkToSizeMap; + /** Map from begin of free chunk to its node in size_to_free_chunk */ + ChunkToSizeMap chunks_free; + /** Map from end of free chunk to its node in size_to_free_chunk */ + ChunkToSizeMap chunks_free_end; + + /** Map from begin of used chunk to its size */ + std::unordered_map chunks_used; + + /** Base address of arena */ + char* base; + /** End address of arena */ + char* end; + /** Minimum chunk alignment */ + size_t alignment; +}; + +/** Pool for locked memory chunks. + * + * To avoid sensitive key data from being swapped to disk, the memory in this pool + * is locked/pinned. + * + * An arena manages a contiguous region of memory. The pool starts out with one arena + * but can grow to multiple arenas if the need arises. + * + * Unlike a normal C heap, the administrative structures are separate from the managed + * memory. This has been done as the sizes and bases of objects are not in themselves sensitive + * information, as to conserve precious locked memory. In some operating systems + * the amount of memory that can be locked is small. + */ +class LockedPool +{ +public: + /** Size of one arena of locked memory. This is a compromise. + * Do not set this too low, as managing many arenas will increase + * allocation and deallocation overhead. Setting it too high allocates + * more locked memory from the OS than strictly necessary. + */ + static const size_t ARENA_SIZE = 256*1024; + /** Chunk alignment. Another compromise. Setting this too high will waste + * memory, setting it too low will facilitate fragmentation. + */ + static const size_t ARENA_ALIGN = 16; + + /** Callback when allocation succeeds but locking fails. + */ + typedef bool (*LockingFailed_Callback)(); + + /** Memory statistics. */ + struct Stats + { + size_t used; + size_t free; + size_t total; + size_t locked; + size_t chunks_used; + size_t chunks_free; + }; + + /** Create a new LockedPool. This takes ownership of the MemoryPageLocker, + * you can only instantiate this with LockedPool(std::move(...)). + * + * The second argument is an optional callback when locking a newly allocated arena failed. + * If this callback is provided and returns false, the allocation fails (hard fail), if + * it returns true the allocation proceeds, but it could warn. + */ + explicit LockedPool(std::unique_ptr allocator, LockingFailed_Callback lf_cb_in = nullptr); + ~LockedPool(); + + LockedPool(const LockedPool& other) = delete; // non construction-copyable + LockedPool& operator=(const LockedPool&) = delete; // non copyable + + /** Allocate size bytes from this arena. + * Returns pointer on success, or 0 if memory is full or + * the application tried to allocate 0 bytes. + */ + void* alloc(size_t size); + + /** Free a previously allocated chunk of memory. + * Freeing the zero pointer has no effect. + * Raises std::runtime_error in case of error. + */ + void free(void *ptr); + + /** Get pool usage statistics */ + Stats stats() const; +private: + std::unique_ptr allocator; + + /** Create an arena from locked pages */ + class LockedPageArena: public Arena + { + public: + LockedPageArena(LockedPageAllocator *alloc_in, void *base_in, size_t size, size_t align); + ~LockedPageArena(); + private: + void *base; + size_t size; + LockedPageAllocator *allocator; + }; + + bool new_arena(size_t size, size_t align); + + std::list arenas; + LockingFailed_Callback lf_cb; + size_t cumulative_bytes_locked; + /** Mutex protects access to this pool's data structures, including arenas. + */ + mutable std::mutex mutex; +}; + +/** + * Singleton class to keep track of locked (ie, non-swappable) memory, for use in + * std::allocator templates. + * + * Some implementations of the STL allocate memory in some constructors (i.e., see + * MSVC's vector implementation where it allocates 1 byte of memory in the allocator.) + * Due to the unpredictable order of static initializers, we have to make sure the + * LockedPoolManager instance exists before any other STL-based objects that use + * secure_allocator are created. So instead of having LockedPoolManager also be + * static-initialized, it is created on demand. + */ +class LockedPoolManager : public LockedPool +{ +public: + /** Return the current instance, or create it once */ + static LockedPoolManager& Instance() + { + std::call_once(LockedPoolManager::init_flag, LockedPoolManager::CreateInstance); + return *LockedPoolManager::_instance; + } + +private: + explicit LockedPoolManager(std::unique_ptr allocator); + + /** Create a new LockedPoolManager specialized to the OS */ + static void CreateInstance(); + /** Called when locking fails, warn the user here */ + static bool LockingFailed(); + + static LockedPoolManager* _instance; + static std::once_flag init_flag; +}; + +#endif // BITCOIN_SUPPORT_LOCKEDPOOL_H diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index d5cb8e8101..0020f121c3 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -1,115 +1,238 @@ +// Copyright (c) 2012-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +//#include + +//#include + +#include + #include -#include "init.h" -#include "main.h" -#include "util.h" +#include BOOST_AUTO_TEST_SUITE(allocator_tests) -// Dummy memory page locker for platform independent tests -static const void *last_lock_addr, *last_unlock_addr; -static size_t last_lock_len, last_unlock_len; -class TestLocker +BOOST_AUTO_TEST_CASE(arena_tests) { -public: - bool Lock(const void *addr, size_t len) + // Fake memory base address for testing + // without actually using memory. + void *synth_base = reinterpret_cast(0x08000000); + const size_t synth_size = 1024*1024; + Arena b(synth_base, synth_size, 16); + void *chunk = b.alloc(1000); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(chunk != nullptr); + BOOST_CHECK(b.stats().used == 1008); // Aligned to 16 + BOOST_CHECK(b.stats().total == synth_size); // Nothing has disappeared? + b.free(chunk); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 0); + BOOST_CHECK(b.stats().free == synth_size); + try { // Test exception on double-free + b.free(chunk); + BOOST_CHECK(0); + } catch(std::runtime_error &) { - last_lock_addr = addr; - last_lock_len = len; - return true; } - bool Unlock(const void *addr, size_t len) - { - last_unlock_addr = addr; - last_unlock_len = len; - return true; + + void *a0 = b.alloc(128); + void *a1 = b.alloc(256); + void *a2 = b.alloc(512); + BOOST_CHECK(b.stats().used == 896); + BOOST_CHECK(b.stats().total == synth_size); +#ifdef ARENA_DEBUG + b.walk(); +#endif + b.free(a0); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 768); + b.free(a1); + BOOST_CHECK(b.stats().used == 512); + void *a3 = b.alloc(128); +#ifdef ARENA_DEBUG + b.walk(); +#endif + BOOST_CHECK(b.stats().used == 640); + b.free(a2); + BOOST_CHECK(b.stats().used == 128); + b.free(a3); + BOOST_CHECK(b.stats().used == 0); + BOOST_CHECK_EQUAL(b.stats().chunks_used, 0U); + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); + BOOST_CHECK_EQUAL(b.stats().chunks_free, 1U); + + std::vector addr; + BOOST_CHECK(b.alloc(0) == nullptr); // allocating 0 always returns nullptr +#ifdef ARENA_DEBUG + b.walk(); +#endif + // Sweeping allocate all memory + for (int x=0; x<1024; ++x) + addr.push_back(b.alloc(1024)); + BOOST_CHECK(b.stats().free == 0); + BOOST_CHECK(b.alloc(1024) == nullptr); // memory is full, this must return nullptr + BOOST_CHECK(b.alloc(0) == nullptr); + for (int x=0; x<1024; ++x) + b.free(addr[x]); + addr.clear(); + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); + + // Now in the other direction... + for (int x=0; x<1024; ++x) + addr.push_back(b.alloc(1024)); + for (int x=0; x<1024; ++x) + b.free(addr[1023-x]); + addr.clear(); + + // Now allocate in smaller unequal chunks, then deallocate haphazardly + // Not all the chunks will succeed allocating, but freeing nullptr is + // allowed so that is no problem. + for (int x=0; x<2048; ++x) + addr.push_back(b.alloc(x+1)); + for (int x=0; x<2048; ++x) + b.free(addr[((x*23)%2048)^242]); + addr.clear(); + + // Go entirely wild: free and alloc interleaved, + // generate targets and sizes using pseudo-randomness. + for (int x=0; x<2048; ++x) + addr.push_back(nullptr); + uint32_t s = 0x12345678; + for (int x=0; x<5000; ++x) { + int idx = s & (addr.size()-1); + if (s & 0x80000000) { + b.free(addr[idx]); + addr[idx] = nullptr; + } else if(!addr[idx]) { + addr[idx] = b.alloc((s >> 16) & 2047); + } + bool lsb = s & 1; + s >>= 1; + if (lsb) + s ^= 0xf00f00f0; // LFSR period 0xf7ffffe0 } -}; + for (void *ptr: addr) + b.free(ptr); + addr.clear(); + + BOOST_CHECK(b.stats().total == synth_size); + BOOST_CHECK(b.stats().free == synth_size); +} -BOOST_AUTO_TEST_CASE(test_LockedPageManagerBase) +/** Mock LockedPageAllocator for testing */ +class TestLockedPageAllocator: public LockedPageAllocator { - const size_t test_page_size = 4096; - LockedPageManagerBase lpm(test_page_size); - size_t addr; - last_lock_addr = last_unlock_addr = 0; - last_lock_len = last_unlock_len = 0; - - /* Try large number of small objects */ - addr = 0; - for(int i=0; i<1000; ++i) - { - lpm.LockRange(reinterpret_cast(addr), 33); - addr += 33; - } - /* Try small number of page-sized objects, straddling two pages */ - addr = test_page_size*100 + 53; - for(int i=0; i<100; ++i) - { - lpm.LockRange(reinterpret_cast(addr), test_page_size); - addr += test_page_size; - } - /* Try small number of page-sized objects aligned to exactly one page */ - addr = test_page_size*300; - for(int i=0; i<100; ++i) - { - lpm.LockRange(reinterpret_cast(addr), test_page_size); - addr += test_page_size; - } - /* one very large object, straddling pages */ - lpm.LockRange(reinterpret_cast(test_page_size*600+1), test_page_size*500); - BOOST_CHECK(last_lock_addr == reinterpret_cast(test_page_size*(600+500))); - /* one very large object, page aligned */ - lpm.LockRange(reinterpret_cast(test_page_size*1200), test_page_size*500-1); - BOOST_CHECK(last_lock_addr == reinterpret_cast(test_page_size*(1200+500-1))); - - BOOST_CHECK(lpm.GetLockedPageCount() == ( - (1000*33+test_page_size-1)/test_page_size + // small objects - 101 + 100 + // page-sized objects - 501 + 500)); // large objects - BOOST_CHECK((last_lock_len & (test_page_size-1)) == 0); // always lock entire pages - BOOST_CHECK(last_unlock_len == 0); // nothing unlocked yet - - /* And unlock again */ - addr = 0; - for(int i=0; i<1000; ++i) +public: + TestLockedPageAllocator(int count_in, int lockedcount_in): count(count_in), lockedcount(lockedcount_in) {} + void* AllocateLocked(size_t len, bool *lockingSuccess) override { - lpm.UnlockRange(reinterpret_cast(addr), 33); - addr += 33; + *lockingSuccess = false; + if (count > 0) { + --count; + + if (lockedcount > 0) { + --lockedcount; + *lockingSuccess = true; + } + + return reinterpret_cast(uint64_t{static_cast(0x08000000) + (count << 24)}); // Fake address, do not actually use this memory + } + return nullptr; } - addr = test_page_size*100 + 53; - for(int i=0; i<100; ++i) + void FreeLocked(void* addr, size_t len) override { - lpm.UnlockRange(reinterpret_cast(addr), test_page_size); - addr += test_page_size; } - addr = test_page_size*300; - for(int i=0; i<100; ++i) + size_t GetLimit() override { - lpm.UnlockRange(reinterpret_cast(addr), test_page_size); - addr += test_page_size; + return std::numeric_limits::max(); } - lpm.UnlockRange(reinterpret_cast(test_page_size*600+1), test_page_size*500); - lpm.UnlockRange(reinterpret_cast(test_page_size*1200), test_page_size*500-1); +private: + int count; + int lockedcount; +}; - /* Check that everything is released */ - BOOST_CHECK(lpm.GetLockedPageCount() == 0); +BOOST_AUTO_TEST_CASE(lockedpool_tests_mock) +{ + // Test over three virtual arenas, of which one will succeed being locked + std::unique_ptr x = MakeUnique(3, 1); + LockedPool pool(std::move(x)); + BOOST_CHECK(pool.stats().total == 0); + BOOST_CHECK(pool.stats().locked == 0); - /* A few and unlocks of size zero (should have no effect) */ - addr = 0; - for(int i=0; i<1000; ++i) - { - lpm.LockRange(reinterpret_cast(addr), 0); - addr += 1; - } - BOOST_CHECK(lpm.GetLockedPageCount() == 0); - addr = 0; - for(int i=0; i<1000; ++i) + // Ensure unreasonable requests are refused without allocating anything + void *invalid_toosmall = pool.alloc(0); + BOOST_CHECK(invalid_toosmall == nullptr); + BOOST_CHECK(pool.stats().used == 0); + BOOST_CHECK(pool.stats().free == 0); + void *invalid_toobig = pool.alloc(LockedPool::ARENA_SIZE+1); + BOOST_CHECK(invalid_toobig == nullptr); + BOOST_CHECK(pool.stats().used == 0); + BOOST_CHECK(pool.stats().free == 0); + + void *a0 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a0); + BOOST_CHECK(pool.stats().locked == LockedPool::ARENA_SIZE); + void *a1 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a1); + void *a2 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a2); + void *a3 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a3); + void *a4 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a4); + void *a5 = pool.alloc(LockedPool::ARENA_SIZE / 2); + BOOST_CHECK(a5); + // We've passed a count of three arenas, so this allocation should fail + void *a6 = pool.alloc(16); + BOOST_CHECK(!a6); + + pool.free(a0); + pool.free(a2); + pool.free(a4); + pool.free(a1); + pool.free(a3); + pool.free(a5); + BOOST_CHECK(pool.stats().total == 3*LockedPool::ARENA_SIZE); + BOOST_CHECK(pool.stats().locked == LockedPool::ARENA_SIZE); + BOOST_CHECK(pool.stats().used == 0); +} + +// These tests used the live LockedPoolManager object, this is also used +// by other tests so the conditions are somewhat less controllable and thus the +// tests are somewhat more error-prone. +BOOST_AUTO_TEST_CASE(lockedpool_tests_live) +{ + LockedPoolManager &pool = LockedPoolManager::Instance(); + LockedPool::Stats initial = pool.stats(); + + void *a0 = pool.alloc(16); + BOOST_CHECK(a0); + // Test reading and writing the allocated memory + *((uint32_t*)a0) = 0x1234; + BOOST_CHECK(*((uint32_t*)a0) == 0x1234); + + pool.free(a0); + try { // Test exception on double-free + pool.free(a0); + BOOST_CHECK(0); + } catch(std::runtime_error &) { - lpm.UnlockRange(reinterpret_cast(addr), 0); - addr += 1; } - BOOST_CHECK(lpm.GetLockedPageCount() == 0); - BOOST_CHECK((last_unlock_len & (test_page_size-1)) == 0); // always unlock entire pages + // If more than one new arena was allocated for the above tests, something is wrong + BOOST_CHECK(pool.stats().total <= (initial.total + LockedPool::ARENA_SIZE)); + // Usage must be back to where it started + BOOST_CHECK(pool.stats().used == initial.used); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util.cpp b/src/util.cpp index ed1512160f..a40f041ff4 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -112,8 +112,6 @@ void locking_callback(int mode, int i, const char* file, int line) NO_THREAD_SAF } } -LockedPageManager LockedPageManager::instance; - // Init class CInit { diff --git a/src/util/memory.h b/src/util/memory.h new file mode 100644 index 0000000000..15ecf8f80d --- /dev/null +++ b/src/util/memory.h @@ -0,0 +1,19 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_MEMORY_H +#define BITCOIN_UTIL_MEMORY_H + +#include +#include + +//! Substitute for C++14 std::make_unique. +template +std::unique_ptr MakeUnique(Args&&... args) +{ + return std::unique_ptr(new T(std::forward(args)...)); +} + +#endif From c78743d997f2d2bb198a505b399119b6f03f9502 Mon Sep 17 00:00:00 2001 From: Cy Rossignol Date: Sun, 22 Sep 2019 13:09:41 -0500 Subject: [PATCH 2/7] Use agnostic memory zeroing from Bitcoin This replaces the remaining OPENSSL_cleanse() calls with Bitcoin's memory_cleanse() function. --- src/crypter.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crypter.cpp b/src/crypter.cpp index 98d57945b2..7213bfc3c5 100644 --- a/src/crypter.cpp +++ b/src/crypter.cpp @@ -29,14 +29,14 @@ bool CCrypter::SetKeyFromPassphrase(const SecureString& strKeyData, const std::v i = EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha512(), &chSalt[0], (unsigned char *)&scryptHash, sizeof scryptHash, nRounds, vchKey.data(), vchIV.data()); - OPENSSL_cleanse(&scryptHash, sizeof scryptHash); + memory_cleanse(&scryptHash, sizeof scryptHash); } if (i != (int)WALLET_CRYPTO_KEY_SIZE) { - OPENSSL_cleanse(vchKey.data(), vchKey.size()); - OPENSSL_cleanse(vchIV.data(), vchIV.size()); + memory_cleanse(vchKey.data(), vchKey.size()); + memory_cleanse(vchIV.data(), vchIV.size()); return false; } From ec20ed758bdf73ae62a58c679ed01b35b1c40dc0 Mon Sep 17 00:00:00 2001 From: Cy Rossignol Date: Sun, 22 Sep 2019 13:35:57 -0500 Subject: [PATCH 3/7] Port streams.h from Bitcoin Bitcoin moved the stream-like classes out of serialize.h into streams.h and added several new classes. This refactors the application to use the new file for code that uses the existing stream objects. --- src/Makefile.am | 1 + src/addrman.cpp | 2 + src/alert.cpp | 1 + src/db.h | 1 + src/kernel.cpp | 1 + src/main.cpp | 1 + src/net.h | 1 + src/qt/signverifymessagedialog.cpp | 1 + src/rpcnet.cpp | 1 + src/rpcrawtransaction.cpp | 1 + src/rpcwallet.cpp | 1 + src/scraper_net.cpp | 4 +- src/scraper_net.h | 10 +- src/script.cpp | 1 + src/serialize.h | 572 +--------------- src/streams.h | 857 ++++++++++++++++++++++++ src/test/neuralnet/cpid_tests.cpp | 1 + src/test/neuralnet/superblock_tests.cpp | 1 + src/test/serialize_tests.cpp | 1 + src/test/transaction_tests.cpp | 1 + src/test/uint256_tests.cpp | 1 + src/txdb-leveldb.h | 1 + src/wallet.h | 1 + src/walletdb.cpp | 1 + 24 files changed, 886 insertions(+), 578 deletions(-) create mode 100644 src/streams.h diff --git a/src/Makefile.am b/src/Makefile.am index ddd5b42515..a68c0435c2 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -115,6 +115,7 @@ GRIDCOIN_CORE_H = \ script.h \ scrypt.h \ serialize.h \ + streams.h \ strlcpy.h \ support/allocators/secure.h \ support/allocators/zeroafterfree.h \ diff --git a/src/addrman.cpp b/src/addrman.cpp index 99f783892e..4819051cc2 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -3,6 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "addrman.h" +#include "streams.h" + using namespace std; int CAddrInfo::GetTriedBucket(const std::vector &nKey) const diff --git a/src/alert.cpp b/src/alert.cpp index 35ce9bf877..3790e22680 100644 --- a/src/alert.cpp +++ b/src/alert.cpp @@ -11,6 +11,7 @@ #include "alert.h" #include "key.h" #include "net.h" +#include "streams.h" #include "sync.h" #include "ui_interface.h" #include "uint256.h" diff --git a/src/db.h b/src/db.h index c7eff08816..89787936d5 100644 --- a/src/db.h +++ b/src/db.h @@ -6,6 +6,7 @@ #define BITCOIN_DB_H #include "main.h" +#include "streams.h" #include #include diff --git a/src/kernel.cpp b/src/kernel.cpp index 308e76834a..5333e61b8c 100644 --- a/src/kernel.cpp +++ b/src/kernel.cpp @@ -6,6 +6,7 @@ #include "kernel.h" #include "txdb.h" #include "main.h" +#include "streams.h" #include "util.h" using namespace std; diff --git a/src/main.cpp b/src/main.cpp index 25de8e7882..17c91cb1dc 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -5,6 +5,7 @@ #include "util.h" #include "net.h" +#include "streams.h" #include "alert.h" #include "checkpoints.h" #include "db.h" diff --git a/src/net.h b/src/net.h index 16ae94aab7..8be272ae9b 100644 --- a/src/net.h +++ b/src/net.h @@ -14,6 +14,7 @@ #include "netbase.h" #include "mruset.h" #include "protocol.h" +#include "streams.h" #include "addrman.h" #ifndef WIN32 diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index bb9dba9a9f..73d21f7d3a 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -7,6 +7,7 @@ #include "init.h" #include "main.h" #include "optionsmodel.h" +#include "streams.h" #include "walletmodel.h" #include "wallet.h" diff --git a/src/rpcnet.cpp b/src/rpcnet.cpp index 59ac8effe5..b995ec4037 100644 --- a/src/rpcnet.cpp +++ b/src/rpcnet.cpp @@ -7,6 +7,7 @@ #include "alert.h" #include "wallet.h" #include "db.h" +#include "streams.h" #include "walletdb.h" #include "net.h" #include "banman.h" diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index aa82f937f4..3c1fc1be08 100755 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -12,6 +12,7 @@ #include "init.h" #include "main.h" #include "net.h" +#include "streams.h" #include "wallet.h" #include "coincontrol.h" diff --git a/src/rpcwallet.cpp b/src/rpcwallet.cpp index f3ac64c335..c6c9d42869 100644 --- a/src/rpcwallet.cpp +++ b/src/rpcwallet.cpp @@ -12,6 +12,7 @@ #include "init.h" #include "base58.h" #include "backup.h" +#include "streams.h" #include "util.h" #include diff --git a/src/scraper_net.cpp b/src/scraper_net.cpp index 25612a6630..5ec2ade630 100644 --- a/src/scraper_net.cpp +++ b/src/scraper_net.cpp @@ -226,7 +226,7 @@ void CScraperManifest::dentry::Serialize(CDataStream& ss, int nType, int nVersio ss<< current; ss<< last; } -void CScraperManifest::dentry::Unserialize(CReaderStream& ss, int nType, int nVersion) +void CScraperManifest::dentry::Unserialize(CDataStream& ss, int nType, int nVersion) { ss>> project; ss>> ETag; @@ -368,7 +368,7 @@ bool CScraperManifest::IsManifestAuthorized(CPubKey& PubKey, unsigned int& bansc } -void CScraperManifest::UnserializeCheck(CReaderStream& ss, unsigned int& banscore_out) +void CScraperManifest::UnserializeCheck(CDataStream& ss, unsigned int& banscore_out) { const auto pbegin = ss.begin(); diff --git a/src/scraper_net.h b/src/scraper_net.h index 838ba7f5d6..db5b948e47 100755 --- a/src/scraper_net.h +++ b/src/scraper_net.h @@ -7,10 +7,11 @@ * polymorphism. */ +#include #include "net.h" +#include "streams.h" #include "sync.h" -#include #include @@ -27,7 +28,7 @@ class CSplitBlob CPart(const uint256& ihash) :hash(ihash) {} - CReaderStream getReader() const { return CReaderStream(&data); } + CDataStream getReader() const { return CDataStream(&data); } bool present() const {return !this->data.empty();} }; @@ -141,7 +142,7 @@ class CScraperManifest bool last =0; void Serialize(CDataStream& s, int nType, int nVersion) const; - void Unserialize(CReaderStream& s, int nType, int nVersion); + void Unserialize(CDataStream& s, int nType, int nVersion); UniValue ToJson() const; }; @@ -169,10 +170,9 @@ class CScraperManifest void Serialize(CDataStream& s, int nType, int nVersion) const; void SerializeWithoutSignature(CDataStream& s, int nType, int nVersion) const; void SerializeForManifestCompare(CDataStream& ss, int nType, int nVersion) const; - void UnserializeCheck(CReaderStream& s, unsigned int& banscore_out); + void UnserializeCheck(CDataStream& s, unsigned int& banscore_out); bool IsManifestCurrent() const; UniValue ToJson() const; - }; diff --git a/src/script.cpp b/src/script.cpp index 42a0dbd29e..6b53f2f0bb 100644 --- a/src/script.cpp +++ b/src/script.cpp @@ -10,6 +10,7 @@ using namespace std; #include "bignum.h" #include "key.h" #include "main.h" +#include "streams.h" #include "sync.h" #include "util.h" diff --git a/src/serialize.h b/src/serialize.h index 29276d47fb..d925e57e60 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -23,8 +23,6 @@ #include "support/allocators/zeroafterfree.h" #include "version.h" -class CAutoFile; -class CDataStream; class CScript; static const unsigned int MAX_SIZE = 0x02000000; @@ -37,23 +35,6 @@ inline T& REF(const T& val) return const_cast(val); } -/** - * Used to acquire a non-const pointer "this" to generate bodies - * of const serialization operations from a template - */ -template -inline T* NCONST_PTR(const T* val) -{ - return const_cast(val); -} - -//! Safely convert odd char pointer types to standard ones. -inline char* CharCast(char* c) { return c; } -inline char* CharCast(unsigned char* c) { return (char*)c; } -inline const char* CharCast(const char* c) { return c; } -inline const char* CharCast(const unsigned char* c) { return (const char*)c; } - - ///////////////////////////////////////////////////////////////// // // Templates for serializing to anything that looks like a stream, @@ -70,8 +51,6 @@ enum // modifiers SER_SKIPSIG = (1 << 16), SER_BLOCKHEADERONLY = (1 << 17), - - // Bits 24-31 are reserved for implementation-specific modifiers. }; #define IMPLEMENT_SERIALIZE(statements) \ @@ -115,28 +94,6 @@ enum #define READWRITE(obj) (nSerSize += ::SerReadWrite(s, (obj), nType, nVersion, ser_action)) -//! Convert the reference base type to X, without changing constness or reference type. -template X& ReadWriteAsHelper(X& x) { return x; } -template const X& ReadWriteAsHelper(const X& x) { return x; } - -#define READWRITEVARIADIC(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__)) -#define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper(obj))) - -/** - * Implement three methods for serializable objects. These are actually wrappers over - * "SerializationOp" template, which implements the body of each class' serialization - * code. Adding "ADD_SERIALIZE_METHODS" in the body of the class causes these wrappers to be - * added as members. - */ -#define ADD_SERIALIZE_METHODS \ - template \ - void Serialize(Stream& s, int, int) const { \ - NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \ - } \ - template \ - void Unserialize(Stream& s, int, int) { \ - SerializationOp(s, CSerActionUnserialize()); \ - } @@ -174,8 +131,6 @@ template inline void Serialize(Stream& s, signed long long a, template inline void Serialize(Stream& s, unsigned long long a, int, int=0) { WRITEDATA(s, a); } template inline void Serialize(Stream& s, float a, int, int=0) { WRITEDATA(s, a); } template inline void Serialize(Stream& s, double a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, const char (&a)[N], int , int=0) { s.write(a, N); } -template inline void Serialize(Stream& s, const unsigned char (&a)[N], int , int=0) { s.write(CharCast(a), N); } template inline void Unserialize(Stream& s, char& a, int, int=0) { READDATA(s, a); } template inline void Unserialize(Stream& s, signed char& a, int, int=0) { READDATA(s, a); } @@ -190,8 +145,6 @@ template inline void Unserialize(Stream& s, signed long long& a template inline void Unserialize(Stream& s, unsigned long long& a, int, int=0) { READDATA(s, a); } template inline void Unserialize(Stream& s, float& a, int, int=0) { READDATA(s, a); } template inline void Unserialize(Stream& s, double& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, char (&a)[N], int, int=0) { s.read(a, N); } -template inline void Unserialize(Stream& s, unsigned char (&a)[N], int , int=0) { s.read(CharCast(a), N); } inline unsigned int GetSerializeSize(bool a, int, int=0) { return sizeof(char); } template inline void Serialize(Stream& s, bool a, int, int=0) { char f=a; WRITEDATA(s, f); } @@ -201,6 +154,7 @@ template inline void Unserialize(Stream& s, bool& a, int, int=0 + // // Compact size // size < 253 -- 1 byte @@ -913,50 +867,6 @@ struct ser_streamplaceholder }; -template -void SerializeMany(Stream& s) -{ -} - -template -void SerializeMany(Stream& s, const Arg& arg, const Args&... args) -{ - ::Serialize(s, arg, 0 /* type, unused */, 0 /* version, unused */); - ::SerializeMany(s, args...); -} - -template -inline void UnserializeMany(Stream& s) -{ -} - -template -inline void UnserializeMany(Stream& s, Arg&& arg, Args&&... args) -{ - ::Unserialize(s, arg, 0 /* type, unused */, 0 /* version, unused */); - ::UnserializeMany(s, args...); -} - -template -inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, const Args&... args) -{ - ::SerializeMany(s, args...); -} - -template -inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&&... args) -{ - ::UnserializeMany(s, args...); -} - - - - - - - - - typedef std::vector > CSerializeData; class CSizeComputer @@ -988,484 +898,4 @@ class CSizeComputer } }; -/** Double ended buffer wrapper combining vector and stream-like interfaces. - * - * Operator >> read unformatted data using the above serialization templates. - * Takes the buffer by reference which means it must reamin valid. - */ -class CReaderStream -{ - protected: - typedef CSerializeData vector_type; - const vector_type* vchro; - protected: - unsigned int nReadPos; - short state; - short exceptmask; - - public: - - typedef vector_type::size_type size_type; - typedef vector_type::difference_type difference_type; - typedef vector_type::const_reference const_reference; - typedef vector_type::value_type value_type; - typedef vector_type::const_iterator const_iterator; - - int nType; - int nVersion; - - CReaderStream(const CSerializeData* ivch, unsigned int iPos = 0, int iType = SER_NETWORK, int iVersion = 1) - :vchro(ivch) - ,nReadPos(iPos) - ,state(0) - ,exceptmask(std::ios::badbit | std::ios::failbit) - ,nType(iType) - ,nVersion(iVersion) - {}; - - std::string str() const - { - return (std::string(begin(), end())); - } - - // - // Vector subset - // - const_iterator begin() const { return vchro->begin() + nReadPos; } - const_iterator end() const { return vchro->end(); } - size_type size() const { return vchro->size() - nReadPos; } - bool empty() const { return vchro->size() == nReadPos; } - - bool Rewind(size_type n) - { - // Rewind by n characters if the buffer hasn't been compacted yet - if (n > nReadPos) - return false; - nReadPos -= n; - return true; - } - - - // - // Stream subset - // - void setstate(short bits, const char* psz) - { - state |= bits; - if (state & exceptmask) - throw std::ios_base::failure(psz); - } - - bool eof() const { return size() == 0; } - bool fail() const { return state & (std::ios::badbit | std::ios::failbit); } - bool good() const { return !eof() && (state == 0); } - void clear(short n) { state = n; } // name conflict with vector clear() - short exceptions() { return exceptmask; } - short exceptions(short mask) { short prev = exceptmask; exceptmask = mask; setstate(0, "CDataStream"); return prev; } - CReaderStream* rdbuf() { return this; } - - int in_avail() { return size(); } - void SetType(int n) { nType = n; } - int GetType() { return nType; } - void SetVersion(int n) { nVersion = n; } - int GetVersion() { return nVersion; } - void ReadVersion() { *this >> nVersion; } - - CReaderStream& read(char* pch, int nSize) - { - // Read from the beginning of the buffer - assert(nSize >= 0); - unsigned int nReadPosNext = nReadPos + nSize; - if (nReadPosNext > vchro->size()) - { - setstate(std::ios::failbit, "CDataStream::read() : end of data"); - memset(pch, 0, nSize); - if (vchro->size() > nReadPos) - nSize = vchro->size() - nReadPos; - else nSize= 0; - assert((nReadPos + nSize) == vchro->size()); - nReadPosNext= nReadPos + nSize; - } - memcpy(pch, &(*vchro)[nReadPos], nSize); - nReadPos = nReadPosNext; - return (*this); - } - - CReaderStream& ignore(int nSize) - { - // Ignore from the beginning of the buffer - assert(nSize >= 0); - unsigned int nReadPosNext = nReadPos + nSize; - if (nReadPosNext >= vchro->size()) - { - if (nReadPosNext > vchro->size()) - setstate(std::ios::failbit, "CDataStream::ignore() : end of data"); - /* FIXME: this was here in CDataStream, but now vchro is const so we can not do this. - nReadPos = 0; - vchro->clear(); - */ - return (*this); - } - nReadPos = nReadPosNext; - return (*this); - } - - template - void Serialize(Stream& s, int nType, int nVersion) const - { - // Special case: stream << stream concatenates like stream += stream - if (!vchro->empty()) - s.write((char*)&(*vchro)[0], vchro->size() * sizeof((*vchro)[0])); - } - - template - CReaderStream& operator>>(T&& obj) - { - // Unserialize from this stream - ::Unserialize(*this, obj, nType, nVersion); - return (*this); - } - -}; - -/** Double ended buffer combining vector and stream-like interfaces. - * - * >> and << read and write unformatted data using the above serialization templates. - * Fills with data in linear time; some stringstream implementations take N^2 time. - */ -class CDataStream - :public CReaderStream -{ -protected: - vector_type vch; -public: - - typedef vector_type::allocator_type allocator_type; - typedef vector_type::reference reference; - typedef vector_type::iterator iterator; - typedef vector_type::const_iterator const_iterator; - typedef vector_type::reverse_iterator reverse_iterator; - - explicit CDataStream(int nTypeIn, int nVersionIn) - :CReaderStream(&vch,0,nTypeIn,nVersionIn) - {assert(&vch==vchro);} - - CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) - :CReaderStream(&vch,0,nTypeIn,nVersionIn) - ,vch(pbegin, pend) - {assert(&vch==vchro);} - -#if !defined(_MSC_VER) || _MSC_VER >= 1300 - CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) - :CReaderStream(&vch,0,nTypeIn,nVersionIn) - ,vch(pbegin, pend) - {assert(&vch==vchro);} -#endif - - CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) - :CReaderStream(&vch,0,nTypeIn,nVersionIn) - ,vch(vchIn.begin(), vchIn.end()) - {assert(&vch==vchro);} - - CDataStream(const std::vector& vchIn, int nTypeIn, int nVersionIn) - :CReaderStream(&vch,0,nTypeIn,nVersionIn) - ,vch(vchIn.begin(), vchIn.end()) - {assert(&vch==vchro);} - - CDataStream(const std::vector& vchIn, int nTypeIn, int nVersionIn) - :CReaderStream(&vch,0,nTypeIn,nVersionIn) - ,vch((char*)&vchIn.begin()[0], (char*)&vchIn.end()[0]) - {assert(&vch==vchro);} - - CDataStream& operator+=(const CReaderStream& b) - { - vch.insert(vch.end(), b.begin(), b.end()); - return *this; - } - - - // - // Vector subset - // - iterator begin() { return vch.begin() + nReadPos; } - const_iterator begin() const { return vch.begin() + nReadPos; } - iterator end() { return vch.end(); } - const_iterator end() const { return vch.end(); } - void resize(size_type n, value_type c=0) { vch.resize(n + nReadPos, c); } - void reserve(size_type n) { vch.reserve(n + nReadPos); } - const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } - reference operator[](size_type pos) { return vch[pos + nReadPos]; } - void clear() { vch.clear(); nReadPos = 0; } - iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); } - void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); } - - void insert(iterator it, std::vector::const_iterator first, std::vector::const_iterator last) - { - assert(last - first >= 0); - if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) - { - // special case for inserting at the front when there's room - nReadPos -= (last - first); - memcpy(&vch[nReadPos], &first[0], last - first); - } - else - vch.insert(it, first, last); - } - -#if !defined(_MSC_VER) || _MSC_VER >= 1300 - void insert(iterator it, const char* first, const char* last) - { - assert(last - first >= 0); - if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) - { - // special case for inserting at the front when there's room - nReadPos -= (last - first); - memcpy(&vch[nReadPos], &first[0], last - first); - } - else - vch.insert(it, first, last); - } -#endif - - iterator erase(iterator it) - { - if (it == vch.begin() + nReadPos) - { - // special case for erasing from the front - if (++nReadPos >= vch.size()) - { - // whenever we reach the end, we take the opportunity to clear the buffer - nReadPos = 0; - return vch.erase(vch.begin(), vch.end()); - } - return vch.begin() + nReadPos; - } - else - return vch.erase(it); - } - - iterator erase(iterator first, iterator last) - { - if (first == vch.begin() + nReadPos) - { - // special case for erasing from the front - if (last == vch.end()) - { - nReadPos = 0; - return vch.erase(vch.begin(), vch.end()); - } - else - { - nReadPos = (last - vch.begin()); - return last; - } - } - else - return vch.erase(first, last); - } - - inline void Compact() - { - vch.erase(vch.begin(), vch.begin() + nReadPos); - nReadPos = 0; - } - - - // - // Stream subset - // - - void WriteVersion() { *this << nVersion; } - - CDataStream& write(const char* pch, int nSize) - { - // Write to the end of the buffer - assert(nSize >= 0); - vch.insert(vch.end(), pch, pch + nSize); - return (*this); - } - - template - unsigned int GetSerializeSize(const T& obj) - { - // Tells the size of the object if serialized to this stream - return ::GetSerializeSize(obj, nType, nVersion); - } - - template - CDataStream& operator<<(const T& obj) - { - // Serialize to this stream - ::Serialize(*this, obj, nType, nVersion); - return (*this); - } - - void GetAndClear(CSerializeData &data) { - data.insert(data.end(), begin(), end()); - clear(); - } - - - CDataStream& operator= (CDataStream&& s) { - /* c++ is being mean so I had to define this */ - vch= std::move(s.vch); - vchro = &vch; - nReadPos= s.nReadPos; - state= s.state; - exceptmask= s.exceptmask; - nType= s.nType; - nVersion= s.nVersion; - return *this; - } - - CDataStream (const CDataStream& s) - :CReaderStream(&vch,s.nType,s.nVersion) - { - /* c++ is being mean so I had to define this */ - vch= s.vch; - vchro = &vch; - nReadPos= s.nReadPos; - state= s.state; - exceptmask= s.exceptmask; - nType= s.nType; - nVersion= s.nVersion; - } - CDataStream (const CDataStream&& s) = delete; - CDataStream& operator= (CDataStream& s) = delete; -}; - - - - - - - - - - -/** RAII wrapper for FILE*. - * - * Will automatically close the file when it goes out of scope if not null. - * If you're returning the file pointer, return file.release(). - * If you need to close the file early, use file.fclose() instead of fclose(file). - */ -class CAutoFile -{ -protected: - FILE* file; - short state; - short exceptmask; -public: - int nType; - int nVersion; - - CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) - { - file = filenew; - nType = nTypeIn; - nVersion = nVersionIn; - state = 0; - exceptmask = std::ios::badbit | std::ios::failbit; - } - - ~CAutoFile() - { - fclose(); - } - - void fclose() - { - if (file != NULL && file != stdin && file != stdout && file != stderr) - ::fclose(file); - file = NULL; - } - - FILE* release() { FILE* ret = file; file = NULL; return ret; } - operator FILE*() { return file; } - FILE* operator->() { return file; } - FILE& operator*() { return *file; } - // FILE** operator&() { return &file; } - FILE* operator=(FILE* pnew) { return file = pnew; } - bool operator!() { return (file == NULL); } - - - // - // Stream subset - // - void setstate(short bits, const char* psz) - { - state |= bits; - if (state & exceptmask) - throw std::ios_base::failure(psz); - } - - bool fail() const { return state & (std::ios::badbit | std::ios::failbit); } - bool good() const { return state == 0; } - void clear(short n = 0) { state = n; } - short exceptions() { return exceptmask; } - short exceptions(short mask) { short prev = exceptmask; exceptmask = mask; setstate(0, "CAutoFile"); return prev; } - - void SetType(int n) { nType = n; } - int GetType() { return nType; } - void SetVersion(int n) { nVersion = n; } - int GetVersion() { return nVersion; } - void ReadVersion() { *this >> nVersion; } - void WriteVersion() { *this << nVersion; } - - /** Get wrapped FILE* without transfer of ownership. - * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the - * CAutoFile outlives use of the passed pointer. - */ - FILE* Get() const { return file; } - - /** Return true if the wrapped FILE* is nullptr, false otherwise. - */ - bool IsNull() const { return (file == nullptr); } - - CAutoFile& read(char* pch, size_t nSize) - { - if (!file) - throw std::ios_base::failure("CAutoFile::read : file handle is NULL"); - if (fread(pch, 1, nSize, file) != nSize) - setstate(std::ios::failbit, feof(file) ? "CAutoFile::read : end of file" : "CAutoFile::read : fread failed"); - return (*this); - } - - CAutoFile& write(const char* pch, size_t nSize) - { - if (!file) - throw std::ios_base::failure("CAutoFile::write : file handle is NULL"); - if (fwrite(pch, 1, nSize, file) != nSize) - setstate(std::ios::failbit, "CAutoFile::write : write failed"); - return (*this); - } - - template - unsigned int GetSerializeSize(const T& obj) - { - // Tells the size of the object if serialized to this stream - return ::GetSerializeSize(obj, nType, nVersion); - } - - template - CAutoFile& operator<<(const T& obj) - { - // Serialize to this stream - if (!file) - throw std::ios_base::failure("CAutoFile::operator<< : file handle is NULL"); - ::Serialize(*this, obj, nType, nVersion); - return (*this); - } - - template - CAutoFile& operator>>(T& obj) - { - // Unserialize from this stream - if (!file) - throw std::ios_base::failure("CAutoFile::operator>> : file handle is NULL"); - ::Unserialize(*this, obj, nType, nVersion); - return (*this); - } -}; - #endif diff --git a/src/streams.h b/src/streams.h new file mode 100644 index 0000000000..4e600f1826 --- /dev/null +++ b/src/streams.h @@ -0,0 +1,857 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_STREAMS_H +#define BITCOIN_STREAMS_H + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +template +class OverrideStream +{ + Stream* stream; + + const int nType; + const int nVersion; + +public: + OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {} + + template + OverrideStream& operator<<(const T& obj) + { + // Serialize to this stream + ::Serialize(*this, obj); + return (*this); + } + + template + OverrideStream& operator>>(T&& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } + + void write(const char* pch, size_t nSize) + { + stream->write(pch, nSize); + } + + void read(char* pch, size_t nSize) + { + stream->read(pch, nSize); + } + + int GetVersion() const { return nVersion; } + int GetType() const { return nType; } + size_t size() const { return stream->size(); } +}; + +/* Minimal stream for overwriting and/or appending to an existing byte vector + * + * The referenced vector will grow as necessary + */ +class CVectorWriter +{ + public: + +/* + * @param[in] nTypeIn Serialization Type + * @param[in] nVersionIn Serialization Version (including any flags) + * @param[in] vchDataIn Referenced byte vector to overwrite/append + * @param[in] nPosIn Starting position. Vector index where writes should start. The vector will initially + * grow as necessary to max(nPosIn, vec.size()). So to append, use vec.size(). +*/ + CVectorWriter(int nTypeIn, int nVersionIn, std::vector& vchDataIn, size_t nPosIn) : nType(nTypeIn), nVersion(nVersionIn), vchData(vchDataIn), nPos(nPosIn) + { + if(nPos > vchData.size()) + vchData.resize(nPos); + } +/* + * (other params same as above) + * @param[in] args A list of items to serialize starting at nPosIn. +*/ + template + CVectorWriter(int nTypeIn, int nVersionIn, std::vector& vchDataIn, size_t nPosIn, Args&&... args) : CVectorWriter(nTypeIn, nVersionIn, vchDataIn, nPosIn) + { + ::SerializeMany(*this, std::forward(args)...); + } + void write(const char* pch, size_t nSize) + { + assert(nPos <= vchData.size()); + size_t nOverwrite = std::min(nSize, vchData.size() - nPos); + if (nOverwrite) { + memcpy(vchData.data() + nPos, reinterpret_cast(pch), nOverwrite); + } + if (nOverwrite < nSize) { + vchData.insert(vchData.end(), reinterpret_cast(pch) + nOverwrite, reinterpret_cast(pch) + nSize); + } + nPos += nSize; + } + template + CVectorWriter& operator<<(const T& obj) + { + // Serialize to this stream + ::Serialize(*this, obj); + return (*this); + } + int GetVersion() const + { + return nVersion; + } + int GetType() const + { + return nType; + } +private: + const int nType; + const int nVersion; + std::vector& vchData; + size_t nPos; +}; + +/** Minimal stream for reading from an existing vector by reference + */ +class VectorReader +{ +private: + const int m_type; + const int m_version; + const std::vector& m_data; + size_t m_pos = 0; + +public: + + /** + * @param[in] type Serialization Type + * @param[in] version Serialization Version (including any flags) + * @param[in] data Referenced byte vector to overwrite/append + * @param[in] pos Starting position. Vector index where reads should start. + */ + VectorReader(int type, int version, const std::vector& data, size_t pos) + : m_type(type), m_version(version), m_data(data), m_pos(pos) + { + if (m_pos > m_data.size()) { + throw std::ios_base::failure("VectorReader(...): end of data (m_pos > m_data.size())"); + } + } + + /** + * (other params same as above) + * @param[in] args A list of items to deserialize starting at pos. + */ + template + VectorReader(int type, int version, const std::vector& data, size_t pos, + Args&&... args) + : VectorReader(type, version, data, pos) + { + ::UnserializeMany(*this, std::forward(args)...); + } + + template + VectorReader& operator>>(T& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } + + int GetVersion() const { return m_version; } + int GetType() const { return m_type; } + + size_t size() const { return m_data.size() - m_pos; } + bool empty() const { return m_data.size() == m_pos; } + + void read(char* dst, size_t n) + { + if (n == 0) { + return; + } + + // Read from the beginning of the buffer + size_t pos_next = m_pos + n; + if (pos_next > m_data.size()) { + throw std::ios_base::failure("VectorReader::read(): end of data"); + } + memcpy(dst, m_data.data() + m_pos, n); + m_pos = pos_next; + } +}; + +/** Double ended buffer combining vector and stream-like interfaces. + * + * >> and << read and write unformatted data using the above serialization templates. + * Fills with data in linear time; some stringstream implementations take N^2 time. + */ +class CDataStream +{ +protected: + typedef CSerializeData vector_type; + vector_type vch; + unsigned int nReadPos; + + int nType; + int nVersion; +public: + + typedef vector_type::allocator_type allocator_type; + typedef vector_type::size_type size_type; + typedef vector_type::difference_type difference_type; + typedef vector_type::reference reference; + typedef vector_type::const_reference const_reference; + typedef vector_type::value_type value_type; + typedef vector_type::iterator iterator; + typedef vector_type::const_iterator const_iterator; + typedef vector_type::reverse_iterator reverse_iterator; + + explicit CDataStream(int nTypeIn, int nVersionIn) + { + Init(nTypeIn, nVersionIn); + } + + CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) + { + Init(nTypeIn, nVersionIn); + } + + CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) + { + Init(nTypeIn, nVersionIn); + } + + CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) + { + Init(nTypeIn, nVersionIn); + } + + CDataStream(const std::vector& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) + { + Init(nTypeIn, nVersionIn); + } + + CDataStream(const std::vector& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) + { + Init(nTypeIn, nVersionIn); + } + + template + CDataStream(int nTypeIn, int nVersionIn, Args&&... args) + { + Init(nTypeIn, nVersionIn); + ::SerializeMany(*this, std::forward(args)...); + } + + void Init(int nTypeIn, int nVersionIn) + { + nReadPos = 0; + nType = nTypeIn; + nVersion = nVersionIn; + } + + CDataStream& operator+=(const CDataStream& b) + { + vch.insert(vch.end(), b.begin(), b.end()); + return *this; + } + + friend CDataStream operator+(const CDataStream& a, const CDataStream& b) + { + CDataStream ret = a; + ret += b; + return (ret); + } + + std::string str() const + { + return (std::string(begin(), end())); + } + + + // + // Vector subset + // + const_iterator begin() const { return vch.begin() + nReadPos; } + iterator begin() { return vch.begin() + nReadPos; } + const_iterator end() const { return vch.end(); } + iterator end() { return vch.end(); } + size_type size() const { return vch.size() - nReadPos; } + bool empty() const { return vch.size() == nReadPos; } + void resize(size_type n, value_type c=0) { vch.resize(n + nReadPos, c); } + void reserve(size_type n) { vch.reserve(n + nReadPos); } + const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } + reference operator[](size_type pos) { return vch[pos + nReadPos]; } + void clear() { vch.clear(); nReadPos = 0; } + iterator insert(iterator it, const char x=char()) { return vch.insert(it, x); } + void insert(iterator it, size_type n, const char x) { vch.insert(it, n, x); } + value_type* data() { return vch.data() + nReadPos; } + const value_type* data() const { return vch.data() + nReadPos; } + + void insert(iterator it, std::vector::const_iterator first, std::vector::const_iterator last) + { + if (last == first) return; + assert(last - first > 0); + if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) + { + // special case for inserting at the front when there's room + nReadPos -= (last - first); + memcpy(&vch[nReadPos], &first[0], last - first); + } + else + vch.insert(it, first, last); + } + + void insert(iterator it, const char* first, const char* last) + { + if (last == first) return; + assert(last - first > 0); + if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) + { + // special case for inserting at the front when there's room + nReadPos -= (last - first); + memcpy(&vch[nReadPos], &first[0], last - first); + } + else + vch.insert(it, first, last); + } + + iterator erase(iterator it) + { + if (it == vch.begin() + nReadPos) + { + // special case for erasing from the front + if (++nReadPos >= vch.size()) + { + // whenever we reach the end, we take the opportunity to clear the buffer + nReadPos = 0; + return vch.erase(vch.begin(), vch.end()); + } + return vch.begin() + nReadPos; + } + else + return vch.erase(it); + } + + iterator erase(iterator first, iterator last) + { + if (first == vch.begin() + nReadPos) + { + // special case for erasing from the front + if (last == vch.end()) + { + nReadPos = 0; + return vch.erase(vch.begin(), vch.end()); + } + else + { + nReadPos = (last - vch.begin()); + return last; + } + } + else + return vch.erase(first, last); + } + + inline void Compact() + { + vch.erase(vch.begin(), vch.begin() + nReadPos); + nReadPos = 0; + } + + bool Rewind(size_type n) + { + // Rewind by n characters if the buffer hasn't been compacted yet + if (n > nReadPos) + return false; + nReadPos -= n; + return true; + } + + + // + // Stream subset + // + bool eof() const { return size() == 0; } + CDataStream* rdbuf() { return this; } + int in_avail() const { return size(); } + + void SetType(int n) { nType = n; } + int GetType() const { return nType; } + void SetVersion(int n) { nVersion = n; } + int GetVersion() const { return nVersion; } + + void read(char* pch, size_t nSize) + { + if (nSize == 0) return; + + // Read from the beginning of the buffer + unsigned int nReadPosNext = nReadPos + nSize; + if (nReadPosNext > vch.size()) { + throw std::ios_base::failure("CDataStream::read(): end of data"); + } + memcpy(pch, &vch[nReadPos], nSize); + if (nReadPosNext == vch.size()) + { + nReadPos = 0; + vch.clear(); + return; + } + nReadPos = nReadPosNext; + } + + void ignore(int nSize) + { + // Ignore from the beginning of the buffer + if (nSize < 0) { + throw std::ios_base::failure("CDataStream::ignore(): nSize negative"); + } + unsigned int nReadPosNext = nReadPos + nSize; + if (nReadPosNext >= vch.size()) + { + if (nReadPosNext > vch.size()) + throw std::ios_base::failure("CDataStream::ignore(): end of data"); + nReadPos = 0; + vch.clear(); + return; + } + nReadPos = nReadPosNext; + } + + void write(const char* pch, size_t nSize) + { + // Write to the end of the buffer + vch.insert(vch.end(), pch, pch + nSize); + } + + template + void Serialize(Stream& s) const + { + // Special case: stream << stream concatenates like stream += stream + if (!vch.empty()) + s.write((char*)vch.data(), vch.size() * sizeof(value_type)); + } + + template + CDataStream& operator<<(const T& obj) + { + // Serialize to this stream + ::Serialize(*this, obj); + return (*this); + } + + template + CDataStream& operator>>(T&& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } + + void GetAndClear(CSerializeData &d) { + d.insert(d.end(), begin(), end()); + clear(); + } + + /** + * XOR the contents of this stream with a certain key. + * + * @param[in] key The key used to XOR the data in this stream. + */ + void Xor(const std::vector& key) + { + if (key.size() == 0) { + return; + } + + for (size_type i = 0, j = 0; i != size(); i++) { + vch[i] ^= key[j++]; + + // This potentially acts on very many bytes of data, so it's + // important that we calculate `j`, i.e. the `key` index in this + // way instead of doing a %, which would effectively be a division + // for each byte Xor'd -- much slower than need be. + if (j == key.size()) + j = 0; + } + } +}; + +template +class BitStreamReader +{ +private: + IStream& m_istream; + + /// Buffered byte read in from the input stream. A new byte is read into the + /// buffer when m_offset reaches 8. + uint8_t m_buffer{0}; + + /// Number of high order bits in m_buffer already returned by previous + /// Read() calls. The next bit to be returned is at this offset from the + /// most significant bit position. + int m_offset{8}; + +public: + explicit BitStreamReader(IStream& istream) : m_istream(istream) {} + + /** Read the specified number of bits from the stream. The data is returned + * in the nbits least significant bits of a 64-bit uint. + */ + uint64_t Read(int nbits) { + if (nbits < 0 || nbits > 64) { + throw std::out_of_range("nbits must be between 0 and 64"); + } + + uint64_t data = 0; + while (nbits > 0) { + if (m_offset == 8) { + m_istream >> m_buffer; + m_offset = 0; + } + + int bits = std::min(8 - m_offset, nbits); + data <<= bits; + data |= static_cast(m_buffer << m_offset) >> (8 - bits); + m_offset += bits; + nbits -= bits; + } + return data; + } +}; + +template +class BitStreamWriter +{ +private: + OStream& m_ostream; + + /// Buffered byte waiting to be written to the output stream. The byte is + /// written buffer when m_offset reaches 8 or Flush() is called. + uint8_t m_buffer{0}; + + /// Number of high order bits in m_buffer already written by previous + /// Write() calls and not yet flushed to the stream. The next bit to be + /// written to is at this offset from the most significant bit position. + int m_offset{0}; + +public: + explicit BitStreamWriter(OStream& ostream) : m_ostream(ostream) {} + + ~BitStreamWriter() + { + Flush(); + } + + /** Write the nbits least significant bits of a 64-bit int to the output + * stream. Data is buffered until it completes an octet. + */ + void Write(uint64_t data, int nbits) { + if (nbits < 0 || nbits > 64) { + throw std::out_of_range("nbits must be between 0 and 64"); + } + + while (nbits > 0) { + int bits = std::min(8 - m_offset, nbits); + m_buffer |= (data << (64 - nbits)) >> (64 - 8 + m_offset); + m_offset += bits; + nbits -= bits; + + if (m_offset == 8) { + Flush(); + } + } + } + + /** Flush any unwritten bits to the output stream, padding with 0's to the + * next byte boundary. + */ + void Flush() { + if (m_offset == 0) { + return; + } + + m_ostream << m_buffer; + m_buffer = 0; + m_offset = 0; + } +}; + + + +/** Non-refcounted RAII wrapper for FILE* + * + * Will automatically close the file when it goes out of scope if not null. + * If you're returning the file pointer, return file.release(). + * If you need to close the file early, use file.fclose() instead of fclose(file). + */ +class CAutoFile +{ +private: + const int nType; + const int nVersion; + + FILE* file; + +public: + CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) + { + file = filenew; + } + + ~CAutoFile() + { + fclose(); + } + + // Disallow copies + CAutoFile(const CAutoFile&) = delete; + CAutoFile& operator=(const CAutoFile&) = delete; + + void fclose() + { + if (file) { + ::fclose(file); + file = nullptr; + } + } + + /** Get wrapped FILE* with transfer of ownership. + * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller + * of this function to clean up the returned FILE*. + */ + FILE* release() { FILE* ret = file; file = nullptr; return ret; } + + /** Get wrapped FILE* without transfer of ownership. + * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the + * CAutoFile outlives use of the passed pointer. + */ + FILE* Get() const { return file; } + + /** Return true if the wrapped FILE* is nullptr, false otherwise. + */ + bool IsNull() const { return (file == nullptr); } + + // + // Stream subset + // + int GetType() const { return nType; } + int GetVersion() const { return nVersion; } + + void read(char* pch, size_t nSize) + { + if (!file) + throw std::ios_base::failure("CAutoFile::read: file handle is nullptr"); + if (fread(pch, 1, nSize, file) != nSize) + throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed"); + } + + void ignore(size_t nSize) + { + if (!file) + throw std::ios_base::failure("CAutoFile::ignore: file handle is nullptr"); + unsigned char data[4096]; + while (nSize > 0) { + size_t nNow = std::min(nSize, sizeof(data)); + if (fread(data, 1, nNow, file) != nNow) + throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed"); + nSize -= nNow; + } + } + + void write(const char* pch, size_t nSize) + { + if (!file) + throw std::ios_base::failure("CAutoFile::write: file handle is nullptr"); + if (fwrite(pch, 1, nSize, file) != nSize) + throw std::ios_base::failure("CAutoFile::write: write failed"); + } + + template + CAutoFile& operator<<(const T& obj) + { + // Serialize to this stream + if (!file) + throw std::ios_base::failure("CAutoFile::operator<<: file handle is nullptr"); + ::Serialize(*this, obj); + return (*this); + } + + template + CAutoFile& operator>>(T&& obj) + { + // Unserialize from this stream + if (!file) + throw std::ios_base::failure("CAutoFile::operator>>: file handle is nullptr"); + ::Unserialize(*this, obj); + return (*this); + } +}; + +/** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to + * deserialize from. It guarantees the ability to rewind a given number of bytes. + * + * Will automatically close the file when it goes out of scope if not null. + * If you need to close the file early, use file.fclose() instead of fclose(file). + */ +class CBufferedFile +{ +private: + const int nType; + const int nVersion; + + FILE *src; //!< source file + uint64_t nSrcPos; //!< how many bytes have been read from source + uint64_t nReadPos; //!< how many bytes have been read from this + uint64_t nReadLimit; //!< up to which position we're allowed to read + uint64_t nRewind; //!< how many bytes we guarantee to rewind + std::vector vchBuf; //!< the buffer + +protected: + //! read data from the source to fill the buffer + bool Fill() { + unsigned int pos = nSrcPos % vchBuf.size(); + unsigned int readNow = vchBuf.size() - pos; + unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind; + if (nAvail < readNow) + readNow = nAvail; + if (readNow == 0) + return false; + size_t nBytes = fread((void*)&vchBuf[pos], 1, readNow, src); + if (nBytes == 0) { + throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill: end of file" : "CBufferedFile::Fill: fread failed"); + } else { + nSrcPos += nBytes; + return true; + } + } + +public: + CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) : + nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), nReadPos(0), nReadLimit(std::numeric_limits::max()), nRewind(nRewindIn), vchBuf(nBufSize, 0) + { + src = fileIn; + } + + ~CBufferedFile() + { + fclose(); + } + + // Disallow copies + CBufferedFile(const CBufferedFile&) = delete; + CBufferedFile& operator=(const CBufferedFile&) = delete; + + int GetVersion() const { return nVersion; } + int GetType() const { return nType; } + + void fclose() + { + if (src) { + ::fclose(src); + src = nullptr; + } + } + + //! check whether we're at the end of the source file + bool eof() const { + return nReadPos == nSrcPos && feof(src); + } + + //! read a number of bytes + void read(char *pch, size_t nSize) { + if (nSize + nReadPos > nReadLimit) + throw std::ios_base::failure("Read attempted past buffer limit"); + if (nSize + nRewind > vchBuf.size()) + throw std::ios_base::failure("Read larger than buffer size"); + while (nSize > 0) { + if (nReadPos == nSrcPos) + Fill(); + unsigned int pos = nReadPos % vchBuf.size(); + size_t nNow = nSize; + if (nNow + pos > vchBuf.size()) + nNow = vchBuf.size() - pos; + if (nNow + nReadPos > nSrcPos) + nNow = nSrcPos - nReadPos; + memcpy(pch, &vchBuf[pos], nNow); + nReadPos += nNow; + pch += nNow; + nSize -= nNow; + } + } + + //! return the current reading position + uint64_t GetPos() const { + return nReadPos; + } + + //! rewind to a given reading position + bool SetPos(uint64_t nPos) { + nReadPos = nPos; + if (nReadPos + nRewind < nSrcPos) { + nReadPos = nSrcPos - nRewind; + return false; + } else if (nReadPos > nSrcPos) { + nReadPos = nSrcPos; + return false; + } else { + return true; + } + } + + bool Seek(uint64_t nPos) { + long nLongPos = nPos; + if (nPos != (uint64_t)nLongPos) + return false; + if (fseek(src, nLongPos, SEEK_SET)) + return false; + nLongPos = ftell(src); + nSrcPos = nLongPos; + nReadPos = nLongPos; + return true; + } + + //! prevent reading beyond a certain position + //! no argument removes the limit + bool SetLimit(uint64_t nPos = std::numeric_limits::max()) { + if (nPos < nReadPos) + return false; + nReadLimit = nPos; + return true; + } + + template + CBufferedFile& operator>>(T&& obj) { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } + + //! search for a given byte in the stream, and remain positioned on it + void FindByte(char ch) { + while (true) { + if (nReadPos == nSrcPos) + Fill(); + if (vchBuf[nReadPos % vchBuf.size()] == ch) + break; + nReadPos++; + } + } +}; + +#endif // BITCOIN_STREAMS_H diff --git a/src/test/neuralnet/cpid_tests.cpp b/src/test/neuralnet/cpid_tests.cpp index 4bf37dcbf9..5ea9b0a163 100644 --- a/src/test/neuralnet/cpid_tests.cpp +++ b/src/test/neuralnet/cpid_tests.cpp @@ -1,4 +1,5 @@ #include "neuralnet/cpid.h" +#include "streams.h" #include #include diff --git a/src/test/neuralnet/superblock_tests.cpp b/src/test/neuralnet/superblock_tests.cpp index 29d55dc3b9..78b713873b 100644 --- a/src/test/neuralnet/superblock_tests.cpp +++ b/src/test/neuralnet/superblock_tests.cpp @@ -1,5 +1,6 @@ #include "compat/endian.h" #include "neuralnet/superblock.h" +#include "streams.h" #include #include diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index a04e759c64..c8551dfc88 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -1,4 +1,5 @@ #include "serialize.h" +#include "streams.h" #include "util.h" #include diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 3a13706641..9e6e0cf062 100755 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -3,6 +3,7 @@ #include #include "main.h" +#include "streams.h" #include "wallet.h" #include "data/tx_valid.json.h" diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 7ce98b6104..c29807463e 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -2,6 +2,7 @@ #include "uint256.h" #include "serialize.h" +#include "streams.h" #include "util.h" #include diff --git a/src/txdb-leveldb.h b/src/txdb-leveldb.h index d1c44937a2..ec271f0840 100644 --- a/src/txdb-leveldb.h +++ b/src/txdb-leveldb.h @@ -7,6 +7,7 @@ #define BITCOIN_LEVELDB_H #include "main.h" +#include "streams.h" #include #include diff --git a/src/wallet.h b/src/wallet.h index 31ec4018fe..14d0ae93db 100644 --- a/src/wallet.h +++ b/src/wallet.h @@ -13,6 +13,7 @@ #include "key.h" #include "keystore.h" #include "script.h" +#include "streams.h" #include "ui_interface.h" #include "walletdb.h" diff --git a/src/walletdb.cpp b/src/walletdb.cpp index ab963cc597..938c39855c 100644 --- a/src/walletdb.cpp +++ b/src/walletdb.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "streams.h" #include "walletdb.h" #include "wallet.h" #include "init.h" From 530c302229e6b568784781809d9418435955fa84 Mon Sep 17 00:00:00 2001 From: Cy Rossignol Date: Sun, 22 Sep 2019 13:42:39 -0500 Subject: [PATCH 4/7] Update prevector.h from Bitcoin --- src/prevector.h | 55 ++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/prevector.h b/src/prevector.h index 103ead82cc..9d576321b6 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2017 The Bitcoin Core developers +// Copyright (c) 2015-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,12 +10,11 @@ #include #include +#include #include #include #include -#include - #pragma pack(push, 1) /** Implements a drop-in replacement for std::vector which stores up to N * elements directly (without heap allocation). The types Size and Diff are @@ -148,14 +147,14 @@ class prevector { }; private: - size_type _size; + size_type _size = 0; union direct_or_indirect { char direct[sizeof(T) * N]; struct { size_type capacity; char* indirect; }; - } _union; + } _union = {}; T* direct_ptr(difference_type pos) { return reinterpret_cast(_union.direct) + pos; } const T* direct_ptr(difference_type pos) const { return reinterpret_cast(_union.direct) + pos; } @@ -197,23 +196,8 @@ class prevector { T* item_ptr(difference_type pos) { return is_direct() ? direct_ptr(pos) : indirect_ptr(pos); } const T* item_ptr(difference_type pos) const { return is_direct() ? direct_ptr(pos) : indirect_ptr(pos); } - void fill(T* dst, ptrdiff_t count) { - if (IS_TRIVIALLY_CONSTRUCTIBLE::value) { - // The most common use of prevector is where T=unsigned char. For - // trivially constructible types, we can use memset() to avoid - // looping. - ::memset(dst, 0, count * sizeof(T)); - } else { - for (auto i = 0; i < count; ++i) { - new(static_cast(dst + i)) T(); - } - } - } - - void fill(T* dst, ptrdiff_t count, const T& value) { - for (auto i = 0; i < count; ++i) { - new(static_cast(dst + i)) T(value); - } + void fill(T* dst, ptrdiff_t count, const T& value = T{}) { + std::fill_n(dst, count, value); } template @@ -246,34 +230,34 @@ class prevector { fill(item_ptr(0), first, last); } - prevector() : _size(0), _union{{}} {} + prevector() {} - explicit prevector(size_type n) : _size(0) { + explicit prevector(size_type n) { resize(n); } - explicit prevector(size_type n, const T& val = T()) : _size(0) { + explicit prevector(size_type n, const T& val) { change_capacity(n); _size += n; fill(item_ptr(0), n, val); } template - prevector(InputIterator first, InputIterator last) : _size(0) { + prevector(InputIterator first, InputIterator last) { size_type n = last - first; change_capacity(n); _size += n; fill(item_ptr(0), first, last); } - prevector(const prevector& other) : _size(0) { + prevector(const prevector& other) { size_type n = other.size(); change_capacity(n); _size += n; fill(item_ptr(0), other.begin(), other.end()); } - prevector(prevector&& other) : _size(0) { + prevector(prevector&& other) { swap(other); } @@ -394,6 +378,21 @@ class prevector { fill(ptr, first, last); } + inline void resize_uninitialized(size_type new_size) { + // resize_uninitialized changes the size of the prevector but does not initialize it. + // If size < new_size, the added elements must be initialized explicitly. + if (capacity() < new_size) { + change_capacity(new_size); + _size += new_size - size(); + return; + } + if (new_size < size()) { + erase(item_ptr(new_size), end()); + } else { + _size += new_size - size(); + } + } + iterator erase(iterator pos) { return erase(pos, pos + 1); } From 8b0aa20cf5b632a4c0dbfafc84ea09ac2308740d Mon Sep 17 00:00:00 2001 From: Cy Rossignol Date: Sun, 22 Sep 2019 13:44:32 -0500 Subject: [PATCH 5/7] Install span.h from Bitcoin --- src/Makefile.am | 1 + src/span.h | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 src/span.h diff --git a/src/Makefile.am b/src/Makefile.am index a68c0435c2..aa313abbde 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -115,6 +115,7 @@ GRIDCOIN_CORE_H = \ script.h \ scrypt.h \ serialize.h \ + span.h \ streams.h \ strlcpy.h \ support/allocators/secure.h \ diff --git a/src/span.h b/src/span.h new file mode 100644 index 0000000000..77de059fa6 --- /dev/null +++ b/src/span.h @@ -0,0 +1,60 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SPAN_H +#define BITCOIN_SPAN_H + +#include +#include +#include + +/** A Span is an object that can refer to a contiguous sequence of objects. + * + * It implements a subset of C++20's std::span. + */ +template +class Span +{ + C* m_data; + std::ptrdiff_t m_size; + +public: + constexpr Span() noexcept : m_data(nullptr), m_size(0) {} + constexpr Span(C* data, std::ptrdiff_t size) noexcept : m_data(data), m_size(size) {} + constexpr Span(C* data, C* end) noexcept : m_data(data), m_size(end - data) {} + + constexpr C* data() const noexcept { return m_data; } + constexpr C* begin() const noexcept { return m_data; } + constexpr C* end() const noexcept { return m_data + m_size; } + constexpr std::ptrdiff_t size() const noexcept { return m_size; } + constexpr C& operator[](std::ptrdiff_t pos) const noexcept { return m_data[pos]; } + + constexpr Span subspan(std::ptrdiff_t offset) const noexcept { return Span(m_data + offset, m_size - offset); } + constexpr Span subspan(std::ptrdiff_t offset, std::ptrdiff_t count) const noexcept { return Span(m_data + offset, count); } + constexpr Span first(std::ptrdiff_t count) const noexcept { return Span(m_data, count); } + constexpr Span last(std::ptrdiff_t count) const noexcept { return Span(m_data + m_size - count, count); } + + friend constexpr bool operator==(const Span& a, const Span& b) noexcept { return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin()); } + friend constexpr bool operator!=(const Span& a, const Span& b) noexcept { return !(a == b); } + friend constexpr bool operator<(const Span& a, const Span& b) noexcept { return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); } + friend constexpr bool operator<=(const Span& a, const Span& b) noexcept { return !(b < a); } + friend constexpr bool operator>(const Span& a, const Span& b) noexcept { return (b < a); } + friend constexpr bool operator>=(const Span& a, const Span& b) noexcept { return !(a < b); } +}; + +/** Create a span to a container exposing data() and size(). + * + * This correctly deals with constness: the returned Span's element type will be + * whatever data() returns a pointer to. If either the passed container is const, + * or its element type is const, the resulting span will have a const element type. + * + * std::span will have a constructor that implements this functionality directly. + */ +template +constexpr Span MakeSpan(A (&a)[N]) { return Span(a, N); } + +template +constexpr Span().data())>::type> MakeSpan(V& v) { return Span().data())>::type>(v.data(), v.size()); } + +#endif From 39bb55d82c15fd84635311934753f4beabbc3cfe Mon Sep 17 00:00:00 2001 From: Cy Rossignol Date: Mon, 23 Sep 2019 19:25:59 -0500 Subject: [PATCH 6/7] Port serialize.h from Bitcoin Bitcoin moved its stream classes out from serialize.h into streams.h. These changes update the application serialization and stream code to match Bitcoin's newer implementation with the following exceptions: - CSizeComputer retains nType for legacy code with type-dependent serialization (SER_NETWORK, SER_DISK, SER_GETHASH, ...) - The retention of serialization implementations for std::tuple for the deprecated accounting API The backport includes some new stream classes from Bitcoin not used in Gridcoin. We may refactor code to use these new classes in the future. --- src/addrdb.cpp | 4 +- src/addrdb.h | 8 +- src/addrman.h | 22 +- src/alert.h | 23 +- src/bignum.h | 13 +- src/crypter.h | 10 +- src/db.cpp | 16 +- src/hash.h | 4 +- src/key.h | 10 +- src/main.cpp | 12 +- src/main.h | 220 ++--- src/netbase.h | 37 +- src/neuralnet/cpid.cpp | 11 - src/neuralnet/cpid.h | 55 +- src/neuralnet/superblock.cpp | 12 - src/neuralnet/superblock.h | 162 ++-- src/protocol.h | 63 +- src/rpcdataacq.cpp | 4 +- src/scraper/fwd.h | 1 + src/scraper/scraper.cpp | 16 +- src/scraper_net.cpp | 16 +- src/scraper_net.h | 12 +- src/script.h | 11 + src/serialize.h | 1093 +++++++++++++---------- src/test/neuralnet/cpid_tests.cpp | 8 +- src/test/neuralnet/superblock_tests.cpp | 22 +- src/test/serialize_tests.cpp | 333 ++++++- src/uint256.h | 9 +- src/wallet.h | 153 ++-- src/walletdb.h | 12 +- 30 files changed, 1454 insertions(+), 918 deletions(-) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 20d5a5fa39..a83cb011df 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -26,8 +26,8 @@ bool SerializeDB(Stream& stream, const Data& data) // Write and commit header, data try { CHashWriter hasher(SER_DISK, CLIENT_VERSION); - stream << FLATDATA(pchMessageStart) << data; - hasher << FLATDATA(pchMessageStart) << data; + stream << pchMessageStart << data; + hasher << pchMessageStart << data; stream << hasher.GetHash(); } catch (const std::exception& e) { return error("%s: Serialize or I/O error - %s", __func__, e.what()); diff --git a/src/addrdb.h b/src/addrdb.h index bd2f6d16cc..3495155ad1 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -52,10 +52,10 @@ class CBanEntry template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEVARIADIC(this->nVersion); - READWRITEVARIADIC(nCreateTime); - READWRITEVARIADIC(nBanUntil); - READWRITEVARIADIC(banReason); + READWRITE(this->nVersion); + READWRITE(nCreateTime); + READWRITE(nBanUntil); + READWRITE(banReason); } void SetNull() diff --git a/src/addrman.h b/src/addrman.h index 9ccdbe9fc4..2b04ef717a 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -45,13 +45,16 @@ class CAddrInfo : public CAddress public: - IMPLEMENT_SERIALIZE( - CAddress* pthis = (CAddress*)(this); - READWRITE(*pthis); + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITEAS(CAddress, *this); READWRITE(source); READWRITE(nLastSuccess); READWRITE(nAttempts); - ) + } void Init() { @@ -265,10 +268,10 @@ class CAddrMan // This format is more complex, but significantly smaller (at most 1.5 MiB), and supports // changes to the ADDRMAN_ parameters without breaking the on-disk structure. // - // We don't use IMPLEMENT_SERIALIZE since the serialization and deserialization code has + // We don't use ADD_SERIALIZE_METHODS since the serialization and deserialization code has // very little in common. template - void Serialize(Stream &s, int nType, int nVersionDummy) const + void Serialize(Stream &s) const { LOCK(cs); @@ -312,7 +315,7 @@ class CAddrMan } template - void Unserialize(Stream& s, int nType, int nVersionDummy) + void Unserialize(Stream& s) { LOCK(cs); @@ -376,11 +379,6 @@ class CAddrMan } } - unsigned int GetSerializeSize(int nType, int nVersion) const - { - return (CSizeComputer(nType, nVersion) << *this).size(); - } - CAddrMan() : vRandom(0), vvTried(ADDRMAN_TRIED_BUCKET_COUNT, std::vector(0)), vvNew(ADDRMAN_NEW_BUCKET_COUNT, std::set()) { nKey.resize(32); diff --git a/src/alert.h b/src/alert.h index a51c0ad73c..cdbd06f0f8 100644 --- a/src/alert.h +++ b/src/alert.h @@ -39,10 +39,14 @@ class CUnsignedAlert std::string strStatusBar; std::string strReserved; - IMPLEMENT_SERIALIZE - ( - READWRITE(this->nVersion); - nVersion = this->nVersion; + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(nVersion); + s.SetVersion(nVersion); + READWRITE(nRelayUntil); READWRITE(nExpiration); READWRITE(nID); @@ -56,7 +60,7 @@ class CUnsignedAlert READWRITE(strComment); READWRITE(strStatusBar); READWRITE(strReserved); - ) + } void SetNull(); @@ -76,11 +80,14 @@ class CAlert : public CUnsignedAlert SetNull(); } - IMPLEMENT_SERIALIZE - ( + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { READWRITE(vchMsg); READWRITE(vchSig); - ) + } void SetNull(); bool IsNull() const; diff --git a/src/bignum.h b/src/bignum.h index cee71b1e79..8bae7986ae 100644 --- a/src/bignum.h +++ b/src/bignum.h @@ -411,22 +411,17 @@ class CBigNum : public CBigNumBase return ToString(16); } - unsigned int GetSerializeSize(int nType=0, int nVersion=PROTOCOL_VERSION) const - { - return ::GetSerializeSize(getvch(), nType, nVersion); - } - template - void Serialize(Stream& s, int nType=0, int nVersion=PROTOCOL_VERSION) const + void Serialize(Stream& s) const { - ::Serialize(s, getvch(), nType, nVersion); + ::Serialize(s, getvch()); } template - void Unserialize(Stream& s, int nType=0, int nVersion=PROTOCOL_VERSION) + void Unserialize(Stream& s) { std::vector vch; - ::Unserialize(s, vch, nType, nVersion); + ::Unserialize(s, vch); setvch(vch); } diff --git a/src/crypter.h b/src/crypter.h index a703c454fd..4313a5ae8e 100644 --- a/src/crypter.h +++ b/src/crypter.h @@ -39,14 +39,18 @@ class CMasterKey // such as the various parameters to scrypt std::vector vchOtherDerivationParameters; - IMPLEMENT_SERIALIZE - ( + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { READWRITE(vchCryptedKey); READWRITE(vchSalt); READWRITE(nDerivationMethod); READWRITE(nDeriveIterations); READWRITE(vchOtherDerivationParameters); - ) + } + CMasterKey() { // 25000 rounds is just under 0.1 seconds on a 1.86 GHz Pentium M diff --git a/src/db.cpp b/src/db.cpp index d44a80cb4c..fb26c9b45b 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -237,7 +237,7 @@ void CDBEnv::CheckpointLSN(std::string strFile) void CDBEnv::lsn_reset(const std::string& strFile) { dbenv.lsn_reset(strFile.c_str(),0); -} +} CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL), activeTxn(NULL) @@ -506,7 +506,7 @@ bool CAddrDB::Write(const CAddrMan& addr) // serialize addresses, checksum data up to that point, then append csum CDataStream ssPeers(SER_DISK, CLIENT_VERSION); - ssPeers << FLATDATA(pchMessageStart); + ssPeers << pchMessageStart; ssPeers << addr; uint256 hash = Hash(ssPeers.begin(), ssPeers.end()); ssPeers << hash; @@ -514,8 +514,8 @@ bool CAddrDB::Write(const CAddrMan& addr) // open temp output file, and associate with CAutoFile boost::filesystem::path pathTmp = GetDataDir() / tmpfn; FILE *file = fopen(pathTmp.string().c_str(), "wb"); - CAutoFile fileout = CAutoFile(file, SER_DISK, CLIENT_VERSION); - if (!fileout) + CAutoFile fileout(file, SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) return error("CAddrman::Write() : open failed"); // Write and commit header, data @@ -525,7 +525,7 @@ bool CAddrDB::Write(const CAddrMan& addr) catch (std::exception &e) { return error("CAddrman::Write() : I/O error"); } - FileCommit(fileout); + FileCommit(fileout.Get()); fileout.fclose(); // replace existing peers.dat, if any, with new peers.dat.XXXX @@ -539,8 +539,8 @@ bool CAddrDB::Read(CAddrMan& addr) { // open input file, and associate with CAutoFile FILE *file = fopen(pathAddr.string().c_str(), "rb"); - CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION); - if (!filein) + CAutoFile filein(file, SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) return error("CAddrman::Read() : open failed"); // use file size to size memory buffer @@ -573,7 +573,7 @@ bool CAddrDB::Read(CAddrMan& addr) unsigned char pchMsgTmp[4]; try { // de-serialize file header (pchMessageStart magic number) and - ssPeers >> FLATDATA(pchMsgTmp); + ssPeers >> pchMsgTmp; // verify the network matches ours if (memcmp(pchMsgTmp, pchMessageStart, sizeof(pchMsgTmp))) diff --git a/src/hash.h b/src/hash.h index ca317c5c69..3933841580 100644 --- a/src/hash.h +++ b/src/hash.h @@ -162,7 +162,7 @@ class CHashWriter template CHashWriter& operator<<(const T& obj) { // Serialize to this stream - ::Serialize(*this, obj, nType, nVersion); + ::Serialize(*this, obj); return (*this); } }; @@ -197,7 +197,7 @@ class CHashVerifier : public CHashWriter CHashVerifier& operator>>(T&& obj) { // Unserialize from this stream - ::Unserialize(*this, obj, nType, nVersion); + ::Unserialize(*this, obj); return (*this); } }; diff --git a/src/key.h b/src/key.h index f31f88472d..c4a991f146 100644 --- a/src/key.h +++ b/src/key.h @@ -73,9 +73,13 @@ class CPubKey { friend bool operator!=(const CPubKey &a, const CPubKey &b) { return a.vchPubKey != b.vchPubKey; } friend bool operator<(const CPubKey &a, const CPubKey &b) { return a.vchPubKey < b.vchPubKey; } - IMPLEMENT_SERIALIZE( + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { READWRITE(vchPubKey); - ) + } CKeyID GetID() const { return CKeyID(Hash160(vchPubKey)); @@ -149,7 +153,7 @@ class CKey // This is only slightly more CPU intensive than just verifying it. // If this function succeeds, the recovered public key is guaranteed to be valid // (the signature is a valid signature of the given data for that key) - + // Ensure that signature is DER-encoded static bool ReserealizeSignature(std::vector& vchSig); diff --git a/src/main.cpp b/src/main.cpp index 17c91cb1dc..9f546c7fe5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -821,7 +821,7 @@ bool AddOrphanTx(const CTransaction& tx) // 10,000 orphans, each of which is at most 5,000 bytes big is // at most 500 megabytes of orphans: - size_t nSize = tx.GetSerializeSize(SER_NETWORK, CTransaction::CURRENT_VERSION); + size_t nSize = GetSerializeSize(tx, SER_NETWORK, CTransaction::CURRENT_VERSION); if (nSize > 5000) { @@ -987,7 +987,7 @@ bool IsStandardTx(const CTransaction& tx) // almost as much to process as they cost the sender in fees, because // computing signature hashes is O(ninputs*txsize). Limiting transactions // to MAX_STANDARD_TX_SIZE mitigates CPU exhaustion attacks. - unsigned int sz = tx.GetSerializeSize(SER_NETWORK, CTransaction::CURRENT_VERSION); + unsigned int sz = GetSerializeSize(tx, SER_NETWORK, CTransaction::CURRENT_VERSION); if (sz >= MAX_STANDARD_TX_SIZE) return false; @@ -5321,12 +5321,12 @@ bool LoadExternalBlockFile(FILE* fileIn) try { CAutoFile blkdat(fileIn, SER_DISK, CLIENT_VERSION); unsigned int nPos = 0; - while (nPos != (unsigned int)-1 && blkdat.good() && !fRequestShutdown) + while (nPos != (unsigned int)-1 && !fRequestShutdown) { unsigned char pchData[65536]; do { - fseek(blkdat, nPos, SEEK_SET); - int nRead = fread(pchData, 1, sizeof(pchData), blkdat); + fseek(blkdat.Get(), nPos, SEEK_SET); + int nRead = fread(pchData, 1, sizeof(pchData), blkdat.Get()); if (nRead <= 8) { nPos = (unsigned int)-1; @@ -5347,7 +5347,7 @@ bool LoadExternalBlockFile(FILE* fileIn) } while(!fRequestShutdown); if (nPos == (unsigned int)-1) break; - fseek(blkdat, nPos, SEEK_SET); + fseek(blkdat.Get(), nPos, SEEK_SET); unsigned int nSize; blkdat >> nSize; if (nSize > 0 && nSize <= MAX_BLOCK_SIZE) diff --git a/src/main.h b/src/main.h index 2e9f82b81f..81116f6eeb 100644 --- a/src/main.h +++ b/src/main.h @@ -325,6 +325,16 @@ class CDiskTxPos unsigned int nBlockPos; unsigned int nTxPos; + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(nFile); + READWRITE(nBlockPos); + READWRITE(nTxPos); + } + CDiskTxPos() { SetNull(); @@ -337,7 +347,6 @@ class CDiskTxPos nTxPos = nTxPosIn; } - IMPLEMENT_SERIALIZE( READWRITE(FLATDATA(*this)); ) void SetNull() { nFile = (unsigned int) -1; nBlockPos = 0; nTxPos = 0; } bool IsNull() const { return (nFile == (unsigned int) -1); } @@ -392,9 +401,17 @@ class COutPoint uint256 hash; unsigned int n; + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(hash); + READWRITE(n); + } + COutPoint() { SetNull(); } COutPoint(uint256 hashIn, unsigned int nIn) { hash = hashIn; n = nIn; } - IMPLEMENT_SERIALIZE( READWRITE(FLATDATA(*this)); ) void SetNull() { hash = 0; n = (unsigned int) -1; } bool IsNull() const { return (hash == 0 && n == (unsigned int) -1); } @@ -457,12 +474,15 @@ class CTxIn nSequence = nSequenceIn; } - IMPLEMENT_SERIALIZE - ( + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { READWRITE(prevout); READWRITE(scriptSig); READWRITE(nSequence); - ) + } bool IsFinal() const { @@ -530,11 +550,14 @@ class CTxOut scriptPubKey = scriptPubKeyIn; } - IMPLEMENT_SERIALIZE - ( + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { READWRITE(nValue); READWRITE(scriptPubKey); - ) + } void SetNull() { @@ -626,17 +649,19 @@ class CTransaction SetNull(); } - IMPLEMENT_SERIALIZE - ( - READWRITE(this->nVersion); - nVersion = this->nVersion; + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(nVersion); READWRITE(nTime); READWRITE(vin); READWRITE(vout); READWRITE(nLockTime); READWRITE(hashBoinc); - ) + } void SetNull() { @@ -753,12 +778,12 @@ class CTransaction bool ReadFromDisk(CDiskTxPos pos, FILE** pfileRet=NULL) { - CAutoFile filein = CAutoFile(OpenBlockFile(pos.nFile, 0, pfileRet ? "rb+" : "rb"), SER_DISK, CLIENT_VERSION); - if (!filein) + CAutoFile filein(OpenBlockFile(pos.nFile, 0, pfileRet ? "rb+" : "rb"), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) return error("CTransaction::ReadFromDisk() : OpenBlockFile failed"); // Read transaction - if (fseek(filein, pos.nTxPos, SEEK_SET) != 0) + if (fseek(filein.Get(), pos.nTxPos, SEEK_SET) != 0) return error("CTransaction::ReadFromDisk() : fseek failed"); try { @@ -771,7 +796,7 @@ class CTransaction // Return file pointer if (pfileRet) { - if (fseek(filein, pos.nTxPos, SEEK_SET) != 0) + if (fseek(filein.Get(), pos.nTxPos, SEEK_SET) != 0) return error("CTransaction::ReadFromDisk() : second fseek failed"); *pfileRet = filein.release(); } @@ -902,16 +927,16 @@ class CMerkleTx : public CTransaction fMerkleVerified = false; } + ADD_SERIALIZE_METHODS; - IMPLEMENT_SERIALIZE - ( - nSerSize += SerReadWrite(s, *(CTransaction*)this, nType, nVersion, ser_action); - nVersion = this->nVersion; + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITEAS(CTransaction, *this); READWRITE(hashBlock); READWRITE(vMerkleBranch); READWRITE(nIndex); - ) - + } int SetMerkleBranch(const CBlock* pblock=NULL); @@ -950,13 +975,19 @@ class CTxIndex vSpent.resize(nOutputs); } - IMPLEMENT_SERIALIZE - ( - if (!(nType & SER_GETHASH)) + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (!(s.GetType() & SER_GETHASH)) { + int nVersion = s.GetVersion(); READWRITE(nVersion); + } + READWRITE(pos); READWRITE(vSpent); - ) + } void SetNull() { @@ -1035,47 +1066,26 @@ class CBlock SetNull(); } - IMPLEMENT_SERIALIZE - ( - //Gridcoin - - // + ADD_SERIALIZE_METHODS; - - READWRITE(this->nVersion); - nVersion = this->nVersion; + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(nVersion); READWRITE(hashPrevBlock); READWRITE(hashMerkleRoot); READWRITE(nTime); READWRITE(nBits); READWRITE(nNonce); - - // ConnectBlock depends on vtx following header to generate CDiskTxPos - if (!(nType & (SER_GETHASH|SER_BLOCKHEADERONLY))) - { + if (!(s.GetType() & (SER_GETHASH|SER_BLOCKHEADERONLY))) { READWRITE(vtx); - READWRITE(vchBlockSig); - - /* - - */ - - - - } - else if (fRead) - { + } else if (ser_action.ForRead()) { const_cast(this)->vtx.clear(); const_cast(this)->vchBlockSig.clear(); } - - - - - - ) + } void SetNull() { @@ -1203,25 +1213,25 @@ class CBlock bool WriteToDisk(unsigned int& nFileRet, unsigned int& nBlockPosRet) { // Open history file to append - CAutoFile fileout = CAutoFile(AppendBlockFile(nFileRet), SER_DISK, CLIENT_VERSION); - if (!fileout) + CAutoFile fileout(AppendBlockFile(nFileRet), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) return error("CBlock::WriteToDisk() : AppendBlockFile failed"); // Write index header - unsigned int nSize = fileout.GetSerializeSize(*this); - fileout << FLATDATA(pchMessageStart) << nSize; + unsigned int nSize = GetSerializeSize(fileout, *this); + fileout << pchMessageStart << nSize; // Write block - long fileOutPos = ftell(fileout); + long fileOutPos = ftell(fileout.Get()); if (fileOutPos < 0) return error("CBlock::WriteToDisk() : ftell failed"); nBlockPosRet = fileOutPos; fileout << *this; // Flush stdio buffers and commit to disk before returning - fflush(fileout); + fflush(fileout.Get()); if (!IsInitialBlockDownload() || (nBestHeight+1) % 500 == 0) - FileCommit(fileout); + FileCommit(fileout.Get()); return true; } @@ -1230,12 +1240,12 @@ class CBlock { SetNull(); + const int ser_flags = SER_DISK | (fReadTransactions ? 0 : SER_BLOCKHEADERONLY); + // Open history file to read - CAutoFile filein = CAutoFile(OpenBlockFile(nFile, nBlockPos, "rb"), SER_DISK, CLIENT_VERSION); - if (!filein) + CAutoFile filein(OpenBlockFile(nFile, nBlockPos, "rb"), ser_flags, CLIENT_VERSION); + if (filein.IsNull()) return error("CBlock::ReadFromDisk() : OpenBlockFile failed"); - if (!fReadTransactions) - filein.nType |= SER_BLOCKHEADERONLY; // Read block try { @@ -1357,7 +1367,7 @@ class CBlockIndex SetNull(); nFile = nFileIn; - nBlockPos = nBlockPosIn; + nBlockPos = nBlockPosIn; if (block.IsProofOfStake()) { SetProofOfStake(); @@ -1572,10 +1582,14 @@ class CDiskBlockIndex : public CBlockIndex hashNext = (pnext ? pnext->GetBlockHash() : 0); } - IMPLEMENT_SERIALIZE - ( - if (!(nType & SER_GETHASH)) + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (!(s.GetType() & SER_GETHASH)) { READWRITE(nVersion); + } READWRITE(hashNext); READWRITE(nFile); @@ -1585,53 +1599,53 @@ class CDiskBlockIndex : public CBlockIndex READWRITE(nMoneySupply); READWRITE(nFlags); READWRITE(nStakeModifier); - if (IsProofOfStake()) - { + + if (IsProofOfStake()) { READWRITE(prevoutStake); READWRITE(nStakeTime); - } - else if (fRead) - { + } else if (ser_action.ForRead()) { const_cast(this)->prevoutStake.SetNull(); const_cast(this)->nStakeTime = 0; } + READWRITE(hashProof); // block header - READWRITE(this->nVersion); + READWRITE(nVersion); READWRITE(hashPrev); READWRITE(hashMerkleRoot); READWRITE(nTime); READWRITE(nBits); READWRITE(nNonce); READWRITE(blockHash); - //7-11-2015 - Gridcoin - New Accrual Fields (Note, Removing the determinstic block number to make this happen all the time): + + //7-11-2015 - Gridcoin - New Accrual Fields (Note, Removing the determinstic block number to make this happen all the time): std::string cpid_hex = GetCPID(); READWRITE(cpid_hex); - if(fRead) - const_cast(this)->SetCPID(cpid_hex); - READWRITE(nResearchSubsidy); - READWRITE(nInterestSubsidy); - READWRITE(nMagnitude); - //9-13-2015 - Indicators - if (this->nHeight > nNewIndex2) - { - READWRITE(nIsSuperBlock); - READWRITE(nIsContract); + if (ser_action.ForRead()) { + const_cast(this)->SetCPID(cpid_hex); + } - std::string dummy; + READWRITE(nResearchSubsidy); + READWRITE(nInterestSubsidy); + READWRITE(nMagnitude); - // Blocks used to contain the GRC address. - READWRITE(dummy); + //9-13-2015 - Indicators + if (this->nHeight > nNewIndex2) { + READWRITE(nIsSuperBlock); + READWRITE(nIsContract); - // Blocks used to come with a reserved string. Keep (de)serializing - // it until it's used. - READWRITE(dummy); - } + std::string dummy; + // Blocks used to contain the GRC address. + READWRITE(dummy); - ) + // Blocks used to come with a reserved string. Keep (de)serializing + // it until it's used. + READWRITE(dummy); + } + } uint256 GetBlockHash() const { @@ -1703,12 +1717,18 @@ class CBlockLocator vHave = vHaveIn; } - IMPLEMENT_SERIALIZE - ( - if (!(nType & SER_GETHASH)) + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (!(s.GetType() & SER_GETHASH)) { + int nVersion = s.GetVersion(); READWRITE(nVersion); + } + READWRITE(vHave); - ) + } void SetNull() { diff --git a/src/netbase.h b/src/netbase.h index bb10e0bfce..ad8af7b5c1 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -78,10 +78,13 @@ class CNetAddr friend bool operator!=(const CNetAddr& a, const CNetAddr& b); friend bool operator<(const CNetAddr& a, const CNetAddr& b); - IMPLEMENT_SERIALIZE - ( - READWRITE(FLATDATA(ip)); - ) + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(ip); + } friend class CSubNet; }; @@ -118,10 +121,11 @@ class CSubNet ADD_SERIALIZE_METHODS; template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEVARIADIC(network); - READWRITEVARIADIC(netmask); - READWRITEVARIADIC(valid); + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(network); + READWRITE(netmask); + READWRITE(valid); } }; @@ -158,15 +162,14 @@ class CService : public CNetAddr CService(const struct in6_addr& ipv6Addr, unsigned short port); CService(const struct sockaddr_in6& addr); - IMPLEMENT_SERIALIZE - ( - CService* pthis = const_cast(this); - READWRITE(FLATDATA(ip)); - unsigned short portN = htons(port); - READWRITE(portN); - if (fRead) - pthis->port = ntohs(portN); - ) + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(ip); + READWRITE(WrapBigEndian(port)); + } }; typedef std::pair proxyType; diff --git a/src/neuralnet/cpid.cpp b/src/neuralnet/cpid.cpp index 141cb2ae70..720bc3f130 100644 --- a/src/neuralnet/cpid.cpp +++ b/src/neuralnet/cpid.cpp @@ -219,17 +219,6 @@ std::string MiningId::ToString() const return boost::apply_visitor(MiningIdToStringVisitor(), m_variant); } -unsigned int MiningId::GetSerializeSize(int nType, int nVersion) const -{ - if (Which() == Kind::CPID) { - return 1 + 16; // Variant tag byte + CPID bytes - } - - // For variants without any associated data, we serialize the variant tag - // only as a single byte: - return 1; -} - // ----------------------------------------------------------------------------- // Class: MiningId::Invalid // ----------------------------------------------------------------------------- diff --git a/src/neuralnet/cpid.h b/src/neuralnet/cpid.h index 28ddb993f8..ddad1e93aa 100644 --- a/src/neuralnet/cpid.h +++ b/src/neuralnet/cpid.h @@ -146,10 +146,27 @@ class Cpid //! std::string ToString() const; - IMPLEMENT_SERIALIZE - ( - READWRITE(FLATDATA(m_bytes)); - ) + //! + //! \brief Serialize the object to the provided stream. + //! + //! \param stream The output stream. + //! + template + void Serialize(Stream& stream) const + { + stream.write(CharCast(m_bytes.data()), m_bytes.size()); + } + + //! + //! \brief Deserialize the object from the provided stream. + //! + //! \param stream The input stream. + //! + template + void Unserialize(Stream& stream) + { + stream.read(CharCast(m_bytes.data()), m_bytes.size()); + } private: //! @@ -307,48 +324,34 @@ class MiningId //! std::string ToString() const; - //! - //! \brief Get the size of the data to serialize. - //! - //! \param nType Target protocol type (network, disk, etc.). - //! \param nVersion Protocol version. - //! - //! \return Size of the data in bytes. - //! - unsigned int GetSerializeSize(int nType, int nVersion) const; - //! //! \brief Serialize the object to the provided stream. //! - //! \param stream The output stream. - //! \param nType Target protocol type (network, disk, etc.). - //! \param nVersion Protocol version. + //! \param stream The output stream. //! template - void Serialize(Stream& stream, int nType, int nVersion) const + void Serialize(Stream& stream) const { unsigned char kind = m_variant.which(); - ::Serialize(stream, kind, nType, nVersion); + ::Serialize(stream, kind); if (static_cast(kind) == Kind::CPID) { - boost::get(m_variant).Serialize(stream, nType, nVersion); + boost::get(m_variant).Serialize(stream); } } //! //! \brief Deserialize the object from the provided stream. //! - //! \param stream The input stream. - //! \param nType Target protocol type (network, disk, etc.). - //! \param nVersion Protocol version. + //! \param stream The input stream. //! template - void Unserialize(Stream& stream, int nType, int nVersion) + void Unserialize(Stream& stream) { unsigned char kind; - ::Unserialize(stream, kind, nType, nVersion); + ::Unserialize(stream, kind); switch (static_cast(kind)) { case Kind::INVESTOR: @@ -357,7 +360,7 @@ class MiningId case Kind::CPID: { Cpid cpid; - cpid.Unserialize(stream, nType, nVersion); + cpid.Unserialize(stream); m_variant = std::move(cpid); } diff --git a/src/neuralnet/superblock.cpp b/src/neuralnet/superblock.cpp index 2d661f84b5..aaa0462aa2 100644 --- a/src/neuralnet/superblock.cpp +++ b/src/neuralnet/superblock.cpp @@ -1605,15 +1605,3 @@ std::string QuorumHash::ToString() const { return boost::apply_visitor(QuorumHashToStringVisitor(), m_hash); } - -unsigned int QuorumHash::GetSerializeSize(int nType, int nVersion) const -{ - switch (Which()) { - case Kind::SHA256: return 1 + sizeof(uint256); - case Kind::MD5: return 1 + sizeof(Md5Sum); - - // For variants without any associated data, we serialize the variant - // tag only as a single byte: - default: return 1; - } -} diff --git a/src/neuralnet/superblock.h b/src/neuralnet/superblock.h index ef54cd619d..a7de50a9de 100644 --- a/src/neuralnet/superblock.h +++ b/src/neuralnet/superblock.h @@ -140,42 +140,30 @@ class QuorumHash //! std::string ToString() const; - //! - //! \brief Get the size of the data to serialize. - //! - //! \param nType Target protocol type (network, disk, etc.). - //! \param nVersion Protocol version. - //! - //! \return Size of the data in bytes. - //! - unsigned int GetSerializeSize(int nType, int nVersion) const; - //! //! \brief Serialize the object to the provided stream. //! - //! \param stream The output stream. - //! \param nType Target protocol type (network, disk, etc.). - //! \param nVersion Protocol version. + //! \param stream The output stream. //! template - void Serialize(Stream& stream, int nType, int nVersion) const + void Serialize(Stream& stream) const { unsigned char kind = m_hash.which(); - ::Serialize(stream, kind, nType, nVersion); + ::Serialize(stream, kind); switch (static_cast(kind)) { case Kind::INVALID: break; // Suppress warning. case Kind::SHA256: - boost::get(m_hash).Serialize(stream, nType, nVersion); + boost::get(m_hash).Serialize(stream); break; case Kind::MD5: { const Md5Sum& hash = boost::get(m_hash); - FLATDATA(hash).Serialize(stream, nType, nVersion); + stream.write(CharCast(hash.data()), hash.size()); break; } } @@ -185,20 +173,18 @@ class QuorumHash //! \brief Deserialize the object from the provided stream. //! //! \param stream The input stream. - //! \param nType Target protocol type (network, disk, etc.). - //! \param nVersion Protocol version. //! template - void Unserialize(Stream& stream, int nType, int nVersion) + void Unserialize(Stream& stream) { unsigned char kind; - ::Unserialize(stream, kind, nType, nVersion); + ::Unserialize(stream, kind); switch (static_cast(kind)) { case Kind::SHA256: { uint256 hash; - hash.Unserialize(stream, nType, nVersion); + hash.Unserialize(stream); m_hash = hash; break; @@ -206,7 +192,7 @@ class QuorumHash case Kind::MD5: { Md5Sum hash; - FLATDATA(hash).Unserialize(stream, nType, nVersion); + stream.read(CharCast(hash.data()), hash.size()); m_hash = hash; break; @@ -390,20 +376,18 @@ class Superblock void Add(const MiningId id, const uint16_t magnitude); //! - //! \brief Get the size of the data to serialize. - //! - //! \param nType Target protocol type (network, disk, etc.). - //! \param nVersion Protocol version. + //! \brief Serialize the object to the provided stream. //! - //! \return Size of the data in bytes. + //! \param stream The output stream. //! - unsigned int GetSerializeSize(int nType, int nVersion) const + template + void Serialize(Stream& stream) const { - unsigned int size = GetSizeOfCompactSize(m_magnitudes.size()) - + m_magnitudes.size() * sizeof(Cpid) - + VARINT(m_zero_magnitude_count).GetSerializeSize(); + WriteCompactSize(stream, m_magnitudes.size()); for (const auto& cpid_pair : m_magnitudes) { + cpid_pair.first.Serialize(stream); + // Compact size encoding provides better compression for the // magnitude values than VARINT because most CPIDs have mags // less than 253: @@ -411,28 +395,6 @@ class Superblock // Note: This encoding imposes an upper limit of MAX_SIZE on // the encoded value. Magnitudes fall well within the limit. // - size += GetSizeOfCompactSize(cpid_pair.second); - } - - return size; - } - - //! - //! \brief Serialize the object to the provided stream. - //! - //! \param stream The output stream. - //! \param nType Target protocol type (network, disk, etc.). - //! \param nVersion Protocol version. - //! - template - void Serialize(Stream& stream, int nType, int nVersion) const - { - WriteCompactSize(stream, m_magnitudes.size()); - - for (const auto& cpid_pair : m_magnitudes) { - cpid_pair.first.Serialize(stream, nType, nVersion); - - // Write magnitude using compact-size encoding: WriteCompactSize(stream, cpid_pair.second); } @@ -447,7 +409,7 @@ class Superblock //! \param nVersion Protocol version. //! template - void Unserialize(Stream& stream, int nType, int nVersion) + void Unserialize(Stream& stream) { m_magnitudes.clear(); m_total_magnitude = 0; @@ -456,7 +418,7 @@ class Superblock for (size_t i = 0; i < size; i++) { Cpid cpid; - cpid.Unserialize(stream, nType, nVersion); + cpid.Unserialize(stream); // Read magnitude using compact-size encoding: uint16_t magnitude = ReadCompactSize(stream); @@ -543,20 +505,20 @@ class Superblock //! uint32_t m_convergence_hint; - IMPLEMENT_SERIALIZE - ( + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { READWRITE(VARINT(m_total_credit)); READWRITE(VARINT(m_average_rac)); READWRITE(VARINT(m_rac)); - // Only serialize and deserialize the convegence hint in fallback- - // to-project-level convergences: - // - if (nType & ProjectIndex::SER_CONVERGED_BY_PROJECT) { - READWRITE(m_convergence_hint); - } - ) - }; + // ProjectIndex handles serialization of m_convergence_hint + // when creating superblocks from fallback-to-project-level + // convergence scenarios. + } + }; // ProjectStats //! //! \brief An optional type that either contains some project statistics or @@ -659,31 +621,60 @@ class Superblock //! void SetHint(const std::string& name, const CSerializeData& part_data); - IMPLEMENT_SERIALIZE - ( - if (!(nType & SER_GETHASH)) { - READWRITE(m_converged_by_project); + //! + //! \brief Serialize the object to the provided stream. + //! + //! \param stream The output stream. + //! + template + void Serialize(Stream& stream) const + { + if (!(stream.GetType() & SER_GETHASH)) { + stream << m_converged_by_project; + } + + WriteCompactSize(stream, m_projects.size()); + + for (const auto& project_pair : m_projects) { + stream << project_pair; // Trigger serialization of ProjectStats convergence hints for // superblocks generated by a fallback-to-project convergence: // if (m_converged_by_project) { - nType |= SER_CONVERGED_BY_PROJECT; + stream << project_pair.second.m_convergence_hint; } } + } - READWRITE(m_projects); + //! + //! \brief Deserialize the object from the provided stream. + //! + //! \param stream The input stream. + //! + template + void Unserialize(Stream& stream) + { + m_projects.clear(); + m_total_rac = 0; + + stream >> m_converged_by_project; + + const unsigned int project_count = ReadCompactSize(stream); + auto iter = m_projects.begin(); - // Tally up the recent average credit after deserializing. - // - if (fRead) { - REF(m_total_rac) = 0; + for (unsigned int i = 0; i < project_count; i++) { + std::pair project_pair; + stream >> project_pair; - for (const auto& project_pair : m_projects) { - REF(m_total_rac) += project_pair.second.m_rac; + if (m_converged_by_project) { + stream >> project_pair.second.m_convergence_hint; } + + m_total_rac += project_pair.second.m_rac; + iter = m_projects.insert(iter, project_pair); } - ) + } private: //! @@ -734,20 +725,21 @@ class Superblock int64_t m_height; //!< Height of the block that contains the contract. int64_t m_timestamp; //!< Timestamp of the block that contains the contract. - IMPLEMENT_SERIALIZE - ( - if (!(nType & SER_GETHASH)) { + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (!(s.GetType() & SER_GETHASH)) { READWRITE(m_version); READWRITE(m_convergence_hint); READWRITE(m_manifest_content_hint); } - nVersion = m_version; - READWRITE(m_cpids); READWRITE(m_projects); //READWRITE(m_verified_beacons); - ) + } //! //! \brief Initialize an empty superblock object. diff --git a/src/protocol.h b/src/protocol.h index 2bc458980e..36d6ca2e78 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -12,8 +12,10 @@ #include "netbase.h" #include "serialize.h" -#include #include "uint256.h" +#include "version.h" + +#include extern bool fTestNet; static inline unsigned short GetDefaultPort(const bool testnet = fTestNet) @@ -39,13 +41,16 @@ class CMessageHeader std::string GetCommand() const; bool IsValid() const; - IMPLEMENT_SERIALIZE - ( - READWRITE(FLATDATA(pchMessageStart)); - READWRITE(FLATDATA(pchCommand)); + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(pchMessageStart); + READWRITE(pchCommand); READWRITE(nMessageSize); READWRITE(nChecksum); - ) + } // TODO: make private (improves encapsulation) //HALFORD: 12-26-2014 - Add Encryption to messages - Increase size by 32 for checksum + delimiters + 10 for timestamp = 50 = 62 (vs 12) @@ -81,20 +86,29 @@ class CAddress : public CService void Init(); - IMPLEMENT_SERIALIZE - ( - CAddress* pthis = const_cast(this); - CService* pip = (CService*)pthis; - if (fRead) - pthis->Init(); - if (nType & SER_DISK) - READWRITE(nVersion); - if ((nType & SER_DISK) || - (nVersion >= CADDR_TIME_VERSION && !(nType & SER_GETHASH))) - READWRITE(nTime); - READWRITE(nServices); - READWRITE(*pip); - ) + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (ser_action.ForRead()) { + Init(); + } + + int nVersion = s.GetVersion(); + if (s.GetType() & SER_DISK) { + READWRITE(nVersion); + } + + if ((s.GetType() & SER_DISK) + || (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) + { + READWRITE(nTime); + } + + READWRITE(nServices); + READWRITEAS(CService, *this); + } void print() const; @@ -117,11 +131,14 @@ class CInv CInv(int typeIn, const uint256& hashIn); CInv(const std::string& strType, const uint256& hashIn); - IMPLEMENT_SERIALIZE - ( + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { READWRITE(type); READWRITE(hash); - ) + } friend bool operator<(const CInv& a, const CInv& b); diff --git a/src/rpcdataacq.cpp b/src/rpcdataacq.cpp index 8a3950cf57..ec6e0c916c 100644 --- a/src/rpcdataacq.cpp +++ b/src/rpcdataacq.cpp @@ -145,7 +145,7 @@ UniValue rpc_getblockstats(const UniValue& params, bool fHelp) interesttotal+=bb.InterestSubsidy; researchcount+=(bb.ResearchSubsidy>0.001); minttotal+=cur->nMint; - unsigned sizeblock = block.GetSerializeSize(SER_NETWORK, PROTOCOL_VERSION); + unsigned sizeblock = GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION); size_min_blk=std::min(size_min_blk,sizeblock); size_max_blk=std::max(size_max_blk,sizeblock); size_sum_blk+=sizeblock; @@ -513,7 +513,7 @@ UniValue rpc_exportstats(const UniValue& params, bool fHelp) cnt_trans += block.vtx.size()-2; /* 2 transactions are special */ cnt_empty += ( block.vtx.size()<=2 ); - double i_size = block.GetSerializeSize(SER_NETWORK, PROTOCOL_VERSION); + double i_size = GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION); sum_size= sum_size + i_size; min_size=std::min(min_size,i_size); max_size=std::max(max_size,i_size); diff --git a/src/scraper/fwd.h b/src/scraper/fwd.h index 79371d4003..6ca8ee5ecb 100644 --- a/src/scraper/fwd.h +++ b/src/scraper/fwd.h @@ -5,6 +5,7 @@ #include #include +#include "support/allocators/zeroafterfree.h" #include "util.h" /********************* diff --git a/src/scraper/scraper.cpp b/src/scraper/scraper.cpp index ddb73a01c6..abee4e392f 100755 --- a/src/scraper/scraper.cpp +++ b/src/scraper/scraper.cpp @@ -2032,10 +2032,10 @@ uint256 GetFileHash(const fs::path& inputfile) { // open input file, and associate with CAutoFile FILE *file = fopen(inputfile.string().c_str(), "rb"); - CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION); + CAutoFile filein(file, SER_DISK, CLIENT_VERSION); uint256 nHash = 0; - if (!filein) + if (filein.IsNull()) return nHash; // use file size to size memory buffer @@ -3610,9 +3610,9 @@ bool ScraperSendFileManifestContents(CBitcoinAddress& Address, CKey& Key) // open input file, and associate with CAutoFile FILE *file = fopen(inputfilewpath.string().c_str(), "rb"); - CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION); + CAutoFile filein(file, SER_DISK, CLIENT_VERSION); - if (!filein) + if (filein.IsNull()) { _log(logattribute::ERR, "ScraperSendFileManifestContents", "Failed to open file (" + inputfile.string() + ")"); return false; @@ -3661,9 +3661,9 @@ bool ScraperSendFileManifestContents(CBitcoinAddress& Address, CKey& Key) // open input file, and associate with CAutoFile FILE *file = fopen(inputfilewpath.string().c_str(), "rb"); - CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION); + CAutoFile filein(file, SER_DISK, CLIENT_VERSION); - if (!filein) + if (filein.IsNull()) { _log(logattribute::ERR, "ScraperSendFileManifestContents", "Failed to open file (" + inputfile.string() + ")"); return false; @@ -5572,7 +5572,7 @@ UniValue testnewsb(const UniValue& params, bool fHelp) _log(logattribute::INFO, "testnewsb", "zero-mag count = " + std::to_string(NewFormatSuperblock.m_cpids.Zeros())); res.pushKV("zero-mag count", (uint64_t) NewFormatSuperblock.m_cpids.Zeros()); - nNewFormatSuperblockSerSize = NewFormatSuperblock.GetSerializeSize(SER_NETWORK, 1); + nNewFormatSuperblockSerSize = GetSerializeSize(NewFormatSuperblock, SER_NETWORK, 1); nNewFormatSuperblockHash = NewFormatSuperblock.GetHash(); _log(logattribute::INFO, "testnewsb", "NewFormatSuperblock.m_version = " + std::to_string(NewFormatSuperblock.m_version)); @@ -5587,7 +5587,7 @@ UniValue testnewsb(const UniValue& params, bool fHelp) ss << NewFormatSuperblock; ss >> NewFormatSuperblock_out; - nNewFormatSuperblock_outSerSize = NewFormatSuperblock_out.GetSerializeSize(SER_NETWORK, 1); + nNewFormatSuperblock_outSerSize = GetSerializeSize(NewFormatSuperblock_out, SER_NETWORK, 1); nNewFormatSuperblock_outHash = NewFormatSuperblock_out.GetHash(); _log(logattribute::INFO, "testnewsb", "nNewFormatSuperblock_outSerSize = " + std::to_string(nNewFormatSuperblock_outSerSize)); diff --git a/src/scraper_net.cpp b/src/scraper_net.cpp index 5ec2ade630..886456c966 100644 --- a/src/scraper_net.cpp +++ b/src/scraper_net.cpp @@ -216,7 +216,7 @@ bool CScraperManifest::SendManifestTo(CNode* pto, const uint256& hash) } -void CScraperManifest::dentry::Serialize(CDataStream& ss, int nType, int nVersion) const +void CScraperManifest::dentry::Serialize(CDataStream& ss) const { /* TODO: remove this redundant code */ ss<< project; ss<< ETag; @@ -226,7 +226,7 @@ void CScraperManifest::dentry::Serialize(CDataStream& ss, int nType, int nVersio ss<< current; ss<< last; } -void CScraperManifest::dentry::Unserialize(CDataStream& ss, int nType, int nVersion) +void CScraperManifest::dentry::Unserialize(CDataStream& ss) { ss>> project; ss>> ETag; @@ -238,7 +238,7 @@ void CScraperManifest::dentry::Unserialize(CDataStream& ss, int nType, int nVers } -void CScraperManifest::SerializeWithoutSignature(CDataStream& ss, int nType, int nVersion) const +void CScraperManifest::SerializeWithoutSignature(CDataStream& ss) const { WriteCompactSize(ss, vParts.size()); for( const CPart* part : vParts ) @@ -253,7 +253,7 @@ void CScraperManifest::SerializeWithoutSignature(CDataStream& ss, int nType, int } // This is to compare manifest content quickly. We just need the parts and the consensus block. -void CScraperManifest::SerializeForManifestCompare(CDataStream& ss, int nType, int nVersion) const +void CScraperManifest::SerializeForManifestCompare(CDataStream& ss) const { WriteCompactSize(ss, vParts.size()); for( const CPart* part : vParts ) @@ -262,9 +262,9 @@ void CScraperManifest::SerializeForManifestCompare(CDataStream& ss, int nType, i } -void CScraperManifest::Serialize(CDataStream& ss, int nType, int nVersion) const +void CScraperManifest::Serialize(CDataStream& ss) const { - SerializeWithoutSignature(ss, nType, nVersion); + SerializeWithoutSignature(ss); ss << signature; } @@ -571,12 +571,12 @@ bool CScraperManifest::addManifest(std::unique_ptr&& m, CKey& // serialize the content for comparison purposes and put in manifest. CDataStream sscomp(SER_NETWORK,1); - m->SerializeForManifestCompare(sscomp, SER_NETWORK, 1); + m->SerializeForManifestCompare(sscomp); m->nContentHash = Hash(sscomp.begin(), sscomp.end()); /* serialize and hash the object */ CDataStream ss(SER_NETWORK,1); - m->SerializeWithoutSignature(ss, SER_NETWORK, 1); + m->SerializeWithoutSignature(ss); //ss << *m; /* sign the serialized manifest and append the signature */ diff --git a/src/scraper_net.h b/src/scraper_net.h index db5b948e47..f017fed4ca 100755 --- a/src/scraper_net.h +++ b/src/scraper_net.h @@ -28,7 +28,7 @@ class CSplitBlob CPart(const uint256& ihash) :hash(ihash) {} - CDataStream getReader() const { return CDataStream(&data); } + CDataStream getReader() const { return CDataStream(data.begin(), data.end(), SER_NETWORK, PROTOCOL_VERSION); } bool present() const {return !this->data.empty();} }; @@ -141,8 +141,8 @@ class CScraperManifest bool current =0; bool last =0; - void Serialize(CDataStream& s, int nType, int nVersion) const; - void Unserialize(CDataStream& s, int nType, int nVersion); + void Serialize(CDataStream& s) const; + void Unserialize(CDataStream& s); UniValue ToJson() const; }; @@ -167,9 +167,9 @@ class CScraperManifest void Complete() override; /** Serialize this object for seding over the network. */ - void Serialize(CDataStream& s, int nType, int nVersion) const; - void SerializeWithoutSignature(CDataStream& s, int nType, int nVersion) const; - void SerializeForManifestCompare(CDataStream& ss, int nType, int nVersion) const; + void Serialize(CDataStream& s) const; + void SerializeWithoutSignature(CDataStream& s) const; + void SerializeForManifestCompare(CDataStream& ss) const; void UnserializeCheck(CDataStream& s, unsigned int& banscore_out); bool IsManifestCurrent() const; diff --git a/src/script.h b/src/script.h index 980ab8e7cf..317a370e0d 100644 --- a/src/script.h +++ b/src/script.h @@ -250,6 +250,9 @@ inline std::string ValueString(const std::vector& vch) return HexStr(vch); } +// TODO: change to prevector: +typedef std::vector CScriptBase; + /** Serialized script, used inside transaction inputs and outputs */ class CScript : public std::vector { @@ -290,6 +293,14 @@ class CScript : public std::vector CScript(const unsigned char* pbegin, const unsigned char* pend) : std::vector(pbegin, pend) { } #endif + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITEAS(CScriptBase, *this); + } + CScript& operator+=(const CScript& b) { insert(end(), b.begin(), b.end()); diff --git a/src/serialize.h b/src/serialize.h index d925e57e60..54feb58e9d 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1,46 +1,174 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2012 The Bitcoin developers -// Distributed under the MIT/X11 software license, see the accompanying +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SERIALIZE_H #define BITCOIN_SERIALIZE_H -#include -#include -#include -#include -#include +#include + +#include +#include #include -#include #include +#include +#include +#include #include -#include -#include - -#include - -#include "support/allocators/zeroafterfree.h" -#include "version.h" +#include +#include +#include +#include -class CScript; +#include +#include static const unsigned int MAX_SIZE = 0x02000000; -// Used to bypass the rule against non-const reference to temporary -// where it makes sense with wrappers such as CFlatData or CTxDB +/** + * Dummy data type to identify deserializing constructors. + * + * By convention, a constructor of a type T with signature + * + * template T::T(deserialize_type, Stream& s) + * + * is a deserializing constructor, which builds the type by + * deserializing it from s. If T contains const fields, this + * is likely the only way to do so. + */ +struct deserialize_type {}; +constexpr deserialize_type deserialize {}; + +/** + * Used to bypass the rule against non-const reference to temporary + * where it makes sense with wrappers. + */ template inline T& REF(const T& val) { return const_cast(val); } +/** + * Used to acquire a non-const pointer "this" to generate bodies + * of const serialization operations from a template + */ +template +inline T* NCONST_PTR(const T* val) +{ + return const_cast(val); +} + +//! Safely convert odd char pointer types to standard ones. +inline char* CharCast(char* c) { return c; } +inline char* CharCast(unsigned char* c) { return (char*)c; } +inline const char* CharCast(const char* c) { return c; } +inline const char* CharCast(const unsigned char* c) { return (const char*)c; } + +/* + * Lowest-level serialization and conversion. + * @note Sizes of these types are verified in the tests + */ +template inline void ser_writedata8(Stream &s, uint8_t obj) +{ + s.write((char*)&obj, 1); +} +template inline void ser_writedata16(Stream &s, uint16_t obj) +{ + obj = htole16(obj); + s.write((char*)&obj, 2); +} +template inline void ser_writedata16be(Stream &s, uint16_t obj) +{ + obj = htobe16(obj); + s.write((char*)&obj, 2); +} +template inline void ser_writedata32(Stream &s, uint32_t obj) +{ + obj = htole32(obj); + s.write((char*)&obj, 4); +} +template inline void ser_writedata32be(Stream &s, uint32_t obj) +{ + obj = htobe32(obj); + s.write((char*)&obj, 4); +} +template inline void ser_writedata64(Stream &s, uint64_t obj) +{ + obj = htole64(obj); + s.write((char*)&obj, 8); +} +template inline uint8_t ser_readdata8(Stream &s) +{ + uint8_t obj; + s.read((char*)&obj, 1); + return obj; +} +template inline uint16_t ser_readdata16(Stream &s) +{ + uint16_t obj; + s.read((char*)&obj, 2); + return le16toh(obj); +} +template inline uint16_t ser_readdata16be(Stream &s) +{ + uint16_t obj; + s.read((char*)&obj, 2); + return be16toh(obj); +} +template inline uint32_t ser_readdata32(Stream &s) +{ + uint32_t obj; + s.read((char*)&obj, 4); + return le32toh(obj); +} +template inline uint32_t ser_readdata32be(Stream &s) +{ + uint32_t obj; + s.read((char*)&obj, 4); + return be32toh(obj); +} +template inline uint64_t ser_readdata64(Stream &s) +{ + uint64_t obj; + s.read((char*)&obj, 8); + return le64toh(obj); +} +inline uint64_t ser_double_to_uint64(double x) +{ + union { double x; uint64_t y; } tmp; + tmp.x = x; + return tmp.y; +} +inline uint32_t ser_float_to_uint32(float x) +{ + union { float x; uint32_t y; } tmp; + tmp.x = x; + return tmp.y; +} +inline double ser_uint64_to_double(uint64_t y) +{ + union { double x; uint64_t y; } tmp; + tmp.y = y; + return tmp.x; +} +inline float ser_uint32_to_float(uint32_t y) +{ + union { float x; uint32_t y; } tmp; + tmp.y = y; + return tmp.x; +} + + ///////////////////////////////////////////////////////////////// // // Templates for serializing to anything that looks like a stream, -// i.e. anything that supports .read(char*, int) and .write(char*, int) +// i.e. anything that supports .read(char*, size_t) and .write(char*, size_t) // +class CSizeComputer; + enum { // primary actions @@ -53,115 +181,79 @@ enum SER_BLOCKHEADERONLY = (1 << 17), }; -#define IMPLEMENT_SERIALIZE(statements) \ - unsigned int GetSerializeSize(int nType, int nVersion) const \ - { \ - CSerActionGetSerializeSize ser_action; \ - const bool fGetSize = true; \ - const bool fWrite = false; \ - const bool fRead = false; \ - unsigned int nSerSize = 0; \ - ser_streamplaceholder s; \ - assert(fGetSize||fWrite||fRead); /* suppress warning */ \ - s.nType = nType; \ - s.nVersion = nVersion; \ - {statements} \ - return nSerSize; \ - } \ - template \ - void Serialize(Stream& s, int nType, int nVersion) const \ - { \ - CSerActionSerialize ser_action; \ - const bool fGetSize = false; \ - const bool fWrite = true; \ - const bool fRead = false; \ - unsigned int nSerSize = 0; \ - assert(fGetSize||fWrite||fRead); /* suppress warning */ \ - {statements} \ - } \ - template \ - void Unserialize(Stream& s, int nType, int nVersion) \ - { \ - CSerActionUnserialize ser_action; \ - const bool fGetSize = false; \ - const bool fWrite = false; \ - const bool fRead = true; \ - unsigned int nSerSize = 0; \ - assert(fGetSize||fWrite||fRead); /* suppress warning */ \ - {statements} \ - } - -#define READWRITE(obj) (nSerSize += ::SerReadWrite(s, (obj), nType, nVersion, ser_action)) - - +//! Convert the reference base type to X, without changing constness or reference type. +template X& ReadWriteAsHelper(X& x) { return x; } +template const X& ReadWriteAsHelper(const X& x) { return x; } +#define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__)) +#define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper(obj))) +/** + * Implement three methods for serializable objects. These are actually wrappers over + * "SerializationOp" template, which implements the body of each class' serialization + * code. Adding "ADD_SERIALIZE_METHODS" in the body of the class causes these wrappers to be + * added as members. + */ +#define ADD_SERIALIZE_METHODS \ + template \ + void Serialize(Stream& s) const { \ + NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \ + } \ + template \ + void Unserialize(Stream& s) { \ + SerializationOp(s, CSerActionUnserialize()); \ + } +#ifndef CHAR_EQUALS_INT8 +template inline void Serialize(Stream& s, char a ) { ser_writedata8(s, a); } // TODO Get rid of bare char +#endif +template inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); } +template inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); } +template inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); } +template inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); } +template inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); } +template inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); } +template inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); } +template inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } +template inline void Serialize(Stream& s, float a ) { ser_writedata32(s, ser_float_to_uint32(a)); } +template inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); } +template inline void Serialize(Stream& s, const char (&a)[N]) { s.write(a, N); } +template inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(CharCast(a), N); } +template inline void Serialize(Stream& s, const Span& span) { s.write(CharCast(span.data()), span.size()); } +template inline void Serialize(Stream& s, const Span& span) { s.write(CharCast(span.data()), span.size()); } + +#ifndef CHAR_EQUALS_INT8 +template inline void Unserialize(Stream& s, char& a ) { a = ser_readdata8(s); } // TODO Get rid of bare char +#endif +template inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); } +template inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); } +template inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); } +template inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); } +template inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); } +template inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); } +template inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); } +template inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } +template inline void Unserialize(Stream& s, float& a ) { a = ser_uint32_to_float(ser_readdata32(s)); } +template inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); } +template inline void Unserialize(Stream& s, char (&a)[N]) { s.read(a, N); } +template inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(CharCast(a), N); } +template inline void Unserialize(Stream& s, Span& span) { s.read(CharCast(span.data()), span.size()); } -// -// Basic types -// -#define WRITEDATA(s, obj) s.write((char*)&(obj), sizeof(obj)) -#define READDATA(s, obj) s.read((char*)&(obj), sizeof(obj)) - -inline unsigned int GetSerializeSize(char a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(signed char a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(unsigned char a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(signed short a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(unsigned short a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(signed int a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(unsigned int a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(signed long a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(unsigned long a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(signed long long a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(unsigned long long a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(float a, int, int=0) { return sizeof(a); } -inline unsigned int GetSerializeSize(double a, int, int=0) { return sizeof(a); } - -template inline void Serialize(Stream& s, char a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, signed char a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, unsigned char a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, signed short a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, unsigned short a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, signed int a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, unsigned int a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, signed long a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, unsigned long a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, signed long long a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, unsigned long long a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, float a, int, int=0) { WRITEDATA(s, a); } -template inline void Serialize(Stream& s, double a, int, int=0) { WRITEDATA(s, a); } - -template inline void Unserialize(Stream& s, char& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, signed char& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, unsigned char& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, signed short& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, unsigned short& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, signed int& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, unsigned int& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, signed long& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, unsigned long& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, signed long long& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, unsigned long long& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, float& a, int, int=0) { READDATA(s, a); } -template inline void Unserialize(Stream& s, double& a, int, int=0) { READDATA(s, a); } - -inline unsigned int GetSerializeSize(bool a, int, int=0) { return sizeof(char); } -template inline void Serialize(Stream& s, bool a, int, int=0) { char f=a; WRITEDATA(s, f); } -template inline void Unserialize(Stream& s, bool& a, int, int=0) { char f; READDATA(s, f); a=f; } +template inline void Serialize(Stream& s, bool a) { char f=a; ser_writedata8(s, f); } +template inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; } -// -// Compact size -// size < 253 -- 1 byte -// size <= USHRT_MAX -- 3 bytes (253 + 2 bytes) -// size <= UINT_MAX -- 5 bytes (254 + 4 bytes) -// size > UINT_MAX -- 9 bytes (255 + 8 bytes) -// +/** + * Compact Size + * size < 253 -- 1 byte + * size <= USHRT_MAX -- 3 bytes (253 + 2 bytes) + * size <= UINT_MAX -- 5 bytes (254 + 4 bytes) + * size > UINT_MAX -- 9 bytes (255 + 8 bytes) + */ inline unsigned int GetSizeOfCompactSize(uint64_t nSize) { if (nSize < 253) return sizeof(unsigned char); @@ -170,34 +262,29 @@ inline unsigned int GetSizeOfCompactSize(uint64_t nSize) else return sizeof(unsigned char) + sizeof(uint64_t); } +inline void WriteCompactSize(CSizeComputer& os, uint64_t nSize); + template void WriteCompactSize(Stream& os, uint64_t nSize) { if (nSize < 253) { - unsigned char chSize = nSize; - WRITEDATA(os, chSize); + ser_writedata8(os, nSize); } else if (nSize <= std::numeric_limits::max()) { - unsigned char chSize = 253; - unsigned short xSize = nSize; - WRITEDATA(os, chSize); - WRITEDATA(os, xSize); + ser_writedata8(os, 253); + ser_writedata16(os, nSize); } else if (nSize <= std::numeric_limits::max()) { - unsigned char chSize = 254; - unsigned int xSize = nSize; - WRITEDATA(os, chSize); - WRITEDATA(os, xSize); + ser_writedata8(os, 254); + ser_writedata32(os, nSize); } else { - unsigned char chSize = 255; - uint64_t xSize = nSize; - WRITEDATA(os, chSize); - WRITEDATA(os, xSize); + ser_writedata8(os, 255); + ser_writedata64(os, nSize); } return; } @@ -205,8 +292,7 @@ void WriteCompactSize(Stream& os, uint64_t nSize) template uint64_t ReadCompactSize(Stream& is) { - unsigned char chSize; - READDATA(is, chSize); + uint8_t chSize = ser_readdata8(is); uint64_t nSizeRet = 0; if (chSize < 253) { @@ -214,35 +300,27 @@ uint64_t ReadCompactSize(Stream& is) } else if (chSize == 253) { - unsigned short xSize; - READDATA(is, xSize); - nSizeRet = xSize; + nSizeRet = ser_readdata16(is); if (nSizeRet < 253) throw std::ios_base::failure("non-canonical ReadCompactSize()"); } else if (chSize == 254) { - unsigned int xSize; - READDATA(is, xSize); - nSizeRet = xSize; + nSizeRet = ser_readdata32(is); if (nSizeRet < 0x10000u) throw std::ios_base::failure("non-canonical ReadCompactSize()"); } else { - uint64_t xSize; - READDATA(is, xSize); - nSizeRet = xSize; - if (nSizeRet < 0x100000000LLu) + nSizeRet = ser_readdata64(is); + if (nSizeRet < 0x100000000ULL) throw std::ios_base::failure("non-canonical ReadCompactSize()"); } if (nSizeRet > (uint64_t)MAX_SIZE) - throw std::ios_base::failure("ReadCompactSize() : size too large"); + throw std::ios_base::failure("ReadCompactSize(): size too large"); return nSizeRet; } -class CSizeComputer; - /** * Variable-length integers: bytes are a MSB base-128 encoding of the number. * The high bit in each byte signifies whether another digit follows. To make @@ -319,7 +397,7 @@ void WriteVarInt(Stream& os, I n) len++; } do { - Serialize(os, tmp[len], SER_NETWORK, 1); + ser_writedata8(os, tmp[len]); } while(len--); } @@ -329,8 +407,7 @@ I ReadVarInt(Stream& is) CheckVarIntMode(); I n = 0; while(true) { - unsigned char chData; - Unserialize(is, chData, SER_NETWORK, 1); + unsigned char chData = ser_readdata8(is); if (n > (std::numeric_limits::max() >> 7)) { throw std::ios_base::failure("ReadVarInt(): size too large"); } @@ -346,6 +423,10 @@ I ReadVarInt(Stream& is) } } +#define VARINT(obj, ...) WrapVarInt<__VA_ARGS__>(REF(obj)) +#define COMPACTSIZE(obj) CCompactSize(REF(obj)) +#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) + template class CVarInt { @@ -354,225 +435,244 @@ class CVarInt public: explicit CVarInt(I& nIn) : n(nIn) { } - unsigned int GetSerializeSize(int nType = 0, int nVersion = 0) const - { - return GetSizeOfVarInt(n); - } - template - void Serialize(Stream &s, int nType = 0, int nVersion = 0) const { + void Serialize(Stream &s) const { WriteVarInt(s, n); } template - void Unserialize(Stream& s, int nType = 0, int nVersion = 0) { + void Unserialize(Stream& s) { n = ReadVarInt(s); } }; -template -CVarInt WrapVarInt(I& n) { return CVarInt{n}; } - -#define VARINT(obj, ...) WrapVarInt<__VA_ARGS__>(REF(obj)) -#define FLATDATA(obj) REF(CFlatData((char*)&(obj), (char*)&(obj) + sizeof(obj))) - -/** Wrapper for serializing arrays and POD. +/** Serialization wrapper class for big-endian integers. + * + * Use this wrapper around integer types that are stored in memory in native + * byte order, but serialized in big endian notation. This is only intended + * to implement serializers that are compatible with existing formats, and + * its use is not recommended for new data structures. + * + * Only 16-bit types are supported for now. */ -class CFlatData +template +class BigEndian { protected: - char* pbegin; - char* pend; + I& m_val; public: - CFlatData(void* pbeginIn, void* pendIn) : pbegin((char*)pbeginIn), pend((char*)pendIn) { } - char* begin() { return pbegin; } - const char* begin() const { return pbegin; } - char* end() { return pend; } - const char* end() const { return pend; } - - unsigned int GetSerializeSize(int, int=0) const + explicit BigEndian(I& val) : m_val(val) { - return pend - pbegin; + static_assert(std::is_unsigned::value, "BigEndian type must be unsigned integer"); + static_assert(sizeof(I) == 2 && std::numeric_limits::min() == 0 && std::numeric_limits::max() == std::numeric_limits::max(), "Unsupported BigEndian size"); } template - void Serialize(Stream& s, int, int=0) const + void Serialize(Stream& s) const { - s.write(pbegin, pend - pbegin); + ser_writedata16be(s, m_val); } template - void Unserialize(Stream& s, int, int=0) + void Unserialize(Stream& s) { - s.read(pbegin, pend - pbegin); + m_val = ser_readdata16be(s); } }; -// -// Forward declarations -// +class CCompactSize +{ +protected: + uint64_t &n; +public: + explicit CCompactSize(uint64_t& nIn) : n(nIn) { } -// string -template unsigned int GetSerializeSize(const std::basic_string& str, int, int=0); -template void Serialize(Stream& os, const std::basic_string& str, int, int=0); -template void Unserialize(Stream& is, std::basic_string& str, int, int=0); + template + void Serialize(Stream &s) const { + WriteCompactSize(s, n); + } -// vector -template unsigned int GetSerializeSize_impl(const std::vector& v, int nType, int nVersion, const boost::true_type&); -template unsigned int GetSerializeSize_impl(const std::vector& v, int nType, int nVersion, const boost::false_type&); -template inline unsigned int GetSerializeSize(const std::vector& v, int nType, int nVersion); -template void Serialize_impl(Stream& os, const std::vector& v, int nType, int nVersion, const boost::true_type&); -template void Serialize_impl(Stream& os, const std::vector& v, int nType, int nVersion, const boost::false_type&); -template inline void Serialize(Stream& os, const std::vector& v, int nType, int nVersion); -template void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, const boost::true_type&); -template void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, const boost::false_type&); -template inline void Unserialize(Stream& is, std::vector& v, int nType, int nVersion); + template + void Unserialize(Stream& s) { + n = ReadCompactSize(s); + } +}; -// others derived from vector -extern inline unsigned int GetSerializeSize(const CScript& v, int nType, int nVersion); -template void Serialize(Stream& os, const CScript& v, int nType, int nVersion); -template void Unserialize(Stream& is, CScript& v, int nType, int nVersion); +template +class LimitedString +{ +protected: + std::string& string; +public: + explicit LimitedString(std::string& _string) : string(_string) {} -// pair -template unsigned int GetSerializeSize(const std::pair& item, int nType, int nVersion); -template void Serialize(Stream& os, const std::pair& item, int nType, int nVersion); -template void Unserialize(Stream& is, std::pair& item, int nType, int nVersion); + template + void Unserialize(Stream& s) + { + size_t size = ReadCompactSize(s); + if (size > Limit) { + throw std::ios_base::failure("String length limit exceeded"); + } + string.resize(size); + if (size != 0) + s.read((char*)string.data(), size); + } -// 3 tuple -template unsigned int GetSerializeSize(const std::tuple& item, int nType, int nVersion); -template void Serialize(Stream& os, const std::tuple& item, int nType, int nVersion); -template void Unserialize(Stream& is, std::tuple& item, int nType, int nVersion); + template + void Serialize(Stream& s) const + { + WriteCompactSize(s, string.size()); + if (!string.empty()) + s.write((char*)string.data(), string.size()); + } +}; -// 4 tuple -template unsigned int GetSerializeSize(const std::tuple& item, int nType, int nVersion); -template void Serialize(Stream& os, const std::tuple& item, int nType, int nVersion); -template void Unserialize(Stream& is, std::tuple& item, int nType, int nVersion); +template +CVarInt WrapVarInt(I& n) { return CVarInt{n}; } -// map -template unsigned int GetSerializeSize(const std::map& m, int nType, int nVersion); -template void Serialize(Stream& os, const std::map& m, int nType, int nVersion); -template void Unserialize(Stream& is, std::map& m, int nType, int nVersion); +template +BigEndian WrapBigEndian(I& n) { return BigEndian(n); } -// unordered_map -template unsigned int GetSerializeSize(const std::unordered_map& m, int nType, int nVersion); -template void Serialize(Stream& os, const std::unordered_map& m, int nType, int nVersion); -template void Unserialize(Stream& is, std::unordered_map& m, int nType, int nVersion); +/** + * Forward declarations + */ -// set -template unsigned int GetSerializeSize(const std::set& m, int nType, int nVersion); -template void Serialize(Stream& os, const std::set& m, int nType, int nVersion); -template void Unserialize(Stream& is, std::set& m, int nType, int nVersion); +/** + * string + */ +template void Serialize(Stream& os, const std::basic_string& str); +template void Unserialize(Stream& is, std::basic_string& str); +/** + * prevector + * prevectors of unsigned char are a special case and are intended to be serialized as a single opaque blob. + */ +template void Serialize_impl(Stream& os, const prevector& v, const unsigned char&); +template void Serialize_impl(Stream& os, const prevector& v, const V&); +template inline void Serialize(Stream& os, const prevector& v); +template void Unserialize_impl(Stream& is, prevector& v, const unsigned char&); +template void Unserialize_impl(Stream& is, prevector& v, const V&); +template inline void Unserialize(Stream& is, prevector& v); +/** + * vector + * vectors of unsigned char are a special case and are intended to be serialized as a single opaque blob. + */ +template void Serialize_impl(Stream& os, const std::vector& v, const unsigned char&); +template void Serialize_impl(Stream& os, const std::vector& v, const bool&); +template void Serialize_impl(Stream& os, const std::vector& v, const V&); +template inline void Serialize(Stream& os, const std::vector& v); +template void Unserialize_impl(Stream& is, std::vector& v, const unsigned char&); +template void Unserialize_impl(Stream& is, std::vector& v, const V&); +template inline void Unserialize(Stream& is, std::vector& v); +/** + * pair + */ +template void Serialize(Stream& os, const std::pair& item); +template void Unserialize(Stream& is, std::pair& item); +/** + * 3 tuple (deprecated - used for accounting API) + */ +template void Serialize(Stream& os, const std::tuple& item); +template void Unserialize(Stream& is, std::tuple& item); + +/** + * map + */ +template void Serialize(Stream& os, const std::map& m); +template void Unserialize(Stream& is, std::map& m); + +/** + * set + */ +template void Serialize(Stream& os, const std::set& m); +template void Unserialize(Stream& is, std::set& m); + +/** + * shared_ptr + */ +template void Serialize(Stream& os, const std::shared_ptr& p); +template void Unserialize(Stream& os, std::shared_ptr& p); + +/** + * unique_ptr + */ +template void Serialize(Stream& os, const std::unique_ptr& p); +template void Unserialize(Stream& os, std::unique_ptr& p); -// -// If none of the specialized versions above matched, default to calling member function. -// "int nType" is changed to "long nType" to keep from getting an ambiguous overload error. -// The compiler will only cast int to long if none of the other templates matched. -// Thanks to Boost serialization for this idea. -// -template -inline unsigned int GetSerializeSize(const T& a, long nType, int nVersion) -{ - return a.GetSerializeSize((int)nType, nVersion); -} + +/** + * If none of the specialized versions above matched, default to calling member function. + */ template -inline void Serialize(Stream& os, const T& a, long nType, int nVersion) +inline void Serialize(Stream& os, const T& a) { - a.Serialize(os, (int)nType, nVersion); + a.Serialize(os); } template -inline void Unserialize(Stream& is, T&& a, long nType, int nVersion) +inline void Unserialize(Stream& is, T&& a) { - a.Unserialize(is, (int)nType, nVersion); + a.Unserialize(is); } -// -// string -// -template -unsigned int GetSerializeSize(const std::basic_string& str, int, int) -{ - return GetSizeOfCompactSize(str.size()) + str.size() * sizeof(str[0]); -} - +/** + * string + */ template -void Serialize(Stream& os, const std::basic_string& str, int, int) +void Serialize(Stream& os, const std::basic_string& str) { WriteCompactSize(os, str.size()); if (!str.empty()) - os.write((char*)&str[0], str.size() * sizeof(str[0])); + os.write((char*)str.data(), str.size() * sizeof(C)); } template -void Unserialize(Stream& is, std::basic_string& str, int, int) +void Unserialize(Stream& is, std::basic_string& str) { unsigned int nSize = ReadCompactSize(is); str.resize(nSize); if (nSize != 0) - is.read((char*)&str[0], nSize * sizeof(str[0])); -} - - - -// -// vector -// -template -unsigned int GetSerializeSize_impl(const std::vector& v, int nType, int nVersion, const boost::true_type&) -{ - return (GetSizeOfCompactSize(v.size()) + v.size() * sizeof(T)); -} - -template -unsigned int GetSerializeSize_impl(const std::vector& v, int nType, int nVersion, const boost::false_type&) -{ - unsigned int nSize = GetSizeOfCompactSize(v.size()); - for (typename std::vector::const_iterator vi = v.begin(); vi != v.end(); ++vi) - nSize += GetSerializeSize((*vi), nType, nVersion); - return nSize; + is.read((char*)str.data(), nSize * sizeof(C)); } -template -inline unsigned int GetSerializeSize(const std::vector& v, int nType, int nVersion) -{ - return GetSerializeSize_impl(v, nType, nVersion, boost::is_fundamental()); -} -template -void Serialize_impl(Stream& os, const std::vector& v, int nType, int nVersion, const boost::true_type&) +/** + * prevector + */ +template +void Serialize_impl(Stream& os, const prevector& v, const unsigned char&) { WriteCompactSize(os, v.size()); if (!v.empty()) - os.write((char*)&v[0], v.size() * sizeof(T)); + os.write((char*)v.data(), v.size() * sizeof(T)); } -template -void Serialize_impl(Stream& os, const std::vector& v, int nType, int nVersion, const boost::false_type&) +template +void Serialize_impl(Stream& os, const prevector& v, const V&) { WriteCompactSize(os, v.size()); - for (typename std::vector::const_iterator vi = v.begin(); vi != v.end(); ++vi) - ::Serialize(os, (*vi), nType, nVersion); + for (typename prevector::const_iterator vi = v.begin(); vi != v.end(); ++vi) + ::Serialize(os, (*vi)); } -template -inline void Serialize(Stream& os, const std::vector& v, int nType, int nVersion) +template +inline void Serialize(Stream& os, const prevector& v) { - Serialize_impl(os, v, nType, nVersion, boost::is_fundamental()); + Serialize_impl(os, v, T()); } -template -void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, const boost::true_type&) +template +void Unserialize_impl(Stream& is, prevector& v, const unsigned char&) { // Limit size per read so bogus size value won't cause out of memory v.clear(); @@ -581,14 +681,14 @@ void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, while (i < nSize) { unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T))); - v.resize(i + blk); + v.resize_uninitialized(i + blk); is.read((char*)&v[i], blk * sizeof(T)); i += blk; } } -template -void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, const boost::false_type&) +template +void Unserialize_impl(Stream& is, prevector& v, const V&) { v.clear(); unsigned int nSize = ReadCompactSize(is); @@ -599,226 +699,177 @@ void Unserialize_impl(Stream& is, std::vector& v, int nType, int nVersion, nMid += 5000000 / sizeof(T); if (nMid > nSize) nMid = nSize; - v.resize(nMid); - for (; i < nMid; i++) - Unserialize(is, v[i], nType, nVersion); + v.resize_uninitialized(nMid); + for (; i < nMid; ++i) + Unserialize(is, v[i]); } } -template -inline void Unserialize(Stream& is, std::vector& v, int nType, int nVersion) +template +inline void Unserialize(Stream& is, prevector& v) { - Unserialize_impl(is, v, nType, nVersion, boost::is_fundamental()); + Unserialize_impl(is, v, T()); } -// -// others derived from vector -// -inline unsigned int GetSerializeSize(const CScript& v, int nType, int nVersion) -{ - return GetSerializeSize((const std::vector&)v, nType, nVersion); -} - -template -void Serialize(Stream& os, const CScript& v, int nType, int nVersion) -{ - Serialize(os, (const std::vector&)v, nType, nVersion); -} - -template -void Unserialize(Stream& is, CScript& v, int nType, int nVersion) +/** + * vector + */ +template +void Serialize_impl(Stream& os, const std::vector& v, const unsigned char&) { - Unserialize(is, (std::vector&)v, nType, nVersion); + WriteCompactSize(os, v.size()); + if (!v.empty()) + os.write((char*)v.data(), v.size() * sizeof(T)); } - - -// -// pair -// -template -unsigned int GetSerializeSize(const std::pair& item, int nType, int nVersion) +template +void Serialize_impl(Stream& os, const std::vector& v, const bool&) { - return GetSerializeSize(item.first, nType, nVersion) + GetSerializeSize(item.second, nType, nVersion); + // A special case for std::vector, as dereferencing + // std::vector::const_iterator does not result in a const bool& + // due to std::vector's special casing for bool arguments. + WriteCompactSize(os, v.size()); + for (bool elem : v) { + ::Serialize(os, elem); + } } -template -void Serialize(Stream& os, const std::pair& item, int nType, int nVersion) +template +void Serialize_impl(Stream& os, const std::vector& v, const V&) { - Serialize(os, item.first, nType, nVersion); - Serialize(os, item.second, nType, nVersion); + WriteCompactSize(os, v.size()); + for (typename std::vector::const_iterator vi = v.begin(); vi != v.end(); ++vi) + ::Serialize(os, (*vi)); } -template -void Unserialize(Stream& is, std::pair& item, int nType, int nVersion) +template +inline void Serialize(Stream& os, const std::vector& v) { - Unserialize(is, item.first, nType, nVersion); - Unserialize(is, item.second, nType, nVersion); + Serialize_impl(os, v, T()); } - -// -// 3 tuple -// -template -unsigned int GetSerializeSize(const std::tuple& item, int nType, int nVersion) +template +void Unserialize_impl(Stream& is, std::vector& v, const unsigned char&) { - unsigned int nSize = 0; - nSize += GetSerializeSize(std::get<0>(item), nType, nVersion); - nSize += GetSerializeSize(std::get<1>(item), nType, nVersion); - nSize += GetSerializeSize(std::get<2>(item), nType, nVersion); - return nSize; + // Limit size per read so bogus size value won't cause out of memory + v.clear(); + unsigned int nSize = ReadCompactSize(is); + unsigned int i = 0; + while (i < nSize) + { + unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T))); + v.resize(i + blk); + is.read((char*)&v[i], blk * sizeof(T)); + i += blk; + } } -template -void Serialize(Stream& os, const std::tuple& item, int nType, int nVersion) +template +void Unserialize_impl(Stream& is, std::vector& v, const V&) { - Serialize(os, std::get<0>(item), nType, nVersion); - Serialize(os, std::get<1>(item), nType, nVersion); - Serialize(os, std::get<2>(item), nType, nVersion); + v.clear(); + unsigned int nSize = ReadCompactSize(is); + unsigned int i = 0; + unsigned int nMid = 0; + while (nMid < nSize) + { + nMid += 5000000 / sizeof(T); + if (nMid > nSize) + nMid = nSize; + v.resize(nMid); + for (; i < nMid; i++) + Unserialize(is, v[i]); + } } -template -void Unserialize(Stream& is, std::tuple& item, int nType, int nVersion) +template +inline void Unserialize(Stream& is, std::vector& v) { - Unserialize(is, std::get<0>(item), nType, nVersion); - Unserialize(is, std::get<1>(item), nType, nVersion); - Unserialize(is, std::get<2>(item), nType, nVersion); + Unserialize_impl(is, v, T()); } -// -// 4 tuple -// -template -unsigned int GetSerializeSize(const std::tuple& item, int nType, int nVersion) -{ - unsigned int nSize = 0; - nSize += GetSerializeSize(std::get<0>(item), nType, nVersion); - nSize += GetSerializeSize(std::get<1>(item), nType, nVersion); - nSize += GetSerializeSize(std::get<2>(item), nType, nVersion); - nSize += GetSerializeSize(std::get<3>(item), nType, nVersion); - return nSize; -} - -template -void Serialize(Stream& os, const std::tuple& item, int nType, int nVersion) +/** + * pair + */ +template +void Serialize(Stream& os, const std::pair& item) { - Serialize(os, std::get<0>(item), nType, nVersion); - Serialize(os, std::get<1>(item), nType, nVersion); - Serialize(os, std::get<2>(item), nType, nVersion); - Serialize(os, std::get<3>(item), nType, nVersion); + Serialize(os, item.first); + Serialize(os, item.second); } -template -void Unserialize(Stream& is, std::tuple& item, int nType, int nVersion) +template +void Unserialize(Stream& is, std::pair& item) { - Unserialize(is, std::get<0>(item), nType, nVersion); - Unserialize(is, std::get<1>(item), nType, nVersion); - Unserialize(is, std::get<2>(item), nType, nVersion); - Unserialize(is, std::get<3>(item), nType, nVersion); + Unserialize(is, item.first); + Unserialize(is, item.second); } - -// -// map -// -template -unsigned int GetSerializeSize(const std::map& m, int nType, int nVersion) -{ - unsigned int nSize = GetSizeOfCompactSize(m.size()); - for (typename std::map::const_iterator mi = m.begin(); mi != m.end(); ++mi) - nSize += GetSerializeSize((*mi), nType, nVersion); - return nSize; -} - -template -void Serialize(Stream& os, const std::map& m, int nType, int nVersion) +/** + * 3 tuple (deprecated - used for accounting API) + */ +template +void Serialize(Stream& os, const std::tuple& item) { - WriteCompactSize(os, m.size()); - for (typename std::map::const_iterator mi = m.begin(); mi != m.end(); ++mi) - Serialize(os, (*mi), nType, nVersion); + Serialize(os, std::get<0>(item)); + Serialize(os, std::get<1>(item)); + Serialize(os, std::get<2>(item)); } -template -void Unserialize(Stream& is, std::map& m, int nType, int nVersion) -{ - m.clear(); - unsigned int nSize = ReadCompactSize(is); - typename std::map::iterator mi = m.begin(); - for (unsigned int i = 0; i < nSize; i++) - { - std::pair item; - Unserialize(is, item, nType, nVersion); - mi = m.insert(mi, item); - } -} - - - - -// -// unordered_map -// -template -unsigned int GetSerializeSize(const std::unordered_map& m, int nType, int nVersion) +template +void Unserialize(Stream& is, std::tuple& item) { - unsigned int nSize = GetSizeOfCompactSize(m.size()); - for (typename std::unordered_map::const_iterator mi = m.begin(); mi != m.end(); ++mi) - nSize += GetSerializeSize((*mi), nType, nVersion); - return nSize; + Unserialize(is, std::get<0>(item)); + Unserialize(is, std::get<1>(item)); + Unserialize(is, std::get<2>(item)); } +/** + * map + */ template -void Serialize(Stream& os, const std::unordered_map& m, int nType, int nVersion) +void Serialize(Stream& os, const std::map& m) { WriteCompactSize(os, m.size()); - for (typename std::unordered_map::const_iterator mi = m.begin(); mi != m.end(); ++mi) - Serialize(os, (*mi), nType, nVersion); + for (const auto& entry : m) + Serialize(os, entry); } template -void Unserialize(Stream& is, std::unordered_map& m, int nType, int nVersion) +void Unserialize(Stream& is, std::map& m) { m.clear(); unsigned int nSize = ReadCompactSize(is); - typename std::unordered_map::iterator mi = m.begin(); + typename std::map::iterator mi = m.begin(); for (unsigned int i = 0; i < nSize; i++) { std::pair item; - Unserialize(is, item, nType, nVersion); + Unserialize(is, item); mi = m.insert(mi, item); } } -// -// set -// -template -unsigned int GetSerializeSize(const std::set& m, int nType, int nVersion) -{ - unsigned int nSize = GetSizeOfCompactSize(m.size()); - for (typename std::set::const_iterator it = m.begin(); it != m.end(); ++it) - nSize += GetSerializeSize((*it), nType, nVersion); - return nSize; -} +/** + * set + */ template -void Serialize(Stream& os, const std::set& m, int nType, int nVersion) +void Serialize(Stream& os, const std::set& m) { WriteCompactSize(os, m.size()); for (typename std::set::const_iterator it = m.begin(); it != m.end(); ++it) - Serialize(os, (*it), nType, nVersion); + Serialize(os, (*it)); } template -void Unserialize(Stream& is, std::set& m, int nType, int nVersion) +void Unserialize(Stream& is, std::set& m) { m.clear(); unsigned int nSize = ReadCompactSize(is); @@ -826,76 +877,178 @@ void Unserialize(Stream& is, std::set& m, int nType, int nVersion) for (unsigned int i = 0; i < nSize; i++) { K key; - Unserialize(is, key, nType, nVersion); + Unserialize(is, key); it = m.insert(it, key); } } -// -// Support for IMPLEMENT_SERIALIZE and READWRITE macro -// -class CSerActionGetSerializeSize { }; -class CSerActionSerialize { }; -class CSerActionUnserialize { }; +/** + * unique_ptr + */ +template void +Serialize(Stream& os, const std::unique_ptr& p) +{ + Serialize(os, *p); +} template -inline unsigned int SerReadWrite(Stream& s, const T& obj, int nType, int nVersion, CSerActionGetSerializeSize ser_action) +void Unserialize(Stream& is, std::unique_ptr& p) { - return ::GetSerializeSize(obj, nType, nVersion); + p.reset(new T(deserialize, is)); } -template -inline unsigned int SerReadWrite(Stream& s, const T& obj, int nType, int nVersion, CSerActionSerialize ser_action) + + +/** + * shared_ptr + */ +template void +Serialize(Stream& os, const std::shared_ptr& p) { - ::Serialize(s, obj, nType, nVersion); - return 0; + Serialize(os, *p); } template -inline unsigned int SerReadWrite(Stream& s, T&& obj, int nType, int nVersion, CSerActionUnserialize ser_action) +void Unserialize(Stream& is, std::shared_ptr& p) { - ::Unserialize(s, obj, nType, nVersion); - return 0; + p = std::make_shared(deserialize, is); } -struct ser_streamplaceholder + + +/** + * Support for ADD_SERIALIZE_METHODS and READWRITE macro + */ +struct CSerActionSerialize +{ + constexpr bool ForRead() const { return false; } +}; +struct CSerActionUnserialize { - int nType; - int nVersion; + constexpr bool ForRead() const { return true; } }; -typedef std::vector > CSerializeData; + + + + + +/* ::GetSerializeSize implementations + * + * Computing the serialized size of objects is done through a special stream + * object of type CSizeComputer, which only records the number of bytes written + * to it. + * + * If your Serialize or SerializationOp method has non-trivial overhead for + * serialization, it may be worthwhile to implement a specialized version for + * CSizeComputer, which uses the s.seek() method to record bytes that would + * be written instead. + */ class CSizeComputer { protected: size_t nSize; + const int nType; + const int nVersion; public: - int nType; - int nVersion; + explicit CSizeComputer(int nTypeIn, int nVersionIn) : nSize(0), nType(nTypeIn), nVersion(nVersionIn) {} - CSizeComputer(int nTypeIn, int nVersionIn) : nSize(0), nType(nTypeIn), nVersion(nVersionIn) {} + void write(const char *psz, size_t _nSize) + { + this->nSize += _nSize; + } - CSizeComputer& write(const char *psz, int nSize) + /** Pretend _nSize bytes are written, without specifying them. */ + void seek(size_t _nSize) { - this->nSize += nSize; - return *this; + this->nSize += _nSize; } template CSizeComputer& operator<<(const T& obj) { - ::Serialize(*this, obj, nType, nVersion); + ::Serialize(*this, obj); return (*this); } size_t size() const { return nSize; } + + int GetVersion() const { return nVersion; } + int GetType() const { return nType; } }; -#endif +template +void SerializeMany(Stream& s) +{ +} + +template +void SerializeMany(Stream& s, const Arg& arg, const Args&... args) +{ + ::Serialize(s, arg); + ::SerializeMany(s, args...); +} + +template +inline void UnserializeMany(Stream& s) +{ +} + +template +inline void UnserializeMany(Stream& s, Arg&& arg, Args&&... args) +{ + ::Unserialize(s, arg); + ::UnserializeMany(s, args...); +} + +template +inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, const Args&... args) +{ + ::SerializeMany(s, args...); +} + +template +inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&&... args) +{ + ::UnserializeMany(s, args...); +} + +template +inline void WriteVarInt(CSizeComputer &s, I n) +{ + s.seek(GetSizeOfVarInt(n)); +} + +inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize) +{ + s.seek(GetSizeOfCompactSize(nSize)); +} + +template +size_t GetSerializeSize(const T& t, int nType, int nVersion = 0) +{ + return (CSizeComputer(nType, nVersion) << t).size(); +} + +template +size_t GetSerializeSize(const S& s, const T& t) +{ + return (CSizeComputer(s.GetType(), s.GetVersion()) << t).size(); +} + +template +size_t GetSerializeSizeMany(int nVersion, const T&... t) +{ + CSizeComputer sc(nVersion); + SerializeMany(sc, t...); + return sc.size(); +} + +#endif // BITCOIN_SERIALIZE_H diff --git a/src/test/neuralnet/cpid_tests.cpp b/src/test/neuralnet/cpid_tests.cpp index 5ea9b0a163..73785f9216 100644 --- a/src/test/neuralnet/cpid_tests.cpp +++ b/src/test/neuralnet/cpid_tests.cpp @@ -210,7 +210,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream) NN::Cpid cpid(expected); - BOOST_CHECK(cpid.GetSerializeSize(SER_NETWORK, 1) == 16); + BOOST_CHECK(GetSerializeSize(cpid, SER_NETWORK, 1) == 16); CDataStream stream(SER_NETWORK, 1); stream << cpid; @@ -438,7 +438,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream_for_invalid) { NN::MiningId mining_id; - BOOST_CHECK(mining_id.GetSerializeSize(SER_NETWORK, 1) == 1); + BOOST_CHECK(GetSerializeSize(mining_id, SER_NETWORK, 1) == 1); CDataStream stream(SER_NETWORK, 1); stream << mining_id; @@ -451,7 +451,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream_for_investor) { NN::MiningId mining_id = NN::MiningId::ForInvestor(); - BOOST_CHECK(mining_id.GetSerializeSize(SER_NETWORK, 1) == 1); + BOOST_CHECK(GetSerializeSize(mining_id, SER_NETWORK, 1) == 1); CDataStream stream(SER_NETWORK, 1); stream << mining_id; @@ -469,7 +469,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream_for_cpid) NN::MiningId mining_id{NN::Cpid(expected)}; - BOOST_CHECK(mining_id.GetSerializeSize(SER_NETWORK, 1) == 17); + BOOST_CHECK(GetSerializeSize(mining_id, SER_NETWORK, 1) == 17); CDataStream stream(SER_NETWORK, 1); stream << mining_id; diff --git a/src/test/neuralnet/superblock_tests.cpp b/src/test/neuralnet/superblock_tests.cpp index 78b713873b..196d36e6e2 100644 --- a/src/test/neuralnet/superblock_tests.cpp +++ b/src/test/neuralnet/superblock_tests.cpp @@ -842,7 +842,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream) NN::Superblock superblock = NN::Superblock::FromConvergence(GetTestConvergence()); - BOOST_CHECK(superblock.GetSerializeSize(SER_NETWORK, 1) == expected.size()); + BOOST_CHECK(GetSerializeSize(superblock, SER_NETWORK, 1) == expected.size()); CDataStream stream(SER_NETWORK, 1); stream << superblock; @@ -959,7 +959,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream_for_fallback_convergences) NN::Superblock superblock = NN::Superblock::FromConvergence( GetTestConvergence(true)); // Set fallback by project flag - BOOST_CHECK(superblock.GetSerializeSize(SER_NETWORK, 1) == expected.size()); + BOOST_CHECK(GetSerializeSize(superblock, SER_NETWORK, 1) == expected.size()); CDataStream stream(SER_NETWORK, 1); stream << superblock; @@ -1285,7 +1285,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream) cpids.Add(NN::Cpid::Parse("00010203040506070809101112131415"), 123); cpids.Add(NN::Cpid(), 0); - BOOST_CHECK(cpids.GetSerializeSize(SER_NETWORK, 1) == expected.size()); + BOOST_CHECK(GetSerializeSize(cpids, SER_NETWORK, 1) == expected.size()); CDataStream stream(SER_NETWORK, 1); stream << cpids; @@ -1367,7 +1367,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream) NN::Superblock::ProjectStats project(1, 2, 3); - BOOST_CHECK(project.GetSerializeSize(SER_NETWORK, 1) == expected.size()); + BOOST_CHECK(GetSerializeSize(project, SER_NETWORK, 1) == expected.size()); CDataStream stream(SER_NETWORK, 1); stream << project; @@ -1622,7 +1622,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream) projects.Add("project_1", NN::Superblock::ProjectStats(1, 2, 3)); projects.Add("project_2", NN::Superblock::ProjectStats(1, 2, 3)); - BOOST_CHECK(projects.GetSerializeSize(SER_NETWORK, 1) == expected.size()); + BOOST_CHECK(GetSerializeSize(projects, SER_NETWORK, 1) == expected.size()); CDataStream stream(SER_NETWORK, 1); stream << projects; @@ -1708,7 +1708,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream_for_fallback_convergences) projects.SetHint("project_1", CSerializeData()); projects.SetHint("project_2", CSerializeData()); - BOOST_CHECK(projects.GetSerializeSize(SER_NETWORK, 1) == expected.size()); + BOOST_CHECK(GetSerializeSize(projects, SER_NETWORK, 1) == expected.size()); CDataStream stream(SER_NETWORK, 1); stream << projects; @@ -2091,7 +2091,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream_for_invalid) { const NN::QuorumHash hash; - BOOST_CHECK(hash.GetSerializeSize(SER_NETWORK, 1) == 1); + BOOST_CHECK(GetSerializeSize(hash, SER_NETWORK, 1) == 1); CDataStream stream(SER_NETWORK, 1); stream << hash; @@ -2111,7 +2111,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream_for_sha256) const NN::QuorumHash hash(expected); - BOOST_CHECK(hash.GetSerializeSize(SER_NETWORK, 1) == 33); + BOOST_CHECK(GetSerializeSize(hash, SER_NETWORK, 1) == 33); CDataStream stream(SER_NETWORK, 1); stream << hash; @@ -2135,7 +2135,7 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream_for_md5) const NN::QuorumHash hash(expected); - BOOST_CHECK(hash.GetSerializeSize(SER_NETWORK, 1) == 17); + BOOST_CHECK(GetSerializeSize(hash, SER_NETWORK, 1) == 17); CDataStream stream(SER_NETWORK, 1); stream << hash; @@ -2175,7 +2175,7 @@ BOOST_AUTO_TEST_CASE(it_deserializes_from_a_stream_for_sha256) CDataStream stream(SER_NETWORK, 1); stream << (unsigned char)0x01; // QuorumHash::Kind::SHA256 - stream << FLATDATA(expected); + stream.write(CharCast(expected.data()), expected.size()); stream >> hash; BOOST_CHECK(hash.Which() == NN::QuorumHash::Kind::SHA256); @@ -2198,7 +2198,7 @@ BOOST_AUTO_TEST_CASE(it_deserializes_from_a_stream_for_md5) CDataStream stream(SER_NETWORK, 1); stream << (unsigned char)0x02; // QuorumHash::Kind::MD5 - stream << FLATDATA(expected); + stream.write(CharCast(expected.data()), expected.size()); stream >> hash; BOOST_CHECK(hash.Which() == NN::QuorumHash::Kind::MD5); diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index c8551dfc88..19f04ba899 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -1,9 +1,180 @@ -#include "serialize.h" -#include "streams.h" -#include "util.h" +// Copyright (c) 2012-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include + +#include #include +BOOST_AUTO_TEST_SUITE(serialize_tests) + +class CSerializeMethodsTestSingle +{ +protected: + int intval; + bool boolval; + std::string stringval; + char charstrval[16]; + CTransaction txval; +public: + CSerializeMethodsTestSingle() = default; + CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char* charstrvalin, const CTransaction& txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), txval(txvalin) + { + memcpy(charstrval, charstrvalin, sizeof(charstrval)); + } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(intval); + READWRITE(boolval); + READWRITE(stringval); + READWRITE(charstrval); + READWRITE(txval); + } + + bool operator==(const CSerializeMethodsTestSingle& rhs) + { + return intval == rhs.intval && \ + boolval == rhs.boolval && \ + stringval == rhs.stringval && \ + strcmp(charstrval, rhs.charstrval) == 0 && \ + txval == rhs.txval; + } +}; + +class CSerializeMethodsTestMany : public CSerializeMethodsTestSingle +{ +public: + using CSerializeMethodsTestSingle::CSerializeMethodsTestSingle; + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(intval, boolval, stringval, charstrval, txval); + } +}; + +BOOST_AUTO_TEST_CASE(sizes) +{ + BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(char(0), 0)); + BOOST_CHECK_EQUAL(sizeof(int8_t), GetSerializeSize(int8_t(0), 0)); + BOOST_CHECK_EQUAL(sizeof(uint8_t), GetSerializeSize(uint8_t(0), 0)); + BOOST_CHECK_EQUAL(sizeof(int16_t), GetSerializeSize(int16_t(0), 0)); + BOOST_CHECK_EQUAL(sizeof(uint16_t), GetSerializeSize(uint16_t(0), 0)); + BOOST_CHECK_EQUAL(sizeof(int32_t), GetSerializeSize(int32_t(0), 0)); + BOOST_CHECK_EQUAL(sizeof(uint32_t), GetSerializeSize(uint32_t(0), 0)); + BOOST_CHECK_EQUAL(sizeof(int64_t), GetSerializeSize(int64_t(0), 0)); + BOOST_CHECK_EQUAL(sizeof(uint64_t), GetSerializeSize(uint64_t(0), 0)); + BOOST_CHECK_EQUAL(sizeof(float), GetSerializeSize(float(0), 0)); + BOOST_CHECK_EQUAL(sizeof(double), GetSerializeSize(double(0), 0)); + // Bool is serialized as char + BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(bool(0), 0)); + + // Sanity-check GetSerializeSize and c++ type matching + BOOST_CHECK_EQUAL(GetSerializeSize(char(0), 0), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(int8_t(0), 0), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(uint8_t(0), 0), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(int16_t(0), 0), 2U); + BOOST_CHECK_EQUAL(GetSerializeSize(uint16_t(0), 0), 2U); + BOOST_CHECK_EQUAL(GetSerializeSize(int32_t(0), 0), 4U); + BOOST_CHECK_EQUAL(GetSerializeSize(uint32_t(0), 0), 4U); + BOOST_CHECK_EQUAL(GetSerializeSize(int64_t(0), 0), 8U); + BOOST_CHECK_EQUAL(GetSerializeSize(uint64_t(0), 0), 8U); + BOOST_CHECK_EQUAL(GetSerializeSize(float(0), 0), 4U); + BOOST_CHECK_EQUAL(GetSerializeSize(double(0), 0), 8U); + BOOST_CHECK_EQUAL(GetSerializeSize(bool(0), 0), 1U); +} + +BOOST_AUTO_TEST_CASE(floats_conversion) +{ + // Choose values that map unambiguously to binary floating point to avoid + // rounding issues at the compiler side. + BOOST_CHECK_EQUAL(ser_uint32_to_float(0x00000000), 0.0F); + BOOST_CHECK_EQUAL(ser_uint32_to_float(0x3f000000), 0.5F); + BOOST_CHECK_EQUAL(ser_uint32_to_float(0x3f800000), 1.0F); + BOOST_CHECK_EQUAL(ser_uint32_to_float(0x40000000), 2.0F); + BOOST_CHECK_EQUAL(ser_uint32_to_float(0x40800000), 4.0F); + BOOST_CHECK_EQUAL(ser_uint32_to_float(0x44444444), 785.066650390625F); + + BOOST_CHECK_EQUAL(ser_float_to_uint32(0.0F), 0x00000000U); + BOOST_CHECK_EQUAL(ser_float_to_uint32(0.5F), 0x3f000000U); + BOOST_CHECK_EQUAL(ser_float_to_uint32(1.0F), 0x3f800000U); + BOOST_CHECK_EQUAL(ser_float_to_uint32(2.0F), 0x40000000U); + BOOST_CHECK_EQUAL(ser_float_to_uint32(4.0F), 0x40800000U); + BOOST_CHECK_EQUAL(ser_float_to_uint32(785.066650390625F), 0x44444444U); +} + +BOOST_AUTO_TEST_CASE(doubles_conversion) +{ + // Choose values that map unambiguously to binary floating point to avoid + // rounding issues at the compiler side. + BOOST_CHECK_EQUAL(ser_uint64_to_double(0x0000000000000000ULL), 0.0); + BOOST_CHECK_EQUAL(ser_uint64_to_double(0x3fe0000000000000ULL), 0.5); + BOOST_CHECK_EQUAL(ser_uint64_to_double(0x3ff0000000000000ULL), 1.0); + BOOST_CHECK_EQUAL(ser_uint64_to_double(0x4000000000000000ULL), 2.0); + BOOST_CHECK_EQUAL(ser_uint64_to_double(0x4010000000000000ULL), 4.0); + BOOST_CHECK_EQUAL(ser_uint64_to_double(0x4088888880000000ULL), 785.066650390625); + + BOOST_CHECK_EQUAL(ser_double_to_uint64(0.0), 0x0000000000000000ULL); + BOOST_CHECK_EQUAL(ser_double_to_uint64(0.5), 0x3fe0000000000000ULL); + BOOST_CHECK_EQUAL(ser_double_to_uint64(1.0), 0x3ff0000000000000ULL); + BOOST_CHECK_EQUAL(ser_double_to_uint64(2.0), 0x4000000000000000ULL); + BOOST_CHECK_EQUAL(ser_double_to_uint64(4.0), 0x4010000000000000ULL); + BOOST_CHECK_EQUAL(ser_double_to_uint64(785.066650390625), 0x4088888880000000ULL); +} +/* +Python code to generate the below hashes: + + def reversed_hex(x): + return binascii.hexlify(''.join(reversed(x))) + def dsha256(x): + return hashlib.sha256(hashlib.sha256(x).digest()).digest() + + reversed_hex(dsha256(''.join(struct.pack('> j; + BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); + } +} + +BOOST_AUTO_TEST_CASE(doubles) +{ + CDataStream ss(SER_DISK, 0); + // encode + for (int i = 0; i < 1000; i++) { + ss << double(i); + } + BOOST_CHECK(Hash(ss.begin(), ss.end()) == uint256("43d0c82591953c4eafe114590d392676a01585d25b25d433557f0d7878b23f96")); + + // decode + for (int i = 0; i < 1000; i++) { + double j; + ss >> j; + BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); + } +} + BOOST_AUTO_TEST_CASE(varints) { // encode @@ -12,13 +183,13 @@ BOOST_AUTO_TEST_CASE(varints) CDataStream::size_type size = 0; for (int i = 0; i < 100000; i++) { ss << VARINT(i, VarIntMode::NONNEGATIVE_SIGNED); - size += ::GetSerializeSize(VARINT(i, VarIntMode::NONNEGATIVE_SIGNED), 0, 0); + size += ::GetSerializeSize(VARINT(i, VarIntMode::NONNEGATIVE_SIGNED), 0); BOOST_CHECK(size == ss.size()); } for (uint64_t i = 0; i < 100000000000ULL; i += 999999937) { ss << VARINT(i); - size += ::GetSerializeSize(VARINT(i), 0, 0); + size += ::GetSerializeSize(VARINT(i), 0); BOOST_CHECK(size == ss.size()); } @@ -56,3 +227,155 @@ BOOST_AUTO_TEST_CASE(varints_bitpatterns) ss << VARINT(0x7fffffffffffffffLL, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "fefefefefefefefe7f"); ss.clear(); ss << VARINT(0xffffffffffffffffULL); BOOST_CHECK_EQUAL(HexStr(ss), "80fefefefefefefefe7f"); ss.clear(); } + +BOOST_AUTO_TEST_CASE(compactsize) +{ + CDataStream ss(SER_DISK, 0); + std::vector::size_type i, j; + + for (i = 1; i <= MAX_SIZE; i *= 2) + { + WriteCompactSize(ss, i-1); + WriteCompactSize(ss, i); + } + for (i = 1; i <= MAX_SIZE; i *= 2) + { + j = ReadCompactSize(ss); + BOOST_CHECK_MESSAGE((i-1) == j, "decoded:" << j << " expected:" << (i-1)); + j = ReadCompactSize(ss); + BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); + } +} + +static bool isCanonicalException(const std::ios_base::failure& ex) +{ + std::ios_base::failure expectedException("non-canonical ReadCompactSize()"); + + // The string returned by what() can be different for different platforms. + // Instead of directly comparing the ex.what() with an expected string, + // create an instance of exception to see if ex.what() matches + // the expected explanatory string returned by the exception instance. + return strcmp(expectedException.what(), ex.what()) == 0; +} + +BOOST_AUTO_TEST_CASE(vector_bool) +{ + std::vector vec1{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; + std::vector vec2{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; + + BOOST_CHECK(vec1 == std::vector(vec2.begin(), vec2.end())); + BOOST_CHECK(SerializeHash(vec1) == SerializeHash(vec2)); +} + +BOOST_AUTO_TEST_CASE(noncanonical) +{ + // Write some non-canonical CompactSize encodings, and + // make sure an exception is thrown when read back. + CDataStream ss(SER_DISK, 0); + std::vector::size_type n; + + // zero encoded with three bytes: + ss.write("\xfd\x00\x00", 3); + BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); + + // 0xfc encoded with three bytes: + ss.write("\xfd\xfc\x00", 3); + BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); + + // 0xfd encoded with three bytes is OK: + ss.write("\xfd\xfd\x00", 3); + n = ReadCompactSize(ss); + BOOST_CHECK(n == 0xfd); + + // zero encoded with five bytes: + ss.write("\xfe\x00\x00\x00\x00", 5); + BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); + + // 0xffff encoded with five bytes: + ss.write("\xfe\xff\xff\x00\x00", 5); + BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); + + // zero encoded with nine bytes: + ss.write("\xff\x00\x00\x00\x00\x00\x00\x00\x00", 9); + BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); + + // 0x01ffffff encoded with nine bytes: + ss.write("\xff\xff\xff\xff\x01\x00\x00\x00\x00", 9); + BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); +} + +BOOST_AUTO_TEST_CASE(insert_delete) +{ + // Test inserting/deleting bytes. + CDataStream ss(SER_DISK, 0); + BOOST_CHECK_EQUAL(ss.size(), 0U); + + ss.write("\x00\x01\x02\xff", 4); + BOOST_CHECK_EQUAL(ss.size(), 4U); + + char c = (char)11; + + // Inserting at beginning/end/middle: + ss.insert(ss.begin(), c); + BOOST_CHECK_EQUAL(ss.size(), 5U); + BOOST_CHECK_EQUAL(ss[0], c); + BOOST_CHECK_EQUAL(ss[1], 0); + + ss.insert(ss.end(), c); + BOOST_CHECK_EQUAL(ss.size(), 6U); + BOOST_CHECK_EQUAL(ss[4], (char)0xff); + BOOST_CHECK_EQUAL(ss[5], c); + + ss.insert(ss.begin()+2, c); + BOOST_CHECK_EQUAL(ss.size(), 7U); + BOOST_CHECK_EQUAL(ss[2], c); + + // Delete at beginning/end/middle + ss.erase(ss.begin()); + BOOST_CHECK_EQUAL(ss.size(), 6U); + BOOST_CHECK_EQUAL(ss[0], 0); + + ss.erase(ss.begin()+ss.size()-1); + BOOST_CHECK_EQUAL(ss.size(), 5U); + BOOST_CHECK_EQUAL(ss[4], (char)0xff); + + ss.erase(ss.begin()+1); + BOOST_CHECK_EQUAL(ss.size(), 4U); + BOOST_CHECK_EQUAL(ss[0], 0); + BOOST_CHECK_EQUAL(ss[1], 1); + BOOST_CHECK_EQUAL(ss[2], 2); + BOOST_CHECK_EQUAL(ss[3], (char)0xff); + + // Make sure GetAndClear does the right thing: + CSerializeData d; + ss.GetAndClear(d); + BOOST_CHECK_EQUAL(ss.size(), 0U); +} + +BOOST_AUTO_TEST_CASE(class_methods) +{ + int intval(100); + bool boolval(true); + std::string stringval("testing"); + const char charstrval[16] = "testing charstr"; + CTransaction txval; + CSerializeMethodsTestSingle methodtest1(intval, boolval, stringval, charstrval, txval); + CSerializeMethodsTestMany methodtest2(intval, boolval, stringval, charstrval, txval); + CSerializeMethodsTestSingle methodtest3; + CSerializeMethodsTestMany methodtest4; + CDataStream ss(SER_DISK, PROTOCOL_VERSION); + BOOST_CHECK(methodtest1 == methodtest2); + ss << methodtest1; + ss >> methodtest4; + ss << methodtest2; + ss >> methodtest3; + BOOST_CHECK(methodtest1 == methodtest2); + BOOST_CHECK(methodtest2 == methodtest3); + BOOST_CHECK(methodtest3 == methodtest4); + + CDataStream ss2(SER_DISK, PROTOCOL_VERSION, intval, boolval, stringval, charstrval, txval); + ss2 >> methodtest3; + BOOST_CHECK(methodtest3 == methodtest4); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/uint256.h b/src/uint256.h index fa66d2342f..c57073ab57 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -367,19 +367,14 @@ class base_uint return pn[2*n] | (uint64_t)pn[2*n+1] << 32; } - unsigned int GetSerializeSize(int nType, int nVersion) const - { - return sizeof(pn); - } - template - void Serialize(Stream& s, int nType, int nVersion) const + void Serialize(Stream& s) const { s.write((char*)pn, sizeof(pn)); } template - void Unserialize(Stream& s, int nType, int nVersion) + void Unserialize(Stream& s) { s.read((char*)pn, sizeof(pn)); } diff --git a/src/wallet.h b/src/wallet.h index 14d0ae93db..96e6f7a17e 100644 --- a/src/wallet.h +++ b/src/wallet.h @@ -64,13 +64,19 @@ class CKeyPool vchPubKey = vchPubKeyIn; } - IMPLEMENT_SERIALIZE - ( - if (!(nType & SER_GETHASH)) + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (!(s.GetType() & SER_GETHASH)) { + int nVersion = s.GetVersion(); READWRITE(nVersion); + } + READWRITE(nTime); READWRITE(vchPubKey); - ) + } }; /** A CWallet is an extension of a keystore, which also maintains a set of transactions and balances, @@ -484,16 +490,17 @@ class CWalletTx : public CMerkleTx nOrderPos = -1; } - IMPLEMENT_SERIALIZE - ( - CWalletTx* pthis = const_cast(this); - if (fRead) - pthis->Init(NULL); + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { char fSpent = false; - if (!fRead) - { - pthis->mapValue["fromaccount"] = pthis->strFromAccount; + if (ser_action.ForRead()) { + Init(NULL); + } else { + mapValue["fromaccount"] = strFromAccount; std::string str; for (auto const& f : vfSpent) @@ -502,15 +509,17 @@ class CWalletTx : public CMerkleTx if (f) fSpent = true; } - pthis->mapValue["spent"] = str; - WriteOrderPos(pthis->nOrderPos, pthis->mapValue); + mapValue["spent"] = str; + + WriteOrderPos(nOrderPos, mapValue); - if (nTimeSmart) - pthis->mapValue["timesmart"] = strprintf("%u", nTimeSmart); + if (nTimeSmart) { + mapValue["timesmart"] = strprintf("%u", nTimeSmart); + } } - nSerSize += SerReadWrite(s, *(CMerkleTx*)this, nType, nVersion,ser_action); + READWRITEAS(CMerkleTx, *this); READWRITE(vtxPrev); READWRITE(mapValue); READWRITE(vOrderForm); @@ -519,32 +528,30 @@ class CWalletTx : public CMerkleTx READWRITE(fFromMe); READWRITE(fSpent); - //Serialize two additional fields for Gridcoin (Reserved for future use): - //READWRITE(nResearchSubsidy); - //READWRITE(nInterestSubsidy); - + if (ser_action.ForRead()) { + strFromAccount = mapValue["fromaccount"]; - if (fRead) - { - pthis->strFromAccount = pthis->mapValue["fromaccount"]; - - if (mapValue.count("spent")) - for (auto const& c : pthis->mapValue["spent"]) - pthis->vfSpent.push_back(c != '0'); - else - pthis->vfSpent.assign(vout.size(), fSpent); + if (mapValue.count("spent")) { + for (auto const& c : mapValue["spent"]) { + vfSpent.push_back(c != '0'); + } + } else { + vfSpent.assign(vout.size(), fSpent); + } - ReadOrderPos(pthis->nOrderPos, pthis->mapValue); + ReadOrderPos(nOrderPos, mapValue); - pthis->nTimeSmart = mapValue.count("timesmart") ? (unsigned int)atoi64(pthis->mapValue["timesmart"]) : 0; + nTimeSmart = mapValue.count("timesmart") + ? (unsigned int)atoi64(mapValue["timesmart"]) + : 0; } - pthis->mapValue.erase("fromaccount"); - pthis->mapValue.erase("version"); - pthis->mapValue.erase("spent"); - pthis->mapValue.erase("n"); - pthis->mapValue.erase("timesmart"); - ) + mapValue.erase("fromaccount"); + mapValue.erase("version"); + mapValue.erase("spent"); + mapValue.erase("n"); + mapValue.erase("timesmart"); + } // marks certain txout's as spent // returns true if any update took place @@ -827,15 +834,21 @@ class CWalletKey nTimeExpires = nExpires; } - IMPLEMENT_SERIALIZE - ( - if (!(nType & SER_GETHASH)) + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (!(s.GetType() & SER_GETHASH)) { + int nVersion = s.GetVersion(); READWRITE(nVersion); + } + READWRITE(vchPrivKey); READWRITE(nTimeCreated); READWRITE(nTimeExpires); READWRITE(strComment); - ) + } }; @@ -861,12 +874,18 @@ class CAccount vchPubKey = CPubKey(); } - IMPLEMENT_SERIALIZE - ( - if (!(nType & SER_GETHASH)) + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (!(s.GetType() & SER_GETHASH)) { + int nVersion = s.GetVersion(); READWRITE(nVersion); + } + READWRITE(vchPubKey); - ) + } }; @@ -901,23 +920,28 @@ class CAccountingEntry nOrderPos = -1; } - IMPLEMENT_SERIALIZE - ( + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { CAccountingEntry& me = *const_cast(this); - if (!(nType & SER_GETHASH)) + int nVersion = s.GetVersion(); + + if (!(s.GetType() & SER_GETHASH)) { READWRITE(nVersion); + } + // Note: strAccount is serialized as part of the key, not here. READWRITE(nCreditDebit); READWRITE(nTime); READWRITE(strOtherAccount); - if (!fRead) - { + if (!ser_action.ForRead()) { WriteOrderPos(nOrderPos, me.mapValue); - if (!(mapValue.empty() && _ssExtra.empty())) - { - CDataStream ss(nType, nVersion); + if (!(mapValue.empty() && _ssExtra.empty())) { + CDataStream ss(s.GetType(), nVersion); ss.insert(ss.begin(), '\0'); ss << mapValue; ss.insert(ss.end(), _ssExtra.begin(), _ssExtra.end()); @@ -928,22 +952,29 @@ class CAccountingEntry READWRITE(strComment); size_t nSepPos = strComment.find("\0", 0, 1); - if (fRead) - { + + if (ser_action.ForRead()) { me.mapValue.clear(); - if (std::string::npos != nSepPos) - { - CDataStream ss(std::vector(strComment.begin() + nSepPos + 1, strComment.end()), nType, nVersion); + + if (std::string::npos != nSepPos) { + CDataStream ss( + std::vector(strComment.begin() + nSepPos + 1, strComment.end()), + s.GetType(), + nVersion); + ss >> me.mapValue; me._ssExtra = std::vector(ss.begin(), ss.end()); } + ReadOrderPos(me.nOrderPos, me.mapValue); } - if (std::string::npos != nSepPos) + + if (std::string::npos != nSepPos) { me.strComment.erase(nSepPos); + } me.mapValue.erase("n"); - ) + } private: std::vector _ssExtra; diff --git a/src/walletdb.h b/src/walletdb.h index 050233d058..2d7d2f17dd 100644 --- a/src/walletdb.h +++ b/src/walletdb.h @@ -40,12 +40,14 @@ class CKeyMetadata nCreateTime = nCreateTime_; } - IMPLEMENT_SERIALIZE - ( - READWRITE(this->nVersion); - nVersion = this->nVersion; + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(nVersion); READWRITE(nCreateTime); - ) + } void SetNull() { From 61b776bd6ae87db379e0e8e84b6d177152acf2c6 Mon Sep 17 00:00:00 2001 From: Cy Rossignol Date: Wed, 25 Sep 2019 12:58:10 -0500 Subject: [PATCH 7/7] Fix serialization for NN::Superblock::ProjectIndex hashing The ProjectStats convergence hint is not part of the superblock hash, so don't serialize the hint while hashing the superblock. --- src/neuralnet/superblock.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neuralnet/superblock.h b/src/neuralnet/superblock.h index a7de50a9de..3b05a9d3d9 100644 --- a/src/neuralnet/superblock.h +++ b/src/neuralnet/superblock.h @@ -641,7 +641,7 @@ class Superblock // Trigger serialization of ProjectStats convergence hints for // superblocks generated by a fallback-to-project convergence: // - if (m_converged_by_project) { + if (!(stream.GetType() & SER_GETHASH) && m_converged_by_project) { stream << project_pair.second.m_convergence_hint; } }