Merge pull request #1837 from jamescowens/integrated_scraper_2

scraper: Scraper global statistics cache optimization

src/neuralnet/quorum.cpp

@@ -788,11 +788,11 @@ class SuperblockValidator
         //! \param project_name      Identifies the project to add.
         //! \param project_part_data Serialized project stats of the part.
         //!
-        void AddPart(std::string project_name, CSerializeData project_part_data)
+        void AddPart(std::string project_name, CSplitBlob::CPart* project_part_ptr)
         {
-            m_convergence.ConvergedManifestPartsMap.emplace(
+            m_convergence.ConvergedManifestPartPtrsMap.emplace(
                 std::move(project_name),
-                std::move(project_part_data));
+                std::move(project_part_ptr));
         }
 
         //!
@@ -944,7 +944,7 @@ class SuperblockValidator
 
                 convergence.AddPart(
                     project_pair.first, // project name
-                    GetResolvedPartData(resolved_part.m_part_hash));
+                    GetResolvedPartPtr(resolved_part.m_part_hash));
 
                 remainder -= part_index * project.m_combiner_mask;
 
@@ -981,7 +981,7 @@ class SuperblockValidator
         //!
         //! \return Serialized binary data of the part to add to a convergence.
         //!
-        static CSerializeData GetResolvedPartData(const uint256& part_hash)
+        static CSplitBlob::CPart* GetResolvedPartPtr(const uint256& part_hash)
         {
             LOCK(CSplitBlob::cs_mapParts);
 
@@ -991,10 +991,10 @@ class SuperblockValidator
             // the most recent project part should always exist:
             if (iter == CSplitBlob::mapParts.end()) {
                 LogPrintf("ValidateSuperblock(): project part disappeared.");
-                return CSerializeData();
+                return nullptr;
             }
 
-            return iter->second.data;
+            return &(iter->second);
         }
 
         //!
@@ -1029,7 +1029,7 @@ class SuperblockValidator
                 return;
             }
 
-            convergence.AddPart("BeaconList", manifest.vParts[0]->data);
+            convergence.AddPart("BeaconList", manifest.vParts[0]);
 
             // Find the offset of the verified beacons project part. Typically
             // this exists at vParts offset 1 when a scraper verified at least
@@ -1054,7 +1054,7 @@ class SuperblockValidator
                 return;
             }
 
-            convergence.AddPart("VerifiedBeacons", manifest.vParts[part_offset]->data);
+            convergence.AddPart("VerifiedBeacons", manifest.vParts[part_offset]);
         }
     }; // ProjectCombiner
 

src/neuralnet/quorum.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <string>
+#include "scraper_net.h"
 
 class CBlockIndex;
 

src/neuralnet/superblock.cpp

@@ -557,11 +557,11 @@ Superblock Superblock::FromConvergence(
     // Add hints created from the hashes of converged manifest parts to each
     // superblock project section to assist receiving nodes with validation:
     //
-    for (const auto& part_pair : stats.Convergence.ConvergedManifestPartsMap) {
+    for (const auto& part_pair : stats.Convergence.ConvergedManifestPartPtrsMap) {
         const std::string& project_name = part_pair.first;
-        const CSerializeData& part_data = part_pair.second;
+        const CSplitBlob::CPart* part_data_ptr = part_pair.second;
 
-        projects.SetHint(project_name, part_data);
+        projects.SetHint(project_name, part_data_ptr);
     }
 
     return superblock;
@@ -954,7 +954,7 @@ void Superblock::ProjectIndex::Add(std::string name, const ProjectStats& stats)
 
 void Superblock::ProjectIndex::SetHint(
     const std::string& name,
-    const CSerializeData& part_data)
+    const CSplitBlob::CPart* part_data_ptr)
 {
     auto iter = std::lower_bound(
         m_projects.begin(),
@@ -966,7 +966,7 @@ void Superblock::ProjectIndex::SetHint(
         return;
     }
 
-    const uint256 part_hash = Hash(part_data.begin(), part_data.end());
+    const uint256 part_hash = Hash(part_data_ptr->data.begin(), part_data_ptr->data.end());
     iter->second.m_convergence_hint = part_hash.GetUint64() >> 32;
 
     m_converged_by_project = true;

src/neuralnet/superblock.h

@@ -1087,7 +1087,7 @@ class Superblock
         //!
         //! \param part_data The convergence part to create the hint from.
         //!
-        void SetHint(const std::string& name, const CSerializeData& part_data);
+        void SetHint(const std::string& name, const CSplitBlob::CPart *part_data_ptr);
 
         //!
         //! \brief Serialize the object to the provided stream.
@@ -1533,6 +1533,22 @@ struct hash<NN::QuorumHash>
 // This is part of the scraper but is put here, because it needs the complete NN:Superblock class.
 struct ConvergedScraperStats
 {
+    ConvergedScraperStats() : Convergence(), NewFormatSuperblock()
+    {
+        bClean = false;
+
+        nTime = 0;
+        mScraperConvergedStats = {};
+        PastConvergences = {};
+    }
+
+    ConvergedScraperStats(const int64_t nTime_in, const ConvergedManifest& Convergence) : Convergence(Convergence)
+    {
+        bClean = false;
+
+        nTime = nTime_in;
+    }
+
     // Flag to indicate cache is clean or dirty (i.e. state change of underlying statistics has occurred.
     // This flag is marked true in ScraperGetSuperblockContract() and false on receipt or deletion of
     // statistics objects.
@@ -1558,6 +1574,7 @@ struct ConvergedScraperStats
         {
             // This is specifically this form of insert to insure that if there is a hint "collision" the referenced
             // SB Hash and Convergence stored will be the LATER one.
+
             PastConvergences[nReducedContentHash] = std::make_pair(NewFormatSuperblock.GetHash(), Convergence);
         }
     }

src/qt/transactionrecord.cpp

@@ -65,7 +65,8 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const CWallet *
     std::map<std::string, std::string> mapValue = wtx.mapValue;
 
     bool fContractPresent = false;
-    NN::ContractType ContractType;
+    // Initialize to unknown to prevent a possible uninitialized warning.
+    NN::ContractType ContractType = NN::ContractType::UNKNOWN;
 
     if (!wtx.GetContracts().empty())
     {

src/rpcserver.cpp

@@ -399,6 +399,7 @@ static const CRPCCommand vRPCCommands[] =
     { "archivelog",              &archivelog,              cat_developer     },
     { "testnewsb",               &testnewsb,               cat_developer     },
     { "convergencereport",       &convergencereport,       cat_developer     },
+    { "scraperreport",           &scraperreport,           cat_developer     },
 
   // Network commands
     { "addnode",                 &addnode,                 cat_network       },

src/rpcserver.h

@@ -213,6 +213,7 @@ extern UniValue deletecscrapermanifest(const UniValue& params, bool fHelp);
 extern UniValue archivelog(const UniValue& params, bool fHelp);
 extern UniValue testnewsb(const UniValue& params, bool fHelp);
 extern UniValue convergencereport(const UniValue& params, bool fHelp);
+extern UniValue scraperreport(const UniValue& params, bool fHelp);
 
 // Network
 extern UniValue addnode(const UniValue& params, bool fHelp);

src/scraper/fwd.h

@@ -9,6 +9,8 @@
 #include "util.h"
 #include "streams.h"
 
+#include "scraper_net.h"
+
 /*********************
 * Scraper ENUMS      *
 *********************/
@@ -55,13 +57,80 @@ typedef std::multimap<int64_t, std::pair<uint256, uint256>, std::greater <int64_
 // See the ScraperID typedef above.
 typedef std::map<ScraperID, mCSManifest> mmCSManifestsBinnedByScraper;
 
-// -------------- Project ---- Converged Part
-typedef std::map<std::string, CSerializeData> mConvergedManifestParts;
-// Note that this IS a copy not a pointer. Since manifests and parts can be deleted because of aging rules,
-// it is dangerous to save memory and point to the actual part objects themselves.
+// Note the CParts pointed to by this map are safe to access, because the pointers are guaranteed valid
+// as long as the holding CScraperManifests (both in the CScaperManifest global map, and this cache)
+// still exist. So the safety of these pointers is coincident with the lifespan of CScraperManifests
+// that have reference to them. If you have questions about this, you should review the CSplitBlob abstract
+// class, which is the base class of the CScraperManifest class, and provides the mechanisms for part
+// control. Note that two LOCKS are used to protect the integrity of the underlying global maps,
+// CScraperManifest::cs_mapManifest and CSplitBlob::cs_mapParts.
+// -------------- Project -- Converged Part Pointer
+typedef std::map<std::string, CSplitBlob::CPart*> mConvergedManifestPart_ptrs;
 
 struct ConvergedManifest
 {
+    // Empty converged manifest constructor
+    ConvergedManifest()
+    {
+        nContentHash = {};
+        ConsensusBlock = {};
+        timestamp = 0;
+        bByParts = false;
+
+        CScraperConvergedManifest_ptr = nullptr;
+
+        ConvergedManifestPartPtrsMap = {};
+
+        mIncludedScraperManifests = {};
+
+        nUnderlyingManifestContentHash = {};
+
+        vIncludedScrapers = {};
+        vExcludedScrapers = {};
+        vScrapersNotPublishing = {};
+
+        mIncludedScrapersbyProject = {};
+        mIncludedProjectsbyScraper = {};
+
+        mScraperConvergenceCountbyProject = {};
+
+        vExcludedProjects = {};
+    }
+
+    // For constructing a dummy converged manifest from a single manifest
+    ConvergedManifest(CScraperManifest& in)
+    {
+        ConsensusBlock = in.ConsensusBlock;
+        timestamp = GetAdjustedTime();
+        bByParts = false;
+
+        CScraperConvergedManifest_ptr = std::make_shared<CScraperManifest>(in);
+
+        PopulateConvergedManifestPartPtrsMap();
+
+        ComputeConvergedContentHash();
+
+        nUnderlyingManifestContentHash = in.nContentHash;
+    }
+
+    // Call operator to update an already initialized ConvergedManifest with a passed in CScraperManifest
+    bool operator()(const CScraperManifest& in)
+    {
+        ConsensusBlock = in.ConsensusBlock;
+        timestamp = GetAdjustedTime();
+        bByParts = false;
+
+        CScraperConvergedManifest_ptr = std::make_shared<CScraperManifest>(in);
+
+        bool bConvergedContentHashMatches = PopulateConvergedManifestPartPtrsMap();
+
+        ComputeConvergedContentHash();
+
+        nUnderlyingManifestContentHash = in.nContentHash;
+
+        return bConvergedContentHashMatches;
+    }
+
     // IMPORTANT... nContentHash is NOT the hash of part hashes in the order of vParts unlike CScraper::manifest.
     // It is the hash of the data in the ConvergedManifestPartsMap in the order of the key. It represents
     // the composite convergence by taking parts piecewise in the case of the fallback to bByParts (project) level.
@@ -70,7 +139,9 @@ struct ConvergedManifest
     int64_t timestamp;
     bool bByParts;
 
-    mConvergedManifestParts ConvergedManifestPartsMap;
+    std::shared_ptr<CScraperManifest> CScraperConvergedManifest_ptr;
+
+    mConvergedManifestPart_ptrs ConvergedManifestPartPtrsMap;
 
     // Used when convergence is at the manifest level (normal)
     std::map<ScraperID, uint256> mIncludedScraperManifests;
@@ -97,6 +168,62 @@ struct ConvergedManifest
 
     // --------- project
     std::vector<std::string> vExcludedProjects;
+
+    bool PopulateConvergedManifestPartPtrsMap()
+    {
+        if (CScraperConvergedManifest_ptr == nullptr) return false;
+
+        int iPartNum = 0;
+        CDataStream ss(SER_NETWORK,1);
+        WriteCompactSize(ss, CScraperConvergedManifest_ptr->vParts.size());
+        uint256 nContentHashCheck;
+
+        for (const auto& iter : CScraperConvergedManifest_ptr->vParts)
+        {
+            std::string sProject;
+
+            if (iPartNum == 0)
+                sProject = "BeaconList";
+            else
+                sProject = CScraperConvergedManifest_ptr->projects[iPartNum-1].project;
+
+            // Copy the pointer to the CPart into the map. This is ok, because the parts will be held
+            // until the CScraperManifest in this object is destroyed and all of the manifest refs to the part
+            // are gone.
+            ConvergedManifestPartPtrsMap.insert(std::make_pair(sProject, iter));
+
+            // Serialize the hash to doublecheck the content hash.
+            ss << iter->hash;
+
+            iPartNum++;
+        }
+
+        ss << CScraperConvergedManifest_ptr->ConsensusBlock;
+
+        nContentHashCheck = Hash(ss.begin(), ss.end());
+
+        if (nContentHashCheck != CScraperConvergedManifest_ptr->nContentHash)
+        {
+            LogPrintf("ERROR: PopulateConvergedManifestPartPtrsMap(): Selected Manifest content hash check failed! "
+                      "nContentHashCheck = %s and nContentHash = %s.",
+                      nContentHashCheck.GetHex(), CScraperConvergedManifest_ptr->nContentHash.GetHex());
+            return false;
+        }
+
+        return true;
+    }
+
+    void ComputeConvergedContentHash()
+    {
+        CDataStream ss(SER_NETWORK,1);
+
+        for (const auto& iter : ConvergedManifestPartPtrsMap)
+        {
+            ss << iter.second->data;
+        }
+
+        nContentHash = Hash(ss.begin(), ss.end());
+    }
 };