From 330f9e7227b3b631a9bcfede11a4ee846ad0db70 Mon Sep 17 00:00:00 2001 From: jamescowens Date: Tue, 18 Aug 2020 00:32:06 -0400 Subject: [PATCH] Implement scraper global statistics cache optimization This commit changes the ConvergedManifestPartsMap to a map of pointers, ConvergedManifestPartPtrsMap. All of the pointer ownership problems may not be solved yet by this commit, but most of them are. The CScraperManifest held by the ConvergedManifest is via a shared_ptr. In turn the CParts in the CScraperManifest::CSplitBlob vParts are also mapped into the ConvergedManifestPartPtrsMap, which is indexed by project. The part pointers in the ConvergedManifestPartPtrsMap are valid for the same lifetime as the CScraperManifest held by shared pointer in the ConvergedManifest, so I think this is safe. Thjs minimizes the changes to the rest of the scraper to achieve this optimization. I am less certain of the pointer safety in the changes I made in the quorum and superblock classes. --- src/neuralnet/quorum.cpp | 18 +- src/neuralnet/quorum.h | 1 + src/neuralnet/superblock.cpp | 4 +- src/neuralnet/superblock.h | 25 +++ src/scraper/fwd.h | 234 ++++++++++++++++++++- src/scraper/scraper.cpp | 265 +++++++++++++++--------- src/test/neuralnet/superblock_tests.cpp | 21 +- 7 files changed, 451 insertions(+), 117 deletions(-) diff --git a/src/neuralnet/quorum.cpp b/src/neuralnet/quorum.cpp index 8f1052db3e..40504b1cfb 100644 --- a/src/neuralnet/quorum.cpp +++ b/src/neuralnet/quorum.cpp @@ -788,11 +788,11 @@ class SuperblockValidator //! \param project_name Identifies the project to add. //! \param project_part_data Serialized project stats of the part. //! - void AddPart(std::string project_name, CSerializeData project_part_data) + void AddPart(std::string project_name, CSplitBlob::CPart* project_part_ptr) { - m_convergence.ConvergedManifestPartsMap.emplace( + m_convergence.ConvergedManifestPartPtrsMap.emplace( std::move(project_name), - std::move(project_part_data)); + std::move(project_part_ptr)); } //! @@ -944,7 +944,7 @@ class SuperblockValidator convergence.AddPart( project_pair.first, // project name - GetResolvedPartData(resolved_part.m_part_hash)); + GetResolvedPartPtr(resolved_part.m_part_hash)); remainder -= part_index * project.m_combiner_mask; @@ -981,7 +981,7 @@ class SuperblockValidator //! //! \return Serialized binary data of the part to add to a convergence. //! - static CSerializeData GetResolvedPartData(const uint256& part_hash) + static CSplitBlob::CPart* GetResolvedPartPtr(const uint256& part_hash) { LOCK(CSplitBlob::cs_mapParts); @@ -991,10 +991,10 @@ class SuperblockValidator // the most recent project part should always exist: if (iter == CSplitBlob::mapParts.end()) { LogPrintf("ValidateSuperblock(): project part disappeared."); - return CSerializeData(); + return nullptr; } - return iter->second.data; + return &(iter->second); } //! @@ -1029,7 +1029,7 @@ class SuperblockValidator return; } - convergence.AddPart("BeaconList", manifest.vParts[0]->data); + convergence.AddPart("BeaconList", manifest.vParts[0]); // Find the offset of the verified beacons project part. Typically // this exists at vParts offset 1 when a scraper verified at least @@ -1054,7 +1054,7 @@ class SuperblockValidator return; } - convergence.AddPart("VerifiedBeacons", manifest.vParts[part_offset]->data); + convergence.AddPart("VerifiedBeacons", manifest.vParts[part_offset]); } }; // ProjectCombiner diff --git a/src/neuralnet/quorum.h b/src/neuralnet/quorum.h index d177f4f17e..f65919fe33 100644 --- a/src/neuralnet/quorum.h +++ b/src/neuralnet/quorum.h @@ -1,6 +1,7 @@ #pragma once #include +#include "scraper_net.h" class CBlockIndex; diff --git a/src/neuralnet/superblock.cpp b/src/neuralnet/superblock.cpp index d27e19dc72..4f2270e298 100644 --- a/src/neuralnet/superblock.cpp +++ b/src/neuralnet/superblock.cpp @@ -557,9 +557,9 @@ Superblock Superblock::FromConvergence( // Add hints created from the hashes of converged manifest parts to each // superblock project section to assist receiving nodes with validation: // - for (const auto& part_pair : stats.Convergence.ConvergedManifestPartsMap) { + for (const auto& part_pair : stats.Convergence.ConvergedManifestPartPtrsMap) { const std::string& project_name = part_pair.first; - const CSerializeData& part_data = part_pair.second; + const CSerializeData& part_data = part_pair.second->data; projects.SetHint(project_name, part_data); } diff --git a/src/neuralnet/superblock.h b/src/neuralnet/superblock.h index fac5e4ff6f..15121020ec 100644 --- a/src/neuralnet/superblock.h +++ b/src/neuralnet/superblock.h @@ -1533,6 +1533,20 @@ struct hash // This is part of the scraper but is put here, because it needs the complete NN:Superblock class. struct ConvergedScraperStats { + ConvergedScraperStats() : Convergence(), NewFormatSuperblock() + { + bClean = false; + + nTime = 0; + mScraperConvergedStats = {}; + PastConvergences = {}; + } + + ConvergedScraperStats(const int64_t nTime_in, const ConvergedManifest& Convergence) : Convergence(Convergence) + { + nTime = nTime_in; + } + // Flag to indicate cache is clean or dirty (i.e. state change of underlying statistics has occurred. // This flag is marked true in ScraperGetSuperblockContract() and false on receipt or deletion of // statistics objects. @@ -1558,7 +1572,18 @@ struct ConvergedScraperStats { // This is specifically this form of insert to insure that if there is a hint "collision" the referenced // SB Hash and Convergence stored will be the LATER one. + PastConvergences[nReducedContentHash] = std::make_pair(NewFormatSuperblock.GetHash(), Convergence); + + /* + if (PastConvergences.find(nReducedContentHash) != PastConvergences.end()) + { + PastConvergences.erase(nReducedContentHash); + } + + PastConvergences.emplace(std::make_pair(nReducedContentHash, Convergence)); + */ + } } diff --git a/src/scraper/fwd.h b/src/scraper/fwd.h index 9675ee0292..b60052b5fd 100644 --- a/src/scraper/fwd.h +++ b/src/scraper/fwd.h @@ -9,6 +9,8 @@ #include "util.h" #include "streams.h" +#include "scraper_net.h" + /********************* * Scraper ENUMS * *********************/ @@ -60,8 +62,178 @@ typedef std::map mConvergedManifestParts; // Note that this IS a copy not a pointer. Since manifests and parts can be deleted because of aging rules, // it is dangerous to save memory and point to the actual part objects themselves. +typedef std::map mConvergedManifestPart_ptrs; + struct ConvergedManifest { + ConvergedManifest() + { + nContentHash = {}; + ConsensusBlock = {}; + timestamp = 0; + bByParts = false; + + CScraperConvergedManifest_ptr = nullptr; + + //ConvergedManifestPartsMap = {}; + + ConvergedManifestPartPtrsMap = {}; + + mIncludedScraperManifests = {}; + + nUnderlyingManifestContentHash = {}; + + vIncludedScrapers = {}; + vExcludedScrapers = {}; + vScrapersNotPublishing = {}; + + mIncludedScrapersbyProject = {}; + mIncludedProjectsbyScraper = {}; + + mScraperConvergenceCountbyProject = {}; + + vExcludedProjects = {}; + } + + ConvergedManifest(const ConvergedManifest& in) + { + // We can use the content hash from the specified converged manifest. We do not need to recompute it. + nContentHash = in.nContentHash; + + ConsensusBlock = in.ConsensusBlock; + timestamp = in.timestamp; + bByParts = in.bByParts; + + CScraperConvergedManifest_ptr = in.CScraperConvergedManifest_ptr; + + PopulateConvergedManifestPartPtrsMap(); + + // CScraperConvergedManifest_ptr = std::move(in.CScraperConvergedManifest_ptr); + + // We are going to make a copy of the manifest here and create a new pointer to it. We + // need to do this because of the const qualifier (for use in const iterators). + + //CScraperManifest CScraperConvergedManifest = *(in.CScraperConvergedManifest_ptr); + + //std::shared_ptr ptr(&CScraperConvergedManifest); + + //CScraperConvergedManifest_ptr = std::move(ptr); + + //ConvergedManifestPartsMap = in.ConvergedManifestPartsMap; + + mIncludedScraperManifests = in.mIncludedScraperManifests; + + nUnderlyingManifestContentHash = in.nUnderlyingManifestContentHash; + + vIncludedScrapers = in.vIncludedScrapers; + vExcludedScrapers = in.vExcludedScrapers; + vScrapersNotPublishing = in.vScrapersNotPublishing; + + mIncludedScrapersbyProject = in.mIncludedScrapersbyProject; + mIncludedProjectsbyScraper = in.mIncludedProjectsbyScraper; + + mScraperConvergenceCountbyProject = in.mScraperConvergenceCountbyProject; + + vExcludedProjects = in.vExcludedProjects; + } + + // For constructing a dummy converged manifest from a single manifest + ConvergedManifest(CScraperManifest& in) + { + ConsensusBlock = in.ConsensusBlock; + timestamp = GetAdjustedTime(); + bByParts = false; + + CScraperConvergedManifest_ptr = std::make_shared(in); + + PopulateConvergedManifestPartPtrsMap(); + + ComputeConvergedContentHash(); + + nUnderlyingManifestContentHash = in.nContentHash; + } + + + void operator()(ConvergedManifest& in) + { + // We can use the content hash from the specified converged manifest. We do not need to recompute it. + nContentHash = in.nContentHash; + + ConsensusBlock = in.ConsensusBlock; + timestamp = in.timestamp; + bByParts = in.bByParts; + + CScraperConvergedManifest_ptr = in.CScraperConvergedManifest_ptr; + + //CScraperConvergedManifest_ptr = std::make_shared(in); + + PopulateConvergedManifestPartPtrsMap(); + + //ConvergedManifestPartsMap = in.ConvergedManifestPartsMap; + + mIncludedScraperManifests = in.mIncludedScraperManifests; + + nUnderlyingManifestContentHash = in.nUnderlyingManifestContentHash; + + vIncludedScrapers = in.vIncludedScrapers; + vExcludedScrapers = in.vExcludedScrapers; + vScrapersNotPublishing = in.vScrapersNotPublishing; + + mIncludedScrapersbyProject = in.mIncludedScrapersbyProject; + mIncludedProjectsbyScraper = in.mIncludedProjectsbyScraper; + + mScraperConvergenceCountbyProject = in.mScraperConvergenceCountbyProject; + + vExcludedProjects = in.vExcludedProjects; + } + + bool operator()(const CScraperManifest& in) + { + ConsensusBlock = in.ConsensusBlock; + timestamp = GetAdjustedTime(); + bByParts = false; + + CScraperConvergedManifest_ptr = std::make_shared(in); + + bool bConvergedContentHashMatches = PopulateConvergedManifestPartPtrsMap(); + + ComputeConvergedContentHash(); + + nUnderlyingManifestContentHash = in.nContentHash; + + return bConvergedContentHashMatches; + } + + void Reset() + { + nContentHash = {}; + ConsensusBlock = {}; + timestamp = 0; + bByParts = false; + + CScraperConvergedManifest_ptr = nullptr; + + //ConvergedManifestPartsMap = {}; + + ConvergedManifestPartPtrsMap = {}; + + mIncludedScraperManifests = {}; + + nUnderlyingManifestContentHash = {}; + + vIncludedScrapers = {}; + vExcludedScrapers = {}; + vScrapersNotPublishing = {}; + + mIncludedScrapersbyProject = {}; + mIncludedProjectsbyScraper = {}; + + mScraperConvergenceCountbyProject = {}; + + vExcludedProjects = {}; + } + + // IMPORTANT... nContentHash is NOT the hash of part hashes in the order of vParts unlike CScraper::manifest. // It is the hash of the data in the ConvergedManifestPartsMap in the order of the key. It represents // the composite convergence by taking parts piecewise in the case of the fallback to bByParts (project) level. @@ -70,7 +242,11 @@ struct ConvergedManifest int64_t timestamp; bool bByParts; - mConvergedManifestParts ConvergedManifestPartsMap; + std::shared_ptr CScraperConvergedManifest_ptr; + + // mConvergedManifestParts ConvergedManifestPartsMap; + + mConvergedManifestPart_ptrs ConvergedManifestPartPtrsMap; // Used when convergence is at the manifest level (normal) std::map mIncludedScraperManifests; @@ -97,6 +273,62 @@ struct ConvergedManifest // --------- project std::vector vExcludedProjects; + + bool PopulateConvergedManifestPartPtrsMap() + { + if (CScraperConvergedManifest_ptr == nullptr) return false; + + int iPartNum = 0; + CDataStream ss(SER_NETWORK,1); + WriteCompactSize(ss, CScraperConvergedManifest_ptr->vParts.size()); + uint256 nContentHashCheck; + + for (const auto& iter : CScraperConvergedManifest_ptr->vParts) + { + std::string sProject; + + if (iPartNum == 0) + sProject = "BeaconList"; + else + sProject = CScraperConvergedManifest_ptr->projects[iPartNum-1].project; + + // Copy the pointer to the CPart into the map. This is ok, because the parts will be held + // until the CScraperManifest in this object is destroyed and all of the manifest refs to the part + // are gone. + ConvergedManifestPartPtrsMap.insert(std::make_pair(sProject, iter)); + + // Serialize the hash to doublecheck the content hash. + ss << iter->hash; + + iPartNum++; + } + + ss << CScraperConvergedManifest_ptr->ConsensusBlock; + + nContentHashCheck = Hash(ss.begin(), ss.end()); + + if (nContentHashCheck != CScraperConvergedManifest_ptr->nContentHash) + { + LogPrintf("ERROR: PopulateConvergedManifestPartPtrsMap(): Selected Manifest content hash check failed! " + "nContentHashCheck = %s and nContentHash = %s.", + nContentHashCheck.GetHex(), CScraperConvergedManifest_ptr->nContentHash.GetHex()); + return false; + } + + return true; + } + + void ComputeConvergedContentHash() + { + CDataStream ss(SER_NETWORK,1); + + for (const auto& iter : ConvergedManifestPartPtrsMap) + { + ss << iter.second->data; + } + + nContentHash = Hash(ss.begin(), ss.end()); + } }; diff --git a/src/scraper/scraper.cpp b/src/scraper/scraper.cpp index daa74f1e54..a64c059d87 100755 --- a/src/scraper/scraper.cpp +++ b/src/scraper/scraper.cpp @@ -3433,10 +3433,10 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByConvergedManifest(const Converge int exclude_parts_from_count = 1; - const auto& iter = StructConvergedManifest.ConvergedManifestPartsMap.find("VerifiedBeacons"); - if (iter != StructConvergedManifest.ConvergedManifestPartsMap.end()) + const auto& iter = StructConvergedManifest.ConvergedManifestPartPtrsMap.find("VerifiedBeacons"); + if (iter != StructConvergedManifest.ConvergedManifestPartPtrsMap.end()) { - CDataStream part(iter->second, SER_NETWORK, 1); + CDataStream part(iter->second->data, SER_NETWORK, 1); try { @@ -3452,14 +3452,14 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByConvergedManifest(const Converge stats_and_verified_beacons.mVerifiedMap = VerifiedBeaconMap; - unsigned int nActiveProjects = StructConvergedManifest.ConvergedManifestPartsMap.size() - exclude_parts_from_count; + unsigned int nActiveProjects = StructConvergedManifest.ConvergedManifestPartPtrsMap.size() - exclude_parts_from_count; _log(logattribute::INFO, "GetScraperStatsByConvergedManifest", "Number of active projects in converged manifest = " + std::to_string(nActiveProjects)); double dMagnitudePerProject = NEURALNETWORKMULTIPLIER / nActiveProjects; ScraperStats mScraperStats; - for (auto entry = StructConvergedManifest.ConvergedManifestPartsMap.begin(); entry != StructConvergedManifest.ConvergedManifestPartsMap.end(); ++entry) + for (auto entry = StructConvergedManifest.ConvergedManifestPartPtrsMap.begin(); entry != StructConvergedManifest.ConvergedManifestPartPtrsMap.end(); ++entry) { std::string project = entry->first; ScraperStats mProjectScraperStats; @@ -3469,7 +3469,7 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByConvergedManifest(const Converge { _log(logattribute::INFO, "GetScraperStatsByConvergedManifest", "Processing stats for project: " + project); - LoadProjectObjectToStatsByCPID(project, entry->second, dMagnitudePerProject, mProjectScraperStats); + LoadProjectObjectToStatsByCPID(project, entry->second->data, dMagnitudePerProject, mProjectScraperStats); // Insert into overall map. for (auto const& entry2 : mProjectScraperStats) @@ -3493,18 +3493,16 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsFromSingleManifest(CScraperManifes { _log(logattribute::INFO, "GetScraperStatsFromSingleManifest", "Beginning stats processing."); - // Create a dummy converged manifest - ConvergedManifest StructDummyConvergedManifest; + // Create a dummy converged manifest and fill out the dummy ConvergedManifest structure from the provided + // manifest. + ConvergedManifest StructDummyConvergedManifest(manifest); ScraperStatsAndVerifiedBeacons stats_and_verified_beacons {}; - // Fill out the dummy ConvergedManifest structure. Note this assumes one-to-one part to project statistics BLOB. Needs to - // be fixed for more than one part per BLOB. This is easy in this case, because it is all from/referring to one manifest. - - StructDummyConvergedManifest.ConsensusBlock = manifest.ConsensusBlock; - StructDummyConvergedManifest.timestamp = GetAdjustedTime(); - StructDummyConvergedManifest.bByParts = false; - + /* + // Fill out the dummy ConvergedManifest structure. Note this assumes one-to-one part to project statistics BLOB. + // Needs to be fixed for more than one part per BLOB. This is easy in this case, because it is all from/referring to + // one manifest. int iPartNum = 0; CDataStream ss(SER_NETWORK,1); WriteCompactSize(ss, manifest.vParts.size()); @@ -3519,9 +3517,6 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsFromSingleManifest(CScraperManifes else sProject = manifest.projects[iPartNum-1].project; - // Copy the parts data into the map keyed by project. - StructDummyConvergedManifest.ConvergedManifestPartsMap.insert(std::make_pair(sProject, iter->data)); - // Serialize the hash to doublecheck the content hash. ss << iter->hash; @@ -3548,6 +3543,7 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsFromSingleManifest(CScraperManifes StructDummyConvergedManifest.nContentHash = Hash(ss2.begin(), ss2.end()); } + */ // Enumerate the count of active projects from the dummy converged manifest. One of the parts // is the beacon list, is not a project, which is why that should not be included in the count. @@ -3556,10 +3552,10 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsFromSingleManifest(CScraperManifes int exclude_parts_from_count = 1; - const auto& iter = StructDummyConvergedManifest.ConvergedManifestPartsMap.find("VerifiedBeacons"); - if (iter != StructDummyConvergedManifest.ConvergedManifestPartsMap.end()) + const auto& iter = StructDummyConvergedManifest.ConvergedManifestPartPtrsMap.find("VerifiedBeacons"); + if (iter != StructDummyConvergedManifest.ConvergedManifestPartPtrsMap.end()) { - CDataStream part(iter->second, SER_NETWORK, 1); + CDataStream part(iter->second->data, SER_NETWORK, 1); try { @@ -3575,12 +3571,12 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsFromSingleManifest(CScraperManifes stats_and_verified_beacons.mVerifiedMap = VerifiedBeaconMap; - unsigned int nActiveProjects = StructDummyConvergedManifest.ConvergedManifestPartsMap.size() - exclude_parts_from_count; + unsigned int nActiveProjects = StructDummyConvergedManifest.ConvergedManifestPartPtrsMap.size() - exclude_parts_from_count; _log(logattribute::INFO, "GetScraperStatsFromSingleManifest", "Number of active projects in converged manifest = " + std::to_string(nActiveProjects)); double dMagnitudePerProject = NEURALNETWORKMULTIPLIER / nActiveProjects; - for (auto entry = StructDummyConvergedManifest.ConvergedManifestPartsMap.begin(); entry != StructDummyConvergedManifest.ConvergedManifestPartsMap.end(); ++entry) + for (auto entry = StructDummyConvergedManifest.ConvergedManifestPartPtrsMap.begin(); entry != StructDummyConvergedManifest.ConvergedManifestPartPtrsMap.end(); ++entry) { std::string project = entry->first; ScraperStats mProjectScraperStats; @@ -3590,7 +3586,7 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsFromSingleManifest(CScraperManifes { _log(logattribute::INFO, "GetScraperStatsFromSingleManifest", "Processing stats for project: " + project); - LoadProjectObjectToStatsByCPID(project, entry->second, dMagnitudePerProject, mProjectScraperStats); + LoadProjectObjectToStatsByCPID(project, entry->second->data, dMagnitudePerProject, mProjectScraperStats); // Insert into overall map. stats_and_verified_beacons.mScraperStats.insert(mProjectScraperStats.begin(), mProjectScraperStats.end()); @@ -4384,7 +4380,9 @@ bool ScraperConstructConvergedManifest(ConvergedManifest& StructConvergedManifes // Fill out the ConvergedManifest structure. Note this assumes one-to-one part to project statistics BLOB. Needs to // be fixed for more than one part per BLOB. This is easy in this case, because it is all from/referring to one manifest. + bool bConvergedContentHashMatches = StructConvergedManifest(manifest); + /* StructConvergedManifest.ConsensusBlock = manifest.ConsensusBlock; StructConvergedManifest.timestamp = GetAdjustedTime(); StructConvergedManifest.bByParts = false; @@ -4416,16 +4414,18 @@ bool ScraperConstructConvergedManifest(ConvergedManifest& StructConvergedManifes nContentHashCheck = Hash(ss.begin(), ss.end()); - if (nContentHashCheck != convergence->first) + */ + + if (!bConvergedContentHashMatches) { bConvergenceSuccessful = false; - _log(logattribute::ERR, "ScraperConstructConvergedManifest", "Selected Converged Manifest content hash check failed! nContentHashCheck = " - + nContentHashCheck.GetHex() + " and nContentHash = " + StructConvergedManifest.nContentHash.GetHex()); + _log(logattribute::ERR, "ScraperConstructConvergedManifest", "Selected Converged Manifest content hash check failed!"); // Reinitialize StructConvergedManifest - StructConvergedManifest = {}; + StructConvergedManifest.Reset(); } else // Content matches so we have a confirmed convergence. { + /* // Copy the MANIFEST content hash into the ConvergedManifest. StructConvergedManifest.nUnderlyingManifestContentHash = convergence->first; @@ -4436,11 +4436,12 @@ bool ScraperConstructConvergedManifest(ConvergedManifest& StructConvergedManifes ss2 << iter.second; StructConvergedManifest.nContentHash = Hash(ss2.begin(), ss2.end()); + */ // Determine if there is an excluded project. If so, set convergence back to false and drop back to project level to try and recover project by project. for (const auto& iProjects : projectWhitelist) { - if (StructConvergedManifest.ConvergedManifestPartsMap.find(iProjects.m_name) == StructConvergedManifest.ConvergedManifestPartsMap.end()) + if (StructConvergedManifest.ConvergedManifestPartPtrsMap.find(iProjects.m_name) == StructConvergedManifest.ConvergedManifestPartPtrsMap.end()) { _log(logattribute::WARNING, "ScraperConstructConvergedManifest", "Project " + iProjects.m_name @@ -4453,7 +4454,7 @@ bool ScraperConstructConvergedManifest(ConvergedManifest& StructConvergedManifes break; } - if (StructConvergedManifest.ConvergedManifestPartsMap.find("BeaconList") == StructConvergedManifest.ConvergedManifestPartsMap.end()) + if (StructConvergedManifest.ConvergedManifestPartPtrsMap.find("BeaconList") == StructConvergedManifest.ConvergedManifestPartPtrsMap.end()) { _log(logattribute::WARNING, "ScraperConstructConvergedManifest", "BeaconList was not found in the converged manifests from the scrapers. \n" "Falling back to attempt convergence by project."); @@ -4474,7 +4475,7 @@ bool ScraperConstructConvergedManifest(ConvergedManifest& StructConvergedManifes _log(logattribute::INFO, "ScraperConstructConvergedManifest", "No convergence on manifests by content at the manifest level."); // Reinitialize StructConvergedManifest - StructConvergedManifest = {}; + StructConvergedManifest.Reset(); // Try to form a convergence by project objects (parts)... bConvergenceSuccessful = ScraperConstructConvergedManifestByProject(projectWhitelist, mMapCSManifestsBinnedByScraper, StructConvergedManifest); @@ -4482,7 +4483,7 @@ bool ScraperConstructConvergedManifest(ConvergedManifest& StructConvergedManifes // If we have reached here. All attempts at convergence have failed. Reinitialize StructConvergedManifest to eliminate stale or // partially filled-in data. if (!bConvergenceSuccessful) - StructConvergedManifest = {}; + StructConvergedManifest.Reset(); } // Signal UI of the status of convergence attempt. @@ -4508,6 +4509,8 @@ bool ScraperConstructConvergedManifestByProject(const NN::WhitelistSnapshot& pro int64_t nConvergedConsensusTime = 0; uint256 nManifestHashForConvergedBeaconList; + StructConvergedManifest.CScraperConvergedManifest_ptr = std::unique_ptr(new CScraperManifest); + // We are going to do this for each project in the whitelist. unsigned int iCountSuccessfulConvergedProjects = 0; unsigned int nScraperCount = mMapCSManifestsBinnedByScraper.size(); @@ -4644,9 +4647,8 @@ bool ScraperConstructConvergedManifestByProject(const NN::WhitelistSnapshot& pro StructConvergedManifest.mIncludedProjectsbyScraper.insert(std::make_pair(iter2->second.first, iter2->second.second)); } - // Put Project Object (Part) in StructConvergedManifest keyed by project. - StructConvergedManifest.ConvergedManifestPartsMap.insert(std::make_pair(iWhitelistProject.m_name, iPart->second.data)); + StructConvergedManifest.ConvergedManifestPartPtrsMap.insert(std::make_pair(iWhitelistProject.m_name, &(iPart->second))); // If the indirectly referenced manifest has a consensus time that is greater than already recorded, replace with that time, and also // change the consensus block to the referred to consensus block. (Note that this is scoped at even above the individual project level, so @@ -4665,7 +4667,7 @@ bool ScraperConstructConvergedManifestByProject(const NN::WhitelistSnapshot& pro break; } } - } + } // projectWhitelist for loop // If we meet the rule of CONVERGENCE_BY_PROJECT_RATIO, then proceed to fill out the rest of the map. if ((double)iCountSuccessfulConvergedProjects / (double)projectWhitelist.size() >= CONVERGENCE_BY_PROJECT_RATIO) @@ -4675,7 +4677,7 @@ bool ScraperConstructConvergedManifestByProject(const NN::WhitelistSnapshot& pro // Lets use the BeaconList from the manifest referred to by nManifestHashForConvergedBeaconList. Technically there is no exact answer to // the BeaconList that should be used in the convergence when putting it together at the individual part level, because each project part - // could have used a different BeaconList (subject to the consensus ladder. It makes sense to use the "newest" one that is associated + // could have used a different BeaconList (subject to the consensus ladder). It makes sense to use the "newest" one that is associated // with a manifest that has the newest part associated with a successful part (project) level convergence. LOCK(CScraperManifest::cs_mapManifest); @@ -4696,7 +4698,7 @@ bool ScraperConstructConvergedManifestByProject(const NN::WhitelistSnapshot& pro else { // The vParts[0] is always the BeaconList. - StructConvergedManifest.ConvergedManifestPartsMap.insert(std::make_pair("BeaconList", manifest.vParts[0]->data)); + StructConvergedManifest.ConvergedManifestPartPtrsMap.insert(std::make_pair("BeaconList", manifest.vParts[0])); // Also include the VerifiedBeaconList "project" if present in the parts. int nPart = -1; @@ -4715,16 +4717,58 @@ bool ScraperConstructConvergedManifestByProject(const NN::WhitelistSnapshot& pro // converged manifest if it is. if (nPart > 0) { - StructConvergedManifest.ConvergedManifestPartsMap.insert(std::make_pair("VerifiedBeacons", manifest.vParts[nPart]->data)); + StructConvergedManifest.ConvergedManifestPartPtrsMap.insert(std::make_pair("VerifiedBeacons", manifest.vParts[nPart])); } StructConvergedManifest.ConsensusBlock = nConvergedConsensusBlock; + // At this point all of the projects that meet convergence rules, along with the + // BeaconList and the VerfiedBeacons are now in the ConvergedManifestPartPtrsMap. + // We also need to populate them into the underlying CScraperConvergedManifest, because + // that manifest will hold the references to the part pointers to ensure they don't disappear + // until the converged manifest is removed from the global cache. + + // The BeaconList is element 0, do that first. + + { + LOCK(CSplitBlob::cs_mapParts); + _log(logattribute::INFO, "LOCK", "CSplitBlob::cs_mapParts"); + + auto iter = StructConvergedManifest.ConvergedManifestPartPtrsMap.find("BeaconList"); + + StructConvergedManifest.CScraperConvergedManifest_ptr->addPart(iter->second->hash); + + iter = StructConvergedManifest.ConvergedManifestPartPtrsMap.find("VerifiedBeacons"); + + if (iter != StructConvergedManifest.ConvergedManifestPartPtrsMap.end()) + { + StructConvergedManifest.CScraperConvergedManifest_ptr->addPart(iter->second->hash); + } + + // Now the rest of the projects (parts). + + for (iter = StructConvergedManifest.ConvergedManifestPartPtrsMap.begin(); + iter != StructConvergedManifest.ConvergedManifestPartPtrsMap.end(); ++iter) + { + if (iter->first != "BeaconList" && iter->first != "VerifiedBeacons") + { + StructConvergedManifest.CScraperConvergedManifest_ptr->addPart(iter->second->hash); + } + } + + _log(logattribute::INFO, "ENDLOCK", "CSplitBlob::cs_mapParts"); + } + + /* + // The ConvergedManifest content hash is in the order of the map key and on the data. - for (const auto& iter : StructConvergedManifest.ConvergedManifestPartsMap) - ss << iter.second; + for (const auto& iter : StructConvergedManifest.ConvergedManifestPartPtrsMap) + ss << iter.second->data; StructConvergedManifest.nContentHash = Hash(ss.begin(), ss.end()); + */ + StructConvergedManifest.ComputeConvergedContentHash(); + StructConvergedManifest.timestamp = GetAdjustedTime(); StructConvergedManifest.bByParts = true; @@ -4738,7 +4782,7 @@ bool ScraperConstructConvergedManifestByProject(const NN::WhitelistSnapshot& pro // Fill out the the excluded projects vector and the included scraper count (by project) map for (const auto& iProjects : projectWhitelist) { - if (StructConvergedManifest.ConvergedManifestPartsMap.find(iProjects.m_name) == StructConvergedManifest.ConvergedManifestPartsMap.end()) + if (StructConvergedManifest.ConvergedManifestPartPtrsMap.find(iProjects.m_name) == StructConvergedManifest.ConvergedManifestPartPtrsMap.end()) { // Project in whitelist was not in the map, so it goes in the exclusion vector. StructConvergedManifest.vExcludedProjects.push_back(iProjects.m_name); @@ -4801,7 +4845,7 @@ bool ScraperConstructConvergedManifestByProject(const NN::WhitelistSnapshot& pro if (!bConvergenceSuccessful) { // Reinitialize StructConvergedManifest. - StructConvergedManifest = {}; + StructConvergedManifest.Reset(); _log(logattribute::INFO, "ScraperConstructConvergedManifestByProject", "No convergence on manifests by projects."); } @@ -4946,12 +4990,15 @@ mmCSManifestsBinnedByScraper ScraperCullAndBinCScraperManifests() bool LoadBeaconListFromConvergedManifest(const ConvergedManifest& StructConvergedManifest, ScraperBeaconMap& mBeaconMap) { // Find the beacon list. - auto iter = StructConvergedManifest.ConvergedManifestPartsMap.find("BeaconList"); + auto iter = StructConvergedManifest.ConvergedManifestPartPtrsMap.find("BeaconList"); // Bail if the beacon list is not found, or the part is zero size (missing referenced part) - if (iter == StructConvergedManifest.ConvergedManifestPartsMap.end() || iter->second.size() == 0) return false; + if (iter == StructConvergedManifest.ConvergedManifestPartPtrsMap.end() || iter->second->data.size() == 0) + { + return false; + } - boostio::basic_array_source input_source(&iter->second[0], iter->second.size()); + boostio::basic_array_source input_source(&iter->second->data[0], iter->second->data.size()); boostio::stream> ingzss(input_source); boostio::filtering_istream in; @@ -4996,10 +5043,10 @@ std::vector GetVerifiedBeaconIDs(const ConvergedManifest& StructConverg std::vector result; ScraperPendingBeaconMap VerifiedBeaconMap; - const auto& iter = StructConvergedManifest.ConvergedManifestPartsMap.find("VerifiedBeacons"); - if (iter != StructConvergedManifest.ConvergedManifestPartsMap.end()) + const auto& iter = StructConvergedManifest.ConvergedManifestPartPtrsMap.find("VerifiedBeacons"); + if (iter != StructConvergedManifest.ConvergedManifestPartPtrsMap.end()) { - CDataStream part(iter->second, SER_NETWORK, 1); + CDataStream part(iter->second->data, SER_NETWORK, 1); try { @@ -5044,10 +5091,10 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsAndVerifiedBeacons(const Converged { ScraperStatsAndVerifiedBeacons stats_and_verified_beacons; - const auto& iter = stats.Convergence.ConvergedManifestPartsMap.find("VerifiedBeacons"); - if (iter != stats.Convergence.ConvergedManifestPartsMap.end()) + const auto& iter = stats.Convergence.ConvergedManifestPartPtrsMap.find("VerifiedBeacons"); + if (iter != stats.Convergence.ConvergedManifestPartPtrsMap.end()) { - CDataStream part(iter->second, SER_NETWORK, 1); + CDataStream part(iter->second->data, SER_NETWORK, 1); try { @@ -5174,7 +5221,7 @@ NN::Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats, bool bCon ConvergedScraperStatsCache.mScraperConvergedStats = mScraperConvergedStats; ConvergedScraperStatsCache.nTime = GetAdjustedTime(); - ConvergedScraperStatsCache.Convergence = StructConvergedManifest; + ConvergedScraperStatsCache.Convergence(StructConvergedManifest); if (IsV11Enabled(nBestHeight + 1)) { superblock = NN::Superblock::FromConvergence(ConvergedScraperStatsCache); @@ -5373,19 +5420,22 @@ UniValue ConvergedScraperStatsToJson(ConvergedScraperStats& ConvergedScraperStat { UniValue entry(UniValue::VOBJ); - ConvergedScraperStats dummy_converged_stats; + const ConvergedManifest& PastConvergence = iter.second.second; + //ScraperStats& PastConvergenceStats = PastConvergence. - dummy_converged_stats.Convergence = iter.second.second; - dummy_converged_stats.nTime = dummy_converged_stats.Convergence.timestamp; + //dummy_converged_stats.Convergence = iter.second.second; + //dummy_converged_stats.nTime = dummy_converged_stats.Convergence.timestamp; - dummy_converged_stats.mScraperConvergedStats = GetScraperStatsByConvergedManifest(dummy_converged_stats.Convergence).mScraperStats; + ScraperStats mScraperConvergedStats = GetScraperStatsByConvergedManifest(PastConvergence).mScraperStats; - entry.pushKV("past_convergence_timestamp", dummy_converged_stats.nTime); - entry.pushKV("past_convergence_datetime", DateTimeStrFormat("%x %H:%M:%S UTC", dummy_converged_stats.nTime)); + entry.pushKV("past_convergence_timestamp", PastConvergence.timestamp); + entry.pushKV("past_convergence_datetime", DateTimeStrFormat("%x %H:%M:%S UTC", PastConvergence.timestamp)); entry.pushKV("past_convergence_content_hash", iter.second.first.ToString()); entry.pushKV("past_convergence_reduced_content_hash", (uint64_t) iter.first); + const ConvergedScraperStats dummy_converged_stats(PastConvergence.timestamp, PastConvergence); + NN::Superblock superblock = NN::Superblock::FromConvergence(dummy_converged_stats); entry.pushKV("superblock_from_this_past_convergence_quorumhash", superblock.GetHash().ToString()); @@ -5662,14 +5712,14 @@ UniValue testnewsb(const UniValue& params, bool fHelp) std::advance(iPastSB, i); - RandomPastConvergedManifest = iPastSB->second.second; + RandomPastConvergedManifest(iPastSB->second.second); bPastConvergencesEmpty = false; } else if (PastConvergencesSize == 1) { //Use the first and only element. - RandomPastConvergedManifest = iPastSB->second.second; + RandomPastConvergedManifest(iPastSB->second.second); bPastConvergencesEmpty = false; } @@ -5692,10 +5742,10 @@ UniValue testnewsb(const UniValue& params, bool fHelp) // Add hints created from the hashes of converged manifest parts to each // superblock project section to assist receiving nodes with validation: // - for (const auto& part_pair : RandomPastConvergedManifest.ConvergedManifestPartsMap) + for (const auto& part_pair : RandomPastConvergedManifest.ConvergedManifestPartPtrsMap) { const std::string& project_name = part_pair.first; - const CSerializeData& part_data = part_pair.second; + const CSerializeData& part_data = part_pair.second->data; projects.SetHint(project_name, part_data); // This also sets m_converged_by_project to true. } @@ -5749,76 +5799,97 @@ UniValue scraperreport(const UniValue& params, bool fHelp) UniValue ret(UniValue::VOBJ); - uint64_t global_manifest_map_size = 0; - uint64_t global_parts_map_size = 0; - uint64_t global_stats_cache_current_convergence_included_scrapers = 0; - uint64_t global_stats_cache_current_convergence_parts_map_size = 0; - uint64_t global_stats_cache_past_convergence_map_size = 0; - uint64_t global_stats_cache_total_past_convergences_parts_maps_size = 0; - uint64_t global_stats_cache_avg_parts_map_size_per_convergence = 0; + UniValue global_scraper_net(UniValue::VOBJ); + UniValue converged_scraper_stats_cache(UniValue::VOBJ); + + uint64_t manifest_map_size = 0; + uint64_t parts_map_size = 0; + uint64_t current_convergence_publishing_scrapers = 0; + uint64_t current_convergence_part_pointer_map_size = 0; + uint64_t past_convergence_map_size = 0; + uint64_t total_past_convergences_part_pointer_maps_size = 0; + uint64_t total_past_convergences_part_unique_pointer_maps_size = 0; + int64_t part_objects_reduced = 0; + + std::set global_cache_unique_parts; { LOCK(CScraperManifest::cs_mapManifest); - global_manifest_map_size = CScraperManifest::mapManifest.size(); + manifest_map_size = CScraperManifest::mapManifest.size(); } - ret.pushKV("global_manifest_map_size", global_manifest_map_size); + global_scraper_net.pushKV("manifest_map_size", manifest_map_size); { LOCK(CSplitBlob::cs_mapParts); - global_parts_map_size = CSplitBlob::mapParts.size(); + parts_map_size = CSplitBlob::mapParts.size(); } - ret.pushKV("global_parts_map_size", global_parts_map_size); + global_scraper_net.pushKV("parts_map_size", parts_map_size); { LOCK(cs_ConvergedScraperStatsCache); - unsigned int i = 0; - if (ConvergedScraperStatsCache.NewFormatSuperblock.WellFormed()) { - global_stats_cache_current_convergence_included_scrapers = - ConvergedScraperStatsCache.Convergence.vIncludedScrapers.size(); + current_convergence_publishing_scrapers = + ConvergedScraperStatsCache.Convergence.vIncludedScrapers.size() + + ConvergedScraperStatsCache.Convergence.vExcludedScrapers.size(); - global_stats_cache_current_convergence_parts_map_size = - ConvergedScraperStatsCache.Convergence.ConvergedManifestPartsMap.size(); + current_convergence_part_pointer_map_size = + ConvergedScraperStatsCache.Convergence.ConvergedManifestPartPtrsMap.size(); - global_stats_cache_past_convergence_map_size = + past_convergence_map_size = ConvergedScraperStatsCache.PastConvergences.size(); + // This next section will form a set of hashs from the pointers in the global cache + // and also add the pointers up arithmetically. The difference is the efficiency gain + // from using pointers rather than copies into the global cache. + for (const auto& iter : ConvergedScraperStatsCache.Convergence.ConvergedManifestPartPtrsMap) + { + global_cache_unique_parts.insert(iter.second->hash); + } + for (const auto& iter : ConvergedScraperStatsCache.PastConvergences) { - global_stats_cache_total_past_convergences_parts_maps_size += - iter.second.second.ConvergedManifestPartsMap.size(); - ++i; + for (const auto& iter2 : iter.second.second.ConvergedManifestPartPtrsMap) + { + global_cache_unique_parts.insert(iter2.second->hash); + } + + total_past_convergences_part_pointer_maps_size += + iter.second.second.ConvergedManifestPartPtrsMap.size(); } - global_stats_cache_avg_parts_map_size_per_convergence = - (global_stats_cache_current_convergence_parts_map_size - + global_stats_cache_total_past_convergences_parts_maps_size) / - (i + 1); + total_past_convergences_part_unique_pointer_maps_size = global_cache_unique_parts.size(); + + part_objects_reduced = total_past_convergences_part_pointer_maps_size + - total_past_convergences_part_unique_pointer_maps_size; - ret.pushKV("global_stats_cache_current_convergence_included_scrapers", - global_stats_cache_current_convergence_included_scrapers); - ret.pushKV("global_stats_cache_current_convergence_parts_map_size", - global_stats_cache_current_convergence_parts_map_size); + converged_scraper_stats_cache.pushKV("current_convergence_included_scrapers", + current_convergence_publishing_scrapers); - ret.pushKV("global_stats_cache_past_convergence_map_size", - global_stats_cache_past_convergence_map_size); + converged_scraper_stats_cache.pushKV("current_convergence_part_pointer_map_size", + current_convergence_part_pointer_map_size); - ret.pushKV("global_stats_cache_total_past_convergences_parts_maps_size", - global_stats_cache_total_past_convergences_parts_maps_size); + converged_scraper_stats_cache.pushKV("past_convergence_map_size", + past_convergence_map_size); - ret.pushKV("global_stats_cache_avg_parts_map_size_per_convergence", - global_stats_cache_avg_parts_map_size_per_convergence); + converged_scraper_stats_cache.pushKV("total_past_convergences_part_pointer_maps_size", + total_past_convergences_part_pointer_maps_size); + + converged_scraper_stats_cache.pushKV("total_past_convergences_part_unique_pointer_maps_size", + total_past_convergences_part_unique_pointer_maps_size); } } + ret.pushKV("global_scraper_net", global_scraper_net); + ret.pushKV("converged_scraper_stats_cache", converged_scraper_stats_cache); + return ret; } diff --git a/src/test/neuralnet/superblock_tests.cpp b/src/test/neuralnet/superblock_tests.cpp index a0959cc0c9..7b33a16528 100644 --- a/src/test/neuralnet/superblock_tests.cpp +++ b/src/test/neuralnet/superblock_tests.cpp @@ -1,6 +1,7 @@ #include "base58.h" #include "compat/endian.h" #include "neuralnet/superblock.h" +#include "scraper_net.h" #include "streams.h" #include @@ -432,22 +433,26 @@ ConvergedScraperStats GetTestConvergence( // Add a verified beacons project part. Technically, this is the second // part for a manifest (offset 1). We skipped adding the beacon list part. // - CDataStream verified_beacons_part(SER_NETWORK, PROTOCOL_VERSION); - verified_beacons_part + CDataStream verified_beacons_part_data(SER_NETWORK, PROTOCOL_VERSION); + verified_beacons_part_data << ScraperPendingBeaconMap { *stats.mVerifiedMap.begin(), *++stats.mVerifiedMap.begin(), }; - convergence.Convergence.ConvergedManifestPartsMap.emplace( - "VerifiedBeacons", - CSerializeData(verified_beacons_part.begin() , verified_beacons_part.end())); + CSplitBlob::CPart verified_beacons_part(Hash(verified_beacons_part_data.begin(), + verified_beacons_part_data.end())); + + verified_beacons_part.data = CSerializeData(verified_beacons_part_data.begin() , verified_beacons_part_data.end()); + + convergence.Convergence.ConvergedManifestPartPtrsMap.emplace("VerifiedBeacons", + &verified_beacons_part); // Add some project parts with the same names as the projects in the stats. - // The part data doesn't matter, so we just add empty containers. + // The part data pointers don't matter, so we just add nullptrs. // - convergence.Convergence.ConvergedManifestPartsMap.emplace("project_1", CSerializeData()); - convergence.Convergence.ConvergedManifestPartsMap.emplace("project_2", CSerializeData()); + convergence.Convergence.ConvergedManifestPartPtrsMap.emplace("project_1", nullptr); + convergence.Convergence.ConvergedManifestPartPtrsMap.emplace("project_2", nullptr); return convergence; }