Skip to content

Commit

Permalink
Further lock adjustments and put in missed IsComplete()
Browse files Browse the repository at this point in the history
  • Loading branch information
@jamescowens
jamescowens committed Aug 22, 2020
1 parent c64a17e commit 8c1eafa
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 72 deletions.
3 changes: 1 addition & 2 deletions src/scraper/fwd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,7 @@ typedef std::map<ScraperID, mCSManifest> mmCSManifestsBinnedByScraper;
// -------------- Project -- Converged Part Pointer
typedef std::map<std::string, CSplitBlob::CPart*> mConvergedManifestPart_ptrs;

// The below type is still used for situations where a long global lock would be held during stats processing
// primarily in GetScraperStatsByConvergedManifest because of the intense work done in LoadProjectObjectToStatsByCPID.
// The below type may still used for situations where a long global lock would be held.
// -------------- Project ---- Converged Part
typedef std::map<std::string, CSerializeData> mConvergedManifestParts;

Expand Down
86 changes: 16 additions & 70 deletions src/scraper/scraper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3457,45 +3457,8 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByConvergedManifest(const Converge

double dMagnitudePerProject = NEURALNETWORKMULTIPLIER / nActiveProjects;

mConvergedManifestParts ConvergedManifestPartsMap;
ScraperStats mScraperStats;

/*
// This is a lock and copy block specifically because LoadProjectObjectToStatsByCPID is very expensive and in
// this particular case it will reduce lock time to make a copy of the parts temporarily for processing. We have
// to take these locks now because of the conversion of the StructConvergedManifest parts map to pointers.
{
LOCK(CSplitBlob::cs_mapParts);
_log(logattribute::INFO, "LOCK", "CSplitBlob::cs_mapParts");
for (auto entry = StructConvergedManifest.ConvergedManifestPartPtrsMap.begin(); entry != StructConvergedManifest.ConvergedManifestPartPtrsMap.end(); ++entry)
{
// Do not process the BeaconList or VerifiedBeacons as a project stats file.
if (entry->first != "BeaconList" && entry->first != "VerifiedBeacons")
{
ConvergedManifestPartsMap[entry->first] = entry->second->data;
}
}
_log(logattribute::INFO, "ENDLOCK", "CSplitBlob::cs_mapParts");
}
// Now run through the copy and do the stats processing.
for (const auto& entry : ConvergedManifestPartsMap)
{
_log(logattribute::INFO, "GetScraperStatsByConvergedManifest", "Processing stats for project: " + entry.first);
ScraperStats mProjectScraperStats;
LoadProjectObjectToStatsByCPID(entry.first, entry.second, dMagnitudePerProject, mProjectScraperStats);
// Insert into overall map.
for (auto const& entry2 : mProjectScraperStats)
{
mScraperStats[entry2.first] = entry2.second;
}
}
*/

for (auto entry = StructConvergedManifest.ConvergedManifestPartPtrsMap.begin(); entry != StructConvergedManifest.ConvergedManifestPartPtrsMap.end(); ++entry)
{
Expand Down Expand Up @@ -4704,14 +4667,6 @@ bool ScraperConstructConvergedManifestByProject(const NN::WhitelistSnapshot& pro
_log(logattribute::INFO, "ENDLOCK", "CSplitBlob::cs_mapParts");
}

/*
// The ConvergedManifest content hash is in the order of the map key and on the data.
for (const auto& iter : StructConvergedManifest.ConvergedManifestPartPtrsMap)
ss << iter.second->data;
StructConvergedManifest.nContentHash = Hash(ss.begin(), ss.end());
*/
StructConvergedManifest.ComputeConvergedContentHash();

StructConvergedManifest.timestamp = GetAdjustedTime();
Expand Down Expand Up @@ -4809,6 +4764,9 @@ mmCSManifestsBinnedByScraper BinCScraperManifestsByScraper()
{
CScraperManifest& manifest = *iter->second;

// Do not consider manifests that do not have all of their parts.
if (!manifest.isComplete()) continue;

std::string sManifestName = manifest.sCManifestName;
int64_t nTime = manifest.nTime;
uint256 nHash = *manifest.phash;
Expand Down Expand Up @@ -4843,7 +4801,7 @@ mmCSManifestsBinnedByScraper ScraperCullAndBinCScraperManifests()

_log(logattribute::INFO, "ScraperDeleteCScraperManifests", "Deleting old CScraperManifests.");

LOCK2(CScraperManifest::cs_mapManifest, CSplitBlob::cs_mapParts);
LOCK(CScraperManifest::cs_mapManifest);
_log(logattribute::INFO, "LOCK", "CScraperManifest::cs_mapManifest");

// First check for unauthorized manifests just in case a scraper has been deauthorized.
Expand Down Expand Up @@ -4924,7 +4882,7 @@ mmCSManifestsBinnedByScraper ScraperCullAndBinCScraperManifests()
// that large. (The lock on CScraperManifest::cs_mapManifest is still held from above.)
mMapCSManifestsBinnedByScraper = BinCScraperManifestsByScraper();

_log(logattribute::INFO, "ENDLOCK2", "CScraperManifest::cs_mapManifest, CSplitBlob::cs_mapParts");
_log(logattribute::INFO, "ENDLOCK", "CScraperManifest::cs_mapManifest");

return mMapCSManifestsBinnedByScraper;
}
Expand All @@ -4934,29 +4892,21 @@ mmCSManifestsBinnedByScraper ScraperCullAndBinCScraperManifests()
// ---------------------------------------------- In ---------------------------------------- Out
bool LoadBeaconListFromConvergedManifest(const ConvergedManifest& StructConvergedManifest, ScraperBeaconMap& mBeaconMap)
{
boostio::filtering_istream in;
// Find the beacon list.
auto iter = StructConvergedManifest.ConvergedManifestPartPtrsMap.find("BeaconList");

// Bail if the beacon list is not found, or the part is zero size (missing referenced part)
if (iter == StructConvergedManifest.ConvergedManifestPartPtrsMap.end() || iter->second->data.size() == 0)
{
LOCK(CSplitBlob::cs_mapParts);
_log(logattribute::INFO, "LOCK", "CSplitBlob::cs_mapParts");

// Find the beacon list.
auto iter = StructConvergedManifest.ConvergedManifestPartPtrsMap.find("BeaconList");

// Bail if the beacon list is not found, or the part is zero size (missing referenced part)
if (iter == StructConvergedManifest.ConvergedManifestPartPtrsMap.end() || iter->second->data.size() == 0)
{
return false;
}

boostio::basic_array_source<char> input_source(&iter->second->data[0], iter->second->data.size());
boostio::stream<boostio::basic_array_source<char>> ingzss(input_source);
return false;
}

in.push(boostio::gzip_decompressor());
in.push(ingzss);
boostio::basic_array_source<char> input_source(&iter->second->data[0], iter->second->data.size());
boostio::stream<boostio::basic_array_source<char>> ingzss(input_source);

_log(logattribute::INFO, "ENDLOCK", "CSplitBlob::cs_mapParts");
}
boostio::filtering_istream in;
in.push(boostio::gzip_decompressor());
in.push(ingzss);

std::string line;

Expand Down Expand Up @@ -5374,10 +5324,6 @@ UniValue ConvergedScraperStatsToJson(ConvergedScraperStats& ConvergedScraperStat
UniValue entry(UniValue::VOBJ);

const ConvergedManifest& PastConvergence = iter.second.second;
//ScraperStats& PastConvergenceStats = PastConvergence.

//dummy_converged_stats.Convergence = iter.second.second;
//dummy_converged_stats.nTime = dummy_converged_stats.Convergence.timestamp;

ScraperStats mScraperConvergedStats = GetScraperStatsByConvergedManifest(PastConvergence).mScraperStats;

Expand Down

0 comments on commit 8c1eafa

Please sign in to comment.