From 8a6867f7b2a4a3b64b13186f53e35b1c69919c51 Mon Sep 17 00:00:00 2001 From: Simon Haegler Date: Sat, 3 Mar 2018 18:40:52 +0100 Subject: [PATCH] #58: introduce ResolveMapCache and trigger force cook on RPK change --- src/palladio/CMakeLists.txt | 1 + src/palladio/MultiWatch.h | 2 +- src/palladio/NodeParameter.cpp | 9 ++-- src/palladio/NodeParameter.h | 9 ++-- src/palladio/PRTContext.cpp | 90 +++++++++++++------------------- src/palladio/PRTContext.h | 11 +--- src/palladio/PalladioMain.cpp | 8 +-- src/palladio/PalladioMain.h | 4 ++ src/palladio/ResolveMapCache.cpp | 73 ++++++++++++++++++++++++++ src/palladio/ResolveMapCache.h | 37 +++++++++++++ src/palladio/SOPAssign.h | 1 + 11 files changed, 165 insertions(+), 80 deletions(-) create mode 100644 src/palladio/PalladioMain.h create mode 100644 src/palladio/ResolveMapCache.cpp create mode 100644 src/palladio/ResolveMapCache.h diff --git a/src/palladio/CMakeLists.txt b/src/palladio/CMakeLists.txt index d356c47e..1c357417 100644 --- a/src/palladio/CMakeLists.txt +++ b/src/palladio/CMakeLists.txt @@ -17,6 +17,7 @@ add_library(${PROJECT_NAME} SHARED ShapeGenerator.cpp NodeParameter.cpp PRTContext.cpp + ResolveMapCache.cpp SOPAssign.cpp SOPGenerate.cpp PrimitivePartition.cpp diff --git a/src/palladio/MultiWatch.h b/src/palladio/MultiWatch.h index 626fe9cb..ce64bd7e 100644 --- a/src/palladio/MultiWatch.h +++ b/src/palladio/MultiWatch.h @@ -24,7 +24,7 @@ #define WA(x) //WatchAgent wa_(x, __PRETTY_FUNCTION__) #define WA_NEW_LAP //theWatch.newLap(); -#define WA_PRINT_TIMINGS //theWatch.printTimings(); +#define WA_PRINT_TIMINGS //theWatch.printTimings(); // TODO: use scoped printer class MultiWatch { diff --git a/src/palladio/NodeParameter.cpp b/src/palladio/NodeParameter.cpp index 9d3963bc..f3b7c06a 100644 --- a/src/palladio/NodeParameter.cpp +++ b/src/palladio/NodeParameter.cpp @@ -84,12 +84,6 @@ int updateRPK(void* data, int, fpreal32 time, const PRM_Template*) { node->evalString(utNextRPKStr, AssignNodeParams::RPK.getToken(), 0, time); const boost::filesystem::path nextRPK(utNextRPKStr.toStdString()); - // -- early exit if rpk path is not valid - if (!boost::filesystem::exists(nextRPK)) { - LOG_WRN << "non-existent rpk"; - return NOT_CHANGED; - } - const ResolveMapUPtr& resolveMap = prtCtx->getResolveMap(nextRPK); if (!resolveMap ) { LOG_WRN << "invalid resolve map"; @@ -135,6 +129,7 @@ int updateRPK(void* data, int, fpreal32 time, const PRM_Template*) { return CHANGED; } +// TODO: called multiple times if assign node is activated (clicked on), optimize void buildStartRuleMenu(void* data, PRM_Name* theMenu, int theMaxSize, const PRM_SpareData*, const PRM_Parm* parm) { constexpr bool DBG = false; @@ -205,6 +200,7 @@ void buildStartRuleMenu(void* data, PRM_Name* theMenu, int theMaxSize, const PRM } } +// TODO: called multiple times if assign node is activated (clicked on), optimize void buildRuleFileMenu(void* data, PRM_Name* theMenu, int theMaxSize, const PRM_SpareData*, const PRM_Parm* parm) { const auto* node = static_cast(data); const auto& prtCtx = node->getPRTCtx(); @@ -235,6 +231,7 @@ std::string extractStyle(const prt::RuleFileInfo::Entry* re) { return toOSNarrowFromUTF16(style); } +// TODO: called multiple times if assign node is activated (clicked on), optimize void buildStyleMenu(void* data, PRM_Name* theMenu, int theMaxSize, const PRM_SpareData*, const PRM_Parm*) { const auto* node = static_cast(data); const PRTContextUPtr& prtCtx = node->getPRTCtx(); diff --git a/src/palladio/NodeParameter.h b/src/palladio/NodeParameter.h index 135b40ce..d9dca023 100644 --- a/src/palladio/NodeParameter.h +++ b/src/palladio/NodeParameter.h @@ -70,8 +70,7 @@ const std::string RULE_FILE_HELP = "Sets value for primitive attribute '" + PLD_ void buildRuleFileMenu(void *data, PRM_Name *theMenu, int theMaxSize, const PRM_SpareData *, const PRM_Parm *); -static PRM_ChoiceList ruleFileMenu(static_cast(PRM_CHOICELIST_EXCLUSIVE | PRM_CHOICELIST_REPLACE), - &buildRuleFileMenu); +static PRM_ChoiceList ruleFileMenu(static_cast(PRM_CHOICELIST_REPLACE), &buildRuleFileMenu); auto getRuleFile = [](const OP_Node* node, fpreal t) -> std::wstring { UT_String s; @@ -91,8 +90,7 @@ const std::string STYLE_HELP = "Sets value for primitive attribute '" + PLD_STYL void buildStyleMenu(void *data, PRM_Name *theMenu, int theMaxSize, const PRM_SpareData *, const PRM_Parm *); -static PRM_ChoiceList styleMenu(static_cast(PRM_CHOICELIST_EXCLUSIVE | PRM_CHOICELIST_REPLACE), - &buildStyleMenu); +static PRM_ChoiceList styleMenu(static_cast(PRM_CHOICELIST_REPLACE), &buildStyleMenu); auto getStyle = [](const OP_Node* node, fpreal t) -> std::wstring { UT_String s; @@ -112,8 +110,7 @@ const std::string START_RULE_HELP = "Sets value for primitive attribute '" + PLD void buildStartRuleMenu(void *data, PRM_Name *theMenu, int theMaxSize, const PRM_SpareData *, const PRM_Parm *); -static PRM_ChoiceList startRuleMenu(static_cast(PRM_CHOICELIST_EXCLUSIVE | PRM_CHOICELIST_REPLACE), - &buildStartRuleMenu); +static PRM_ChoiceList startRuleMenu(static_cast(PRM_CHOICELIST_REPLACE), &buildStartRuleMenu); auto getStartRule = [](const OP_Node* node, fpreal t) -> std::wstring { UT_String s; diff --git a/src/palladio/PRTContext.cpp b/src/palladio/PRTContext.cpp index df1d7693..0ceb5264 100644 --- a/src/palladio/PRTContext.cpp +++ b/src/palladio/PRTContext.cpp @@ -15,14 +15,15 @@ */ #include "PRTContext.h" +#include "PalladioMain.h" #include "LogHandler.h" +#include "SOPAssign.h" + +#include "OP/OP_Node.h" +#include "OP/OP_Director.h" +#include "OP/OP_Network.h" #include -#include -#include -#ifndef WIN32 -# include -#endif namespace { @@ -97,15 +98,24 @@ uint32_t getNumCores() { return n > 0 ? n : 1; } -const timespec NO_MODIFICATION_TIME{0, 0}; +/** + * schedule recook of all assign nodes with matching rpk + */ +void scheduleRecook(const boost::filesystem::path& rpk) { + auto checkAssignNodes = [](OP_Node& n, void* data) -> bool { + if (n.getOperator()->getName().equal(OP_PLD_ASSIGN)) { + auto rpk = reinterpret_cast(data); + SOPAssign& sa = static_cast(n); + if (sa.getRPK() == *rpk) { + LOG_DBG << "forcing recook of: " << n.getName() << ", " << n.getOpType() << ", " << n.getOperator()->getName(); + sa.forceRecook(); + } + } + return false; + }; -timespec getFileModificationTime(const boost::filesystem::path& p) { - struct stat result; - if (stat(p.c_str(), &result) == 0) { - return result.st_mtim; - } - else - return NO_MODIFICATION_TIME; + OP_Network* objMgr = OPgetDirector()->getManager("obj"); + objMgr->traverseChildren(checkAssignNodes, const_cast(reinterpret_cast(&rpk)), true); } } // namespace @@ -115,8 +125,8 @@ PRTContext::PRTContext(const std::vector& addExtDirs) : mLogHandler(new logging::LogHandler(PLD_LOG_PREFIX)), mLicHandle{nullptr}, mPRTCache{prt::CacheObject::create(prt::CacheObject::CACHE_TYPE_DEFAULT)}, - mRPKUnpackPath{boost::filesystem::temp_directory_path() / (PLD_TMP_PREFIX + std::to_string(::getpid()))}, - mCores{getNumCores()} + mCores{getNumCores()}, + mResolveMapCache{new ResolveMapCache(boost::filesystem::temp_directory_path() / (PLD_TMP_PREFIX + std::to_string(::getpid())))} { const prt::LogLevel logLevel = getLogLevel(); prt::setLogLevel(logLevel); @@ -162,53 +172,23 @@ PRTContext::PRTContext(const std::vector& addExtDirs) } PRTContext::~PRTContext() { - mPRTCache.reset(); + mResolveMapCache.reset(); + LOG_INF << "Released RPK Cache"; + + mPRTCache.reset(); // calling reset manually to ensure order LOG_INF << "Released PRT cache"; - mLicHandle.reset(); + mLicHandle.reset(); // same here LOG_INF << "Shutdown PRT & returned license"; - boost::filesystem::remove_all(mRPKUnpackPath); - LOG_INF << "Removed RPK unpack directory"; - prt::removeLogHandler(mLogHandler.get()); } const ResolveMapUPtr& PRTContext::getResolveMap(const boost::filesystem::path& rpk) { - if (rpk.empty()) - return mResolveMapNone; - - const auto timeStamp = getFileModificationTime(rpk); - LOG_DBG << "rpk: current timestamp: " << timeStamp.tv_sec << "s " << timeStamp.tv_nsec << "ns"; - - bool reload = false; - auto it = mResolveMapCache.find(rpk); - if (it != mResolveMapCache.end()) { - LOG_DBG << "rpk: cache timestamp: " << it->second.mTimeStamp.tv_sec << "s " << it->second.mTimeStamp.tv_nsec << "ns"; - if (it->second.mTimeStamp.tv_sec != timeStamp.tv_sec || it->second.mTimeStamp.tv_nsec != timeStamp.tv_nsec) { - mResolveMapCache.erase(it); - const auto cnt = boost::filesystem::remove_all(mRPKUnpackPath / rpk.leaf()); - mPRTCache->flushAll(); // TODO: or do we need to explicitely remove cgbs from cache? - LOG_DBG << "RPK change detected, refreshing unpack cache: " << rpk << "( removed " << cnt << " files)"; - reload = true; - } + auto lookupResult = mResolveMapCache->get(rpk); + if (lookupResult.second == ResolveMapCache::CacheStatus::MISS) { + mPRTCache->flushAll(); + scheduleRecook(rpk); } - else - reload = true; - - if (reload) { - const auto rpkURI = toFileURI(rpk); - - ResolveMapCacheEntry rmce; - rmce.mTimeStamp = timeStamp; - - prt::Status status = prt::STATUS_UNSPECIFIED_ERROR; - rmce.mResolveMap.reset(prt::createResolveMap(rpkURI.c_str(), mRPKUnpackPath.wstring().c_str(), &status)); - if (status != prt::STATUS_OK) - return mResolveMapNone; - - it = mResolveMapCache.emplace(rpk, std::move(rmce)).first; - } - - return it->second.mResolveMap; + return lookupResult.first; } diff --git a/src/palladio/PRTContext.h b/src/palladio/PRTContext.h index 34d7a2e1..dd606f64 100644 --- a/src/palladio/PRTContext.h +++ b/src/palladio/PRTContext.h @@ -16,6 +16,7 @@ #pragma once +#include "ResolveMapCache.h" #include "Utils.h" #include "prt/Object.h" @@ -47,16 +48,8 @@ struct PRTContext final { logging::LogHandlerPtr mLogHandler; ObjectUPtr mLicHandle; CacheObjectUPtr mPRTCache; - boost::filesystem::path mRPKUnpackPath; const uint32_t mCores; - - struct ResolveMapCacheEntry { - ResolveMapUPtr mResolveMap; - timespec mTimeStamp; - }; - using ResolveMapCache = std::map; - ResolveMapCache mResolveMapCache; - const ResolveMapUPtr mResolveMapNone; + ResolveMapCacheUPtr mResolveMapCache; }; using PRTContextUPtr = std::unique_ptr; diff --git a/src/palladio/PalladioMain.cpp b/src/palladio/PalladioMain.cpp index ce451c05..2dc86082 100644 --- a/src/palladio/PalladioMain.cpp +++ b/src/palladio/PalladioMain.cpp @@ -14,9 +14,11 @@ * limitations under the License. */ +#include "PalladioMain.h" + +#include "PRTContext.h" #include "SOPAssign.h" #include "SOPGenerate.h" -#include "PRTContext.h" #include "NodeParameter.h" #include "OP/OP_OperatorTable.h" @@ -47,7 +49,7 @@ void newSopOperator(OP_OperatorTable *table) { auto createSOPAssign = [](OP_Network *net, const char *name, OP_Operator *op) -> OP_Node* { return new SOPAssign(prtCtx, net, name, op); }; - table->addOperator(new OP_Operator("pldAssign", "pldAssign", createSOPAssign, + table->addOperator(new OP_Operator(OP_PLD_ASSIGN, OP_PLD_ASSIGN, createSOPAssign, AssignNodeParams::PARAM_TEMPLATES, 1, 1, nullptr, OP_FLAG_GENERATOR )); @@ -55,7 +57,7 @@ void newSopOperator(OP_OperatorTable *table) { auto createSOPGenerate = [](OP_Network *net, const char *name, OP_Operator *op) -> OP_Node* { return new SOPGenerate(prtCtx, net, name, op); }; - table->addOperator(new OP_Operator("pldGenerate", "pldGenerate", createSOPGenerate, + table->addOperator(new OP_Operator(OP_PLD_GENERATE, OP_PLD_GENERATE, createSOPGenerate, GenerateNodeParams::PARAM_TEMPLATES, 1, 1, nullptr, OP_FLAG_GENERATOR )); } diff --git a/src/palladio/PalladioMain.h b/src/palladio/PalladioMain.h new file mode 100644 index 00000000..7b16a3a1 --- /dev/null +++ b/src/palladio/PalladioMain.h @@ -0,0 +1,4 @@ +#pragma once + +constexpr const char* OP_PLD_ASSIGN = "pldAssign"; +constexpr const char* OP_PLD_GENERATE = "pldGenerate"; diff --git a/src/palladio/ResolveMapCache.cpp b/src/palladio/ResolveMapCache.cpp new file mode 100644 index 00000000..75ca74ff --- /dev/null +++ b/src/palladio/ResolveMapCache.cpp @@ -0,0 +1,73 @@ +// +// Created by shaegler on 3/3/18. +// + +#include "ResolveMapCache.h" +#include "LogHandler.h" + + +namespace { + +const ResolveMapUPtr RESOLVE_MAP_NONE; +const timespec NO_MODIFICATION_TIME{0, 0}; + +timespec getFileModificationTime(const boost::filesystem::path& p) { + struct stat result; + if (stat(p.c_str(), &result) == 0) { + return result.st_mtim; + } + else + return NO_MODIFICATION_TIME; +} + +} // namespace + + +ResolveMapCache::~ResolveMapCache() { + boost::filesystem::remove_all(mRPKUnpackPath); + LOG_INF << "Removed RPK unpack directory"; +} + +ResolveMapCache::LookupResult ResolveMapCache::get(const boost::filesystem::path& rpk) { + if (rpk.empty()) + return { RESOLVE_MAP_NONE, CacheStatus::MISS }; + + if (!boost::filesystem::exists(rpk)) + return { RESOLVE_MAP_NONE, CacheStatus::MISS }; + + const auto timeStamp = getFileModificationTime(rpk); + LOG_DBG << "rpk: current timestamp: " << timeStamp.tv_sec << "s " << timeStamp.tv_nsec << "ns"; + + CacheStatus cs = CacheStatus::HIT; + auto it = mCache.find(rpk); + if (it != mCache.end()) { + LOG_DBG << "rpk: cache timestamp: " << it->second.mTimeStamp.tv_sec << "s " << it->second.mTimeStamp.tv_nsec << "ns"; + if (it->second.mTimeStamp.tv_sec != timeStamp.tv_sec || it->second.mTimeStamp.tv_nsec != timeStamp.tv_nsec) { + mCache.erase(it); + const auto cnt = boost::filesystem::remove_all(mRPKUnpackPath / rpk.leaf()); + LOG_INF << "RPK change detected, forcing reload and clearing cache for " << rpk << " (removed " << cnt << " files)"; + cs = CacheStatus::MISS; + } + } + else + cs = CacheStatus::MISS; + + if (cs == CacheStatus::MISS) { + const auto rpkURI = toFileURI(rpk); + + ResolveMapCacheEntry rmce; + rmce.mTimeStamp = timeStamp; + + prt::Status status = prt::STATUS_UNSPECIFIED_ERROR; + rmce.mResolveMap.reset(prt::createResolveMap(rpkURI.c_str(), mRPKUnpackPath.wstring().c_str(), &status)); + if (status != prt::STATUS_OK) + return { RESOLVE_MAP_NONE, CacheStatus::MISS }; + + it = mCache.emplace(rpk, std::move(rmce)).first; + LOG_INF << "Upacked RPK " << rpk << " to " << mRPKUnpackPath; + } + + return { it->second.mResolveMap, cs }; + +} + diff --git a/src/palladio/ResolveMapCache.h b/src/palladio/ResolveMapCache.h new file mode 100644 index 00000000..33eec05d --- /dev/null +++ b/src/palladio/ResolveMapCache.h @@ -0,0 +1,37 @@ +#pragma once + +#include "Utils.h" + +#include "boost/filesystem.hpp" + +#include +#include +#ifndef WIN32 +# include +#endif + + +class ResolveMapCache { +public: + ResolveMapCache(const boost::filesystem::path& unpackPath) : mRPKUnpackPath{unpackPath} { } + ResolveMapCache(const ResolveMapCache&) = delete; + ResolveMapCache(ResolveMapCache&&) = delete; + ResolveMapCache& operator=(ResolveMapCache&) = delete; + ~ResolveMapCache(); + + enum class CacheStatus { HIT, MISS }; + using LookupResult = std::pair; + LookupResult get(const boost::filesystem::path& rpk); + +private: + struct ResolveMapCacheEntry { + ResolveMapUPtr mResolveMap; + timespec mTimeStamp; + }; + using Cache = std::map; + Cache mCache; + + const boost::filesystem::path mRPKUnpackPath; +}; + +using ResolveMapCacheUPtr = std::unique_ptr; \ No newline at end of file diff --git a/src/palladio/SOPAssign.h b/src/palladio/SOPAssign.h index 2abb7238..6fa58d4f 100644 --- a/src/palladio/SOPAssign.h +++ b/src/palladio/SOPAssign.h @@ -28,6 +28,7 @@ class SOPAssign : public SOP_Node { ~SOPAssign() override = default; const PRTContextUPtr& getPRTCtx() const { return mPRTCtx; } + const boost::filesystem::path& getRPK() const { return mShapeConverter->mRPK; } void opChanged(OP_EventType reason, void* data = nullptr) override;