Port scraper log archiving functionality to main logging class

Note that the archivelog function needs to be moved out of the scraper file now.

Also next the scraper logger will be retired in favor of the using another instance
of the main logger class.

src/init.cpp

@@ -349,7 +349,8 @@ void InitLogging()
 
     if (LogInstance().m_print_to_file)
     {
-        if (GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile()))
+        // Only shrink debug file at start if log archiving is set to false.
+        if (!GetBoolArg("-logarchivedaily", true) && GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile()))
         {
             // Do this first since it both loads a bunch of debug.log into memory,
             // and because this needs to happen before any other debug.log printing

src/logging.cpp

@@ -7,7 +7,20 @@
 #include <util/threadnames.h>
 #include "util/time.h"
 
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/copy.hpp>
+#include <boost/filesystem/fstream.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/iostreams/filter/gzip.hpp>
+
 #include <mutex>
+#include <set>
+
+// Unavoidable because these are in util.h.
+extern fs::path &GetDataDir(bool fNetSpecific);
+extern std::string DateTimeStrFormat(const char* pszFormat, int64_t nTime);
+extern bool GetBoolArg(const std::string& strArg, bool fDefault);
+extern int64_t GetArg(const std::string& strArg, int64_t nDefault);
 
 const char * const DEFAULT_DEBUGLOGFILE = "debug.log";
 
@@ -26,12 +39,13 @@ BCLog::Logger& LogInstance()
  * have a trivial destructor.
  *
  * This method of initialization was originally introduced in
- * ee3374234c60aba2cc4c5cd5cac1c0aefc2d817c.
+ * Bitcoin core commit ee3374234c60aba2cc4c5cd5cac1c0aefc2d817c.
  */
     static BCLog::Logger* g_logger{new BCLog::Logger()};
     return *g_logger;
 }
 
+boost::gregorian::date BCLog::Logger::PrevArchiveCheckDate = boost::posix_time::from_time_t(GetAdjustedTime()).date();
 bool fLogIPs = DEFAULT_LOGIPS;
 
 static int FileWriteStr(const std::string &str, FILE *fp)
@@ -342,3 +356,143 @@ void BCLog::Logger::ShrinkDebugFile()
     else if (file != nullptr)
         fclose(file);
 }
+
+bool BCLog::Logger::archive(bool fImmediate, fs::path pfile_out)
+{
+    bool fArchiveDaily = GetBoolArg("-logarchivedaily", true);
+
+    int64_t nTime = GetAdjustedTime();
+    boost::gregorian::date ArchiveCheckDate = boost::posix_time::from_time_t(nTime).date();
+    fs::path plogfile;
+    fs::path pfile_temp;
+    fs::path pathDataDir = GetDataDir(false);
+
+    std::stringstream ssArchiveCheckDate, ssPrevArchiveCheckDate;
+
+    ssArchiveCheckDate << ArchiveCheckDate;
+    ssPrevArchiveCheckDate << PrevArchiveCheckDate;
+
+    fs::path LogArchiveDir = pathDataDir / "logarchive";
+
+    // Check to see if the log archive directory exists and is a directory. If not create it.
+    if (fs::exists(LogArchiveDir))
+    {
+        // If it is a normal file, this is not right. Remove the file and replace with the log archive directory.
+        if (fs::is_regular_file(LogArchiveDir))
+        {
+            fs::remove(LogArchiveDir);
+            fs::create_directory(LogArchiveDir);
+        }
+    }
+    else
+    {
+        fs::create_directory(LogArchiveDir);
+    }
+
+    if (fImmediate || (fArchiveDaily && ArchiveCheckDate > PrevArchiveCheckDate))
+    {
+        {
+            std::lock_guard<std::mutex> scoped_lock(m_cs);
+
+            fclose(m_fileout);
+
+            plogfile = m_file_path;
+
+            pfile_temp = static_cast<fs::path>(m_file_path.stem().string() + "-" + DateTimeStrFormat("%Y%m%d%H%M%S", nTime) + m_file_path.extension().string());
+
+            pfile_out = LogArchiveDir / static_cast<fs::path>((m_file_path.filename().stem().string() + "-" + DateTimeStrFormat("%Y%m%d%H%M%S", nTime)
+                                         + m_file_path.filename().extension().string() + ".gz"));
+
+            try
+            {
+                fs::rename(plogfile, pfile_temp);
+            }
+            catch(...)
+            {
+                LogPrintf("ERROR: Logger: archive: Failed to rename logging file\n");
+                return false;
+            }
+
+            // Re-open logging file. (This is subtly different than the flag based reopen above, because the file must be closed first, renamed for compression,
+            // and then a new one opened.
+            FILE* new_fileout = fsbridge::fopen(m_file_path, "a");
+            if (new_fileout)
+            {
+                setbuf(new_fileout, nullptr); // unbuffered
+                m_fileout = new_fileout;
+            }
+
+            PrevArchiveCheckDate = ArchiveCheckDate;
+        }
+
+        fsbridge::ifstream infile(pfile_temp, std::ios_base::in | std::ios_base::binary);
+
+        if (!infile)
+        {
+            LogPrintf("ERROR: Logger: Failed to open archive log file for compression %s.", pfile_temp.string());
+            return false;
+        }
+
+        fsbridge::ofstream outgzfile(pfile_out, std::ios_base::out | std::ios_base::binary);
+
+        if (!outgzfile)
+        {
+            LogPrintf("ERROR: Logger: Failed to open archive gzip file %s.", pfile_out.string());
+            return false;
+        }
+
+        boost::iostreams::filtering_ostream out;
+        out.push(boost::iostreams::gzip_compressor());
+        out.push(outgzfile);
+
+        boost::iostreams::copy(infile, out);
+
+        infile.close();
+        outgzfile.flush();
+        outgzfile.close();
+
+        fs::remove(pfile_temp);
+
+        bool fDeleteOldLogArchives = GetBoolArg("-deleteoldlogarchives", true);
+
+        if (fDeleteOldLogArchives)
+        {
+            unsigned int nRetention = (unsigned int)GetArg("-logarchiveretainnumfiles", 14);
+            LogPrintf ("INFO: Logger: nRetention %i.", nRetention);
+
+            std::set<fs::directory_entry, std::greater <fs::directory_entry>> SortedDirEntries;
+
+            // Iterate through the log archive directory and delete the oldest files beyond the retention rule
+            // The names are in format <logname base>-YYYYMMDDHHMMSS.log for the logs, so filter by containing <logname base>.
+            // The greater than sort in the set should then return descending order by datetime.
+            for (fs::directory_entry& DirEntry : fs::directory_iterator(LogArchiveDir))
+            {
+                std::string sFilename = DirEntry.path().filename().string();
+                size_t FoundPos = sFilename.find(m_file_path.filename().stem().string());
+
+                if (FoundPos != std::string::npos) SortedDirEntries.insert(DirEntry);
+            }
+
+            // Now iterate through set of filtered filenames. Delete all files greater than retention count.
+            unsigned int i = 0;
+            for (auto const& iter : SortedDirEntries)
+            {
+                if (i >= nRetention)
+                {
+                    fs::remove(iter.path());
+
+                    LogPrintf("INFO: Logger: Removed old archive gzip file %s.", iter.path().filename().string());
+                }
+
+                ++i;
+            }
+        }
+
+        return true; // archive condition was satisfied. Return true after rotating and archiving.
+    }
+    else
+    {
+        return false; // archive condition was not satisfied. Do nothing and return false.
+    }
+}
+

src/logging.h

@@ -16,6 +16,10 @@
 #include <string>
 #include <vector>
 
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/date_time/gregorian/gregorian.hpp>
+#include <boost/date_time/gregorian/greg_date.hpp>
+
 static const bool DEFAULT_LOGTIMEMICROS = false;
 static const bool DEFAULT_LOGIPS        = false;
 static const bool DEFAULT_LOGTIMESTAMPS = true;
@@ -24,6 +28,9 @@ extern const char* const DEFAULT_DEBUGLOGFILE;
 
 extern bool fLogIPs;
 
+// Unavoidable because this is in util.h.
+extern int64_t GetAdjustedTime();
+
 struct CLogCategoryActive
 {
     std::string category;
@@ -90,6 +97,7 @@ namespace BCLog {
 
         fs::path m_file_path;
         std::atomic<bool> m_reopen_file{false};
+        static boost::gregorian::date PrevArchiveCheckDate;
 
         /** Send a string to the log output */
         void LogPrintStr(const std::string& str);
@@ -123,6 +131,8 @@ namespace BCLog {
 
         void ShrinkDebugFile();
 
+        bool archive(bool fImmediate, fs::path pfile_out);
+
         uint32_t GetCategoryMask() const { return m_categories.load(); }
 
         void EnableCategory(LogFlags flag);

src/rpcserver.cpp

@@ -396,7 +396,7 @@ static const CRPCCommand vRPCCommands[] =
     { "sendscraperfilemanifest", &sendscraperfilemanifest, cat_developer     },
     { "savescraperfilemanifest", &savescraperfilemanifest, cat_developer     },
     { "deletecscrapermanifest",  &deletecscrapermanifest,  cat_developer     },
-    { "archivescraperlog",       &archivescraperlog,       cat_developer     },
+    { "archivelog",              &archivelog,              cat_developer     },
     { "testnewsb",               &testnewsb,               cat_developer     },
 
   // Network commands

src/rpcserver.h

@@ -215,7 +215,7 @@ extern UniValue getmpart(const UniValue& params, bool fHelp);
 extern UniValue sendscraperfilemanifest(const UniValue& params, bool fHelp);
 extern UniValue savescraperfilemanifest(const UniValue& params, bool fHelp);
 extern UniValue deletecscrapermanifest(const UniValue& params, bool fHelp);
-extern UniValue archivescraperlog(const UniValue& params, bool fHelp);
+extern UniValue archivelog(const UniValue& params, bool fHelp);
 extern UniValue testnewsb(const UniValue& params, bool fHelp);
 
 // Network