From c77b57049baac80302eb02e88bdd054c0a987ecd Mon Sep 17 00:00:00 2001
From: Felix Weiglhofer <weiglhofer@fias.uni-frankfurt.de>
Date: Tue, 16 Apr 2024 10:27:17 +0000
Subject: [PATCH] algo: Add MemoryLogger to log memory usage of process.
---
algo/CMakeLists.txt | 3 +++
algo/base/System.cxx | 46 ++++++++++++++++++++++++++++++++
algo/base/System.h | 31 ++++++++++++++++++++++
algo/base/util/MemoryLogger.cxx | 32 ++++++++++++++++++++++
algo/base/util/MemoryLogger.h | 47 +++++++++++++++++++++++++++++++++
reco/app/cbmreco/main.cxx | 8 ++++++
6 files changed, 167 insertions(+)
create mode 100644 algo/base/System.cxx
create mode 100644 algo/base/System.h
create mode 100644 algo/base/util/MemoryLogger.cxx
create mode 100644 algo/base/util/MemoryLogger.h
diff --git a/algo/CMakeLists.txt b/algo/CMakeLists.txt
index 85b74016e4..7cbf4a011a 100644
--- a/algo/CMakeLists.txt
+++ b/algo/CMakeLists.txt
@@ -74,6 +74,8 @@ set(SRCS
base/Options.cxx
base/MainConfig.cxx
base/RecoParams.cxx
+ base/System.cxx
+ base/util/MemoryLogger.cxx
base/util/StlUtils.cxx
base/util/EnumDict.cxx
base/util/TimingsFormat.cxx
@@ -248,6 +250,7 @@ install(
base/Options.h
base/RecoParams.h
base/SubChain.h
+ base/System.h
base/PartitionedVector.h
base/PartitionedSpan.h
global/Reco.h
diff --git a/algo/base/System.cxx b/algo/base/System.cxx
new file mode 100644
index 0000000000..88bf1ac516
--- /dev/null
+++ b/algo/base/System.cxx
@@ -0,0 +1,46 @@
+/* Copyright (C) 2024 FIAS Frankfurt Institute for Advanced Studies, Frankfurt / Main
+ SPDX-License-Identifier: GPL-3.0-only
+ Authors: Felix Weiglhofer [committer] */
+
+#include "System.h"
+
+#include <cstdio>
+
+#ifdef __linux__
+#include <sys/resource.h>
+#include <unistd.h>
+#endif
+
+
+size_t cbm::algo::GetCurrentRSS()
+{
+ // Implementation copied from https://stackoverflow.com/a/14927379
+#ifndef __linux__
+ return 0;
+#else
+ unsigned long rss = 0L;
+ FILE* fp = nullptr;
+ if ((fp = fopen("/proc/self/statm", "r")) == nullptr) {
+ return size_t(0L); /* Can't open? */
+ }
+ if (fscanf(fp, "%*s%lu", &rss) != 1) {
+ fclose(fp);
+ return size_t(0L); /* Can't read? */
+ }
+ fclose(fp);
+ return size_t(rss) * size_t(sysconf(_SC_PAGESIZE));
+#endif
+}
+
+size_t cbm::algo::GetPeakRSS()
+{
+ // Implementation copied from https://stackoverflow.com/a/14927379
+#ifndef __linux__
+ return 0;
+#else
+ struct rusage rusage;
+ getrusage(RUSAGE_SELF, &rusage);
+
+ return size_t(rusage.ru_maxrss * 1024L);
+#endif
+}
diff --git a/algo/base/System.h b/algo/base/System.h
new file mode 100644
index 0000000000..3c415ba131
--- /dev/null
+++ b/algo/base/System.h
@@ -0,0 +1,31 @@
+/* Copyright (C) 2024 FIAS Frankfurt Institute for Advanced Studies, Frankfurt / Main
+ SPDX-License-Identifier: GPL-3.0-only
+ Authors: Felix Weiglhofer [committer] */
+
+#pragma once
+
+#include <cstddef>
+
+/**
+ * @file System.h
+ * @brief System functions
+**/
+
+namespace cbm::algo
+{
+
+ /**
+ * @brief Get the current resident set size (pyhysical memory usage) of the process
+ * @return The current resident set size in bytes
+ * @note Returns zero if the value cannot be determined
+ **/
+ size_t GetCurrentRSS();
+
+ /**
+ * @brief Get the peak resident set size (pyhysical memory usage) of the process
+ * @return The peak resident set size in bytes
+ * @note Returns zero if the value cannot be determined
+ **/
+ size_t GetPeakRSS();
+
+} // namespace cbm::algo
diff --git a/algo/base/util/MemoryLogger.cxx b/algo/base/util/MemoryLogger.cxx
new file mode 100644
index 0000000000..61c4777219
--- /dev/null
+++ b/algo/base/util/MemoryLogger.cxx
@@ -0,0 +1,32 @@
+/* Copyright (C) 2024 FIAS Frankfurt Institute for Advanced Studies, Frankfurt / Main
+ SPDX-License-Identifier: GPL-3.0-only
+ Authors: Felix Weiglhofer [committer] */
+
+#include "MemoryLogger.h"
+
+#include "System.h"
+
+#include <log.hpp>
+
+using namespace cbm::algo;
+
+template<typename T>
+T MemoryLogger::BytesToMB(T bytes) const
+{
+ return bytes / (1024 * 1024);
+}
+
+void MemoryLogger::Log()
+{
+ size_t currentRSS = GetCurrentRSS();
+ size_t peakRSS = GetPeakRSS();
+
+ ptrdiff_t deltaRSS = currentRSS - mLastRSS;
+ float deltaPercent = 100.0f * deltaRSS / currentRSS;
+
+ L_(debug) << "Current memory usage: " << BytesToMB(currentRSS) << "MB (delta " << BytesToMB(deltaRSS) << "MB / "
+ << deltaPercent << "%)"
+ << ", peak: " << BytesToMB(peakRSS) << "MB";
+
+ mLastRSS = currentRSS;
+}
diff --git a/algo/base/util/MemoryLogger.h b/algo/base/util/MemoryLogger.h
new file mode 100644
index 0000000000..5b732ccd96
--- /dev/null
+++ b/algo/base/util/MemoryLogger.h
@@ -0,0 +1,47 @@
+/* Copyright (C) 2024 FIAS Frankfurt Institute for Advanced Studies, Frankfurt / Main
+ SPDX-License-Identifier: GPL-3.0-only
+ Authors: Felix Weiglhofer [committer] */
+
+#pragma once
+
+#include <cstddef>
+
+/**
+ * @file MemoryLogger.h
+ * @brief Memory logging
+**/
+
+namespace cbm::algo
+{
+
+ /**
+ * @brief Track the memory usage of the process and write it to the log
+ **/
+ class MemoryLogger {
+
+ public:
+ /**
+ * @brief Constructor
+ **/
+ MemoryLogger() = default;
+
+ /**
+ * @brief Destructor
+ **/
+ ~MemoryLogger() = default;
+
+ /**
+ * @brief Log the current memory usage
+ **/
+ void Log();
+
+ private:
+ size_t mLastRSS = 0;
+
+ // Convert bytes to MB
+ // Template to allow for different integer types
+ template<typename T>
+ T BytesToMB(T bytes) const;
+ };
+
+} // namespace cbm::algo
diff --git a/reco/app/cbmreco/main.cxx b/reco/app/cbmreco/main.cxx
index 21ef274235..b078661f1a 100644
--- a/reco/app/cbmreco/main.cxx
+++ b/reco/app/cbmreco/main.cxx
@@ -8,8 +8,10 @@
#include "Reco.h"
#include "RecoResultsInputArchive.h"
#include "RecoResultsOutputArchive.h"
+#include "System.h"
#include "compat/OpenMP.h"
#include "gpu/DeviceImage.h"
+#include "util/MemoryLogger.h"
#include <TimesliceAutoSource.hpp>
@@ -160,6 +162,7 @@ int main(int argc, char** argv)
if (dumpArchive(opts)) return 0;
Reco reco;
+ MemoryLogger memoryLogger;
auto startProcessing = std::chrono::high_resolution_clock::now();
reco.Init(opts);
@@ -198,6 +201,11 @@ int main(int argc, char** argv)
L_(error) << "Caught ProcessingError while processing timeslice " << tsIdx << ": " << e.what();
}
+ // Release memory after each timeslice and log memory usage
+ // This is useful to detect memory leaks as the memory usage should be constant between timeslices
+ ts.reset();
+ memoryLogger.Log();
+
tsIdx++;
if (num_ts > 0 && tsIdx >= num_ts) break;
--
GitLab