From a188a8f5b75d8e501f7ab145e99b439ae7d17896 Mon Sep 17 00:00:00 2001
From: "s.zharko@gsi.de" <s.zharko@gsi.de>
Date: Tue, 12 Nov 2024 16:43:14 +0100
Subject: [PATCH] QA-checker: bugfix + reducing memory consumption
---
core/qa/checker/CbmQaCheckerCore.cxx | 33 +++++++++++----------
core/qa/checker/CbmQaCheckerCore.h | 5 +++-
core/qa/checker/CbmQaCheckerFileHandler.cxx | 28 ++++++++++++++---
core/qa/checker/CbmQaCheckerFileHandler.h | 7 +++--
core/qa/checker/CbmQaCheckerObjectDB.cxx | 7 ++++-
core/qa/checker/CbmQaCheckerObjectDB.h | 8 +++++
macro/qa/qa_compare_ca.C | 2 +-
7 files changed, 65 insertions(+), 25 deletions(-)
diff --git a/core/qa/checker/CbmQaCheckerCore.cxx b/core/qa/checker/CbmQaCheckerCore.cxx
index 546d7bb68a..3125a699e1 100644
--- a/core/qa/checker/CbmQaCheckerCore.cxx
+++ b/core/qa/checker/CbmQaCheckerCore.cxx
@@ -13,7 +13,6 @@
#include "Logger.h"
#include "TClonesArray.h"
#include "TFile.h"
-#include "TFolder.h"
//#include <boost/filesystem.hpp>
#include <regex>
@@ -40,7 +39,7 @@ void Core::AddDataset(const char* dataset) { fpObjDB->AddDataset(dataset); }
void Core::RegisterOutFile(const char* filename)
{
LOG(info) << "Core: Registering output file: " << filename;
- fpOutFile = std::make_unique<TFile>(filename, "RECREATE");
+ fpObjDB->SetOutputPath(filename);
}
// ---------------------------------------------------------------------------------------------------------------------
@@ -50,27 +49,15 @@ void Core::Process(Option_t* opt)
// ----- Init the object database
fpObjDB->Init();
- // ----- Register output file
- if (fpOutFile == nullptr) {
- this->RegisterOutFile("QaCheckerOutput.root");
- }
+ PrepareOutputFile();
// ----- Process datasets and files
int nDatasets = fpObjDB->GetNofDatasets();
- std::vector<TFolder*> vDSFolders(nDatasets, nullptr);
-
for (int iDS = 0; iDS < nDatasets; ++iDS) {
- // Create a new folder for dataset (/dataset)
- auto* pDSDir = fpOutFile->mkdir(fpObjDB->GetDataset(iDS).data());
-
- // Loop over files
for (int iFile = 0; iFile < fpObjDB->GetNofFiles(); ++iFile) {
- // Define output folder for file (/dataset/file)
- auto* pFileDir = pDSDir->mkdir(fpObjDB->GetFileLabel(iFile).data());
-
// Create and process a file handler
- auto pFileHandler = std::make_unique<FileHandler>(fpObjDB, pFileDir, iDS, iFile);
+ auto pFileHandler = std::make_unique<FileHandler>(fpObjDB, iDS, iFile);
pFileHandler->Process(opt);
} // iFile
} // iDS
@@ -80,6 +67,20 @@ void Core::Process(Option_t* opt)
//
void Core::SetFromYAML(const char* configName) { fpObjDB->ReadFromYAML(configName); }
+// ---------------------------------------------------------------------------------------------------------------------
+//
+void Core::PrepareOutputFile()
+{
+ TFile outFile(fpObjDB->GetOutputPath().c_str(), "RECREATE");
+ for (int iDS = 0; iDS < fpObjDB->GetNofDatasets(); ++iDS) {
+ auto* pDSDir = outFile.mkdir(fpObjDB->GetDataset(iDS).c_str());
+ for (int iFile = 0; iFile < fpObjDB->GetNofFiles(); ++iFile) {
+ pDSDir->mkdir(fpObjDB->GetFileLabel(iFile).c_str());
+ }
+ }
+ outFile.Close();
+}
+
// ---------------------------------------------------------------------------------------------------------------------
//
bool Core::Scan() const
diff --git a/core/qa/checker/CbmQaCheckerCore.h b/core/qa/checker/CbmQaCheckerCore.h
index 5390e4f387..7dae9e67d0 100644
--- a/core/qa/checker/CbmQaCheckerCore.h
+++ b/core/qa/checker/CbmQaCheckerCore.h
@@ -98,7 +98,10 @@ namespace cbm::qa::checker
void SetInputRootPath(const char* pathName) { fpObjDB->SetInputRootPath(pathName); }
private:
- std::shared_ptr<TFile> fpOutFile = nullptr; ///< Output file
+ /// @brief Prepares output file (creates directory structure)
+ void PrepareOutputFile();
+
+
std::shared_ptr<ObjectDB> fpObjDB = nullptr; ///< Database of names
FlagBitSet_t fControlBits; ///< Control bit flags
diff --git a/core/qa/checker/CbmQaCheckerFileHandler.cxx b/core/qa/checker/CbmQaCheckerFileHandler.cxx
index dd799211d5..a1517d6fbb 100644
--- a/core/qa/checker/CbmQaCheckerFileHandler.cxx
+++ b/core/qa/checker/CbmQaCheckerFileHandler.cxx
@@ -34,12 +34,12 @@ using cbm::qa::checker::FileHandler;
// ---------------------------------------------------------------------------------------------------------------------
//
-FileHandler::FileHandler(std::shared_ptr<ObjectDB>& pObjDB, TDirectory* pOutDir, int iDataset, int iFile)
+FileHandler::FileHandler(std::shared_ptr<ObjectDB>& pObjDB, int iDataset, int iFile)
: fFileID(iFile)
, fDatasetID(iDataset)
, fpObjDB(pObjDB)
- , fpOutDir(pOutDir)
{
+ ReOpenOutputFile();
fpInputFiles = std::make_unique<TClonesArray>("TFile");
for (int iVer = 0; iVer < fpObjDB->GetNofVersions(); ++iVer) {
@@ -63,6 +63,8 @@ FileHandler::~FileHandler()
// ----- Clean pointers
fpInputFiles->Delete();
fpInputFiles = nullptr;
+ fpOutputFile->Close();
+ fpOutputFile = nullptr;
}
// ---------------------------------------------------------------------------------------------------------------------
@@ -114,8 +116,12 @@ void FileHandler::Process(Option_t* opt)
}
// Create an instance of an object handler
+ if (iObj != 0 && (20 * iObj) % nObjects == 0) {
+ ReOpenOutputFile();
+ LOG(info) << "FileHandler: processing object " << iObj << " of " << nObjects;
+ }
std::unique_ptr<ObjectHandler> pObjHandler = nullptr;
- //LOG(info) << "FileHandler: processing object \"" << vpObjects[0]->GetName() << '\"';
+ LOG(info) << "FileHandler: processing object \"" << vpObjects[0]->GetName() << '\"';
if (dynamic_cast<TH2*>(vpObjects[0])) {
pObjHandler = std::make_unique<Hist2DHandler>(iObj, fFileID, fDatasetID);
if (bCmpPointByPoint) {
@@ -170,6 +176,7 @@ void FileHandler::Process(Option_t* opt)
}
}
pObjHandler->Write();
+ gFile->Flush();
// Clean memory
for (auto* pObj : vpObjects) {
@@ -178,6 +185,19 @@ void FileHandler::Process(Option_t* opt)
pObj = nullptr;
}
}
-
} // iObj
}
+
+// ---------------------------------------------------------------------------------------------------------------------
+//
+void FileHandler::ReOpenOutputFile()
+{
+ if (fpOutputFile.get()) {
+ fpOutputFile->Close();
+ fpOutputFile = nullptr;
+ fpOutDir = nullptr;
+ }
+ fpOutputFile = std::make_unique<TFile>(fpObjDB->GetOutputPath().c_str(), "UPDATE");
+ TString dirName = fpObjDB->GetDataset(fDatasetID) + "/" + fpObjDB->GetFileLabel(fFileID);
+ fpOutDir = fpOutputFile->Get<TDirectory>(dirName);
+}
diff --git a/core/qa/checker/CbmQaCheckerFileHandler.h b/core/qa/checker/CbmQaCheckerFileHandler.h
index 580fae408f..c8128b0f2c 100644
--- a/core/qa/checker/CbmQaCheckerFileHandler.h
+++ b/core/qa/checker/CbmQaCheckerFileHandler.h
@@ -29,10 +29,9 @@ namespace cbm::qa::checker
public:
/// @brief Constructor
/// @param pObjDB Shared pointer to object database
- /// @param pOutDir Directory to store the output (/dataset/file)
/// @param iDataset Index of dataset
/// @param iFile Index of file
- FileHandler(std::shared_ptr<ObjectDB>& pObjDB, TDirectory* pOutDir, int iDataset, int iFile);
+ FileHandler(std::shared_ptr<ObjectDB>& pObjDB, int iDataset, int iFile);
/// @brief Destructor
~FileHandler();
@@ -71,12 +70,16 @@ namespace cbm::qa::checker
/// @return Pointer to created TDirectory object
TDirectory* CreateNestedDirectory(const std::string& path);
+ /// @brief Closes and opens output file
+ void ReOpenOutputFile();
+
int fFileID = -1; ///< Index of file
int fDatasetID = -1; ///< Index of dataset
std::shared_ptr<ObjectDB> fpObjDB = nullptr; ///< Pointer to object database
TDirectory* fpOutDir = nullptr; ///< Pointer to output directory
std::unique_ptr<TClonesArray> fpInputFiles = nullptr; ///< Pointer to input files array
+ std::unique_ptr<TFile> fpOutputFile = nullptr; ///< Pointer to output file
// TODO: replace fpOutputFolder with shared_ptr
};
diff --git a/core/qa/checker/CbmQaCheckerObjectDB.cxx b/core/qa/checker/CbmQaCheckerObjectDB.cxx
index 44f4ed3d9d..87853e20d7 100644
--- a/core/qa/checker/CbmQaCheckerObjectDB.cxx
+++ b/core/qa/checker/CbmQaCheckerObjectDB.cxx
@@ -71,6 +71,11 @@ void ObjectDB::Init()
<< GetNofVersions() << " were provided)";
+ // ----- Define output file
+ if (fsOutputPath.empty()) {
+ fsOutputPath = "QaCheckerOutput.root";
+ }
+
// ----- Define default version index
if (fsDefaultLabel.size()) {
auto it = std::find(fvVersionLabels.cbegin(), fvVersionLabels.cend(), fsDefaultLabel);
@@ -249,7 +254,7 @@ void ObjectDB::ReadObjectList(int iFile)
std::set<std::string> objectPaths;
LOG(info) << "Reading object list from files: ...";
for (int iDs = 0; iDs < static_cast<int>(fvDatasets.size()); ++iDs) {
- const char* fileName = this->GetInputFileName(fDefVersionID, iFile, iDs).c_str();
+ TString fileName = this->GetInputFileName(fDefVersionID, iFile, iDs);
LOG(info) << "- file: " << fileName;
TFile fileIn{fileName, "READONLY"};
fileIn.cd();
diff --git a/core/qa/checker/CbmQaCheckerObjectDB.h b/core/qa/checker/CbmQaCheckerObjectDB.h
index dc73e1e4fd..ffb3d01706 100644
--- a/core/qa/checker/CbmQaCheckerObjectDB.h
+++ b/core/qa/checker/CbmQaCheckerObjectDB.h
@@ -122,6 +122,9 @@ namespace cbm::qa::checker
/// @brief Gets number of versions
int GetNofVersions() const { return fvVersionLabels.size(); }
+ /// @brief Gets output path
+ const std::string& GetOutputPath() const { return fsOutputPath; }
+
/// @brief Gets version label
/// @param iVersion Index of version
const std::string& GetVersionLabel(int iVersion) const { return fvVersionLabels[iVersion]; }
@@ -165,6 +168,10 @@ namespace cbm::qa::checker
/// @param pathName Relative or absolute root path to input the input directories
void SetInputRootPath(const char* pathName) { fsInputRootPath = pathName; }
+ /// @brief Sets the output path
+ /// @param path Path to the output ROOT-file
+ void SetOutputPath(const char* path) { fsOutputPath = path; }
+
private:
/// @brief Reads list of histograms from file
/// @param iFile Index of file
@@ -180,6 +187,7 @@ namespace cbm::qa::checker
int fDefVersionID = -1; ///< Index of default version
std::string fsInputRootPath = ""; ///< Root path for input files
+ std::string fsOutputPath = ""; ///< Path to the output file
std::string fsDefaultLabel = ""; ///< Name of default version label
std::vector<std::string> fvDatasets; ///< Container of dataset names
diff --git a/macro/qa/qa_compare_ca.C b/macro/qa/qa_compare_ca.C
index 04541db7dc..ddb750158a 100644
--- a/macro/qa/qa_compare_ca.C
+++ b/macro/qa/qa_compare_ca.C
@@ -36,7 +36,7 @@ int qa_compare_ca(
pQaChecker->SetFromYAML(configName); // Read file-object map
//// ----- Run comparision routine
- pQaChecker->Process("PRB"); // P -
+ pQaChecker->Process("PRC"); // P -
//// ----- Scan results
bool res = pQaChecker->Scan(); // true - objects are the same, false - objects differ
--
GitLab