diff --git a/core/qa/checker/CbmQaCheckerCore.cxx b/core/qa/checker/CbmQaCheckerCore.cxx
index 2f832c1bc4231d6108146a84722042aaeaf4412f..193648d1f32b08f6a00eee39e5a19c8650d3317b 100644
--- a/core/qa/checker/CbmQaCheckerCore.cxx
+++ b/core/qa/checker/CbmQaCheckerCore.cxx
@@ -44,23 +44,46 @@ void Core::RegisterOutFile(const char* filename)
// ---------------------------------------------------------------------------------------------------------------------
//
-void Core::Process(Option_t* opt)
+int Core::Process(Option_t* opt)
{
// ----- Init the object database
fpObjDB->Init();
PrepareOutputFile();
- // ----- Process datasets and files
int nDatasets = fpObjDB->GetNofDatasets();
+ int nFiles = fpObjDB->GetNofFiles();
+ int nVersions = fpObjDB->GetNofVersions();
+
+ std::vector<ECmpInference> cmpSummary(nVersions * nFiles * nDatasets, ECmpInference::StronglyEqual);
+ // ----- Process datasets and files
for (int iDS = 0; iDS < nDatasets; ++iDS) {
- for (int iFile = 0; iFile < fpObjDB->GetNofFiles(); ++iFile) {
+ for (int iFile = 0; iFile < nFiles; ++iFile) {
// Create and process a file handler
auto pFileHandler = std::make_unique<FileHandler>(fpObjDB, iDS, iFile);
- pFileHandler->Process(opt);
+ auto cmpRes = pFileHandler->Process(opt);
+ for (int iVer = 0; iVer < nVersions; ++iVer) {
+ cmpSummary[nVersions * (iDS * nFiles + iFile) + iVer] = cmpRes[iVer];
+ }
} // iFile
} // iDS
+
+ ECmpInference res = ECmpInference::StronglyEqual;
+ LOG(info) << "Summary:";
+ for (int iDS{0}; iDS < nDatasets; ++iDS) {
+ LOG(info) << "\tDataset: " << fpObjDB->GetDataset(iDS);
+ for (int iFile{0}; iFile < nFiles; ++iFile) {
+ LOG(info) << "\t\tFile: " << fpObjDB->GetFileLabel(iDS);
+ for (int iVer{0}; iVer < nVersions; ++iVer) {
+ auto versionInference = cmpSummary[nVersions * (iDS * nFiles + iFile) + iVer];
+ LOG(info) << "\t\t\tVersion: " << fpObjDB->GetVersionLabel(iVer)
+ << " (path: " << fpObjDB->GetInputFileName(iVer, iFile, iDS) << "): " << ToString(versionInference);
+ res = std::max(versionInference, res);
+ }
+ }
+ }
+ return static_cast<int>(res);
}
// ---------------------------------------------------------------------------------------------------------------------
@@ -80,52 +103,3 @@ void Core::PrepareOutputFile()
}
outFile.Close();
}
-
-// ---------------------------------------------------------------------------------------------------------------------
-//
-bool Core::Scan() const
-{
- LOG(info) << "Core: Scanning comparison result ...";
- bool res = true; /// Equal
- for (int iFile = 0; iFile < fpObjDB->GetNofFiles(); ++iFile) {
- for (int iObj = 0; iObj < fpObjDB->GetNofObjects(iFile); ++iObj) {
- for (int iDS = 0; iDS < fpObjDB->GetNofDatasets(); ++iDS) {
- for (int iVer = 0; iVer < fpObjDB->GetNofVersions(); ++iVer) {
- // For now we check, if anything has been changed with respect to default. Later one can add a flag to test
- // compatibility for different levels (for example, two histograms could be obtained from different systems,
- // or different seeds, but still be consistent).
- const auto& cmp = fpObjDB->GetCmpResult(iDS, iFile, iObj, iVer);
- bool bEqual = true;
- {
- std::stringstream msg;
- constexpr const char* kEqual = "\e[1;32mequal\e[0m";
- constexpr const char* kDiff = "\e[1;32mdifferent\e[0m";
- msg << "Found mismatch: file: " << fpObjDB->GetInputFileName(iVer, iFile, iDS)
- << ", object: " << fpObjDB->GetObject(iFile, iObj);
- if (cmp.fbPointByPointTested) {
- msg << "\tpoint-by-point -> " << (cmp.fbPointByPoint ? kEqual : kDiff);
- bEqual &= cmp.fbPointByPoint;
- }
- if (cmp.fbRatioTested) {
- bool bOk = (cmp.fRatioLo >= kRatioMin && cmp.fRatioUp <= kRatioMax);
- msg << "\tratio -> " << (bOk ? kEqual : kDiff) << "(lo: " << cmp.fRatioLo << ", up: " << cmp.fRatioUp
- << ')';
- bEqual &= bOk;
- }
- if (cmp.fbChi2Tested) {
- bool bOk = cmp.fPval >= kPvalThrsh;
- msg << "\tchi2 test -> " << (bOk ? kEqual : kDiff) << "(p-val: " << cmp.fPval << ')';
- bEqual &= bOk;
- }
- if (!bEqual) {
- LOG(info) << msg.str();
- }
- }
- } // iVer
- } // iDS
- } // iObj
- } // iFile
- LOG(info) << "Core: Scanning comparison result ... done";
-
- return res;
-}
diff --git a/core/qa/checker/CbmQaCheckerCore.h b/core/qa/checker/CbmQaCheckerCore.h
index a86d2c29490d07698ccaf466f17d0e86e526e8d0..d15a1c5310b3ab5702dc5cab3acb20b6595957ed 100644
--- a/core/qa/checker/CbmQaCheckerCore.h
+++ b/core/qa/checker/CbmQaCheckerCore.h
@@ -66,23 +66,15 @@ namespace cbm::qa::checker
/// "P": enables bin-by-bin comparison
/// "S": enables statistical hypothesis test, where is possible
/// "U": enables interval comparison, where is possible
- void Process(Option_t* opt = "P");
+ /// @return 0: All versions are identical
+ /// @return 1: Some checks for some histograms did not pass, but the histograms are consistent
+ /// @return 2: Some histograms are different
+ int Process(Option_t* comparisonMethod = "P");
/// @brief Registers root-file for storing output
/// @param filename Name of file
void RegisterOutFile(const char* filename);
- /// @brief Scans comparison results
- /// @return Comparison flag:
- /// - true: All histograms are the same under conditions
- /// - false: Some of the histograms were changed
- bool Scan() const;
-
- /// @brief Sets control flag
- /// @param flag Flag label
- /// @param value Flag value
- void SetControlBitFlag(EFlagBit flag, bool value = true) { fControlBits[static_cast<int>(flag)] = value; }
-
/// @brief Sets default version label
/// @param defaultLabel Name of default label
///
@@ -97,18 +89,20 @@ namespace cbm::qa::checker
/// @param pathName Relative or absolute root path to input the input directories
void SetInputRootPath(const char* pathName) { fpObjDB->SetInputRootPath(pathName); }
+ /// @brief Sets P-value threshold
+ /// @param pVal P-value threshold
+ void SetPvalThreshold(float pVal) { fpObjDB->SetPvalThreshold(pVal); }
+
+ /// @brief Sets ratio accepted range
+ /// @param min Lower boundary
+ /// @param max Upper boundary
+ void SetRatioRange(float min, float max) { fpObjDB->SetRatioRange(min, max); }
+
private:
/// @brief Prepares output file (creates directory structure)
void PrepareOutputFile();
- // TODO: provide setter/config entry
- static constexpr float kRatioMin = 0.95;
- static constexpr float kRatioMax = 1.05;
- static constexpr float kPvalThrsh = 0.05;
-
std::shared_ptr<ObjectDB> fpObjDB = nullptr; ///< Database of names
-
- FlagBitSet_t fControlBits; ///< Control bit flags
};
} // namespace cbm::qa::checker
diff --git a/core/qa/checker/CbmQaCheckerFileHandler.cxx b/core/qa/checker/CbmQaCheckerFileHandler.cxx
index b5cb57de0da0ce5395cf519e05cc609b7a0a4683..5852beb3b73657ccbe59b53ed952df09fa58c02c 100644
--- a/core/qa/checker/CbmQaCheckerFileHandler.cxx
+++ b/core/qa/checker/CbmQaCheckerFileHandler.cxx
@@ -30,6 +30,7 @@
#include <string>
+using cbm::qa::checker::ECmpInference;
using cbm::qa::checker::FileHandler;
// ---------------------------------------------------------------------------------------------------------------------
@@ -77,15 +78,17 @@ TDirectory* FileHandler::CreateNestedDirectory(const std::string& path)
// ---------------------------------------------------------------------------------------------------------------------
//
-void FileHandler::Process(Option_t* opt)
+std::vector<ECmpInference> FileHandler::Process(Option_t* opt)
{
+ std::vector<ECmpInference> bCmpResult;
int nObjects = fpObjDB->GetNofObjects(fFileID);
int nVersions = fpObjDB->GetNofVersions();
+ bCmpResult.resize(nVersions, ECmpInference::StronglyEqual);
// ----- Initial checks
if (!nObjects) {
LOG(warn) << "FileHandler: No objects were passed to file \"" << nObjects << ". Skipping file";
- return;
+ return bCmpResult;
}
// ----- Option parsing
@@ -94,12 +97,12 @@ void FileHandler::Process(Option_t* opt)
ch = std::tolower(ch);
}
bool bSuppressCanvases = sOption.find("b") != std::string::npos;
- //bool bForceCanvases = sOption.find("c") != std::string::npos;
- bool bCmpPointByPoint = sOption.find("p") != std::string::npos;
- bool bCmpStatAny = sOption.find("s") != std::string::npos;
+ bool bForceCanvases = sOption.find("c") != std::string::npos;
+ bool bCmpExact = sOption.find("p") != std::string::npos;
+ bool bCmpChi2 = sOption.find("s") != std::string::npos;
bool bCmpRatio = sOption.find("u") != std::string::npos;
- LOG(info) << "FileHandler: processing objects: ";
+ LOG(info) << "FileHandler: processing objects: ...";
std::vector<TNamed*> vpObjects(nVersions, nullptr); // vector to keep object different versions
for (int iObj = 0; iObj < nObjects; ++iObj) {
bool skipObj = false;
@@ -107,8 +110,8 @@ void FileHandler::Process(Option_t* opt)
auto* pInputFile = static_cast<TFile*>(fpInputFiles->At(iVer));
vpObjects[iVer] = pInputFile->Get<TNamed>(fpObjDB->GetObject(fFileID, iObj).c_str());
if (!vpObjects[iVer]) {
- LOG(warn) << "FileHandler: object " << fpObjDB->GetObject(fFileID, iObj) << " is undefined for version "
- << fpObjDB->GetVersionLabel(iVer) << ". This object will be skipped";
+ //LOG(warn) << "FileHandler: object " << fpObjDB->GetObject(fFileID, iObj) << " is undefined for version "
+ // << fpObjDB->GetVersionLabel(iVer) << ". This object will be skipped";
skipObj = true;
}
} // iVer
@@ -116,13 +119,15 @@ void FileHandler::Process(Option_t* opt)
continue;
}
- // Create an instance of an object handler
- if (iObj != 0 && (20 * iObj) % nObjects == 0) {
+ //
+ if (iObj != 0 && nObjects % 500 == 0) {
ReOpenOutputFile();
- LOG(info) << "FileHandler: processing object " << iObj << " of " << nObjects;
+ LOG(info) << "FileHandler: object " << iObj << " / " << nObjects;
}
+
+ // Create an instance of an object handler
std::unique_ptr<ObjectHandler> pObjHandler = nullptr;
- LOG(info) << "FileHandler: processing object \"" << vpObjects[0]->GetName() << '\"';
+ //LOG(info) << "FileHandler: processing object \"" << vpObjects[0]->GetName() << '\"';
if (dynamic_cast<TProfile*>(vpObjects[0])) {
pObjHandler = std::make_unique<Profile1DHandler>(iObj, fFileID, fDatasetID);
}
@@ -138,37 +143,29 @@ void FileHandler::Process(Option_t* opt)
<< "\", which is unknown to the cbm::qa::checker framework, it will be skipped";
continue;
}
- if (bCmpPointByPoint) {
- pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kPOINT); // Compare point-by-point (exact equality)
+ if (bCmpExact) {
+ pObjHandler->SetComparisonMethod(ECmpMethod::Exact); // Compare point-by-point (exact equality)
}
- if (bCmpStatAny) {
- pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kCHI2); // Compare with chi2 test
+ if (bCmpChi2) {
+ pObjHandler->SetComparisonMethod(ECmpMethod::Chi2); // Compare with chi2 test
}
if (bCmpRatio) {
- pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kRATIO);
+ pObjHandler->SetComparisonMethod(ECmpMethod::Ratio);
}
pObjHandler->SetObjectDB(fpObjDB);
pObjHandler->SetOutputDirectory(CreateNestedDirectory((fpObjDB->GetObject(fFileID, iObj))));
pObjHandler->AddObjects(vpObjects);
- pObjHandler->CompareWithDefault();
+ auto cmpResultObject = pObjHandler->CompareWithDefault();
+ for (int iVer = 0; iVer < nVersions; ++iVer) {
+ bCmpResult[iVer] = std::max(cmpResultObject[iVer], bCmpResult[iVer]);
+ }
- // Create comparison canvas
- if (!bSuppressCanvases) {
+ if (!bSuppressCanvases
+ && (bForceCanvases || std::any_of(cmpResultObject.begin(), cmpResultObject.end(), [](auto i) {
+ return i != ECmpInference::StronglyEqual;
+ }))) {
pObjHandler->CreateCanvases(opt);
}
-
- // TODO: re-work the suppress and force canvas features
- //if (!bSuppressCanvases) {
- // bool areDifferent = false;
- // for (int iVer = 0; iVer < nVersions; ++iVer) {
- // if (fpObjDB->GetCmpResult(fDatasetID, fFileID, iObj, iVer)) {
- // areDifferent = true;
- // }
- // }
- // if (areDifferent || bForceCanvases) {
- // pObjHandler->CreateCanvases(opt);
- // }
- //}
pObjHandler->Write();
gFile->Flush();
@@ -180,6 +177,8 @@ void FileHandler::Process(Option_t* opt)
}
}
} // iObj
+ LOG(info) << "FileHandler: processing objects: done";
+ return bCmpResult;
}
// ---------------------------------------------------------------------------------------------------------------------
@@ -187,6 +186,7 @@ void FileHandler::Process(Option_t* opt)
void FileHandler::ReOpenOutputFile()
{
if (fpOutputFile.get()) {
+ fpOutputFile->Flush();
fpOutputFile->Close();
fpOutputFile = nullptr;
fpOutDir = nullptr;
diff --git a/core/qa/checker/CbmQaCheckerFileHandler.h b/core/qa/checker/CbmQaCheckerFileHandler.h
index c8128b0f2ced4c4ceeffce721dafdea78d0a5011..ca678bcbcb8c044ce26959828175ba8563ffd635 100644
--- a/core/qa/checker/CbmQaCheckerFileHandler.h
+++ b/core/qa/checker/CbmQaCheckerFileHandler.h
@@ -62,7 +62,8 @@ namespace cbm::qa::checker
/// "P": enables bin-by-bin comparison
/// "S": enables statistical hypothesis test, where is possible
/// "U": enables interval comparison
- void Process(Option_t* opt = "");
+ /// @return a vector of comparison inferences (for each version)
+ std::vector<ECmpInference> Process(Option_t* opt = "");
private:
/// @brief Creates nested directory from a given path
diff --git a/core/qa/checker/CbmQaCheckerHist1DHandler.cxx b/core/qa/checker/CbmQaCheckerHist1DHandler.cxx
index c2263f051bf345b7dac5758f39e7cf6cd7ae6664..6b6056c61444f4963def433ad1df6039945bc763 100644
--- a/core/qa/checker/CbmQaCheckerHist1DHandler.cxx
+++ b/core/qa/checker/CbmQaCheckerHist1DHandler.cxx
@@ -25,7 +25,7 @@
#include <bitset>
#include <limits>
-using cbm::qa::checker::CmpResult;
+using cbm::qa::checker::ECmpInference;
using cbm::qa::checker::Hist1DHandler;
// ---------------------------------------------------------------------------------------------------------------------
@@ -37,77 +37,134 @@ Hist1DHandler::Hist1DHandler(int iObject, int iFile, int iDataset) : ObjectHandl
// ---------------------------------------------------------------------------------------------------------------------
-//
-CmpResult Hist1DHandler::Compare(const TNamed* pNum, const TNamed* pDenom) const
+// TODO: move somewhere else (probably unite with the CbmQaCompare in a future merger request)
+ECmpInference Hist1DHandler::Compare(int iVersion) const
{
- CmpResult res;
- auto* pNumerator = static_cast<const TH1*>(pNum);
- auto* pDenominator = static_cast<const TH1*>(pDenom);
+ auto* pNumerator = static_cast<const TH1*>(fvpObjects[iVersion]);
+ auto* pDenominator = static_cast<const TH1*>(fvpObjects[fpObjDB->GetDefaultID()]);
if (pNumerator->GetNbinsX() != pDenominator->GetNbinsX() || pNumerator->GetNbinsY() != pDenominator->GetNbinsY()
|| pNumerator->GetNbinsZ() != pDenominator->GetNbinsZ()) {
- // TODO: Provide an appropriate case handling
- return res;
+ return ECmpInference::Different;
}
- res.fbPointByPointTested = fOptionBits[EFlags::kPOINT];
- res.fbRatioTested = fOptionBits[EFlags::kRATIO];
- res.fbChi2Tested = fOptionBits[EFlags::kCHI2];
+ bool bCompareExact{fCmpBits[static_cast<uint8_t>(ECmpMethod::Exact)]};
+ bool bCompareRatio{fCmpBits[static_cast<uint8_t>(ECmpMethod::Ratio)]};
+ bool bCompareChi2{fCmpBits[static_cast<uint8_t>(ECmpMethod::Chi2)]};
- if (res.fbPointByPointTested || res.fbRatioTested) {
- res.fbPointByPoint = res.fbPointByPointTested;
- if (res.fbRatioTested) {
- res.fRatioLo = 1.;
- res.fRatioUp = 1.;
- }
- for (int iBinX = 0; iBinX <= pNumerator->GetNbinsX(); ++iBinX) {
- for (int iBinY = 0; iBinY <= pNumerator->GetNbinsY(); ++iBinY) {
- for (int iBinZ = 0; iBinZ <= pNumerator->GetNbinsZ(); ++iBinZ) {
- auto numBinContent = pNumerator->GetBinContent(iBinX, iBinY, iBinZ);
- auto denBinContent = pDenominator->GetBinContent(iBinX, iBinY, iBinZ);
- float ratio = res.fbRatioTested ? static_cast<double>(numBinContent) / denBinContent
- : std::numeric_limits<float>::signaling_NaN();
+ bool bEqualExactly{true};
+ bool bEqualByRatio{true};
+
+ double ratioLo{1.};
+ double ratioUp{1.};
+
+ if (bCompareExact || bCompareRatio) {
+ bool bRatioMayBeWrong =
+ false; // handling a case, when the histograms contain different empty bins, but the ratio remains 1
+ for (int iBinX{0}; iBinX <= pNumerator->GetNbinsX(); ++iBinX) {
+ for (int iBinY{0}; iBinY <= pNumerator->GetNbinsY(); ++iBinY) {
+ for (int iBinZ{0}; iBinZ <= pNumerator->GetNbinsZ(); ++iBinZ) {
+ auto numBinContent{pNumerator->GetBinContent(iBinX, iBinY, iBinZ)};
+ auto denBinContent{pDenominator->GetBinContent(iBinX, iBinY, iBinZ)};
+ double ratio{static_cast<double>(numBinContent) / denBinContent};
// Check bin content
if (!TMath::IsNaN(numBinContent) && !TMath::IsNaN(denBinContent)) {
if (numBinContent != denBinContent) {
- res.fbPointByPoint = false;
- if (res.fbRatioTested) {
- res.fRatioLo = std::min(res.fRatioLo, ratio);
- res.fRatioUp = std::max(res.fRatioUp, ratio);
+ bEqualExactly = false;
+ if (bCompareRatio) {
+ bool numEmpty{numBinContent < 1.e-4};
+ bool denEmpty{denBinContent < 1.e-4};
+ if (numEmpty || denEmpty) {
+ bRatioMayBeWrong &= (!numEmpty || !denEmpty);
+ continue; // Ignoring empty bin in ratio estimation (but only, if it is empty in both histograms)
+ }
+ ratioLo = std::min(ratioLo, ratio);
+ ratioUp = std::max(ratioUp, ratio);
}
}
}
else {
if (TMath::IsNaN(numBinContent) != TMath::IsNaN(denBinContent)) {
- res.fbPointByPoint = false;
+ bEqualExactly = false;
}
}
- auto numBinError = pNumerator->GetBinError(iBinX, iBinY, iBinZ);
- auto denBinError = pDenominator->GetBinError(iBinX, iBinY, iBinZ);
+ auto numBinError{pNumerator->GetBinError(iBinX, iBinY, iBinZ)};
+ auto denBinError{pDenominator->GetBinError(iBinX, iBinY, iBinZ)};
// Check bin error
if (!TMath::IsNaN(numBinError) && !TMath::IsNaN(denBinError)) {
if (numBinError != denBinError) {
- res.fbPointByPoint = false;
+ bEqualExactly = false;
}
}
else {
if (TMath::IsNaN(numBinError) != TMath::IsNaN(denBinError)) {
- res.fbPointByPoint = false;
+ bEqualExactly = false;
}
}
}
}
}
+
+ if (bCompareExact && bEqualExactly) {
+ return ECmpInference::StronglyEqual; // chi2 and ratio tests can be ignored
+ }
+
+ if (bRatioMayBeWrong && (std::fabs(1 - 1.e-4) < ratioLo) && std::fabs(1 + 1.e+4) > ratioUp) {
+ LOG(warn) << "Hist1DHandler::Compare: file " << fpObjDB->GetInputFileName(fDatasetID, fFileID, 0)
+ << ", object: " << pNumerator->GetName()
+ << " has a ratio equal to 1., but some of bins are still different "
+ << "(empty/non-empty bin case)";
+ }
+
+ if (bCompareRatio) {
+ bEqualByRatio = (ratioLo >= fpObjDB->GetRatioRangeMin() && ratioUp <= fpObjDB->GetRatioRangeMax());
+ }
}
- if (res.fbPointByPoint) {
- res.fRatioLo = std::numeric_limits<float>::signaling_NaN();
- res.fRatioUp = std::numeric_limits<float>::signaling_NaN();
- res.fbChi2Tested = false;
- res.fbRatioTested = false;
+ bool bEqualByChi2{true};
+ double pVal{0.};
+ if (bCompareChi2) { // perform only, if the histograms were different
+ auto currErrorLevel{gErrorIgnoreLevel};
+ gErrorIgnoreLevel = kFatal;
+ double chi2{0.};
+ int ndf{0};
+ int igood{0};
+ pVal = pDenominator->Chi2TestX(pNumerator, chi2, ndf, igood);
+ bEqualByChi2 = (pVal >= fpObjDB->GetPvalThreshold());
+ gErrorIgnoreLevel = currErrorLevel;
}
- if (res.fbChi2Tested) { // perform only, if the histograms were different
- res.fPval = pDenominator->Chi2Test(pNumerator, "P");
+ // Do not account for method, if it was not required
+ // strongly eq = (!bCmp_0 v bEq_0) ^ (!bCmp_1 v bEq_1) ^ .. ^ (!bCmp_N v bEq_N)
+ // weakly eq = (!bCmp_0 v bEq_0) v (!bCmp_1 v bEq_1) v .. v (!bCmp_N v bEq_N)
+ bEqualExactly = !bCompareExact || bEqualExactly;
+ bEqualByRatio = !bCompareRatio || bEqualByRatio;
+ bEqualByChi2 = !bCompareChi2 || bEqualByChi2;
+ bool bEqualStrongly = bEqualExactly && bEqualByRatio && bEqualByChi2;
+ bool bEqualWeakly = bEqualExactly || bEqualByRatio || bEqualByChi2;
+ ECmpInference res = bEqualStrongly ? ECmpInference::StronglyEqual
+ : (bEqualWeakly ? ECmpInference::WeaklyEqual : ECmpInference::Different);
+
+ if (ECmpInference::StronglyEqual != res) {
+ auto ResultMsg = [](bool flag) -> std::string {
+ constexpr const char* kEqual = "\e[1;32mconsistent\e[0m";
+ constexpr const char* kDiff = "\e[1;31minconsistent\e[0m";
+ return flag ? kEqual : kDiff;
+ };
+ std::stringstream msg;
+ msg << "Found mismatch: file: " << fpObjDB->GetInputFileName(iVersion, fFileID, fDatasetID)
+ << ", object: " << fpObjDB->GetObject(fFileID, fObjectID) << "\n";
+ if (bCompareExact) {
+ msg << "\texact -> " << ResultMsg(bEqualExactly) << '\n';
+ }
+ if (bCompareRatio) {
+ msg << "\tratio -> " << ResultMsg(bEqualByRatio) << "(lo: " << ratioLo << ", up: " << ratioUp << ")\n";
+ }
+ if (bCompareChi2) {
+ msg << "\tchi2 -> " << ResultMsg(bEqualByChi2) << "(p-val: " << pVal << ")\n";
+ }
+ msg << "Inference: " << ToString(res);
+
+ LOG(info) << msg.str();
}
return res;
diff --git a/core/qa/checker/CbmQaCheckerHist1DHandler.h b/core/qa/checker/CbmQaCheckerHist1DHandler.h
index 714c46cee7fedbc84d2ac62545347b2935261448..3d3c4d99f8e6907143186459d5824d552ba5febe 100644
--- a/core/qa/checker/CbmQaCheckerHist1DHandler.h
+++ b/core/qa/checker/CbmQaCheckerHist1DHandler.h
@@ -28,14 +28,6 @@ namespace cbm::qa::checker
///
class Hist1DHandler : public ObjectHandler {
public:
- /// Enumeration for bit-flags of different comparison methods
- enum EFlags
- {
- kPOINT, ///< Point-by-point exact comparison
- kRATIO, ///< Comparison within of ratio difference from unity
- kCHI2 ///< Statistical hypothesis test using chi2 test
- };
-
/// @brief Constructor
/// @param iObject Index of object
/// @param iFile Index of file
@@ -45,18 +37,14 @@ namespace cbm::qa::checker
/// @brief Destructor
~Hist1DHandler() = default;
+ /// @brief Compares objects to default
+ /// @param iVersion Version index
+ /// @return Comparison inference
+ ECmpInference Compare(int iVersion) const override;
+
/// @brief Creates object comparison canvas
/// @param opt Canvas options
void CreateCanvases(Option_t* opt = "") override;
-
- /// @brief Compares objects to default
- /// @param pNum Pointer to "numerator" object
- /// @param pDenom Pointer to "denominator" object
- /// @return Comparison result
- ///
- /// The function provides comparison of objects and stores the comparison result in the field of the ObjectDB
- /// instance for a given version, file, object and dataset ids.
- CmpResult Compare(const TNamed* pNum, const TNamed* pDenom) const override;
};
} // namespace cbm::qa::checker
diff --git a/core/qa/checker/CbmQaCheckerObjectDB.cxx b/core/qa/checker/CbmQaCheckerObjectDB.cxx
index 87853e20d7aad87af91dfb3e5e31f14e74ac902e..9328a28449a8088f0633cc2978cd2032bdb71291 100644
--- a/core/qa/checker/CbmQaCheckerObjectDB.cxx
+++ b/core/qa/checker/CbmQaCheckerObjectDB.cxx
@@ -34,7 +34,6 @@ void ObjectDB::Clear()
fvVersionLabels.clear();
fvVersionPaths.clear();
fvObjectFirstGlobIndex.clear();
- fvCmpResults.clear();
}
// ---------------------------------------------------------------------------------------------------------------------
@@ -94,7 +93,6 @@ void ObjectDB::Init()
LOG(warn) << "ObjectDB: default version was not registered. Using the first version as the default one (\""
<< fvVersionLabels[fDefVersionID] << "\")";
}
- LOG(info) << this->ToString();
// ----- Read object list from file
for (size_t iFile = 0; iFile < fvObjects.size(); ++iFile) {
@@ -103,6 +101,8 @@ void ObjectDB::Init()
}
}
+ LOG(info) << this->ToString();
+
// ----- Init the object index vector
fvObjectFirstGlobIndex.clear();
fvObjectFirstGlobIndex.resize(fvObjects.size() + 1, 0);
@@ -110,9 +110,6 @@ void ObjectDB::Init()
fvObjectFirstGlobIndex[iFile] = fvObjectFirstGlobIndex[iFile - 1] + fvObjects[iFile - 1].size();
}
- // ----- Reserve space for object comparison results
- fvCmpResults.resize(fvVersionLabels.size() * fvObjectFirstGlobIndex.back() * fvDatasets.size());
-
// ----- Add root path of input, if it were defined
auto regexSlashes = std::regex("(/+)"); // regular expression for a sequence of consecutive slashes
for (auto& path : fvVersionPaths) {
@@ -269,31 +266,40 @@ void ObjectDB::ReadObjectList(int iFile)
// ---------------------------------------------------------------------------------------------------------------------
//
-std::string ObjectDB::ToString() const
+std::string ObjectDB::ToString(int verbose) const
{
std::stringstream msg;
- msg << "*** CBM QA-Checker: defined files ****\n\n";
-
- msg << "----- Versions:\n";
- for (size_t iV = 0; iV < fvVersionLabels.size(); ++iV) {
- if (iV == (size_t) fDefVersionID) {
- msg << "\033[1;32m- " << fvVersionLabels[iV] << " (path: " << fvVersionPaths[iV] << ")\n ---> DEFAULT\033[0m";
+ if (verbose > 0) {
+ msg << '\n';
+ msg << " ********************\n";
+ msg << " ** CBM QA-Checker **\n";
+ msg << " ********************\n\n";
+
+ msg << "\e[1mVersions\e[0m:\n";
+ for (size_t iV = 0; iV < fvVersionLabels.size(); ++iV) {
+ if (iV == (size_t) fDefVersionID) {
+ msg << "\t- " << fvVersionLabels[iV] << " (path: " << fvVersionPaths[iV] << ") -> \e[1;33mDEFAULT\e[0m\n";
+ }
+ else {
+ msg << "\t- " << fvVersionLabels[iV] << " (path: " << fvVersionPaths[iV] << ")\n";
+ }
}
- else {
- msg << "- " << fvVersionLabels[iV] << " (path: " << fvVersionPaths[iV] << ")\n";
+ msg << "\e[1mDatasets\e[0m:\n";
+ for (const auto& dataset : fvDatasets) {
+ msg << "\t- " << dataset << "\n";
}
- }
-
- msg << "----- Datasets:\n";
- for (const auto& dataset : fvDatasets) {
- msg << "- \033[1m" << dataset << "\033[0m\n";
- }
-
- msg << "----- Objects\n";
- for (size_t iF = 0; iF < fvFiles.size(); ++iF) {
- msg << "- \033[1m" << fvFiles[iF] << "\033[0m\n";
- for (const auto& object : this->GetObjects(iF)) {
- msg << "\t- " << object << '\n';
+ msg << "\e[1mFiles\e[0m:\n";
+ for (size_t iF = 0; iF < fvFiles.size(); ++iF) {
+ msg << "\t- " << fvFiles[iF];
+ if (verbose > 1) {
+ msg << " with objects:\n";
+ for (const auto& object : fvObjects[iF]) {
+ msg << "\t\t- " << object << '\n';
+ }
+ }
+ else {
+ msg << '\n';
+ }
}
}
return msg.str();
diff --git a/core/qa/checker/CbmQaCheckerObjectDB.h b/core/qa/checker/CbmQaCheckerObjectDB.h
index f5e81503192b8d6f1b245776470715a673058d54..fecf0f49fcb422c53d37c0fb057b1d3b023acfaa 100644
--- a/core/qa/checker/CbmQaCheckerObjectDB.h
+++ b/core/qa/checker/CbmQaCheckerObjectDB.h
@@ -71,28 +71,7 @@ namespace cbm::qa::checker
/// @return Label of file
const std::string& GetFileLabel(int iFile) const { return fvFileLabels[iFile]; }
- private:
- /// @brief Gets iterator range for object names stored in a file
- /// @note The iterator_range object behaves itself just like an ordinary STL container.
- /// @param iFile Index of file
- /// @return iterator_range for ROOT objects of this file
- /// TODO: Understand, which type is deduced
- const std::vector<std::string>& GetObjects(int iFile) const { return fvObjects[iFile]; }
-
public:
- /// @brief Gets comparison result
- /// @param iDS Index of dataset
- /// @param iFile Index of file
- /// @param iObj Index of object within the file
- /// @param iVer Index of version
- /// @return Value of comparison result
- const CmpResult& GetCmpResult(int iDS, int iFile, int iObj, int iVer) const
- {
- int iObjGlob = fvObjectFirstGlobIndex[iFile] + iObj;
- int iRes = iVer + fvVersionLabels.size() * (iObjGlob + iDS * fvObjectFirstGlobIndex.back());
- return fvCmpResults[iRes];
- }
-
/// @brief Gets name of file from indexes of version, file and dataset
/// @param iVersion Index of version
/// @param iFile Index of file
@@ -104,7 +83,7 @@ namespace cbm::qa::checker
/// @param iFile Index of file
/// @param iObject Index of object for a given file
/// @return Name of object
- const std::string& GetObject(int iFile, int iObject) const { return GetObjects(iFile)[iObject]; }
+ const std::string& GetObject(int iFile, int iObject) const { return fvObjects[iFile][iObject]; }
/// @brief Gets number of datasets
int GetNofDatasets() const { return fvDatasets.size(); }
@@ -114,7 +93,7 @@ namespace cbm::qa::checker
/// @brief Gets number of objects in file
/// @param iFile Index of file
- int GetNofObjects(int iFile) const { return GetObjects(iFile).size(); }
+ int GetNofObjects(int iFile) const { return fvObjects[iFile].size(); }
/// @brief Gets number of files
int GetNofFiles() const { return fvFiles.size(); }
@@ -125,6 +104,15 @@ namespace cbm::qa::checker
/// @brief Gets output path
const std::string& GetOutputPath() const { return fsOutputPath; }
+ /// @brief Gets p-value threshold
+ double GetPvalThreshold() const { return fPvalThresh; }
+
+ /// @brief Gets upper bound of the accepted ratio range
+ double GetRatioRangeMax() const { return fRatioMax; }
+
+ /// @brief Gets lower bound of the accepted ratio range
+ double GetRatioRangeMin() const { return fRatioMin; }
+
/// @brief Gets version label
/// @param iVersion Index of version
const std::string& GetVersionLabel(int iVersion) const { return fvVersionLabels[iVersion]; }
@@ -140,22 +128,13 @@ namespace cbm::qa::checker
/// @param config Root node of the YAML file
void ReadFromYAML(const char* configName);
- /// @brief Saves content to string
+ /// @brief String representation of the content
+ /// @param verbose Verbosity level:
+ /// 0: no output,
+ /// 1: minimal output (versions, datasets and file names),
+ /// 2: objects as well
/// @return A string representation of the DB contents
- std::string ToString() const;
-
- /// @brief Sets comparison result
- /// @param iDS Index of dataset
- /// @param iFile Index of file
- /// @param iObj Index of object within the file
- /// @param iVer Index of version
- /// @param value Value of comparison result
- void SetCmpResult(int iDS, int iFile, int iObj, int iVer, CmpResult value)
- {
- int iObjGlob = fvObjectFirstGlobIndex[iFile] + iObj;
- int iRes = iVer + fvVersionLabels.size() * (iObjGlob + iDS * fvObjectFirstGlobIndex.back());
- fvCmpResults[iRes] = value;
- }
+ std::string ToString(int verbose = 1) const;
/// @brief Sets default version label
/// @param defaultLabel Name of default label
@@ -172,6 +151,19 @@ namespace cbm::qa::checker
/// @param path Path to the output ROOT-file
void SetOutputPath(const char* path) { fsOutputPath = path; }
+ /// @brief Sets P-value threshold
+ /// @param pVal P-value threshold
+ void SetPvalThreshold(float pVal) { fPvalThresh = pVal; }
+
+ /// @brief Sets ratio accepted range
+ /// @param min Lower boundary
+ /// @param max Upper boundary
+ void SetRatioRange(float min, float max)
+ {
+ fRatioMin = min;
+ fRatioMax = max;
+ }
+
private:
/// @brief Reads list of histograms from file
/// @param iFile Index of file
@@ -198,7 +190,10 @@ namespace cbm::qa::checker
std::vector<std::vector<std::string>> fvObjects; ///< Container of object names vs file id
std::vector<std::string> fvVersionLabels; ///< Container of version labels
std::vector<std::string> fvVersionPaths; ///< Container of version paths
- std::vector<CmpResult> fvCmpResults; ///< Comparison results vs. dataset, object and version
+
+ float fPvalThresh{0.05}; ///< P-value threshold for histograms equality
+ float fRatioMax{1.05}; ///< Upper boundary for ratio deviation
+ float fRatioMin{0.95}; ///< Lower boundary for ratio deviation
};
} // namespace cbm::qa::checker
diff --git a/core/qa/checker/CbmQaCheckerObjectHandler.cxx b/core/qa/checker/CbmQaCheckerObjectHandler.cxx
index f28573caec5137e8c0b77482b8d1c29ce697bb50..ef9027416fe0e3cc7eafc9acc26327fdc9a3f547 100644
--- a/core/qa/checker/CbmQaCheckerObjectHandler.cxx
+++ b/core/qa/checker/CbmQaCheckerObjectHandler.cxx
@@ -14,15 +14,16 @@
#include "TDirectory.h"
#include "TNamed.h"
+using cbm::qa::checker::ECmpInference;
using cbm::qa::checker::ObjectHandler;
// ---------------------------------------------------------------------------------------------------------------------
//
ObjectHandler::ObjectHandler(int iObject, int iFile, int iDataset, const char* objType)
- : fObjectID(iObject)
+ : fsObjType(objType)
+ , fObjectID(iObject)
, fFileID(iFile)
, fDatasetID(iDataset)
- , fsObjType(objType)
{
}
@@ -66,16 +67,18 @@ void ObjectHandler::AddObjects(const std::vector<TNamed*>& vpObj)
// ---------------------------------------------------------------------------------------------------------------------
//
-void ObjectHandler::CompareWithDefault()
+std::vector<ECmpInference> ObjectHandler::CompareWithDefault()
{
+ // All object versions are equal to default one (H0)
+ std::vector<ECmpInference> res(fpObjDB->GetNofVersions(), ECmpInference::StronglyEqual);
int iDef = fpObjDB->GetDefaultID();
for (int iV = 0; iV < (int) fvpObjects.size(); ++iV) {
if (iV == iDef) {
continue;
}
- auto res = this->Compare(fvpObjects[iV], fvpObjects[iDef]);
- fpObjDB->SetCmpResult(fDatasetID, fFileID, fObjectID, iV, res);
+ res[iV] = this->Compare(iV);
}
+ return res;
}
// ---------------------------------------------------------------------------------------------------------------------
diff --git a/core/qa/checker/CbmQaCheckerObjectHandler.h b/core/qa/checker/CbmQaCheckerObjectHandler.h
index 8618b618ff08117c0ed3480736b03d5e2fc75531..353ebb5b6aa409594ba7ffdf8585a5c0052e2efe 100644
--- a/core/qa/checker/CbmQaCheckerObjectHandler.h
+++ b/core/qa/checker/CbmQaCheckerObjectHandler.h
@@ -28,7 +28,6 @@ namespace cbm::qa::checker
///
/// The class provides interface for handling objects of the same type, obtained under different versions of the code
/// base.
- ///
class ObjectHandler {
public:
/// @brief Default constructor
@@ -49,15 +48,14 @@ namespace cbm::qa::checker
/// @param opt Canvas options
virtual void CreateCanvases(Option_t*){};
- /// @brief Compares two objects with different methods
- /// @param pNum Pointer to "numerator" object
- /// @param pDenom Pointer to "denominator" object
- /// @return Comparison result represented as a bitset in unsigned integer
- /// TODO: .....
- virtual CmpResult Compare(const TNamed* pNum, const TNamed* pDenom) const = 0;
+ /// @brief Compares objects to default
+ /// @param iVersion Version index
+ /// @return Comparison inference
+ virtual ECmpInference Compare(int iVersion) const = 0;
- /// @brief Compares versions with default and writes result into DB
- void CompareWithDefault();
+ /// @brief Compares different versions with default
+ /// @return Vector of comparison inferences for different versions
+ std::vector<ECmpInference> CompareWithDefault();
/// @brief Sets folder to store output
/// @param pDir Pointer to folder instance
@@ -67,34 +65,27 @@ namespace cbm::qa::checker
/// @param pObjDB Shared pointer to object database
void SetObjectDB(std::shared_ptr<ObjectDB>& pObjDB) { fpObjDB = pObjDB; }
- /// @brief Sets verbose level
- /// @param verbose Verbose level:
- /// - 0: Silent mode
- /// - 1:
- void SetVerbose(int verbose) { fVerbose = verbose; }
-
/// @brief Sets bit flag to control handler behaviour
/// @param bit Bit index
///
/// The bit flags should be defined in an enumeration of the default class
- void SetBitFlag(uint8_t bit) { fOptionBits.set(bit); }
+ void SetComparisonMethod(ECmpMethod method) { fCmpBits.set(static_cast<uint8_t>(method)); }
/// @brief Writes objects to file
void Write();
protected:
- int fObjectID = -1; ///< Index of object
- int fFileID = -1; ///< Index of file
- int fDatasetID = -1; ///< Index of dataset
- int fVerbose = 0; ///< Verbosity level
- TDirectory* fpOutDir = nullptr; ///< Pointer to directory
- std::shared_ptr<ObjectDB> fpObjDB = nullptr; ///< Pointer to object database
- std::shared_ptr<TCanvas> fpCanvas = nullptr; ///< Comparison canvas
- std::bitset<4> fOptionBits; ///< Bitset for option
-
- std::string fsObjType = ""; ///< Base type of the object to be handled
- std::string fsBaseName = ""; ///< Base names of the object
- std::vector<TNamed*> fvpObjects; ///< Vector of objects
+ std::string fsObjType{""}; ///< Base type of the object to be handled
+ std::string fsBaseName{""}; ///< Base names of the object
+ std::vector<TNamed*> fvpObjects; ///< Vector of objects
+ std::shared_ptr<ObjectDB> fpObjDB{nullptr}; ///< Pointer to object database
+ std::shared_ptr<TCanvas> fpCanvas{nullptr}; ///< Comparison canvas
+ TDirectory* fpOutDir{nullptr}; ///< Pointer to directory
+ int fObjectID{-1}; ///< Index of object
+ int fFileID{-1}; ///< Index of file
+ int fDatasetID{-1}; ///< Index of dataset
+
+ std::bitset<static_cast<size_t>(ECmpMethod::END)> fCmpBits; ///< Bitset for comparison methods
};
} // namespace cbm::qa::checker
diff --git a/core/qa/checker/CbmQaCheckerTypedefs.h b/core/qa/checker/CbmQaCheckerTypedefs.h
index 2b120c6176a28ee9e013e4b2674e8e523af30da5..d5474c92fed5c067e72e5b4581fbfba61ddf83b3 100644
--- a/core/qa/checker/CbmQaCheckerTypedefs.h
+++ b/core/qa/checker/CbmQaCheckerTypedefs.h
@@ -20,38 +20,50 @@
namespace cbm::qa::checker
{
- /// @brief Enumerations for QA-Checker execution control
- ///
- enum class EFlagBit
+ /// @enum ECmpMethod
+ /// @brief Comparison method
+ enum class ECmpMethod : uint8_t
{
- kSKIP_LOST_OBJECTS, //< Skips objects, which are defined in config, but not presented in file
- kEND
+ Exact, ///< exact equality (point-by-point, error-by-error)
+ Ratio, ///< ratio equality (max and min ratios do not exceed a required range)
+ Chi2, ///< equality within a chi2 hypothesis test
+ END ///< end of enumeration
};
- /// @struct CmpResult
- /// @brief Comparison result
- struct CmpResult {
- // results
- float fRatioUp{std::numeric_limits<float>::signaling_NaN()};
- float fRatioLo{std::numeric_limits<float>::signaling_NaN()};
- float fPval{std::numeric_limits<float>::signaling_NaN()};
- bool fbPointByPoint{false}; // non-equal
- // if test was performed
- bool fbPointByPointTested{false};
- bool fbRatioTested{false};
- bool fbChi2Tested{false};
+ /// @enum ECmpInference
+ /// @brief The object comparison inference
+ /// @note The sequence of the elements is important, please, do not shuffle!
+ enum class ECmpInference : uint8_t
+ {
+ StronglyEqual = 0, ///< All the comparison methods gave equality
+ WeaklyEqual, ///< At least one of the comparison methods showed equality
+ Different ///< Neither of the comparison methods showed equality
};
- // ----- Aliases
+ /// @brief String representation of the ECmpInference enum
+ inline std::string ToString(ECmpInference inference)
+ {
+ switch (inference) {
+ case ECmpInference::StronglyEqual: return "\e[1;32msame\e[0m";
+ case ECmpInference::WeaklyEqual: return "\e[1;33mconsistent\e[0m";
+ case ECmpInference::Different: return "\e[1;31mdifferent\e[0m";
+ default: return "";
+ }
+ }
+
+ // Aliases
using MapStrToStr_t = std::unordered_map<std::string, std::string>;
using MapStrToStrVect_t = std::unordered_map<std::string, std::vector<std::string>>;
- using FlagBitSet_t = std::bitset<static_cast<int>(EFlagBit::kEND)>;
template<class T>
using VectRange_t = boost::iterator_range<typename std::vector<T>::iterator>;
- // ----- Constants
- constexpr double kLegendSize[2] = {.3, .05}; // width and height in % of the pad size
+ // Constants
+ constexpr double kLegendSize[2] = {.3, .05}; ///< width and height in % of the pad size
+ constexpr float kRatioMin = 0.95; ///< Minimal acceptable ratio
+ constexpr float kRatioMax = 1.05; ///< Maximal acceptable ratio
+ constexpr float kPvalThrsh = 0.05; ///< P-value threshold
+
} // namespace cbm::qa::checker
diff --git a/macro/qa/qa_compare.C b/macro/qa/qa_compare.C
index d5cda2386a92d94cf3dbc3c5783300679ec45b5b..1323ff04f9ec573c483f1e33c45da662e883fe08 100644
--- a/macro/qa/qa_compare.C
+++ b/macro/qa/qa_compare.C
@@ -50,10 +50,7 @@ int qa_compare(
pQaChecker->SetDefaultVersion("old");
//// ----- Run comparision routine
- pQaChecker->Process("P");
-
- //// ----- Scan results
- bool res = pQaChecker->Scan(); // true - objects are the same, false - objects differ
+ bool res = pQaChecker->Process("P");
std::cout << "Macro finished successfully." << std::endl;
- return !res;
+ return res;
}
diff --git a/macro/qa/qa_compare_ca.C b/macro/qa/qa_compare_ca.C
index ddb750158a839df51c2040e3433076b142bfa2e9..ea2530bb38604be0fbe7b67554545cf8efe191a7 100644
--- a/macro/qa/qa_compare_ca.C
+++ b/macro/qa/qa_compare_ca.C
@@ -36,10 +36,7 @@ int qa_compare_ca(
pQaChecker->SetFromYAML(configName); // Read file-object map
//// ----- Run comparision routine
- pQaChecker->Process("PRC"); // P -
-
- //// ----- Scan results
- bool res = pQaChecker->Scan(); // true - objects are the same, false - objects differ
+ int res = pQaChecker->Process("UPS");
std::cout << "Macro finished successfully." << std::endl;
- return !res;
+ return res;
}