From c7cc0bce984d8d3cd375faac7b37a8d2e86f0b19 Mon Sep 17 00:00:00 2001 From: "s.zharko@gsi.de" <s.zharko@gsi.de> Date: Tue, 12 Nov 2024 19:45:02 +0100 Subject: [PATCH] QA-checker: added comparison using chi2 test and ratio --- core/qa/checker/CbmQaCheckerCore.cxx | 30 +++- core/qa/checker/CbmQaCheckerCore.h | 4 + core/qa/checker/CbmQaCheckerFileHandler.cxx | 64 ++++---- core/qa/checker/CbmQaCheckerHist1DHandler.cxx | 147 ++++++++---------- core/qa/checker/CbmQaCheckerHist1DHandler.h | 48 +----- core/qa/checker/CbmQaCheckerObjectDB.h | 6 +- core/qa/checker/CbmQaCheckerObjectHandler.h | 4 +- core/qa/checker/CbmQaCheckerTypedefs.h | 16 +- 8 files changed, 148 insertions(+), 171 deletions(-) diff --git a/core/qa/checker/CbmQaCheckerCore.cxx b/core/qa/checker/CbmQaCheckerCore.cxx index 3125a699e1..2f832c1bc4 100644 --- a/core/qa/checker/CbmQaCheckerCore.cxx +++ b/core/qa/checker/CbmQaCheckerCore.cxx @@ -94,10 +94,32 @@ bool Core::Scan() const // For now we check, if anything has been changed with respect to default. Later one can add a flag to test // compatibility for different levels (for example, two histograms could be obtained from different systems, // or different seeds, but still be consistent). - if (fpObjDB->GetCmpResult(iDS, iFile, iObj, iVer)) { - LOG(info) << "\t file: " << fpObjDB->GetInputFileName(iVer, iFile, iDS) - << ", object: " << fpObjDB->GetObject(iFile, iObj); - res = false; + const auto& cmp = fpObjDB->GetCmpResult(iDS, iFile, iObj, iVer); + bool bEqual = true; + { + std::stringstream msg; + constexpr const char* kEqual = "\e[1;32mequal\e[0m"; + constexpr const char* kDiff = "\e[1;32mdifferent\e[0m"; + msg << "Found mismatch: file: " << fpObjDB->GetInputFileName(iVer, iFile, iDS) + << ", object: " << fpObjDB->GetObject(iFile, iObj); + if (cmp.fbPointByPointTested) { + msg << "\tpoint-by-point -> " << (cmp.fbPointByPoint ? kEqual : kDiff); + bEqual &= cmp.fbPointByPoint; + } + if (cmp.fbRatioTested) { + bool bOk = (cmp.fRatioLo >= kRatioMin && cmp.fRatioUp <= kRatioMax); + msg << "\tratio -> " << (bOk ? kEqual : kDiff) << "(lo: " << cmp.fRatioLo << ", up: " << cmp.fRatioUp + << ')'; + bEqual &= bOk; + } + if (cmp.fbChi2Tested) { + bool bOk = cmp.fPval >= kPvalThrsh; + msg << "\tchi2 test -> " << (bOk ? kEqual : kDiff) << "(p-val: " << cmp.fPval << ')'; + bEqual &= bOk; + } + if (!bEqual) { + LOG(info) << msg.str(); + } } } // iVer } // iDS diff --git a/core/qa/checker/CbmQaCheckerCore.h b/core/qa/checker/CbmQaCheckerCore.h index 7dae9e67d0..a86d2c2949 100644 --- a/core/qa/checker/CbmQaCheckerCore.h +++ b/core/qa/checker/CbmQaCheckerCore.h @@ -101,6 +101,10 @@ namespace cbm::qa::checker /// @brief Prepares output file (creates directory structure) void PrepareOutputFile(); + // TODO: provide setter/config entry + static constexpr float kRatioMin = 0.95; + static constexpr float kRatioMax = 1.05; + static constexpr float kPvalThrsh = 0.05; std::shared_ptr<ObjectDB> fpObjDB = nullptr; ///< Database of names diff --git a/core/qa/checker/CbmQaCheckerFileHandler.cxx b/core/qa/checker/CbmQaCheckerFileHandler.cxx index a1517d6fbb..b5cb57de0d 100644 --- a/core/qa/checker/CbmQaCheckerFileHandler.cxx +++ b/core/qa/checker/CbmQaCheckerFileHandler.cxx @@ -94,9 +94,10 @@ void FileHandler::Process(Option_t* opt) ch = std::tolower(ch); } bool bSuppressCanvases = sOption.find("b") != std::string::npos; - bool bForceCanvases = sOption.find("c") != std::string::npos; + //bool bForceCanvases = sOption.find("c") != std::string::npos; bool bCmpPointByPoint = sOption.find("p") != std::string::npos; bool bCmpStatAny = sOption.find("s") != std::string::npos; + bool bCmpRatio = sOption.find("u") != std::string::npos; LOG(info) << "FileHandler: processing objects: "; std::vector<TNamed*> vpObjects(nVersions, nullptr); // vector to keep object different versions @@ -122,35 +123,14 @@ void FileHandler::Process(Option_t* opt) } std::unique_ptr<ObjectHandler> pObjHandler = nullptr; LOG(info) << "FileHandler: processing object \"" << vpObjects[0]->GetName() << '\"'; - if (dynamic_cast<TH2*>(vpObjects[0])) { - pObjHandler = std::make_unique<Hist2DHandler>(iObj, fFileID, fDatasetID); - if (bCmpPointByPoint) { - pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kPOINT); // Compare point-by-point (exact equality) - } - if (bCmpStatAny) { - pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kCHI2); // Compare with chi2 test - pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kKOLM); // Compare with Kolmogorov test - } - } - else if (dynamic_cast<TProfile*>(vpObjects[0])) { + if (dynamic_cast<TProfile*>(vpObjects[0])) { pObjHandler = std::make_unique<Profile1DHandler>(iObj, fFileID, fDatasetID); - if (bCmpPointByPoint) { - pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kPOINT); // Compare point-by-point (exact equality) - } - if (bCmpStatAny) { - pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kCHI2); // Compare with chi2 test - pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kKOLM); // Compare with Kolmogorov test - } + } + else if (dynamic_cast<TH2*>(vpObjects[0])) { + pObjHandler = std::make_unique<Hist2DHandler>(iObj, fFileID, fDatasetID); } else if (dynamic_cast<TH1*>(vpObjects[0])) { pObjHandler = std::make_unique<Hist1DHandler>(iObj, fFileID, fDatasetID); - if (bCmpPointByPoint) { - pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kPOINT); // Compare point-by-point (exact equality) - } - if (bCmpStatAny) { - pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kCHI2); // Compare with chi2 test - pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kKOLM); // Compare with Kolmogorov test - } } else { LOG(warn) << "FileHandler: Object " << fpObjDB->GetObject(fFileID, iObj) << " has a type \"" @@ -158,6 +138,15 @@ void FileHandler::Process(Option_t* opt) << "\", which is unknown to the cbm::qa::checker framework, it will be skipped"; continue; } + if (bCmpPointByPoint) { + pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kPOINT); // Compare point-by-point (exact equality) + } + if (bCmpStatAny) { + pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kCHI2); // Compare with chi2 test + } + if (bCmpRatio) { + pObjHandler->SetBitFlag(Hist1DHandler::EFlags::kRATIO); + } pObjHandler->SetObjectDB(fpObjDB); pObjHandler->SetOutputDirectory(CreateNestedDirectory((fpObjDB->GetObject(fFileID, iObj)))); pObjHandler->AddObjects(vpObjects); @@ -165,16 +154,21 @@ void FileHandler::Process(Option_t* opt) // Create comparison canvas if (!bSuppressCanvases) { - bool areDifferent = false; - for (int iVer = 0; iVer < nVersions; ++iVer) { - if (fpObjDB->GetCmpResult(fDatasetID, fFileID, iObj, iVer)) { - areDifferent = true; - } - } - if (areDifferent || bForceCanvases) { - pObjHandler->CreateCanvases(opt); - } + pObjHandler->CreateCanvases(opt); } + + // TODO: re-work the suppress and force canvas features + //if (!bSuppressCanvases) { + // bool areDifferent = false; + // for (int iVer = 0; iVer < nVersions; ++iVer) { + // if (fpObjDB->GetCmpResult(fDatasetID, fFileID, iObj, iVer)) { + // areDifferent = true; + // } + // } + // if (areDifferent || bForceCanvases) { + // pObjHandler->CreateCanvases(opt); + // } + //} pObjHandler->Write(); gFile->Flush(); diff --git a/core/qa/checker/CbmQaCheckerHist1DHandler.cxx b/core/qa/checker/CbmQaCheckerHist1DHandler.cxx index e107920b3e..c2263f051b 100644 --- a/core/qa/checker/CbmQaCheckerHist1DHandler.cxx +++ b/core/qa/checker/CbmQaCheckerHist1DHandler.cxx @@ -25,7 +25,7 @@ #include <bitset> #include <limits> -using cbm::qa::checker::CmpResult_t; +using cbm::qa::checker::CmpResult; using cbm::qa::checker::Hist1DHandler; // --------------------------------------------------------------------------------------------------------------------- @@ -38,33 +38,79 @@ Hist1DHandler::Hist1DHandler(int iObject, int iFile, int iDataset) : ObjectHandl // --------------------------------------------------------------------------------------------------------------------- // -CmpResult_t Hist1DHandler::Compare(const TNamed* pNum, const TNamed* pDenom) const +CmpResult Hist1DHandler::Compare(const TNamed* pNum, const TNamed* pDenom) const { - std::bitset<sizeof(CmpResult_t) * 8> res; + CmpResult res; auto* pNumerator = static_cast<const TH1*>(pNum); auto* pDenominator = static_cast<const TH1*>(pDenom); if (pNumerator->GetNbinsX() != pDenominator->GetNbinsX() || pNumerator->GetNbinsY() != pDenominator->GetNbinsY() || pNumerator->GetNbinsZ() != pDenominator->GetNbinsZ()) { - res[EFlags::kPOINT] = false; - res[EFlags::kRATIO] = false; - res[EFlags::kCHI2] = false; - res[EFlags::kKOLM] = false; - return res.to_ulong(); + // TODO: Provide an appropriate case handling + return res; } - if (fOptionBits[EFlags::kPOINT] && ComparePointToPoint(pNumerator, pDenominator)) { - res[EFlags::kPOINT] = true; - } - if (fOptionBits[EFlags::kRATIO] && CompareRatioDeviation(pNumerator, pDenominator)) { - res[EFlags::kRATIO] = true; + res.fbPointByPointTested = fOptionBits[EFlags::kPOINT]; + res.fbRatioTested = fOptionBits[EFlags::kRATIO]; + res.fbChi2Tested = fOptionBits[EFlags::kCHI2]; + + if (res.fbPointByPointTested || res.fbRatioTested) { + res.fbPointByPoint = res.fbPointByPointTested; + if (res.fbRatioTested) { + res.fRatioLo = 1.; + res.fRatioUp = 1.; + } + for (int iBinX = 0; iBinX <= pNumerator->GetNbinsX(); ++iBinX) { + for (int iBinY = 0; iBinY <= pNumerator->GetNbinsY(); ++iBinY) { + for (int iBinZ = 0; iBinZ <= pNumerator->GetNbinsZ(); ++iBinZ) { + auto numBinContent = pNumerator->GetBinContent(iBinX, iBinY, iBinZ); + auto denBinContent = pDenominator->GetBinContent(iBinX, iBinY, iBinZ); + float ratio = res.fbRatioTested ? static_cast<double>(numBinContent) / denBinContent + : std::numeric_limits<float>::signaling_NaN(); + // Check bin content + if (!TMath::IsNaN(numBinContent) && !TMath::IsNaN(denBinContent)) { + if (numBinContent != denBinContent) { + res.fbPointByPoint = false; + if (res.fbRatioTested) { + res.fRatioLo = std::min(res.fRatioLo, ratio); + res.fRatioUp = std::max(res.fRatioUp, ratio); + } + } + } + else { + if (TMath::IsNaN(numBinContent) != TMath::IsNaN(denBinContent)) { + res.fbPointByPoint = false; + } + } + auto numBinError = pNumerator->GetBinError(iBinX, iBinY, iBinZ); + auto denBinError = pDenominator->GetBinError(iBinX, iBinY, iBinZ); + // Check bin error + if (!TMath::IsNaN(numBinError) && !TMath::IsNaN(denBinError)) { + if (numBinError != denBinError) { + res.fbPointByPoint = false; + } + } + else { + if (TMath::IsNaN(numBinError) != TMath::IsNaN(denBinError)) { + res.fbPointByPoint = false; + } + } + } + } + } } - if (fOptionBits[EFlags::kCHI2] && CompareWithChi2(pNumerator, pDenominator)) { - res[EFlags::kCHI2] = true; + + if (res.fbPointByPoint) { + res.fRatioLo = std::numeric_limits<float>::signaling_NaN(); + res.fRatioUp = std::numeric_limits<float>::signaling_NaN(); + res.fbChi2Tested = false; + res.fbRatioTested = false; } - if (fOptionBits[EFlags::kKOLM] && CompareWithKolmogorov(pNumerator, pDenominator)) { - res[EFlags::kKOLM] = true; + + if (res.fbChi2Tested) { // perform only, if the histograms were different + res.fPval = pDenominator->Chi2Test(pNumerator, "P"); } - return res.to_ulong(); + + return res; } // --------------------------------------------------------------------------------------------------------------------- @@ -193,68 +239,3 @@ void Hist1DHandler::CreateCanvases(Option_t* opt) if (pPadDiff) pPadDiff->Draw(); } -// --------------------------------------------------------------------------------------------------------------------- -// -bool Hist1DHandler::ComparePointToPoint(const TH1* pNumerator, const TH1* pDenominator) -{ - bool ifDifferent = false; - // Compare content and error of each bin - // TODO: write a comment about - for (int iBinX = 0; iBinX <= pNumerator->GetNbinsX(); ++iBinX) { - for (int iBinY = 0; iBinY <= pNumerator->GetNbinsY(); ++iBinY) { - for (int iBinZ = 0; iBinZ <= pNumerator->GetNbinsZ(); ++iBinZ) { - auto numBinContent = pNumerator->GetBinContent(iBinX, iBinY, iBinZ); - auto denBinContent = pDenominator->GetBinContent(iBinX, iBinY, iBinZ); - // Check bin content - if (!TMath::IsNaN(numBinContent) && !TMath::IsNaN(denBinContent)) { - if (numBinContent != denBinContent) { - return true; - } - } - else { - if (TMath::IsNaN(numBinContent) != TMath::IsNaN(denBinContent)) { - return true; - } - } - auto numBinError = pNumerator->GetBinError(iBinX, iBinY, iBinZ); - auto denBinError = pDenominator->GetBinError(iBinX, iBinY, iBinZ); - // Check bin error - if (!TMath::IsNaN(numBinError) && !TMath::IsNaN(denBinError)) { - if (numBinError != denBinError) { - return true; - } - } - else { - if (TMath::IsNaN(numBinError) != TMath::IsNaN(denBinError)) { - return true; - } - } - } - } - } - return ifDifferent; -} - -// --------------------------------------------------------------------------------------------------------------------- -// -bool Hist1DHandler::CompareRatioDeviation(const TH1* /*pNumerator*/, const TH1* /*pDenominator*/, double /*allowedDev*/) -{ - LOG(warn) << "Hist1DHandler::CompareRatioDeviation function was not implemented"; - return false; -} - -// --------------------------------------------------------------------------------------------------------------------- -// -bool Hist1DHandler::CompareWithChi2(const TH1* /*pNumerator*/, const TH1* /*pDenominator*/) -{ - LOG(warn) << "Hist1DHandler::CompareWthChi2 function was not implemented"; - return false; -} - -// --------------------------------------------------------------------------------------------------------------------- -// -bool Hist1DHandler::CompareWithKolmogorov(const TH1* /*pNumerator*/, const TH1* /*pDenominator*/) -{ - LOG(warn) << "Hist1DHandler::CompareWithKolmogorov function was not implemented"; - return false; -} diff --git a/core/qa/checker/CbmQaCheckerHist1DHandler.h b/core/qa/checker/CbmQaCheckerHist1DHandler.h index bef26ff20a..714c46cee7 100644 --- a/core/qa/checker/CbmQaCheckerHist1DHandler.h +++ b/core/qa/checker/CbmQaCheckerHist1DHandler.h @@ -24,7 +24,7 @@ namespace cbm::qa::checker /// /// The handler keeps one-dimensional histogram objects of the same quantity within different code versions /// and provides several comparison methods including point-by-point comparison, test of ratio deviation and - /// statistical hypothesis tests (Chi2 test and Kolmogorov test). + /// statistical hypothesis tests (Chi2 test). /// class Hist1DHandler : public ObjectHandler { public: @@ -33,8 +33,7 @@ namespace cbm::qa::checker { kPOINT, ///< Point-by-point exact comparison kRATIO, ///< Comparison within of ratio difference from unity - kCHI2, ///< Statistical hypothesis test using chi2 test - kKOLM ///< Statistical hypothesis test using Kolmogorov test + kCHI2 ///< Statistical hypothesis test using chi2 test }; /// @brief Constructor @@ -53,48 +52,11 @@ namespace cbm::qa::checker /// @brief Compares objects to default /// @param pNum Pointer to "numerator" object /// @param pDenom Pointer to "denominator" object - /// @return Comparison result represented as a bitset in unsigned integer + /// @return Comparison result /// /// The function provides comparison of objects and stores the comparison result in the field of the ObjectDB - /// instance for a given version, file, object and dataset ids. The comparison result is represented with the - /// unsigned integer, each raised bit of which declares, that the objects are different within the corresponding - /// comparison method. - /// Bits used: - /// 0: Point to point comparison - /// 1: Comparison based on the ratio deviation - /// 2: Statistical test comparison with chi2-test - /// 3: Statistical test comparison with Kolmogorov test - CmpResult_t Compare(const TNamed* pNum, const TNamed* pDenom) const override; - - private: - /// @brief Compares two histograms point-to-point - /// @param pNumerator Pointer to the numerator object of comparison (usually new version) - /// @param pDenominator Pointer to the denominator object for comparison (usually default) - /// @return true Histograms are different - /// @return false Histograms are equal - static bool ComparePointToPoint(const TH1* pNumerator, const TH1* pDenominator); - - /// @brief Compares two histograms by maximum deviation of the ratio - /// @param pNumerator Pointer to the numerator object of comparison (usually new version) - /// @param pDenominator Pointer to the denominator object for comparison (usually default) - /// @param allowedDev Max allowed ratio deviation from unity - /// @return true Histograms are different - /// @return false Histograms are equal - static bool CompareRatioDeviation(const TH1* pNumerator, const TH1* pDenominator, double allowedDev = 10); - - /// @brief Compares two histograms with chi2-test - /// @param pNumerator Pointer to the numerator object of comparison (usually new version) - /// @param pDenominator Pointer to the denominator object for comparison (usually default) - /// @return true Histograms are different - /// @return false Histograms are equal - static bool CompareWithChi2(const TH1* pNumerator, const TH1* pDenominator); - - /// @brief Compares two histograms with Kolmogorov test - /// @param pNumerator Pointer to the numerator object of comparison (usually new version) - /// @param pDenominator Pointer to the denominator object for comparison (usually default) - /// @return true Histograms are different - /// @return false Histograms are equal - static bool CompareWithKolmogorov(const TH1* pNumerator, const TH1* pDenominator); + /// instance for a given version, file, object and dataset ids. + CmpResult Compare(const TNamed* pNum, const TNamed* pDenom) const override; }; } // namespace cbm::qa::checker diff --git a/core/qa/checker/CbmQaCheckerObjectDB.h b/core/qa/checker/CbmQaCheckerObjectDB.h index ffb3d01706..f5e8150319 100644 --- a/core/qa/checker/CbmQaCheckerObjectDB.h +++ b/core/qa/checker/CbmQaCheckerObjectDB.h @@ -86,7 +86,7 @@ namespace cbm::qa::checker /// @param iObj Index of object within the file /// @param iVer Index of version /// @return Value of comparison result - CmpResult_t GetCmpResult(int iDS, int iFile, int iObj, int iVer) const + const CmpResult& GetCmpResult(int iDS, int iFile, int iObj, int iVer) const { int iObjGlob = fvObjectFirstGlobIndex[iFile] + iObj; int iRes = iVer + fvVersionLabels.size() * (iObjGlob + iDS * fvObjectFirstGlobIndex.back()); @@ -150,7 +150,7 @@ namespace cbm::qa::checker /// @param iObj Index of object within the file /// @param iVer Index of version /// @param value Value of comparison result - void SetCmpResult(int iDS, int iFile, int iObj, int iVer, CmpResult_t value) + void SetCmpResult(int iDS, int iFile, int iObj, int iVer, CmpResult value) { int iObjGlob = fvObjectFirstGlobIndex[iFile] + iObj; int iRes = iVer + fvVersionLabels.size() * (iObjGlob + iDS * fvObjectFirstGlobIndex.back()); @@ -198,7 +198,7 @@ namespace cbm::qa::checker std::vector<std::vector<std::string>> fvObjects; ///< Container of object names vs file id std::vector<std::string> fvVersionLabels; ///< Container of version labels std::vector<std::string> fvVersionPaths; ///< Container of version paths - std::vector<CmpResult_t> fvCmpResults; ///< Comparison results vs. dataset, object and version + std::vector<CmpResult> fvCmpResults; ///< Comparison results vs. dataset, object and version }; } // namespace cbm::qa::checker diff --git a/core/qa/checker/CbmQaCheckerObjectHandler.h b/core/qa/checker/CbmQaCheckerObjectHandler.h index 2d188b0bbe..8618b618ff 100644 --- a/core/qa/checker/CbmQaCheckerObjectHandler.h +++ b/core/qa/checker/CbmQaCheckerObjectHandler.h @@ -54,7 +54,7 @@ namespace cbm::qa::checker /// @param pDenom Pointer to "denominator" object /// @return Comparison result represented as a bitset in unsigned integer /// TODO: ..... - virtual CmpResult_t Compare(const TNamed* pNum, const TNamed* pDenom) const = 0; + virtual CmpResult Compare(const TNamed* pNum, const TNamed* pDenom) const = 0; /// @brief Compares versions with default and writes result into DB void CompareWithDefault(); @@ -90,7 +90,7 @@ namespace cbm::qa::checker TDirectory* fpOutDir = nullptr; ///< Pointer to directory std::shared_ptr<ObjectDB> fpObjDB = nullptr; ///< Pointer to object database std::shared_ptr<TCanvas> fpCanvas = nullptr; ///< Comparison canvas - std::bitset<sizeof(CmpResult_t) * 8> fOptionBits; ///< Bitset for option + std::bitset<4> fOptionBits; ///< Bitset for option std::string fsObjType = ""; ///< Base type of the object to be handled std::string fsBaseName = ""; ///< Base names of the object diff --git a/core/qa/checker/CbmQaCheckerTypedefs.h b/core/qa/checker/CbmQaCheckerTypedefs.h index 97b7e879d6..2b120c6176 100644 --- a/core/qa/checker/CbmQaCheckerTypedefs.h +++ b/core/qa/checker/CbmQaCheckerTypedefs.h @@ -13,6 +13,7 @@ #include <boost/range/iterator_range.hpp> #include <bitset> +#include <limits> #include <string> #include <unordered_map> #include <vector> @@ -27,11 +28,24 @@ namespace cbm::qa::checker kEND }; + /// @struct CmpResult + /// @brief Comparison result + struct CmpResult { + // results + float fRatioUp{std::numeric_limits<float>::signaling_NaN()}; + float fRatioLo{std::numeric_limits<float>::signaling_NaN()}; + float fPval{std::numeric_limits<float>::signaling_NaN()}; + bool fbPointByPoint{false}; // non-equal + // if test was performed + bool fbPointByPointTested{false}; + bool fbRatioTested{false}; + bool fbChi2Tested{false}; + }; + // ----- Aliases using MapStrToStr_t = std::unordered_map<std::string, std::string>; using MapStrToStrVect_t = std::unordered_map<std::string, std::vector<std::string>>; using FlagBitSet_t = std::bitset<static_cast<int>(EFlagBit::kEND)>; - using CmpResult_t = uint16_t; ///< Bitset to keep the comparison result template<class T> using VectRange_t = boost::iterator_range<typename std::vector<T>::iterator>; -- GitLab