From 1815aaab6081068d32af72ea44a9734a107fd677 Mon Sep 17 00:00:00 2001
From: "s.zharko@gsi.de" <s.zharko@gsi.de>
Date: Sat, 10 Feb 2024 04:09:11 +0100
Subject: [PATCH] Ca: multithreaded track finding
---
algo/ca/core/tracking/CaFramework.h | 7 +--
algo/ca/core/tracking/CaTrackFinder.cxx | 57 +++++++++++++++----
algo/ca/core/tracking/CaTrackFinder.h | 3 +
algo/ca/core/tracking/CaTrackFinderWindow.cxx | 24 ++++----
algo/ca/core/tracking/CaTrackFinderWindow.h | 10 +++-
5 files changed, 72 insertions(+), 29 deletions(-)
diff --git a/algo/ca/core/tracking/CaFramework.h b/algo/ca/core/tracking/CaFramework.h
index 50e1a7069f..9d9ec0488b 100644
--- a/algo/ca/core/tracking/CaFramework.h
+++ b/algo/ca/core/tracking/CaFramework.h
@@ -226,7 +226,7 @@ namespace cbm::algo::ca
TrackingMonitorData fMonitorData{}; ///< Tracking monitor data (statistics per call)
- int fNofThreads = 1; ///< Number of threads to execute the track-finder
+ int fNofThreads = 10; ///< Number of threads to execute the track-finder
public:
Vector<CaHitTimeInfo> fHitTimeInfo;
@@ -243,11 +243,6 @@ namespace cbm::algo::ca
Vector<Track> fRecoTracks{"Framework::fRecoTracks"}; ///< reconstructed tracks
Vector<ca::HitIndex_t> fRecoHits{"Framework::fRecoHits"}; ///< packed hits of reconstructed tracks
-
- // WARN: Potential race conditions ->
- Vector<int> fHitKeyToTrack{"Framework::fHitKeyToTrack"}; // strip to track pointers
-
-
fvec EventTime{0.f};
fvec Err{0.f};
diff --git a/algo/ca/core/tracking/CaTrackFinder.cxx b/algo/ca/core/tracking/CaTrackFinder.cxx
index e7679e3efe..2dfedb8f9a 100644
--- a/algo/ca/core/tracking/CaTrackFinder.cxx
+++ b/algo/ca/core/tracking/CaTrackFinder.cxx
@@ -23,6 +23,7 @@
#include "CaTrack.h"
#include <chrono>
+#include <thread>
using namespace cbm::algo::ca;
@@ -57,12 +58,15 @@ void TrackFinder::FindTracks()
// ----- Reset data arrays -------------------------------------------------------------------------------------------
frAlgo.fvHitKeyFlags.reset(frAlgo.fInputData.GetNhitKeys(), 0);
- frAlgo.fRecoTracks.clear();
- frAlgo.fRecoHits.clear();
- frAlgo.fRecoHits.reserve(2 * frAlgo.fInputData.GetNhits());
- frAlgo.fRecoTracks.reserve(2 * frAlgo.fInputData.GetNhits() / frAlgo.GetParameters().GetNstationsActive());
+ size_t nHitsExpected = 2 * frAlgo.fInputData.GetNhits();
+ size_t nTracksExpected = 2 * frAlgo.fInputData.GetNhits() / frAlgo.GetParameters().GetNstationsActive();
for (int iThread = 0; iThread < frAlgo.GetNofThreads(); ++iThread) {
+ fvRecoTracks[iThread].clear();
+ fvRecoTracks[iThread].reserve(nTracksExpected / frAlgo.GetNofThreads());
+ fvRecoHitIndices[iThread].clear();
+ fvRecoHitIndices[iThread].reserve(nHitsExpected / frAlgo.GetNofThreads());
+ frAlgo.fvTrackFinderWindow[iThread].InitTimeslice();
for (int iS = 0; iS < frAlgo.GetParameters().GetNstationsActive(); ++iS) {
frAlgo.fvWData[iThread].TsHitIndices(iS).clear();
frAlgo.fvWData[iThread].TsHitIndices(iS).reserve(frAlgo.fInputData.GetNhits());
@@ -70,7 +74,6 @@ void TrackFinder::FindTracks()
}
frAlgo.fvHitKeyFlags.reset(frAlgo.fInputData.GetNhitKeys(), 0);
- frAlgo.fHitKeyToTrack.reset(frAlgo.fInputData.GetNhitKeys(), -1);
frAlgo.fHitTimeInfo.reset(frAlgo.fInputData.GetNhits());
@@ -207,8 +210,32 @@ void TrackFinder::FindTracks()
<< fvWindowEndThread[iThread] / 1.e6 << " ms";
}
- for (int iThread = 0; iThread < frAlgo.GetNofThreads(); ++iThread) {
- this->FindTracksThread(iThread);
+ // Save tracks
+ frAlgo.fRecoTracks.clear();
+ frAlgo.fRecoHits.clear();
+ if (frAlgo.GetNofThreads() == 1) {
+ this->FindTracksThread(0);
+ frAlgo.fRecoTracks = std::move(fvRecoTracks[0]);
+ frAlgo.fRecoHits = std::move(fvRecoHitIndices[0]);
+ }
+ else {
+ std::vector<std::thread> vThreadList;
+ vThreadList.reserve(frAlgo.GetNofThreads());
+ for (int iTh = 0; iTh < frAlgo.GetNofThreads(); ++iTh) {
+ vThreadList.emplace_back(&TrackFinder::FindTracksThread, this, iTh);
+ }
+ for (auto& th: vThreadList) {
+ if (th.joinable()) { th.join(); }
+ }
+ auto Operation = [](size_t acc, const Vector<auto>& v) { return acc + v.size(); };
+ int nRecoTracks = std::accumulate(fvRecoTracks.begin(), fvRecoTracks.end(), 0, Operation);
+ int nRecoHits = std::accumulate(fvRecoHitIndices.begin(), fvRecoHitIndices.end(), 0, Operation);
+ frAlgo.fRecoTracks.reserve(nRecoTracks);
+ frAlgo.fRecoHits.reserve(nRecoHits);
+ for (int iTh = 0; iTh < frAlgo.GetNofThreads(); ++iTh) {
+ frAlgo.fRecoTracks.insert(frAlgo.fRecoTracks.end(), fvRecoTracks[iTh].begin(), fvRecoTracks[iTh].end());
+ frAlgo.fRecoHits.insert(frAlgo.fRecoHits.end(), fvRecoHitIndices[iTh].begin(), fvRecoHitIndices[iTh].end());
+ }
}
frAlgo.fMonitorData.StopTimer(ETimer::FindTracks);
@@ -331,7 +358,7 @@ void TrackFinder::FindTracksThread(int iThread)
<< statNwindowHits << " hits. "
<< " Processing " << dataRead << " % of the TS time and "
<< 100. * fvStatNhitsProcessed[iThread] / fStatNhitsTotal << " % of TS hits."
- << " Already reconstructed " << frAlgo.fRecoTracks.size() << " tracks ";
+ << " Already reconstructed " << fvRecoTracks[iThread].size() << " tracks on thread #" << iThread;
}
}
@@ -389,14 +416,14 @@ void TrackFinder::FindTracksThread(int iThread)
}
}
else { // save the track
- frAlgo.fRecoTracks.push_back(track);
+ fvRecoTracks[iThread].push_back(track);
// mark the track hits as used
for (int i = 0; i < track.fNofHits; i++) {
int caHitId = frAlgo.fvWData[iThread].RecoHitIndex(trackFirstHit + i);
const auto& h = frAlgo.fInputData.GetHit(caHitId);
frAlgo.fvHitKeyFlags[h.FrontKey()] = 1;
frAlgo.fvHitKeyFlags[h.BackKey()] = 1;
- frAlgo.fRecoHits.push_back(caHitId);
+ fvRecoHitIndices[iThread].push_back(caHitId);
}
}
trackFirstHit += track.fNofHits;
@@ -421,12 +448,20 @@ void TrackFinder::Init()
fvWindowEndThread.clear();
fvStatNwindows.clear();
fvStatNhitsProcessed.clear();
-
+ fvRecoTracks.clear();
+ fvRecoHitIndices.clear();
+
fvWindowStartThread.resize(frAlgo.GetNofThreads());
fvWindowEndThread.resize(frAlgo.GetNofThreads());
fvStatNwindows.resize(frAlgo.GetNofThreads());
fvStatNhitsProcessed.resize(frAlgo.GetNofThreads());
+ fvRecoTracks.resize(frAlgo.GetNofThreads());
+ fvRecoHitIndices.resize(frAlgo.GetNofThreads());
+ for (int iThread = 0; iThread < frAlgo.GetNofThreads(); ++iThread) {
+ fvRecoTracks[iThread].SetName(std::string("TrackFinder::fvRecoTracks_") + std::to_string(iThread));
+ fvRecoHitIndices[iThread].SetName(std::string("TrackFinder::fvRecoHitIndices_") + std::to_string(iThread));
+ }
fWindowLength = (ca::Framework::TrackingMode::kMcbm == frAlgo.fTrackingMode) ? 500 : 10000;
}
diff --git a/algo/ca/core/tracking/CaTrackFinder.h b/algo/ca/core/tracking/CaTrackFinder.h
index ba5eca97e0..a8f33ab701 100644
--- a/algo/ca/core/tracking/CaTrackFinder.h
+++ b/algo/ca/core/tracking/CaTrackFinder.h
@@ -70,6 +70,9 @@ namespace cbm::algo::ca
std::vector<int> fvStatNwindows;
std::vector<int> fvStatNhitsProcessed;
+ std::vector<Vector<Track>> fvRecoTracks; ///< reconstructed tracks
+ std::vector<Vector<ca::HitIndex_t>> fvRecoHitIndices; ///< packed hits of reconstructed tracks
+
float fWindowLength = 0.;
float fWindowOverlap = 15.; // ns
diff --git a/algo/ca/core/tracking/CaTrackFinderWindow.cxx b/algo/ca/core/tracking/CaTrackFinderWindow.cxx
index f25135dc4c..5e8f267609 100644
--- a/algo/ca/core/tracking/CaTrackFinderWindow.cxx
+++ b/algo/ca/core/tracking/CaTrackFinderWindow.cxx
@@ -119,6 +119,10 @@ bool TrackFinderWindow::checkTripletMatch(const ca::Triplet& l, const ca::Triple
return true;
}
+void TrackFinderWindow::InitTimeslice()
+{
+ fvHitKeyToTrack.reset(frAlgo.fInputData.GetNhitKeys(), -1);
+}
// **************************************************************************************************
// * *
@@ -428,8 +432,8 @@ void TrackFinderWindow::CaTrackFinderSlice()
fvTrackCandidates.clear();
for (const auto& h : frWData.Hits()) {
- frAlgo.fHitKeyToTrack[h.FrontKey()] = -1;
- frAlgo.fHitKeyToTrack[h.BackKey()] = -1;
+ fvHitKeyToTrack[h.FrontKey()] = -1;
+ fvHitKeyToTrack[h.BackKey()] = -1;
}
//== Loop over triplets with the required level, find and store track candidates
@@ -542,7 +546,7 @@ void TrackFinderWindow::CaTrackFinderSlice()
const ca::Hit& h = frAlgo.fInputData.GetHit(hitId);
bool isAlive = true;
{ // front strip
- auto& stripF = (frAlgo.fHitKeyToTrack)[h.FrontKey()];
+ auto& stripF = fvHitKeyToTrack[h.FrontKey()];
if ((stripF >= 0) && (stripF != tr.Id())) { // strip is used by other candidate
const auto& other = fvTrackCandidates[stripF];
if (!other.IsAlive() && tr.IsBetterThan(other)) {
@@ -561,7 +565,7 @@ void TrackFinderWindow::CaTrackFinderSlice()
}
{ // back strip
- auto& stripB = (frAlgo.fHitKeyToTrack)[h.BackKey()];
+ auto& stripB = fvHitKeyToTrack[h.BackKey()];
if ((stripB >= 0) && (stripB != tr.Id())) { // strip is used by other candidate
const auto& other = fvTrackCandidates[stripB];
if (!other.IsAlive() && tr.IsBetterThan(other)) {
@@ -591,18 +595,18 @@ void TrackFinderWindow::CaTrackFinderSlice()
tr.SetAlive(true);
for (int iHit = 0; tr.IsAlive() && (iHit < (int) tr.Hits().size()); ++iHit) {
const ca::Hit& h = frAlgo.fInputData.GetHit(tr.Hits()[iHit]);
- tr.SetAlive(tr.IsAlive() && ((frAlgo.fHitKeyToTrack)[h.FrontKey()] == tr.Id())
- && ((frAlgo.fHitKeyToTrack)[h.BackKey()] == tr.Id()));
+ tr.SetAlive(tr.IsAlive() && (fvHitKeyToTrack[h.FrontKey()] == tr.Id())
+ && (fvHitKeyToTrack[h.BackKey()] == tr.Id()));
}
if (!tr.IsAlive()) { // release strips
for (auto hitId : tr.Hits()) {
const ca::Hit& h = frAlgo.fInputData.GetHit(hitId);
- if (frAlgo.fHitKeyToTrack[h.FrontKey()] == tr.Id()) {
- frAlgo.fHitKeyToTrack[h.FrontKey()] = -1;
+ if (fvHitKeyToTrack[h.FrontKey()] == tr.Id()) {
+ fvHitKeyToTrack[h.FrontKey()] = -1;
}
- if (frAlgo.fHitKeyToTrack[h.BackKey()] == tr.Id()) {
- frAlgo.fHitKeyToTrack[h.BackKey()] = -1;
+ if (fvHitKeyToTrack[h.BackKey()] == tr.Id()) {
+ fvHitKeyToTrack[h.BackKey()] = -1;
}
}
}
diff --git a/algo/ca/core/tracking/CaTrackFinderWindow.h b/algo/ca/core/tracking/CaTrackFinderWindow.h
index 2813e5ccc2..b47eab9807 100644
--- a/algo/ca/core/tracking/CaTrackFinderWindow.h
+++ b/algo/ca/core/tracking/CaTrackFinderWindow.h
@@ -60,6 +60,9 @@ namespace cbm::algo::ca
void CAFindTrack(int ista, ca::Branch& best_tr, const ca::Triplet* curr_trip, ca::Branch& curr_tr,
unsigned char min_best_l, ca::Branch* new_tr);
+ /// \note The function initializes global arrays for a given thread
+ void InitTimeslice();
+
private:
///-------------------------------
/// Private methods
@@ -78,8 +81,11 @@ namespace cbm::algo::ca
/// \note The candidates may share any amount of hits.
Vector<ca::Branch> fvTrackCandidates{"TrackFinderWindow::fTrackCandidates"};
- Vector<int> fvHitFirstTriplet{"Framework::fHitFirstTriplet"}; /// link hit -> first triplet { hit, *, *}
- Vector<int> fvHitNofTriplets{"Framework::fHitNofTriplets"}; /// link hit ->n triplets { hit, *, *}
+ Vector<int> fvHitFirstTriplet{"TrackFinderWindow::fvHitFirstTriplet"}; /// link hit -> first triplet { hit, *, *}
+ Vector<int> fvHitNofTriplets{"TrackFinderWindow::fvHitNofTriplets"}; /// link hit ->n triplets { hit, *, *}
+
+ /// \note Global array for a given thread
+ Vector<int> fvHitKeyToTrack{"TrackFinderWindow::fvHitKeyToTrack"};
ca::Framework& frAlgo; ///< Reference to the main track finder algorithm class
static constexpr bool fDebug = false; // print debug info
--
GitLab