From 86ba019d32e52f87d681d0ed954b9351a6c0013d Mon Sep 17 00:00:00 2001
From: P-A Loizeau <p.-a.loizeau@gsi.de>
Date: Mon, 8 Jul 2024 12:21:12 +0200
Subject: [PATCH] [mCBM 2024] Improve chain reco script w/ step check + bug
fixes + fix L1 macro
- Store logs of each step in temp file + check if success pattern found
- Stop early if a step fails
- Fix some warnings for printf usage from ShellCheck online checker
- Fix some variables names (typos?) found by ShellCheck
- Avoid overwriting reco output with symbolic link
- In L1 macro, avoir error if output folder already existing (mkdir option)
- In L1 macro, set verbosity one level up to LOW to see severity
---
macro/beamtime/mcbm2024/mcbm_event_reco_L1.C | 4 +-
macro/beamtime/mcbm2024/reco_mcbm.sh | 97 +++++++++++++++++---
2 files changed, 84 insertions(+), 17 deletions(-)
diff --git a/macro/beamtime/mcbm2024/mcbm_event_reco_L1.C b/macro/beamtime/mcbm2024/mcbm_event_reco_L1.C
index f0b54c19a2..a924f1fd3d 100644
--- a/macro/beamtime/mcbm2024/mcbm_event_reco_L1.C
+++ b/macro/beamtime/mcbm2024/mcbm_event_reco_L1.C
@@ -43,7 +43,7 @@ Bool_t mcbm_event_reco_L1(UInt_t uRunId = 3105,
// --- Logger settings ----------------------------------------------------
TString logLevel = "INFO"; //"INFO";
- TString logVerbosity = "VERYLOW"; //"VERYLOW";
+ TString logVerbosity = "LOW"; //"VERYLOW";
// ------------------------------------------------------------------------
// ----- Environment --------------------------------------------------
@@ -56,7 +56,7 @@ Bool_t mcbm_event_reco_L1(UInt_t uRunId = 3105,
/// Standardized RUN ID
TString sRunId = TString::Format("%04u", uRunId);
- gSystem->Exec("mkdir " + sOutDir);
+ gSystem->Exec("mkdir -p " + sOutDir);
gSystem->Exec("cp $VMCWORKDIR/macro/run/.rootrc .");
TString outFile = sOutDir + "/" + sRunId;
diff --git a/macro/beamtime/mcbm2024/reco_mcbm.sh b/macro/beamtime/mcbm2024/reco_mcbm.sh
index f3266b2bf3..0f17266377 100755
--- a/macro/beamtime/mcbm2024/reco_mcbm.sh
+++ b/macro/beamtime/mcbm2024/reco_mcbm.sh
@@ -50,9 +50,25 @@
#
# 4. Knwon missing features
#
-# - Automated detection of success/failure if a given step to avoid running unnecessary steps
-# - Proper evaluation and printout of script result (reaching the end is not always success)
# - Script and its sub-process is not killed if CTEST is stopped with CTRL+C
+# - Error in reco of run 3105:
+# CbmAlgoBuildRawEvents::filterBmon: Bmon digi with wrong address [GetSmType() != 5]. Skip.
+# - LOG to change from INFO to DEBUG in RECO CA step:
+#[2024-07-08 10:11:10.301054] [0x00007f96766b4a40] [info] ca: Hit out of range: iHit = 35, time = 126.389 ms, window = 1
+#
+# - LOG to change from INFO to DEBUG in RECO CA step:
+# ======= Ca Track finder: process event 17297 ...
+# [2024-07-08 10:11:10.375592] [0x00007f96766b4a40] [info] CA tracker process time slice 127.431 -- 127.431 [ms] with 47 hits
+# [2024-07-08 10:11:10.375597] [0x00007f96766b4a40] [info] Fraction of hits from monster events: 0
+# [2024-07-08 10:11:10.375603] [0x00007f96766b4a40] [info] Thread: 0 from 127.431 ms to 127.431 ms (delta = 0.00012 ms)
+# [2024-07-08 10:11:10.376800] [0x00007f96766b4a40] [info] CA tracker: time slice finished. Reconstructed 4 tracks with 16 hits. Processed 47 hits in 1 time windows. Reco time 1.21725e-12 s
+# Ca Track Finder finished, found 4 tracks
+# CA Track Finder: 0.00121725 s/sub-ts
+# Tracking performance... done
+# End of CA
+# CA: N hits used/tot = 46/49
+#
+# - LOG to change from INFO to DEBUG in QA step: [root.exe][10:11:24][INFO] CA: N hits used/tot = 17/18
#
# *********************
# ** USER PARAMETERS **
@@ -101,10 +117,10 @@ ONLINE_PAR=${VMCWORKDIR}/parameters/online
RUN=-1
TSA=
N_TS=-1
-while [[ $# > 0 ]]; do
+while [[ $# -gt 0 ]]; do
case ${1} in
--setup )
- CREATE_SETUP=1
+ DO_CREATE_SETUP=1
;;
--unpack )
DO_UNPACK=1
@@ -197,7 +213,7 @@ elif [[ ${RUN} -ge 2918 ]]; then
SETUP_NAME="mcbm_beam_2024_05_08_nickel"
MACRO_RECO="${VMCWORKDIR}/macro/beamtime/mcbm2024/mcbm_event_reco_L1.C"
else
- printf "E- Run ${RUN} is undefined. Exiting"
+ printf "E- Run %5d is undefined. Exiting" "${RUN}"
exit 4
fi
@@ -207,13 +223,29 @@ FILE_LABEL=$(printf $(basename ${TSA_INP}) | cut -f 1 -d '.')
OUT_DIGI="${DATA_TOP_DIR}/${FILE_LABEL}.digi.out"
INP_FILE="${DATA_TOP_DIR}/${FILE_LABEL}.digi.root"
+# -----
+# Log Files
+SETUP_LOG="run_mcbm_setup.log"
+UNPACK_LOG="run_mcbm_unpack.log"
+RECO_LOG="run_mcbm_reco.log"
+RECO_QA_LOG="run_mcbm_reco_qa.log"
+QA_LOG="run_mcbm_qa.log"
+
# *********************
# ** Steps execution **
# *********************
# ----- Create setup files
-if [[ ${CREATE_SETUP} -eq 1 ]]; then
- root -b -l -q ${MACRO_SETUP}"(${RUN}, \"${DATA_TOP_DIR}\")"
+if [[ ${DO_CREATE_SETUP} -eq 1 ]]; then
+ root -b -l -q ${MACRO_SETUP}"(${RUN}, \"${DATA_TOP_DIR}\")" &> ${SETUP_LOG}
+ cat ${SETUP_LOG}
+
+ if [[ (1 -ne $(grep -c " Test passed" "${SETUP_LOG}")) || (1 -ne $(grep -c " All ok " "${SETUP_LOG}")) ]]; then
+ printf "\nSetup file creation for %s failed, stopping there\n" "${TSA_INP}"
+ rm ${SETUP_LOG}
+ exit 5
+ fi
+ rm ${SETUP_LOG}
fi
# ----- Run unpacker
@@ -221,8 +253,16 @@ if [[ ${DO_UNPACK} -eq 1 ]]; then
# TODO: Define unpack options for different setups
${ONLINE_BINARY} --steps Unpack -i ${TSA_INP} -r ${RUN} -p ${ONLINE_PAR} -O DigiTimeslice -o ${OUT_DIGI} -s STS BMON TOF TRD RICH -n ${N_TS}
- root -l -b -q ${VMCWORKDIR}/macro/run/run_inspect_digi_timeslice.C"(\"${OUT_DIGI}\", \"${INP_FILE}\")"
+ root -l -b -q ${VMCWORKDIR}/macro/run/run_inspect_digi_timeslice.C"(\"${OUT_DIGI}\", \"${INP_FILE}\")" &> ${UNPACK_LOG}
+ cat ${UNPACK_LOG}
rm ${OUT_DIGI}
+
+ if [[ 1 -ne $(grep -c "Macro finished successfully." "${UNPACK_LOG}") ]]; then
+ printf "\nUnpacked data file conversion of %s failed, stopping there\n" "${TSA_INP}"
+ rm ${UNPACK_LOG}
+ exit 6
+ fi
+ rm ${UNPACK_LOG}
fi
# ----- Run reconstruction
@@ -233,18 +273,36 @@ if [[ ${DO_RECO} -eq 1 ]]; then
PARS="${RUN},${N_TS},\"${DATA_PREF}\",\"${DATA_PREF}\",${UNP_FILE_ID},${RECO_MVD},${RECO_STS},${RECO_TRD}"
PARS="${PARS},${RECO_TRD2d},${RECO_RICH},${RECO_MUCH},${RECO_TOF},${RECO_TOFtr},${RECO_PSD},${RECO_ALI},${RECO_EvB}"
PARS="${PARS},${RECO_CA},${RECO_QA},${RECO_FSD},\"${INP_FILE}\",${RECO_PV}"
- root -b -l -q ${MACRO_RECO}"(${PARS})"
- ln -s "${FILE_LABEL}.digi.root" "${DATA_TOP_DIR}/${FILE_LABEL}.raw.root" # TMP for QA
- ln -s "${FILE_LABEL}.reco.root" "${DATA_TOP_DIR}/${FILE_LABEL}.rec.root" # TMP for QA
+ root -b -l -q ${MACRO_RECO}"(${PARS})" &> ${RECO_LOG}
+ cat ${RECO_LOG}
+
+ if [[ (1 -ne $(grep -c " Test passed" "${RECO_LOG}")) || (1 -ne $(grep -c " All ok " "${RECO_LOG}")) ]]; then
+ printf "\nReconstruction of %s failed, stopping there\n" "${TSA_INP}"
+ rm ${RECO_LOG}
+ exit 7
+ fi
+ rm ${RECO_LOG}
+
+ ln -s -f "${FILE_LABEL}.digi.root" "${DATA_TOP_DIR}/${FILE_LABEL}.raw.root" # TMP for QA
+ # Commented out as the output of mcbm_event_reco_L1.C is already [...].rec.root
+ # ln -s -f "${FILE_LABEL}.reco.root" "${DATA_TOP_DIR}/${FILE_LABEL}.rec.root" # TMP for QA
fi
# ----- Run QA of reco modules
if [[ ${DO_QA_MODULE} -eq 1 ]]; then
QA_GEO="${DATA_TOP_DIR}/${SETUP_NAME}.geo.root"
QA_REC="${DATA_TOP_DIR}/${FILE_LABEL}.rec.root"
- ln -s ${QA_GEO} "${SETUP_NAME}.geo.root"
+ ln -s -f ${QA_GEO} "${SETUP_NAME}.geo.root"
PARS="-1,\"${QA_REC}\",\"${SETUP_NAME}\",kFALSE"
- root -b -l -q ${MACRO_QA_ALEX}"(${PARS})"
+ root -b -l -q ${MACRO_QA_MODULE}"(${PARS})" &> ${RECO_QA_LOG}
+ cat ${RECO_QA_LOG}
+
+ if [[ (1 -ne $(grep -c " Test passed" "${RECO_QA_LOG}")) || (1 -ne $(grep -c " All ok " "${RECO_QA_LOG}")) ]]; then
+ printf "\nReco Modules QA for %s failed, stopping there\n" "${TSA_INP}"
+ rm ${RECO_QA_LOG}
+ exit 8
+ fi
+ rm ${RECO_QA_LOG}
fi
# ----- Run QA
@@ -261,7 +319,16 @@ if [[ ${DO_QA} -eq 1 ]]; then
PARS="0,\"\",\"${QA_RAW}\",\"${QA_REC}\",\"${QA_PAR}\",\"${QA_GEO}\",\"${QA_OUT}\",\"${SETUP_NAME}\""
PARS="${PARS},${USE_MC},\"${CONFIG}\",\"${BENCHMARK}\""
- root -b -l -q ${MACRO_QA}"(${PARS})"
+ root -b -l -q ${MACRO_QA}"(${PARS})" &> ${QA_LOG}
+ cat ${QA_LOG}
+
+ if [[ (1 -eq $(grep -c " QA checks failed" "${QA_LOG}")) || (1 -ne $(grep -c " Test passed" "${QA_LOG}"))
+ || (1 -ne $(grep -c " All ok " "${QA_LOG}")) ]]; then
+ printf "\nFull QA for %s failed, stopping there\n" "${TSA_INP}"
+ rm ${QA_LOG}
+ exit 9
+ fi
+ rm ${QA_LOG}
fi
-printf "Reconstruction of ${TSA_INP} succeeded\n"
+printf "Reconstruction of %s succeeded\n" "${TSA_INP}"
--
GitLab