From ce68b9c1bec03471d7e7aaddba072305616ad1f5 Mon Sep 17 00:00:00 2001
From: P-A Loizeau <p.-a.loizeau@gsi.de>
Date: Fri, 24 May 2024 15:38:43 +0200
Subject: [PATCH] [CI/CD/QA] Fix-improve Weekly-SLURM-job-End-time Finder
script for Virgo3
---
scripts/find_slurm_ctest_stop_time.sh | 50 ++++++++++++++++-----------
1 file changed, 30 insertions(+), 20 deletions(-)
diff --git a/scripts/find_slurm_ctest_stop_time.sh b/scripts/find_slurm_ctest_stop_time.sh
index 2f1e7bb17e..fcf534e3b3 100755
--- a/scripts/find_slurm_ctest_stop_time.sh
+++ b/scripts/find_slurm_ctest_stop_time.sh
@@ -19,28 +19,38 @@
#>> ./Dart.sh Weekly ./${DART_CFG} &> ${OUT_DIR}/Dart_weekly.log
# SLURM end time
-END_TIME=`squeue -j ${SLURM_JOB_ID} -h --Format EndTime`
-CET_CEST=`date +"%Z"`
-echo "SLURM Job end time ${END_TIME} probably ${CET_CEST}"
+# 1. From SLURM env variables if possible (virgo3)
+if [[ -n ${SLURM_JOB_END_TIME} ]]; then
+ echo "Env variable SLURM_JOB_END_TIME=${SLURM_JOB_END_TIME}"
+ SLURM_JOB_END_TIME_DATE=`date -d "@${SLURM_JOB_END_TIME}"`
+ echo "SLURM_JOB_END_TIME in human format: ${SLURM_JOB_END_TIME_DATE}"
+ CTEST_END_TIME_LIMIT=`date -d "${SLURM_JOB_END_TIME_DATE} -5minutes" +"%H:%M:%S"`
+else
+ # 2. From what squeue return if not possible directly from environment (virgo2)
+ # ==> Broken from bare-metal submit nodes end May 2024, maybe bug?
+ END_TIME=`squeue -j ${SLURM_JOB_ID} -h --Format EndTime`
+ CET_CEST=`date +"%Z"`
+ echo "SLURM Job end time ${END_TIME} probably ${CET_CEST}"
-# Ctest end time = SLURM -5 minutes (should be enough for coverage and uploads)
-# => Not working as ctest expects the "time point in hours-minutes-seconds within current day"
-# + hardcode the day/year/month to the current day...
-# https://gitlab.kitware.com/cmake/cmake/-/blob/master/Help/manual/ctest.1.rst#L440
-# https://gitlab.kitware.com/cmake/cmake/-/blob/master/Source/cmCTest.cxx#L1991
-# https://gitlab.kitware.com/cmake/cmake/-/blob/master/Source/cmCTest.cxx#L3012
-# https://gitlab.kitware.com/cmake/cmake/-/blob/master/Source/CTest/cmCTestRunTest.cxx#L811
-# => Does not work for long running tests close to end of the day
-# CTEST_END_TIME_LIMIT=`date -d "${END_TIME}CET -5minutes" +"%Y-%m-%d %H:%M:%S %z"`
-# => But following should be ok for weeklies starting early in the morning
-# => Does not work at least with cmake 3.22 and 3.24, seesm that timezone is breaking decoding
-# (tried all timezone options of date, with and without space between time and timezone)
-#CTEST_END_TIME_LIMIT=`date -d "${END_TIME}CET -5minutes" +"%H:%M:%S %z"`
-# => Working! but probably unsafe on day of Summer time swaps + if tests run close to midnight...
-# => Fixed for summer time but will probably still fail on the day of the swap itself... so twice a year
-CTEST_END_TIME_LIMIT=`date -d "${END_TIME}${CET_CEST} -5minutes" +"%H:%M:%S"`
-echo "Setting the job time limit for ctest to ${CTEST_END_TIME_LIMIT} to make sure CDASH data is uploaded"
+ # Ctest end time = SLURM -5 minutes (should be enough for coverage and uploads)
+ # => Not working as ctest expects the "time point in hours-minutes-seconds within current day"
+ # + hardcode the day/year/month to the current day...
+ # https://gitlab.kitware.com/cmake/cmake/-/blob/master/Help/manual/ctest.1.rst#L440
+ # https://gitlab.kitware.com/cmake/cmake/-/blob/master/Source/cmCTest.cxx#L1991
+ # https://gitlab.kitware.com/cmake/cmake/-/blob/master/Source/cmCTest.cxx#L3012
+ # https://gitlab.kitware.com/cmake/cmake/-/blob/master/Source/CTest/cmCTestRunTest.cxx#L811
+ # => Does not work for long running tests close to end of the day
+ # CTEST_END_TIME_LIMIT=`date -d "${END_TIME}CET -5minutes" +"%Y-%m-%d %H:%M:%S %z"`
+ # => But following should be ok for weeklies starting early in the morning
+ # => Does not work at least with cmake 3.22 and 3.24, seesm that timezone is breaking decoding
+ # (tried all timezone options of date, with and without space between time and timezone)
+ #CTEST_END_TIME_LIMIT=`date -d "${END_TIME}CET -5minutes" +"%H:%M:%S %z"`
+ # => Working! but probably unsafe on day of Summer time swaps + if tests run close to midnight...
+ # => Fixed for summer time but will probably still fail on the day of the swap itself... so twice a year
+ CTEST_END_TIME_LIMIT=`date -d "${END_TIME}${CET_CEST} -5minutes" +"%H:%M:%S"`
+fi
+echo "Setting the job time limit for ctest to ${CTEST_END_TIME_LIMIT} to make sure CDASH data is uploaded"
if [[ $# -eq 1 ]]; then
# Set the export in the chosen Dart.cfg file
echo "export CTEST_END_TIME_LIMIT=\"${CTEST_END_TIME_LIMIT}\"" >> $1
--
GitLab