[CI] Add detection of job timeout and total test time TO check to CI

.gitlab-ci.yml

@@ -233,6 +233,10 @@ FileLicenceCheck:
     - if [[ -n $INSTALL_PROJECT ]]; then
     -   echo "export INSTALL_PROJECT=TRUE" >> Dart.cfg
     - fi
+    - ./scripts/find_cijob_ctest_stop_time.sh Dart.cfg
+    - if [[ -n $CI_TEST_STAGE_TOTAL_TIME_LIMIT ]]; then
+    -   echo "export CI_TEST_STAGE_TOTAL_TIME_LIMIT=$CI_TEST_STAGE_TOTAL_TIME_LIMIT"  >> Dart.cfg
+    - fi
     - if [[ -n $CONFIGFILE ]]; then
     -   source $CONFIGFILE $FAIRSOFT_VERSION
     - fi
@@ -258,6 +262,7 @@ FileLicenceCheck:
     SIMPATH: "$BASE_PATH/fairsoft/$FAIRSOFT_VERSION"
     FAIRROOTPATH: "$BASE_PATH/fairroot/${FAIRROOT_VERSION}_${FAIRSOFT_VERSION}"
     CHECK_GEO_HASH_CHANGE: "1"
+    CI_TEST_STAGE_TOTAL_TIME_LIMIT: 1500
 
 .apptainer_tag: &apptainer_tag
   stage: build

CbmRoot_test.cmake

@@ -119,6 +119,25 @@ If(NOT _RETVAL)
     endif()
   EndIf()
 
+  If(DEFINED ENV{CI_TEST_STAGE_TOTAL_TIME_LIMIT})
+    # Compute test stage timeout based on current time + limit from environment variable
+    If(DEFINED ENV{CTEST_END_TIME_LIMIT})
+      # If also full DASH run timeout defined, also compare the two and keep the earliest one
+      execute_process (
+        COMMAND scripts/find_citests_ctest_stop_time.sh $ENV{CTEST_END_TIME_LIMIT}
+        OUTPUT_VARIABLE CI_TESTS_TOTAL_END_TIME_DATE
+      )
+    Else()
+      # Else just get the CI jpb timeout
+      execute_process (
+        COMMAND scripts/find_citests_ctest_stop_time.sh
+        OUTPUT_VARIABLE CI_TESTS_TOTAL_END_TIME_DATE
+      )
+    EndIf()
+    set(stop_time STOP_TIME "${CI_TESTS_TOTAL_END_TIME_DATE}")
+    message(STATUS "Due to CI mode, Set end time for the ctest test runs to ${stop_time}")
+  EndIf()
+
   Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}"
              PARALLEL_LEVEL $ENV{number_of_processors}
              ${repeat}

scripts/find_cijob_ctest_stop_time.sh

+#!/bin/bash
+# Copyright (C) 2024 GSI Helmholtzzentrum fuer Schwerionenforschung, Darmstadt
+# SPDX-License-Identifier: GPL-3.0-only
+# First commited by Pierre-Alain Loizeau
+
+# Script used to compute a "safe" end time for the ctest test stage of a Dart/Cdash chain in order
+# to have enough time to upload the CDASH results before a job is killed in a Gitlab CI pipeline
+# Uses the GITLAB CI automatically pre-defined variable ${CI_JOB_TIMEOUT}
+#
+# Usage:
+# - with a full/relative/absolute path to an output Dart.cfg file as parameter, where the export will be added
+# - without any parameter, then setting an export
+#
+# => Typical use case would be calling it in your sbatch script after setting up the other parts of the
+#    Dart.cfg gile and before the call to Dart.sh
+#>> echo "export NCPU=10" >> ${DART_CFG}
+#>> [...]
+#>> ./scripts/find_cijob_ctest_stop_time.sh ${DART_CFG}
+#>> [...]
+#>> ./Dart.sh Weekly ./${DART_CFG} &> ${OUT_DIR}/Dart_weekly.log
+
+# Job end time
+# 1. Make end time from current time + job timeout (not really precise due to job start time until here but should be
+#    good enough)
+echo "Env. variable CI_JOB_TIMEOUT = ${CI_JOB_TIMEOUT} s"
+CI_JOB_END_TIME_DATE=$(date -d"now +${CI_JOB_TIMEOUT}seconds")
+
+# 2. Remove 5 minutes at the end for safety
+echo "Now in human format: $(date +\"%H:%M:%S\")"
+echo "CI_END_TIME in human format: ${CI_JOB_END_TIME_DATE}"
+CTEST_END_TIME_LIMIT=$(date -d "${CI_JOB_END_TIME_DATE} -5minutes" +"%H:%M:%S")
+
+echo "Setting time limit for ctest to ${CTEST_END_TIME_LIMIT} (5min bef. job TO) to make sure CDASH data is uploaded"
+if [[ $# -eq 1 ]]; then
+  # Set the export in the chosen Dart.cfg file
+  echo "export CTEST_END_TIME_LIMIT=\"${CTEST_END_TIME_LIMIT}\"" >> $1
+else
+  export CTEST_END_TIME_LIMIT=\"${CTEST_END_TIME_LIMIT}\"
+fi

scripts/find_citests_ctest_stop_time.sh

+#!/bin/bash
+# Copyright (C) 2024 GSI Helmholtzzentrum fuer Schwerionenforschung, Darmstadt
+# SPDX-License-Identifier: GPL-3.0-only
+# First commited by Pierre-Alain Loizeau
+
+# Script used to compute a "total timeout" end time for the ctest test stage of a Dart/Cdash chain
+#
+# Usage:
+# - without any parameter, result sent to stdout
+# - with a single parameter, compare the two dates (after conversion to seconds) and return the earliest one
+#   => !!! No check of the format of the parameter before usage, meant to be "HH:MM:SS" !!!
+#
+# => Typical use case would be calling it in the CDASH/ctest cmake between the build and test stages
+
+# Job end time
+# 1. Make end time from current time + job timeout (not really precise due to job start time until here but should be
+#    good enough)
+
+
+CI_TESTS_TOTAL_END_TIME_DATE=$(date -d "now +${CI_TEST_STAGE_TOTAL_TIME_LIMIT}seconds" +"%H:%M:%S")
+
+if [[ $# -eq 1 ]]; then
+  if [[ $(date --date="${1}" +%s) -lt $(date --date="${CI_TESTS_TOTAL_END_TIME_DATE}" +%s) ]]; then
+    CI_TESTS_TOTAL_END_TIME_DATE=$1
+  fi
+fi
+echo ${CI_TESTS_TOTAL_END_TIME_DATE}
+