diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 02dcaa54619e533ef125a8605b22b4e1ced4bab3..5e801e5496acbccce12262bba05033c82f40bb54 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -233,6 +233,10 @@ FileLicenceCheck: - if [[ -n $INSTALL_PROJECT ]]; then - echo "export INSTALL_PROJECT=TRUE" >> Dart.cfg - fi + - ./scripts/find_cijob_ctest_stop_time.sh Dart.cfg + - if [[ -n $CI_TEST_STAGE_TOTAL_TIME_LIMIT ]]; then + - echo "export CI_TEST_STAGE_TOTAL_TIME_LIMIT=$CI_TEST_STAGE_TOTAL_TIME_LIMIT" >> Dart.cfg + - fi - if [[ -n $CONFIGFILE ]]; then - source $CONFIGFILE $FAIRSOFT_VERSION - fi @@ -258,6 +262,7 @@ FileLicenceCheck: SIMPATH: "$BASE_PATH/fairsoft/$FAIRSOFT_VERSION" FAIRROOTPATH: "$BASE_PATH/fairroot/${FAIRROOT_VERSION}_${FAIRSOFT_VERSION}" CHECK_GEO_HASH_CHANGE: "1" + CI_TEST_STAGE_TOTAL_TIME_LIMIT: 1500 .apptainer_tag: &apptainer_tag stage: build diff --git a/CbmRoot_test.cmake b/CbmRoot_test.cmake index 2bf9c186d78e84888be00e0915ab8f7aa9b62e7e..14c9de60e2e3ba9f0c21997168c0b929ff21c1c0 100644 --- a/CbmRoot_test.cmake +++ b/CbmRoot_test.cmake @@ -119,6 +119,25 @@ If(NOT _RETVAL) endif() EndIf() + If(DEFINED ENV{CI_TEST_STAGE_TOTAL_TIME_LIMIT}) + # Compute test stage timeout based on current time + limit from environment variable + If(DEFINED ENV{CTEST_END_TIME_LIMIT}) + # If also full DASH run timeout defined, also compare the two and keep the earliest one + execute_process ( + COMMAND scripts/find_citests_ctest_stop_time.sh $ENV{CTEST_END_TIME_LIMIT} + OUTPUT_VARIABLE CI_TESTS_TOTAL_END_TIME_DATE + ) + Else() + # Else just get the CI jpb timeout + execute_process ( + COMMAND scripts/find_citests_ctest_stop_time.sh + OUTPUT_VARIABLE CI_TESTS_TOTAL_END_TIME_DATE + ) + EndIf() + set(stop_time STOP_TIME "${CI_TESTS_TOTAL_END_TIME_DATE}") + message(STATUS "Due to CI mode, Set end time for the ctest test runs to ${stop_time}") + EndIf() + Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}" PARALLEL_LEVEL $ENV{number_of_processors} ${repeat} diff --git a/scripts/find_cijob_ctest_stop_time.sh b/scripts/find_cijob_ctest_stop_time.sh new file mode 100755 index 0000000000000000000000000000000000000000..2924d8b447c64a86fdf8492d1812795d4ac30a7c --- /dev/null +++ b/scripts/find_cijob_ctest_stop_time.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright (C) 2024 GSI Helmholtzzentrum fuer Schwerionenforschung, Darmstadt +# SPDX-License-Identifier: GPL-3.0-only +# First commited by Pierre-Alain Loizeau + +# Script used to compute a "safe" end time for the ctest test stage of a Dart/Cdash chain in order +# to have enough time to upload the CDASH results before a job is killed in a Gitlab CI pipeline +# Uses the GITLAB CI automatically pre-defined variable ${CI_JOB_TIMEOUT} +# +# Usage: +# - with a full/relative/absolute path to an output Dart.cfg file as parameter, where the export will be added +# - without any parameter, then setting an export +# +# => Typical use case would be calling it in your sbatch script after setting up the other parts of the +# Dart.cfg gile and before the call to Dart.sh +#>> echo "export NCPU=10" >> ${DART_CFG} +#>> [...] +#>> ./scripts/find_cijob_ctest_stop_time.sh ${DART_CFG} +#>> [...] +#>> ./Dart.sh Weekly ./${DART_CFG} &> ${OUT_DIR}/Dart_weekly.log + +# Job end time +# 1. Make end time from current time + job timeout (not really precise due to job start time until here but should be +# good enough) +echo "Env. variable CI_JOB_TIMEOUT = ${CI_JOB_TIMEOUT} s" +CI_JOB_END_TIME_DATE=$(date -d"now +${CI_JOB_TIMEOUT}seconds") + +# 2. Remove 5 minutes at the end for safety +echo "Now in human format: $(date +\"%H:%M:%S\")" +echo "CI_END_TIME in human format: ${CI_JOB_END_TIME_DATE}" +CTEST_END_TIME_LIMIT=$(date -d "${CI_JOB_END_TIME_DATE} -5minutes" +"%H:%M:%S") + +echo "Setting time limit for ctest to ${CTEST_END_TIME_LIMIT} (5min bef. job TO) to make sure CDASH data is uploaded" +if [[ $# -eq 1 ]]; then + # Set the export in the chosen Dart.cfg file + echo "export CTEST_END_TIME_LIMIT=\"${CTEST_END_TIME_LIMIT}\"" >> $1 +else + export CTEST_END_TIME_LIMIT=\"${CTEST_END_TIME_LIMIT}\" +fi diff --git a/scripts/find_citests_ctest_stop_time.sh b/scripts/find_citests_ctest_stop_time.sh new file mode 100755 index 0000000000000000000000000000000000000000..d4a5f530a88f290929c9fe8a7fc47c569696b101 --- /dev/null +++ b/scripts/find_citests_ctest_stop_time.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2024 GSI Helmholtzzentrum fuer Schwerionenforschung, Darmstadt +# SPDX-License-Identifier: GPL-3.0-only +# First commited by Pierre-Alain Loizeau + +# Script used to compute a "total timeout" end time for the ctest test stage of a Dart/Cdash chain +# +# Usage: +# - without any parameter, result sent to stdout +# - with a single parameter, compare the two dates (after conversion to seconds) and return the earliest one +# => !!! No check of the format of the parameter before usage, meant to be "HH:MM:SS" !!! +# +# => Typical use case would be calling it in the CDASH/ctest cmake between the build and test stages + +# Job end time +# 1. Make end time from current time + job timeout (not really precise due to job start time until here but should be +# good enough) + + +CI_TESTS_TOTAL_END_TIME_DATE=$(date -d "now +${CI_TEST_STAGE_TOTAL_TIME_LIMIT}seconds" +"%H:%M:%S") + +if [[ $# -eq 1 ]]; then + if [[ $(date --date="${1}" +%s) -lt $(date --date="${CI_TESTS_TOTAL_END_TIME_DATE}" +%s) ]]; then + CI_TESTS_TOTAL_END_TIME_DATE=$1 + fi +fi +echo ${CI_TESTS_TOTAL_END_TIME_DATE} +