Skip to content
Snippets Groups Projects
Commit 63a6baf1 authored by Pierre-Alain Loizeau's avatar Pierre-Alain Loizeau Committed by Florian Uhlig
Browse files

[CI] Add detection of job timeout and total test time TO check to CI

parent 324f5817
No related branches found
No related tags found
1 merge request!1951[CI] Add detection of job timeout and total test time TO check to CI
......@@ -233,6 +233,10 @@ FileLicenceCheck:
- if [[ -n $INSTALL_PROJECT ]]; then
- echo "export INSTALL_PROJECT=TRUE" >> Dart.cfg
- fi
- ./scripts/find_cijob_ctest_stop_time.sh Dart.cfg
- if [[ -n $CI_TEST_STAGE_TOTAL_TIME_LIMIT ]]; then
- echo "export CI_TEST_STAGE_TOTAL_TIME_LIMIT=$CI_TEST_STAGE_TOTAL_TIME_LIMIT" >> Dart.cfg
- fi
- if [[ -n $CONFIGFILE ]]; then
- source $CONFIGFILE $FAIRSOFT_VERSION
- fi
......@@ -258,6 +262,7 @@ FileLicenceCheck:
SIMPATH: "$BASE_PATH/fairsoft/$FAIRSOFT_VERSION"
FAIRROOTPATH: "$BASE_PATH/fairroot/${FAIRROOT_VERSION}_${FAIRSOFT_VERSION}"
CHECK_GEO_HASH_CHANGE: "1"
CI_TEST_STAGE_TOTAL_TIME_LIMIT: 1500
.apptainer_tag: &apptainer_tag
stage: build
......
......@@ -119,6 +119,25 @@ If(NOT _RETVAL)
endif()
EndIf()
If(DEFINED ENV{CI_TEST_STAGE_TOTAL_TIME_LIMIT})
# Compute test stage timeout based on current time + limit from environment variable
If(DEFINED ENV{CTEST_END_TIME_LIMIT})
# If also full DASH run timeout defined, also compare the two and keep the earliest one
execute_process (
COMMAND scripts/find_citests_ctest_stop_time.sh $ENV{CTEST_END_TIME_LIMIT}
OUTPUT_VARIABLE CI_TESTS_TOTAL_END_TIME_DATE
)
Else()
# Else just get the CI jpb timeout
execute_process (
COMMAND scripts/find_citests_ctest_stop_time.sh
OUTPUT_VARIABLE CI_TESTS_TOTAL_END_TIME_DATE
)
EndIf()
set(stop_time STOP_TIME "${CI_TESTS_TOTAL_END_TIME_DATE}")
message(STATUS "Due to CI mode, Set end time for the ctest test runs to ${stop_time}")
EndIf()
Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}"
PARALLEL_LEVEL $ENV{number_of_processors}
${repeat}
......
#!/bin/bash
# Copyright (C) 2024 GSI Helmholtzzentrum fuer Schwerionenforschung, Darmstadt
# SPDX-License-Identifier: GPL-3.0-only
# First commited by Pierre-Alain Loizeau
# Script used to compute a "safe" end time for the ctest test stage of a Dart/Cdash chain in order
# to have enough time to upload the CDASH results before a job is killed in a Gitlab CI pipeline
# Uses the GITLAB CI automatically pre-defined variable ${CI_JOB_TIMEOUT}
#
# Usage:
# - with a full/relative/absolute path to an output Dart.cfg file as parameter, where the export will be added
# - without any parameter, then setting an export
#
# => Typical use case would be calling it in your sbatch script after setting up the other parts of the
# Dart.cfg gile and before the call to Dart.sh
#>> echo "export NCPU=10" >> ${DART_CFG}
#>> [...]
#>> ./scripts/find_cijob_ctest_stop_time.sh ${DART_CFG}
#>> [...]
#>> ./Dart.sh Weekly ./${DART_CFG} &> ${OUT_DIR}/Dart_weekly.log
# Job end time
# 1. Make end time from current time + job timeout (not really precise due to job start time until here but should be
# good enough)
echo "Env. variable CI_JOB_TIMEOUT = ${CI_JOB_TIMEOUT} s"
CI_JOB_END_TIME_DATE=$(date -d"now +${CI_JOB_TIMEOUT}seconds")
# 2. Remove 5 minutes at the end for safety
echo "Now in human format: $(date +\"%H:%M:%S\")"
echo "CI_END_TIME in human format: ${CI_JOB_END_TIME_DATE}"
CTEST_END_TIME_LIMIT=$(date -d "${CI_JOB_END_TIME_DATE} -5minutes" +"%H:%M:%S")
echo "Setting time limit for ctest to ${CTEST_END_TIME_LIMIT} (5min bef. job TO) to make sure CDASH data is uploaded"
if [[ $# -eq 1 ]]; then
# Set the export in the chosen Dart.cfg file
echo "export CTEST_END_TIME_LIMIT=\"${CTEST_END_TIME_LIMIT}\"" >> $1
else
export CTEST_END_TIME_LIMIT=\"${CTEST_END_TIME_LIMIT}\"
fi
#!/bin/bash
# Copyright (C) 2024 GSI Helmholtzzentrum fuer Schwerionenforschung, Darmstadt
# SPDX-License-Identifier: GPL-3.0-only
# First commited by Pierre-Alain Loizeau
# Script used to compute a "total timeout" end time for the ctest test stage of a Dart/Cdash chain
#
# Usage:
# - without any parameter, result sent to stdout
# - with a single parameter, compare the two dates (after conversion to seconds) and return the earliest one
# => !!! No check of the format of the parameter before usage, meant to be "HH:MM:SS" !!!
#
# => Typical use case would be calling it in the CDASH/ctest cmake between the build and test stages
# Job end time
# 1. Make end time from current time + job timeout (not really precise due to job start time until here but should be
# good enough)
CI_TESTS_TOTAL_END_TIME_DATE=$(date -d "now +${CI_TEST_STAGE_TOTAL_TIME_LIMIT}seconds" +"%H:%M:%S")
if [[ $# -eq 1 ]]; then
if [[ $(date --date="${1}" +%s) -lt $(date --date="${CI_TESTS_TOTAL_END_TIME_DATE}" +%s) ]]; then
CI_TESTS_TOTAL_END_TIME_DATE=$1
fi
fi
echo ${CI_TESTS_TOTAL_END_TIME_DATE}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment